From 2838b25806a3742814d8066a13aa70897adeeef5 Mon Sep 17 00:00:00 2001 From: Berk <12572227+par274@users.noreply.github.com> Date: Tue, 13 Oct 2026 05:21:41 +0300 Subject: [PATCH] Render pipeline throughput (#1082) * [GPU] Run the command stream on its own thread and speed up draw execution ASTRO BOT crowd scene: ~25 FPS -> ~55-58 FPS. - The PM4 command stream is interpreted on its own thread. Draws, dispatches, barriers, end-of-pipe writes and the interpreter's stores are handed to the render thread in stream order through a short queue (64 items); callbacks that return a result wait for the work before them. - Graphics program resolution runs on a worker thread ahead of the draws, from copy-on-write register bank snapshots, and is validated against the GPU ownership state before use. - The resource materialization cache rebases buffer base addresses instead of missing on every moved descriptor. - DCC metadata fills that cannot be absorbed run on the host, so binding the surface no longer waits for a GPU readback. - Cheaper per-draw work: cached pipeline lookups no longer wait on program preparation, redundant dynamic state is skipped, image bindings and target requests are memoized, fewer allocations. - Guest thread stack/TLS slots are chosen and mapped under one lock, so two threads can no longer share a stack. - Wave lowering split into native wave32, native wave64 and emulated wave64. * [GPU] Record a frame's draws before its boundary returns to the guest With the command stream on its own thread, a frame boundary only posted its flush, so the guest resumed while up to 64 items of the frame were still queued for the render thread. The guest then rewrote per-draw memory (descriptor tables, rings) that those draws read when recorded, which showed as black frames and flicker. The boundary now waits for the render thread to run everything queued before it. * [ShaderCompiler] Prove wave64 programs whose lanes never read each other * [ShaderCompiler] Find vector writes that no later instruction observes * [ShaderCompiler] Declare the SubgroupId built-in * [ShaderCompiler] Let compile requests ask for 32-lane compute subgroups * [ShaderCompiler] Run wave64 compute as paired lanes on 32-lane subgroups * [ShaderCompiler] Run lane-local wave64 programs as independent half waves * [GPU] Compile wave64 compute for 32-lane subgroups when the host lacks wave64 * [ShaderCompiler] Skip EXEC selects for vector writes no lane observes * [Diagnostics] Count GPU barriers and rendering scopes * [GPU] Merge consecutive memory barrier requests in a recording buffer * [GPU] Request guest memory barriers through the recording buffer * [GPU] Track GPU write ticks per buffer range * [GPU] Publish read-back pieces that no later write covers * [GPU] Stop reading ahead windows whose copies keep going stale * [GPU] Skip the upload walk when no CPU-dirty page is in range * [ShaderCompiler] Count the finest mip each pixel sample reaches * [GPU] Pack LOD statistics reports from counter words * [GPU] Report LOD statistics measured on the GPU * [ShaderCompiler] Start pixel append programs without helper lanes * [GPU] Advance a binding epoch when helper passes bind their own state * [GPU] Skip native binds that repeat the bound state * [GPU] Split the post-lookup work out of image lookups * [GPU] Memoize texture lookups of DCC images * [Core] Map guest thread priorities one band lower * [HLE] Write single-view guest memory without the region lock * [HLE] Answer GPU span coverage from a snapshot * [HLE] Skip the region lock for writes to CPU-dirty pages * [AGC] Emit packets with one write and read the buffer state at once * [AGC] Emit dispatch, memory and marker packets with one write * [AGC] Emit indirect buffer packets with one write * [AGC] Emit draw packets with one write * [AGC] Emit event and flip packets with one write * [AGC] Emit register packets with one write * [Tests] Cover wave64 subgroups, readbacks, barriers, LOD statistics and packet writes * [Profile] Count resource cache refusal reasons * [Profile] Classify what changed in stale resource cache entries * [Profile] Attribute render thread GPU waits to their call sites * [Profile] Count DCC metadata synchronizations and their outcome * [Diagnostics] Report the GPU writers of synchronized DCC metadata * [GPU] Fill metadata-shaped bounded fills on the host * [Presenter] Merge overlapping concurrent resolve ownership answers * [Tests] Cover concurrent resolve answers and release device slabs * [GPU] Write the mip statistics report with its ready marker * [GPU] Write the interpreter's LOD statistics through the shared report writer * [Presenter] Write the bound mip statistics in stream order on the render thread * [Presenter] Report bound mip statistics by default and measure on the GPU on request * [Tests] Cover the written mip statistics report * [GPU] Decode repeated FP16 and uint buffer fill words * [GPU] Clear image buffer owners with format-correct fill values * [Tests] Cover FP16 and uint buffer fills and raw fallback --------- Co-authored-by: Astellou --- .../Cpu/Native/DirectExecutionBackend.cs | 37 +- .../Memory/PhysicalVirtualMemory.cs | 7 + src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs | 13 +- .../GpuMemory/GuestPageTracker.cs | 8 + src/SharpEmu.HLE/GpuMemory/SpanSet.cs | 29 + src/SharpEmu.HLE/GpuMemory/TrackedRegion.cs | 14 + .../GuestMemory/GuestSpaceOwner.cs | 2 + .../GuestMemory/SharedBackingViews.cs | 18 + .../Agc/AgcExports.CommandPackets.cs | 91 +- .../Agc/AgcExports.CommandStream.cs | 244 +- .../Agc/AgcExports.ControlFlow.cs | 6 +- .../Agc/AgcExports.DrawPackets.cs | 64 +- src/SharpEmu.Libs/Agc/AgcExports.Events.cs | 21 +- .../Agc/AgcExports.RegisterPackets.cs | 14 +- src/SharpEmu.Libs/Agc/AgcExports.cs | 35 +- .../Diagnostics/DccWriterTrace.cs | 36 + .../Gpu/Buffers/BdaFaultProcessor.cs | 1 + src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs | 29 +- .../Gpu/Buffers/GuestBufferCache.cs | 536 +++- src/SharpEmu.Libs/Gpu/Buffers/WriteTickMap.cs | 143 ++ .../Gpu/GpuCommands/CommandStreamQueue.cs | 2 +- .../GpuCommandInterpreter.Adaptations.cs | 6 +- .../GpuCommandInterpreter.Memory.cs | 12 +- .../GpuCommandInterpreter.Registers.cs | 16 +- .../GpuCommandInterpreter.Waits.cs | 2 +- .../Gpu/GpuCommands/GpuCommandInterpreter.cs | 25 +- .../Gpu/GpuCommands/ICommandStreamHost.cs | 14 + .../GpuCommands/Registers/ContextRegisters.cs | 12 +- .../GpuCommands/Registers/RegisterBanks.cs | 311 ++- .../Registers/RegisterWriteTable.cs | 18 +- .../Registers/RegisterWriters.Context.cs | 118 +- .../Registers/ShaderProgramRegisters.cs | 10 + .../Gpu/Images/CachedImage.Transfers.cs | 13 +- .../Gpu/Images/ColorToMultisampleDepthBlit.cs | 2 + src/SharpEmu.Libs/Gpu/Images/GpuTiler.cs | 9 +- .../Gpu/Images/GuestImageCache.Lookup.cs | 19 + .../Gpu/Images/GuestImageCache.Metadata.cs | 29 +- .../Gpu/Images/GuestImageCache.Transfers.cs | 3 +- .../Gpu/Images/GuestImageCache.cs | 7 +- .../Gpu/Images/ImageDescription.cs | 24 + .../ImageRequestBuilders.ColorTarget.cs | 51 +- .../ImageRequestBuilders.DepthTarget.cs | 24 +- .../Images/ImageRequestBuilders.Texture.cs | 3 +- src/SharpEmu.Libs/Gpu/Images/MipStatistics.cs | 51 + src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs | 12 +- .../Gpu/Images/TextureTransferLayout.cs | 3 +- src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs | 18 + .../Gpu/Pipelines/DescriptorWriter.cs | 3 +- .../Gpu/Pipelines/IShaderPipelineHost.cs | 7 + .../Gpu/Pipelines/PipelineKeys.cs | 30 +- .../Gpu/Pipelines/ShaderPipelineCache.cs | 210 +- .../Gpu/Pipelines/ShaderProgramCache.cs | 65 +- .../Gpu/Rendering/ParallelProgramPrefetch.cs | 282 +++ .../Gpu/Rendering/RenderExecutor.Compute.cs | 11 +- .../Gpu/Rendering/RenderExecutor.Prefetch.cs | 115 + .../Gpu/Rendering/RenderExecutor.Resources.cs | 5 +- .../Gpu/Rendering/RenderExecutor.Targets.cs | 6 + .../Gpu/Rendering/RenderExecutor.cs | 27 +- .../Gpu/Rendering/RenderingState.cs | 24 +- .../Gpu/Scheduling/GpuWritebackEpoch.cs | 16 + .../Gpu/Scheduling/RecordingBuffer.cs | 81 +- .../Gpu/Scheduling/SubmissionScheduler.cs | 14 + .../Gpu/Vulkan/GpuSyncProfile.cs | 79 + .../Gpu/Vulkan/GraphicsDynamicStateEpoch.cs | 16 + .../Gpu/Vulkan/NativeBindingEpoch.cs | 13 + .../Gpu/Vulkan/VulkanAsyncReadback.cs | 3 + .../Gpu/Vulkan/VulkanSynchronization.cs | 5 + .../Gpu/Vulkan/VulkanTickDevice.cs | 3 + .../VideoOut/GpuWaitSiteProfile.cs | 96 + .../VideoOut/RenderPhaseProfile.cs | 3 + .../VulkanVideoPresenter.BindingShadow.cs | 249 ++ .../VulkanVideoPresenter.CachedImages.cs | 33 + .../VulkanVideoPresenter.CommandStream.cs | 435 +++- .../VulkanVideoPresenter.CommandThread.cs | 511 ++++ .../VulkanVideoPresenter.ConcurrentResolve.cs | 152 ++ .../VulkanVideoPresenter.Descriptors.cs | 159 +- .../VulkanVideoPresenter.Device.Setup.cs | 13 +- .../VulkanVideoPresenter.Draws.Recording.cs | 4 + .../VulkanVideoPresenter.GuestSubmissions.cs | 40 +- .../VulkanVideoPresenter.LodStatistics.cs | 199 ++ .../VulkanVideoPresenter.Pipelines.cs | 65 +- .../VulkanVideoPresenter.Present.Hdr.cs | 2 + .../VulkanVideoPresenter.RenderLoop.cs | 81 +- .../VulkanVideoPresenter.Rendering.cs | 179 +- .../VulkanVideoPresenter.Scheduler.cs | 24 + .../VulkanVideoPresenter.ShaderPrewarm.cs | 6 +- .../Gen5SpirvTranslator.Alu.cs | 121 +- .../Gen5SpirvTranslator.CommonWave.cs | 27 + ...5SpirvTranslator.EmulatedWave64Lowering.cs | 578 +++++ .../Gen5SpirvTranslator.HalfWave64Lowering.cs | 158 ++ ...en5SpirvTranslator.NativeWave32Lowering.cs | 156 ++ ...en5SpirvTranslator.NativeWave64Lowering.cs | 160 ++ .../Gen5SpirvTranslator.PairedLanes.cs | 464 ++++ .../Gen5SpirvTranslator.Resources.cs | 189 +- .../Gen5SpirvTranslator.WaveLowering.cs | 34 + .../Gen5SpirvTranslator.cs | 585 ++--- .../SpirvModuleBuilder.cs | 5 + .../Gen5PixelInputMapping.cs | 14 +- .../Ir/Gen5VectorObservationAnalysis.cs | 454 ++++ .../Ir/Gen5WaveLaneLocalAnalysis.cs | 2244 +++++++++++++++++ .../Resources/BindingLayout.cs | 32 +- .../Resources/ResourceMaterializationCache.cs | 1587 +++++++++++- .../Resources/ResourceMaterializer.cs | 7 + .../Resources/ResourceSpecialization.cs | 3 +- .../Resources/ShaderResourceInfo.cs | 2 + .../ShaderCompileRequest.cs | 5 + .../Agc/AgcSetShRegisterRangeTests.cs | 8 +- .../Buffers/GuestBufferAsyncReadbackTests.cs | 156 ++ .../Gpu/Buffers/WriteTickMapTests.cs | 62 + .../Registers/RegisterBanksSnapshotTests.cs | 236 ++ .../Images/GuestImageCacheTests.Metadata.cs | 47 + .../Images/GuestImageCacheTests.Ownership.cs | 52 + .../Gpu/Images/ImageDescriptionTests.cs | 41 + .../Gpu/Images/MipStatisticsTests.cs | 53 + .../Gpu/Pipelines/ShaderPipelineCacheTests.cs | 3 + .../Rendering/RenderExecutorComputeTests.cs | 16 + .../Gpu/Rendering/RenderExecutorDrawTests.cs | 16 +- .../Rendering/RenderExecutorPrefetchTests.cs | 87 + .../Rendering/RenderExecutorTestSupport.cs | 32 +- .../Scheduling/RecordingBufferBarrierTests.cs | 72 + .../Gpu/Vulkan/HeadlessVulkan.cs | 1 + .../Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs | 94 +- .../Gpu/Vulkan/Wave64Subgroup32DeviceTests.cs | 183 ++ .../Gpu/Vulkan/WaveMaxReductionDeviceTests.cs | 10 +- .../Gpu/Vulkan/WaveShuffleDeviceTests.cs | 18 +- .../VideoOut/ConcurrentResolveTests.cs | 75 + .../VideoOut/PresenterImageBindingTests.cs | 35 + .../Gen5MipStatisticsTests.cs | 118 + .../Gen5VectorObservationAnalysisTests.cs | 98 + .../Gen5WaveLaneLocalAnalysisTests.cs | 262 ++ .../Gen5WaveLoweringTests.cs | 250 ++ .../Resources/BindingLayoutTests.cs | 3 +- .../ResourceMaterializationCacheTests.cs | 290 +++ .../Resources/SpirvBindingDeclarationTests.cs | 19 + 134 files changed, 13303 insertions(+), 1287 deletions(-) create mode 100644 src/SharpEmu.Libs/Gpu/Buffers/WriteTickMap.cs create mode 100644 src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs create mode 100644 src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs create mode 100644 src/SharpEmu.Libs/Gpu/Scheduling/GpuWritebackEpoch.cs create mode 100644 src/SharpEmu.Libs/Gpu/Vulkan/GpuSyncProfile.cs create mode 100644 src/SharpEmu.Libs/Gpu/Vulkan/GraphicsDynamicStateEpoch.cs create mode 100644 src/SharpEmu.Libs/Gpu/Vulkan/NativeBindingEpoch.cs create mode 100644 src/SharpEmu.Libs/VideoOut/GpuWaitSiteProfile.cs create mode 100644 src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.BindingShadow.cs create mode 100644 src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandThread.cs create mode 100644 src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ConcurrentResolve.cs create mode 100644 src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.LodStatistics.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.CommonWave.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.EmulatedWave64Lowering.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.HalfWave64Lowering.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave32Lowering.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave64Lowering.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.PairedLanes.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.WaveLowering.cs create mode 100644 src/SharpEmu.ShaderCompiler/Ir/Gen5VectorObservationAnalysis.cs create mode 100644 src/SharpEmu.ShaderCompiler/Ir/Gen5WaveLaneLocalAnalysis.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Buffers/WriteTickMapTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Scheduling/RecordingBufferBarrierTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64Subgroup32DeviceTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/VideoOut/ConcurrentResolveTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Tests/Gen5MipStatisticsTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Tests/Gen5VectorObservationAnalysisTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLaneLocalAnalysisTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLoweringTests.cs diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs index 331b974..442a67b 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs @@ -5604,6 +5604,11 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return false; } + // Map does not reject a range that is already mapped: it reuses and zero-fills it. Two threads + // creating guest threads at once could both find the same slot free and share one stack, so + // choosing and mapping a slot happen under one lock. + private static readonly object GuestThreadRegionGate = new(); + private static bool TryMapGuestThreadRegion( IVirtualMemory virtualMemory, ulong baseAddress, @@ -5611,6 +5616,20 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ProgramHeaderFlags protection, out ulong mappedBase, out string? error) + { + lock (GuestThreadRegionGate) + { + return TryMapGuestThreadRegionLocked(virtualMemory, baseAddress, size, protection, out mappedBase, out error); + } + } + + private static bool TryMapGuestThreadRegionLocked( + IVirtualMemory virtualMemory, + ulong baseAddress, + ulong size, + ProgramHeaderFlags protection, + out ulong mappedBase, + out string? error) { for (int i = 0; i < GuestThreadRegionSlots; i++) { @@ -5645,6 +5664,17 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I IVirtualMemory virtualMemory, out ulong tlsBase, out string? error) + { + lock (GuestThreadRegionGate) + { + return TryMapGuestThreadTlsRegionLocked(virtualMemory, out tlsBase, out error); + } + } + + private static bool TryMapGuestThreadTlsRegionLocked( + IVirtualMemory virtualMemory, + out ulong tlsBase, + out string? error) { for (int i = 0; i < GuestThreadRegionSlots; i++) { @@ -5727,18 +5757,21 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return true; } + private static readonly bool LegacyGuestPriorities = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_GUEST_PRIORITY"), "legacy", StringComparison.OrdinalIgnoreCase); + private static ThreadPriority MapGuestThreadPriority(int priority) { if (priority <= 478) { - return ThreadPriority.Highest; + return LegacyGuestPriorities ? ThreadPriority.Highest : ThreadPriority.Normal; } if (priority >= 733) { return ThreadPriority.Lowest; } - return ThreadPriority.Normal; + return LegacyGuestPriorities ? ThreadPriority.Normal : ThreadPriority.BelowNormal; } private void ApplyGuestThreadAffinity(ulong guestAffinityMask) diff --git a/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs b/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs index fe3b026..ea343fd 100644 --- a/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs +++ b/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs @@ -2070,6 +2070,13 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA GuestImageWriteTracker.NotifyManagedWrite(virtualAddress, (ulong)source.Length); GuestGpuMemoryHook.MarkCpuWrite(virtualAddress, (ulong)source.Length); + if (!source.IsEmpty && _backedSpace is { } backed && !_disposed && + backed.TryWriteSingleView(virtualAddress, source)) + { + NotifyGuestWriteWatch(virtualAddress, source); + return true; + } + var requiresExclusiveAccess = false; _gate.EnterReadLock(); try diff --git a/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs b/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs index a99f4b5..c46e01c 100644 --- a/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs +++ b/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs @@ -30,6 +30,7 @@ public sealed class GuestGpuMemory : IDisposable private readonly PageGuard _pages; private readonly ReaderWriterLockSlim _spansLock = new(); private readonly SpanSet _spans = new(); + private SpanSnapshot _spanSnapshot = SpanSnapshot.Empty; private long _spanVersion; public long SpanVersion => Interlocked.Read(ref _spanVersion); @@ -153,15 +154,7 @@ public sealed class GuestGpuMemory : IDisposable return false; } - _spansLock.EnterReadLock(); - try - { - return _spans.Contains(address, size); - } - finally - { - _spansLock.ExitReadLock(); - } + return Volatile.Read(ref _spanSnapshot).Contains(address, size); } public void ForEachSpan(Action visit) @@ -207,6 +200,7 @@ public sealed class GuestGpuMemory : IDisposable try { _spans.Add(address, size); + Volatile.Write(ref _spanSnapshot, _spans.Snapshot()); Interlocked.Increment(ref _spanVersion); } finally @@ -280,6 +274,7 @@ public sealed class GuestGpuMemory : IDisposable try { _spans.Remove(address, size); + Volatile.Write(ref _spanSnapshot, _spans.Snapshot()); Interlocked.Increment(ref _spanVersion); } finally diff --git a/src/SharpEmu.HLE/GpuMemory/GuestPageTracker.cs b/src/SharpEmu.HLE/GpuMemory/GuestPageTracker.cs index 0ac893a..2032867 100644 --- a/src/SharpEmu.HLE/GpuMemory/GuestPageTracker.cs +++ b/src/SharpEmu.HLE/GpuMemory/GuestPageTracker.cs @@ -198,6 +198,14 @@ public sealed class GuestPageTracker if (Volatile.Read(ref _regions[index]) is { } region) { tracked = true; + if (region.IsCpuDirtyThroughout(offset, bytes)) + { + remaining -= bytes; + offset = 0; + index++; + continue; + } + using (region.Lock.Hold()) { if (region.IsModified(WriteOrigin.Gpu, offset, bytes)) diff --git a/src/SharpEmu.HLE/GpuMemory/SpanSet.cs b/src/SharpEmu.HLE/GpuMemory/SpanSet.cs index fcf861f..291417f 100644 --- a/src/SharpEmu.HLE/GpuMemory/SpanSet.cs +++ b/src/SharpEmu.HLE/GpuMemory/SpanSet.cs @@ -3,6 +3,22 @@ namespace SharpEmu.HLE.GpuMemory; +public sealed class SpanSnapshot(ulong[] starts, ulong[] ends) +{ + public static readonly SpanSnapshot Empty = new([], []); + + public bool Contains(ulong address, ulong size) + { + var index = Array.BinarySearch(starts, address); + if (index < 0) + { + index = ~index - 1; + } + + return index >= 0 && ends[index] >= address + size; + } +} + public sealed class SpanSet { private readonly SortedList _spans = new(); @@ -79,6 +95,19 @@ public sealed class SpanSet return index > 0 && _spans.GetValueAtIndex(index - 1) > address; } + public SpanSnapshot Snapshot() + { + var starts = new ulong[_spans.Count]; + var ends = new ulong[_spans.Count]; + for (var index = 0; index < starts.Length; index++) + { + starts[index] = _spans.GetKeyAtIndex(index); + ends[index] = _spans.GetValueAtIndex(index); + } + + return new SpanSnapshot(starts, ends); + } + public void ForEach(Action visit) { foreach (var (start, end) in _spans) diff --git a/src/SharpEmu.HLE/GpuMemory/TrackedRegion.cs b/src/SharpEmu.HLE/GpuMemory/TrackedRegion.cs index 7dfe95c..9519cf3 100644 --- a/src/SharpEmu.HLE/GpuMemory/TrackedRegion.cs +++ b/src/SharpEmu.HLE/GpuMemory/TrackedRegion.cs @@ -46,6 +46,20 @@ public sealed class TrackedRegion return GetDirtyMask(side).AnyInRange(start, end); } + public bool IsCpuDirtyThroughout(ulong offset, ulong size) + { + var (start, end) = GetPageRange(BaseAddress + offset, size); + for (var page = start; page < end; page++) + { + if (!_cpuDirty.Get(page)) + { + return false; + } + } + + return true; + } + public bool IsCpuWriteHot(ulong offset, ulong size) { var (start, end) = GetPageRange(BaseAddress + offset, size); diff --git a/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs b/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs index 2b44b28..1387921 100644 --- a/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs +++ b/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs @@ -67,6 +67,8 @@ public sealed class GuestSpaceOwner : IDisposable // Reads through a view that covers the whole range without any lock; false otherwise. public bool TryReadSingleView(ulong address, Span data) => _views.TryReadSingleView(address, data); + public bool TryWriteSingleView(ulong address, ReadOnlySpan data) => _views.TryWriteSingleView(address, data); + public object AliasSnapshot => _views.AliasSnapshot; public bool TryEnterAliasAccess() => _views.TryEnterAliasAccess(); diff --git a/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs b/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs index 43be043..1698426 100644 --- a/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs +++ b/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs @@ -118,6 +118,24 @@ public sealed unsafe class SharedBackingViews : IDisposable } } + public bool TryWriteSingleView(ulong address, ReadOnlySpan data) + { + if (!TryAccessSingleView(address, (ulong)data.Length, out var target)) + { + return false; + } + + try + { + data.CopyTo(new Span((void*)target, data.Length)); + return true; + } + finally + { + Interlocked.Decrement(ref _activeAccesses); + } + } + public bool TryReadBacking(ulong address, Span data) { if (TryAccessSingleView(address, (ulong)data.Length, out var source)) diff --git a/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs index 75f378a..906ed65 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs @@ -136,12 +136,7 @@ public static partial class AgcExports var groupCountZ = (uint)ctx[CpuRegister.Rcx]; var modifier = (uint)ctx[CpuRegister.R8]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItDispatchDirect, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, groupCountX) || - !TryWriteUInt32(ctx, commandAddress + 8, groupCountY) || - !TryWriteUInt32(ctx, commandAddress + 12, groupCountZ) || - !TryWriteUInt32(ctx, commandAddress + 16, DirectDispatchInitiator(modifier))) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(5, ItDispatchDirect, 0), groupCountX, groupCountY, groupCountZ, DirectDispatchInitiator(modifier))) { return ReturnPointer(ctx, 0); } @@ -166,9 +161,7 @@ public static partial class AgcExports { var commandBufferAddress = ctx[CpuRegister.Rdi]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RAcbReset)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNop, RAcbReset), 0)) { return ReturnPointer(ctx, 0); } @@ -198,15 +191,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RAcquireMem)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0x8000_0000u) || - !TryWriteUInt32(ctx, commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || - !TryWriteUInt32(ctx, commandAddress + 12, 0) || - !TryWriteUInt32(ctx, commandAddress + 16, (uint)(baseAddress >> 8)) || - !TryWriteUInt32(ctx, commandAddress + 20, 0) || - !TryWriteUInt32(ctx, commandAddress + 24, pollCycles / 40) || - !TryWriteUInt32(ctx, commandAddress + 28, gcrControl)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(8, ItNop, RAcquireMem), 0x8000_0000u, noSize ? 0 : (uint)(sizeBytes >> 8), 0, (uint)(baseAddress >> 8), 0, pollCycles / 40, gcrControl)) { return ReturnPointer(ctx, 0); } @@ -301,11 +286,7 @@ public static partial class AgcExports var argumentsAddress = ctx[CpuRegister.Rsi]; var modifier = (uint)ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItDispatchIndirect, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, (uint)argumentsAddress) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)(argumentsAddress >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 12, (modifier & 0xA038u) | 0x41u)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(4, ItDispatchIndirect, 0), (uint)argumentsAddress, (uint)(argumentsAddress >> 32), (modifier & 0xA038u) | 0x41u)) { return ReturnPointer(ctx, 0); } @@ -360,18 +341,7 @@ public static partial class AgcExports var packetAddressValue = interrupt == 4 ? 0UL : destinationAddress; var packetData = interrupt == 4 ? 0UL : data; - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RReleaseMem)) || - !TryWriteUInt32(ctx, commandAddress + 4, action | (cachePolicy << 8)) || - !TryWriteUInt32( - ctx, - commandAddress + 8, - gcrControl | (dataSelection << 16) | (interrupt << 24)) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)packetAddressValue) || - !TryWriteUInt32(ctx, commandAddress + 16, (uint)(packetAddressValue >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 20, (uint)packetData) || - !TryWriteUInt32(ctx, commandAddress + 24, (uint)(packetData >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 28, interruptContextId & 0x07FF_FFFFu)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(8, ItNop, RReleaseMem), action | (cachePolicy << 8), gcrControl | (dataSelection << 16) | (interrupt << 24), (uint)packetAddressValue, (uint)(packetAddressValue >> 32), (uint)packetData, (uint)(packetData >> 32), interruptContextId & 0x07FF_FFFFu)) { return ReturnPointer(ctx, 0); } @@ -397,9 +367,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RDrawReset)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNop, RDrawReset), 0)) { return ReturnPointer(ctx, 0); } @@ -448,12 +416,7 @@ public static partial class AgcExports (resetCounters << 18) | (counterMask << 10) | (counterSelect << 2); - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItGetLodStats, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, control) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)destinationAddress & ~0x3Fu) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)(destinationAddress >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 16, packetControl)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(5, ItGetLodStats, 0), control, (uint)destinationAddress & ~0x3Fu, (uint)(destinationAddress >> 32), packetControl)) { return ReturnPointer(ctx, 0); } @@ -493,15 +456,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RAcquireMem)) || - !TryWriteUInt32(ctx, commandAddress + 4, (engine << 31) | cbDbOp) || - !TryWriteUInt32(ctx, commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || - !TryWriteUInt32(ctx, commandAddress + 12, 0) || - !TryWriteUInt32(ctx, commandAddress + 16, (uint)(baseAddress >> 8)) || - !TryWriteUInt32(ctx, commandAddress + 20, 0) || - !TryWriteUInt32(ctx, commandAddress + 24, pollCycles / 40) || - !TryWriteUInt32(ctx, commandAddress + 28, gcrControl)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(8, ItNop, RAcquireMem), (engine << 31) | cbDbOp, noSize ? 0 : (uint)(sizeBytes >> 8), 0, (uint)(baseAddress >> 8), 0, pollCycles / 40, gcrControl)) { return ReturnPointer(ctx, 0); } @@ -631,16 +586,7 @@ public static partial class AgcExports var control = 0x10u | compareFunction | (writeSpace == 1 ? 0x100u : 0u); - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 9, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(9, ItCondWrite, RZero)) || - !TryWriteUInt32(ctx, commandAddress + 4, control) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)readAddress & ~0x3u) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)(readAddress >> 32) & 0xFFFFu) || - !TryWriteUInt32(ctx, commandAddress + 16, reference) || - !TryWriteUInt32(ctx, commandAddress + 20, mask) || - !TryWriteUInt32(ctx, commandAddress + 24, (uint)writeAddress & ~0x3u) || - !TryWriteUInt32(ctx, commandAddress + 28, (uint)(writeAddress >> 32) & 0xFFFFu) || - !TryWriteUInt32(ctx, commandAddress + 32, writeValue)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(9, ItCondWrite, RZero), control, (uint)readAddress & ~0x3u, (uint)(readAddress >> 32) & 0xFFFFu, reference, mask, (uint)writeAddress & ~0x3u, (uint)(writeAddress >> 32) & 0xFFFFu, writeValue)) { return ReturnPointer(ctx, 0); } @@ -752,9 +698,7 @@ public static partial class AgcExports var address = ctx[CpuRegister.Rdx]; var reference = ctx[CpuRegister.Rcx]; if (commandBufferAddress == 0 || size > 1 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RZero)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNop, RZero), 0)) { return ReturnPointer(ctx, 0); } @@ -790,11 +734,7 @@ public static partial class AgcExports var baseIndex = (uint)ctx[CpuRegister.Rsi]; var address = ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItSetBase, 0) | (baseIndex << 1)) || - !TryWriteUInt32(ctx, commandAddress + 4, 1) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)address & ~7u) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)(address >> 32))) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(4, ItSetBase, 0) | (baseIndex << 1), 1, (uint)address & ~7u, (uint)(address >> 32))) { return ReturnPointer(ctx, 0); } @@ -813,10 +753,7 @@ public static partial class AgcExports var dataOffset = (uint)ctx[CpuRegister.Rsi]; var modifier = (uint)ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 3, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(3, ItDispatchIndirect, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, dataOffset) || - !TryWriteUInt32(ctx, commandAddress + 8, (modifier & 0xA038u) | 0x41u)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(3, ItDispatchIndirect, 0), dataOffset, (modifier & 0xA038u) | 0x41u)) { return ReturnPointer(ctx, 0); } @@ -884,9 +821,7 @@ public static partial class AgcExports { var commandBufferAddress = ctx[CpuRegister.Rdi]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RPopMarker)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNop, RPopMarker), 0)) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs index 73abc74..ec78bd1 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs @@ -23,6 +23,21 @@ public static partial class AgcExports internal sealed record RetainedTargetlessDraw(RegisterBanks Banks, TargetlessDrawArguments Arguments); + // A draw or dispatch the translation deferred, with the banks it was issued under. + internal abstract record DeferredWork; + + // Takes the deferred work of a command stream interpreted on its own thread. + internal interface IDeferredWorkSink + { + // True on the thread that interprets the stream; elsewhere the translation runs work itself. + bool IsProducerThread { get; } + + // The non-draw items handed over so far; a draw's resolution waits until they ran. + long Gate { get; } + + void EnqueueDeferred(DeferredWork work, bool isDraw); + } + // The snapshots the submit-time prepass captured for one submission. private sealed record SubmittedGeometrySnapshots( Dictionary? IndexSnapshots, @@ -160,9 +175,18 @@ public static partial class AgcExports { _pipelines = new ShaderPipelineCache(_context, pipelineHost, GuestGpu.Current, CreateShaderHeaderRegistry(_context)); _executor = new RenderExecutor(renderHost, _pipelines); + if (host is IConcurrentResolveHost concurrentHost) + { + _parallel = new ParallelProgramPrefetch(_executor, renderHost, concurrentHost); + _executor.Prefetch = _parallel; + } } } + // Resolves the programs of deferred draws ahead of them, for a host that answers reads from + // another thread; draws are deferred whenever it exists. + private readonly ParallelProgramPrefetch? _parallel; + // The context bank of the queue whose slice runs now; the Metal host rebuilds its records from it. internal ContextRegisters? CurrentContextRegisters => _current?.TypedRegisters?.Context; @@ -196,17 +220,194 @@ public static partial class AgcExports } } + private sealed record PendingDraw( + ulong SubmitId, + SubmittedDcbState State, + RegisterBanks Banks, + bool Indexed, + DrawIndexedArguments IndexedArguments, + DrawAutoArguments AutoArguments, + SubmittedVertexSnapshot? VertexSnapshot, + SubmittedIndexSnapshot? IndexSnapshot, + ParallelProgramPrefetch.Job? Job) : DeferredWork; + + private sealed record PendingDispatch( + ulong SubmitId, + SubmittedDcbState State, + RegisterBanks Banks, + uint GroupsX, + uint GroupsY, + uint GroupsZ, + uint DispatchInitiator, + ulong IndirectArgumentsAddress) : DeferredWork; + + // With the command stream on its own thread, its draws and dispatches reach the render + // thread through the sink, which runs them with RunDeferred in stream order. + internal IDeferredWorkSink? Sink { get; set; } + + private bool Producing => Sink is { IsProducerThread: true }; + + // The queue state of the work the render thread runs now; _current belongs to the thread + // interpreting the stream. + private SubmittedDcbState? _executing; + + public void RunDeferred(DeferredWork work) + { + switch (work) + { + case PendingDraw draw: + { + var executing = _executing; + _executing = draw.State; + RecordKnownColorTargets(draw.State, draw.Banks); + if (_parallel is not null) + { + _parallel.Current = draw.Job; + } + + try + { + RunExecutorDraw(draw.SubmitId, draw.State, draw.Banks, draw.Indexed, draw.IndexedArguments, draw.AutoArguments, + clearSnapshots: false); + } + finally + { + _executing = executing; + if (_parallel is not null) + { + _parallel.Current = null; + } + } + + break; + } + case PendingDispatch dispatch: + { + var executing = _executing; + _executing = dispatch.State; + try + { + ExecuteDispatch(_executor!, dispatch.SubmitId, dispatch.Banks, dispatch.GroupsX, dispatch.GroupsY, dispatch.GroupsZ, + dispatch.DispatchInitiator, dispatch.IndirectArgumentsAddress); + } + finally + { + _executing = executing; + } + + break; + } + default: + throw _host.Fatal($"The deferred work is unknown: {work.GetType().Name}."); + } + } + + // The resolution worker, so the render thread can order it with the stream's other work. + internal ParallelProgramPrefetch? Prefetch => _parallel; + + private readonly Queue _pendingDraws = new(); + + public bool HasPendingDraws => _pendingDraws.Count != 0; + + private bool _draining; + + // Runs every deferred draw in issue order with the banks and snapshots it was issued with. + // A draw calls back into the host, whose entry points drain too: those calls run inside + // the draw and must not start the next one. + public void DrainPendingDraws() + { + if (_draining) + { + return; + } + + _draining = true; + try + { + DrainPendingDrawsCore(); + } + finally + { + _draining = false; + } + } + + private void DrainPendingDrawsCore() + { + while (_pendingDraws.TryDequeue(out var pending)) + { + var current = _current; + var state = pending.State; + _current = state; + state.CurrentVertexSnapshot = pending.VertexSnapshot; + state.CurrentIndexSnapshot = pending.IndexSnapshot; + if (_parallel is not null) + { + _parallel.Current = pending.Job; + } + + try + { + RunExecutorDraw(pending.SubmitId, state, pending.Banks, pending.Indexed, pending.IndexedArguments, pending.AutoArguments); + } + finally + { + _current = current; + if (_parallel is not null) + { + _parallel.Current = null; + } + } + } + + // What follows a drain may change guest memory and cache state the worker reads. + _parallel?.WaitIdle(); + } + private bool TryRunExecutorDraw(ulong submitId, SubmittedDcbState state, bool indexed, in DrawIndexedArguments indexedArguments, in DrawAutoArguments autoArguments) { - if (_executor is not { } executor) + if (_executor is null) { return false; } var banks = RequireTypedRegisters(state); - RecordKnownColorTargets(state, banks); + var producing = Producing; + if (!producing) + { + // A draw run by the render thread records its targets there. + RecordKnownColorTargets(state, banks); + } + state.FrameDrawCount++; state.SawIndexedDraw |= indexed; + if (_parallel is not null || producing) + { + var snapshot = banks.Snapshot(); + var pending = new PendingDraw(submitId, state, snapshot, indexed, indexedArguments, autoArguments, + state.CurrentVertexSnapshot, state.CurrentIndexSnapshot, _parallel?.Submit(snapshot, producing ? Sink!.Gate : 0)); + if (producing) + { + Sink!.EnqueueDeferred(pending, isDraw: true); + } + else + { + _pendingDraws.Enqueue(pending); + } + + state.CurrentIndexSnapshot = null; + state.CurrentVertexSnapshot = null; + return true; + } + + RunExecutorDraw(submitId, state, banks, indexed, indexedArguments, autoArguments); + return true; + } + + private void RunExecutorDraw(ulong submitId, SubmittedDcbState state, RegisterBanks banks, bool indexed, in DrawIndexedArguments indexedArguments, in DrawAutoArguments autoArguments, + bool clearSnapshots = true) + { + var executor = _executor!; var drawStarted = DcbParseProfile.Begin(); try { @@ -222,17 +423,19 @@ public static partial class AgcExports finally { DcbParseProfile.RecordDraw(drawStarted); - state.CurrentIndexSnapshot = null; - state.CurrentVertexSnapshot = null; + // The snapshot fields belong to the interpreting thread when it is another one. + if (clearSnapshots) + { + state.CurrentIndexSnapshot = null; + state.CurrentVertexSnapshot = null; + } } - - return true; } // Keeps the banks the draw was issued under; the flip replays it into the display buffer. internal void RetainTargetlessDraw(RegisterBanks banks, in TargetlessDrawArguments arguments) { - var state = RequireCurrent(); + var state = _executing ?? RequireCurrent(); if (RenderTrace.Enabled) { RenderTrace.Write($"TargetlessRetention submit={arguments.SubmitId} replaced={state.RetainedTargetlessDraw is not null} previousSubmit={state.RetainedTargetlessDraw?.Arguments.SubmitId} export=0x{banks.Shader.Vertex.ExportAddress:X16} pixel=0x{banks.Shader.Pixel.Address:X16}"); @@ -363,22 +566,35 @@ public static partial class AgcExports { var banks = RequireTypedRegisters(state); state.FrameDispatchCount++; - var executorStarted = DcbParseProfile.Begin(); - try + if (Producing) { - executor.Dispatch(submitId, banks, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress); - } - finally - { - DcbParseProfile.RecordDispatch(executorStarted); + var snapshot = banks.Snapshot(); + Sink!.EnqueueDeferred(new PendingDispatch(submitId, state, snapshot, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress), + isDraw: false); + return; } + ExecuteDispatch(executor, submitId, banks, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress); return; } throw _host.Fatal("The command stream has no render executor."); } + private static void ExecuteDispatch(RenderExecutor executor, ulong submitId, RegisterBanks banks, uint endX, uint endY, uint endZ, + uint dispatchInitiator, ulong indirectArgumentsAddress) + { + var executorStarted = DcbParseProfile.Begin(); + try + { + executor.Dispatch(submitId, banks, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress); + } + finally + { + DcbParseProfile.RecordDispatch(executorStarted); + } + } + // The retained draw runs into the display buffer with the target words it was last bound with. private void ReplayRetainedTargetlessDraw(RenderExecutor executor, SubmittedDcbState state, int handle, int displayBufferIndex) { diff --git a/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs b/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs index 125c2c8..b5bcf3f 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs @@ -249,11 +249,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItIndirectBuffer, RZero)) || - !TryWriteUInt32(ctx, commandAddress + 4, (uint)(target & 0xFFFF_FFFFUL)) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)((target >> 32) & 0xFFFFUL)) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)targetDwords & 0xFFFFFu)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(4, ItIndirectBuffer, RZero), (uint)(target & 0xFFFF_FFFFUL), (uint)((target >> 32) & 0xFFFFUL), (uint)targetDwords & 0xFFFFFu)) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs index dfb8c60..62d31f2 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs @@ -46,16 +46,7 @@ public static partial class AgcExports uint perInstanceObjectIdSupport) { if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 3, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(3, ItSetUconfigRegIndex, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0x2000_0000u | VgtIndexType) || - !TryWriteUInt32( - ctx, - commandAddress + 8, - 0x400u | - (indexSize & 0x3u) | - ((cachePolicy & 0x3u) << 6) | - ((perInstanceObjectIdSupport & 0x1u) << 14))) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(3, ItSetUconfigRegIndex, 0), 0x2000_0000u | VgtIndexType, 0x400u | (indexSize & 0x3u) | ((cachePolicy & 0x3u) << 6) | ((perInstanceObjectIdSupport & 0x1u) << 14))) { return ReturnPointer(ctx, 0); } @@ -77,9 +68,7 @@ public static partial class AgcExports var commandBufferAddress = ctx[CpuRegister.Rdi]; var indexCount = (uint)ctx[CpuRegister.Rsi]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RIndexCount)) || - !TryWriteUInt32(ctx, commandAddress + 4, indexCount)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNop, RIndexCount), indexCount)) { return ReturnPointer(ctx, 0); } @@ -112,9 +101,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNumInstances, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, instanceCount)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNumInstances, 0), instanceCount)) { return ReturnPointer(ctx, 0); } @@ -140,12 +127,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var baseCommand) || - !TryWriteUInt32(ctx, baseCommand, Pm4(3, ItIndexBase, 0)) || - !TryWriteUInt32(ctx, baseCommand + 4, (uint)indexAddress) || - !TryWriteUInt32(ctx, baseCommand + 8, (uint)(indexAddress >> 32)) || - !TryWriteUInt32(ctx, baseCommand + 12, Pm4(2, ItIndexBufferSize, 0)) || - !TryWriteUInt32(ctx, baseCommand + 16, indexCount)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var baseCommand, Pm4(3, ItIndexBase, 0), (uint)indexAddress, (uint)(indexAddress >> 32), Pm4(2, ItIndexBufferSize, 0), indexCount)) { return ReturnPointer(ctx, 0); } @@ -155,13 +137,7 @@ public static partial class AgcExports // former five-dword packet omitted both the real base and the count // field, so every call made by Unity looked like a zero-count draw to // the submitted-command parser and the complete scene was discarded. - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 6, out var drawCommand) || - !TryWriteUInt32(ctx, drawCommand, Pm4(6, ItDrawIndex2, 0)) || - !TryWriteUInt32(ctx, drawCommand + 4, indexCount) || - !TryWriteUInt32(ctx, drawCommand + 8, (uint)indexAddress) || - !TryWriteUInt32(ctx, drawCommand + 12, (uint)(indexAddress >> 32)) || - !TryWriteUInt32(ctx, drawCommand + 16, indexCount) || - !TryWriteUInt32(ctx, drawCommand + 20, 0)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var drawCommand, Pm4(6, ItDrawIndex2, 0), indexCount, (uint)indexAddress, (uint)(indexAddress >> 32), indexCount, 0)) { return ReturnPointer(ctx, 0); } @@ -204,12 +180,7 @@ public static partial class AgcExports ? 2u : ((modifierBits >> 3) & 0x20u) | 2u; - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var drawCommand) || - !TryWriteUInt32(ctx, drawCommand, Pm4(5, ItDrawIndirect, 0)) || - !TryWriteUInt32(ctx, drawCommand + 4, dataOffset) || - !TryWriteUInt32(ctx, drawCommand + 8, firstVertexRegister) || - !TryWriteUInt32(ctx, drawCommand + 12, firstInstanceRegister) || - !TryWriteUInt32(ctx, drawCommand + 16, drawInitiator)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var drawCommand, Pm4(5, ItDrawIndirect, 0), dataOffset, firstVertexRegister, firstInstanceRegister, drawInitiator)) { var rejects = Interlocked.Increment(ref _indirectDrawEmitRejectCount); if (rejects <= 8 || rejects % 250 == 0) @@ -244,14 +215,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 7, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(7, ItNop, RDrawIndexAuto)) || - !TryWriteUInt32(ctx, commandAddress + 4, indexCount) || - !TryWriteUInt32(ctx, commandAddress + 8, 0) || - !TryWriteUInt32(ctx, commandAddress + 12, 0) || - !TryWriteUInt32(ctx, commandAddress + 16, 0) || - !TryWriteUInt32(ctx, commandAddress + 20, 0) || - !TryWriteUInt32(ctx, commandAddress + 24, 0)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(7, ItNop, RDrawIndexAuto), indexCount, 0, 0, 0, 0, 0)) { return ReturnPointer(ctx, 0); } @@ -390,12 +354,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(3, ItIndexBase, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, (uint)(indexBufferAddress & 0xFFFF_FFFFUL)) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)(indexBufferAddress >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 12, Pm4(2, ItIndexBufferSize, 0)) || - !TryWriteUInt32(ctx, commandAddress + 16, indexCount)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(3, ItIndexBase, 0), (uint)(indexBufferAddress & 0xFFFF_FFFFUL), (uint)(indexBufferAddress >> 32), Pm4(2, ItIndexBufferSize, 0), indexCount)) { return ReturnPointer(ctx, 0); } @@ -424,12 +383,7 @@ public static partial class AgcExports var drawInitiator = (drawModifier & (1UL << 32)) != 0 ? 0u : ((uint)drawModifier >> 3) & 0x20u; - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItDrawIndexOffset2, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, Math.Max(indexCount, 1u)) || - !TryWriteUInt32(ctx, commandAddress + 8, indexOffset) || - !TryWriteUInt32(ctx, commandAddress + 12, indexCount) || - !TryWriteUInt32(ctx, commandAddress + 16, drawInitiator)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(5, ItDrawIndexOffset2, 0), Math.Max(indexCount, 1u), indexOffset, indexCount, drawInitiator)) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.Events.cs b/src/SharpEmu.Libs/Agc/AgcExports.Events.cs index d21a8b6..432a2ec 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.Events.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.Events.cs @@ -69,9 +69,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItEventWrite, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, eventType)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItEventWrite, 0), eventType)) { return ReturnPointer(ctx, 0); } @@ -95,14 +93,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 7, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(7, ItNop, RWaitFlipDone)) || - !TryWriteUInt32(ctx, commandAddress + 4, videoOutHandle) || - !TryWriteUInt32(ctx, commandAddress + 8, displayBufferIndex) || - !TryWriteUInt32(ctx, commandAddress + 12, 0) || - !TryWriteUInt32(ctx, commandAddress + 16, 0) || - !TryWriteUInt32(ctx, commandAddress + 20, 0) || - !TryWriteUInt32(ctx, commandAddress + 24, 0)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(7, ItNop, RWaitFlipDone), videoOutHandle, displayBufferIndex, 0, 0, 0, 0)) { return ReturnPointer(ctx, 0); } @@ -128,13 +119,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 6, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(6, ItNop, RFlip)) || - !TryWriteUInt32(ctx, commandAddress + 4, videoOutHandle) || - !TryWriteUInt32(ctx, commandAddress + 8, unchecked((uint)displayBufferIndex)) || - !TryWriteUInt32(ctx, commandAddress + 12, flipMode) || - !TryWriteUInt32(ctx, commandAddress + 16, (uint)(flipArg & 0xFFFF_FFFFUL)) || - !TryWriteUInt32(ctx, commandAddress + 20, (uint)(flipArg >> 32))) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(6, ItNop, RFlip), videoOutHandle, unchecked((uint)displayBufferIndex), flipMode, (uint)(flipArg & 0xFFFF_FFFFUL), (uint)(flipArg >> 32))) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs index 06f2b71..9aeb373 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs @@ -109,12 +109,9 @@ public static partial class AgcExports ulong commandAddress; using (AgcRegisterPacketProfile.Measure(AgcRegisterPacketProfile.Phase.PacketSetup)) { - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var markerAddress) || - !TryWriteUInt32(ctx, markerAddress, Pm4(2, ItNop, RZero)) || - !TryWriteUInt32(ctx, markerAddress + 4, CbSetShRegisterRangeMarker) || + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out _, Pm4(2, ItNop, RZero), CbSetShRegisterRangeMarker) || !TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItSetShReg, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, offset)) + !TryWriteCommandDwords(ctx, commandAddress, Pm4(packetDwords, ItSetShReg, 0), offset)) { AgcRegisterPacketProfile.RecordResult(false); return ReturnPointer(ctx, 0); @@ -288,12 +285,7 @@ public static partial class AgcExports _ => 0u, }; if (op == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(5, op, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, (uint)(registersAddress & 0xFFFF_FFFCUL)) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)(registersAddress >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 12, 0x8000_0000u) || - !TryWriteUInt32(ctx, commandAddress + 16, registerCount & 0x3FFFu)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(5, op, 0), (uint)(registersAddress & 0xFFFF_FFFCUL), (uint)(registersAddress >> 32), 0x8000_0000u, registerCount & 0x3FFFu)) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.cs b/src/SharpEmu.Libs/Agc/AgcExports.cs index 89fb747..b32f20a 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.cs @@ -491,15 +491,40 @@ public static partial class AgcExports private static bool TryAllocateCommandDwords(CpuContext ctx, ulong commandBufferAddress, uint sizeDwords, out ulong commandAddress) => TryPrepareCommandDwords(ctx, commandBufferAddress, sizeDwords, true, out commandAddress); + private static bool TryEmitCommandDwords(CpuContext ctx, ulong commandBufferAddress, out ulong commandAddress, params ReadOnlySpan dwords) + { + return TryAllocateCommandDwords(ctx, commandBufferAddress, (uint)dwords.Length, out commandAddress) && + TryWriteCommandDwords(ctx, commandAddress, dwords); + } + + private static bool TryWriteCommandDwords(CpuContext ctx, ulong address, params ReadOnlySpan dwords) => + ctx.Memory.TryWrite(address, System.Runtime.InteropServices.MemoryMarshal.AsBytes(dwords)); + + private const int CommandBufferStateBytes = (int)(CommandBufferReservedDwOffset + sizeof(uint) - CommandBufferCursorUpOffset); + + private static bool TryReadCommandBufferState(CpuContext ctx, ulong commandBufferAddress, out ulong cursorUp, out ulong cursorDown, out ulong callback, out ulong userData, out uint reservedDwords) + { + Span state = stackalloc byte[CommandBufferStateBytes]; + if (!ctx.Memory.TryRead(commandBufferAddress + CommandBufferCursorUpOffset, state)) + { + cursorUp = cursorDown = callback = userData = 0; + reservedDwords = 0; + return false; + } + + cursorUp = BinaryPrimitives.ReadUInt64LittleEndian(state); + cursorDown = BinaryPrimitives.ReadUInt64LittleEndian(state[(int)(CommandBufferCursorDownOffset - CommandBufferCursorUpOffset)..]); + callback = BinaryPrimitives.ReadUInt64LittleEndian(state[(int)(CommandBufferCallbackOffset - CommandBufferCursorUpOffset)..]); + userData = BinaryPrimitives.ReadUInt64LittleEndian(state[(int)(CommandBufferUserDataOffset - CommandBufferCursorUpOffset)..]); + reservedDwords = BinaryPrimitives.ReadUInt32LittleEndian(state[(int)(CommandBufferReservedDwOffset - CommandBufferCursorUpOffset)..]); + return true; + } + private static bool TryPrepareCommandDwords(CpuContext ctx, ulong commandBufferAddress, uint sizeDwords, bool advanceCursor, out ulong commandAddress) { commandAddress = 0; if (sizeDwords == 0 || - !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorUpOffset, out var cursorUp) || - !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorDownOffset, out var cursorDown) || - !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCallbackOffset, out var callback) || - !TryReadUInt64(ctx, commandBufferAddress + CommandBufferUserDataOffset, out var userData) || - !TryReadUInt32(ctx, commandBufferAddress + CommandBufferReservedDwOffset, out var reservedDwords)) + !TryReadCommandBufferState(ctx, commandBufferAddress, out var cursorUp, out var cursorDown, out var callback, out var userData, out var reservedDwords)) { return false; } diff --git a/src/SharpEmu.Libs/Diagnostics/DccWriterTrace.cs b/src/SharpEmu.Libs/Diagnostics/DccWriterTrace.cs index ed00daf..322c317 100644 --- a/src/SharpEmu.Libs/Diagnostics/DccWriterTrace.cs +++ b/src/SharpEmu.Libs/Diagnostics/DccWriterTrace.cs @@ -28,6 +28,42 @@ internal static class DccWriterTrace private static readonly HashSet<(ulong Address, ulong Size, ulong Program)> _reported = new(); private static readonly HashSet<(ulong Hash, string Reason)> _refusals = new(); + private static readonly System.Collections.Concurrent.ConcurrentDictionary _writes = new(); + private static readonly HashSet _syncReported = new(); + + [ThreadStatic] + public static string? CurrentKind; + + public static void RecordWrite(ulong address, ulong size) + { + if (DccEnabled) + { + _writes[address] = (size, CurrentProgram, CurrentKind ?? "?"); + } + } + + public static void ReportSync(ulong metadataAddress, ulong size) + { + if (!DccEnabled) + { + return; + } + + lock (_syncReported) + { + if (_syncReported.Count >= 64 || !_syncReported.Add(metadataAddress)) + { + return; + } + } + + var writers = _writes + .Where(pair => pair.Key < metadataAddress + size && metadataAddress < pair.Key + pair.Value.Size) + .Take(8) + .Select(pair => $"0x{pair.Key:X}+0x{pair.Value.Size:X}:{pair.Value.Kind}:0x{pair.Value.Program:X16}"); + Console.Error.WriteLine( + $"[GPU][DCC_SYNC_WRITER] metadata=0x{metadataAddress:X16} size=0x{size:X} writers={string.Join(" ", writers)}"); + } private static ulong ParseAddress(string? value) { diff --git a/src/SharpEmu.Libs/Gpu/Buffers/BdaFaultProcessor.cs b/src/SharpEmu.Libs/Gpu/Buffers/BdaFaultProcessor.cs index febe4ce..5b9458b 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/BdaFaultProcessor.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/BdaFaultProcessor.cs @@ -216,6 +216,7 @@ public sealed unsafe class BdaFaultProcessor : IDisposable vk.CmdBindPipeline(command, PipelineBindPoint.Compute, _pipeline); var set = _sets[_currentArea]; vk.CmdBindDescriptorSets(command, PipelineBindPoint.Compute, _pipelineLayout, 0, 1, &set, 0, null); + NativeBindingEpoch.Advance(); var threads = _pageCount / 32; vk.CmdDispatch(command, (uint)((threads + 63) / 64), 1, 1); VulkanSynchronization.PipelineBarrier(vk, diff --git a/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs b/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs index fec9cf7..6250279 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs @@ -172,16 +172,39 @@ public unsafe class GpuBuffer : IDisposable // A readback of it only has to wait for that tick, not for all queued work. public ulong LastGpuWriteTick { get; private set; } + private readonly WriteTickMap _writeTicks = new(); + private int _writeTickPruneCount = 64; + // Called by every path that records a GPU write into this buffer. - public void NoteGpuWrite() + public void NoteGpuWrite() => NoteGpuWrite(0, Size); + + public void NoteGpuWrite(ulong offset, ulong size) { var tick = _scheduler.CurrentTick; if (tick > LastGpuWriteTick) { LastGpuWriteTick = tick; } + + if (size == 0 || offset >= Size) + { + return; + } + + _writeTicks.Assign(offset, Math.Min(size, Size - offset), tick); + if (_writeTicks.Count >= _writeTickPruneCount) + { + if (Volatile.Read(ref _foreignReads) == 0) + { + _writeTicks.Prune(_scheduler.Timeline.CompletedTick + 1); + } + + _writeTickPruneCount = Math.Max(64, _writeTicks.Count * 2); + } } + public ulong LastGpuWriteTickIn(ulong offset, ulong size) => _writeTicks.MaxTick(offset, size); + public void Write(ulong offset, ReadOnlySpan source) { if (_mapped == null || offset > Size || (ulong)source.Length > Size - offset) @@ -243,7 +266,7 @@ public unsafe class GpuBuffer : IDisposable } command.EndRendering(); - NoteGpuWrite(); + NoteGpuWrite(destinationOffset, size); var vk = _device.Vk; var native = new CommandBuffer(command.Handle); var before = stackalloc BufferMemoryBarrier2[2]; @@ -271,7 +294,7 @@ public unsafe class GpuBuffer : IDisposable var command = _scheduler.Current; command.EndRendering(); - NoteGpuWrite(); + NoteGpuWrite(offset, size); var vk = _device.Vk; var native = new CommandBuffer(command.Handle); var before = CreateBarrier(offset, size, MemoryAccess, AccessFlags.TransferWriteBit); diff --git a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs index 183f607..1a77b94 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs @@ -240,10 +240,11 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable _ = SynchronizeBuffer(buffer, guestAddress, size, isWritten, isTexelBuffer, preserveCpuWriteHotPages: false); if (isWritten) { - buffer.NoteGpuWrite(); + buffer.NoteGpuWrite(buffer.Offset(guestAddress), size); if (Diagnostics.DccWriterTrace.Enabled) { Diagnostics.DccWriterTrace.Record(RequireImageCache(), guestAddress, size); + Diagnostics.DccWriterTrace.RecordWrite(guestAddress, size); } if (!_gpuModifiedRanges.Contains(guestAddress, size)) @@ -333,6 +334,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable throw SubmissionScheduler.Fatal("The host DMA write range is invalid."); } + GpuWritebackEpoch.Advance(); if (!_backing.TryWriteBacking(guestAddress, data)) { throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{guestAddress:X16} size=0x{data.Length:X16}"); @@ -359,7 +361,10 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } } - public void FillBuffer(ulong guestAddress, ulong size, uint value, bool isGds) + public void FillBuffer(ulong guestAddress, ulong size, uint value, bool isGds) => + FillBuffer(guestAddress, size, value, isGds, publishFill: false); + + private void FillBuffer(ulong guestAddress, ulong size, uint value, bool isGds, bool publishFill) { if ((guestAddress & 3) != 0 || size == 0 || (size & 3) != 0 || size > ulong.MaxValue - guestAddress) { @@ -408,56 +413,116 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable images.InvalidateMemoryFromGpu(guestAddress, size); var bufferIdentifier = FindBuffer(guestAddress, size); + if (Diagnostics.DccWriterTrace.Enabled) + { + Diagnostics.DccWriterTrace.CurrentKind = "fill"; + } + var (destination, destinationOffset) = ObtainBuffer(guestAddress, size, true, true, bufferIdentifier); destination.Fill(destinationOffset, size, value); - if (images.OverlapsDccMetadata(guestAddress, size) && TryWriteFillToBacking(guestAddress, size, value)) + if ((publishFill || images.OverlapsDccMetadata(guestAddress, size)) && TryWriteFillToBacking(guestAddress, size, value)) { _gpuModifiedRanges.Remove(guestAddress, size); - var firstPage = guestAddress & ~(TrackerLayout.PageBytes - 1); - for (var page = firstPage; page < guestAddress + size; page += TrackerLayout.PageBytes) - { - if (!_gpuModifiedRanges.Overlaps(page, TrackerLayout.PageBytes)) - { - _tracker.ClearGpuDirtyPages(page, TrackerLayout.PageBytes); - } - } + Interlocked.Increment(ref _gpuModifiedVersion); + ClearGpuDirtyPagesOfRemovedRange(guestAddress, size); + } + } + + // After a range left the GPU-modified set: the pages inside it are clean in one tracker walk, + // and only a page the range shares at either end can still hold another GPU-modified range. + private void ClearGpuDirtyPagesOfRemovedRange(ulong guestAddress, ulong size) + { + const ulong pageBytes = TrackerLayout.PageBytes; + var end = guestAddress + size; + var firstPage = guestAddress & ~(pageBytes - 1); + var lastPage = (end - 1) & ~(pageBytes - 1); + var startPartial = guestAddress != firstPage; + var endPartial = end != lastPage + pageBytes; + var innerBegin = startPartial ? firstPage + pageBytes : firstPage; + var innerEnd = endPartial ? lastPage : end; + if (innerBegin < innerEnd) + { + _tracker.ClearGpuDirtyPages(innerBegin, innerEnd - innerBegin); + } + + if (startPartial) + { + ClearGpuDirtyPageIfUnmodified(firstPage); + } + + if (endPartial && (lastPage != firstPage || !startPartial)) + { + ClearGpuDirtyPageIfUnmodified(lastPage); + } + } + + private void ClearGpuDirtyPageIfUnmodified(ulong page) + { + if (!_gpuModifiedRanges.Overlaps(page, TrackerLayout.PageBytes)) + { + _tracker.ClearGpuDirtyPages(page, TrackerLayout.PageBytes); } } public void FillDccMetadata(ulong guestAddress, ulong size, uint value) { if (guestAddress == 0 || (guestAddress & 3) != 0 || size == 0 || (size & 3) != 0 || size > ulong.MaxValue - guestAddress || - HasGpuDirtyBytes(guestAddress, size) || RequireImageCache().QueryRegion(guestAddress, size).ImageBytes) + HasGpuDirtyBytes(guestAddress, size)) { - FillBuffer(guestAddress, size, value, false); + FillBuffer(guestAddress, size, value, false, publishFill: true); return; } + // Images over the range read it again; only GPU-owned image bytes need the general fill. + // The buffer copy is filled on the GPU rather than uploaded from the written bytes. + var images = RequireImageCache(); + var region = images.QueryRegion(guestAddress, size); + if (region.GpuImageBytes) + { + FillBuffer(guestAddress, size, value, false, publishFill: true); + return; + } + + if (region.ImageBytes) + { + images.InvalidateMemory(guestAddress, size); + } + var (destination, destinationOffset) = ObtainBuffer(guestAddress, size, false, false, FindBuffer(guestAddress, size)); destination.Fill(destinationOffset, size, value); if (!TryWriteFillToBacking(guestAddress, size, value)) { - FillBuffer(guestAddress, size, value, false); + FillBuffer(guestAddress, size, value, false, publishFill: true); } } private bool TryWriteFillToBacking(ulong guestAddress, ulong size, uint value) { - var values = new uint[(int)Math.Min(size / sizeof(uint), 4096)]; - Array.Fill(values, value); - var bytes = MemoryMarshal.AsBytes(values); - for (ulong offset = 0; offset < size;) + // Metadata fills repeat every frame: the pattern buffer is pooled. + var count = (int)Math.Min(size / sizeof(uint), 4096); + var values = System.Buffers.ArrayPool.Shared.Rent(count); + try { - var chunk = (int)Math.Min(size - offset, (ulong)bytes.Length); - if (!_backing.TryWriteBacking(guestAddress + offset, bytes[..chunk])) + values.AsSpan(0, count).Fill(value); + var bytes = MemoryMarshal.AsBytes(values.AsSpan(0, count)); + for (ulong offset = 0; offset < size;) { - return false; + var chunk = (int)Math.Min(size - offset, (ulong)bytes.Length); + GpuWritebackEpoch.Advance(); + if (!_backing.TryWriteBacking(guestAddress + offset, bytes[..chunk])) + { + return false; + } + + offset += (ulong)chunk; } - offset += (ulong)chunk; + return true; + } + finally + { + System.Buffers.ArrayPool.Shared.Return(values); } - - return true; } public const ulong MaxHostCopyWords = 64 * 1024; @@ -583,6 +648,11 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var srcId = srcMemory ? FindBuffer(srcVaddr, size) : default; var dstId = dstMemory ? FindBuffer(dstVaddr, size) : default; var (src, srcOffset) = srcMemory ? ObtainBuffer(srcVaddr, size, false, true, srcId) : (_gds, srcVaddr); + if (Diagnostics.DccWriterTrace.Enabled) + { + Diagnostics.DccWriterTrace.CurrentKind = $"copy_from_0x{srcVaddr:X}"; + } + var (dst, dstOffset) = dstMemory ? ObtainBuffer(dstVaddr, size, true, true, dstId) : (_gds, dstVaddr); if (ReferenceEquals(src, dst) && srcOffset < dstOffset + size && dstOffset < srcOffset + size) { @@ -898,22 +968,34 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable if (!_relay.IsGpuQueueThread && AsyncReadback is { } readback && GuestReadsAwaitOffQueue) { - PendingDownload? pending = null; - if (_relay.TryRunOnGpuQueue(() => pending = BeginReadMemoryOnGpu(guestAddress, size, isWrite, source, allowPending: true))) + for (var attempt = 0; ; attempt++) { + var allowPending = attempt < MaxAsyncReadbackAttempts; + var markHot = attempt == 0 && !isWrite; + PendingDownload? pending = null; + if (!_relay.TryRunOnGpuQueue(() => + pending = BeginReadMemoryOnGpu(guestAddress, size, isWrite, source, allowPending, markHot))) + { + return AwaitShutdown(); + } + if (pending is null) { return true; } readback.Wait(pending.Ticket); - if (_relay.TryRunOnGpuQueue(() => CompleteReadMemoryOnGpu(pending, retrySynchronously: true))) + var complete = false; + if (!_relay.TryRunOnGpuQueue(() => complete = CompleteReadMemoryOnGpu(pending, consumed: true))) + { + return AwaitShutdown(); + } + + if (complete) { return true; } } - - return AwaitShutdown(); } var onQueue = _relay.IsGpuQueueThread; @@ -934,6 +1016,9 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable private const ulong ReadbackWindowBytes = 512 * 1024; private const long HotWindowLifetime = 512; private const int MaxEagerReadbacks = 4; + private const int MaxAsyncReadbackAttempts = 4; + private const int EagerStrikeLimit = 3; + private const long EagerMuteBatches = 4096; private readonly Dictionary _hotWindows = new(); private readonly HashSet _eagerCandidates = new(); private readonly List _eagerDownloads = new(); @@ -941,8 +1026,9 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable private static long _reportedEagerStarted; private static long _reportedEagerUsed; private static long _reportedEagerStale; + private static long _reportedEagerMuted; - private readonly record struct HotWindow(ulong Address, ulong Size, long LastHit); + private readonly record struct HotWindow(ulong Address, ulong Size, long LastHit, int Strikes = 0, long MutedUntil = 0); internal long EagerReadbacksStarted { get; private set; } @@ -950,7 +1036,44 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable private void NoteHotReadback(ulong guestAddress, ulong size) { - _hotWindows[guestAddress & ~(ReadbackWindowBytes - 1)] = new HotWindow(guestAddress, size, _batchSerial); + var key = guestAddress & ~(ReadbackWindowBytes - 1); + _hotWindows[key] = _hotWindows.TryGetValue(key, out var hot) + ? hot with { Address = guestAddress, Size = size, LastHit = _batchSerial } + : new HotWindow(guestAddress, size, _batchSerial); + } + + internal long EagerReadbacksWasted { get; private set; } + + private void NoteEagerOutcome(ulong key, bool wasted) + { + if (wasted) + { + EagerReadbacksWasted++; + } + + if (!_hotWindows.TryGetValue(key, out var hot)) + { + return; + } + + if (!wasted) + { + if (hot.Strikes != 0) + { + _hotWindows[key] = hot with { Strikes = 0 }; + } + + return; + } + + var strikes = hot.Strikes + 1; + _hotWindows[key] = strikes >= EagerStrikeLimit + ? hot with { Strikes = 0, MutedUntil = _batchSerial + EagerMuteBatches } + : hot with { Strikes = strikes }; + if (strikes >= EagerStrikeLimit) + { + Interlocked.Increment(ref _reportedEagerMuted); + } } private void NoteHotWindowWrite(ulong guestAddress, ulong size) @@ -963,10 +1086,11 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var last = (guestAddress + size - 1) & ~(ReadbackWindowBytes - 1); for (var key = guestAddress & ~(ReadbackWindowBytes - 1); key <= last; key += ReadbackWindowBytes) { - if (_hotWindows.TryGetValue(key, out var hot) && _batchSerial - hot.LastHit <= HotWindowLifetime) + if (_hotWindows.TryGetValue(key, out var hot) && _batchSerial - hot.LastHit <= HotWindowLifetime && + _batchSerial >= hot.MutedUntil) { _eagerCandidates.Add(key); - HotWritePending = true; + HotWritePending |= hot.Strikes == 0; } } } @@ -988,16 +1112,14 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable if (readback.IsComplete(pending.Ticket)) { _eagerDownloads.RemoveAt(index); - if (!CompleteReadMemoryOnGpu(pending, retrySynchronously: false)) - { - Interlocked.Increment(ref _reportedEagerStale); - } + _ = CompleteReadMemoryOnGpu(pending, consumed: false); } } if ((_batchSerial & 1023) == 0) { - foreach (var key in _hotWindows.Where(pair => _batchSerial - pair.Value.LastHit > HotWindowLifetime).Select(pair => pair.Key).ToList()) + foreach (var key in _hotWindows.Where(pair => _batchSerial - pair.Value.LastHit > HotWindowLifetime && _batchSerial >= pair.Value.MutedUntil) + .Select(pair => pair.Key).ToList()) { _hotWindows.Remove(key); } @@ -1008,6 +1130,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable return; } + List? settled = null; foreach (var key in _eagerCandidates) { if (_eagerDownloads.Count >= MaxEagerReadbacks) @@ -1015,26 +1138,52 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable break; } - if (_hotWindows.TryGetValue(key, out var hot)) + if (!_hotWindows.TryGetValue(key, out var hot) || _batchSerial - hot.LastHit > HotWindowLifetime || + _batchSerial < hot.MutedUntil || StartEagerReadback(hot) != EagerStart.Deferred) { - StartEagerReadback(hot); + (settled ??= []).Add(key); } } - _eagerCandidates.Clear(); - } - - private void StartEagerReadback(HotWindow hot) - { - if (FindOwner(hot.Address, hot.Size) is not { } buffer) + if (settled is null) { return; } - var copies = CollectReadbackWindow(buffer, hot.Address, hot.Size, out var windowBegin, out var windowEnd); - if (copies.Count == 0 || !TryBeginDownloadAsync(copies, out var ticket, out var writeTicks)) + foreach (var key in settled) { - return; + _eagerCandidates.Remove(key); + } + } + + private enum EagerStart + { + Started, + Deferred, + Nothing, + } + + private EagerStart StartEagerReadback(HotWindow hot) + { + if (FindOwner(hot.Address, hot.Size) is not { } buffer) + { + return EagerStart.Nothing; + } + + if (OverlapsEagerReadback(hot.Address, hot.Size)) + { + return EagerStart.Deferred; + } + + var copies = CollectReadbackWindow(buffer, hot.Address, hot.Size, out var windowBegin, out var windowEnd); + if (copies.Count == 0) + { + return EagerStart.Nothing; + } + + if (!TryBeginDownloadAsync(copies, out var ticket, out var writeTicks)) + { + return EagerStart.Deferred; } foreach (var copy in copies) @@ -1044,6 +1193,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable _eagerDownloads.Add(new PendingDownload { + HotKey = hot.Address & ~(ReadbackWindowBytes - 1), Ticket = ticket, Copies = copies, WriteTicks = writeTicks, @@ -1052,11 +1202,26 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable GuestAddress = hot.Address, Size = hot.Size, IsWrite = false, + Eager = true, Source = GuestMemoryProfile.ReadbackSource.ShaderResourceRead, Started = System.Diagnostics.Stopwatch.GetTimestamp(), }); EagerReadbacksStarted++; Interlocked.Increment(ref _reportedEagerStarted); + return EagerStart.Started; + } + + private bool OverlapsEagerReadback(ulong guestAddress, ulong size) + { + foreach (var pending in _eagerDownloads) + { + if (guestAddress < pending.WindowEnd && pending.WindowBegin < guestAddress + size) + { + return true; + } + } + + return false; } private PendingDownload? TakeEagerReadback(ulong guestAddress, ulong size) @@ -1068,6 +1233,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable guestAddress - pending.WindowBegin <= pending.WindowEnd - pending.WindowBegin - size) { _eagerDownloads.RemoveAt(index); + NoteHotReadback(guestAddress, size); return pending; } } @@ -1083,15 +1249,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } readback.Wait(pending.Ticket); - if (!CompleteReadMemoryOnGpu(pending, retrySynchronously: false)) - { - Interlocked.Increment(ref _reportedEagerStale); - return false; - } - - EagerReadbacksUsed++; - Interlocked.Increment(ref _reportedEagerUsed); - return !HasGpuDirtyBytes(guestAddress, size); + return CompleteReadMemoryOnGpu(pending with { GuestAddress = guestAddress, Size = size }, consumed: true); } private void AbandonEagerReadbacks() @@ -1131,9 +1289,6 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } } - private void ReadMemoryOnGpu(ulong guestAddress, ulong size, bool isWrite, GuestMemoryProfile.ReadbackSource source) => - _ = BeginReadMemoryOnGpu(guestAddress, size, isWrite, source, allowPending: false); - internal long PendingReadbacksApplied { get; private set; } internal long PendingReadbacksRetried { get; private set; } @@ -1141,8 +1296,13 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable private static long _reportedApplied; private static long _reportedRetried; + private static long _reportedStalePieces; + private static long _reportedPublishedPieces; + + internal long StalePiecesKept { get; private set; } + public static string TakeAsyncReadbackReport() => FormattableString.Invariant( - $"[PERF][ASYNC_READBACK] applied={Interlocked.Exchange(ref _reportedApplied, 0)} retried={Interlocked.Exchange(ref _reportedRetried, 0)} eager_started={Interlocked.Exchange(ref _reportedEagerStarted, 0)} eager_used={Interlocked.Exchange(ref _reportedEagerUsed, 0)} eager_stale={Interlocked.Exchange(ref _reportedEagerStale, 0)}"); + $"[PERF][ASYNC_READBACK] applied={Interlocked.Exchange(ref _reportedApplied, 0)} retried={Interlocked.Exchange(ref _reportedRetried, 0)} published_pieces={Interlocked.Exchange(ref _reportedPublishedPieces, 0)} stale_pieces={Interlocked.Exchange(ref _reportedStalePieces, 0)} eager_started={Interlocked.Exchange(ref _reportedEagerStarted, 0)} eager_used={Interlocked.Exchange(ref _reportedEagerUsed, 0)} eager_stale={Interlocked.Exchange(ref _reportedEagerStale, 0)} eager_muted={Interlocked.Exchange(ref _reportedEagerMuted, 0)}"); private sealed record PendingDownload { @@ -1154,6 +1314,8 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable public required ulong GuestAddress { get; init; } public required ulong Size { get; init; } public required bool IsWrite { get; init; } + public bool Eager { get; init; } + public ulong HotKey { get; init; } public required GuestMemoryProfile.ReadbackSource Source { get; init; } public required long Started { get; init; } } @@ -1169,7 +1331,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable return null; } - if (allowPending && TakeEagerReadback(guestAddress, size) is { } eager) + if (!isWrite && allowPending && TakeEagerReadback(guestAddress, size) is { } eager) { return eager with { GuestAddress = guestAddress, Size = size, IsWrite = isWrite, Source = source }; } @@ -1216,20 +1378,87 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable return null; } - private bool CompleteReadMemoryOnGpu(PendingDownload pending, bool retrySynchronously) + private bool CompleteReadMemoryOnGpu(PendingDownload pending, bool consumed) { using var readbackScope = GuestMemoryProfile.Measure(GuestMemoryProfile.Operation.BufferReadback); var readback = AsyncReadback ?? throw SubmissionScheduler.Fatal("The pending readback lost its queue."); try { - if (!IsPendingDownloadCurrent(pending)) + var copies = pending.Copies; + var current = new bool[copies.Count]; + var published = 0; + for (var index = 0; index < copies.Count; index++) { - readback.Complete(pending.Ticket, null); - if (retrySynchronously) + current[index] = IsPieceCurrent(pending, index); + if (current[index]) { - PendingReadbacksRetried++; - Interlocked.Increment(ref _reportedRetried); - ReadMemoryOnGpu(pending.GuestAddress, pending.Size, pending.IsWrite, pending.Source); + published++; + } + } + + readback.Complete(pending.Ticket, published == 0 ? null : (index, bytes) => + { + if (!current[index]) + { + return; + } + + GpuWritebackEpoch.Advance(); + if (!_backing.TryWriteBacking(copies[index].Address, bytes)) + { + throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copies[index].Address:X16} size=0x{(ulong)bytes.Length:X16}"); + } + }); + for (var index = 0; index < copies.Count; index++) + { + if (current[index]) + { + _gpuModifiedRanges.Remove(copies[index].Address, copies[index].Size); + Interlocked.Increment(ref _gpuModifiedVersion); + } + } + + for (var index = 0; index < copies.Count; index++) + { + if (current[index]) + { + ClearGpuDirtyPagesOfRemovedRange(copies[index].Address, copies[index].Size); + } + } + + var stale = copies.Count - published; + if (pending.Eager) + { + NoteEagerOutcome(pending.HotKey, stale != 0); + } + + if (stale != 0) + { + StalePiecesKept += stale; + Interlocked.Add(ref _reportedStalePieces, stale); + } + + Interlocked.Add(ref _reportedPublishedPieces, published); + var clean = pending.IsWrite + ? !_tracker.HasGpuDirtyPages(pending.GuestAddress, pending.Size) + : !HasGpuDirtyBytes(pending.GuestAddress, pending.Size); + if (!consumed) + { + if (pending.Eager && stale != 0) + { + Interlocked.Increment(ref _reportedEagerStale); + } + + return clean; + } + + if (!clean) + { + PendingReadbacksRetried++; + Interlocked.Increment(ref _reportedRetried); + if (pending.Eager) + { + Interlocked.Increment(ref _reportedEagerStale); } return false; @@ -1237,21 +1466,12 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable PendingReadbacksApplied++; Interlocked.Increment(ref _reportedApplied); - - var copies = pending.Copies; - readback.Complete(pending.Ticket, (index, bytes) => + if (pending.Eager) { - if (!_backing.TryWriteBacking(copies[index].Address, bytes)) - { - throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copies[index].Address:X16} size=0x{(ulong)bytes.Length:X16}"); - } - }); - foreach (var copy in copies) - { - _gpuModifiedRanges.Remove(copy.Address, copy.Size); + EagerReadbacksUsed++; + Interlocked.Increment(ref _reportedEagerUsed); } - _tracker.ClearGpuDirtyPages(pending.WindowBegin, pending.WindowEnd - pending.WindowBegin); if (pending.IsWrite) { _tracker.MarkCpuDirtyPages(pending.GuestAddress, pending.Size); @@ -1269,29 +1489,12 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } } - private bool IsPendingDownloadCurrent(PendingDownload pending) + private bool IsPieceCurrent(PendingDownload pending, int index) { - var total = 0UL; - for (var index = 0; index < pending.Copies.Count; index++) - { - var copy = pending.Copies[index]; - if (copy.Buffer.LastGpuWriteTick != pending.WriteTicks[index] || - !_gpuModifiedRanges.Contains(copy.Address, copy.Size) || - !ReferenceEquals(FindOwner(copy.Address, copy.Size), copy.Buffer)) - { - return false; - } - - total += copy.Size; - } - - var current = 0UL; - foreach (var range in _gpuModifiedRanges.GetOverlappingRanges(pending.WindowBegin, pending.WindowEnd - pending.WindowBegin)) - { - current += range.Size; - } - - return current == total; + var copy = pending.Copies[index]; + return copy.Buffer.LastGpuWriteTickIn(copy.SourceOffset, copy.Size) == pending.WriteTicks[index] && + _gpuModifiedRanges.Contains(copy.Address, copy.Size) && + ReferenceEquals(FindOwner(copy.Address, copy.Size), copy.Buffer); } private static void RecordReadback(ulong windowBegin, ulong windowEnd, bool isWrite, List copies, long started, @@ -1456,6 +1659,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable foreach (var copy in copies) { _gpuModifiedRanges.Remove(copy.Address, copy.Size); + Interlocked.Increment(ref _gpuModifiedVersion); } return; @@ -1492,6 +1696,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable foreach (var copy in copies) { _gpuModifiedRanges.Remove(copy.Address, copy.Size); + Interlocked.Increment(ref _gpuModifiedVersion); } } @@ -1509,7 +1714,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var ticks = new ulong[copies.Count]; for (var index = 0; index < copies.Count; index++) { - var written = copies[index].Buffer.LastGpuWriteTick; + var written = copies[index].Buffer.LastGpuWriteTickIn(copies[index].SourceOffset, copies[index].Size); if (written == 0) { return false; @@ -1559,7 +1764,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var total = 0UL; foreach (var copy in copies) { - var written = copy.Buffer.LastGpuWriteTick; + var written = copy.Buffer.LastGpuWriteTickIn(copy.SourceOffset, copy.Size); // A GPU-modified range with no recorded writer: keep the conservative path. if (written == 0) { @@ -1590,6 +1795,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable readback.Read(pieces, waitTick, (index, bytes) => { + GpuWritebackEpoch.Advance(); if (!_backing.TryWriteBacking(copies[index].Address, bytes)) { throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copies[index].Address:X16} size=0x{(ulong)bytes.Length:X16}"); @@ -1622,6 +1828,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var placement = copy.Placement; var offset = baseOffset + placement.DataOffset; _download.Invalidate(offset, placement.DataSize); + GpuWritebackEpoch.Advance(); if (!_backing.TryWriteBacking(copy.Address, _download.Mapped.Slice((int)offset, (int)placement.DataSize))) { throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copy.Address:X16} size=0x{placement.DataSize:X16}"); @@ -1702,34 +1909,99 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.BufferDirtySynchronization); var startedAt = BufferUploadProfile.Enabled ? System.Diagnostics.Stopwatch.GetTimestamp() : 0; - if ((!preserveCpuWriteHotPages && !isWritten && !isTexelBuffer && !_tracker.MayHaveCpuDirtyPages(guestAddress, size)) || - (isWritten && !isTexelBuffer && _gpuModifiedRanges.Contains(guestAddress, size))) + var nothingToUpload = isWritten + ? _gpuModifiedRanges.Contains(guestAddress, size) + : !_tracker.MayHaveCpuDirtyPages(guestAddress, size); + if (nothingToUpload) { if (BufferUploadProfile.Enabled) BufferUploadProfile.Record(guestAddress, size, 0, 0, 0, System.Diagnostics.Stopwatch.GetTimestamp() - startedAt); - return false; + return isTexelBuffer && SynchronizeTexelFromImage(buffer, guestAddress, size); } profileScope.SwitchPhase(isWritten ? RenderPhaseProfile.Phase.BufferDirtySyncWritten : isTexelBuffer ? RenderPhaseProfile.Phase.BufferDirtySyncTexel : RenderPhaseProfile.Phase.BufferDirtySyncUpload); - var copies = new List(); - var totalSize = 0UL; - GpuBuffer? source = null; - _tracker.ForEachUploadRange( - guestAddress, - size, - isWritten, - (address, bytes) => + var collector = _uploadCollector; + if (collector is null || collector.InUse) + { + // The first call on a thread keeps its collector; a nested one gets its own. + collector = new UploadCollector(); + _uploadCollector ??= collector; + } + + collector.Begin(this, buffer, guestAddress, size); + try + { + return SynchronizeBufferCore(collector, buffer, guestAddress, size, isWritten, isTexelBuffer, preserveCpuWriteHotPages, startedAt); + } + finally + { + collector.End(); + } + } + + // The ranges of one synchronization. The delegates are bound once, so a call that has dirty + // pages to upload allocates no closure and no list. + private sealed class UploadCollector + { + private GuestBufferCache _cache = null!; + private GpuBuffer _buffer = null!; + private ulong _guestAddress; + private ulong _size; + public readonly List Copies = new(); + public ulong TotalSize; + public GpuBuffer? Source; + public bool InUse; + public readonly Action Collect; + public readonly Action Upload; + + public UploadCollector() + { + Collect = (address, bytes) => { - copies.Add(new BufferCopy(totalSize, buffer.Offset(address), bytes)); - totalSize += bytes; - }, - () => source = _uploader.PrepareSource(buffer.CpuAddress, CollectionsMarshal.AsSpan(copies), totalSize, guestAddress, size), - preserveCpuWriteHotPages); + Copies.Add(new BufferCopy(TotalSize, _buffer.Offset(address), bytes)); + TotalSize += bytes; + }; + Upload = () => Source = _cache._uploader.PrepareSource(_buffer.CpuAddress, CollectionsMarshal.AsSpan(Copies), TotalSize, _guestAddress, _size); + } + + public void Begin(GuestBufferCache cache, GpuBuffer buffer, ulong guestAddress, ulong size) + { + InUse = true; + _cache = cache; + _buffer = buffer; + _guestAddress = guestAddress; + _size = size; + } + + public void End() + { + InUse = false; + _cache = null!; + _buffer = null!; + Copies.Clear(); + TotalSize = 0; + Source = null; + } + } + + [ThreadStatic] private static UploadCollector? _uploadCollector; + + private bool SynchronizeBufferCore(UploadCollector collector, GpuBuffer buffer, ulong guestAddress, ulong size, bool isWritten, + bool isTexelBuffer, bool preserveCpuWriteHotPages, long startedAt) + { + var copies = collector.Copies; + _tracker.ForEachUploadRange(guestAddress, size, isWritten, collector.Collect, collector.Upload, preserveCpuWriteHotPages); + var source = collector.Source; + var totalSize = collector.TotalSize; if (source != null) { - buffer.NoteGpuWrite(); + foreach (var copy in copies) + { + buffer.NoteGpuWrite(copy.DstOffset, copy.Size); + } + var command = _scheduler.Current; command.EndRendering(); var native = new CommandBuffer(command.Handle); @@ -1771,18 +2043,18 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable BufferUploadProfile.Record(guestAddress, size, copies.Count, totalSize, hotBytes, elapsedTicks); } - if (isTexelBuffer) - { - var copiedFromImage = RequireImageCache().TrySynchronizeBufferFromImage(buffer, guestAddress, size); - if (copiedFromImage) - { - buffer.NoteGpuWrite(); - } + return isTexelBuffer && SynchronizeTexelFromImage(buffer, guestAddress, size); + } - return copiedFromImage; + private bool SynchronizeTexelFromImage(GpuBuffer buffer, ulong guestAddress, ulong size) + { + var copiedFromImage = RequireImageCache().TrySynchronizeBufferFromImage(buffer, guestAddress, size); + if (copiedFromImage) + { + buffer.NoteGpuWrite(buffer.Offset(guestAddress), size); } - return false; + return copiedFromImage; } private GpuBuffer? FindOwner(ulong guestAddress, ulong size) diff --git a/src/SharpEmu.Libs/Gpu/Buffers/WriteTickMap.cs b/src/SharpEmu.Libs/Gpu/Buffers/WriteTickMap.cs new file mode 100644 index 0000000..66d8d5d --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Buffers/WriteTickMap.cs @@ -0,0 +1,143 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Gpu.Buffers; + +public sealed class WriteTickMap +{ + private readonly List _entries = []; + private ulong _floor; + + private readonly record struct Entry(ulong Begin, ulong End, ulong Tick); + + public int Count => _entries.Count; + + public ulong Floor => _floor; + + public void Assign(ulong begin, ulong size, ulong tick) + { + if (size == 0) + { + return; + } + + var end = begin + size; + var index = FirstEndingAfter(begin); + var removed = 0; + while (index + removed < _entries.Count && _entries[index + removed].Begin < end) + { + removed++; + } + + Entry? left = null; + Entry? right = null; + if (removed != 0) + { + var first = _entries[index]; + if (first.Begin < begin) + { + left = first with { End = begin }; + } + + var last = _entries[index + removed - 1]; + if (last.End > end) + { + right = last with { Begin = end }; + } + + _entries.RemoveRange(index, removed); + } + + var insert = index; + if (left is { } leftPiece) + { + _entries.Insert(insert++, leftPiece); + } + + _entries.Insert(insert, new Entry(begin, end, tick)); + var middle = insert++; + if (right is { } rightPiece) + { + _entries.Insert(insert, rightPiece); + } + + if (middle + 1 < _entries.Count) + { + _ = TryMerge(middle, middle + 1); + } + + if (middle > 0) + { + _ = TryMerge(middle - 1, middle); + } + } + + public ulong MaxTick(ulong begin, ulong size) + { + var tick = _floor; + if (size == 0) + { + return tick; + } + + var end = begin + size; + for (var index = FirstEndingAfter(begin); index < _entries.Count && _entries[index].Begin < end; index++) + { + tick = Math.Max(tick, _entries[index].Tick); + } + + return tick; + } + + public void Prune(ulong below) + { + var kept = 0; + for (var index = 0; index < _entries.Count; index++) + { + var entry = _entries[index]; + if (entry.Tick < below) + { + _floor = Math.Max(_floor, entry.Tick); + continue; + } + + _entries[kept++] = entry; + } + + _entries.RemoveRange(kept, _entries.Count - kept); + } + + private bool TryMerge(int left, int right) + { + var a = _entries[left]; + var b = _entries[right]; + if (a.End != b.Begin || a.Tick != b.Tick) + { + return false; + } + + _entries[left] = a with { End = b.End }; + _entries.RemoveAt(right); + return true; + } + + private int FirstEndingAfter(ulong address) + { + var low = 0; + var high = _entries.Count; + while (low < high) + { + var middle = low + ((high - low) >> 1); + if (_entries[middle].End <= address) + { + low = middle + 1; + } + else + { + high = middle; + } + } + + return low; + } +} diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs index 1db6ec3..2c6fd44 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs @@ -529,7 +529,7 @@ public sealed class CommandStreamQueue if (submission.Kind == CommandSubmissionKind.FrameBoundary) { SlicesRun++; - _host.Flush(); + _host.FlushFrame(); return true; } diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs index 268275b..23adb2a 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs @@ -26,7 +26,7 @@ public sealed partial class GpuCommandInterpreter } // A compare-swap loop applies once per try and suspends while the compare fails. - if (!atomic.TryApply(_host.TryReadGuest, _host.Memory, out var priorValue, out _, out var comparePassed)) + if (!atomic.TryApply(_host.TryReadGuestOperand, _host.Memory, out var priorValue, out _, out var comparePassed)) { throw _host.Fatal($"The atomic address cannot be accessed: address=0x{atomic.Address:X16} size={atomic.ByteCount}."); } @@ -68,7 +68,7 @@ public sealed partial class GpuCommandInterpreter if (semaphore.IsSignal) { - if (!Packets.MemorySemaphorePacket.TrySignal(_host.TryReadGuest, _host.Memory, semaphore.Address, semaphore.WriteSignal, out _)) + if (!Packets.MemorySemaphorePacket.TrySignal(_host.TryReadGuestOperand, _host.Memory, semaphore.Address, semaphore.WriteSignal, out _)) { throw _host.Fatal($"The semaphore cannot be signaled: address=0x{semaphore.Address:X16}."); } @@ -77,7 +77,7 @@ public sealed partial class GpuCommandInterpreter } // A wait takes one token, or suspends until a signal adds one. - if (!Packets.MemorySemaphorePacket.TryConsume(_host.TryReadGuest, _host.Memory, semaphore.Address, out var priorValue)) + if (!Packets.MemorySemaphorePacket.TryConsume(_host.TryReadGuestOperand, _host.Memory, semaphore.Address, out var priorValue)) { throw _host.Fatal($"The semaphore cannot be read: address=0x{semaphore.Address:X16}."); } diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs index 3b98876..1fec559 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs @@ -11,7 +11,6 @@ namespace SharpEmu.Libs.Gpu.GpuCommands; public sealed partial class GpuCommandInterpreter { private const uint DmaSkippedDestination = 0x3022C; - private const uint LodStatsWithoutCounters = 1; // Destination selectors 1, 2, 4 and 5 are memory; registers and GDS are not supported. internal void WriteData(ulong destination, ReadOnlySpan source, uint writeControl) @@ -367,15 +366,16 @@ public sealed partial class GpuCommandInterpreter var bufferSize = payload[0]; var destination = (payload[1] & 0xFFFF_FFC0u) | ((ulong)payload[2] << 32); + if (destination != 0 && bufferSize != 0 && _host.TryRecordLodStatistics(destination, bufferSize)) + { + return 4; + } + if (destination != 0 && bufferSize != 0) { var report = new byte[bufferSize]; - var hasCounters = MipStatistics.Shared.TryWriteReport(report); + MipStatistics.Shared.WriteReport(report); WriteBytes(destination, report); - if (bufferSize >= sizeof(uint)) - { - WriteDword(destination, hasCounters ? MipStatistics.FirstCounterOffset : LodStatsWithoutCounters); - } } return 4; diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs index a168128..9b42efc 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs @@ -82,8 +82,8 @@ public sealed partial class GpuCommandInterpreter var next = packet.PacketAddress + (ulong)packet.Length * sizeof(uint); var expectedHeader = PacketHeader.Make(centroidPacketDwords, PacketOpcode.SetContextRegister); if (packet.Remaining < packet.Length + centroidPacketDwords || - ReadDword(next) != expectedHeader || - RegisterBankLayout.Normalize(ReadDword(next + sizeof(uint))) != ContextRegisterOffset.PaScCentroidPriority0) + ReadDword(next, RenderPhaseProfile.CommandReadKind.Payload) != expectedHeader || + RegisterBankLayout.Normalize(ReadDword(next + sizeof(uint), RenderPhaseProfile.CommandReadKind.Payload)) != ContextRegisterOffset.PaScCentroidPriority0) { throw _host.Fatal($"The sample location packet is not followed by the centroid priority packet: header=0x{packet.Header:X8} address=0x{packet.PacketAddress:X16} next=0x{next:X16}."); } @@ -102,16 +102,16 @@ public sealed partial class GpuCommandInterpreter var address = packet.PacketAddress; var registerHeader = PacketHeader.Make(3, PacketOpcode.SetContextRegister); - if (ReadDword(address + 40) != registerHeader || ReadDword(address + 44) != RegisterBankLayout.DepthInfo || - ReadDword(address + 52) != registerHeader || ReadDword(address + 56) != RegisterBankLayout.DepthView || - ReadDword(address + 64) != registerHeader || ReadDword(address + 68) != RegisterBankLayout.HtileDataBase || - ReadDword(address + 76) != registerHeader || ReadDword(address + 80) != RegisterBankLayout.HtileSurface || - ReadDword(address + 88) != PacketHeader.Make(2, PacketOpcode.Nop)) + if (ReadDword(address + 40, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 44, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.DepthInfo || + ReadDword(address + 52, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 56, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.DepthView || + ReadDword(address + 64, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 68, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.HtileDataBase || + ReadDword(address + 76, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 80, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.HtileSurface || + ReadDword(address + 88, RenderPhaseProfile.CommandReadKind.Payload) != PacketHeader.Make(2, PacketOpcode.Nop)) { return; } - var sizeXy = ReadDword(address + 92); + var sizeXy = ReadDword(address + 92, RenderPhaseProfile.CommandReadKind.Payload); if (sizeXy != 0) { TypedRegisters.CompositeDepthSizeXy = sizeXy; diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs index 7e6f8fd..97b7928 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs @@ -28,7 +28,7 @@ public sealed partial class GpuCommandInterpreter return; } - var value = is64Bit ? ReadQword(address) : ReadDword(address); + var value = ReadWaitOperand(address, is64Bit); if (!WaitOperation.TryCompare(value, reference, mask, compareFunction, out var satisfied)) { throw _host.Fatal($"The wait compare function is unknown: function={compareFunction} address=0x{address:X16}."); diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs index ef91e56..d10d372 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs @@ -394,7 +394,12 @@ public sealed partial class GpuCommandInterpreter { RenderPhaseProfile.RecordCommandRead(readKind, sizeof(uint)); Span bytes = stackalloc byte[sizeof(uint)]; - if (!_host.TryReadGuest(address, bytes)) + // The packets themselves are command data; anything else is an operand earlier work may write. + var read = readKind is RenderPhaseProfile.CommandReadKind.Header or RenderPhaseProfile.CommandReadKind.Payload or + RenderPhaseProfile.CommandReadKind.RegisterTable + ? _host.TryReadGuest(address, bytes) + : _host.TryReadGuestOperand(address, bytes); + if (!read) { throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size=4."); } @@ -406,7 +411,7 @@ public sealed partial class GpuCommandInterpreter { RenderPhaseProfile.RecordCommandRead(RenderPhaseProfile.CommandReadKind.Operand64, sizeof(ulong)); Span bytes = stackalloc byte[sizeof(ulong)]; - if (!_host.TryReadGuest(address, bytes)) + if (!_host.TryReadGuestOperand(address, bytes)) { throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size=8."); } @@ -414,10 +419,24 @@ public sealed partial class GpuCommandInterpreter return BinaryPrimitives.ReadUInt64LittleEndian(bytes); } + private ulong ReadWaitOperand(ulong address, bool is64Bit) + { + RenderPhaseProfile.RecordCommandRead(is64Bit ? RenderPhaseProfile.CommandReadKind.Operand64 : RenderPhaseProfile.CommandReadKind.Operand32, + is64Bit ? sizeof(ulong) : sizeof(uint)); + Span bytes = stackalloc byte[sizeof(ulong)]; + var size = is64Bit ? sizeof(ulong) : sizeof(uint); + if (!_host.TryReadGuestWaitOperand(address, bytes[..size])) + { + throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size={size}."); + } + + return is64Bit ? BinaryPrimitives.ReadUInt64LittleEndian(bytes) : BinaryPrimitives.ReadUInt32LittleEndian(bytes); + } + internal void ReadBytes(ulong address, Span destination) { RenderPhaseProfile.RecordCommandRead(RenderPhaseProfile.CommandReadKind.Other, destination.Length); - if (!_host.TryReadGuest(address, destination)) + if (!_host.TryReadGuestOperand(address, destination)) { throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size={destination.Length}."); } diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs index 6196265..f1e0b9d 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs @@ -81,9 +81,19 @@ public interface ICommandStreamHost { ICpuMemory Memory { get; } + bool TryRecordLodStatistics(ulong destination, uint size) => false; + // Reads guest memory the GPU may have written; the host synchronizes GPU-owned pages first. bool TryReadGuest(ulong address, Span destination); + // Reads a packet operand (a label, a predicate, indirect arguments): memory earlier work may + // write, unlike the packets themselves. A host that defers work completes it first. + bool TryReadGuestOperand(ulong address, Span destination) => TryReadGuest(address, destination); + + // Reads the label a wait polls. A stale value only suspends the wait until a retry, so a host + // that runs work behind the stream need not complete it first. + bool TryReadGuestWaitOperand(ulong address, Span destination) => TryReadGuestOperand(address, destination); + // Runs commands other threads posted to this worker. Called before every packet. void RunPendingCommands(); @@ -92,6 +102,10 @@ public interface ICommandStreamHost void Flush(); + // A frame boundary: submits what the frame recorded. The submitter resumes once it returns, so a + // host that runs the stream ahead of its recording returns only after the frame is recorded. + void FlushFrame() => Flush(); + void FlushAndWait(); void SynchronizeGpu(); diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs index 5ad7830..1e6f6a5 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs @@ -338,8 +338,15 @@ public sealed class ScreenViewportRegisters public ScreenViewportRegisters Copy() { - var copy = (ScreenViewportRegisters)MemberwiseClone(); + var copy = CopySharingViewports(); copy.Viewports = (ViewportRegisters[])Viewports.Clone(); + return copy; + } + + // A copy that shares the viewport array, the largest part, until RegisterBanks copies it. + public ScreenViewportRegisters CopySharingViewports() + { + var copy = (ScreenViewportRegisters)MemberwiseClone(); copy.ClipRectangleLeft = (int[])ClipRectangleLeft.Clone(); copy.ClipRectangleTop = (int[])ClipRectangleTop.Clone(); copy.ClipRectangleRight = (int[])ClipRectangleRight.Clone(); @@ -388,6 +395,9 @@ public sealed class ContextRegisters public EnhancedQualityAntialiasingRegisters EnhancedQualityAntialiasing; public ShaderInterfaceRegisters ShaderInterface = new(); + // A copy that shares every nested part; RegisterBanks copies a part before writing it. + public ContextRegisters ShallowCopy() => (ContextRegisters)MemberwiseClone(); + public uint RenderTargetMaskForSlot(uint slot) => (RenderTargetMask >> (int)(slot * 4)) & 0xFu; public ContextRegisters Copy() diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs index 8715653..7c20201 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs @@ -1,6 +1,8 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using SharpEmu.Libs.Gpu.Images; + namespace SharpEmu.Libs.Gpu.GpuCommands.Registers; // The three typed register banks of one queue, with the state the register writers carry between packets. @@ -9,15 +11,31 @@ public sealed class RegisterBanks private ContextRegisters? _savedContext; private uint? _savedCompositeDepthSizeXy; - public RegisterBanks(Func fatal) => Fatal = fatal; + public RegisterBanks(Func fatal) + : this(fatal, new ContextRegisters(), new ShaderProgramRegisters(), new UserConfigRegisters()) + { + } + + // Banks over given parts, so a copy or a snapshot does not first build empty ones. + private RegisterBanks( + Func fatal, + ContextRegisters context, + ShaderProgramRegisters shader, + UserConfigRegisters userConfig) + { + Fatal = fatal; + Context = context; + Shader = shader; + UserConfig = userConfig; + } public Func Fatal { get; } - public ContextRegisters Context { get; private set; } = new(); + public ContextRegisters Context { get; private set; } - public ShaderProgramRegisters Shader { get; private set; } = new(); + public ShaderProgramRegisters Shader { get; private set; } - public UserConfigRegisters UserConfig { get; private set; } = new(); + public UserConfigRegisters UserConfig { get; private set; } public bool ContextPushed => _savedContext is not null; @@ -31,11 +49,8 @@ public sealed class RegisterBanks // A copy of the live banks; the saved context and the draw-time markers are not part of it. public RegisterBanks Clone() { - var clone = new RegisterBanks(Fatal) + var clone = new RegisterBanks(Fatal, Context.Copy(), Shader.Copy(), UserConfig.Copy()) { - Context = Context.Copy(), - Shader = Shader.Copy(), - UserConfig = UserConfig.Copy(), CompositeDepthSizeXy = CompositeDepthSizeXy, UserDataMarker = UserDataMarker, IndexTypeAndSize = IndexTypeAndSize, @@ -43,9 +58,281 @@ public sealed class RegisterBanks return clone; } + // The parts a snapshot shares with these banks; each is copied before its next write here. + private bool _contextShared; + private bool _vertexShared; + private bool _pixelShared; + private bool _computeShared; + private bool _userConfigShared; + + // A read-only copy that shares every part with the live banks until they next write it: a + // deferred draw takes one per draw, and between draws only a few parts change. + public RegisterBanks Snapshot() + { + var snapshot = new RegisterBanks(Fatal, Context, Shader.ShareStages(), UserConfig) + { + CompositeDepthSizeXy = CompositeDepthSizeXy, + UserDataMarker = UserDataMarker, + IndexTypeAndSize = IndexTypeAndSize, + }; + _contextShared = _vertexShared = _pixelShared = _computeShared = _userConfigShared = true; + _pixelScalarsShared = _hullScalarsShared = _geometryScalarsShared = _legacyVertexScalarsShared = _exportScalarsShared = _computeScalarsShared = true; + return snapshot; + } + + // Called by the register writers before they store into a part. + public void PrepareContextWrite() + { + if (_contextShared) + { + Context = Context.ShallowCopy(); + _contextShared = false; + SetContextPartsShared(true); + } + } + + // The context is copied without its nested parts, and a part only when a writer asks for it + // through these accessors: a draw usually rewrites a few plain fields. + private bool _screenViewportShared; + private bool _viewportsShared; + private bool _sampleLocationsShared; + private bool _shaderInterfaceShared; + private bool _blendControlsShared; + private bool _colorTargetsShared; + private bool _colorClearWord1Shared; + private bool _unmodeledShared; + + private void SetContextPartsShared(bool shared) => + _screenViewportShared = _viewportsShared = _sampleLocationsShared = _shaderInterfaceShared = + _blendControlsShared = _colorTargetsShared = _colorClearWord1Shared = _unmodeledShared = shared; + + private void OwnContext(ContextRegisters context) + { + Context = context; + _contextShared = false; + SetContextPartsShared(false); + } + + public ScreenViewportRegisters ScreenViewportForWrite + { + get + { + PrepareContextWrite(); + if (_screenViewportShared) + { + Context.ScreenViewport = Context.ScreenViewport.CopySharingViewports(); + _screenViewportShared = false; + } + + return Context.ScreenViewport; + } + } + + public ViewportRegisters[] ViewportsForWrite + { + get + { + var screenViewport = ScreenViewportForWrite; + if (_viewportsShared) + { + screenViewport.Viewports = (ViewportRegisters[])screenViewport.Viewports.Clone(); + _viewportsShared = false; + } + + return screenViewport.Viewports; + } + } + + public SampleLocationRegisters SampleLocationsForWrite + { + get + { + PrepareContextWrite(); + if (_sampleLocationsShared) + { + Context.SampleLocations = Context.SampleLocations.Copy(); + _sampleLocationsShared = false; + } + + return Context.SampleLocations; + } + } + + public ShaderInterfaceRegisters ShaderInterfaceForWrite + { + get + { + PrepareContextWrite(); + if (_shaderInterfaceShared) + { + Context.ShaderInterface = Context.ShaderInterface.Copy(); + _shaderInterfaceShared = false; + } + + return Context.ShaderInterface; + } + } + + public BlendRegisters[] BlendControlsForWrite + { + get + { + PrepareContextWrite(); + if (_blendControlsShared) + { + Context.BlendControls = (BlendRegisters[])Context.BlendControls.Clone(); + _blendControlsShared = false; + } + + return Context.BlendControls; + } + } + + public ColorTargetWords[] ColorTargetsForWrite + { + get + { + PrepareContextWrite(); + if (_colorTargetsShared) + { + Context.ColorTargets = (ColorTargetWords[])Context.ColorTargets.Clone(); + _colorTargetsShared = false; + } + + return Context.ColorTargets; + } + } + + public uint[] ColorClearWord1ForWrite + { + get + { + PrepareContextWrite(); + if (_colorClearWord1Shared) + { + Context.ColorClearWord1 = (uint[])Context.ColorClearWord1.Clone(); + _colorClearWord1Shared = false; + } + + return Context.ColorClearWord1; + } + } + + public Dictionary UnmodeledTableRegistersForWrite + { + get + { + PrepareContextWrite(); + if (_unmodeledShared) + { + Context.UnmodeledTableRegisters = new Dictionary(Context.UnmodeledTableRegisters); + _unmodeledShared = false; + } + + return Context.UnmodeledTableRegisters; + } + } + + public void PrepareUserConfigWrite() + { + if (_userConfigShared) + { + UserConfig = UserConfig.Copy(); + _userConfigShared = false; + } + } + + // COMPUTE_DISPATCH_INITIATOR, the first register of the compute range. + private const uint ComputeRangeStart = 0x200; + + // A stage is copied without its user scalar sets, and a set only when its window is written: + // a draw usually rewrites one set of one or two stages. + private bool _pixelScalarsShared; + private bool _hullScalarsShared; + private bool _geometryScalarsShared; + private bool _legacyVertexScalarsShared; + private bool _exportScalarsShared; + private bool _computeScalarsShared; + private const uint UserScalarWindow = UserScalarRegisters.Capacity; + + private static bool Overlaps(uint first, uint last, uint windowStart) => + first < windowStart + UserScalarWindow && last >= windowStart; + + // The shader bank's offsets: pixel below SPI_SHADER_PGM_LO_VS, the geometry stages up to the + // compute range, compute from there. + public void PrepareShaderWrite(uint offset, uint count) + { + var last = offset + Math.Max(count, 1) - 1; + if (offset < ShaderRegisterOffset.SpiShaderPgmLoVs) + { + if (_pixelShared) + { + Shader.Pixel = Shader.Pixel.ShallowCopy(); + _pixelShared = false; + } + + if (_pixelScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataPs0)) + { + Shader.Pixel.UserScalars = Shader.Pixel.UserScalars.Copy(); + _pixelScalarsShared = false; + } + } + + if (offset < ComputeRangeStart && last >= ShaderRegisterOffset.SpiShaderPgmLoVs) + { + if (_vertexShared) + { + Shader.Vertex = Shader.Vertex.ShallowCopy(); + _vertexShared = false; + } + + var vertex = Shader.Vertex; + if (_hullScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataHs0)) + { + vertex.HullUserScalars = vertex.HullUserScalars.Copy(); + _hullScalarsShared = false; + } + + if (_geometryScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataGs0)) + { + vertex.GeometryUserScalars = vertex.GeometryUserScalars.Copy(); + _geometryScalarsShared = false; + } + + if (_legacyVertexScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataVs0)) + { + vertex.LegacyVertexUserScalars = vertex.LegacyVertexUserScalars.Copy(); + _legacyVertexScalarsShared = false; + } + + if (_exportScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataEs0)) + { + vertex.ExportUserScalars = vertex.ExportUserScalars.Copy(); + _exportScalarsShared = false; + } + } + + if (last >= ComputeRangeStart) + { + if (_computeShared) + { + Shader.Compute = Shader.Compute.ShallowCopy(); + _computeShared = false; + } + + if (_computeScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.ComputeUserData0)) + { + Shader.Compute.UserScalars = Shader.Compute.UserScalars.Copy(); + _computeScalarsShared = false; + } + } + } + public void Reset() { - Context = new ContextRegisters(); + _contextShared = _vertexShared = _pixelShared = _computeShared = _userConfigShared = false; + _pixelScalarsShared = _hullScalarsShared = _geometryScalarsShared = _legacyVertexScalarsShared = _exportScalarsShared = _computeScalarsShared = false; + OwnContext(new ContextRegisters()); Shader = new ShaderProgramRegisters(); UserConfig = new UserConfigRegisters(); _savedContext = null; @@ -61,7 +348,7 @@ public sealed class RegisterBanks switch (operation) { case ContextStateOperation.Clear: - Context = new ContextRegisters(); + OwnContext(new ContextRegisters()); CompositeDepthSizeXy = null; break; case ContextStateOperation.Push: @@ -73,14 +360,14 @@ public sealed class RegisterBanks throw Fatal("The context state is not pushed."); } - Context = _savedContext; + OwnContext(_savedContext); CompositeDepthSizeXy = _savedCompositeDepthSizeXy; _savedContext = null; _savedCompositeDepthSizeXy = null; break; case ContextStateOperation.PushClear: Push(); - Context = new ContextRegisters(); + OwnContext(new ContextRegisters()); CompositeDepthSizeXy = null; break; default: diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs index 3592016..a7258f8 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs @@ -28,6 +28,7 @@ public static class RegisterWriteTable public static uint WriteContextPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) { + banks.PrepareContextWrite(); var consumed = WritePacket(banks, in packet, offset, values, Context, ContextIndirect, "context", emptyIsHandled: true); if (Rendering.RenderTrace.Enabled) { @@ -43,6 +44,7 @@ public static class RegisterWriteTable // A shader packet without values has no writer; the other banks accept it. public static uint WriteShaderPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) { + banks.PrepareShaderWrite(offset, (uint)values.Length); var consumed = WritePacket(banks, in packet, offset, values, Shader, ShaderIndirect, "shader", emptyIsHandled: false); if (Rendering.RenderTrace.Enabled) { @@ -52,15 +54,19 @@ public static class RegisterWriteTable return consumed; } - public static uint WriteUserConfigPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) => - WritePacket(banks, in packet, offset, values, UserConfig, UserConfigIndirect, "user-config", emptyIsHandled: true); + public static uint WriteUserConfigPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) + { + banks.PrepareUserConfigWrite(); + return WritePacket(banks, in packet, offset, values, UserConfig, UserConfigIndirect, "user-config", emptyIsHandled: true); + } public static void WriteContextEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) { + banks.PrepareContextWrite(); if (offset < ContextIndirect.Length && ContextIndirect[offset] is null) { // Keep unknown in-bank table values so a later renderer implementation can decode them. - banks.Context.UnmodeledTableRegisters[offset] = value; + banks.UnmodeledTableRegistersForWrite[offset] = value; } else { @@ -88,6 +94,7 @@ public static class RegisterWriteTable public static void WriteShaderEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) { + banks.PrepareShaderWrite(offset, 1); WriteEntry(banks, offset, value, ShaderIndirect, "shader", tableAddress); if (Rendering.RenderTrace.Enabled) TracePixelRegisterWrite(banks, offset, value, tableAddress, "table"); @@ -104,8 +111,11 @@ public static class RegisterWriteTable $"value=0x{value:X8} shader=0x{banks.Shader.Pixel.Address:X16}"); } - public static void WriteUserConfigEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) => + public static void WriteUserConfigEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) + { + banks.PrepareUserConfigWrite(); WriteEntry(banks, offset, value, UserConfigIndirect, "user-config", tableAddress); + } private static uint WritePacket( RegisterBanks banks, diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs index ea46226..706e224 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs @@ -98,8 +98,8 @@ internal static partial class RegisterWriters { indirect[offset] = static (banks, register, value) => { - banks.Context.ShaderInterface.PixelInterpolatorSettings[register - SpiPsInputCntl0] = value; - banks.Context.ShaderInterface.PixelInterpolatorWritten |= 1u << (int)(register - SpiPsInputCntl0); + banks.ShaderInterfaceForWrite.PixelInterpolatorSettings[register - SpiPsInputCntl0] = value; + banks.ShaderInterfaceForWrite.PixelInterpolatorWritten |= 1u << (int)(register - SpiPsInputCntl0); }; } @@ -160,7 +160,7 @@ internal static partial class RegisterWriters for (var offset = PaScAaSampleLocations0; offset < PaScAaSampleLocations0 + SampleLocationRegisters.LocationCount; offset++) { - indirect[offset] = static (banks, register, value) => banks.Context.SampleLocations.Locations[register - PaScAaSampleLocations0] = value; + indirect[offset] = static (banks, register, value) => banks.SampleLocationsForWrite.Locations[register - PaScAaSampleLocations0] = value; } foreach (var offset in new[] @@ -175,36 +175,36 @@ internal static partial class RegisterWriters indirect[offset] = IgnoreEntry; } - indirect[SpiVsOutConfig] = static (banks, _, value) => banks.Context.ShaderInterface.VertexOutputConfiguration = value; + indirect[SpiVsOutConfig] = static (banks, _, value) => banks.ShaderInterfaceForWrite.VertexOutputConfiguration = value; indirect[DbRenderOverride] = static (banks, _, value) => banks.Context.DepthRenderOverride = DepthRenderOverrideRegisters.Decode(value); - indirect[SpiShaderPosFormat] = static (banks, _, value) => banks.Context.ShaderInterface.PositionExportFormat = value; - indirect[SpiShaderIdxFormat] = static (banks, _, value) => banks.Context.ShaderInterface.IndexExportFormat = value; - indirect[PaClVsOutCntl] = static (banks, _, value) => banks.Context.ShaderInterface.VertexOutputControl = value; - indirect[GeNggSubgroupCntl] = static (banks, _, value) => banks.Context.ShaderInterface.PrimitiveShaderSubgroupControl = value; - indirect[VgtGsInstanceCnt] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryInstanceCount = value; - indirect[VgtGsOnchipCntl] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryOnChipControl = value; - indirect[VgtHosMaxTessLevel] = static (banks, _, value) => banks.Context.ShaderInterface.MaxTessellationLevel = value; - indirect[VgtHosMinTessLevel] = static (banks, _, value) => banks.Context.ShaderInterface.MinTessellationLevel = value; - indirect[GeMaxOutputPerSubgroup] = static (banks, _, value) => banks.Context.ShaderInterface.MaxOutputPerSubgroup = value; - indirect[VgtEsgsRingItemSize] = static (banks, _, value) => banks.Context.ShaderInterface.ExportRingItemSize = value; - indirect[VgtGsMaxVertOut] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryMaxVerticesOut = value; - indirect[VgtPrimitiveIdEn] = static (banks, _, value) => banks.Context.ShaderInterface.PrimitiveIdEnable = value; - indirect[VgtReuseOff] = static (banks, _, value) => banks.Context.ShaderInterface.VertexReuseOff = value; + indirect[SpiShaderPosFormat] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PositionExportFormat = value; + indirect[SpiShaderIdxFormat] = static (banks, _, value) => banks.ShaderInterfaceForWrite.IndexExportFormat = value; + indirect[PaClVsOutCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.VertexOutputControl = value; + indirect[GeNggSubgroupCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PrimitiveShaderSubgroupControl = value; + indirect[VgtGsInstanceCnt] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryInstanceCount = value; + indirect[VgtGsOnchipCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryOnChipControl = value; + indirect[VgtHosMaxTessLevel] = static (banks, _, value) => banks.ShaderInterfaceForWrite.MaxTessellationLevel = value; + indirect[VgtHosMinTessLevel] = static (banks, _, value) => banks.ShaderInterfaceForWrite.MinTessellationLevel = value; + indirect[GeMaxOutputPerSubgroup] = static (banks, _, value) => banks.ShaderInterfaceForWrite.MaxOutputPerSubgroup = value; + indirect[VgtEsgsRingItemSize] = static (banks, _, value) => banks.ShaderInterfaceForWrite.ExportRingItemSize = value; + indirect[VgtGsMaxVertOut] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryMaxVerticesOut = value; + indirect[VgtPrimitiveIdEn] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PrimitiveIdEnable = value; + indirect[VgtReuseOff] = static (banks, _, value) => banks.ShaderInterfaceForWrite.VertexReuseOff = value; indirect[VgtMultiPrimIbResetIndex] = static (banks, _, value) => banks.Context.PrimitiveResetIndex = value; - indirect[VgtTessDistribution] = static (banks, _, value) => banks.Context.ShaderInterface.TessellationDistribution = value; + indirect[VgtTessDistribution] = static (banks, _, value) => banks.ShaderInterfaceForWrite.TessellationDistribution = value; indirect[VgtShaderStagesEn] = static (banks, _, value) => banks.Context.ShaderStages = value; - indirect[VgtLsHsConfig] = static (banks, _, value) => banks.Context.ShaderInterface.LocalHullConfiguration = value; - indirect[VgtGsOutPrimType] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryOutputPrimitiveType = value; - indirect[VgtTfParam] = static (banks, _, value) => banks.Context.ShaderInterface.TessellationFactorParameter = value; - indirect[SpiShaderZFormat] = static (banks, _, value) => banks.Context.ShaderInterface.DepthExportFormat = value; + indirect[VgtLsHsConfig] = static (banks, _, value) => banks.ShaderInterfaceForWrite.LocalHullConfiguration = value; + indirect[VgtGsOutPrimType] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryOutputPrimitiveType = value; + indirect[VgtTfParam] = static (banks, _, value) => banks.ShaderInterfaceForWrite.TessellationFactorParameter = value; + indirect[SpiShaderZFormat] = static (banks, _, value) => banks.ShaderInterfaceForWrite.DepthExportFormat = value; indirect[SpiShaderColFormat] = ShaderColorFormatEntry; - indirect[SpiPsInputEna] = static (banks, _, value) => banks.Context.ShaderInterface.PixelInputEnable = value; - indirect[SpiPsInputAddr] = static (banks, _, value) => banks.Context.ShaderInterface.PixelInputAddress = value; - indirect[SpiPsInControl] = static (banks, _, value) => banks.Context.ShaderInterface.PixelInputControl = value; - indirect[SpiBarycCntl] = static (banks, _, value) => banks.Context.ShaderInterface.BarycentricControl = value; - indirect[DbShaderControl] = static (banks, _, value) => banks.Context.ShaderInterface.DepthShaderControl = DepthShaderControlRegisters.Decode(value); - indirect[CbShaderMask] = static (banks, _, value) => banks.Context.ShaderInterface.ColorShaderMask = value; - indirect[PaScShaderControl] = static (banks, _, value) => banks.Context.ShaderInterface.ScanShaderControl = value; + indirect[SpiPsInputEna] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PixelInputEnable = value; + indirect[SpiPsInputAddr] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PixelInputAddress = value; + indirect[SpiPsInControl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PixelInputControl = value; + indirect[SpiBarycCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.BarycentricControl = value; + indirect[DbShaderControl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.DepthShaderControl = DepthShaderControlRegisters.Decode(value); + indirect[CbShaderMask] = static (banks, _, value) => banks.ShaderInterfaceForWrite.ColorShaderMask = value; + indirect[PaScShaderControl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.ScanShaderControl = value; indirect[CbTargetMask] = static (banks, _, value) => banks.Context.RenderTargetMask = value; indirect[PaScScreenScissorTl] = ScreenScissorTopLeftEntry; indirect[PaScScreenScissorBr] = ScreenScissorBottomRightEntry; @@ -248,7 +248,7 @@ internal static partial class RegisterWriters indirect[DbStencilRefMaskBack] = StencilMaskBackEntry; indirect[PaClClipCntl] = static (banks, _, value) => banks.Context.Clip = ClipControlRegisters.Decode(value); indirect[PaSuLineCntl] = LineControlEntry; - indirect[PaClVteCntl] = static (banks, _, value) => banks.Context.ScreenViewport.TransformControl = value; + indirect[PaClVteCntl] = static (banks, _, value) => banks.ScreenViewportForWrite.TransformControl = value; indirect[PaScModeCntl0] = static (banks, _, value) => banks.Context.ScanMode = ScanModeRegisters.Decode(value); indirect[PaScAaConfig] = static (banks, _, value) => banks.Context.AntialiasingConfig = AntialiasingConfigRegisters.Decode(value); indirect[PaScCentroidPriority0] = CentroidPriorityEntry; @@ -369,8 +369,8 @@ internal static partial class RegisterWriters throw PacketError(banks, "The pixel interpolator packet has an invalid count", in packet, offset, values.Length); } - values.CopyTo(banks.Context.ShaderInterface.PixelInterpolatorSettings); - banks.Context.ShaderInterface.PixelInterpolatorWritten |= (uint)((1ul << values.Length) - 1); + values.CopyTo(banks.ShaderInterfaceForWrite.PixelInterpolatorSettings); + banks.ShaderInterfaceForWrite.PixelInterpolatorWritten |= (uint)((1ul << values.Length) - 1); return (uint)values.Length; } @@ -410,7 +410,7 @@ internal static partial class RegisterWriters // One location at any slot, or all sixteen from the first slot. private static uint SampleLocationsPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) { - var locations = banks.Context.SampleLocations.Locations; + var locations = banks.SampleLocationsForWrite.Locations; if (values.Length == 1) { locations[offset - PaScAaSampleLocations0] = values[0]; @@ -506,19 +506,19 @@ internal static partial class RegisterWriters private static void HardwareScreenOffsetEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.HardwareOffsetX = RegisterField.Get(value, 0, 0x1FF); - banks.Context.ScreenViewport.HardwareOffsetY = RegisterField.Get(value, 16, 0x1FF); + banks.ScreenViewportForWrite.HardwareOffsetX = RegisterField.Get(value, 0, 0x1FF); + banks.ScreenViewportForWrite.HardwareOffsetY = RegisterField.Get(value, 16, 0x1FF); } private static void WindowOffsetEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.WindowOffsetX = RegisterField.Coordinate16(value); - banks.Context.ScreenViewport.WindowOffsetY = RegisterField.Coordinate16(value >> 16); + banks.ScreenViewportForWrite.WindowOffsetX = RegisterField.Coordinate16(value); + banks.ScreenViewportForWrite.WindowOffsetY = RegisterField.Coordinate16(value >> 16); } private static void ClipRectangleEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; if (offset == PaScClipRectRule) { viewport.ClipRectangleRule = (ushort)(value & 0xFFFFu); @@ -598,7 +598,7 @@ internal static partial class RegisterWriters private static void CentroidPriorityEntry(RegisterBanks banks, uint offset, uint value) { - var locations = banks.Context.SampleLocations; + var locations = banks.SampleLocationsForWrite; switch (offset) { case PaScCentroidPriority0: @@ -612,7 +612,7 @@ internal static partial class RegisterWriters private static void ShaderColorFormatEntry(RegisterBanks banks, uint offset, uint value) { - var modes = banks.Context.ShaderInterface.TargetOutputModes; + var modes = banks.ShaderInterfaceForWrite.TargetOutputModes; for (var index = 0; index < modes.Length; index++) { modes[index] = (byte)((value >> (index * 4)) & 0xFu); @@ -621,19 +621,19 @@ internal static partial class RegisterWriters private static void ScreenScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.ScreenScissorLeft = RegisterField.Coordinate16(value); - banks.Context.ScreenViewport.ScreenScissorTop = RegisterField.Coordinate16(value >> 16); + banks.ScreenViewportForWrite.ScreenScissorLeft = RegisterField.Coordinate16(value); + banks.ScreenViewportForWrite.ScreenScissorTop = RegisterField.Coordinate16(value >> 16); } private static void ScreenScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.ScreenScissorRight = RegisterField.Coordinate16(value); - banks.Context.ScreenViewport.ScreenScissorBottom = RegisterField.Coordinate16(value >> 16); + banks.ScreenViewportForWrite.ScreenScissorRight = RegisterField.Coordinate16(value); + banks.ScreenViewportForWrite.ScreenScissorBottom = RegisterField.Coordinate16(value >> 16); } private static void GenericScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; viewport.GenericScissorLeft = RegisterField.Coordinate15(value); viewport.GenericScissorTop = RegisterField.Coordinate15(value >> 16); viewport.GenericScissorWindowOffsetEnable = !RegisterField.Bit(value, 31); @@ -641,13 +641,13 @@ internal static partial class RegisterWriters private static void GenericScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.GenericScissorRight = RegisterField.Coordinate15(value); - banks.Context.ScreenViewport.GenericScissorBottom = RegisterField.Coordinate15(value >> 16); + banks.ScreenViewportForWrite.GenericScissorRight = RegisterField.Coordinate15(value); + banks.ScreenViewportForWrite.GenericScissorBottom = RegisterField.Coordinate15(value >> 16); } private static void WindowScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; viewport.WindowScissorLeft = RegisterField.Coordinate15(value); viewport.WindowScissorTop = RegisterField.Coordinate15(value >> 16); viewport.WindowScissorWindowOffsetEnable = !RegisterField.Bit(value, 31); @@ -655,13 +655,13 @@ internal static partial class RegisterWriters private static void WindowScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.WindowScissorRight = RegisterField.Coordinate15(value); - banks.Context.ScreenViewport.WindowScissorBottom = RegisterField.Coordinate15(value >> 16); + banks.ScreenViewportForWrite.WindowScissorRight = RegisterField.Coordinate15(value); + banks.ScreenViewportForWrite.WindowScissorBottom = RegisterField.Coordinate15(value >> 16); } private static void ViewportScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - ref var viewport = ref banks.Context.ScreenViewport.Viewports[(offset - PaScViewportScissor0Tl) / 2]; + ref var viewport = ref banks.ViewportsForWrite[(offset - PaScViewportScissor0Tl) / 2]; viewport.ScissorLeft = RegisterField.Coordinate15(value); viewport.ScissorTop = RegisterField.Coordinate15(value >> 16); viewport.ScissorWindowOffsetEnable = !RegisterField.Bit(value, 31); @@ -669,21 +669,21 @@ internal static partial class RegisterWriters private static void ViewportScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - ref var viewport = ref banks.Context.ScreenViewport.Viewports[(offset - PaScViewportScissor0Br) / 2]; + ref var viewport = ref banks.ViewportsForWrite[(offset - PaScViewportScissor0Br) / 2]; viewport.ScissorRight = RegisterField.Coordinate15(value); viewport.ScissorBottom = RegisterField.Coordinate15(value >> 16); } private static void ViewportZMinEntry(RegisterBanks banks, uint offset, uint value) => - banks.Context.ScreenViewport.Viewports[(offset - PaScViewportZMin0) / 2].MinDepth = RegisterField.AsFloat(value); + banks.ViewportsForWrite[(offset - PaScViewportZMin0) / 2].MinDepth = RegisterField.AsFloat(value); private static void ViewportZMaxEntry(RegisterBanks banks, uint offset, uint value) => - banks.Context.ScreenViewport.Viewports[(offset - PaScViewportZMax0) / 2].MaxDepth = RegisterField.AsFloat(value); + banks.ViewportsForWrite[(offset - PaScViewportZMax0) / 2].MaxDepth = RegisterField.AsFloat(value); private static void ViewportScaleOffsetEntry(RegisterBanks banks, uint offset, uint value) { var relative = offset - PaClViewportXScale; - ref var viewport = ref banks.Context.ScreenViewport.Viewports[relative / ViewportFieldCount]; + ref var viewport = ref banks.ViewportsForWrite[relative / ViewportFieldCount]; var number = RegisterField.AsFloat(value); switch (relative % ViewportFieldCount) { @@ -718,7 +718,7 @@ internal static partial class RegisterWriters private static void GuardBandEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; var number = RegisterField.AsFloat(value); switch (offset) { @@ -760,7 +760,7 @@ internal static partial class RegisterWriters } private static void BlendControlEntry(RegisterBanks banks, uint offset, uint value) => - banks.Context.BlendControls[offset - CbBlend0Control] = BlendRegisters.Decode(value); + banks.BlendControlsForWrite[offset - CbBlend0Control] = BlendRegisters.Decode(value); private static void DepthZInfoEntry(RegisterBanks banks, uint offset, uint value) { @@ -804,7 +804,7 @@ internal static partial class RegisterWriters } private static ref ColorTargetWords ColorTarget(RegisterBanks banks, uint offset, uint first, uint stride) => - ref banks.Context.ColorTargets[(offset - first) / stride]; + ref banks.ColorTargetsForWrite[(offset - first) / stride]; private static void ColorBaseLowEntry(RegisterBanks banks, uint offset, uint value) { @@ -878,7 +878,7 @@ internal static partial class RegisterWriters } private static void ColorClearWord1Entry(RegisterBanks banks, uint offset, uint value) => - banks.Context.ColorClearWord1[(offset - CbColor0ClearWord1) / ColorSlotStride] = value; + banks.ColorClearWord1ForWrite[(offset - CbColor0ClearWord1) / ColorSlotStride] = value; private static void ColorDccBaseLowEntry(RegisterBanks banks, uint offset, uint value) { diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs index e2d3e45..b29fe93 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs @@ -199,6 +199,9 @@ public sealed class VertexStageRegisters public UserScalarRegisters LegacyVertexUserScalars = new(); public UserScalarRegisters ExportUserScalars = new(); + // The same registers sharing the user scalar sets, which a writer then copies one by one. + public VertexStageRegisters ShallowCopy() => (VertexStageRegisters)MemberwiseClone(); + public VertexStageRegisters Copy() { var copy = (VertexStageRegisters)MemberwiseClone(); @@ -217,6 +220,8 @@ public sealed class PixelStageRegisters public PixelResource2 Resource2; public UserScalarRegisters UserScalars = new(); + public PixelStageRegisters ShallowCopy() => (PixelStageRegisters)MemberwiseClone(); + public PixelStageRegisters Copy() { var copy = (PixelStageRegisters)MemberwiseClone(); @@ -253,6 +258,8 @@ public sealed class ComputeStageRegisters public byte SharedVectorRegisters; public UserScalarRegisters UserScalars = new(); + public ComputeStageRegisters ShallowCopy() => (ComputeStageRegisters)MemberwiseClone(); + public ComputeStageRegisters Copy() { var copy = (ComputeStageRegisters)MemberwiseClone(); @@ -264,6 +271,9 @@ public sealed class ComputeStageRegisters // The shader bank: the program registers of every stage and their user scalars. public sealed class ShaderProgramRegisters { + // A bank holding the same stage objects, for a snapshot that shares them. + public ShaderProgramRegisters ShareStages() => (ShaderProgramRegisters)MemberwiseClone(); + public VertexStageRegisters Vertex = new(); public PixelStageRegisters Pixel = new(); public ComputeStageRegisters Compute = new(); diff --git a/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs b/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs index f244c77..ae91f2d 100644 --- a/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs +++ b/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs @@ -31,10 +31,13 @@ public sealed unsafe partial class CachedImage SubresourceRange = new ImageSubresourceRange(ViewFormatRules.FullAspects(Backing.Format), baseLevel, levelCount, baseLayer, layerCount), }; + // What a state that needs no barrier returns, so the common case allocates nothing. Callers only read it. + private static readonly List NoBarriers = []; + // Barriers for the requested state; a repeated write always gets a barrier. public (List Barriers, PipelineStageFlags SourceStages) GetBarriers(ImageLayout layout, AccessFlags access, PipelineStageFlags stage, SubresourceRange? range) { - var barriers = new List(); + List? barriers = null; PipelineStageFlags sourceStages = 0; if ((access & (WriteAccess | AccessFlags.ColorAttachmentWriteBit | AccessFlags.DepthStencilAttachmentWriteBit)) != 0) { @@ -83,7 +86,7 @@ public sealed unsafe partial class CachedImage var repeatedWrite = (state.Access & WriteAccess) != 0; if (state.Layout != layout || state.Access != access || repeatedWrite) { - barriers.Add(MakeBarrier(state, layout, access, level, 1, layer, 1)); + (barriers ??= []).Add(MakeBarrier(state, layout, access, level, 1, layer, 1)); sourceStages |= state.Stage; states[index] = new ImageAccessState(stage, access, layout); } @@ -101,15 +104,15 @@ public sealed unsafe partial class CachedImage var repeatedWrite = (state.Access & WriteAccess) != 0; if (state.Layout == layout && state.Access == access && !repeatedWrite) { - return (barriers, sourceStages); + return (NoBarriers, sourceStages); } - barriers.Add(MakeBarrier(state, layout, access, 0, Vk.RemainingMipLevels, 0, Vk.RemainingArrayLayers)); + (barriers ??= []).Add(MakeBarrier(state, layout, access, 0, Vk.RemainingMipLevels, 0, Vk.RemainingArrayLayers)); sourceStages |= state.Stage; } Backing.State = new ImageAccessState(stage, access, layout); - return (barriers, sourceStages); + return (barriers ?? NoBarriers, sourceStages); } public void Transition(ImageLayout layout, AccessFlags access, SubresourceRange? range, CommandBuffer command) diff --git a/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs b/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs index 9648265..64f8fc2 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs @@ -303,6 +303,8 @@ public sealed unsafe class ColorToMultisampleDepthBlit : IDisposable vk.CmdBeginRenderPass(command, &begin, SubpassContents.Inline); vk.CmdBindDescriptorSets(command, PipelineBindPoint.Graphics, _pipelineLayout, 0, 1, &set, 0, null); vk.CmdBindPipeline(command, PipelineBindPoint.Graphics, cached.Pipeline); + Vulkan.GraphicsDynamicStateEpoch.Advance(); + Vulkan.NativeBindingEpoch.Advance(); var viewport = new Viewport(0.0f, 0.0f, extent.Width, extent.Height, 0.0f, 1.0f); var scissor = new Rect2D(new Offset2D(0, 0), extent); vk.CmdSetViewport(command, 0, 1, &viewport); diff --git a/src/SharpEmu.Libs/Gpu/Images/GpuTiler.cs b/src/SharpEmu.Libs/Gpu/Images/GpuTiler.cs index 72a42cd..eca0c58 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GpuTiler.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GpuTiler.cs @@ -540,8 +540,11 @@ public sealed unsafe class GpuTiler : IDisposable return set; } - private void BindDescriptorSet(CommandBuffer command, DescriptorSet set) => + private void BindDescriptorSet(CommandBuffer command, DescriptorSet set) + { _device.Vk.CmdBindDescriptorSets(command, PipelineBindPoint.Compute, _pipelineLayout, 0, 1, &set, 0, null); + Vulkan.NativeBindingEpoch.Advance(); + } private void Record(bool tile, VkBuffer source, ulong sourceOffset, ulong sourceCapacity, VkBuffer target, ulong targetOffset, ulong targetCapacity, List dispatches, bool clearTarget) { @@ -587,6 +590,7 @@ public sealed unsafe class GpuTiler : IDisposable infos[2] = new DescriptorBufferInfo(dispatch.ArgumentsBuffer, dispatch.ArgumentsOffset, TileTransferArguments.Size); BindDescriptorSet(command, WriteDescriptorSet(infos, 2)); vk.CmdBindPipeline(command, PipelineBindPoint.Compute, GetPipeline(dispatch.PipelineSlot)); + Vulkan.NativeBindingEpoch.Advance(); vk.CmdDispatch(command, (dispatch.Arguments.Width + 7) / 8, (dispatch.Arguments.Height + 7) / 8, dispatch.Arguments.Depth); } @@ -697,6 +701,7 @@ public sealed unsafe class GpuTiler : IDisposable fixed (TileTransferArguments* pointer = &arguments) { _device.Vk.CmdPushConstants(command, _pipelineLayout, ShaderStageFlags.ComputeBit, 0, TileTransferArguments.Size, pointer); + Vulkan.NativeBindingEpoch.Advance(); } } @@ -743,6 +748,7 @@ public sealed unsafe class GpuTiler : IDisposable AccessFlags.ShaderReadBit | AccessFlags.ShaderWriteBit); VulkanSynchronization.PipelineBarrier(vk,command, PipelineStageFlags.AllCommandsBit | PipelineStageFlags.HostBit, PipelineStageFlags.ComputeShaderBit, 0, 0, null, 2, barriers, 0, null); vk.CmdBindPipeline(command, PipelineBindPoint.Compute, pipeline); + Vulkan.NativeBindingEpoch.Advance(); var descriptorAlignment = StorageAlignment; var groupLimit = _device.MaxComputeWorkGroupCount; @@ -830,6 +836,7 @@ public sealed unsafe class GpuTiler : IDisposable var command = new CommandBuffer(_scheduler.Current.Handle); VulkanSynchronization.PipelineBarrier(vk,command, PipelineStageFlags.AllCommandsBit | PipelineStageFlags.HostBit, PipelineStageFlags.ComputeShaderBit, 0, 0, null, 2, barriers, 0, null); vk.CmdBindPipeline(command, PipelineBindPoint.Compute, _swapBgra16); + Vulkan.NativeBindingEpoch.Advance(); BindDescriptorSet(command, WriteDescriptorSet(infos, -1)); PushArguments(command, new TileTransferArguments { SourceBase = inputBinding.Base, DestinationBase = outputBinding.Base, Width = pixels }); vk.CmdDispatch(command, (pixels + 63) / 64, 1, 1); diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs index 807c88c..3707edb 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs @@ -35,6 +35,25 @@ public sealed partial class GuestImageCache private void InvalidateLookups() => _lookupGeneration++; + // Changes whenever an image a lookup could find is created, removed or replaced, so a caller + // that remembers a lookup's result can tell when it may have changed. + public ulong LookupGeneration => _lookupGeneration; + + // What a remembered lookup still does on reuse: the image counts as used now. False when the + // image is gone. + public bool TryTouchRemembered(ResourceSlotIdentifier imageIdentifier) + { + using var held = _lock.Hold(); + if (_slots.TryGet(imageIdentifier) is not { Registered: true } image) + { + return false; + } + + image.LastAccessTick = _scheduler.CurrentTick; + TouchImage(image); + return true; + } + private static int LookupSlot(in ImageRequest request, bool exactFormat) { var hash = HashCode.Combine(request.Description.Data.Address, request.Description.Data.Size, request.Description.PixelFormat, diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs index f7aa599..c05998f 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs @@ -11,6 +11,14 @@ namespace SharpEmu.Libs.Gpu.Images; // Surface metadata (HTile, DCC, CMask, FMask) keyed by its guest address. public sealed partial class GuestImageCache { + private static long _dccSyncs; + private static long _dccSlicesUndecodable; + private static long _dccSlicesMixed; + private static long _dccSlicesCleared; + + public static string TakeDccReport() => FormattableString.Invariant( + $"[PERF][DCC_SYNC] gpu_syncs={Interlocked.Exchange(ref _dccSyncs, 0)} cleared={Interlocked.Exchange(ref _dccSlicesCleared, 0)} mixed={Interlocked.Exchange(ref _dccSlicesMixed, 0)} undecodable={Interlocked.Exchange(ref _dccSlicesUndecodable, 0)}"); + private unsafe void MaterializeDccClear( ResourceSlotIdentifier imageIdentifier, in ImageRequest request, @@ -96,6 +104,7 @@ public sealed partial class GuestImageCache allowClearToRegister: request.Role == ImageRole.ColorTarget, out var clear)) { + Interlocked.Increment(ref _dccSlicesUndecodable); continue; } @@ -126,9 +135,12 @@ public sealed partial class GuestImageCache if (!uniform) { + Interlocked.Increment(ref _dccSlicesMixed); continue; } + Interlocked.Increment(ref _dccSlicesCleared); + using (var held = _lock.Hold()) { if (_slots.TryGet(imageIdentifier) == null) @@ -287,13 +299,20 @@ public sealed partial class GuestImageCache public bool IsMetadataCleared(ulong address, uint slice) => IsMetadataCleared(address, slice, out _); + // DCC metadata of a bound target, or the range of a DCC fill seen before its target: both are + // filled on the host, which keeps them CPU-readable instead of GPU-written. public bool OverlapsDccMetadata(ulong address, ulong size) { using var held = _lock.Hold(); foreach (var (start, metadata) in _surfaceMetadata) { - if (metadata.Kind == SurfaceMetadataKind.Dcc && metadata.Size != 0 && - address < start + metadata.Size && start < address + size) + var extent = metadata.Kind switch + { + SurfaceMetadataKind.Dcc => metadata.Size, + SurfaceMetadataKind.PendingDcc => metadata.FillSize, + _ => 0UL, + }; + if (extent != 0 && address < start + extent && start < address + size) { return true; } @@ -416,6 +435,12 @@ public sealed partial class GuestImageCache var size = (ulong)layerCount * sliceSize; if (IsValidRange(address, size) && _bufferCache.HasGpuDirtyBytes(address, size)) { + Interlocked.Increment(ref _dccSyncs); + if (Diagnostics.DccWriterTrace.Enabled) + { + Diagnostics.DccWriterTrace.ReportSync(address, size); + } + _ = _bufferCache.TrySynchronizeCpuRead(address, size); } } diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs index 000513d..1d938f7 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs @@ -761,6 +761,7 @@ public sealed unsafe partial class GuestImageCache _scheduler.QueuePriorityCompletionAction(() => { download.Invalidate(offset, range.Size); + GpuWritebackEpoch.Advance(); if (!backing.TryWriteBacking(range.Address, download.Mapped.Slice((int)offset, (int)range.Size))) { throw SubmissionScheduler.Fatal($"The image readback could not be written to guest memory: address=0x{range.Address:X16} size=0x{range.Size:X}."); @@ -833,7 +834,7 @@ public sealed unsafe partial class GuestImageCache byte stencilClear = 0; if (aspect == ImageAspectFlags.ColorBit) { - if (!PackedClearValue.TryDecodeColor(image.Description.PixelFormat, packedClear, out colorClear)) + if (!PackedClearValue.TryDecodeBufferColor(image.Description.PixelFormat, packedClear, out colorClear)) { return false; } diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.cs index 5568e0b..2050915 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.cs @@ -125,7 +125,12 @@ public sealed unsafe partial class GuestImageCache : IGuestImageCache, IGuestIma var metadataBaseLayer = request.View.BaseLayer; var result = LookUpImage(ref request, exactFormat); + CompleteLookup(result, in request, metadataBaseLayer); + return result; + } + public void CompleteLookup(ResourceSlotIdentifier result, in ImageRequest request, uint metadataBaseLayer) + { if (request.Role is ImageRole.Texture or ImageRole.StorageImage) { ulong metadataAddress; @@ -163,8 +168,6 @@ public sealed unsafe partial class GuestImageCache : IGuestImageCache, IGuestIma $"A compressed display surface can only be read from clean native GPU contents: address=0x{image.Description.Data.Address:X16} bufferModified={image.IsBufferModified} cpuDirty={image.IsCpuDirty} gpuModified={image.IsGpuModified}."); } } - - return result; } private ResourceSlotIdentifier LookUpImage(ref ImageRequest request, bool exactFormat) diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs b/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs index 7c772e6..25f2ed8 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs @@ -597,6 +597,30 @@ public static class PackedClearValue _ => 0, }; + // A buffer fill repeats the word in every texel, unlike a DCC clear register. + public static bool TryDecodeBufferColor(Format format, uint packed, out ClearColorValue clear) + { + clear = default; + switch (format) + { + case Format.R16G16Sfloat: + case Format.R16G16B16A16Sfloat: + var low = BitConverter.UInt16BitsToHalf((ushort)packed); + var high = BitConverter.UInt16BitsToHalf((ushort)(packed >> 16)); + // A float conversion may change a NaN payload; keep the raw fill in that case. + if (Half.IsNaN(low) || Half.IsNaN(high)) return false; + clear.Float32_0 = clear.Float32_2 = (float)low; + clear.Float32_1 = clear.Float32_3 = (float)high; + return true; + case Format.R32G32Uint: + case Format.R32G32B32A32Uint: + clear.Uint32_0 = clear.Uint32_1 = clear.Uint32_2 = clear.Uint32_3 = packed; + return true; + default: + return TryDecodeColor(format, packed, out clear); + } + } + public static bool TryDecodeColor(Format format, uint packed, out ClearColorValue clear) { static float Unorm8(uint value) => (value & 0xff) / 255.0f; diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs index 12bac61..be088e2 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs @@ -76,8 +76,56 @@ public static partial class ImageRequestBuilders public static bool SupportsDccFixedClear(Format format) => PackedClearValue.SupportsDccFixedColor(format); + // Every draw rebuilds its targets' requests (a whole mip layout) from the same register words, + // twice: once for the attachments and once for the pixel outputs. The build reads nothing but + // its arguments, so a result is reused while they are equal. + private struct ColorTargetMemo + { + public bool Valid; + public ColorTargetWords Words; + public uint TargetMask; + public uint DrawLayerOffset; + public bool IgnoreTargetMask; + public ColorTargetResolution? Result; + } + + private const int TargetMemoCount = 8; + + [ThreadStatic] + private static ColorTargetMemo[]? _colorTargetMemos; + + private static int TargetMemoSlot(ulong address) => (int)((address >> 12) ^ (address >> 20)) & (TargetMemoCount - 1); + // Builds the request for a bound color target. Null when the slot carries no target. public static ColorTargetResolution? ColorTarget(in ColorTargetWords words, uint targetMask, uint drawLayerOffset, bool ignoreTargetMask) + { + if (Rendering.RenderTrace.Enabled) + { + return ColorTargetCore(in words, targetMask, drawLayerOffset, ignoreTargetMask); + } + + var memos = _colorTargetMemos ??= new ColorTargetMemo[TargetMemoCount]; + ref var memo = ref memos[TargetMemoSlot(words.BaseAddress)]; + if (memo.Valid && memo.TargetMask == targetMask && memo.DrawLayerOffset == drawLayerOffset && + memo.IgnoreTargetMask == ignoreTargetMask && memo.Words.Equals(words)) + { + return memo.Result; + } + + var result = ColorTargetCore(in words, targetMask, drawLayerOffset, ignoreTargetMask); + memo = new ColorTargetMemo + { + Valid = true, + Words = words, + TargetMask = targetMask, + DrawLayerOffset = drawLayerOffset, + IgnoreTargetMask = ignoreTargetMask, + Result = result, + }; + return result; + } + + private static ColorTargetResolution? ColorTargetCore(in ColorTargetWords words, uint targetMask, uint drawLayerOffset, bool ignoreTargetMask) { var mask = targetMask & 0xF; if (ignoreTargetMask && words.BaseAddress != 0 && mask == 0) @@ -235,8 +283,7 @@ public static partial class ImageRequestBuilders pitch = TileGeometry.TexturePitch(transferFormat, width, GuestTileMode.Linear); } - var mipSpans = new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; - var mipPadded = new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + var (mipSpans, mipPadded) = MipLayoutScratch.Rent(); TiledSurfaceLayout? volumeLayout = null; ulong size; ulong backingSize = 0; diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs index 8a0c357..3fed162 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs @@ -59,8 +59,28 @@ public static partial class ImageRequestBuilders } // Builds the request for the bound depth target. Null when no depth or stencil state is active. - public static DepthTargetResolution? DepthTarget(in DepthTargetWords depthWords, IImageFormatSupport device) => - DepthTargetCore(in depthWords, device, copyMode: false, writeBuffer: false); + // Reused while the words and the device are the same, as ColorTarget's requests are. + public static DepthTargetResolution? DepthTarget(in DepthTargetWords depthWords, IImageFormatSupport device) + { + if (Rendering.RenderTrace.Enabled) + { + return DepthTargetCore(in depthWords, device, copyMode: false, writeBuffer: false); + } + + if (_depthTargetMemo is { } memo && ReferenceEquals(memo.Device, device) && memo.Words.Equals(depthWords)) + { + return memo.Result; + } + + var result = DepthTargetCore(in depthWords, device, copyMode: false, writeBuffer: false); + _depthTargetMemo = new DepthTargetMemo(depthWords, device, result); + return result; + } + + private sealed record DepthTargetMemo(DepthTargetWords Words, IImageFormatSupport Device, DepthTargetResolution? Result); + + [ThreadStatic] + private static DepthTargetMemo? _depthTargetMemo; // Builds the source or destination image used by DB_RENDER_OVERRIDE depth/stencil copies. public static DepthTargetResolution? DepthTargetCopy(in DepthTargetWords depthWords, IImageFormatSupport device, bool writeBuffer) => diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs index 1508dd0..f99fe0a 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs @@ -89,8 +89,7 @@ public static partial class ImageRequestBuilders return; } - var spans = new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; - var padded = new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + var (spans, padded) = MipLayoutScratch.Rent(); TileGeometry.TryGetTextureSize( description.GuestFormat, description.Extent.Width, description.Extent.Height, description.Resources.Levels, description.TileMode, out _, spans, padded); diff --git a/src/SharpEmu.Libs/Gpu/Images/MipStatistics.cs b/src/SharpEmu.Libs/Gpu/Images/MipStatistics.cs index e9b555f..a8f46ed 100644 --- a/src/SharpEmu.Libs/Gpu/Images/MipStatistics.cs +++ b/src/SharpEmu.Libs/Gpu/Images/MipStatistics.cs @@ -11,6 +11,10 @@ public sealed class MipStatistics public const uint FirstCounterOffset = 64; public const int ReportBytes = (int)FirstCounterOffset + SlotCount * CounterBytes; public const uint NoMipSampled = 15; + public const uint CounterWords = 257; + public const ulong CounterBufferBytes = 2 * CounterWords * sizeof(uint); + public const uint NoCounterField = 0x8000; + public const uint ReportWithoutCounters = 1; private const int CounterBytes = 2 * sizeof(uint); private const uint FinestMip = 0; @@ -29,6 +33,53 @@ public sealed class MipStatistics } } + public static uint ImageField(ReadOnlySpan texture) + { + if (((texture[5] >> 25) & 1) == 0) + { + return NoCounterField; + } + + var counter = texture[6] & 0xFF; + var baseLevel = (texture[3] >> 12) & 0xF; + var minLod = (texture[1] >> 8) & 0xFFF; + return counter | (baseLevel << 8) | (minLod << 16); + } + + public static void PackReport(ReadOnlySpan words, Span report) + { + report.Clear(); + if (report.Length >= sizeof(uint)) + { + BinaryPrimitives.WriteUInt32LittleEndian(report, FirstCounterOffset); + } + + for (var slot = 0; slot < SlotCount; slot++) + { + var start = (int)FirstCounterOffset + slot * CounterBytes; + if (start + CounterBytes > report.Length) + { + break; + } + + var finest = words[slot]; + var count = Math.Min(words[(int)CounterWords + slot], MaxSamples); + var level = finest == uint.MaxValue ? NoMipSampled : Math.Min(finest, 14u); + var counter = report.Slice(start, CounterBytes); + BinaryPrimitives.WriteUInt32LittleEndian(counter, count | ((uint)slot << 24)); + BinaryPrimitives.WriteUInt32LittleEndian(counter[sizeof(uint)..], level << 24); + } + } + + public void WriteReport(Span report) + { + var hasCounters = TryWriteReport(report); + if (report.Length >= sizeof(uint)) + { + BinaryPrimitives.WriteUInt32LittleEndian(report, hasCounters ? FirstCounterOffset : ReportWithoutCounters); + } + } + public bool TryWriteReport(Span report) { if (report.Length < ReportBytes) diff --git a/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs b/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs index 466023e..1c23e50 100644 --- a/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs +++ b/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs @@ -48,7 +48,7 @@ public enum SamplerBorderColor : uint public sealed unsafe class SamplerStore : IDisposable { private readonly GpuDeviceInfo _device; - private readonly Dictionary<(uint, uint, uint, uint, bool), Sampler> _samplers = new(); + private readonly System.Collections.Concurrent.ConcurrentDictionary<(uint, uint, uint, uint, bool), Sampler> _samplers = new(); private readonly object _gate = new(); public SamplerStore(GpuDeviceInfo device) => _device = device; @@ -57,16 +57,22 @@ public sealed unsafe class SamplerStore : IDisposable public Sampler GetSampler(in SamplerDescriptorWords words, bool integerView) { + var key = (words[0], words[1], words[2], words[3], integerView); + // Every sampler binding of every draw asks; only the first of each kind creates. + if (_samplers.TryGetValue(key, out var found)) + { + return found; + } + lock (_gate) { - var key = (words[0], words[1], words[2], words[3], integerView); if (_samplers.TryGetValue(key, out var existing)) { return existing; } var sampler = CreateSampler(words, integerView); - _samplers.Add(key, sampler); + _samplers[key] = sampler; return sampler; } } diff --git a/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs b/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs index 989e5e3..fef468d 100644 --- a/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs +++ b/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs @@ -233,8 +233,7 @@ public sealed class TextureTransferLayout layout.Pitch = TileGeometry.TexturePitch(format, width, tile); if (tile == GuestTileMode.Linear) { - var levelSpans = new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; - var paddedSizes = new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + var (levelSpans, paddedSizes) = MipLayoutScratch.Rent(); TileGeometry.TryGetTextureSize(format, width, height, levels, tile, out _, levelSpans, paddedSizes); for (uint level = 0; level < levels; level++) { diff --git a/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs b/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs index 2de8d20..837441e 100644 --- a/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs +++ b/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs @@ -21,6 +21,24 @@ public struct TileLevelSpan public readonly record struct TilePaddedSize(uint Width, uint Height); +// The per-level output arrays of the layout queries, kept per thread: every draw that resolves a +// target or a texture needs them for a moment and reads them before the next query. +internal static class MipLayoutScratch +{ + [ThreadStatic] private static TileLevelSpan[]? _spans; + [ThreadStatic] private static TilePaddedSize[]? _padded; + + // Zeroed, as a new array would be, so a level the query leaves out reads as empty. + public static (TileLevelSpan[] Spans, TilePaddedSize[] Padded) Rent() + { + var spans = _spans ??= new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; + var padded = _padded ??= new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + Array.Clear(spans); + Array.Clear(padded); + return (spans, padded); + } +} + public enum TileBlockKind : uint { Standard256B, diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/DescriptorWriter.cs b/src/SharpEmu.Libs/Gpu/Pipelines/DescriptorWriter.cs index 7a2d378..da19348 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/DescriptorWriter.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/DescriptorWriter.cs @@ -28,7 +28,8 @@ public static class DescriptorWriter { DescriptorBindingKind.Samplers => Silk.NET.Vulkan.DescriptorType.Sampler, DescriptorBindingKind.Buffers or DescriptorBindingKind.GlobalDataShare or DescriptorBindingKind.DeviceAddressPageTable or - DescriptorBindingKind.FaultBuffer or DescriptorBindingKind.FlattenedResourceTable or DescriptorBindingKind.ShaderData => + DescriptorBindingKind.FaultBuffer or DescriptorBindingKind.FlattenedResourceTable or DescriptorBindingKind.ShaderData or + DescriptorBindingKind.MipStatistics => Silk.NET.Vulkan.DescriptorType.StorageBuffer, _ => throw SubmissionScheduler.Fatal($"The descriptor binding kind is invalid: kind={kind}."), }; diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs b/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs index 2eff4e4..2595149 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs @@ -36,6 +36,13 @@ internal interface IShaderPipelineHost bool ComputeWave64Supported { get; } + // True when a 64-invocation wave64 compute workgroup runs as one 64-lane host subgroup. + bool ComputeWave64SubgroupNative => false; + + bool ComputeRunsOn32LaneSubgroups(ComputeInputInfo input) => false; + + bool MipStatisticsSubgroupReduction => false; + bool GraphicsSubgroupOperationsEnabled { get; } // The device supports shaderSharedInt64Atomics, so LDS 64-bit atomics can be diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs b/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs index f4eba84..4394dcd 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs @@ -45,6 +45,13 @@ public sealed class PipelineStaticParameters : IEquatable public Format StencilFormat { get; set; } = Format.Undefined; public uint ColorCount { get; set; } + // The state of a new instance, for a lookup object reused across draws. + public void Reset() + { + Array.Clear(ColorFormats); + DepthFormat = Format.Undefined; + StencilFormat = Format.Undefined; + ColorCount = 0; + } + public bool Equals(PipelineRenderingState? other) => other is not null && ColorCount == other.ColorCount && DepthFormat == other.DepthFormat && StencilFormat == other.StencilFormat && ColorFormats.AsSpan().SequenceEqual(other.ColorFormats); @@ -179,6 +195,15 @@ public sealed class PipelineVertexInputState : IEquatable other is not null && BindingCount == other.BindingCount && AttributeCount == other.AttributeCount && Bindings.AsSpan().SequenceEqual(other.Bindings) && Attributes.AsSpan().SequenceEqual(other.Attributes); @@ -207,8 +232,9 @@ public sealed class PipelineVertexInputState : IEquatable { public required PipelineRenderingState Rendering { get; init; } - public ulong VertexProgramId { get; init; } - public ulong PixelProgramId { get; init; } + // Settable for the lookup key reused across draws; a key in the cache is never changed. + public ulong VertexProgramId { get; set; } + public ulong PixelProgramId { get; set; } public required PipelineVertexInputState VertexInput { get; init; } public required PipelineStaticParameters StaticParameters { get; init; } diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs index 96a8f8d..0aafc52 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs @@ -27,6 +27,9 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider private readonly Dictionary _graphicsPipelines = new(); private readonly Dictionary _computePipelines = new(); private readonly object _gate = new(); + // The pipeline maps: a lookup takes this lock or _gate, an addition both; a cached draw so does + // not wait for the program preparation a resolving thread holds _gate for. + private readonly object _pipelineGate = new(); private readonly bool _strictShaders = Environment.GetEnvironmentVariable("SHARPEMU_STRICT_COMPUTE") != "0"; private readonly HashSet<(ShaderStage Stage, ulong Hash, uint CodeSize)> _reportedShaderSkips = []; @@ -70,6 +73,11 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider return new ShaderSource(registered, hash, userData, userDataBase, stage); } + // Program resolution runs on the render thread and on a thread resolving draws ahead of it; + // the program, decode and materialization caches it updates are used under this lock only. + // A pipeline lookup takes _pipelineGate instead, so it does not wait for a resolution. + private readonly object _resolveGate = new(); + public GraphicsPrograms GetGraphicsPrograms( VertexStageRegisters vertex, PixelStageRegisters pixel, @@ -78,6 +86,21 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider ReadOnlySpan targetExportMapping, bool pixelActive, bool depthBound) + { + lock (_resolveGate) + { + return GetGraphicsProgramsCore(vertex, pixel, shaderInterface, context, targetExportMapping, pixelActive, depthBound); + } + } + + private GraphicsPrograms GetGraphicsProgramsCore( + VertexStageRegisters vertex, + PixelStageRegisters pixel, + ShaderInterfaceRegisters shaderInterface, + ContextRegisters context, + ReadOnlySpan targetExportMapping, + bool pixelActive, + bool depthBound) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.ProgramPreparation); var vertexSource = PrepareSource( @@ -209,44 +232,53 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider shaderInterface.VertexOutputControl, clipSpace); } - // One past the highest parameter location the pixel program reads, resolved as its translator does. - private static uint ReadVertexOutputCount(Gen5ShaderProgram pixelProgram, PixelInputInfo info) - { - var attributes = pixelProgram.Instructions + // The interpolated attributes a pixel program reads, ascending. A program never changes, and this + // was rebuilt with LINQ on every draw. + private static readonly System.Runtime.CompilerServices.ConditionalWeakTable InterpolatedAttributeCache = new(); + + private static uint[] InterpolatedAttributesOf(Gen5ShaderProgram program) => + InterpolatedAttributeCache.GetValue(program, static program => program.Instructions .Select(static instruction => instruction.Control) .OfType() .Select(static control => control.Attribute) .Distinct() .Order() - .ToArray(); + .ToArray()); + + // One past the highest parameter location the pixel program reads, resolved as its translator does. + private static uint ReadVertexOutputCount(Gen5ShaderProgram pixelProgram, PixelInputInfo info) + { + var attributes = InterpolatedAttributesOf(pixelProgram); if (attributes.Length == 0) { return 0; } - var controls = new uint[32]; + Span controls = stackalloc uint[32]; for (var index = 0u; index < (uint)controls.Length; index++) { - controls[index] = index < info.InputCount && index < (uint)info.InterpolatorSettings.Length + controls[(int)index] = index < info.InputCount && index < (uint)info.InterpolatorSettings.Length ? info.InterpolatorSettings[index] : index; } - return Gen5PixelInputMapping.ResolveLocations(controls, attributes).Max() + 1; - } - - private static uint InterpolatedAttributeCount(Gen5ShaderProgram program) - { - var maxAttribute = -1; - foreach (var instruction in program.Instructions) + // Attributes are distinct and below 32, so a program never has more than 32 of them. + Span locations = attributes.Length <= 32 ? stackalloc uint[attributes.Length] : new uint[attributes.Length]; + Gen5PixelInputMapping.ResolveLocations(controls, attributes, locations); + var highest = 0u; + foreach (var location in locations) { - if (instruction.Control is Gen5InterpolationControl interpolation) - { - maxAttribute = Math.Max(maxAttribute, (int)interpolation.Attribute); - } + highest = Math.Max(highest, location); } - return (uint)(maxAttribute + 1); + return highest + 1; + } + + // The attribute list is ascending, so its last entry is the highest. + private static uint InterpolatedAttributeCount(Gen5ShaderProgram program) + { + var attributes = InterpolatedAttributesOf(program); + return attributes.Length == 0 ? 0 : attributes[^1] + 1; } // The bound colour slots in order; each output mode names the kind the pixel program exports. @@ -321,6 +353,20 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider uint dimensionX, uint dimensionY, uint dimensionZ) + { + lock (_resolveGate) + { + return GetComputeProgramCore(compute, shaderInterface, dispatchInitiator, dimensionX, dimensionY, dimensionZ); + } + } + + private ComputeProgram GetComputeProgramCore( + ComputeStageRegisters compute, + ShaderInterfaceRegisters shaderInterface, + uint dispatchInitiator, + uint dimensionX, + uint dimensionY, + uint dimensionZ) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.ProgramPreparation); var source = PrepareSource(compute.Address, ShaderStage.Compute, "compute", compute.UserScalars, compute.UserScalarCount, probeWrittenRegisters: false, userDataBase: 0); @@ -417,6 +463,28 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider ShaderProgram pixelProgram) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.PipelineCreation); + // The lookup fills objects this thread reuses for every draw; only a new pipeline gets its own. + var lookup = _graphicsLookup ??= new GraphicsPipelineKey + { + Rendering = new PipelineRenderingState(), + VertexInput = new PipelineVertexInputState(), + StaticParameters = new PipelineStaticParameters(), + }; + lookup.Rendering.Reset(); + lookup.VertexInput.Reset(); + lookup.StaticParameters.Reset(); + FillGraphicsState(colors, in depth, vertexInput, pixelInput, context, in rendering, topology, primitiveRestartEnabled, disableBlending, + vertexProgram, pixelProgram, _host.NoAttachmentSampleCounts, lookup.Rendering, lookup.StaticParameters, lookup.VertexInput); + lookup.VertexProgramId = vertexProgram.Id; + lookup.PixelProgramId = pixelInput is not null ? pixelProgram.Id : 0; + lock (_pipelineGate) + { + if (_graphicsPipelines.TryGetValue(lookup, out var cached)) + { + return cached; + } + } + var description = BuildGraphicsDescription( colors, in depth, vertexInput, pixelInput, context, in rendering, topology, primitiveRestartEnabled, disableBlending, vertexProgram, pixelProgram, _host.NoAttachmentSampleCounts); @@ -437,12 +505,19 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider } var created = _host.CreateGraphicsPipeline(description); - _graphicsPipelines.Add(key, created); + lock (_pipelineGate) + { + _graphicsPipelines.Add(key, created); + } + ShaderCacheCounters.CountGraphicsPipeline(); return created; } } + [ThreadStatic] + private static GraphicsPipelineKey? _graphicsLookup; + // The full static state of one graphics pipeline from the draw's targets, registers and programs. internal static GraphicsPipelineDescription BuildGraphicsDescription( ReadOnlySpan colors, @@ -457,6 +532,43 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider ShaderProgram vertexProgram, ShaderProgram pixelProgram, SampleCountFlags noAttachmentSampleCounts) + { + var renderingState = new PipelineRenderingState(); + var parameters = new PipelineStaticParameters(); + var vertexState = new PipelineVertexInputState(); + var (vertexStage, pixelStage) = FillGraphicsState(colors, in depth, vertexInput, pixelInput, context, in rendering, topology, + primitiveRestartEnabled, disableBlending, vertexProgram, pixelProgram, noAttachmentSampleCounts, renderingState, parameters, vertexState); + return new GraphicsPipelineDescription + { + Rendering = renderingState, + VertexInput = vertexState, + VertexInfo = vertexInput, + VertexProgram = vertexProgram, + VertexStage = vertexStage, + PixelInfo = pixelInput, + PixelProgram = pixelProgram, + PixelStage = pixelStage, + StaticParameters = parameters, + }; + } + + // Writes the pipeline state into objects in their initial state; returns the stage programs. + private static (ShaderProgramInfo VertexStage, ShaderProgramInfo? PixelStage) FillGraphicsState( + ReadOnlySpan colors, + in DepthAttachmentState depth, + VertexInputInfo vertexInput, + PixelInputInfo? pixelInput, + ContextRegisters context, + in RenderingState rendering, + PrimitiveTopology topology, + bool primitiveRestartEnabled, + bool disableBlending, + ShaderProgram vertexProgram, + ShaderProgram pixelProgram, + SampleCountFlags noAttachmentSampleCounts, + PipelineRenderingState renderingState, + PipelineStaticParameters parameters, + PipelineVertexInputState vertexState) { if (colors.Length > PipelineStaticParameters.ColorAttachmentCount) { @@ -477,8 +589,8 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider var vertexStage = vertexInput.Stage.Program ?? throw SubmissionScheduler.Fatal("The vertex stage has no program."); var pixelStage = pixelActive ? pixelInput!.Stage.Program ?? throw SubmissionScheduler.Fatal("The pixel stage has no program.") : null; var colorCount = (uint)colors.Length; - var parameters = new PipelineStaticParameters { ColorCount = colorCount }; - var renderingState = new PipelineRenderingState { ColorCount = colorCount }; + parameters.ColorCount = colorCount; + renderingState.ColorCount = colorCount; for (var index = 0; index < colors.Length; index++) { ref readonly var color = ref colors[index]; @@ -564,18 +676,8 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider parameters.SetBlendBypass(index, ((context.ColorTargets[slot].Info >> 16) & 0x1) != 0); } - return new GraphicsPipelineDescription - { - Rendering = renderingState, - VertexInput = BuildVertexInputState(vertexInput), - VertexInfo = vertexInput, - VertexProgram = vertexProgram, - VertexStage = vertexStage, - PixelInfo = pixelInput, - PixelProgram = pixelProgram, - PixelStage = pixelStage, - StaticParameters = parameters, - }; + FillVertexInputState(vertexInput, vertexState); + return (vertexStage, pixelStage); } internal static GraphicsPipelineKey KeyOf(GraphicsPipelineDescription description) => new() @@ -587,18 +689,15 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider StaticParameters = description.StaticParameters, }; - private static PipelineVertexInputState BuildVertexInputState(VertexInputInfo info) + private static void FillVertexInputState(VertexInputInfo info, PipelineVertexInputState state) { if (info.Buffers.Length > VertexInputInfo.MaxBuffers || info.Attributes.Length > VertexInputInfo.MaxBuffers) { throw SubmissionScheduler.Fatal($"The vertex input is too large: buffers={info.Buffers.Length} attributes={info.Attributes.Length}."); } - var state = new PipelineVertexInputState - { - BindingCount = (byte)info.Buffers.Length, - AttributeCount = (byte)info.Attributes.Length, - }; + state.BindingCount = (byte)info.Buffers.Length; + state.AttributeCount = (byte)info.Attributes.Length; for (var binding = 0; binding < info.Buffers.Length; binding++) { var buffer = info.Buffers[binding]; @@ -615,8 +714,6 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider state.Attributes[index] = new PipelineVertexAttribute(attribute.OffsetBytes, (byte)attribute.BufferIndex); } - - return state; } public PipelineHandle CreateComputePipeline(ComputeInputInfo input, ShaderProgram program) @@ -629,6 +726,14 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider var stage = input.Stage.Program ?? throw SubmissionScheduler.Fatal("The compute stage has no program."); var key = new ComputePipelineKey(program.Id); + lock (_pipelineGate) + { + if (_computePipelines.TryGetValue(key, out var cached)) + { + return cached; + } + } + lock (_gate) { if (_computePipelines.TryGetValue(key, out var cached)) @@ -642,7 +747,11 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider } var created = _host.CreateComputePipeline(new ComputePipelineDescription { Input = input, Program = program, Stage = stage }); - _computePipelines.Add(key, created); + lock (_pipelineGate) + { + _computePipelines.Add(key, created); + } + ShaderCacheCounters.CountComputePipeline(); return created; } @@ -659,6 +768,15 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider var stage = input.Stage.Program ?? throw SubmissionScheduler.Fatal("The compute stage has no program."); var key = new ComputePipelineKey(program.Id); + lock (_pipelineGate) + { + if (_computePipelines.TryGetValue(key, out var cached)) + { + handle = cached; + return true; + } + } + lock (_gate) { if (_computePipelines.TryGetValue(key, out var cached)) @@ -674,7 +792,11 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider return false; } - _computePipelines.Add(key, created); + lock (_pipelineGate) + { + _computePipelines.Add(key, created); + } + ShaderCacheCounters.CountComputePipeline(); handle = created; return true; diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs index 611b9c8..7607f3d 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs @@ -1,6 +1,7 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Runtime.InteropServices; using SharpEmu.HLE; using SharpEmu.Libs.Gpu.Rendering; using SharpEmu.Libs.Gpu.Scheduling; @@ -68,6 +69,37 @@ public sealed class ProgramKey : IEquatable public override int GetHashCode() => HashCode.Combine(Stage, Hash, UserDataCount, CodeSize, StaticState.Length); } +// A program key as a lookup sees it: the static state words stay in the caller's list, so a draw +// that finds its program does not build a key object or copy the words. +internal readonly ref struct ProgramKeyProbe(ShaderStage stage, ulong hash, uint userDataCount, uint codeSize, ReadOnlySpan staticState) +{ + public readonly ShaderStage Stage = stage; + public readonly ulong Hash = hash; + public readonly uint UserDataCount = userDataCount; + public readonly uint CodeSize = codeSize; + public readonly ReadOnlySpan StaticState = staticState; +} + +internal sealed class ProgramKeyComparer : IEqualityComparer, IAlternateEqualityComparer +{ + public static readonly ProgramKeyComparer Instance = new(); + + public bool Equals(ProgramKey? x, ProgramKey? y) => x is null ? y is null : x.Equals(y); + + public int GetHashCode(ProgramKey key) => key.GetHashCode(); + + public bool Equals(ProgramKeyProbe alternate, ProgramKey key) => + alternate.Stage == key.Stage && alternate.Hash == key.Hash && alternate.UserDataCount == key.UserDataCount && + alternate.CodeSize == key.CodeSize && alternate.StaticState.SequenceEqual(key.StaticState); + + // The same value as ProgramKey.GetHashCode. + public int GetHashCode(ProgramKeyProbe alternate) => + HashCode.Combine(alternate.Stage, alternate.Hash, alternate.UserDataCount, alternate.CodeSize, alternate.StaticState.Length); + + public ProgramKey Create(ProgramKeyProbe alternate) => + new(alternate.Stage, alternate.Hash, alternate.UserDataCount, alternate.CodeSize, alternate.StaticState.ToArray()); +} + // One compiled module of a program entry for one specialization and push-data start. internal sealed class ProgramPermutation { @@ -102,7 +134,7 @@ internal sealed class ShaderProgramCache private readonly CpuContext _context; private readonly IGuestGpuBackend _compiler; private readonly IShaderPipelineHost _host; - private readonly Dictionary _programs = new(); + private readonly Dictionary _programs = new(ProgramKeyComparer.Instance); private readonly Dictionary<(ulong Hash, uint CodeSize), Gen5ShaderProgram> _decoded = new(); private readonly Dictionary<(ulong Hash, uint CodeSize), ShaderCodeCapture> _codeCaptures = new(); private readonly List _staticState = new(StageStaticKey.MaxWords); @@ -216,13 +248,14 @@ internal sealed class ShaderProgramCache private ShaderProgram GetOrCompileCore(ShaderSource source, StageCompileOptions options, ref uint pushDataCursor, out ShaderStageResources stage) { - ProgramKey key; + ProgramKey? key = null; ProgramSourceEntry? entry; using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.ProgramCacheLookup)) { BuildStaticState(source.Stage, options); - key = new ProgramKey(source.Stage, source.Hash, (uint)source.UserData.Length, source.CodeSize, _staticState.ToArray()); - _programs.TryGetValue(key, out entry); + var probe = new ProgramKeyProbe(source.Stage, source.Hash, (uint)source.UserData.Length, source.CodeSize, + CollectionsMarshal.AsSpan(_staticState)); + _programs.GetAlternateLookup().TryGetValue(probe, out entry); } var sourceWasCached = entry is not null; if (RenderTrace.Enabled && RenderTrace.Pipeline()) @@ -246,12 +279,14 @@ internal sealed class ShaderProgramCache if (entry is null) { entry = CreateEntry(source, options); + key = BuildProgramKey(source); _programs.Add(key, entry); ShaderCacheCounters.CountProgram(); } - var snapshot = new ResourceSnapshot(); - var specialization = new ResourceSpecialization(); + // Both are assigned by the materialization that follows; a failed one throws. + ResourceSnapshot snapshot = null!; + ResourceSpecialization specialization = null!; var captureIndirectImageFailure = _spirvDumpEnabled ? ShaderPermutationDump.CreateFailureCapture(source) : null; if (Diagnostics.GpuReadTrace.Enabled) { @@ -295,7 +330,7 @@ internal sealed class ShaderProgramCache } } - var permutation = CompilePermutation(source, options, entry, specialization, pushDataCursor, key, sourceWasCached); + var permutation = CompilePermutation(source, options, entry, specialization, pushDataCursor, key ?? BuildProgramKey(source), sourceWasCached); entry.Permutations.Add(permutation); ShaderCacheCounters.CountPermutation(); stage = CreateStageResources(permutation.Program, snapshot, source, options); @@ -308,6 +343,10 @@ internal sealed class ShaderProgramCache return permutation.Handle; } + // The key of the entry the static state was just built for. + private ProgramKey BuildProgramKey(ShaderSource source) => + new(source.Stage, source.Hash, (uint)source.UserData.Length, source.CodeSize, _staticState.ToArray()); + private static ShaderStageResources CreateStageResources( ShaderProgramInfo program, ResourceSnapshot snapshot, ShaderSource source, StageCompileOptions options) { @@ -641,12 +680,14 @@ internal sealed class ShaderProgramCache SupportsPerVertexPixelInputs = _host.PerVertexPixelInputsSupported, PixelInputAddress = options.PixelInputAddress, PixelInputCntl = interpolators, + MipStatisticsSubgroupReduction = _host.MipStatisticsSubgroupReduction, }; } default: return BuildComputeRequest(entry.Plan, resources, layout, options.ComputeInfo!, options.ComputeSystemRegisters, - sharedInt64Atomics, _host.ExecGuardElisionEnabled, signedZeroInfNanPreserve); + sharedInt64Atomics, _host.ExecGuardElisionEnabled, signedZeroInfNanPreserve, _host.ComputeWave64SubgroupNative, + _host.ComputeRunsOn32LaneSubgroups(options.ComputeInfo!)); } } @@ -663,10 +704,12 @@ internal sealed class ShaderProgramCache private static ShaderCompileRequest BuildComputeRequest(ShaderResourcePlan plan, SpecializedResourceInfo resources, BindingLayout layout, ComputeInputInfo info, Gen5ComputeSystemRegisters? systemRegisters, bool sharedInt64Atomics, bool execGuardElision, - bool signedZeroInfNanPreserve) => + bool signedZeroInfNanPreserve, bool hostWave64Supported = false, bool hostComputeSubgroup32 = false) => new(plan, resources, layout) { WaveSize = info.WaveSize, + HostWave64Supported = info.WaveSize == 64 && hostWave64Supported, + HostComputeSubgroup32 = info.WaveSize == 64 && !hostWave64Supported && hostComputeSubgroup32, EnableExecGuardElision = info.WaveSize != 64 || execGuardElision, TraceDeviceAddressFaults = SharpEmu.HLE.GpuMemory.GuestGpuMemoryHook.TraceEnabled, ScratchDwords = info.ScratchDwords, @@ -681,7 +724,7 @@ internal sealed class ShaderProgramCache internal static bool TryCompilePrewarm(ComputePrewarmRecord record, ShaderCodeCapture code, IGuestGpuBackend compiler, bool sharedInt64Atomics, bool execGuardElision, bool signedZeroInfNanPreserve, out IGuestCompiledShader? compiled, - out BindingLayout? layout, out string error) + out BindingLayout? layout, out string error, bool hostWave64Supported = false, bool hostComputeSubgroup32 = false) { compiled = null; layout = null; @@ -698,7 +741,7 @@ internal sealed class ShaderProgramCache layout = AllocateLayout(program, plan, resources, record.UserDataBase, record.UserDataCount, record.PushDataCursor, record.Info.DispatchThreadDimensions); var request = BuildComputeRequest(plan, resources, layout, record.Info, record.SystemRegisters, - sharedInt64Atomics, execGuardElision, signedZeroInfNanPreserve); + sharedInt64Atomics, execGuardElision, signedZeroInfNanPreserve, hostWave64Supported, hostComputeSubgroup32); return compiler.TryCompileProgram(request, out compiled, out error) && compiled is not null; } catch (Exception exception) diff --git a/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs new file mode 100644 index 0000000..0ea527f --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs @@ -0,0 +1,282 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Scheduling; + +namespace SharpEmu.Libs.Gpu.Rendering; + +// Resolves the programs of deferred draws on a worker thread, in issue order, while the render +// thread interprets and records. The render thread takes a result for the draw it runs only when +// the resolution completed, used the inputs the draw derives and still matches the host's +// GPU-ownership state; otherwise it resolves the draw itself. +public sealed class ParallelProgramPrefetch : IProgramPrefetch, IDisposable +{ + public sealed class Job(RegisterBanks banks, long gate) + { + internal readonly RegisterBanks Banks = banks; + // The count of stream items other than draws that must have run before the job resolves. + internal readonly long Gate = gate; + internal GraphicsProgramInputs Inputs; + internal GraphicsPrograms? Programs; + internal object? Assumptions; + // 0 queued, 1 resolved, 2 abandoned. + internal volatile int State; + } + + private readonly RenderExecutor _executor; + private readonly IRenderHost _host; + private readonly IConcurrentResolveHost _concurrent; + private readonly object _lock = new(); + private readonly Queue _queue = new(); + private readonly Thread _thread; + private int _outstanding; + private long _finished; + private long _executedGate; + // The gate the worker sleeps on; long.MaxValue while it does not. + private long _sleepingOnGate = long.MaxValue; + private bool _paused; + private bool _running; + private bool _stopping; + + public ParallelProgramPrefetch(RenderExecutor executor, IRenderHost host, IConcurrentResolveHost concurrent) + { + _executor = executor; + _host = host; + _concurrent = concurrent; + _thread = new Thread(Run) { IsBackground = true, Name = "SharpEmu program resolve" }; + _thread.Start(); + } + + // The job of the draw the render thread runs now; set around each deferred draw. + public Job? Current { get; set; } + + public long Submitted { get; private set; } + + // A gate above zero holds the job until AdvanceGate reaches it: with the command stream on its own + // thread, draws are submitted ahead of the work before them, which may change what they read. + public Job Submit(RegisterBanks banks, long gate = 0) + { + var job = new Job(banks, gate); + lock (_lock) + { + _queue.Enqueue(job); + Submitted++; + _outstanding++; + Monitor.PulseAll(_lock); + } + + return job; + } + + // Returns once the worker has finished every submitted job: it reads guest memory and cache + // state, which the render thread changes only with the worker idle. + public void WaitIdle() + { + if (Volatile.Read(ref _outstanding) == 0) + { + return; + } + + lock (_lock) + { + while (_outstanding != 0) + { + Monitor.Wait(_lock); + } + } + } + + // The render thread ran this many non-draw items: jobs gated on them may resolve. + public void AdvanceGate(long executed) + { + Interlocked.Exchange(ref _executedGate, executed); + // Only a worker asleep on a gate this reaches needs a wake: one per item costs the render + // thread a kernel call each. + if (Volatile.Read(ref _sleepingOnGate) <= executed) + { + lock (_lock) + { + Monitor.PulseAll(_lock); + } + } + } + + // Returns once the worker finished the first count jobs it was given, so a non-draw item runs + // with no resolution of the draws before it still reading. + public void WaitFinished(long count) + { + if (Volatile.Read(ref _finished) >= count) + { + return; + } + + lock (_lock) + { + while (_finished < count) + { + Monitor.Wait(_lock); + } + } + } + + // Holds the worker between jobs, for host work that changes what a resolution reads. + public void Pause() + { + lock (_lock) + { + _paused = true; + while (_running) + { + Monitor.Wait(_lock); + } + } + } + + public void Resume() + { + lock (_lock) + { + _paused = false; + Monitor.PulseAll(_lock); + } + } + + public bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs) + { + programs = null!; + var job = Current; + if (job is null || !ReferenceEquals(job.Banks, banks)) + { + return false; + } + + Current = null; + WaitFor(job); + if (job.State != 1 || !job.Inputs.Equals(inputs) || !_concurrent.IsStillCurrent(job.Assumptions!)) + { + return false; + } + + programs = job.Programs!; + return true; + } + + private void WaitFor(Job job) + { + var spinner = new SpinWait(); + for (var spin = 0; spin < 64 && job.State == 0; spin++) + { + spinner.SpinOnce(sleep1Threshold: -1); + } + + if (job.State != 0) + { + return; + } + + lock (_lock) + { + while (job.State == 0) + { + Monitor.Wait(_lock); + } + } + } + + private void Run() + { + SubmissionScheduler.ThrowOnFatal = true; + while (true) + { + Job job; + lock (_lock) + { + while (_queue.Count == 0 && !_stopping) + { + Monitor.Wait(_lock); + } + + if (_stopping) + { + return; + } + + job = _queue.Peek(); + if (job.Gate > Volatile.Read(ref _executedGate)) + { + // The gate usually opens within a few items: spin before sleeping. + Monitor.Exit(_lock); + try + { + var spinner = new SpinWait(); + for (var spin = 0; spin < 50 && job.Gate > Volatile.Read(ref _executedGate); spin++) + { + spinner.SpinOnce(sleep1Threshold: -1); + } + } + finally + { + Monitor.Enter(_lock); + } + } + + while ((_paused || job.Gate > Volatile.Read(ref _executedGate)) && !_stopping) + { + Interlocked.Exchange(ref _sleepingOnGate, _paused ? long.MaxValue : job.Gate); + if (!_paused && job.Gate <= Volatile.Read(ref _executedGate)) + { + break; + } + + Monitor.Wait(_lock, 50); + } + + Volatile.Write(ref _sleepingOnGate, long.MaxValue); + if (_stopping) + { + return; + } + + _queue.Dequeue(); + _running = true; + } + + Resolve(job); + lock (_lock) + { + _outstanding--; + _finished++; + _running = false; + Monitor.PulseAll(_lock); + } + } + } + + private void Resolve(Job job) + { + _concurrent.BeginConcurrentResolve(); + try + { + job.Inputs = RenderExecutor.ProgramInputsOf(job.Banks, _host.FormatSupport, _host.Fatal); + job.Programs = _executor.ResolveGraphicsPrograms(job.Banks, in job.Inputs); + job.Assumptions = _concurrent.EndConcurrentResolve(); + job.State = 1; + } + catch (Exception) + { + // The render thread resolves the draw itself, and fails there if the error is real. + job.Assumptions = _concurrent.EndConcurrentResolve(); + job.State = 2; + } + } + + public void Dispose() + { + lock (_lock) + { + _stopping = true; + Monitor.PulseAll(_lock); + } + } +} diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs index bf29ca8..9eafa94 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs @@ -72,6 +72,7 @@ public sealed partial class RenderExecutor if (Diagnostics.DccWriterTrace.Enabled) { Diagnostics.DccWriterTrace.CurrentProgram = program.Hash; + Diagnostics.DccWriterTrace.CurrentKind = "compute"; } if (RenderTrace.Enabled) @@ -714,15 +715,19 @@ public sealed partial class RenderExecutor var hash = input.Stage.Program!.Hash; if (!_host.TryClearImageFromBuffer(address, clear.Size, clear.PackedClear)) { - // A metadata fill may run before its target is bound; the store keeps it pending and the dispatch runs. + // A metadata fill may run before its target is bound; the store keeps it pending. The fill + // itself then runs on the host when the range is known DCC metadata: run as a dispatch it + // leaves the metadata GPU-written, and every later bind of the surface waits for the GPU to + // read it back (Astro Bot fills several surfaces this way each frame). var registered = _host.TryAbsorbDccFill(address, clear.Size, clear.PackedClear); + var filledOnHost = !registered && _host.TryFillDccMetadata(address, clear.Size, clear.PackedClear); if (RenderTrace.Enabled && RenderTrace.MetadataClear()) { RenderTrace.Write( - $"{(registered ? "Tracked" : "Deferred")} a metadata clear: shader=0x{hash:X16} address=0x{address:X16} size=0x{clear.Size:X16} value=0x{clear.PackedClear:X8}"); + $"{(registered ? "Tracked" : filledOnHost ? "Filled on the host" : "Deferred")} a metadata clear: shader=0x{hash:X16} address=0x{address:X16} size=0x{clear.Size:X16} value=0x{clear.PackedClear:X8}"); } - return registered; + return registered || filledOnHost; } if (RenderTrace.Enabled && RenderTrace.ImageClear()) diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs new file mode 100644 index 0000000..31c96ac --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs @@ -0,0 +1,115 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Buffers; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Images; + +namespace SharpEmu.Libs.Gpu.Rendering; + +// What a draw's program resolution reads besides the stage and context registers: the export +// mapping of every bound color slot, whether the pixel stage runs and whether a depth target is +// bound. All three follow from the register banks alone. +public struct GraphicsProgramInputs : IEquatable +{ + [System.Runtime.CompilerServices.InlineArray(RenderingState.ColorAttachmentCapacity)] + public struct ExportMappings + { + private ColorComponentMap _element0; + } + + public ExportMappings Mapping; + public bool PixelActive; + public bool DepthBound; + + public static GraphicsProgramInputs Create() + { + var inputs = new GraphicsProgramInputs(); + ((Span)inputs.Mapping).Fill(ColorComponentMap.Identity); + return inputs; + } + + public readonly bool Equals(GraphicsProgramInputs other) => + PixelActive == other.PixelActive && DepthBound == other.DepthBound && + ((ReadOnlySpan)Mapping).SequenceEqual(other.Mapping); + + public override readonly bool Equals(object? obj) => obj is GraphicsProgramInputs other && Equals(other); + + public override readonly int GetHashCode() + { + var hash = new HashCode(); + foreach (var map in (ReadOnlySpan)Mapping) + { + hash.Add(map.Packed); + } + + hash.Add(PixelActive); + hash.Add(DepthBound); + return hash.ToHashCode(); + } +} + +// The host side of resolving programs on a thread other than the render thread. The resolving +// thread records what it assumed of the host's GPU-ownership state; the render thread checks, +// when it takes the result, that the assumptions still hold. +public interface IConcurrentResolveHost +{ + // On the resolving thread, around one resolution. + void BeginConcurrentResolve(); + + object EndConcurrentResolve(); + + // On the render thread. + bool IsStillCurrent(object assumptions); +} + +// Program resolutions computed ahead of their draws. A draw takes one only when it was resolved +// from the same register banks with the same inputs; otherwise it resolves its programs itself. +public interface IProgramPrefetch +{ + bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs); +} + +public sealed partial class RenderExecutor +{ + public IProgramPrefetch? Prefetch { get; set; } + + // The inputs ResolveShaderPrograms derives from the targets TryResolveDrawTargets binds, from the + // registers alone: the target requests and the pixel and depth activity, without the images. + public static GraphicsProgramInputs ProgramInputsOf(RegisterBanks banks, IImageFormatSupport formats, Func fatal) + { + var inputs = GraphicsProgramInputs.Create(); + var context = banks.Context; + Span mapping = inputs.Mapping; + for (var slot = 0u; slot < ContextRegisters.ColorTargetCount; slot++) + { + if (slot != 0 && (context.RenderTargetMaskForSlot(slot) == 0 || context.ColorTargets[slot].BaseAddress == 0)) + { + continue; + } + + if (ColorTargetResolver.Resolve(context, slot, DrawLayerOffset, ignoreTargetMask: false, out var resolvedSlot) is { } resolution) + { + mapping[(int)resolvedSlot] = resolution.ExportMapping; + } + } + + inputs.DepthBound = DepthTargetResolver.Resolve(context, formats, fatal) is not null; + inputs.PixelActive = HasActivePixelShader(banks); + return inputs; + } + + // The programs of a draw with these banks and inputs, as ResolveShaderPrograms resolves them. + public GraphicsPrograms ResolveGraphicsPrograms(RegisterBanks banks, in GraphicsProgramInputs inputs) + { + var context = banks.Context; + return _pipelines.GetGraphicsPrograms( + banks.Shader.Vertex, + banks.Shader.Pixel, + context.ShaderInterface, + context, + inputs.Mapping, + inputs.PixelActive, + inputs.DepthBound); + } +} diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs index 5831fae..e1569cc 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs @@ -35,7 +35,8 @@ public sealed partial class RenderExecutor throw _host.Fatal($"The vertex input has too many buffers: count={buffers.Length} max={VertexInputInfo.MaxBuffers}."); } - Span ranges = stackalloc VertexBufferRange[VertexInputInfo.MaxBuffers]; + // Sized to this draw's buffers: a MaxBuffers span is over a kilobyte, zeroed on every draw. + Span ranges = stackalloc VertexBufferRange[buffers.Length]; var rangeCount = 0; foreach (ref readonly var vertex in buffers.AsSpan()) { @@ -54,7 +55,7 @@ public sealed partial class RenderExecutor } ranges[..rangeCount].Sort(static (left, right) => left.BaseAddress.CompareTo(right.BaseAddress)); - Span merged = stackalloc VertexBufferRange[VertexInputInfo.MaxBuffers]; + Span merged = stackalloc VertexBufferRange[buffers.Length]; var mergedCount = 0; for (var i = 0; i < rangeCount; i++) { diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs index 38fa148..2a2afcd 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs @@ -17,6 +17,12 @@ public sealed partial class RenderExecutor private bool TryResolveDrawTargets(RegisterBanks banks, in DrawCall draw, ref DrawState state) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.DrawTargetResolution); + if (Diagnostics.DccWriterTrace.Enabled) + { + Diagnostics.DccWriterTrace.CurrentKind = "draw"; + Diagnostics.DccWriterTrace.CurrentProgram = 0; + } + var context = banks.Context; if (TryResolveMultisampleColor(context)) { diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs index 8558cf4..cc6dd14 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs @@ -667,29 +667,22 @@ public sealed partial class RenderExecutor private void ResolveShaderPrograms(RegisterBanks banks, ref DrawState state) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.DrawProgramResolution); - var context = banks.Context; - Span mappingStorage = stackalloc ColorComponentMapArray[1]; - Span targetExportMapping = mappingStorage[0]; - targetExportMapping.Fill(ColorComponentMap.Identity); + var inputs = GraphicsProgramInputs.Create(); + Span targetExportMapping = inputs.Mapping; foreach (ref readonly var color in BoundColors(ref state)) { targetExportMapping[(int)color.Slot] = color.Resolution.ExportMapping; } - state.Programs = _pipelines.GetGraphicsPrograms( - banks.Shader.Vertex, - banks.Shader.Pixel, - context.ShaderInterface, - context, - targetExportMapping, - state.PixelActive, - state.Depth.HasTarget); - } + inputs.PixelActive = state.PixelActive; + inputs.DepthBound = state.Depth.HasTarget; + if (Prefetch is { } prefetch && prefetch.TryTakeGraphics(banks, in inputs, out var prefetched)) + { + state.Programs = prefetched; + return; + } - [System.Runtime.CompilerServices.InlineArray(RenderingState.ColorAttachmentCapacity)] - private struct ColorComponentMapArray - { - private ColorComponentMap _element0; + state.Programs = ResolveGraphicsPrograms(banks, in inputs); } private static void TraceDrawState(ulong submitId, RegisterBanks banks, in DrawCall draw, in DrawState state) diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs index 30b36d8..0f1bd06 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs @@ -19,7 +19,29 @@ public readonly record struct RenderingAttachment( bool HasDepth, bool DepthClear, bool HasStencil, - bool StencilClear); + bool StencilClear) +{ + // Compared by handle value: the generated comparison goes through the object comparer of ImageView + // and boxes both views, which a draw pays for every attachment it re-checks. + public bool Equals(RenderingAttachment other) => + View.Handle == other.View.Handle && + Layout == other.Layout && + Format == other.Format && + ClearWord0 == other.ClearWord0 && + ClearWord1 == other.ClearWord1 && + ClearWord2 == other.ClearWord2 && + ClearWord3 == other.ClearWord3 && + IsClear == other.IsClear && + HasDepth == other.HasDepth && + DepthClear == other.DepthClear && + HasStencil == other.HasStencil && + StencilClear == other.StencilClear; + + public override int GetHashCode() => HashCode.Combine( + View.Handle, + HashCode.Combine(Layout, Format, ClearWord0, ClearWord1, ClearWord2, ClearWord3), + HashCode.Combine(IsClear, HasDepth, DepthClear, HasStencil, StencilClear)); +} [InlineArray(RenderingState.ColorAttachmentCapacity)] public struct RenderingColorAttachments diff --git a/src/SharpEmu.Libs/Gpu/Scheduling/GpuWritebackEpoch.cs b/src/SharpEmu.Libs/Gpu/Scheduling/GpuWritebackEpoch.cs new file mode 100644 index 0000000..5d329cd --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Scheduling/GpuWritebackEpoch.cs @@ -0,0 +1,16 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Gpu.Scheduling; + +// Counts the stores of GPU data into guest memory: buffer and image downloads, and buffer transfers +// the caches complete there. Work done ahead of the render thread from guest memory is still +// current while the count has not moved. +internal static class GpuWritebackEpoch +{ + private static long _value; + + public static long Value => Volatile.Read(ref _value); + + public static void Advance() => Interlocked.Increment(ref _value); +} diff --git a/src/SharpEmu.Libs/Gpu/Scheduling/RecordingBuffer.cs b/src/SharpEmu.Libs/Gpu/Scheduling/RecordingBuffer.cs index ea228a5..6f791f2 100644 --- a/src/SharpEmu.Libs/Gpu/Scheduling/RecordingBuffer.cs +++ b/src/SharpEmu.Libs/Gpu/Scheduling/RecordingBuffer.cs @@ -1,8 +1,23 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using Silk.NET.Vulkan; + namespace SharpEmu.Libs.Gpu.Scheduling; +public readonly record struct MemoryBarrierRequest( + PipelineStageFlags SourceStages, + PipelineStageFlags DestinationStages, + AccessFlags SourceAccess, + AccessFlags DestinationAccess) +{ + public MemoryBarrierRequest Union(in MemoryBarrierRequest other) => new( + SourceStages | other.SourceStages, + DestinationStages | other.DestinationStages, + SourceAccess | other.SourceAccess, + DestinationAccess | other.DestinationAccess); +} + public enum RecordedOperation : uint { DispatchDirect, @@ -23,6 +38,8 @@ public sealed class RecordingBuffer private readonly IRenderingState _rendering; private SubmissionContext? _context; private bool _handleWasRequested; + private MemoryBarrierRequest _pendingBarrier; + private bool _hasPendingBarrier; internal nint Buffer; @@ -51,7 +68,11 @@ public sealed class RecordingBuffer public bool IsInvalid => Buffer == 0; - internal bool HasPendingCommands => _handleWasRequested || _rendering.IsRendering; + internal bool HasPendingCommands => _handleWasRequested || _hasPendingBarrier || _rendering.IsRendering; + + public long MergedBarriers { get; private set; } + + public long RecordedBarriers { get; private set; } // The native handle stays valid only until the next submission. public nint Handle @@ -64,11 +85,68 @@ public sealed class RecordingBuffer } // A caller can record native commands after it obtains the handle. + FlushPendingBarrier(); _handleWasRequested = true; return Buffer; } } + public nint StateHandle + { + get + { + if (IsInvalid) + { + throw SubmissionScheduler.Fatal("The command buffer is not recording."); + } + + _handleWasRequested = true; + return Buffer; + } + } + + public void RequestMemoryBarrier(in MemoryBarrierRequest request) + { + if (IsInvalid || _rendering.IsRendering) + { + throw SubmissionScheduler.Fatal("A memory barrier needs a recording command buffer outside rendering."); + } + + if (_hasPendingBarrier) + { + _pendingBarrier = _pendingBarrier.Union(request); + MergedBarriers++; + if (Vulkan.GpuSyncProfile.Enabled) + { + Vulkan.GpuSyncProfile.CountMergedRequest(); + } + + return; + } + + _pendingBarrier = request; + _hasPendingBarrier = true; + } + + private void FlushPendingBarrier() + { + if (!_hasPendingBarrier) + { + return; + } + + _hasPendingBarrier = false; + if (_rendering.IsRendering) + { + throw SubmissionScheduler.Fatal("A pending memory barrier reached a rendering scope."); + } + + var recorder = Owner.MemoryBarrierRecorder ?? + throw SubmissionScheduler.Fatal("A memory barrier was requested without a recorder."); + recorder(Buffer, _pendingBarrier); + RecordedBarriers++; + } + public SubmissionContext Context => _context ?? throw SubmissionScheduler.Fatal("The command buffer has no submission context."); public void SetDebugInfo(uint op, ulong submitId, uint arg0 = 0, uint arg1 = 0, uint arg2 = 0, uint arg3 = 0, ulong arg4 = 0) @@ -101,6 +179,7 @@ public sealed class RecordingBuffer _device.BeginBuffer(Buffer); _handleWasRequested = false; + _hasPendingBarrier = false; SetDebugInfo((uint)RecordedOperation.Unknown, 0); } diff --git a/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs b/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs index 7999a4d..2a5b2f9 100644 --- a/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs +++ b/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs @@ -84,13 +84,27 @@ public sealed class SubmissionScheduler : IGpuTickScheduler, IDisposable public ulong CurrentTick => _timeline.CurrentTick; + public Action? MemoryBarrierRecorder { get; set; } + public TickTimeline Timeline => _timeline; internal bool PriorityWorkerAlive => _priorityThread.IsAlive; + // Set on a thread that does work ahead of the render thread: there a fatal condition only + // abandons that work, which the render thread then does itself and fails on if it is real. + [ThreadStatic] + internal static bool ThrowOnFatal; + + public sealed class AbandonedWorkException(string message) : Exception(message); + // Report the fatal error, then return an exception to stop the caller. internal static Exception Fatal(string message) { + if (ThrowOnFatal) + { + throw new AbandonedWorkException(message); + } + Console.Error.WriteLine($"[GPU][FATAL] {message}"); OnFatal(message); return new InvalidOperationException(message); diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/GpuSyncProfile.cs b/src/SharpEmu.Libs/Gpu/Vulkan/GpuSyncProfile.cs new file mode 100644 index 0000000..d5fbd62 --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Vulkan/GpuSyncProfile.cs @@ -0,0 +1,79 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using Silk.NET.Vulkan; + +namespace SharpEmu.Libs.Gpu.Vulkan; + +internal enum NativeStateKind +{ + Pipeline, + PushConstants, + Descriptors, + VertexBuffers, + IndexBuffer, + Count, +} + +internal static class GpuSyncProfile +{ + private static readonly long[] _nativeStateRecorded = new long[(int)NativeStateKind.Count]; + private static readonly long[] _nativeStateReused = new long[(int)NativeStateKind.Count]; + private static long _barriers; + private static long _drainingBarriers; + private static long _memoryBarriers; + private static long _bufferBarriers; + private static long _imageBarriers; + private static long _mergedRequests; + private static long _renderingScopes; + private static long _batchFlushes; + + public static bool Enabled => VideoOut.RenderPhaseProfile.Enabled; + + public static void CountBarrier(PipelineStageFlags source, PipelineStageFlags destination, uint memory, uint buffer, uint image) + { + Interlocked.Increment(ref _barriers); + if ((source & PipelineStageFlags.AllCommandsBit) != 0 && (destination & (PipelineStageFlags.AllCommandsBit | PipelineStageFlags.TopOfPipeBit | PipelineStageFlags.DrawIndirectBit)) != 0) + { + Interlocked.Increment(ref _drainingBarriers); + } + + Interlocked.Add(ref _memoryBarriers, memory); + Interlocked.Add(ref _bufferBarriers, buffer); + Interlocked.Add(ref _imageBarriers, image); + } + + public static void CountMergedRequest() => Interlocked.Increment(ref _mergedRequests); + + public static void CountRenderingScope() => Interlocked.Increment(ref _renderingScopes); + + public static void CountBatchFlush() => Interlocked.Increment(ref _batchFlushes); + + public static void CountNativeState(NativeStateKind kind, bool reused) + { + if (reused) + { + Interlocked.Increment(ref _nativeStateReused[(int)kind]); + } + else + { + Interlocked.Increment(ref _nativeStateRecorded[(int)kind]); + } + } + + private static string TakeNativeStateReport() + { + var parts = new string[(int)NativeStateKind.Count]; + for (var index = 0; index < parts.Length; index++) + { + var reused = Interlocked.Exchange(ref _nativeStateReused[index], 0); + var recorded = Interlocked.Exchange(ref _nativeStateRecorded[index], 0); + parts[index] = FormattableString.Invariant($"{(NativeStateKind)index}={reused}/{reused + recorded}"); + } + + return string.Join(" ", parts); + } + + public static string TakeReport() => FormattableString.Invariant( + $"[PERF][GPU_SYNC] barriers={Interlocked.Exchange(ref _barriers, 0)} draining={Interlocked.Exchange(ref _drainingBarriers, 0)} memory={Interlocked.Exchange(ref _memoryBarriers, 0)} buffer={Interlocked.Exchange(ref _bufferBarriers, 0)} image={Interlocked.Exchange(ref _imageBarriers, 0)} merged_requests={Interlocked.Exchange(ref _mergedRequests, 0)} rendering_scopes={Interlocked.Exchange(ref _renderingScopes, 0)} draw_count_flushes={Interlocked.Exchange(ref _batchFlushes, 0)} reused {TakeNativeStateReport()}"); +} diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/GraphicsDynamicStateEpoch.cs b/src/SharpEmu.Libs/Gpu/Vulkan/GraphicsDynamicStateEpoch.cs new file mode 100644 index 0000000..9df63b5 --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Vulkan/GraphicsDynamicStateEpoch.cs @@ -0,0 +1,16 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Gpu.Vulkan; + +// Advanced by every graphics pipeline bind or dynamic state change outside the guest draw path: +// such a pipeline may fix the state the guest pipelines leave dynamic, so the dynamic state a +// guest draw last set no longer holds in the command buffer. +internal static class GraphicsDynamicStateEpoch +{ + private static long _value; + + public static long Value => Volatile.Read(ref _value); + + public static void Advance() => Interlocked.Increment(ref _value); +} diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/NativeBindingEpoch.cs b/src/SharpEmu.Libs/Gpu/Vulkan/NativeBindingEpoch.cs new file mode 100644 index 0000000..91be64c --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Vulkan/NativeBindingEpoch.cs @@ -0,0 +1,13 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Gpu.Vulkan; + +internal static class NativeBindingEpoch +{ + private static long _value; + + public static long Value => Volatile.Read(ref _value); + + public static void Advance() => Interlocked.Increment(ref _value); +} diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanAsyncReadback.cs b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanAsyncReadback.cs index 42d8420..d771e3f 100644 --- a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanAsyncReadback.cs +++ b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanAsyncReadback.cs @@ -172,10 +172,13 @@ internal sealed unsafe class VulkanAsyncReadback : IDisposable PSemaphores = &semaphore, PValues = &signalValue, }; + var waitStart = VideoOut.GpuWaitSiteProfile.Begin(); using (VideoOut.RenderPhaseProfile.MeasureDetail(VideoOut.RenderPhaseProfile.Phase.GpuCompletionWait)) { Require(vk.WaitSemaphores(_device.Device, &waitInfo, ulong.MaxValue), "vkWaitSemaphores(readback)"); } + + VideoOut.GpuWaitSiteProfile.End(waitStart); } finally { diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanSynchronization.cs b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanSynchronization.cs index d275742..93cecea 100644 --- a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanSynchronization.cs +++ b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanSynchronization.cs @@ -22,6 +22,11 @@ internal static unsafe class VulkanSynchronization uint imageMemoryBarrierCount, ImageMemoryBarrier2* imageMemoryBarriers) { + if (GpuSyncProfile.Enabled) + { + GpuSyncProfile.CountBarrier(sourceStages, destinationStages, memoryBarrierCount, bufferMemoryBarrierCount, imageMemoryBarrierCount); + } + var sourceStages2 = (PipelineStageFlags2)(ulong)sourceStages; var destinationStages2 = (PipelineStageFlags2)(ulong)destinationStages; for (var index = 0u; index < memoryBarrierCount; index++) diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs index db3e26f..cae352a 100644 --- a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs +++ b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs @@ -220,10 +220,13 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice PValues = &tick, }; Result result; + var waitStart = GpuWaitSiteProfile.Begin(); using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.GpuCompletionWait)) { result = _vk.WaitSemaphores(_device, &waitInfo, ulong.MaxValue); } + + GpuWaitSiteProfile.End(waitStart); failure = result.ToString(); return result == Result.Success && WaitRetired(tick); } diff --git a/src/SharpEmu.Libs/VideoOut/GpuWaitSiteProfile.cs b/src/SharpEmu.Libs/VideoOut/GpuWaitSiteProfile.cs new file mode 100644 index 0000000..d86cc75 --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/GpuWaitSiteProfile.cs @@ -0,0 +1,96 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; + +namespace SharpEmu.Libs.VideoOut; + +internal static class GpuWaitSiteProfile +{ + private const int SiteFrames = 5; + + private static readonly object _gate = new(); + private static readonly Dictionary _sites = new(StringComparer.Ordinal); + + private static readonly HashSet PlumbingTypes = new(StringComparer.Ordinal) + { + "GpuWaitSiteProfile", + "TickTimeline", + "SubmissionScheduler", + "VulkanTickDevice", + "VulkanAsyncReadback", + "GpuWorkerRelay", + "GpuWaitSpin", + }; + + public static long Begin() => RenderPhaseProfile.DetailMeasurementsEnabled ? Stopwatch.GetTimestamp() : 0; + + public static void End(long start) + { + if (start == 0) + { + return; + } + + var elapsed = Stopwatch.GetTimestamp() - start; + var site = Site(); + lock (_gate) + { + _sites.TryGetValue(site, out var entry); + _sites[site] = (entry.Count + 1, entry.Ticks + elapsed); + } + } + + private static string Site() + { + var names = new List(SiteFrames); + foreach (var frame in new StackTrace(2, false).GetFrames()) + { + if (frame.GetMethod() is not { } method) + { + continue; + } + + var type = method.DeclaringType; + while (type is { DeclaringType: not null } && type.Name.StartsWith('<')) + { + type = type.DeclaringType; + } + + var typeName = type?.Name ?? "?"; + if (PlumbingTypes.Contains(typeName) || method.Name.StartsWith('<')) + { + continue; + } + + names.Add($"{typeName}.{method.Name}"); + if (names.Count == SiteFrames) + { + break; + } + } + + return string.Join(" < ", names); + } + + public static void Report() + { + List> ranked; + lock (_gate) + { + if (_sites.Count == 0) + { + return; + } + + ranked = _sites.OrderByDescending(pair => pair.Value.Ticks).Take(12).ToList(); + _sites.Clear(); + } + + foreach (var (site, (count, ticks)) in ranked) + { + Console.Error.WriteLine( + $"[PERF][GPU_WAIT_SITE] ms={ticks * 1000.0 / Stopwatch.Frequency:F1} n={count} site={site}"); + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs b/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs index 8821b5d..a1f6c38 100644 --- a/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs +++ b/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs @@ -465,11 +465,14 @@ internal static class RenderPhaseProfile $"[PERF][RENDER_MS] window_s={seconds:F1} frames={frames} " + string.Join(" ", parts.Select(part => $"{part.Phase}={part.Milliseconds:F2}ms/n{part.Entries}"))); + GpuWaitSiteProfile.Report(); ReportImageUploads(); BufferUploadProfile.Report(); Console.Error.WriteLine(SharpEmu.ShaderCompiler.Resources.ResourceMaterializationCache.TakeReport()); Console.Error.WriteLine(SharpEmu.Libs.Gpu.Buffers.GuestBufferCache.TakeAsyncReadbackReport()); + Console.Error.WriteLine(SharpEmu.Libs.Gpu.Vulkan.GpuSyncProfile.TakeReport()); Console.Error.WriteLine(SharpEmu.Libs.Gpu.Images.GuestImageCache.TakeLookupReport()); + Console.Error.WriteLine(SharpEmu.Libs.Gpu.Images.GuestImageCache.TakeDccReport()); SharpEmu.ShaderCompiler.Resources.ResourceMaterializationProfile.WriteReport(); SharpEmu.Libs.Diagnostics.AgcRegisterPacketProfile.WriteReport(); SharpEmu.HLE.GpuMemory.GpuMemoryAccessProfile.WriteReport(); diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.BindingShadow.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.BindingShadow.cs new file mode 100644 index 0000000..022640c --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.BindingShadow.cs @@ -0,0 +1,249 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Vulkan; +using SharpEmu.ShaderCompiler.Resources; +using Silk.NET.Vulkan; +using VkBuffer = Silk.NET.Vulkan.Buffer; + +namespace SharpEmu.Libs.VideoOut; + +internal static unsafe partial class VulkanVideoPresenter +{ + private static readonly bool NativeStateReuse = + Environment.GetEnvironmentVariable("SHARPEMU_NATIVE_STATE_REUSE") != "0"; + + private sealed partial class Presenter + { + private const int ShadowVertexBufferCapacity = 32; + + private nint _shadowCommand; + private ulong _shadowTick; + private long _shadowEpoch = -1; + private Pipeline _shadowGraphicsPipeline; + private Pipeline _shadowComputePipeline; + private PipelineLayout _shadowPushLayout; + private ShaderStageFlags _shadowPushStages; + private uint[]? _shadowPushData; + private DescriptorShadow? _graphicsDescriptorShadow; + private DescriptorShadow? _computeDescriptorShadow; + private ulong[]? _shadowVertexBindings; + private int _shadowVertexCount = -1; + private VkBuffer _shadowIndexBuffer; + private ulong _shadowIndexOffset; + private IndexType _shadowIndexType; + private bool _shadowIndexValid; + private ulong[]? _descriptorKeyScratch; + + private sealed class DescriptorShadow + { + public bool Valid; + public PipelineLayout Layout; + public ulong[] Key = new ulong[256]; + public int Length; + } + + private bool SyncNativeShadow(CommandBuffer command) + { + if (!NativeStateReuse) + { + return false; + } + + var tick = _scheduler.CurrentTick; + var epoch = NativeBindingEpoch.Value; + if (command.Handle == _shadowCommand && tick == _shadowTick && epoch == _shadowEpoch) + { + return true; + } + + _shadowCommand = command.Handle; + _shadowTick = tick; + _shadowEpoch = epoch; + _shadowGraphicsPipeline = default; + _shadowComputePipeline = default; + _shadowPushLayout = default; + if (_graphicsDescriptorShadow is not null) + { + _graphicsDescriptorShadow.Valid = false; + } + + if (_computeDescriptorShadow is not null) + { + _computeDescriptorShadow.Valid = false; + } + + _shadowVertexCount = -1; + _shadowIndexValid = false; + return true; + } + + private void BindPipelineShadowed(CommandBuffer command, PipelineBindPoint bindPoint, Pipeline pipeline) + { + if (SyncNativeShadow(command)) + { + ref var bound = ref bindPoint == PipelineBindPoint.Compute ? ref _shadowComputePipeline : ref _shadowGraphicsPipeline; + var reused = bound.Handle == pipeline.Handle; + CountNativeState(NativeStateKind.Pipeline, reused); + if (reused) + { + return; + } + + bound = pipeline; + } + + _vk.CmdBindPipeline(command, bindPoint, pipeline); + } + + private void PushConstantsShadowed(CommandBuffer command, PipelineLayout layout, ShaderStageFlags stages, uint* data) + { + if (SyncNativeShadow(command)) + { + var values = new ReadOnlySpan(data, (int)PushData.DwordCount); + var shadowData = _shadowPushData ??= new uint[PushData.DwordCount]; + var reused = _shadowPushLayout.Handle == layout.Handle && _shadowPushStages == stages && values.SequenceEqual(shadowData); + CountNativeState(NativeStateKind.PushConstants, reused); + if (reused) + { + return; + } + + _shadowPushLayout = layout; + _shadowPushStages = stages; + values.CopyTo(shadowData); + } + + _vk.CmdPushConstants(command, layout, stages, 0, PushData.ByteSize, data); + } + + private bool DescriptorsAlreadyBound(CommandBuffer command, PipelineBindPoint bindPoint, PipelineLayout layout, ReadOnlySpan writes) + { + if (!SyncNativeShadow(command)) + { + return false; + } + + var length = DescriptorKey(writes); + var shadow = bindPoint == PipelineBindPoint.Compute + ? _computeDescriptorShadow ??= new DescriptorShadow() + : _graphicsDescriptorShadow ??= new DescriptorShadow(); + var key = _descriptorKeyScratch.AsSpan(0, length); + var reused = shadow.Valid && shadow.Layout.Handle == layout.Handle && key.SequenceEqual(shadow.Key.AsSpan(0, shadow.Length)); + CountNativeState(NativeStateKind.Descriptors, reused); + if (reused) + { + return true; + } + + if (shadow.Key.Length < length) + { + shadow.Key = new ulong[Math.Max(length, shadow.Key.Length * 2)]; + } + + key.CopyTo(shadow.Key); + shadow.Length = length; + shadow.Layout = layout; + shadow.Valid = true; + return false; + } + + private int DescriptorKey(ReadOnlySpan writes) + { + var length = 0; + _descriptorKeyScratch ??= new ulong[256]; + foreach (ref readonly var write in writes) + { + var needed = length + 2 + (int)write.DescriptorCount * 3; + if (_descriptorKeyScratch.Length < needed) + { + Array.Resize(ref _descriptorKeyScratch, Math.Max(needed, _descriptorKeyScratch.Length * 2)); + } + + var key = _descriptorKeyScratch; + key[length++] = write.DstBinding | ((ulong)write.DescriptorType << 32); + key[length++] = write.DescriptorCount | (write.PBufferInfo != null ? 1UL << 32 : 0) | (write.PImageInfo != null ? 1UL << 33 : 0); + for (var index = 0; index < write.DescriptorCount; index++) + { + if (write.PBufferInfo != null) + { + var info = write.PBufferInfo[index]; + key[length++] = info.Buffer.Handle; + key[length++] = info.Offset; + key[length++] = info.Range; + } + else if (write.PImageInfo != null) + { + var info = write.PImageInfo[index]; + key[length++] = info.Sampler.Handle; + key[length++] = info.ImageView.Handle; + key[length++] = (ulong)info.ImageLayout; + } + } + } + + return length; + } + + private bool VertexBuffersAlreadyBound(CommandBuffer command, ReadOnlySpan bindings) + { + if (!SyncNativeShadow(command) || bindings.Length > ShadowVertexBufferCapacity) + { + _shadowVertexCount = -1; + return false; + } + + var shadow = _shadowVertexBindings ??= new ulong[ShadowVertexBufferCapacity * 2]; + var reused = _shadowVertexCount == bindings.Length; + for (var index = 0; reused && index < bindings.Length; index++) + { + reused = shadow[index * 2] == bindings[index].Handle && shadow[index * 2 + 1] == bindings[index].Offset; + } + + CountNativeState(NativeStateKind.VertexBuffers, reused); + if (reused) + { + return true; + } + + for (var index = 0; index < bindings.Length; index++) + { + shadow[index * 2] = bindings[index].Handle; + shadow[index * 2 + 1] = bindings[index].Offset; + } + + _shadowVertexCount = bindings.Length; + return false; + } + + private bool IndexBufferAlreadyBound(CommandBuffer command, VkBuffer buffer, ulong offset, IndexType type) + { + if (!SyncNativeShadow(command)) + { + return false; + } + + var reused = _shadowIndexValid && _shadowIndexBuffer.Handle == buffer.Handle && _shadowIndexOffset == offset && _shadowIndexType == type; + CountNativeState(NativeStateKind.IndexBuffer, reused); + if (reused) + { + return true; + } + + _shadowIndexBuffer = buffer; + _shadowIndexOffset = offset; + _shadowIndexType = type; + _shadowIndexValid = true; + return false; + } + + private static void CountNativeState(NativeStateKind kind, bool reused) + { + if (GpuSyncProfile.Enabled) + { + GpuSyncProfile.CountNativeState(kind, reused); + } + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs index 8d57211..18d37c3 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs @@ -130,6 +130,37 @@ internal static unsafe partial class VulkanVideoPresenter // A draw-local copy used when the guest samples a depth aspect while the // same image is attached for writes. It retires with the submission. public bool IsFeedbackSnapshot; + // Taken from the presenter's pool; goes back to it when its submission retires. + public bool Pooled; + public bool InPool; + + // Back to the state of a new object, so a reused one never carries a previous binding. + public void Reset() + { + Address = 0; + ImageIdentifier = default; + Request = default; + CachedImage = null; + MipLevel = 0; + Image = default; + View = default; + Layout = ImageLayout.ShaderReadOnlyOptimal; + Sampler = default; + SamplerState = default; + IsStorage = false; + IsHostMovie = false; + HostMoviePlane = -1; + HostMovieFrameSerial = 0; + NeedsUpload = false; + StagingBuffer = default; + StagingMemory = default; + Width = 0; + Height = 0; + RowLength = 0; + DestinationSelect = 0; + MipViews = []; + IsFeedbackSnapshot = false; + } } private sealed partial class Presenter @@ -1017,6 +1048,8 @@ internal static unsafe partial class VulkanVideoPresenter continue; } + _commandBuffer = BeginBatchedGuestCommands(); + var initialized = texture.HostMoviePlane == 0 ? _hostMovieImageInitialized : _hostMovieChromaImageInitialized; var toTransfer = new ImageMemoryBarrier2 { diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs index 36e741d..77b714d 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs @@ -176,9 +176,6 @@ internal static unsafe partial class VulkanVideoPresenter private EndOfPipe _endOfPipe = null!; private AgcExports.CommandStreamTranslation _translation = null!; private ICpuMemory _guestMemory = null!; - private long _lastCommandStreamProgressTicks; - private static readonly long _blockedRetryTicks = - CommandStreamQueue.AllBlockedRetryMilliseconds * System.Diagnostics.Stopwatch.Frequency / 1000L; public CommandStreamQueue CommandStream => _commandStream; @@ -188,7 +185,6 @@ internal static unsafe partial class VulkanVideoPresenter _guestMemory = guest; _endOfPipe = new EndOfPipe(_endOfPipeEvents, this); _translation = AgcExports.CreateCommandStreamTranslation(guest, this); - _lastCommandStreamProgressTicks = System.Diagnostics.Stopwatch.GetTimestamp(); } public void EnqueueCommandStream(uint queue, ulong address, uint dwordCount, ulong submissionId, object? geometrySnapshots) @@ -204,25 +200,42 @@ internal static unsafe partial class VulkanVideoPresenter } } - // Runs slices until the budget ends or nothing is runnable; blocked heads retry every 100 ms. + // The command stream thread interprets the submissions; this thread runs what it hands over + // until the budget ends or nothing is queued. Null until the first render tick. + private CommandThread? _commandThread; + private Action? _betweenCommandItems; + + private bool OnCommandThread => _commandThread is { IsProducerThread: true }; + + // Items run in batches; completions are collected between them so blocked waits retry. + private const int CommandItemBatch = 256; + private void RunCommandStreamSlices(long renderWorkDeadline) { - var slices = 0; - while (slices < _maxGuestWorkPerRender) + if (_commandThread is null) + { + _commandThread = new CommandThread(_commandStream, _guestMemory, RunDeferredWork, _translation.Prefetch); + _betweenCommandItems = RunRelayBetweenCommandItems; + _translation.Sink = _commandThread; + _commandThread.Start(); + } + + var thread = _commandThread; + for (;;) { using (RenderPhaseProfile.Measure(RenderPhaseProfile.Phase.Collect)) { CollectCompletedGuestSubmissions(waitForOldest: false); } - SliceResult result; + int ran; using (RenderPhaseProfile.Measure(RenderPhaseProfile.Phase.CommandStream)) { var aliasAccess = _guestBacking?.TryEnterBackingAliasAccess() == true; _backingAliasAccess = aliasAccess; try { - result = _commandStream.ProcessOne(); + ran = thread.RunBatch(renderWorkDeadline, CommandItemBatch, _betweenCommandItems!); } finally { @@ -234,64 +247,162 @@ internal static unsafe partial class VulkanVideoPresenter } } - switch (result) - { - case SliceResult.NoWork: - return; - case SliceResult.AllBlocked: - RetryBlockedCommandStreamIfDue(); - return; - case SliceResult.BlockedWithoutProgress: - if (!_commandStream.HasUnblockedPending) - { - RetryBlockedCommandStreamIfDue(); - return; - } - - break; - default: - _lastCommandStreamProgressTicks = System.Diagnostics.Stopwatch.GetTimestamp(); - break; - } - - slices++; - if (System.Diagnostics.Stopwatch.GetTimestamp() >= renderWorkDeadline) + if (ran == 0 || System.Diagnostics.Stopwatch.GetTimestamp() >= renderWorkDeadline) { return; } } } - private void RetryBlockedCommandStreamIfDue() + private void RunDeferredWork(AgcExports.DeferredWork work) { - var now = System.Diagnostics.Stopwatch.GetTimestamp(); - if (now - _lastCommandStreamProgressTicks < _blockedRetryTicks) + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + try { - return; + _translation.RunDeferred(work); + } + finally + { + Interlocked.Add(ref _perfDrawTicks, System.Diagnostics.Stopwatch.GetTimestamp() - started); } - - _commandStream.RetryBlocked(); - _lastCommandStreamProgressTicks = now; } - // Milliseconds until the blocked heads are due for a retry; null when none is blocked. - private int? BlockedRetryWaitMilliseconds() + // Commands other threads post to the GPU worker run between items, with the resolution + // worker held: they change the caches it reads. + private void RunRelayBetweenCommandItems() { - if (!_vulkanReady || !_commandStream.HasPending || _commandStream.HasUnblockedPending) + if (_relay.HasPendingCommands) { - return null; + RunRelayCommandsWithPrefetchHeld(); } - - var remaining = _blockedRetryTicks - (System.Diagnostics.Stopwatch.GetTimestamp() - _lastCommandStreamProgressTicks); - return remaining <= 0 ? 0 : (int)Math.Max(1, remaining * 1000 / System.Diagnostics.Stopwatch.Frequency); } - ICpuMemory ICommandStreamHost.Memory => _guestMemory; + private void RunRelayCommandsWithPrefetchHeld() + { + var prefetch = _commandThread is not null ? _translation.Prefetch : null; + prefetch?.Pause(); + try + { + using var relayScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay); + _relay.RunPendingCommands(); + } + finally + { + prefetch?.Resume(); + } + } + + // The interpreter's own stores (labels, atomics, semaphores) follow every deferred draw. + private DrainingWriteMemory? _commandMemory; + + ICpuMemory ICommandStreamHost.Memory => _commandMemory ??= new DrainingWriteMemory(_guestMemory, DrainDraws, this); + + // On the command stream thread a store is queued behind the work before it and reads see + // the queued stores; elsewhere the deferred draws run first. + private sealed class DrainingWriteMemory(ICpuMemory inner, Action drain, Presenter presenter) : ICpuMemory, ICpuMemoryWrapper + { + public ICpuMemory Inner => inner; + + public bool TryRead(ulong virtualAddress, Span destination) => + presenter.OnCommandThread + ? presenter._commandThread!.ReadOverlay(virtualAddress, destination) + : inner.TryRead(virtualAddress, destination); + + public bool TryWrite(ulong virtualAddress, ReadOnlySpan source) + { + if (presenter.OnCommandThread) + { + presenter._commandThread!.PostWrite(virtualAddress, source); + return true; + } + + drain(); + return inner.TryWrite(virtualAddress, source); + } + + public bool TryCompare(ulong virtualAddress, ReadOnlySpan expected, out bool equal) + { + if (presenter.OnCommandThread) + { + var bytes = expected.ToArray(); + var compared = false; + var result = presenter._commandThread!.Run(() => TryCompare(virtualAddress, bytes, out compared)); + equal = compared; + return result; + } + + drain(); + return inner.TryCompare(virtualAddress, expected, out equal); + } + + public bool TryCopy(ulong destinationAddress, ulong sourceAddress, ulong length) + { + if (presenter.OnCommandThread) + { + return presenter._commandThread!.Run(() => TryCopy(destinationAddress, sourceAddress, length)); + } + + drain(); + return inner.TryCopy(destinationAddress, sourceAddress, length); + } + + public bool TryScanCString(ulong address, byte needle, bool findLast, ulong maxLength, out ulong match) => + inner.TryScanCString(address, needle, findLast, maxLength, out match); + + public bool CanRead(ulong address, ulong size) => inner.CanRead(address, size); + + public string DescribeReadRange(ulong address, ulong size) => inner.DescribeReadRange(address, size); + } + + // A read the render thread runs after everything queued before it, for the command thread. + private bool ReadOnRenderThread(ulong address, Span destination, bool operand) + { + var buffer = new byte[destination.Length]; + var read = _commandThread!.Run(() => operand ? TryReadGuestOperand(address, buffer) : TryReadGuest(address, buffer)); + buffer.CopyTo(destination); + return read; + } + + // Command memory and polled labels: GPU-written pages go through the render thread, the + // rest is read here with the stream's queued stores applied. + private bool ReadOnCommandThread(ulong address, Span destination) + { + _commandThread!.WaitForPendingRange(address, (ulong)destination.Length); + return _bufferCache.MayHaveGpuDirtyPages(address, (ulong)destination.Length) + ? ReadOnRenderThread(address, destination, operand: false) + : _commandThread!.ReadOverlay(address, destination); + } + + public bool TryReadGuestWaitOperand(ulong address, Span destination) => + OnCommandThread ? ReadOnCommandThread(address, destination) : TryReadGuestOperand(address, destination); + + public bool TryReadGuestOperand(ulong address, Span destination) + { + if (OnCommandThread) + { + return ReadOnRenderThread(address, destination, operand: true); + } + + DrainDraws(); + return TryReadGuest(address, destination); + } public bool TryReadGuest(ulong address, Span destination) { + if (OnCommandThread) + { + return ReadOnCommandThread(address, destination); + } + using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandMemorySync)) { + // GPU-written command memory is downloaded first: deferred draws are recorded before it. + if (_translation is { HasPendingDraws: true } && _bufferCache.MayHaveGpuDirtyPages(address, (ulong)destination.Length)) + { + DrainDraws(); + } + if (!_bufferCache.TrySynchronizeCpuRead(address, (ulong)destination.Length, SharpEmu.HLE.GuestMemory.GuestMemoryProfile.ReadbackSource.CommandMemoryRead)) { @@ -305,27 +416,79 @@ internal static unsafe partial class VulkanVideoPresenter public void RunPendingCommands() { + // With the stream on its own thread the render thread runs them between its items. + if (OnCommandThread) + { + return; + } + + if (_relay.HasPendingCommands) + { + DrainDraws(); + } + using var relayScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay); _relay.RunPendingCommands(); } public void BeginSubmission(int queueId, ulong submissionId, object? geometrySnapshots) + { + if (OnCommandThread) + { + // The translation state belongs to this thread; the queue context to the render thread. + _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId)); + _commandThread!.Post(() => BeginSubmissionContext(queueId, submissionId)); + return; + } + + DrainDraws(); + BeginSubmissionContext(queueId, submissionId); + _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId)); + } + + private void BeginSubmissionContext(int queueId, ulong submissionId) { using var contextScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueContext); _activeGuestQueue = new VulkanGuestQueueIdentity(_commandQueueNames[queueId], submissionId); BindSubmissionContext(_activeGuestQueue); _ = CurrentRecordingBuffer(); - _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId)); } public void Flush() { + if (OnCommandThread) + { + _commandThread!.Post(Flush); + return; + } + + DrainDraws(); using var flushScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.Flush); FlushBatchedGuestCommands(); } + // The guest reuses the frame's memory (descriptor tables, rings) once the boundary returns, + // so the draws queued before it are recorded first. + public void FlushFrame() + { + if (OnCommandThread) + { + _commandThread!.Run(Flush); + return; + } + + Flush(); + } + public void FlushAndWait() { + if (OnCommandThread) + { + _commandThread!.Run(FlushAndWait); + return; + } + + DrainDraws(); using var waitScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandGpuWait); _ = CurrentRecordingBuffer(); _scheduler.FlushAndWait(); @@ -334,16 +497,40 @@ internal static unsafe partial class VulkanVideoPresenter public void SynchronizeGpu() { + if (OnCommandThread) + { + _commandThread!.Run(SynchronizeGpu); + return; + } + + DrainDraws(); using var waitScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandGpuWait); _ = CurrentRecordingBuffer(); _scheduler.Finish(); CollectCompletedGuestSubmissions(waitForOldest: false); } - public void RunGarbageCollector() => RunGuestCacheCollection(endsFrame: false); + public void RunGarbageCollector() + { + if (OnCommandThread) + { + _commandThread!.Post(RunGarbageCollector); + return; + } + + DrainDraws(); + RunGuestCacheCollection(endsFrame: false); + } public void EmitGlobalBarrier() { + if (OnCommandThread) + { + _commandThread!.Post(EmitGlobalBarrier); + return; + } + + DrainDraws(); // Inside a rendering scope whose draws only wrote attachments, the barrier matters only // to what runs after the scope: everything before the scope is ordered by the barrier // BeginRendering records, and a draw that stores to memory ends a deferred scope first. @@ -354,6 +541,12 @@ internal static unsafe partial class VulkanVideoPresenter } EndRendering(); + if (BatchBarriers) + { + RequestGuestMemoryBarrier(GlobalMemoryBarrier); + return; + } + RecordGlobalBarrier(BeginBatchedGuestCommands()); } @@ -380,6 +573,13 @@ internal static unsafe partial class VulkanVideoPresenter public void FillBuffer(ulong address, ulong size, uint value, bool isGds) { + if (OnCommandThread) + { + _commandThread!.PostRange(() => FillBuffer(address, size, value, isGds), address, isGds ? 0 : size); + return; + } + + DrainDraws(); using var transferScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandMemoryTransfer); // Transfers the buffer cache completes in guest memory leave the rendering scope open; // its GPU paths end the scope before they record. @@ -389,6 +589,13 @@ internal static unsafe partial class VulkanVideoPresenter public void CopyBuffer(ulong destination, ulong source, ulong size, bool destinationIsGds, bool sourceIsGds) { + if (OnCommandThread) + { + _commandThread!.PostRange(() => CopyBuffer(destination, source, size, destinationIsGds, sourceIsGds), destination, destinationIsGds ? 0 : size); + return; + } + + DrainDraws(); using var transferScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandMemoryTransfer); // Transfers the buffer cache completes in guest memory leave the rendering scope open; // its GPU paths end the scope before they record. @@ -396,11 +603,30 @@ internal static unsafe partial class VulkanVideoPresenter _bufferCache.CopyBuffer(destination, source, size, destinationIsGds, sourceIsGds); } - public void ReadGds(Span destination, uint wordOffset, uint wordCount) => + public void ReadGds(Span destination, uint wordOffset, uint wordCount) + { + if (OnCommandThread) + { + var words = new uint[destination.Length]; + _commandThread!.Run(() => ReadGds(words, wordOffset, wordCount)); + words.CopyTo(destination); + return; + } + + DrainDraws(); EndOfPipe.ReadGdsWords(_bufferCache.GdsBuffer.Mapped, destination, wordOffset, wordCount); + } public void RecordEndOfPipe(in EndOfPipeWrite write) { + if (OnCommandThread) + { + var queued = write; + _commandThread!.Post(() => RecordEndOfPipe(in queued)); + return; + } + + DrainDraws(); using var completionScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandEndOfPipe); _ = BeginBatchedGuestCommands(); var buffer = _scheduler.Current; @@ -456,10 +682,26 @@ internal static unsafe partial class VulkanVideoPresenter } } - public void TriggerInterrupt(int eventId, uint contextId) => _endOfPipeEvents.TriggerInterrupt(eventId, contextId); + public void TriggerInterrupt(int eventId, uint contextId) + { + if (OnCommandThread) + { + _commandThread!.Post(() => TriggerInterrupt(eventId, contextId)); + return; + } + + DrainDraws(); + _endOfPipeEvents.TriggerInterrupt(eventId, contextId); + } public bool HasFlipSlot() { + if (OnCommandThread) + { + return _commandThread!.Run(HasFlipSlot); + } + + DrainDraws(); lock (_gate) { return _pendingGuestImagePresentations.Count < MaxPendingGuestFlipVersions; @@ -468,17 +710,47 @@ internal static unsafe partial class VulkanVideoPresenter public ulong PrepareFlip(int handle, int index, int flipMode, long flipArgument) { + if (OnCommandThread) + { + return _commandThread!.Run(() => PrepareFlip(handle, index, flipMode, flipArgument)); + } + + DrainDraws(); _ = BeginBatchedGuestCommands(); return _endOfPipe.PrepareVideoOutFlip(_scheduler.Current, handle, index, flipMode, flipArgument); } - public bool IsFlipDone(int handle, int index) => VideoOutExports.IsFlipDone(handle, index); + public bool IsFlipDone(int handle, int index) + { + if (OnCommandThread) + { + return _commandThread!.Run(() => IsFlipDone(handle, index)); + } - public void PrepareCpuFlip(int handle, int index, ulong requestId) => + DrainDraws(); + return VideoOutExports.IsFlipDone(handle, index); + } + + public void PrepareCpuFlip(int handle, int index, ulong requestId) + { + if (OnCommandThread) + { + _commandThread!.Run(() => PrepareCpuFlip(handle, index, requestId)); + return; + } + + DrainDraws(); CaptureFlip(handle, index, requestId, flipMode: 0, flipArg: 0); + } public void DrawIndexed(ulong submitId, in DrawIndexedArguments arguments) { + if (OnCommandThread) + { + _translation.DrawIndexed(submitId, in arguments); + return; + } + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); var started = System.Diagnostics.Stopwatch.GetTimestamp(); try @@ -493,6 +765,12 @@ internal static unsafe partial class VulkanVideoPresenter public void DrawAuto(ulong submitId, in DrawAutoArguments arguments) { + if (OnCommandThread) + { + _translation.DrawAuto(submitId, in arguments); + return; + } + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); var started = System.Diagnostics.Stopwatch.GetTimestamp(); try @@ -521,6 +799,13 @@ internal static unsafe partial class VulkanVideoPresenter public void DispatchDirect(ulong submitId, uint groupsX, uint groupsY, uint groupsZ, uint dispatchInitiator, ulong indirectArgumentsAddress = 0) { + if (OnCommandThread) + { + _translation.Dispatch(submitId, groupsX, groupsY, groupsZ, dispatchInitiator, indirectArgumentsAddress); + return; + } + + DrainDraws(); using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDispatchTranslation); var started = System.Diagnostics.Stopwatch.GetTimestamp(); try @@ -533,13 +818,48 @@ internal static unsafe partial class VulkanVideoPresenter } } - public void OnQueueReset(int queueId) => _translation.QueueReset(queueId); + public void OnQueueReset(int queueId) + { + if (OnCommandThread) + { + _translation.QueueReset(queueId); + return; + } + + DrainDraws(); + _translation.QueueReset(queueId); + } + + // Deferred draws run before anything else the stream asks of the host: everything but a + // draw may record, read or write what they use. Their time counts as draw time. Completion + // callbacks reach some of the same entry points from the GPU priority worker; they are not + // ordered with the stream and leave the draws to the render thread. + private void DrainDraws() + { + // Hosts built without a command stream have no translation. + if (_translation is not { HasPendingDraws: true } || !_relay.IsGpuQueueThread) + { + return; + } + + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + _translation.DrainPendingDraws(); + } + finally + { + Interlocked.Add(ref _perfDrawTicks, System.Diagnostics.Stopwatch.GetTimestamp() - started); + } + } public Exception Fatal(string message) => SubmissionScheduler.Fatal(message); // The flip packet: translation first, then the request, then the capture in stream order. public int SubmitFlipFromGpu(RecordingBuffer buffer, int handle, int index, int flipMode, long flipArg, out ulong requestId) { + DrainDraws(); _ = buffer; _translation.PrepareFlip(handle, index); var result = VideoOutExports.TryReserveFlipRequest(handle, index, flipMode, flipArg, gpuQueued: true, out requestId); @@ -563,11 +883,16 @@ internal static unsafe partial class VulkanVideoPresenter // The interpreter must suspend before capture so this worker can present queued frames. public void WaitForSubmitSlot() => throw SubmissionScheduler.Fatal("The GPU worker cannot wait for a flip slot. Suspend the command stream before capture."); - public void CompleteFlip(ulong requestId) => VideoOutExports.CompleteFlip(requestId); + public void CompleteFlip(ulong requestId) + { + DrainDraws(); + VideoOutExports.CompleteFlip(requestId); + } // Copies the display surface through the store; presentation shows the copy once its tick retires. internal void CaptureFlip(int handle, int index, ulong requestId, int flipMode, long flipArg) { + DrainDraws(); // A no-buffer flip has no image to capture; its completion still retires in order. if (VideoOutExports.ReleaseNoBufferFlip(index, requestId)) { @@ -586,7 +911,7 @@ internal static unsafe partial class VulkanVideoPresenter FlushBatchedGuestCommands(); RunGuestCacheCollection(endsFrame: true); - EnsureGuestSubmissionCapacity(); + EnsureGuestSubmissionCapacity(collectNow: true); long version; lock (_gate) { diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandThread.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandThread.cs new file mode 100644 index 0000000..0db0cd4 --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandThread.cs @@ -0,0 +1,511 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.VideoOut; + +using System.Diagnostics; +using SharpEmu.HLE; +using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Scheduling; + +internal static partial class VulkanVideoPresenter +{ + // The command stream is interpreted on its own thread, which hands its work to the render thread + // in stream order. Draws, dispatches, barriers, completions and the interpreter's own stores are + // queued; a callback that returns a result waits for the render thread to run everything before it. + private sealed class CommandThread : AgcExports.IDeferredWorkSink + { + // The thread runs this far ahead of the render thread at most: enough to keep it busy, and + // no more, since a GPU round trip (a readback, a label wait) waits for everything queued. + // Measured in Astro Bot: 4096 items halved the frame rate of a crowd scene, 16 starved the + // render thread, 32 to 64 kept it busy. + private const int MaxOutstanding = 64; + + private enum Kind : byte + { + Deferred, + Call, + Write, + } + + private sealed class Item + { + public Kind Kind; + public bool IsDraw; + public AgcExports.DeferredWork? Work; + public Action? Action; + public ulong Address; + public byte[]? Data; + // The guest range a queued fill or copy writes; zero for other items. + public ulong RangeSize; + // Resolution jobs submitted before this non-draw item; they finish before it runs. + public long JobsBefore; + public bool Sync; + public volatile bool Done; + public Exception? Failure; + } + + private readonly CommandStreamQueue _queue; + private readonly ICpuMemory _memory; + private readonly Action _runDeferred; + private readonly ParallelProgramPrefetch? _prefetch; + private readonly Thread _thread; + private readonly object _lock = new(); + private List _incoming = new(); + private List _draining = new(); + private int _drainIndex; + private int _outstanding; + // What the producer sleeps on: room in the queue, or its synchronous item. + private volatile bool _waitingForSpace; + private volatile bool _waitingForSync; + private volatile bool _consumerIdle; + private volatile bool _stopping; + private volatile Exception? _failure; + private long _enqueuedNonDraw; + private long _executedNonDraw; + + // The interpreter's stores the render thread has not applied yet, oldest first; reads on + // this thread see them over guest memory. + private readonly object _writeLock = new(); + private readonly Queue _pendingWrites = new(); + private volatile int _pendingWriteCount; + + // The destinations of queued fills and copies; a read on this thread that overlaps one waits + // for the render thread to run it. + private readonly List _pendingRanges = new(); + private volatile int _pendingRangeCount; + + private static readonly long RetryTicks = CommandStreamQueue.AllBlockedRetryMilliseconds * Stopwatch.Frequency / 1000L; + + public CommandThread(CommandStreamQueue queue, ICpuMemory memory, Action runDeferred, ParallelProgramPrefetch? prefetch) + { + _queue = queue; + _memory = memory; + _runDeferred = runDeferred; + _prefetch = prefetch; + _thread = new Thread(Run) { IsBackground = true, Name = "SharpEmu command stream" }; + } + + public void Start() => _thread.Start(); + + public bool IsProducerThread => Environment.CurrentManagedThreadId == _thread.ManagedThreadId; + + public long Gate => _enqueuedNonDraw; + + public bool HasWork => Volatile.Read(ref _outstanding) != 0 || _failure is not null; + + // Read by the render loop before it waits, so a queued item wakes it. + public bool ConsumerIdle + { + set => _consumerIdle = value; + } + + public bool IsAlive => _thread.IsAlive; + + // The producer side, on the command stream thread. + + public void EnqueueDeferred(AgcExports.DeferredWork work, bool isDraw) => + Enqueue(new Item { Kind = Kind.Deferred, Work = work, IsDraw = isDraw }); + + public void Post(Action action) => Enqueue(new Item { Kind = Kind.Call, Action = action }); + + public void Run(Action action) + { + var item = new Item { Kind = Kind.Call, Action = action, Sync = true }; + Enqueue(item); + WaitFor(item); + } + + public T Run(Func function) + { + T result = default!; + Run(() => { result = function(); }); + return result; + } + + public void PostWrite(ulong address, ReadOnlySpan data) + { + var item = new Item { Kind = Kind.Write, Address = address, Data = data.ToArray() }; + lock (_writeLock) + { + _pendingWrites.Enqueue(item); + _pendingWriteCount = _pendingWrites.Count; + } + + Enqueue(item); + } + + // A fill or copy the render thread runs in order; reads of its destination wait for it. + public void PostRange(Action action, ulong address, ulong size) + { + var item = new Item { Kind = Kind.Call, Action = action, Address = address, RangeSize = size }; + if (size != 0) + { + lock (_writeLock) + { + _pendingRanges.Add(item); + _pendingRangeCount = _pendingRanges.Count; + } + } + + Enqueue(item); + } + + // Runs every queued item when one of them is a fill or copy into this range. + public void WaitForPendingRange(ulong address, ulong size) + { + if (_pendingRangeCount != 0 && OverlapsPendingRange(address, size)) + { + Run(static () => { }); + } + } + + private bool OverlapsPendingRange(ulong address, ulong size) + { + lock (_writeLock) + { + foreach (var pending in _pendingRanges) + { + if (address < pending.Address + pending.RangeSize && pending.Address < address + size) + { + return true; + } + } + + return false; + } + } + + // Guest memory as the stream sees it: with its own stores applied, queued or not. + public bool ReadOverlay(ulong address, Span destination) + { + WaitForPendingRange(address, (ulong)destination.Length); + if (_pendingWriteCount == 0) + { + return _memory.TryRead(address, destination); + } + + lock (_writeLock) + { + if (!_memory.TryRead(address, destination)) + { + return false; + } + + var end = address + (ulong)destination.Length; + foreach (var write in _pendingWrites) + { + var writeEnd = write.Address + (ulong)write.Data!.Length; + if (write.Address >= end || writeEnd <= address) + { + continue; + } + + var start = Math.Max(address, write.Address); + var stop = Math.Min(end, writeEnd); + write.Data.AsSpan((int)(start - write.Address), (int)(stop - start)) + .CopyTo(destination[(int)(start - address)..]); + } + + return true; + } + } + + private void Enqueue(Item item) + { + ThrowIfFailed(); + if (!item.IsDraw) + { + item.JobsBefore = _prefetch?.Submitted ?? 0; + _enqueuedNonDraw++; + } + + if (Volatile.Read(ref _outstanding) >= MaxOutstanding) + { + WaitForSpace(); + } + + lock (_lock) + { + _incoming.Add(item); + } + + Interlocked.Increment(ref _outstanding); + if (_consumerIdle || item.Sync) + { + Presenter.WakeRenderThread(); + } + } + + private void WaitForSpace() + { + lock (_lock) + { + // The flag is set before the check: the render thread reads it after it changes what is checked. + for (;;) + { + _waitingForSpace = true; + Interlocked.MemoryBarrier(); + if (Volatile.Read(ref _outstanding) < MaxOutstanding / 2 || _failure is not null) + { + break; + } + + Monitor.Wait(_lock, 10); + } + + _waitingForSpace = false; + } + + ThrowIfFailed(); + } + + private void WaitFor(Item item) + { + var spinner = new SpinWait(); + for (var spin = 0; spin < 100 && !item.Done; spin++) + { + spinner.SpinOnce(sleep1Threshold: -1); + } + + if (!item.Done) + { + lock (_lock) + { + for (;;) + { + _waitingForSync = true; + Interlocked.MemoryBarrier(); + if (item.Done || _failure is not null) + { + break; + } + + Monitor.Wait(_lock, 10); + } + + _waitingForSync = false; + } + } + + if (item.Failure is { } failure) + { + throw new InvalidOperationException("A command stream callback failed on the render thread.", failure); + } + + if (!item.Done) + { + ThrowIfFailed(); + } + } + + // The consumer side, on the render thread. Runs queued items until the deadline or until + // none is left; returns how many ran. + public int RunBatch(long deadline, int maxItems, Action betweenItems) + { + if (_drainIndex == _draining.Count) + { + _draining.Clear(); + _drainIndex = 0; + lock (_lock) + { + (_incoming, _draining) = (_draining, _incoming); + } + + if (_draining.Count == 0) + { + ThrowIfFailed(); + return 0; + } + } + + var ran = 0; + while (_drainIndex < _draining.Count) + { + var item = _draining[_drainIndex]; + _draining[_drainIndex++] = null!; + Execute(item); + ran++; + var outstanding = Interlocked.Decrement(ref _outstanding); + // One pulse per wait: the producer sets its flag again if it keeps waiting. + var wakeForSync = item.Sync && _waitingForSync; + var wakeForSpace = _waitingForSpace && outstanding < MaxOutstanding / 2; + if (wakeForSync || wakeForSpace) + { + if (wakeForSync) + { + _waitingForSync = false; + } + + if (wakeForSpace) + { + _waitingForSpace = false; + } + + lock (_lock) + { + Monitor.PulseAll(_lock); + } + } + + betweenItems(); + if (ran >= maxItems || ((ran & 31) == 0 && Stopwatch.GetTimestamp() >= deadline)) + { + break; + } + } + + return ran; + } + + private void Execute(Item item) + { + if (item.IsDraw) + { + _runDeferred(item.Work!); + return; + } + + _prefetch?.WaitFinished(item.JobsBefore); + try + { + switch (item.Kind) + { + case Kind.Deferred: + _runDeferred(item.Work!); + break; + case Kind.Call: + item.Action!(); + break; + case Kind.Write: + ApplyWrite(item); + break; + } + } + catch (Exception exception) when (item.Sync) + { + item.Failure = exception; + } + catch (Exception exception) + { + _failure ??= exception; + throw; + } + finally + { + _executedNonDraw++; + if (item.RangeSize != 0) + { + lock (_writeLock) + { + _pendingRanges.Remove(item); + _pendingRangeCount = _pendingRanges.Count; + } + } + + _prefetch?.AdvanceGate(_executedNonDraw); + if (item.Sync) + { + item.Done = true; + } + } + } + + private void ApplyWrite(Item item) + { + lock (_writeLock) + { + var written = _memory.TryWrite(item.Address, item.Data); + if (!ReferenceEquals(_pendingWrites.Peek(), item)) + { + throw SubmissionScheduler.Fatal("The command stream stores ran out of order."); + } + + _pendingWrites.Dequeue(); + _pendingWriteCount = _pendingWrites.Count; + if (!written) + { + throw SubmissionScheduler.Fatal( + $"The command stream cannot write guest memory: address=0x{item.Address:X16} size={item.Data!.Length}."); + } + } + } + + public void ThrowIfFailed() + { + if (_failure is { } failure) + { + throw new InvalidOperationException("The command stream thread failed.", failure); + } + } + + // Ends the thread after its current slice; the render thread keeps running its items + // meanwhile, since the slice may wait for one. + public void Stop(Action runItems) + { + _stopping = true; + _queue.Wake(); + var deadline = Stopwatch.GetTimestamp() + 5 * Stopwatch.Frequency; + while (_thread.IsAlive && Stopwatch.GetTimestamp() < deadline) + { + runItems(); + _thread.Join(1); + } + + runItems(); + } + + private void Run() + { + try + { + var lastProgress = Stopwatch.GetTimestamp(); + while (!_stopping) + { + var result = _queue.ProcessOne(); + switch (result) + { + case SliceResult.NoWork: + _ = _queue.WaitForWork(8); + break; + case SliceResult.AllBlocked: + WaitBlocked(ref lastProgress); + break; + case SliceResult.BlockedWithoutProgress: + if (!_queue.HasUnblockedPending) + { + WaitBlocked(ref lastProgress); + } + + break; + default: + lastProgress = Stopwatch.GetTimestamp(); + break; + } + } + } + catch (Exception exception) + { + _failure ??= exception; + Presenter.WakeRenderThread(); + } + } + + // Blocked heads retry when a GPU completion or a presentation unblocks them, or every + // retry interval without progress. + private void WaitBlocked(ref long lastProgress) + { + var remaining = RetryTicks - (Stopwatch.GetTimestamp() - lastProgress); + if (remaining <= 0) + { + _queue.RetryBlocked(); + lastProgress = Stopwatch.GetTimestamp(); + return; + } + + _ = _queue.WaitForRetryInterval((int)Math.Max(1, remaining * 1000 / Stopwatch.Frequency)); + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ConcurrentResolve.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ConcurrentResolve.cs new file mode 100644 index 0000000..ad0223e --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ConcurrentResolve.cs @@ -0,0 +1,152 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.VideoOut; + +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Scheduling; + +internal static unsafe partial class VulkanVideoPresenter +{ + internal enum ConcurrentQuery : byte + { + DirtyBytes, + CleanWord, + Resident, + ResidentClean, + } + + internal sealed class ConcurrentAssumptions(long bufferVersion, long writebackEpoch) + { + public readonly long BufferVersion = bufferVersion; + public readonly long WritebackEpoch = writebackEpoch; + public readonly List<(ConcurrentQuery Query, ulong Address, ulong Size, bool Answer)> Answers = new(); + public bool ReadsImageOwnership; + + public void Record(ConcurrentQuery query, ulong address, ulong size, bool answer) + { + ReadsImageOwnership |= query is ConcurrentQuery.CleanWord or ConcurrentQuery.ResidentClean; + for (var index = Answers.Count - 1; index >= Math.Max(0, Answers.Count - 8); index--) + { + var previous = Answers[index]; + if (previous.Query != query || previous.Answer != answer) + continue; + if (previous.Address == address && previous.Size == size) + return; + // No owned bytes in either range proves the union, but an owned byte does not. + if (!answer && size <= ulong.MaxValue - address && previous.Size <= ulong.MaxValue - previous.Address && + address <= previous.Address + previous.Size && previous.Address <= address + size) + { + var start = Math.Min(address, previous.Address); + var end = Math.Max(address + size, previous.Address + previous.Size); + Answers[index] = (query, start, end - start, false); + return; + } + } + + Answers.Add((query, address, size, answer)); + } + } + + private sealed partial class Presenter : IConcurrentResolveHost + { + // This partial serves the guest readers of program resolution on a thread resolving draws + // ahead of the render thread. That thread never synchronizes GPU-owned memory: a word the + // GPU owns abandons the resolution, which the render thread then does itself. Every + // ownership answer it relied on is recorded and checked again when the result is taken. + + // Set only on the resolving thread, around one resolution. + [ThreadStatic] + private static ConcurrentAssumptions? _concurrentAssumptions; + + void IConcurrentResolveHost.BeginConcurrentResolve() => + _concurrentAssumptions = new ConcurrentAssumptions(_bufferCache.GpuModifiedVersion, GpuWritebackEpoch.Value); + + object IConcurrentResolveHost.EndConcurrentResolve() + { + var assumptions = _concurrentAssumptions ?? throw new InvalidOperationException("No concurrent resolution is open."); + _concurrentAssumptions = null; + return assumptions; + } + + // GPU data stored into guest memory since the resolution may have changed what it read; + // otherwise the answers stand while the ownership they came from has not changed. + bool IConcurrentResolveHost.IsStillCurrent(object assumptionsObject) + { + var assumptions = (ConcurrentAssumptions)assumptionsObject; + if (assumptions.WritebackEpoch != GpuWritebackEpoch.Value) + { + return false; + } + + if (assumptions.BufferVersion == _bufferCache.GpuModifiedVersion && !assumptions.ReadsImageOwnership) + { + return true; + } + + foreach (var (query, address, size, answer) in assumptions.Answers) + { + var now = query switch + { + ConcurrentQuery.DirtyBytes => _bufferCache.HasGpuDirtyBytes(address, size), + ConcurrentQuery.CleanWord => IsGpuOwnedForCleanRead(address, size), + ConcurrentQuery.Resident => IsGpuOwnedForResidentRead(address, size, clean: false), + _ => IsGpuOwnedForResidentRead(address, size, clean: true), + }; + if (now != answer) + { + return false; + } + } + + return true; + } + + private bool IsGpuOwnedForCleanRead(ulong address, ulong size) => + (_bufferCache.MayHaveGpuDirtyPages(address, size) && _bufferCache.HasGpuDirtyPages(address, size)) || + _bufferCache.HasGpuDirtyBytes(address, size) || + _imageCache.HasGpuModifiedImageBytes(address, size); + + private bool IsGpuOwnedForResidentRead(ulong address, ulong size, bool clean) => + _bufferCache.HasGpuDirtyBytes(address, size) || + (clean && (_bufferCache.HasGpuDirtyPages(address, size) || _imageCache.HasGpuModifiedImageBytes(address, size))); + + private bool TryReadGuestWordConcurrent(ulong address, out uint word, ConcurrentAssumptions assumptions) + { + word = 0; + if (!_guestMemory.CanRead(address, sizeof(uint))) + { + return false; + } + + var dirty = _bufferCache.HasGpuDirtyBytes(address, sizeof(uint)); + assumptions.Record(ConcurrentQuery.DirtyBytes, address, sizeof(uint), dirty); + if (dirty) + { + throw new SubmissionScheduler.AbandonedWorkException($"The resolution reads GPU-owned memory: address=0x{address:X16}."); + } + + Span bytes = stackalloc byte[sizeof(uint)]; + if (!_guestMemory.TryRead(address, bytes)) + { + return false; + } + + word = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes); + return true; + } + + private bool TryReadResidentGuestBytesConcurrent(ulong address, Span destination, bool clean, ConcurrentAssumptions assumptions) + { + var size = (ulong)destination.Length; + if (!_guestMemory.CanRead(address, size)) + { + return false; + } + + var owned = IsGpuOwnedForResidentRead(address, size, clean); + assumptions.Record(clean ? ConcurrentQuery.ResidentClean : ConcurrentQuery.Resident, address, size, owned); + return !owned && _guestMemory.TryRead(address, destination); + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs index cdc3b6d..3200cce 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs @@ -111,21 +111,89 @@ internal static unsafe partial class VulkanVideoPresenter Atomic: image.Atomic); private static readonly System.Collections.Concurrent.ConcurrentDictionary _tracedTextureBindings = new(); + // The last lookup of one image of one program. A draw binds the same descriptor as the + // previous draw of its program most of the time; while no image the lookup could find has + // changed (the cache's lookup generation), the request and the image found are the same. + private struct ImageBindingMemo + { + public uint[]? Words; + public ulong Generation; + public TextureRequestResolution Resolution; + public ResourceSlotIdentifier Found; + public uint MetadataBaseLayer; + public bool CompletesLookup; + + public readonly bool Matches(uint[] words, ulong generation) => + Words is not null && Generation == generation && words.AsSpan().SequenceEqual(Words); + } + + private Dictionary? _imageBindingMemos; + private ShaderProgramInfo? _lastMemoProgram; + private ImageBindingMemo[] _lastMemos = []; + + // A draw's images resolve one after another for one program, so the last program's memos + // are kept at hand. + private ref ImageBindingMemo ImageBindingMemoOf(ShaderProgramInfo program, int index) + { + if (!ReferenceEquals(program, _lastMemoProgram) || index >= _lastMemos.Length) + { + var all = _imageBindingMemos ??= new Dictionary(ReferenceEqualityComparer.Instance); + if (!all.TryGetValue(program, out var memos) || index >= memos.Length) + { + Array.Resize(ref memos, Math.Max(index + 1, program.Resources?.Info.Images.Count ?? 0)); + all[program] = memos; + } + + _lastMemoProgram = program; + _lastMemos = memos; + } + + return ref _lastMemos[index]; + } // Render-state discovery for one shader image; the view is acquired later with the draw. private TextureResource ResolveImageBinding(ImageResource image, uint[] words, ShaderProgramInfo program, int index) { + ref var memo = ref ImageBindingMemoOf(program, index); if (words.Length < 4) { throw SubmissionScheduler.Fatal($"An image descriptor is too short: image={index} words={words.Length} hash=0x{program.Hash:X16}."); } var storage = image.ResourceClass == ShaderCompiler.Resources.ImageResourceClass.Storage; - var resolution = ImageRequestBuilders.Texture(words, ShapeOf(image)); _ = BeginBatchedGuestCommands(); - var request = resolution.Request; - var imageIdentifier = _imageCache.FindImage(ref request, resolution.ExactFormat); - resolution = resolution with { Request = request }; + TextureRequestResolution resolution; + ResourceSlotIdentifier imageIdentifier; + if (memo.Matches(words, _imageCache.LookupGeneration) && _imageCache.TryTouchRemembered(memo.Found)) + { + resolution = memo.Resolution; + imageIdentifier = memo.Found; + if (memo.CompletesLookup) + { + _imageCache.CompleteLookup(imageIdentifier, resolution.Request, memo.MetadataBaseLayer); + } + } + else + { + resolution = ImageRequestBuilders.Texture(words, ShapeOf(image)); + var request = resolution.Request; + var metadataBaseLayer = request.View.BaseLayer; + imageIdentifier = _imageCache.FindImage(ref request, resolution.ExactFormat); + resolution = resolution with { Request = request }; + memo = !ImageDescription.IsEmptyRange(request.Description.Data) + ? new ImageBindingMemo + { + Words = (uint[])words.Clone(), + Generation = _imageCache.LookupGeneration, + Resolution = resolution, + Found = imageIdentifier, + MetadataBaseLayer = metadataBaseLayer, + CompletesLookup = request.Description.Metadata.Kind == MetadataKind.Dcc || + _imageCache.GetImage(imageIdentifier).Description.DccSliceSize != 0, + } + : default; + } + imageIdentifier = ImageRequestBuilders.ValidateTextureOwner(_imageCache, imageIdentifier, resolution); BindImage(imageIdentifier, storage); var descriptor = new TextureDescriptorWords(words); @@ -151,16 +219,47 @@ internal static unsafe partial class VulkanVideoPresenter Console.Error.WriteLine(line); } } - return new TextureResource + var texture = RentTextureResource(); + texture.Address = descriptor.BaseAddress; + texture.ImageIdentifier = imageIdentifier; + texture.Request = resolution.Request; + texture.IsStorage = storage; + texture.DestinationSelect = words[3] & 0xFFFu; + texture.Width = descriptor.Width; + texture.Height = descriptor.Height; + return texture; + } + + // Binding descriptions are large (an image request carries its whole mip layout) and one is made + // per shader image per draw, so they are reused once the submission that read them has retired. + // Only the render thread rents and returns them. + // Sized for the bindings of every draw still in flight (a few frames of a thousand draws each); + // a smaller cap drops returns during busy scenes and the pool allocates again. + private const int MaxPooledTextureResources = 32768; + private Stack? _texturePool; + + private TextureResource RentTextureResource() + { + if (_texturePool is not null && _texturePool.TryPop(out var texture)) { - Address = descriptor.BaseAddress, - ImageIdentifier = imageIdentifier, - Request = request, - IsStorage = storage, - DestinationSelect = words[3] & 0xFFFu, - Width = descriptor.Width, - Height = descriptor.Height, - }; + texture.InPool = false; + return texture; + } + + return new TextureResource { Pooled = true }; + } + + private void ReturnTextureResource(TextureResource texture) + { + _texturePool ??= new Stack(); + if (!texture.Pooled || texture.InPool || _texturePool.Count >= MaxPooledTextureResources) + { + return; + } + + texture.Reset(); + texture.InPool = true; + _texturePool.Push(texture); } // Compare bits stay only on depth-compare samplers; a forced point sampler drops its filters. @@ -572,6 +671,12 @@ internal static unsafe partial class VulkanVideoPresenter } prepared.Descriptors.Buffers = views; + if (layout.MipStatisticsCount != 0) + { + LodStatistics().EnsureInitialized(); + WriteMipStatisticsFields(shaderData, layout, info, snapshot.Images); + } + if (layout.Find(DescriptorBindingKind.FlattenedResourceTable) is not null) { prepared.Descriptors.FlattenedTable = UploadDwords(snapshot.FlattenedResourceTable, "flattened resource table", program); @@ -751,7 +856,7 @@ internal static unsafe partial class VulkanVideoPresenter using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.DescriptorCommit); var preparation = RequirePreparation(); var entry = RequirePipelineEntry(in pipeline); - var command = BeginBatchedGuestCommands(); + var command = BeginBatchedGuestStateCommands(); _commandBuffer = command; var writeCount = 0; var bufferCount = 0; @@ -768,8 +873,11 @@ internal static unsafe partial class VulkanVideoPresenter throw SubmissionScheduler.Fatal($"A stage does not belong to the bind point: stage={stage.Program.Stage} bindPoint={bindPoint}."); } - foreach (var binding in stage.Layout.Descriptors) + // Indexed loops over the layout's lists: a foreach on IReadOnlyList allocates an enumerator per draw. + var countedBindings = stage.Layout.Descriptors; + for (var bindingIndex = 0; bindingIndex < countedBindings.Count; bindingIndex++) { + var binding = countedBindings[bindingIndex]; writeCount++; var count = (int)DescriptorWriter.DescriptorCount(binding); if (ImageDescriptorBinding.ResourceClass(binding.Kind) != ShaderCompiler.Resources.ImageResourceClass.None || binding.Kind == DescriptorBindingKind.Samplers) @@ -862,8 +970,11 @@ internal static unsafe partial class VulkanVideoPresenter var occurrences = occurrenceScratch[..descriptors.Images.Length]; occurrences.Clear(); - foreach (var binding in stage.Layout.Descriptors) + var bindings = stage.Layout.Descriptors; + for (var bindingIndex = 0; bindingIndex < bindings.Count; bindingIndex++) { + var binding = bindings[bindingIndex]; + var resources = binding.Resources; var write = new WriteDescriptorSet { SType = StructureType.WriteDescriptorSet, @@ -875,8 +986,9 @@ internal static unsafe partial class VulkanVideoPresenter var imageStart = imageIndex; if (ImageDescriptorBinding.ResourceClass(binding.Kind) != ShaderCompiler.Resources.ImageResourceClass.None) { - foreach (var resource in binding.Resources) + for (var resourceIndex = 0; resourceIndex < resources.Count; resourceIndex++) { + var resource = resources[resourceIndex]; imageInfos[imageIndex++] = ImageInfo(descriptors.Images[(int)resource], occurrences[(int)resource]++, program, (int)resource); } } @@ -885,8 +997,9 @@ internal static unsafe partial class VulkanVideoPresenter switch (binding.Kind) { case DescriptorBindingKind.Buffers: - foreach (var resource in binding.Resources) + for (var resourceIndex = 0; resourceIndex < resources.Count; resourceIndex++) { + var resource = resources[resourceIndex]; var view = descriptors.Buffers[(int)resource]; if (view.Buffer.Handle == 0) { @@ -896,6 +1009,9 @@ internal static unsafe partial class VulkanVideoPresenter bufferInfos[bufferIndex++] = new DescriptorBufferInfo { Buffer = view.Buffer, Offset = view.Offset, Range = view.Range }; } + break; + case DescriptorBindingKind.MipStatistics: + bufferInfos[bufferIndex++] = MipStatisticsDescriptor(); break; case DescriptorBindingKind.DeviceAddressPageTable: case DescriptorBindingKind.FaultBuffer: @@ -925,8 +1041,9 @@ internal static unsafe partial class VulkanVideoPresenter } case DescriptorBindingKind.Samplers: - foreach (var resource in binding.Resources) + for (var resourceIndex = 0; resourceIndex < resources.Count; resourceIndex++) { + var resource = resources[resourceIndex]; var sampler = descriptors.Samplers[(int)resource]; if (sampler.Handle == 0) { @@ -983,10 +1100,10 @@ internal static unsafe partial class VulkanVideoPresenter if (hasPushData) { var pushStages = bindPoint == PipelineBindPoint.Graphics ? ShaderStageFlags.VertexBit | ShaderStageFlags.FragmentBit : ShaderStageFlags.ComputeBit; - _vk.CmdPushConstants(command, entry.Layout, pushStages, 0, PushData.ByteSize, pushData); + PushConstantsShadowed(command, entry.Layout, pushStages, pushData); } - if (writeIndex != 0) + if (writeIndex != 0 && !DescriptorsAlreadyBound(command, bindPoint, entry.Layout, writes[..writeIndex])) { fixed (WriteDescriptorSet* writePointer = writes) { diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs index 9d0d1d0..08099d2 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs @@ -608,11 +608,16 @@ internal static unsafe partial class VulkanVideoPresenter _supportsShaderSignedZeroInfNanPreserveFloat32 = floatControls.ShaderSignedZeroInfNanPreserveFloat32; SetNativeSubgroupCapabilities(subgroup.SubgroupSize, subgroup.SupportedStages); + SetMipStatisticsCapabilities(true, subgroup.SupportedOperations, subgroup.SupportedStages); _canRequireComputeSubgroup32 = subgroup.SubgroupSize != RdnaSubgroupSize && subgroupSizeControl.MinSubgroupSize <= RdnaSubgroupSize && subgroupSizeControl.MaxSubgroupSize >= RdnaSubgroupSize && (subgroupSizeControl.RequiredSubgroupSizeStages & ShaderStageFlags.ComputeBit) != 0; + _canRequireComputeSubgroup64 = + subgroupSizeControl.MinSubgroupSize <= Wave64SubgroupSize && + subgroupSizeControl.MaxSubgroupSize >= Wave64SubgroupSize && + (subgroupSizeControl.RequiredSubgroupSizeStages & ShaderStageFlags.ComputeBit) != 0; _maxComputeWorkgroupSubgroups = subgroupSizeControl.MaxComputeWorkgroupSubgroups; // MoltenVK reports a zero OpArrayLength for storage buffers bound through push // descriptors, which turns every bounds-checked load into zero and drops every store. @@ -750,6 +755,11 @@ internal static unsafe partial class VulkanVideoPresenter queueInfoCount = 2; } _vk.GetPhysicalDeviceFeatures(_physicalDevice, out var supportedFeatures); + if (!supportedFeatures.FragmentStoresAndAtomics) + { + ShaderCompiler.Resources.ResourceMaterializer.RecordMipStatistics = false; + } + _supportsIndependentBlend = supportedFeatures.IndependentBlend; _supportsDepthBiasClamp = supportedFeatures.DepthBiasClamp; _supportsDepthBounds = supportedFeatures.DepthBounds; @@ -945,6 +955,7 @@ internal static unsafe partial class VulkanVideoPresenter var supportsNullDescriptor = robustness2Features.NullDescriptor; var supportsRobustness2 = supportsRobustImageAccess2 || supportsNullDescriptor; _canRequireComputeSubgroup32 &= vulkan13Features.SubgroupSizeControl; + _canRequireComputeSubgroup64 &= vulkan13Features.SubgroupSizeControl; SetSharedInt64AtomicsCapability(supportsSharedInt64Atomics); if (!supportsSharedInt64Atomics) { @@ -1157,7 +1168,7 @@ internal static unsafe partial class VulkanVideoPresenter SType = StructureType.PhysicalDeviceVulkan13Features, DynamicRendering = true, Synchronization2 = true, - SubgroupSizeControl = _canRequireComputeSubgroup32, + SubgroupSizeControl = _canRequireComputeSubgroup32 || _canRequireComputeSubgroup64, PNext = renderingChain, }; var features2 = new PhysicalDeviceFeatures2 diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs index 08a48bb..552baeb 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs @@ -275,8 +275,12 @@ internal static unsafe partial class VulkanVideoPresenter { RecycleHostBuffer(texture.StagingBuffer, texture.StagingMemory); } + + ReturnTextureResource(texture); } + resources.Textures = []; + foreach (var snapshot in resources.FeedbackSnapshots) { RetireFeedbackSnapshot(snapshot); diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs index 4c56738..4f721ab 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs @@ -47,13 +47,32 @@ internal static unsafe partial class VulkanVideoPresenter private CommandBuffer BeginBatchedGuestCommands() { var commandBuffer = CurrentRecordingBuffer(); + OpenGuestBatch(); + return commandBuffer; + } + + private CommandBuffer BeginBatchedGuestStateCommands() + { + var recording = CurrentRecording(); + OpenGuestBatch(); + return new CommandBuffer(recording.StateHandle); + } + + private void RequestGuestMemoryBarrier(in MemoryBarrierRequest request) + { + var recording = CurrentRecording(); + OpenGuestBatch(); + _scheduler.MemoryBarrierRecorder ??= RecordRequestedMemoryBarrier; + recording.RequestMemoryBarrier(request); + } + + private void OpenGuestBatch() + { if (!_batchOpen) { _batchOpen = true; _batchDrawCount = 0; } - - return commandBuffer; } // Submits the current recording buffer with everything the batch lists own. @@ -108,9 +127,23 @@ internal static unsafe partial class VulkanVideoPresenter retireBuffers)); } - private void EnsureGuestSubmissionCapacity() + // Collecting completed submissions polls the GPU timeline twice and may wait for the priority + // worker. Every draw used to pay that; completed work only frees resources and runs deferred + // callbacks, so a collection a few hundred microseconds later changes nothing but how long + // those resources stay reserved. A full in-flight queue still collects, and waits, at once. + private static readonly long CollectIntervalTicks = System.Diagnostics.Stopwatch.Frequency / 4000; + private long _lastCollectTicks; + + private void EnsureGuestSubmissionCapacity(bool collectNow = false) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.SubmissionCapacity); + var now = System.Diagnostics.Stopwatch.GetTimestamp(); + if (!collectNow && _pendingGuestSubmissions.Count < MaxInFlightGuestSubmissions && now - _lastCollectTicks < CollectIntervalTicks) + { + return; + } + + _lastCollectTicks = now; CollectCompletedGuestSubmissions(waitForOldest: false); if (_pendingGuestSubmissions.Count >= MaxInFlightGuestSubmissions) { @@ -149,6 +182,7 @@ internal static unsafe partial class VulkanVideoPresenter } _commandStream.NotifyCompletedGpuTick(_scheduler.Timeline.CompletedTick); + PublishLodStatistics(); ProcessDeferredTextureDestroys(); } diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.LodStatistics.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.LodStatistics.cs new file mode 100644 index 0000000..6b1d0de --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.LodStatistics.cs @@ -0,0 +1,199 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Collections.Concurrent; +using SharpEmu.Libs.Gpu.Buffers; +using SharpEmu.Libs.Gpu.Images; +using SharpEmu.Libs.Gpu.Scheduling; +using SharpEmu.ShaderCompiler.Resources; +using Silk.NET.Vulkan; + +namespace SharpEmu.Libs.VideoOut; + +internal static unsafe partial class VulkanVideoPresenter +{ + private static readonly bool GpuLodStatistics = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOD_STATS"), "gpu", StringComparison.Ordinal); + + private static int _fragmentSubgroupArithmetic; + + internal static bool FragmentSubgroupArithmetic => Volatile.Read(ref _fragmentSubgroupArithmetic) != 0; + + private static void SetMipStatisticsCapabilities(bool fragmentStoresAndAtomics, SubgroupFeatureFlags operations, ShaderStageFlags stages) + { + ResourceMaterializer.RecordMipStatistics = GpuLodStatistics && fragmentStoresAndAtomics; + var arithmetic = (operations & (SubgroupFeatureFlags.ArithmeticBit | SubgroupFeatureFlags.BallotBit | SubgroupFeatureFlags.BasicBit)) == + (SubgroupFeatureFlags.ArithmeticBit | SubgroupFeatureFlags.BallotBit | SubgroupFeatureFlags.BasicBit) && + (stages & ShaderStageFlags.FragmentBit) != 0; + Volatile.Write(ref _fragmentSubgroupArithmetic, arithmetic ? 1 : 0); + } + + private sealed partial class Presenter + { + private LodStatisticsRecorder? _lodStatistics; + + private LodStatisticsRecorder LodStatistics() => + _lodStatistics ??= new LodStatisticsRecorder(_deviceInfo, _scheduler); + + bool Gpu.Pipelines.IShaderPipelineHost.MipStatisticsSubgroupReduction => FragmentSubgroupArithmetic && GraphicsSubgroupOperationsEnabled; + + bool Gpu.GpuCommands.ICommandStreamHost.TryRecordLodStatistics(ulong destination, uint size) + { + if (!ResourceMaterializer.RecordMipStatistics) + { + if (!OnCommandThread) + { + return false; + } + + _commandThread!.PostRange(() => WriteBindingLodStatistics(destination, size), destination, size); + return true; + } + + if (size < MipStatistics.ReportBytes) + { + return false; + } + + if (OnCommandThread) + { + _commandThread!.Post(() => RecordLodStatistics(destination, size)); + return true; + } + + RecordLodStatistics(destination, size); + return true; + } + + private void RecordLodStatistics(ulong destination, uint size) + { + DrainDraws(); + EndRendering(); + _ = BeginBatchedGuestCommands(); + LodStatistics().Record(destination, size); + } + + private void WriteBindingLodStatistics(ulong destination, uint size) + { + var report = new byte[size]; + MipStatistics.Shared.WriteReport(report); + if (!((Gpu.GpuCommands.ICommandStreamHost)this).Memory.TryWrite(destination, report)) + { + throw SubmissionScheduler.Fatal($"The LOD statistics report could not be written: address=0x{destination:X16} size={report.Length}."); + } + } + + private void PublishLodStatistics() + { + if (_lodStatistics is not { } recorder) + { + return; + } + + while (recorder.TryTakeReport(out var destination, out var report)) + { + if (!((Gpu.GpuCommands.ICommandStreamHost)this).Memory.TryWrite(destination, report)) + { + throw SubmissionScheduler.Fatal($"The LOD statistics report could not be written: address=0x{destination:X16} size={report.Length}."); + } + } + } + + private void WriteMipStatisticsFields(uint[] shaderData, BindingLayout layout, ShaderResourceInfo info, uint[][] images) + { + for (var index = 0; index < layout.MipStatisticsCount; index++) + { + var image = index < info.Images.Count ? info.Images[index] : null; + var words = index < images.Length ? images[index] : null; + shaderData[layout.MipStatisticsDword + (uint)index] = image is { MipStatistics: true } && words is { Length: >= 8 } + ? MipStatistics.ImageField(words) + : MipStatistics.NoCounterField; + } + } + + private DescriptorBufferInfo MipStatisticsDescriptor() + { + var counters = LodStatistics().Counters; + return new DescriptorBufferInfo { Buffer = counters.Handle, Offset = 0, Range = counters.Size }; + } + } + + private sealed class LodStatisticsRecorder + { + private const int Slots = 32; + private const ulong SlotStride = 4096; + + private readonly SubmissionScheduler _scheduler; + private readonly GpuBuffer _publish; + private readonly ulong[] _slotTicks = new ulong[Slots]; + private readonly ConcurrentQueue<(ulong Destination, byte[] Report)> _ready = new(); + private int _nextSlot; + private bool _initialized; + + public LodStatisticsRecorder(GpuDeviceInfo device, SubmissionScheduler scheduler) + { + _scheduler = scheduler; + Counters = new GpuBuffer(device, scheduler, GpuBufferUsage.DeviceLocal, 0, GpuBuffer.AllFlags, MipStatistics.CounterBufferBytes); + _publish = new GpuBuffer(device, scheduler, GpuBufferUsage.Download, 0, GpuBuffer.AllFlags, Slots * SlotStride); + } + + public GpuBuffer Counters { get; } + + public void Reset() + { + Counters.Fill(0, MipStatistics.CounterWords * sizeof(uint), uint.MaxValue); + Counters.Fill(MipStatistics.CounterWords * sizeof(uint), MipStatistics.CounterWords * sizeof(uint), 0); + _initialized = true; + } + + public void EnsureInitialized() + { + if (!_initialized) + { + Reset(); + } + } + + public void Record(ulong destination, uint size) + { + EnsureInitialized(); + var slot = _nextSlot; + _nextSlot = (_nextSlot + 1) % Slots; + if (_slotTicks[slot] != 0 && !_scheduler.IsTickComplete(_slotTicks[slot])) + { + _scheduler.Wait(_slotTicks[slot]); + } + + var offset = (ulong)slot * SlotStride; + _publish.CopyFrom( + _scheduler.Current, Counters, 0, offset, MipStatistics.CounterBufferBytes, + AccessFlags.ShaderWriteBit | AccessFlags.MemoryWriteBit, AccessFlags.None, + AccessFlags.ShaderReadBit | AccessFlags.ShaderWriteBit | AccessFlags.MemoryReadBit | AccessFlags.MemoryWriteBit, AccessFlags.HostReadBit); + Reset(); + _slotTicks[slot] = _scheduler.CurrentTick; + var reportSize = (int)Math.Min(size, (uint)MipStatistics.ReportBytes); + _scheduler.QueueCompletionAction(() => + { + _publish.Invalidate(offset, MipStatistics.CounterBufferBytes); + var words = System.Runtime.InteropServices.MemoryMarshal.Cast( + _publish.Mapped.Slice((int)offset, (int)MipStatistics.CounterBufferBytes)); + var report = new byte[reportSize]; + MipStatistics.PackReport(words, report); + _ready.Enqueue((destination, report)); + }); + } + + public bool TryTakeReport(out ulong destination, out byte[] report) + { + if (_ready.TryDequeue(out var entry)) + { + (destination, report) = entry; + return true; + } + + destination = 0; + report = []; + return false; + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs index 3b0e354..d9cecb5 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs @@ -67,23 +67,47 @@ internal static unsafe partial class VulkanVideoPresenter uint IShaderPipelineHost.MaxPushDescriptors => _maxPushDescriptors; // Wave64 compute runs natively when the device's subgroup is that wide; smaller devices emulate it. + bool IShaderPipelineHost.ComputeWave64SubgroupNative => + Volatile.Read(ref _nativeSubgroupSize) >= 64 && (_canRequireComputeSubgroup64 || Volatile.Read(ref _nativeSubgroupSize) == 64); + bool IShaderPipelineHost.ComputeWave64Supported => Volatile.Read(ref _nativeSubgroupSize) >= 64; - // Only a 64-invocation wave64 workgroup is translated for either host subgroup width. Every - // other compute translation maps a guest wave to 32-lane host subgroups, and a 64-lane host - // subgroup (AMD's default) left lanes 32..63 inactive: a wave64 8x8x8 group lost rows 4..7. + bool IShaderPipelineHost.ComputeRunsOn32LaneSubgroups(ComputeInputInfo input) => + RequiresComputeSubgroup32(input) || + (Volatile.Read(ref _nativeSubgroupSize) == RdnaSubgroupSize && !RequiresComputeSubgroup64(input)); + + // Wave32 compute maps a guest wave to 32-lane host subgroups, and a 64-lane host subgroup + // (AMD's default) left lanes 32..63 inactive, so those pipelines are pinned to 32. Wave64 + // compute is pinned to 64 instead when the host runs it natively. private const uint RdnaSubgroupSize = 32; private bool _canRequireComputeSubgroup32; private uint _maxComputeWorkgroupSubgroups; + private const uint Wave64SubgroupSize = 64; + private bool _canRequireComputeSubgroup64; + + // A native wave64 translation reads one guest wave as one host subgroup, so the driver must + // not pick 32-lane subgroups for it (RDNA drivers may do so for compute). + private bool RequiresComputeSubgroup64(ComputeInputInfo input) + { + return _canRequireComputeSubgroup64 && input.WaveSize == 64 && + ((IShaderPipelineHost)this).ComputeWave64SubgroupNative; + } + private bool RequiresComputeSubgroup32(ComputeInputInfo input) { var invocations = (ulong)Math.Max(input.ThreadsX, 1) * Math.Max(input.ThreadsY, 1) * Math.Max(input.ThreadsZ, 1); return _canRequireComputeSubgroup32 && - !(input.WaveSize == 64 && invocations == 64) && + !(input.WaveSize == 64 && ((IShaderPipelineHost)this).ComputeWave64SubgroupNative) && invocations <= (ulong)_maxComputeWorkgroupSubgroups * RdnaSubgroupSize; } + // The subgroup width a compute pipeline must be pinned to, or 0 to leave it to the driver. + private uint ComputeRequiredSubgroupSize(ComputeInputInfo input) => + RequiresComputeSubgroup64(input) ? Wave64SubgroupSize + : RequiresComputeSubgroup32(input) ? RdnaSubgroupSize + : 0u; + bool IShaderPipelineHost.GraphicsSubgroupOperationsEnabled => GraphicsSubgroupOperationsEnabled; bool IShaderPipelineHost.SharedInt64AtomicsEnabled => SharedInt64AtomicsEnabled; @@ -101,6 +125,11 @@ internal static unsafe partial class VulkanVideoPresenter // A range the GPU wrote is downloaded first, so the word is what the guest CPU would read. public bool TryReadGuestWord(ulong address, out uint word) { + if (_concurrentAssumptions is { } assumptions) + { + return TryReadGuestWordConcurrent(address, out word, assumptions); + } + using var profile = ResourceMaterializationProfile.Measure(ResourceMaterializationProfile.Phase.GuestRead); word = 0; var synchronized = false; @@ -170,9 +199,9 @@ internal static unsafe partial class VulkanVideoPresenter return false; } - if ((_bufferCache.MayHaveGpuDirtyPages(address, sizeof(uint)) && _bufferCache.HasGpuDirtyPages(address, sizeof(uint))) || - _bufferCache.HasGpuDirtyBytes(address, sizeof(uint)) || - _imageCache.HasGpuModifiedImageBytes(address, sizeof(uint))) + var owned = IsGpuOwnedForCleanRead(address, sizeof(uint)); + _concurrentAssumptions?.Record(ConcurrentQuery.CleanWord, address, sizeof(uint), owned); + if (owned) { return false; } @@ -190,6 +219,11 @@ internal static unsafe partial class VulkanVideoPresenter // The same ownership rules as the two word readers, checked once for a whole range. public bool TryReadResidentGuestBytes(ulong address, Span destination, bool clean) { + if (_concurrentAssumptions is { } assumptions) + { + return TryReadResidentGuestBytesConcurrent(address, destination, clean, assumptions); + } + var size = (ulong)destination.Length; if (clean || !IsCleanReadPage(address, size)) { @@ -967,15 +1001,16 @@ internal static unsafe partial class VulkanVideoPresenter Pipeline pipeline; try { + var pin = ComputeRequiredSubgroupSize(description.Input); var requiredSubgroupSize = new PipelineShaderStageRequiredSubgroupSizeCreateInfo { SType = StructureType.PipelineShaderStageRequiredSubgroupSizeCreateInfo, - RequiredSubgroupSize = RdnaSubgroupSize, + RequiredSubgroupSize = pin, }; var stageInfo = new PipelineShaderStageCreateInfo { SType = StructureType.PipelineShaderStageCreateInfo, - PNext = RequiresComputeSubgroup32(description.Input) ? &requiredSubgroupSize : null, + PNext = pin != 0 ? &requiredSubgroupSize : null, Stage = ShaderStageFlags.ComputeBit, Module = computeModule, PName = entryPoint, @@ -1099,6 +1134,7 @@ internal static unsafe partial class VulkanVideoPresenter var device = _device; var cacheSource = GetGuestPipelineCacheSource(ComputeCacheKey(description.Stage.Hash, computeModule.Handle)); var vk = _vk; + var requiredSubgroupSize = ComputeRequiredSubgroupSize(description.Input); var started = new PendingComputePipeline { SetLayout = setLayout, @@ -1122,7 +1158,7 @@ internal static unsafe partial class VulkanVideoPresenter // Importing a MoltenVK cache compiles its MSL libraries. // Keep that work inside the same bounded compiler slot. var cache = ResolveGuestPipelineCache(cacheSource); - return CompileComputePipeline(vk, device, cache, computeModule, layout); + return CompileComputePipeline(vk, device, cache, computeModule, layout, requiredSubgroupSize); } finally { @@ -1146,14 +1182,21 @@ internal static unsafe partial class VulkanVideoPresenter Device device, PipelineCache cache, ShaderModule module, - PipelineLayout layout) + PipelineLayout layout, + uint requiredSubgroupSize) { var entryPoint = (byte*)SilkMarshal.StringToPtr("main"); try { + var subgroupInfo = new PipelineShaderStageRequiredSubgroupSizeCreateInfo + { + SType = StructureType.PipelineShaderStageRequiredSubgroupSizeCreateInfo, + RequiredSubgroupSize = requiredSubgroupSize, + }; var stageInfo = new PipelineShaderStageCreateInfo { SType = StructureType.PipelineShaderStageCreateInfo, + PNext = requiredSubgroupSize != 0 ? &subgroupInfo : null, Stage = ShaderStageFlags.ComputeBit, Module = module, PName = entryPoint, diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs index 025312b..7c8d072 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs @@ -357,6 +357,8 @@ internal static unsafe partial class VulkanVideoPresenter _commandBuffer, PipelineBindPoint.Graphics, isHdr ? _hdrPqPipeline : _hdrPipeline); + SharpEmu.Libs.Gpu.Vulkan.GraphicsDynamicStateEpoch.Advance(); + SharpEmu.Libs.Gpu.Vulkan.NativeBindingEpoch.Advance(); var descriptorSet = isHdr ? _hdrPqDescriptorSets[imageIndex] : _hdrDescriptorSets[imageIndex]; diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs index 307d6f9..f6d0c6d 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs @@ -46,38 +46,50 @@ internal static unsafe partial class VulkanVideoPresenter using var profileScope = RenderPhaseProfile.Measure(RenderPhaseProfile.Phase.Idle); var gpuWorkInFlight = _pendingGuestSubmissions.Count > 0 || Array.Exists(_frameInFlight, static pending => pending); - var blockedRetryWait = BlockedRetryWaitMilliseconds(); - lock (_gate) + // The command stream thread retries its blocked heads itself and wakes this thread for its + // items; before it starts, queued submissions start it on the next tick. + var commandThread = _commandThread; + if (commandThread is not null) { - if (_closed || - Volatile.Read(ref _presenterCloseRequested) || - _relay.HasPendingCommands || - (_vulkanReady && _commandStream.HasUnblockedPending) || - blockedRetryWait == 0 || - HasReadyPresentationLocked()) - { - return; - } + commandThread.ConsumerIdle = true; + } - var waitMilliseconds = gpuWorkInFlight ? 1 : 8; - if (blockedRetryWait is { } retryWait) + try + { + lock (_gate) { - waitMilliseconds = Math.Min(waitMilliseconds, retryWait); - } + if (_closed || + Volatile.Read(ref _presenterCloseRequested) || + _relay.HasPendingCommands || + (commandThread is not null ? commandThread.HasWork : _vulkanReady && _commandStream.HasPending) || + HasReadyPresentationLocked()) + { + return; + } - var waitPhase = _commandStream.HasPending - ? RenderPhaseProfile.Phase.IdleBlockedCommands - : _pendingGuestImagePresentations.Count > 0 || _pendingVideoPresentations.Count > 0 || - (_latestPresentation is { } latest && latest.Sequence != _presentedSequence) - ? RenderPhaseProfile.Phase.IdlePendingPresentation - : RenderPhaseProfile.Phase.IdleNoQueuedWork; - using var waitProfile = RenderPhaseProfile.MeasureDetail(waitPhase); - SubmissionFlowProfile.Record(SubmissionFlowProfile.EventKind.WaitStarted, - detail: waitMilliseconds); - var signaled = System.Threading.Monitor.Wait(_gate, waitMilliseconds); - // The result describes the monitor wait, not the arrival of runnable guest work. - SubmissionFlowProfile.Record(signaled ? SubmissionFlowProfile.EventKind.WaitSignaled - : SubmissionFlowProfile.EventKind.WaitTimedOut, detail: (int)waitPhase); + var waitMilliseconds = gpuWorkInFlight ? 1 : 8; + + var waitPhase = _commandStream.HasPending + ? RenderPhaseProfile.Phase.IdleBlockedCommands + : _pendingGuestImagePresentations.Count > 0 || _pendingVideoPresentations.Count > 0 || + (_latestPresentation is { } latest && latest.Sequence != _presentedSequence) + ? RenderPhaseProfile.Phase.IdlePendingPresentation + : RenderPhaseProfile.Phase.IdleNoQueuedWork; + using var waitProfile = RenderPhaseProfile.MeasureDetail(waitPhase); + SubmissionFlowProfile.Record(SubmissionFlowProfile.EventKind.WaitStarted, + detail: waitMilliseconds); + var signaled = System.Threading.Monitor.Wait(_gate, waitMilliseconds); + // The result describes the monitor wait, not the arrival of runnable guest work. + SubmissionFlowProfile.Record(signaled ? SubmissionFlowProfile.EventKind.WaitSignaled + : SubmissionFlowProfile.EventKind.WaitTimedOut, detail: (int)waitPhase); + } + } + finally + { + if (commandThread is not null) + { + commandThread.ConsumerIdle = false; + } } } @@ -117,9 +129,16 @@ internal static unsafe partial class VulkanVideoPresenter return; } - using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay)) + if (_commandThread is not null) { - _relay.RunPendingCommands(); + RunRelayCommandsWithPrefetchHeld(); + } + else + { + using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay)) + { + _relay.RunPendingCommands(); + } } if (_deviceLost) { @@ -388,6 +407,8 @@ internal static unsafe partial class VulkanVideoPresenter _commandBuffer, PipelineBindPoint.Graphics, _barycentricPipeline); + SharpEmu.Libs.Gpu.Vulkan.GraphicsDynamicStateEpoch.Advance(); + SharpEmu.Libs.Gpu.Vulkan.NativeBindingEpoch.Advance(); _vk.CmdDraw(_commandBuffer, 3, 1, 0, 0); _vk.CmdEndRenderPass(_commandBuffer); waitStage = PipelineStageFlags.ColorAttachmentOutputBit; diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs index 799601f..e602ca2 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs @@ -12,6 +12,7 @@ using SharpEmu.Libs.Gpu.GpuCommands.Registers; using SharpEmu.Libs.Gpu.Images; using SharpEmu.Libs.Gpu.Pipelines; using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Vulkan; using SharpEmu.Libs.Gpu.Scheduling; using SharpEmu.Libs.Gpu.Vulkan; using Silk.NET.Core; @@ -280,14 +281,21 @@ internal static unsafe partial class VulkanVideoPresenter _boundGraphicsPipeline = null; } + // The acquisition below reads the attachment back into its result and keeps no reference to it, + // so one render-thread instance serves every draw. + private ColorAttachment? _colorAttachmentScratch; + ColorAttachmentAcquisition IRenderHost.AcquireColorAttachment(in ColorTargetState target) { - var attachment = new ColorAttachment - { - ImageIdentifier = target.Image, - Request = target.Resolution.Request, - Resolution = target.Resolution, - }; + var attachment = _colorAttachmentScratch ??= new ColorAttachment(); + attachment.ImageIdentifier = target.Image; + attachment.Request = target.Resolution.Request; + attachment.Resolution = target.Resolution; + attachment.Image = null!; + attachment.View = default; + attachment.Layout = default; + attachment.Clear = false; + attachment.ClearValue = default; AcquireColorAttachment(attachment); return new ColorAttachmentAcquisition( attachment.ImageIdentifier, @@ -373,6 +381,11 @@ internal static unsafe partial class VulkanVideoPresenter { if (bindings.IsEmpty) return; var command = BeginBatchedGuestCommands(); + if (VertexBuffersAlreadyBound(command, bindings)) + { + return; + } + var buffers = stackalloc VkBuffer[bindings.Length]; var offsets = stackalloc ulong[bindings.Length]; for (var index = 0; index < bindings.Length; index++) @@ -384,8 +397,14 @@ internal static unsafe partial class VulkanVideoPresenter _vk.CmdBindVertexBuffers(command, 0, (uint)bindings.Length, buffers, offsets); } - public void BindIndexBuffer(BufferBinding binding, IndexType type) => - _vk.CmdBindIndexBuffer(BeginBatchedGuestCommands(), new VkBuffer(binding.Handle), binding.Offset, type); + public void BindIndexBuffer(BufferBinding binding, IndexType type) + { + var command = BeginBatchedGuestCommands(); + if (!IndexBufferAlreadyBound(command, new VkBuffer(binding.Handle), binding.Offset, type)) + { + _vk.CmdBindIndexBuffer(command, new VkBuffer(binding.Handle), binding.Offset, type); + } + } public IResourcePreparation BeginPreparation() { @@ -399,6 +418,11 @@ internal static unsafe partial class VulkanVideoPresenter if (_batchDrawCount >= (_renderingActive ? DrawsPerBatchInRenderPass : DrawsPerBatch)) { + if (GpuSyncProfile.Enabled) + { + GpuSyncProfile.CountBatchFlush(); + } + EndRendering(); FlushBatchedGuestCommands(); } @@ -644,29 +668,80 @@ internal static unsafe partial class VulkanVideoPresenter _vk.CmdClearDepthStencilImage(command, image.Backing.Handle, ImageLayout.TransferDstOptimal, &value, 1, &vkRange); } + // The dynamic state the last guest draw set, and where it holds: the command buffer and its + // tick (a new command buffer starts without state), and no other graphics pipeline since. + private DynamicDrawState _lastDynamicState; + private CommandBuffer _lastDynamicStateBuffer; + private ulong _lastDynamicStateTick; + private long _lastDynamicStateEpoch = -1; + public void SetDynamicState(in DynamicDrawState state) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.DrawDynamicStateRecording); var command = BeginBatchedGuestCommands(); - var viewport = new Viewport(state.ViewportX, state.ViewportY, state.ViewportWidth, state.ViewportHeight, state.ViewportMinDepth, state.ViewportMaxDepth); - _vk.CmdSetViewport(command, 0, 1, &viewport); - var scissor = new Rect2D( - new Offset2D(state.Scissor.Left, state.Scissor.Top), - new Extent2D((uint)(state.Scissor.Right - state.Scissor.Left), (uint)(state.Scissor.Bottom - state.Scissor.Top))); - _vk.CmdSetScissor(command, 0, 1, &scissor); - _vk.CmdSetLineWidth(command, state.LineWidth); - var blendConstants = stackalloc float[4] { state.BlendRed, state.BlendGreen, state.BlendBlue, state.BlendAlpha }; - _vk.CmdSetBlendConstants(command, blendConstants); - _vk.CmdSetDepthTestEnable(command, state.DepthTestEnabled); - _vk.CmdSetDepthWriteEnable(command, state.DepthWriteEnabled); - _vk.CmdSetDepthCompareOp(command, state.DepthCompare); - _vk.CmdSetDepthBiasEnable(command, state.DepthBiasEnabled); - if (state.DepthBiasEnabled) + // Every guest pipeline keeps these states dynamic, so they persist across its binds; + // only what changed since the last draw in this command buffer is set again. + var known = command.Handle == _lastDynamicStateBuffer.Handle && _scheduler.CurrentTick == _lastDynamicStateTick && + GraphicsDynamicStateEpoch.Value == _lastDynamicStateEpoch; + ref readonly var last = ref _lastDynamicState; + if (!known || state.ViewportX != last.ViewportX || state.ViewportY != last.ViewportY || + state.ViewportWidth != last.ViewportWidth || state.ViewportHeight != last.ViewportHeight || + state.ViewportMinDepth != last.ViewportMinDepth || state.ViewportMaxDepth != last.ViewportMaxDepth) + { + var viewport = new Viewport(state.ViewportX, state.ViewportY, state.ViewportWidth, state.ViewportHeight, state.ViewportMinDepth, state.ViewportMaxDepth); + _vk.CmdSetViewport(command, 0, 1, &viewport); + } + + if (!known || state.Scissor != last.Scissor) + { + var scissor = new Rect2D( + new Offset2D(state.Scissor.Left, state.Scissor.Top), + new Extent2D((uint)(state.Scissor.Right - state.Scissor.Left), (uint)(state.Scissor.Bottom - state.Scissor.Top))); + _vk.CmdSetScissor(command, 0, 1, &scissor); + } + + if (!known || state.LineWidth != last.LineWidth) + { + _vk.CmdSetLineWidth(command, state.LineWidth); + } + + if (!known || state.BlendRed != last.BlendRed || state.BlendGreen != last.BlendGreen || + state.BlendBlue != last.BlendBlue || state.BlendAlpha != last.BlendAlpha) + { + var blendConstants = stackalloc float[4] { state.BlendRed, state.BlendGreen, state.BlendBlue, state.BlendAlpha }; + _vk.CmdSetBlendConstants(command, blendConstants); + } + + if (!known || state.DepthTestEnabled != last.DepthTestEnabled) + { + _vk.CmdSetDepthTestEnable(command, state.DepthTestEnabled); + } + + if (!known || state.DepthWriteEnabled != last.DepthWriteEnabled) + { + _vk.CmdSetDepthWriteEnable(command, state.DepthWriteEnabled); + } + + if (!known || state.DepthCompare != last.DepthCompare) + { + _vk.CmdSetDepthCompareOp(command, state.DepthCompare); + } + + if (!known || state.DepthBiasEnabled != last.DepthBiasEnabled) + { + _vk.CmdSetDepthBiasEnable(command, state.DepthBiasEnabled); + } + + // The factors only matter while the bias is on; they are set when it turns on or changes. + if (state.DepthBiasEnabled && (!known || !last.DepthBiasEnabled || + state.DepthBiasConstantFactor != last.DepthBiasConstantFactor || state.DepthBiasClamp != last.DepthBiasClamp || + state.DepthBiasSlopeFactor != last.DepthBiasSlopeFactor)) { _vk.CmdSetDepthBias(command, state.DepthBiasConstantFactor, _supportsDepthBiasClamp ? state.DepthBiasClamp : 0f, state.DepthBiasSlopeFactor); } - if (state.StencilTestEnabled) + if (state.StencilTestEnabled && (!known || !last.StencilTestEnabled || + state.FrontStencil != last.FrontStencil || state.BackStencil != last.BackStencil)) { _vk.CmdSetStencilCompareMask(command, StencilFaceFlags.FaceFrontBit, state.FrontStencil.CompareMask); _vk.CmdSetStencilCompareMask(command, StencilFaceFlags.FaceBackBit, state.BackStencil.CompareMask); @@ -686,6 +761,11 @@ internal static unsafe partial class VulkanVideoPresenter colorWriteEnable.CmdSetColorWriteEnable(command, state.ColorWriteCount, enables); } + + _lastDynamicState = state; + _lastDynamicStateBuffer = command; + _lastDynamicStateTick = _scheduler.CurrentTick; + _lastDynamicStateEpoch = GraphicsDynamicStateEpoch.Value; } @@ -787,12 +867,17 @@ internal static unsafe partial class VulkanVideoPresenter } EndRendering(); - var command = BeginBatchedGuestCommands(); - _commandBuffer = command; - if (DeferGlobalBarriers) + if (DeferGlobalBarriers && BatchBarriers) { // A guest cache flush inside this scope may be deferred to its end; this keeps // everything recorded before the scope ordered before its draws either way. + RequestGuestMemoryBarrier(GlobalMemoryBarrier); + } + + var command = BeginBatchedGuestCommands(); + _commandBuffer = command; + if (DeferGlobalBarriers && !BatchBarriers) + { RecordGlobalBarrier(command); } @@ -863,6 +948,11 @@ internal static unsafe partial class VulkanVideoPresenter }; _vk.CmdBeginRendering(command, &rendering); _renderingScopesBegun++; + if (GpuSyncProfile.Enabled) + { + GpuSyncProfile.CountRenderingScope(); + } + _renderingActive = true; _renderingState = state; } @@ -912,19 +1002,19 @@ internal static unsafe partial class VulkanVideoPresenter public void BindPipeline(PipelineBindPoint bindPoint, in PipelineHandle pipeline) { var entry = RequirePipelineEntry(in pipeline); - var command = BeginBatchedGuestCommands(); + var command = BeginBatchedGuestStateCommands(); if (bindPoint == PipelineBindPoint.Graphics) { _boundGraphicsPipeline = entry; if (!entry.RectangleList) { - _vk.CmdBindPipeline(command, bindPoint, entry.Pipeline); + BindPipelineShadowed(command, bindPoint, entry.Pipeline); } return; } - _vk.CmdBindPipeline(command, bindPoint, entry.Pipeline); + BindPipelineShadowed(command, bindPoint, entry.Pipeline); _profileComputePipeline = entry.Id; } @@ -948,7 +1038,7 @@ internal static unsafe partial class VulkanVideoPresenter variant = CreateRenderPipeline(entry.Description!, strip ? PrimitiveTopology.TriangleStrip : PrimitiveTopology.TriangleList, entry.Layout); } - _vk.CmdBindPipeline(command, PipelineBindPoint.Graphics, variant); + BindPipelineShadowed(command, PipelineBindPoint.Graphics, variant); } private void BindNativeRectangleList(RenderPipelineEntry entry, CommandBuffer command) @@ -959,7 +1049,7 @@ internal static unsafe partial class VulkanVideoPresenter entry.Layout, PolygonMode.FillRectangleNV); } - _vk.CmdBindPipeline(command, PipelineBindPoint.Graphics, entry.RectangleVariant); + BindPipelineShadowed(command, PipelineBindPoint.Graphics, entry.RectangleVariant); } // Rectangle2D consumes three vertices and fills their projected bounding box. @@ -1075,9 +1165,24 @@ internal static unsafe partial class VulkanVideoPresenter return true; } + private static readonly bool BatchBarriers = + Environment.GetEnvironmentVariable("SHARPEMU_BATCH_BARRIERS") != "0"; + + private static readonly MemoryBarrierRequest GlobalMemoryBarrier = new( + PipelineStageFlags.AllCommandsBit, + PipelineStageFlags.AllCommandsBit, + AccessFlags.MemoryWriteBit, + AccessFlags.MemoryReadBit | AccessFlags.MemoryWriteBit); + private void RecordMemoryBarrier(PipelineStageFlags sourceStages, PipelineStageFlags destinationStages, AccessFlags sourceAccess, AccessFlags destinationAccess) { EndRendering(); + if (BatchBarriers) + { + RequestGuestMemoryBarrier(new MemoryBarrierRequest(sourceStages, destinationStages, sourceAccess, destinationAccess)); + return; + } + var command = BeginBatchedGuestCommands(); var barrier = new MemoryBarrier2 { @@ -1194,7 +1299,7 @@ internal static unsafe partial class VulkanVideoPresenter public bool TryFillDccMetadata(ulong address, ulong size, uint fillValue) { - if (!_imageCache.OverlapsDccMetadata(address, size)) + if (!_imageCache.OverlapsDccMetadata(address, size) && !IsMetadataShapedFill(address, size, fillValue)) { return false; } @@ -1203,6 +1308,16 @@ internal static unsafe partial class VulkanVideoPresenter return true; } + private bool IsMetadataShapedFill(ulong address, ulong size, uint fillValue) + { + const ulong metadataBlockSize = 0x1000; + const ulong maxMetadataFill = 0x100000; + return address != 0 && size != 0 && size <= maxMetadataFill && + address % metadataBlockSize == 0 && size % metadataBlockSize == 0 && + fillValue == (fillValue & 0xFF) * 0x01010101u && + !_imageCache.QueryRegion(address, size).ImageBytes; + } + public bool TryCopyWordsOnHost(ulong destination, ulong source, ulong sourceWords, ulong words) => _bufferCache.TryCopyWordsOnHost(destination, source, sourceWords, words); } diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs index dbdff71..a974b1f 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs @@ -74,6 +74,18 @@ internal static unsafe partial class VulkanVideoPresenter _ => WakeRenderThread()); } + private void RecordRequestedMemoryBarrier(nint handle, MemoryBarrierRequest request) + { + var barrier = new MemoryBarrier2 + { + SType = StructureType.MemoryBarrier2, + SrcAccessMask = VulkanSynchronization.Access(request.SourceAccess), + DstAccessMask = VulkanSynchronization.Access(request.DestinationAccess), + }; + VulkanSynchronization.PipelineBarrier(_vk, new CommandBuffer(handle), request.SourceStages, request.DestinationStages, + 0, 1, &barrier, 0, null, 0, null); + } + // Both stores share the manager's page guard and read guest memory through its address space. private static (GuestGpuMemory Memory, ICpuMemory Guest, IGuestBackedSpace Backing) RequireGuestMemory(string store) { @@ -120,6 +132,8 @@ internal static unsafe partial class VulkanVideoPresenter // Drain accepted submissions before closing the relay. // Cancel blocked submissions if a full retry cycle makes no progress. _commandStream.StopAccepting(); + // The command stream thread ends after its slice; the rest drains here. + _commandThread?.Stop(() => RunCommandStreamSlices(long.MaxValue)); var outcome = _commandStream.DrainForShutdown(cancelBlockedOnNoProgress: true); FlushBatchedGuestCommands(); Console.Error.WriteLine( @@ -165,6 +179,16 @@ internal static unsafe partial class VulkanVideoPresenter return new CommandBuffer(_scheduler.Current.Handle); } + private RecordingBuffer CurrentRecording() + { + if (!_scheduler.Active) + { + _scheduler.Begin(_submissionContext); + } + + return _scheduler.Current; + } + private void BindSubmissionContext(VulkanGuestQueueIdentity queue) { _submissionContext.QueueName = queue.Name; diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs index d063eae..06665c1 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs @@ -163,7 +163,9 @@ internal static unsafe partial class VulkanVideoPresenter { if (!ShaderProgramCache.TryCompilePrewarm( record, code, compiler, sharedInt64Atomics, execGuardElision, - _supportsShaderSignedZeroInfNanPreserveFloat32, out var compiled, out var layout, out var error)) + _supportsShaderSignedZeroInfNanPreserveFloat32, out var compiled, out var layout, out var error, + ((IShaderPipelineHost)this).ComputeWave64SubgroupNative, + ((IShaderPipelineHost)this).ComputeRunsOn32LaneSubgroups(record.Info))) { NoteShaderPrewarmFailure(record, error); return; @@ -181,7 +183,7 @@ internal static unsafe partial class VulkanVideoPresenter setLayout = CreateDescriptorSetLayout(bindings, out _, out _); pipelineLayout = CreatePipelineLayout(setLayout, ShaderStageFlags.ComputeBit); module = CreateShaderModule(payload); - var pipeline = CompileComputePipeline(_vk, _device, _pipelineCache, module, pipelineLayout); + var pipeline = CompileComputePipeline(_vk, _device, _pipelineCache, module, pipelineLayout, ComputeRequiredSubgroupSize(record.Info)); _vk.DestroyPipeline(_device, pipeline, null); Interlocked.Increment(ref _shaderPrewarmCompiled); } diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs index bce01f0..cda8ecc 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs @@ -51,40 +51,9 @@ public static partial class Gen5SpirvTranslator return false; } - var value = GetRawSource(instruction, 0); - if (_subgroupInvocationIdInput != 0) - { - if (_emulateWave64) - { - value = BroadcastFirstWave64Active(value); - } - else - { - // SPIR-V's BroadcastFirst uses the first host-active - // invocation. Guest EXEC is modeled as data, so obtain the - // guest-active mask explicitly and broadcast from its first - // set lane instead. This also updates the private SGPR copy - // for lanes that are currently disabled and may be restored - // by a later saveexec sequence. - var activeLanes = _module.AddInstruction( - SpirvOp.GroupNonUniformBallot, - _uvec4Type, - UInt(3), - Load(_boolType, _exec)); - var activeLow = _module.AddInstruction( - SpirvOp.CompositeExtract, - _uintType, - activeLanes, - 0); - var firstActiveLane = Ext(73, _uintType, activeLow); - value = _module.AddInstruction( - SpirvOp.GroupNonUniformBroadcast, - _uintType, - UInt(3), - value, - firstActiveLane); - } - } + var value = _pairedLanes + ? PairedReadFirstLane(instruction) + : BroadcastFirstWaveActive(GetRawSource(instruction, 0)); StoreS(instruction.Destinations[0].Value, value); return true; @@ -2773,7 +2742,7 @@ public static partial class Gen5SpirvTranslator if (instruction.Opcode.EndsWith("SaveexecB32", StringComparison.Ordinal)) { - var oldExec64 = BooleanToWaveMask(Load(_boolType, _exec)); + var oldExec64 = CurrentExecMask(); var oldExec = _module.AddInstruction( SpirvOp.UConvert, _uintType, @@ -3422,9 +3391,7 @@ public static partial class Gen5SpirvTranslator var left = GetRawSource64(instruction, 0); if (instruction.Opcode.EndsWith("SaveexecB64", StringComparison.Ordinal)) { - var oldExec = _emulateWave64 && _subgroupInvocationIdInput != 0 - ? LoadS64(126) - : BooleanToWaveMask(Load(_boolType, _exec)); + var oldExec = CurrentExecMask(); var notLeft = _module.AddInstruction(SpirvOp.Not, _ulongType, left); var newExec = instruction.Opcode switch { @@ -3493,12 +3460,7 @@ public static partial class Gen5SpirvTranslator return false; } - if (!_emulateWave64 && _waveLaneCount != 64) - { - newExec = BitwiseAnd64( - newExec, - _module.Constant64(_ulongType, 0xFFFF_FFFFUL)); - } + newExec = NormalizeExecMask(newExec); StoreS64(destination, oldExec); StoreS64(126, newExec); @@ -4387,6 +4349,12 @@ public static partial class Gen5SpirvTranslator private void StoreS64(uint register, uint value) { + if (_pairedLanes && _inLanePass) + { + StorePairedLaneMaskWord(register, value); + return; + } + StoreS( register, _module.AddInstruction(SpirvOp.UConvert, _uintType, value)); @@ -4958,52 +4926,6 @@ public static partial class Gen5SpirvTranslator private uint GuestLaneSelect(uint selector) => BitwiseAnd(selector, UInt(_waveLaneCount == 64 ? 63u : RdnaWaveLaneCount - 1)); - // An emulated wave64 spans two host subgroups, so a subgroup broadcast cannot reach - // the other half. The selected guest lane publishes through workgroup scratch, like - // BroadcastFirstWave64Active. - private uint BroadcastWave64Lane(uint value, uint lane) => - ExchangeWave64Value(_module.AddInstruction(SpirvOp.IEqual, _boolType, GuestWaveLane(), lane), value); - - private uint BroadcastFirstWave64Active(uint value) - { - var lane = GuestWaveLane(); - var upperHalf = ShiftRightLogical(lane, UInt(5)); - var activeInHalf = OwnHalfBallot(Load(_boolType, _exec)); - var halfHasActive = IsNotZero(activeInHalf); - var firstInHalf = _module.AddInstruction( - SpirvOp.Select, - _uintType, - halfHasActive, - Ext(73, _uintType, activeInHalf), - UInt(0)); - var halfBase = BitwiseAnd(Load(_uintType, _subgroupInvocationIdInput), UInt(~31u)); - var halfValue = _module.AddInstruction( - SpirvOp.GroupNonUniformBroadcast, - _uintType, - UInt(3), - value, - IAdd(halfBase, firstInHalf)); - var exchange = BeginWave64Exchange(); - EmitConditional(IsHalfWaveLeader(lane), () => - { - Store(Wave64ExchangePointer(exchange, upperHalf), halfValue); - Store( - Wave64ExchangePointer(exchange, IAdd(UInt(2), upperHalf)), - _module.AddInstruction(SpirvOp.Select, _uintType, halfHasActive, UInt(1), UInt(0))); - }); - EmitWave64Barrier(); - var lowerValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))); - var upperValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(1))); - var lowerActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(2)))); - var upperActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(3)))); - return _module.AddInstruction( - SpirvOp.Select, - _uintType, - _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, LogicalNot(lowerActive), upperActive), - upperValue, - lowerValue); - } - private void StoreCarryOut( Gen5ShaderInstruction instruction, uint carry) @@ -5035,23 +4957,24 @@ public static partial class Gen5SpirvTranslator } var destination = instruction.Destinations[0].Value; - var sourceValue = GetRawSource(instruction, 0); var selectedLane = BitwiseAnd(GetRawSource(instruction, 1), UInt(LaneSelectMask)); + if (_pairedLanes) + { + StoreS(destination, PairedReadLane(instruction, selectedLane)); + return true; + } - if (_emulateWave64) + var sourceValue = GetRawSource(instruction, 0); + + if (_stage == Gen5SpirvStage.Compute && _waveLowering is EmulatedWave64Lowering) { // The selected guest lane can belong to another host subgroup. // Read it even when the guest execution mask disables that lane. - StoreS(destination, BroadcastWave64Lane(sourceValue, selectedLane)); + StoreS(destination, BroadcastWaveLane(sourceValue, selectedLane)); } else if (_subgroupInvocationIdInput != 0 && _stage == Gen5SpirvStage.Compute) { - var broadcast = _module.AddInstruction( - SpirvOp.GroupNonUniformBroadcast, - _uintType, - UInt(3), - sourceValue, - selectedLane); + var broadcast = BroadcastWaveLane(sourceValue, selectedLane); StoreS(destination, ReadLaneSpillSlot(instruction, selectedLane, broadcast)); } else diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.CommonWave.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.CommonWave.cs new file mode 100644 index 0000000..b843e5c --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.CommonWave.cs @@ -0,0 +1,27 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private uint BroadcastFirstWaveActive(uint value) => + _waveLowering.BroadcastFirstActive(value); + + private uint BroadcastWaveLane(uint value, uint lane) => + _waveLowering.BroadcastLane(value, lane); + + private uint BroadcastWaveValue(uint writer, uint value, uint lane) => + _waveLowering.BroadcastValue(writer, value, lane); + + private uint CurrentExecMask() => + _waveLowering.CurrentExecMask(); + + private uint NormalizeExecMask(uint mask) => + _waveLowering.NormalizeExecMask(mask); + } +} + + diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.EmulatedWave64Lowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.EmulatedWave64Lowering.cs new file mode 100644 index 0000000..7361761 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.EmulatedWave64Lowering.cs @@ -0,0 +1,578 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private sealed class EmulatedWave64Lowering : IWaveLowering + { + private const uint ExchangeSlotCount = 4; + private const uint ExchangeDwordCount = ExchangeSlotCount * 2; + private const uint GuestWaveLaneCount = 64; + private static readonly bool HalfWaveMasks = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_HALF_MASKS"), + "0", + StringComparison.Ordinal); + + private readonly CompilationContext _context; + private uint _exchange; + private uint _exchangeElementPointer; + private uint _exchangeOffset; + private uint _exchangeParity; + // Several guest waves share one workgroup. Each gets its own exchange area plus an + // arrival counter, and its two halves meet at that counter instead of at a workgroup + // barrier, which other waves (possibly in other blocks) would never reach. + private readonly uint _waveCount; + private readonly bool _multiWave; + private readonly uint _waveStride; + private uint _rendezvousEpoch; + private Ir.Gen5Wave64HalfMaskPlan? _halfMaskPlan; + private bool _halfMaskPlanBuilt; + private bool _synchronizeSharedMemory; + private SharedMemoryPhase _sharedMemoryPhase; + private uint? _emittingPc; + + private enum SharedMemoryPhase { None, Read, Write } + + public EmulatedWave64Lowering(CompilationContext context) + { + _context = context; + var threads = (ulong)context._localSizeX * context._localSizeY * context._localSizeZ; + _waveCount = (uint)Math.Max((threads + GuestWaveLaneCount - 1) / GuestWaveLaneCount, 1); + _multiWave = _waveCount > 1; + _waveStride = _multiWave ? ExchangeDwordCount + 1 : ExchangeDwordCount; + } + + private uint ExchangeTotalDwords => _waveCount * _waveStride; + + public bool RequiresCrossSubgroupSynchronization => true; + + private bool UsesExchange => _context.UsesSubgroupOperations(); + + public uint ReserveLdsDwords(uint guestDwordCount) => + UsesExchange && guestDwordCount < LdsDwordCount + ? guestDwordCount + ExchangeTotalDwords + : guestDwordCount; + + public void DeclareResources() + { + var context = _context; + if (!UsesExchange) + { + return; + } + + if (context._lds != 0) + { + // Metal exposes 32 KiB of threadgroup memory on the Apple + // GPUs we target. Reuse the tail of guest LDS so the + // cross-subgroup bridge stays within that limit. + _exchange = context._lds; + _exchangeElementPointer = context._ldsElementPointer; + _exchangeOffset = context._ldsDwordMask + 1 < LdsDwordCount + ? context._ldsDwordMask + 1 + : LdsDwordCount - ExchangeTotalDwords; + return; + } + + var exchangeArrayType = context._module.TypeArray(context._uintType, ExchangeTotalDwords); + _exchangeElementPointer = context._module.TypePointer(SpirvStorageClass.Workgroup, context._uintType); + _exchange = context._module.AddGlobalVariable( + context._module.TypePointer(SpirvStorageClass.Workgroup, exchangeArrayType), + SpirvStorageClass.Workgroup); + _exchangeOffset = 0; + context._module.AddName(_exchange, "wave64Exchange"); + context._interfaces.Add(_exchange); + } + + public void DeclareFunctionResources() + { + if (!UsesExchange) + { + return; + } + + _exchangeParity = _context.DeclareFunctionUint( + _context._module.Constant(_context._uintType, 0)); + _context._module.AddName(_exchangeParity, "wave64ExchangeParity"); + if (_multiWave) + { + _rendezvousEpoch = _context.DeclareFunctionUint( + _context._module.Constant(_context._uintType, 0)); + _context._module.AddName(_rendezvousEpoch, "wave64RendezvousEpoch"); + } + } + + public void InitializeMasks() + { + var context = _context; + if (context._subgroupInvocationIdInput != 0) + { + context.StoreS64(106, BothHalvesOfOwnBallot(context._module.ConstantBool(false))); + context.StoreS64(126, BothHalvesOfOwnBallot(context._module.ConstantBool(true))); + InitializeRendezvous(); + return; + } + + context.StoreS64(106, context._module.Constant64(context._ulongType, 0)); + context.StoreS64(126, context._module.Constant64(context._ulongType, 1)); + } + + public void BeginBlock() + { + _synchronizeSharedMemory = HalfMaskPlan() is null; + _sharedMemoryPhase = SharedMemoryPhase.None; + } + + public void BeforeInstruction(Gen5ShaderInstruction instruction) + { + var context = _context; + _emittingPc = instruction.Pc; + if (HalfMaskPlan() is { } plan) + { + if (plan.ExactPairsBefore.TryGetValue(instruction.Pc, out var exactPairs)) + { + EmitExactWaveMasks(exactPairs); + } + + if (plan.SharedMemoryBarriersBefore.Contains(instruction.Pc)) + { + Barrier(); + } + } + + if (!_synchronizeSharedMemory) + { + return; + } + + var nextPhase = instruction.Control is Gen5DataShareControl { Gds: false } + ? instruction.Opcode.StartsWith("DsRead", StringComparison.Ordinal) ? SharedMemoryPhase.Read + : instruction.Opcode.StartsWith("DsWrite", StringComparison.Ordinal) ? SharedMemoryPhase.Write + : SharedMemoryPhase.None + : SharedMemoryPhase.None; + if (instruction.Opcode == "SBarrier") + { + _sharedMemoryPhase = SharedMemoryPhase.None; + } + if (nextPhase != SharedMemoryPhase.None) + { + if (_sharedMemoryPhase != SharedMemoryPhase.None && _sharedMemoryPhase != nextPhase) + { + Barrier(); + } + _sharedMemoryPhase = nextPhase; + } + } + + public void EndBlock() + { + if (_synchronizeSharedMemory && _sharedMemoryPhase != SharedMemoryPhase.None) + { + Barrier(); + } + } + + public uint SubgroupAny(uint condition) => + _context._subgroupInvocationIdInput == 0 + ? condition + : _context.IsNotZero64(BooleanToWaveMask(condition)); + + public uint CurrentLaneBit() + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.Constant64(context._ulongType, 1); + } + + return context.ShiftLeftLogical64( + context._module.Constant64(context._ulongType, 1), + context._module.AddInstruction( + SpirvOp.UConvert, + context._ulongType, + context.GuestWaveLane())); + } + + public uint BooleanToWaveMask(uint condition) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context.BooleanToLaneMask(condition); + } + + var lane = context.GuestWaveLane(); + var owned = OwnHalfBallot(condition); + var exchange = BeginExchange(); + context.EmitConditional(IsHalfWaveLeader(lane), () => + context.Store(ExchangePointer(exchange, context.ShiftRightLogical(lane, context.UInt(5))), owned)); + Barrier(); + return context.Pair64( + context.Load(context._uintType, ExchangePointer(exchange, context.UInt(0))), + context.Load(context._uintType, ExchangePointer(exchange, context.UInt(1)))); + } + + public uint BroadcastFirstActive(uint value) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return value; + } + + var lane = context.GuestWaveLane(); + var upperHalf = context.ShiftRightLogical(lane, context.UInt(5)); + var activeInHalf = OwnHalfBallot(context.Load(context._boolType, context._exec)); + var halfHasActive = context.IsNotZero(activeInHalf); + var firstInHalf = context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + halfHasActive, + context.Ext(73, context._uintType, activeInHalf), + context.UInt(0)); + var halfBase = context.BitwiseAnd( + context.Load(context._uintType, context._subgroupInvocationIdInput), + context.UInt(~31u)); + var halfValue = context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + context._uintType, + context.UInt(3), + value, + context.IAdd(halfBase, firstInHalf)); + var exchange = BeginExchange(); + context.EmitConditional(IsHalfWaveLeader(lane), () => + { + context.Store(ExchangePointer(exchange, upperHalf), halfValue); + context.Store( + ExchangePointer(exchange, context.IAdd(context.UInt(2), upperHalf)), + context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + halfHasActive, + context.UInt(1), + context.UInt(0))); + }); + Barrier(); + var lowerValue = context.Load(context._uintType, ExchangePointer(exchange, context.UInt(0))); + var upperValue = context.Load(context._uintType, ExchangePointer(exchange, context.UInt(1))); + var lowerActive = context.IsNotZero(context.Load(context._uintType, ExchangePointer(exchange, context.UInt(2)))); + var upperActive = context.IsNotZero(context.Load(context._uintType, ExchangePointer(exchange, context.UInt(3)))); + return context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + context._module.AddInstruction( + SpirvOp.LogicalAnd, + context._boolType, + context.LogicalNot(lowerActive), + upperActive), + upperValue, + lowerValue); + } + + public uint BroadcastLane(uint value, uint lane) => + ExchangeValue( + _context._module.AddInstruction( + SpirvOp.IEqual, + _context._boolType, + _context.GuestWaveLane(), + lane), + value); + + public uint BroadcastValue(uint writer, uint value, uint lane) => ExchangeValue(writer, value); + + public uint WaveMaskAny(uint register, uint laneActive) => + _context._subgroupInvocationIdInput != 0 + ? _context.IsNotZero64(_context.LoadS64(register)) + : SubgroupAny(_context.Load(_context._boolType, laneActive)); + + public uint CurrentExecMask() => + _context._subgroupInvocationIdInput != 0 + ? _context.LoadS64(126) + : BooleanToWaveMask(_context.Load(_context._boolType, _context._exec)); + + public uint NormalizeExecMask(uint mask) => mask; + + public void StoreWaveMask(uint register, uint condition) + { + var context = _context; + if (context._waveLaneCount == 32) + { + context.StoreS(register, context.Narrow(BooleanToWaveMask(condition))); + return; + } + + if (context._subgroupInvocationIdInput != 0 && + _emittingPc is { } pc && + HalfMaskPlan() is { } plan && + plan.HalfMaskWrites.TryGetValue(pc, out var halfRegister) && + halfRegister == register) + { + context.StoreS64(register, BooleanToHalfWaveMask(condition)); + return; + } + + context.StoreS64(register, BooleanToWaveMask(condition)); + } + + private uint OwnHalfBallot(uint condition) + { + var context = _context; + var ballot = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + condition); + return context._module.AddInstruction( + SpirvOp.VectorExtractDynamic, + context._uintType, + ballot, + context.ShiftRightLogical( + context.Load(context._uintType, context._subgroupInvocationIdInput), + context.UInt(5))); + } + + private uint BooleanToHalfWaveMask(uint condition) + { + var context = _context; + var owned = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, OwnHalfBallot(condition)); + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + context._module.AddInstruction( + SpirvOp.UGreaterThanEqual, + context._boolType, + context.GuestWaveLane(), + context.UInt(32)), + context.ShiftLeftLogical64(owned, context._module.Constant64(context._ulongType, 32)), + owned); + } + + private uint BothHalvesOfOwnBallot(uint condition) + { + var context = _context; + var owned = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, OwnHalfBallot(condition)); + return context.BitwiseOr64( + owned, + context.ShiftLeftLogical64(owned, context._module.Constant64(context._ulongType, 32))); + } + + private uint IsHalfWaveLeader(uint lane) => + _context._module.AddInstruction( + SpirvOp.IEqual, + _context._boolType, + _context.BitwiseAnd(lane, _context.UInt(31)), + _context.UInt(0)); + + private uint BeginExchange() + { + var context = _context; + var parity = context.Load(context._uintType, _exchangeParity); + context.Store(_exchangeParity, context.BitwiseXor(parity, context.UInt(1))); + return context.IAdd( + WaveExchangeBase(), + context._module.AddInstruction( + SpirvOp.IMul, + context._uintType, + parity, + context.UInt(ExchangeSlotCount))); + } + + private uint WaveExchangeBase() + { + var context = _context; + if (!_multiWave) + { + return context.UInt(_exchangeOffset); + } + + var wave = context.ShiftRightLogical( + context.Load(context._uintType, context._localInvocationIndexInput), + context.UInt(6)); + return context.IAdd( + context.UInt(_exchangeOffset), + context._module.AddInstruction(SpirvOp.IMul, context._uintType, wave, context.UInt(_waveStride))); + } + + // Zeroes each wave's arrival counter. Runs once at entry, where the whole workgroup is + // converged, so the workgroup barrier is safe here. + private void InitializeRendezvous() + { + if (!_multiWave || !UsesExchange) + { + return; + } + + var context = _context; + context.EmitConditional( + context._module.AddInstruction( + SpirvOp.IEqual, + context._boolType, + context.GuestWaveLane(), + context.UInt(0)), + () => context.Store( + ExchangePointer(WaveExchangeBase(), context.UInt(ExchangeDwordCount)), + context.UInt(0))); + WorkgroupBarrier(); + } + + // Both halves of the wave arrive, then wait until the counter shows two arrivals per + // rendezvous so far. The halves of one wave are co-resident in the workgroup, so the + // wait always ends. + private void WaveRendezvous() + { + var context = _context; + var counter = ExchangePointer(WaveExchangeBase(), context.UInt(ExchangeDwordCount)); + context.EmitConditional( + IsHalfWaveLeader(context.GuestWaveLane()), + () => context._module.AddInstruction( + SpirvOp.AtomicIAdd, + context._uintType, + counter, + context.UInt(2), + context.UInt(0x108), + context.UInt(1))); + var epoch = context.IAdd(context.Load(context._uintType, _rendezvousEpoch), context.UInt(1)); + context.Store(_rendezvousEpoch, epoch); + var target = context._module.AddInstruction(SpirvOp.IMul, context._uintType, epoch, context.UInt(2)); + + var header = context._module.AllocateId(); + var continueLabel = context._module.AllocateId(); + var mergeLabel = context._module.AllocateId(); + context._module.AddStatement(SpirvOp.Branch, header); + context._module.AddLabel(header); + var arrived = context._module.AddInstruction( + SpirvOp.AtomicLoad, + context._uintType, + counter, + context.UInt(2), + context.UInt(0x102)); + var done = context._module.AddInstruction( + SpirvOp.UGreaterThanEqual, + context._boolType, + arrived, + target); + context._module.AddStatement(SpirvOp.LoopMerge, mergeLabel, continueLabel, 0); + context._module.AddStatement(SpirvOp.BranchConditional, done, mergeLabel, continueLabel); + context._module.AddLabel(continueLabel); + context._module.AddStatement(SpirvOp.Branch, header); + context._module.AddLabel(mergeLabel); + } + + private uint ExchangePointer(uint exchange, uint slot) => + _context._module.AddInstruction( + SpirvOp.AccessChain, + _exchangeElementPointer, + _exchange, + _context.IAdd(exchange, slot)); + + private uint ExchangeValue(uint writer, uint value) + { + var context = _context; + var exchange = BeginExchange(); + context.EmitConditional(writer, () => context.Store(ExchangePointer(exchange, context.UInt(0)), value)); + Barrier(); + return context.Load(context._uintType, ExchangePointer(exchange, context.UInt(0))); + } + + private void EmitExactWaveMasks(IReadOnlyList pairs) + { + var context = _context; + var lane = context.GuestWaveLane(); + var upperHalf = context.ShiftRightLogical(lane, context.UInt(5)); + var lowerHalf = context._module.AddInstruction(SpirvOp.IEqual, context._boolType, upperHalf, context.UInt(0)); + const int pairsPerExchange = (int)ExchangeSlotCount / 2; + for (var start = 0; start < pairs.Count; start += pairsPerExchange) + { + var count = Math.Min(pairsPerExchange, pairs.Count - start); + var owned = new uint[count]; + for (var index = 0; index < count; index++) + { + var pair = pairs[start + index]; + owned[index] = context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + lowerHalf, + context.LoadS(pair), + context.LoadS(pair + 1)); + } + + var exchange = BeginExchange(); + context.EmitConditional(IsHalfWaveLeader(lane), () => + { + for (var index = 0; index < count; index++) + { + context.Store( + ExchangePointer(exchange, context.IAdd(context.UInt((uint)index * 2), upperHalf)), + owned[index]); + } + }); + Barrier(); + for (var index = 0; index < count; index++) + { + var pair = pairs[start + index]; + context.StoreS(pair, context.Load(context._uintType, ExchangePointer(exchange, context.UInt((uint)index * 2)))); + context.StoreS(pair + 1, context.Load(context._uintType, ExchangePointer(exchange, context.UInt((uint)index * 2 + 1)))); + } + } + } + + private Ir.Gen5Wave64HalfMaskPlan? HalfMaskPlan() + { + if (_halfMaskPlanBuilt) + { + return _halfMaskPlan; + } + + _halfMaskPlanBuilt = true; + var context = _context; + if (HalfWaveMasks && UsesExchange && context._subgroupInvocationIdInput != 0) + { + var sharedFlatPcs = context._request.Memory.Entries + .Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) + .Select(static memory => memory.Pc) + .ToHashSet(); + _halfMaskPlan = Ir.Gen5Wave64HalfMaskAnalysis.Analyze(context._request.Program, sharedFlatPcs); + } + + return _halfMaskPlan; + } + + private void Barrier() + { + if (_multiWave) + { + // The rendezvous needs the exchange area, declared only with subgroup operations. + if (UsesExchange) + { + WaveRendezvous(); + } + + return; + } + + WorkgroupBarrier(); + } + + private void WorkgroupBarrier() + { + var workgroup = _context.UInt(2); + _context._module.AddStatement( + SpirvOp.ControlBarrier, + workgroup, + workgroup, + _context.UInt(0x108)); + } + } + } +} + + + + + diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.HalfWave64Lowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.HalfWave64Lowering.cs new file mode 100644 index 0000000..95fdc62 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.HalfWave64Lowering.cs @@ -0,0 +1,158 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + // Independent halves duplicate scalar work; a lane-local proof is not a performance proof. + private static readonly bool LaneLocalWave64Enabled = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_LANE_LOCAL"), "1", StringComparison.Ordinal); + + private static bool CanRunHalfWaves(ShaderCompileRequest request, int[] physicalAxisOfLogical) + { + if (!LaneLocalWave64Enabled || + physicalAxisOfLogical is not [0, 1, 2]) + { + return false; + } + + var sharedFlatPcs = request.Memory.Entries + .Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) + .Select(static memory => memory.Pc) + .ToHashSet(); + return Ir.Gen5WaveLaneLocalAnalysis.IsLaneLocal(request.Program, sharedFlatPcs, out _); + } + + private sealed class HalfWave64Lowering : IWaveLowering + { + private readonly CompilationContext _context; + + public HalfWave64Lowering(CompilationContext context) + { + _context = context; + } + + public bool RequiresCrossSubgroupSynchronization => false; + + public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount; + + public void DeclareResources() { } + public void DeclareFunctionResources() { } + public void BeginBlock() { } + public void BeforeInstruction(Gen5ShaderInstruction instruction) { } + public void EndBlock() { } + + public void InitializeMasks() + { + var context = _context; + if (context._subgroupInvocationIdInput != 0) + { + StoreWaveMask(106, context._module.ConstantBool(false)); + StoreWaveMask(126, context._module.ConstantBool(true)); + return; + } + + context.StoreS64(106, context._module.Constant64(context._ulongType, 0)); + context.StoreS64(126, context._module.Constant64(context._ulongType, 1)); + } + + public uint BooleanToWaveMask(uint condition) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + condition, + CurrentLaneBit(), + context._module.Constant64(context._ulongType, 0)); + } + + var ballot = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + condition); + var low = context._module.AddInstruction(SpirvOp.CompositeExtract, context._uintType, ballot, 0); + var widened = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, low); + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + context._module.AddInstruction( + SpirvOp.UGreaterThanEqual, + context._boolType, + context.GuestWaveLane(), + context.UInt(32)), + context.ShiftLeftLogical64(widened, context._module.Constant64(context._ulongType, 32)), + widened); + } + + public uint SubgroupAny(uint condition) => + _context._subgroupInvocationIdInput == 0 + ? condition + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformAny, + _context._boolType, + _context.UInt(3), + condition); + + public uint CurrentLaneBit() + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.Constant64(context._ulongType, 1); + } + + return context.ShiftLeftLogical64( + context._module.Constant64(context._ulongType, 1), + context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, context.GuestWaveLane())); + } + + public uint BroadcastFirstActive(uint value) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return value; + } + + var activeLanes = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + context.Load(context._boolType, context._exec)); + var activeLow = context._module.AddInstruction(SpirvOp.CompositeExtract, context._uintType, activeLanes, 0); + return context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + context._uintType, + context.UInt(3), + value, + context.Ext(73, context._uintType, activeLow)); + } + + public uint BroadcastLane(uint value, uint lane) => + throw new InvalidOperationException("a half-wave program reads no other lane"); + + public uint BroadcastValue(uint writer, uint value, uint lane) => + throw new InvalidOperationException("a half-wave program writes no other lane"); + + public uint WaveMaskAny(uint register, uint laneActive) => + SubgroupAny(_context.Load(_context._boolType, laneActive)); + + public uint CurrentExecMask() => + BooleanToWaveMask(_context.Load(_context._boolType, _context._exec)); + + public uint NormalizeExecMask(uint mask) => mask; + + public void StoreWaveMask(uint register, uint condition) => + _context.StoreS64(register, BooleanToWaveMask(condition)); + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave32Lowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave32Lowering.cs new file mode 100644 index 0000000..3491e4f --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave32Lowering.cs @@ -0,0 +1,156 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + // A guest Wave32 maps directly to one 32-lane host subgroup. This is the + // former implicit/default lowering, made explicit so every guest wave size + // has a concrete lowering strategy. + private sealed class NativeWave32Lowering : IWaveLowering + { + private readonly CompilationContext _context; + + public NativeWave32Lowering(CompilationContext context) + { + _context = context; + } + + public bool RequiresCrossSubgroupSynchronization => false; + + public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount; + + public void DeclareResources() { } + public void DeclareFunctionResources() { } + public void BeginBlock() { } + public void BeforeInstruction(Gen5ShaderInstruction instruction) { } + public void EndBlock() { } + + public void InitializeMasks() + { + var context = _context; + if (context._subgroupInvocationIdInput != 0) + { + StoreWaveMask(106, context._module.ConstantBool(false)); + StoreWaveMask(126, context._module.ConstantBool(true)); + return; + } + + context.StoreS64(106, context._module.Constant64(context._ulongType, 0)); + context.StoreS64(126, context._module.Constant64(context._ulongType, 1)); + } + + public uint BooleanToWaveMask(uint condition) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + condition, + CurrentLaneBit(), + context._module.Constant64(context._ulongType, 0)); + } + + var ballot = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + condition); + var low = context._module.AddInstruction( + SpirvOp.CompositeExtract, + context._uintType, + ballot, + 0); + return context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, low); + } + + public uint SubgroupAny(uint condition) => + _context._subgroupInvocationIdInput == 0 + ? condition + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformAny, + _context._boolType, + _context.UInt(3), + condition); + + public uint CurrentLaneBit() + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.Constant64(context._ulongType, 1); + } + + var shifted = context.ShiftLeftLogical64( + context._module.Constant64(context._ulongType, 1), + context._module.AddInstruction( + SpirvOp.UConvert, + context._ulongType, + context.GuestWaveLane())); + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + context.IsCurrentLaneInRdnaWave(), + shifted, + context._module.Constant64(context._ulongType, 0)); + } + + public uint BroadcastFirstActive(uint value) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return value; + } + + var activeLanes = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + context.Load(context._boolType, context._exec)); + var activeLow = context._module.AddInstruction( + SpirvOp.CompositeExtract, + context._uintType, + activeLanes, + 0); + var firstActiveLane = context.Ext(73, context._uintType, activeLow); + return context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + context._uintType, + context.UInt(3), + value, + firstActiveLane); + } + + public uint BroadcastLane(uint value, uint lane) => + _context._subgroupInvocationIdInput == 0 + ? value + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + _context._uintType, + _context.UInt(3), + value, + lane); + + public uint BroadcastValue(uint writer, uint value, uint lane) => + _context.ShuffleLane(value, lane); + + public uint WaveMaskAny(uint register, uint laneActive) => + SubgroupAny(_context.Load(_context._boolType, laneActive)); + + public uint CurrentExecMask() => + BooleanToWaveMask(_context.Load(_context._boolType, _context._exec)); + + public uint NormalizeExecMask(uint mask) => + _context.BitwiseAnd64(mask, _context._module.Constant64(_context._ulongType, 0xFFFF_FFFFUL)); + + public void StoreWaveMask(uint register, uint condition) => + _context.StoreS(register, _context.Narrow(BooleanToWaveMask(condition))); + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave64Lowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave64Lowering.cs new file mode 100644 index 0000000..d4daa6e --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave64Lowering.cs @@ -0,0 +1,160 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + // One guest wave is one 64-lane host subgroup (ballot.x = lanes 0..31, ballot.y = lanes + // 32..63). It has no workgroup exchange, LDS reservation, half-mask plan or + // cross-subgroup barrier state. + private sealed class NativeWave64Lowering : IWaveLowering + { + private readonly CompilationContext _context; + + public NativeWave64Lowering(CompilationContext context) + { + _context = context; + } + + public bool RequiresCrossSubgroupSynchronization => false; + + public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount; + + public void DeclareResources() { } + public void DeclareFunctionResources() { } + public void BeginBlock() { } + public void BeforeInstruction(Gen5ShaderInstruction instruction) { } + public void EndBlock() { } + + public void InitializeMasks() + { + var context = _context; + if (context._subgroupInvocationIdInput != 0) + { + StoreWaveMask(106, context._module.ConstantBool(false)); + StoreWaveMask(126, context._module.ConstantBool(true)); + } + else + { + context.StoreS64(106, context._module.Constant64(context._ulongType, 0)); + context.StoreS64(126, context._module.Constant64(context._ulongType, 1)); + } + } + + public uint BooleanToWaveMask(uint condition) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + condition, + CurrentLaneBit(), + context._module.Constant64(context._ulongType, 0)); + } + + return BallotToMask(condition); + } + + // The host subgroup is at least 64 wide: lanes 0..31 are ballot.x and lanes 32..63 ballot.y. + private uint BallotToMask(uint condition) + { + var context = _context; + var ballot = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + condition); + var low = context._module.AddInstruction(SpirvOp.CompositeExtract, context._uintType, ballot, 0); + var high = context._module.AddInstruction(SpirvOp.CompositeExtract, context._uintType, ballot, 1); + return context.BitwiseOr64( + context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, low), + context.ShiftLeftLogical64( + context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, high), + context._module.Constant64(context._ulongType, 32))); + } + + public uint SubgroupAny(uint condition) => + _context._subgroupInvocationIdInput == 0 + ? condition + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformAny, + _context._boolType, + _context.UInt(3), + condition); + + public uint CurrentLaneBit() + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.Constant64(context._ulongType, 1); + } + + var shifted = context.ShiftLeftLogical64( + context._module.Constant64(context._ulongType, 1), + context._module.AddInstruction( + SpirvOp.UConvert, + context._ulongType, + context.GuestWaveLane())); + return shifted; + } + + public uint BroadcastFirstActive(uint value) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return value; + } + + var activeLanes = BallotToMask(context.Load(context._boolType, context._exec)); + var activeLow = context._module.AddInstruction(SpirvOp.UConvert, context._uintType, activeLanes); + var activeHigh = context._module.AddInstruction( + SpirvOp.UConvert, + context._uintType, + context.ShiftRightLogical64(activeLanes, context._module.Constant64(context._ulongType, 32))); + var firstActiveLane = context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + context._module.AddInstruction(SpirvOp.IEqual, context._boolType, activeLow, context.UInt(0)), + context.IAdd(context.Ext(73, context._uintType, activeHigh), context.UInt(32)), + context.Ext(73, context._uintType, activeLow)); + return context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + context._uintType, + context.UInt(3), + value, + firstActiveLane); + } + + public uint BroadcastLane(uint value, uint lane) => + _context._subgroupInvocationIdInput == 0 + ? value + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + _context._uintType, + _context.UInt(3), + value, + lane); + + public uint BroadcastValue(uint writer, uint value, uint lane) => + _context.ShuffleLane(value, lane); + + public uint WaveMaskAny(uint register, uint laneActive) => + SubgroupAny(_context.Load(_context._boolType, laneActive)); + + public uint CurrentExecMask() => + BooleanToWaveMask(_context.Load(_context._boolType, _context._exec)); + + public uint NormalizeExecMask(uint mask) => mask; + + public void StoreWaveMask(uint register, uint condition) => + _context.StoreS64(register, BooleanToWaveMask(condition)); + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.PairedLanes.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.PairedLanes.cs new file mode 100644 index 0000000..6e325ac --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.PairedLanes.cs @@ -0,0 +1,464 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private const uint PairedHalfLaneCount = 32; + + private static readonly bool PairedWave64Enabled = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_PAIRED"), "0", StringComparison.Ordinal); + + private readonly bool _pairedLanes; + private int _laneHalf; + private bool _inLanePass; + private readonly Dictionary _upperVectorRegisterVariables = new(); + private readonly uint[] _packedHalfRegistersByHalf = new uint[2]; + private uint _lowerExec; + private uint _upperExec; + private uint _lowerVcc; + private uint _upperVcc; + private uint _lowerScratch; + private uint _upperScratch; + private uint _subgroupIdInput; + private uint _pairedCounterResult; + private Dictionary? _scalarSnapshot; + + private void SelectLaneHalf(int half) + { + if (!_pairedLanes) + { + return; + } + + _packedHalfRegistersByHalf[_laneHalf] = _packedHalfRegisters; + _laneHalf = half; + _packedHalfRegisters = _packedHalfRegistersByHalf[half]; + _exec = half == 0 ? _lowerExec : _upperExec; + _vcc = half == 0 ? _lowerVcc : _upperVcc; + if (_lowerScratch != 0) + { + _scratch = half == 0 ? _lowerScratch : _upperScratch; + } + } + + private void DeclarePairedLaneFlags(uint lowerVcc, uint lowerExec, Func flag) + { + _lowerVcc = lowerVcc; + _lowerExec = lowerExec; + if (!_pairedLanes) + { + return; + } + + _upperVcc = flag(false); + _upperExec = flag(true); + _interfaces.Add(_upperVcc); + _interfaces.Add(_upperExec); + _module.AddName(_upperVcc, "vccUpperHalf"); + _module.AddName(_upperExec, "execUpperHalf"); + } + + private void DeclarePairedScratch(uint arrayPointer, uint arrayType) + { + _lowerScratch = _scratch; + if (!_pairedLanes) + { + return; + } + + _upperScratch = _module.AddGlobalVariable( + arrayPointer, + SpirvStorageClass.Private, + _module.ConstantNull(arrayType)); + _interfaces.Add(_upperScratch); + _module.AddName(_upperScratch, "scratchUpperHalf"); + } + + private void DeclarePairedLaneInputs() + { + if (!_pairedLanes) + { + return; + } + + _subgroupIdInput = _module.AddGlobalVariable( + _module.TypePointer(SpirvStorageClass.Input, _uintType), + SpirvStorageClass.Input); + _module.AddDecoration(_subgroupIdInput, SpirvDecoration.BuiltIn, (uint)SpirvBuiltIn.SubgroupId); + _interfaces.Add(_subgroupIdInput); + } + + private uint PairedHostThreadCount() + { + var guestThreads = checked(_localSizeX * _localSizeY * _localSizeZ); + return (guestThreads + 63) / 64 * PairedHalfLaneCount; + } + + private Dictionary CurrentVectorRegisterVariables() => + _pairedLanes && _laneHalf == 1 ? _upperVectorRegisterVariables : _vectorRegisterVariables; + + private string CurrentVectorRegisterPrefix() => + _pairedLanes && _laneHalf == 1 ? "vu" : "v"; + + private uint PairedSubgroupLane() => Load(_uintType, _subgroupInvocationIdInput); + + private uint PairedGuestLane() => + _laneHalf == 0 ? PairedSubgroupLane() : IAdd(PairedSubgroupLane(), UInt(PairedHalfLaneCount)); + + private uint HalfBallot(uint condition) => + _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + _module.AddInstruction(SpirvOp.GroupNonUniformBallot, _uvec4Type, UInt(3), condition), + 0); + + private uint SubgroupBroadcast(uint value, uint lane) => + _module.AddInstruction(SpirvOp.GroupNonUniformBroadcast, _uintType, UInt(3), value, lane); + + private uint ExecHalfBallot(int half) + { + var current = _laneHalf; + SelectLaneHalf(half); + var ballot = HalfBallot(Load(_boolType, _exec)); + SelectLaneHalf(current); + return ballot; + } + + private uint VectorSourceOfHalf(Gen5ShaderInstruction instruction, int sourceIndex, int half) + { + var current = _laneHalf; + SelectLaneHalf(half); + var value = GetRawSource(instruction, sourceIndex); + SelectLaneHalf(current); + return value; + } + + private static bool IsPairedLaneInstruction(Gen5ShaderInstruction instruction) => + instruction.Opcode is not ("VReadlaneB32" or "VReadfirstlaneB32" or "DsAppend" or "DsConsume") && + instruction.Encoding is + Gen5ShaderEncoding.Mubuf or + Gen5ShaderEncoding.Mtbuf or + Gen5ShaderEncoding.Vop1 or + Gen5ShaderEncoding.Vop2 or + Gen5ShaderEncoding.Vopc or + Gen5ShaderEncoding.Vop3 or + Gen5ShaderEncoding.Vop3p or + Gen5ShaderEncoding.Vintrp or + Gen5ShaderEncoding.Ds or + Gen5ShaderEncoding.Flat or + Gen5ShaderEncoding.Mimg or + Gen5ShaderEncoding.Exp; + + private bool TryEmitPairedLaneInstruction(Gen5ShaderInstruction instruction, out string error) + { + _scalarSnapshot = SnapshotScalarSources(instruction); + _inLanePass = true; + try + { + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + if (!TryEmitInstruction(instruction, out error)) + { + return false; + } + } + + error = string.Empty; + return true; + } + finally + { + _inLanePass = false; + _scalarSnapshot = null; + SelectLaneHalf(0); + } + } + + private Dictionary? SnapshotScalarSources(Gen5ShaderInstruction instruction) + { + Dictionary? snapshot = null; + foreach (var operand in instruction.Sources) + { + if (operand.Kind != Gen5OperandKind.ScalarRegister) + { + continue; + } + + for (var register = operand.Value; register <= operand.Value + 1 && register < ScalarRegisterCount; register++) + { + if (register == 125) + { + continue; + } + + snapshot ??= new Dictionary(); + if (!snapshot.ContainsKey(register)) + { + snapshot[register] = Load(_uintType, ScalarPointer(register)); + } + } + } + + return snapshot; + } + + private void UpdateLaneMaskFlag(uint lowRegister) + { + if (!_pairedLanes || _inLanePass) + { + Store(lowRegister == 106 ? _vcc : _exec, IsLaneSetInMaskRegisters(lowRegister)); + return; + } + + var current = _laneHalf; + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + Store(lowRegister == 106 ? _vcc : _exec, IsLaneSetInMaskRegisters(lowRegister)); + } + + SelectLaneHalf(current); + } + + private void StorePairedLaneMaskWord(uint register, uint value) + { + var word = _laneHalf == 0 + ? Narrow(value) + : Narrow(ShiftRightLogical64(value, _module.Constant64(_ulongType, 32))); + StoreS(register + (uint)_laneHalf, word); + } + + private void EmitPairedComputeInitialState() + { + var workGroupId = Load(_uvec3Type, _workGroupIdInput); + var waveBase = _module.AddInstruction( + SpirvOp.IMul, + _uintType, + Load(_uintType, _subgroupIdInput), + UInt(2 * PairedHalfLaneCount)); + var guestThreads = checked(_localSizeX * _localSizeY * _localSizeZ); + var limits = new uint[3]; + var groups = new uint[3]; + for (uint component = 0; component < 3; component++) + { + limits[component] = ComputeThreadLimit(component); + groups[component] = _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + workGroupId, + (uint)_physicalAxisOfLogical[component]); + } + + var active = new uint[2]; + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + var index = IAdd(waveBase, PairedGuestLane()); + var inBounds = _module.AddInstruction(SpirvOp.ULessThan, _boolType, index, UInt(guestThreads)); + var row = _module.AddInstruction(SpirvOp.UDiv, _uintType, index, UInt(_localSizeX)); + var local = new[] + { + _module.AddInstruction(SpirvOp.UMod, _uintType, index, UInt(_localSizeX)), + _module.AddInstruction(SpirvOp.UMod, _uintType, row, UInt(_localSizeY)), + _module.AddInstruction(SpirvOp.UDiv, _uintType, row, UInt(_localSizeY)), + }; + var sizes = new[] { _localSizeX, _localSizeY, _localSizeZ }; + for (uint component = 0; component < 3; component++) + { + StoreV(component, local[component], guardWithExec: false); + var global = IAdd( + _module.AddInstruction(SpirvOp.IMul, _uintType, groups[component], UInt(sizes[component])), + local[component]); + inBounds = LogicalAnd( + inBounds, + _module.AddInstruction(SpirvOp.ULessThan, _boolType, global, limits[component])); + } + + active[half] = inBounds; + } + + SelectLaneHalf(0); + var lowerActive = HalfBallot(active[0]); + var upperActive = HalfBallot(active[1]); + StoreS(126, lowerActive); + StoreS(127, upperActive); + Store(_programActive, _module.AddInstruction(SpirvOp.LogicalOr, _boolType, active[0], active[1])); + + if (_request.ComputeSystemRegisters is { } registers) + { + StoreComputeSystemRegister(registers.WorkGroupXRegister, workGroupId, (uint)_physicalAxisOfLogical[0]); + StoreComputeSystemRegister(registers.WorkGroupYRegister, workGroupId, (uint)_physicalAxisOfLogical[1]); + StoreComputeSystemRegister(registers.WorkGroupZRegister, workGroupId, (uint)_physicalAxisOfLogical[2]); + if (registers.ThreadGroupSizeRegister is { } sizeRegister) + { + StoreS(sizeRegister, UInt(guestThreads)); + } + } + } + + private uint PairedReadFirstLane(Gen5ShaderInstruction instruction) + { + var lowerActive = ExecHalfBallot(0); + var upperActive = ExecHalfBallot(1); + var lowerValue = VectorSourceOfHalf(instruction, 0, 0); + var upperValue = VectorSourceOfHalf(instruction, 0, 1); + var lowerHasActive = IsNotZero(lowerActive); + var useLower = LogicalOr(lowerHasActive, LogicalNot(IsNotZero(upperActive))); + var lowerLane = _module.AddInstruction( + SpirvOp.Select, + _uintType, + lowerHasActive, + Ext(73, _uintType, lowerActive), + UInt(0)); + var lane = BitwiseAnd( + _module.AddInstruction( + SpirvOp.Select, + _uintType, + useLower, + lowerLane, + Ext(73, _uintType, upperActive)), + UInt(PairedHalfLaneCount - 1)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + useLower, + SubgroupBroadcast(lowerValue, lane), + SubgroupBroadcast(upperValue, lane)); + } + + private uint PairedReadLane(Gen5ShaderInstruction instruction, uint selectedLane) + { + var lowerValue = VectorSourceOfHalf(instruction, 0, 0); + var upperValue = VectorSourceOfHalf(instruction, 0, 1); + var lane = BitwiseAnd(selectedLane, UInt(PairedHalfLaneCount - 1)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction(SpirvOp.ULessThan, _boolType, selectedLane, UInt(PairedHalfLaneCount)), + SubgroupBroadcast(lowerValue, lane), + SubgroupBroadcast(upperValue, lane)); + } + + private uint LogicalOr(uint left, uint right) => + _module.AddInstruction(SpirvOp.LogicalOr, _boolType, left, right); + + private void EmitPairedWaveCounter(uint destination, uint inBounds, Func update) + { + var lowerActive = ExecHalfBallot(0); + var upperActive = ExecHalfBallot(1); + var activeCount = IAdd( + _module.AddInstruction(SpirvOp.BitCount, _uintType, lowerActive), + _module.AddInstruction(SpirvOp.BitCount, _uintType, upperActive)); + var anyActive = IsNotZero(BitwiseOr(lowerActive, upperActive)); + var leaderLane = BitwiseAnd( + _module.AddInstruction( + SpirvOp.Select, + _uintType, + IsNotZero(lowerActive), + Ext(73, _uintType, lowerActive), + Ext(73, _uintType, upperActive)), + UInt(PairedHalfLaneCount - 1)); + if (_pairedCounterResult == 0) + { + _pairedCounterResult = DeclareFunctionUint(UInt(0)); + _module.AddName(_pairedCounterResult, "pairedCounterResult"); + } + + var valid = LogicalAnd(inBounds, anyActive); + var leader = LogicalAnd( + valid, + _module.AddInstruction(SpirvOp.IEqual, _boolType, PairedSubgroupLane(), leaderLane)); + EmitConditional(leader, () => Store(_pairedCounterResult, update(activeCount))); + var original = SubgroupBroadcast(Load(_uintType, _pairedCounterResult), leaderLane); + var result = _module.AddInstruction(SpirvOp.Select, _uintType, valid, original, UInt(0)); + var current = _laneHalf; + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + StoreV(destination, result); + } + + SelectLaneHalf(current); + } + + private sealed class PairedWave64Lowering : IWaveLowering + { + private readonly CompilationContext _context; + + public PairedWave64Lowering(CompilationContext context) + { + _context = context; + } + + public bool RequiresCrossSubgroupSynchronization => false; + + public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount; + + public void DeclareResources() { } + public void DeclareFunctionResources() { } + public void BeginBlock() { } + public void BeforeInstruction(Gen5ShaderInstruction instruction) { } + public void EndBlock() { } + + public void InitializeMasks() => + _context.StoreS64(106, _context._module.Constant64(_context._ulongType, 0)); + + public uint BooleanToWaveMask(uint condition) + { + var context = RequireLanePass(nameof(BooleanToWaveMask)); + var wide = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, context.HalfBallot(condition)); + return context._laneHalf == 0 + ? wide + : context.ShiftLeftLogical64(wide, context._module.Constant64(context._ulongType, 32)); + } + + public uint SubgroupAny(uint condition) + { + var context = RequireLanePass(nameof(SubgroupAny)); + return context._module.AddInstruction(SpirvOp.GroupNonUniformAny, context._boolType, context.UInt(3), condition); + } + + public uint CurrentLaneBit() => + _context.ShiftLeftLogical64( + _context._module.Constant64(_context._ulongType, 1), + _context._module.AddInstruction(SpirvOp.UConvert, _context._ulongType, _context.PairedGuestLane())); + + public uint BroadcastFirstActive(uint value) => + throw new InvalidOperationException("paired wave64 reads the first active lane from both halves"); + + public uint BroadcastLane(uint value, uint lane) => + throw new InvalidOperationException("paired wave64 reads a lane from both halves"); + + public uint BroadcastValue(uint writer, uint value, uint lane) => + throw new InvalidOperationException("paired wave64 broadcasts a wave counter from both halves"); + + public uint WaveMaskAny(uint register, uint laneActive) => + _context.IsNotZero64(_context.LoadS64(register)); + + public uint CurrentExecMask() => _context.LoadS64(126); + + public uint NormalizeExecMask(uint mask) => mask; + + public void StoreWaveMask(uint register, uint condition) + { + var context = RequireLanePass(nameof(StoreWaveMask)); + context.StoreS(register + (uint)context._laneHalf, context.HalfBallot(condition)); + } + + private CompilationContext RequireLanePass(string operation) + { + if (!_context._inLanePass) + { + throw new InvalidOperationException($"paired wave64 {operation} needs a lane half"); + } + + return _context; + } + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs index 9d682a8..ce3d1f4 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs @@ -68,6 +68,8 @@ public static partial class Gen5SpirvTranslator private readonly Dictionary _indirectKeyScratch = []; private uint _pushData; private uint _shaderData; + private uint _mipStatistics; + private uint _helperInvocationInput; private uint _flattenedTable; private uint _pageTable; private uint _faultBuffer; @@ -110,13 +112,43 @@ public static partial class Gen5SpirvTranslator _usesPixelValidMask = _stage == Gen5SpirvStage.Pixel && request.Program.Instructions.Any(static instruction => instruction.Control is Gen5ExportControl { ValidMask: true }); + _pixelExecExcludesHelpers = + _stage == Gen5SpirvStage.Pixel && PixelLiveExec && + request.Program.Instructions.Any(static instruction => instruction.Opcode is "DsAppend" or "DsConsume"); _enableGraphicsSubgroupOperations = _stage == Gen5SpirvStage.Compute || request.EnableGraphicsSubgroupOperations; _waveLaneCount = request.WaveSize == 64 ? 64u : 32u; _localSizeX = Math.Max(request.LocalSizeX, 1); _localSizeY = Math.Max(request.LocalSizeY, 1); _localSizeZ = Math.Max(request.LocalSizeZ, 1); _physicalAxisOfLogical = ComputeWorkgroupAxisOrder(_localSizeX, _localSizeY, _localSizeZ); - _emulateWave64 = _stage == Gen5SpirvStage.Compute && _waveLaneCount == 64 && (ulong)_localSizeX * _localSizeY * _localSizeZ == 64; + // A wave64 runs as one 64-lane host subgroup when the host pins it to one + // (HostWave64Supported). Hosts without 64-lane subgroups bridge the two 32-lane + // halves through workgroup memory. + if (_waveLaneCount == 32) + { + _waveLowering = new NativeWave32Lowering(this); + } + else if (request.HostWave64Supported) + { + _waveLowering = new NativeWave64Lowering(this); + _guestLaneFromSubgroup = true; + } + else if (_stage == Gen5SpirvStage.Compute && + request.HostComputeSubgroup32 && + CanRunHalfWaves(request, _physicalAxisOfLogical)) + { + _waveLowering = new HalfWave64Lowering(this); + } + else if (_stage == Gen5SpirvStage.Compute && request.HostComputeSubgroup32 && PairedWave64Enabled) + { + _pairedLanes = true; + _waveLowering = new PairedWave64Lowering(this); + } + else + { + _waveLowering = new EmulatedWave64Lowering(this); + } + _requiredVertexOutputCount = request.RequiredVertexOutputCount; _pixelInputEnable = request.PixelInputEnable; _pixelInputAddress = request.PixelInputAddress; @@ -174,6 +206,9 @@ public static partial class Gen5SpirvTranslator case DescriptorBindingKind.ShaderData: _shaderData = DeclareWordBlock("shaderData", number); break; + case DescriptorBindingKind.MipStatistics: + _mipStatistics = DeclareWordBlock("mipStatistics", number); + break; default: DeclareImageClass(binding, number); break; @@ -1590,11 +1625,127 @@ public static partial class Gen5SpirvTranslator imageClass.Dimension, imageInfo.ConversionFormat, imageInfo.ShaderSwizzle, - imageInfo.EmulatedCompareFunction); + imageInfo.EmulatedCompareFunction, + resourceIndex); dstSelect = imageInfo.ShaderSwizzle; return true; } + private const uint MipStatisticsEntries = 257; + private const uint MipStatisticsNoCounter = 0x8000; + + private void RecordMipStatistics(SpirvImageResource resource, uint imageObject, uint coordinates, uint? explicitLod, uint bias) + { + if (_mipStatistics == 0 || _stage != Gen5SpirvStage.Pixel || resource.ResourceIndex < 0 || + resource.IsStorage || resource.Multisampled || resource.Cube) + { + return; + } + + var field = LoadShaderDataDword(UInt(_request.Bindings.MipStatisticsDword + (uint)resource.ResourceIndex)); + uint lod; + if (explicitLod is { } value) + { + lod = value; + } + else + { + var queried = _module.AddInstruction( + SpirvOp.ImageQueryLod, + _vec2Type, + imageObject, + SpatialCoordinates(resource, coordinates)); + lod = _module.AddInstruction( + SpirvOp.FAdd, + _floatType, + _module.AddInstruction(SpirvOp.CompositeExtract, _floatType, queried, 1u), + bias); + } + + var counter = BitwiseAnd(field, UInt(0xFF)); + var baseLevel = _module.AddInstruction(SpirvOp.ConvertUToF, _floatType, BitwiseAnd(ShiftRightLogical(field, UInt(8)), UInt(0xF))); + var threshold = _module.AddInstruction( + SpirvOp.FMul, + _floatType, + _module.AddInstruction(SpirvOp.ConvertUToF, _floatType, BitwiseAnd(ShiftRightLogical(field, UInt(16)), UInt(0xFFF))), + Float(1f / 256f)); + var absolute = Ext(43, _floatType, _module.AddInstruction(SpirvOp.FAdd, _floatType, lod, baseLevel), Float(0f), Float(14f)); + var level = _module.AddInstruction(SpirvOp.ConvertFToU, _uintType, absolute); + var clamped = _module.AddInstruction(SpirvOp.FOrdLessThan, _boolType, absolute, threshold); + var record = LogicalAnd( + Load(_boolType, _exec), + _module.AddInstruction(SpirvOp.IEqual, _boolType, BitwiseAnd(field, UInt(MipStatisticsNoCounter)), UInt(0))); + record = LogicalAnd(record, _module.AddInstruction(SpirvOp.LogicalNot, _boolType, Load(_boolType, HelperInvocationInput()))); + var finestPointer = MipStatisticsPointer(counter); + var countPointer = MipStatisticsPointer(IAdd(counter, UInt(MipStatisticsEntries))); + EmitConditional(record, () => + { + if (!_request.MipStatisticsSubgroupReduction) + { + _module.AddInstruction(SpirvOp.AtomicUMin, _uintType, finestPointer, UInt(1), UInt(0), level); + EmitConditional(clamped, () => + _module.AddInstruction(SpirvOp.AtomicIAdd, _uintType, countPointer, UInt(1), UInt(0), UInt(1))); + return; + } + + _module.AddCapability(SpirvCapability.GroupNonUniform); + _module.AddCapability(SpirvCapability.GroupNonUniformBallot); + _module.AddCapability(SpirvCapability.GroupNonUniformArithmetic); + var finest = _module.AddInstruction(SpirvOp.GroupNonUniformUMin, _uintType, UInt(3), 0u, level); + var ballot = _module.AddInstruction(SpirvOp.GroupNonUniformBallot, _uvec4Type, UInt(3), clamped); + var count = _module.AddInstruction(SpirvOp.GroupNonUniformBallotBitCount, _uintType, UInt(3), 0u, ballot); + var leader = _module.AddInstruction(SpirvOp.GroupNonUniformElect, _boolType, UInt(3)); + EmitConditional(leader, () => + { + _module.AddInstruction(SpirvOp.AtomicUMin, _uintType, finestPointer, UInt(1), UInt(0), finest); + EmitConditional(IsNotZero(count), () => + _module.AddInstruction(SpirvOp.AtomicIAdd, _uintType, countPointer, UInt(1), UInt(0), count)); + }); + }); + } + + private uint MipStatisticsPointer(uint index) => + BlockWordPointer(_mipStatistics, index); + + private uint HelperInvocationInput() + { + if (_helperInvocationInput == 0) + { + _helperInvocationInput = _module.AddGlobalVariable( + _module.TypePointer(SpirvStorageClass.Input, _boolType), + SpirvStorageClass.Input); + _module.AddDecoration(_helperInvocationInput, SpirvDecoration.BuiltIn, (uint)SpirvBuiltIn.HelperInvocation); + _module.AddName(_helperInvocationInput, "helperInvocation"); + _interfaces.Add(_helperInvocationInput); + } + + return _helperInvocationInput; + } + + private uint SpatialCoordinates(SpirvImageResource resource, uint coordinates) + { + var spatial = ImageSpatialComponentCountOf(resource.Dimension); + if (!resource.Arrayed) + { + return coordinates; + } + + if (spatial == 1) + { + return _module.AddInstruction(SpirvOp.CompositeExtract, _floatType, coordinates, 0u); + } + + var components = new uint[spatial + 2]; + components[0] = coordinates; + components[1] = coordinates; + for (var index = 0u; index < spatial; index++) + { + components[index + 2] = index; + } + + return _module.AddInstruction(SpirvOp.VectorShuffle, _module.TypeVector(_floatType, spatial), components); + } + private static uint ImageSpatialComponentCountOf(SpirvImageDim dimension) => dimension switch { SpirvImageDim.Dim1D => 1u, @@ -1881,7 +2032,25 @@ public static partial class Gen5SpirvTranslator var index = GlobalDataShareIndex(baseAddress, offset); var inBounds = LogicalAnd(IsNotZero(sizeBytes), IsBlockWordInRange(_globalDataShare, index)); var destination = instruction.Destinations[0].Value; + if (_pairedLanes) + { + EmitPairedWaveCounter(destination, inBounds, count => EmitAtomic( + instruction.Opcode == "DsAppend" ? SpirvOp.AtomicIAdd : SpirvOp.AtomicISub, + _uintType, + BlockWordPointer(_globalDataShare, index), + scope: 1, + semantics: 0x48, + value: () => count, + comparator: () => UInt(0))); + return true; + } + var active = Load(_boolType, _exec); + if (_stage == Gen5SpirvStage.Pixel) + { + active = LogicalAnd(active, _module.AddInstruction(SpirvOp.LogicalNot, _boolType, Load(_boolType, HelperInvocationInput()))); + } + var activeMask = BooleanToWaveMask(active); var activeLow = Narrow(activeMask); var activeHigh = Narrow(ShiftRightLogical64(activeMask, ULong(32))); @@ -1909,21 +2078,7 @@ public static partial class Gen5SpirvTranslator }); var firstValue = LoadV(destination); - uint broadcast; - if (_emulateWave64) - { - broadcast = ExchangeWave64Value(isFirstActive, firstValue); - } - else if (_stage == Gen5SpirvStage.Compute || _enableGraphicsSubgroupOperations) - { - // Every active lane receives the first lane's old counter, in graphics stages too. - broadcast = _module.AddInstruction(SpirvOp.GroupNonUniformShuffle, _uintType, UInt(3), firstValue, firstLane); - } - else - { - // Without subgroup operations a graphics invocation is its own wave of one lane. - broadcast = firstValue; - } + var broadcast = BroadcastWaveValue(isFirstActive, firstValue, firstLane); var validResult = LogicalAnd(inBounds, IsNotZero64(activeMask)); StoreV(destination, _module.AddInstruction(SpirvOp.Select, _uintType, validResult, broadcast, UInt(0))); diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.WaveLowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.WaveLowering.cs new file mode 100644 index 0000000..a0d7296 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.WaveLowering.cs @@ -0,0 +1,34 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private interface IWaveLowering + { + bool RequiresCrossSubgroupSynchronization { get; } + + uint ReserveLdsDwords(uint guestDwordCount); + void DeclareResources(); + void DeclareFunctionResources(); + void InitializeMasks(); + void BeginBlock(); + void BeforeInstruction(Gen5ShaderInstruction instruction); + void EndBlock(); + uint BooleanToWaveMask(uint condition); + uint SubgroupAny(uint condition); + uint CurrentLaneBit(); + uint BroadcastFirstActive(uint value); + uint BroadcastLane(uint value, uint lane); + uint BroadcastValue(uint writer, uint value, uint lane); + uint WaveMaskAny(uint register, uint laneActive); + uint CurrentExecMask(); + uint NormalizeExecMask(uint mask); + void StoreWaveMask(uint register, uint condition); + } + } +} + diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs index 73a8a16..da1c589 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs @@ -17,8 +17,6 @@ public static partial class Gen5SpirvTranslator // so a smaller array is safe. private const uint PrivateLdsDwordCount = 2048; private const uint RdnaWaveLaneCount = 32; - private const uint Wave64ExchangeSlotCount = 4; - private const uint Wave64ExchangeDwordCount = Wave64ExchangeSlotCount * 2; internal static SpirvImageFormat DecodeStorageImageFormat( uint dataFormat, @@ -69,9 +67,14 @@ public static partial class Gen5SpirvTranslator private readonly Gen5SpirvStage _stage; private readonly IReadOnlyList _pixelOutputBindings; private readonly bool _usesPixelValidMask; + private readonly bool _pixelExecExcludesHelpers; + + private static readonly bool PixelLiveExec = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_PS_LIVE_EXEC"), "0", StringComparison.Ordinal); private readonly bool _enableGraphicsSubgroupOperations; private readonly uint _waveLaneCount; - private readonly bool _emulateWave64; + private readonly IWaveLowering _waveLowering; + private bool _guestLaneFromSubgroup; // Safety valve for the PC-dispatcher loop. Each iteration executes one // GCN basic block; a correctly-translated shader always reaches its @@ -195,12 +198,6 @@ public static partial class Gen5SpirvTranslator private uint _workGroupIdInput; private uint _pushConstantUintPointer; private uint _subgroupInvocationIdInput; - private uint _wave64Exchange; - private uint _wave64ExchangeElementPointer; - private uint _wave64ExchangeOffset; - private uint _wave64ExchangeParity; - private Ir.Gen5Wave64HalfMaskPlan? _halfMaskPlan; - private uint? _emittingPc; private uint _glsl; private enum ImageComponentKind @@ -231,7 +228,8 @@ public static partial class Gen5SpirvTranslator SpirvImageDim Dimension = SpirvImageDim.Dim2D, uint ConversionFormat = 0, uint ShaderSwizzle = 0, - int EmulatedCompareFunction = -1); + int EmulatedCompareFunction = -1, + int ResourceIndex = -1); private readonly record struct SpirvVertexInput( uint Variable, @@ -421,8 +419,8 @@ public static partial class Gen5SpirvTranslator var active = Load(_boolType, _programActive); if (_maxDispatcherSteps > 0) { - var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1)); - Store(_iterationGuard, steps); + var steps = IAdd(Load(_uintType, IterationGuard()), UInt(1)); + Store(IterationGuard(), steps); var withinLimit = _module.AddInstruction( SpirvOp.ULessThan, _boolType, @@ -538,12 +536,19 @@ public static partial class Gen5SpirvTranslator physicalSizes[_physicalAxisOfLogical[logical]] = logicalSizes[logical]; } - _module.AddExecutionMode( - main, - SpirvExecutionMode.LocalSize, - physicalSizes[0], - physicalSizes[1], - physicalSizes[2]); + if (_pairedLanes) + { + _module.AddExecutionMode(main, SpirvExecutionMode.LocalSize, PairedHostThreadCount(), 1, 1); + } + else + { + _module.AddExecutionMode( + main, + SpirvExecutionMode.LocalSize, + physicalSizes[0], + physicalSizes[1], + physicalSizes[2]); + } } var attributeCount = _stage == Gen5SpirvStage.Vertex @@ -625,7 +630,14 @@ public static partial class Gen5SpirvTranslator _scc = Flag(false); _vcc = Flag(false); _exec = Flag(true); - _reachedPixelExport = Flag(false); + DeclarePairedLaneFlags(_vcc, _exec, Flag); + // Only the title-shader trace reads it; elsewhere it would be a write-only Private + // global, and AMD's driver crashes linking stages that carry an unused one. + if (TraceTitleShaderState) + { + _reachedPixelExport = Flag(false); + } + if (_usesPixelValidMask) { _pixelValidMaskActive = Flag(true); @@ -635,22 +647,7 @@ public static partial class Gen5SpirvTranslator SpirvStorageClass.Private, _module.Constant(_uintType, 0)); _programActive = Flag(true); - if (UsesWave64Exchange()) - { - _wave64ExchangeParity = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, UInt(0)); - _interfaces.Add(_wave64ExchangeParity); - _module.AddName(_wave64ExchangeParity, "wave64ExchangeParity"); - } - if (_maxDispatcherSteps > 0) - { - _iterationGuard = _module.AddGlobalVariable( - _privateUintPointer, - SpirvStorageClass.Private, - _module.Constant(_uintType, 0)); - _interfaces.Add(_iterationGuard); - _module.AddName(_iterationGuard, "pcGuard"); - } - + _waveLowering.DeclareFunctionResources(); foreach (var slot in FindLaneSpillSlots()) { var variable = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, _module.Constant(_uintType, 0)); @@ -668,7 +665,11 @@ public static partial class Gen5SpirvTranslator _interfaces.Add(_scc); _interfaces.Add(_vcc); _interfaces.Add(_exec); - _interfaces.Add(_reachedPixelExport); + if (_reachedPixelExport != 0) + { + _interfaces.Add(_reachedPixelExport); + } + if (_pixelValidMaskActive != 0) { _interfaces.Add(_pixelValidMaskActive); @@ -681,7 +682,7 @@ public static partial class Gen5SpirvTranslator DeclareLayoutBindings(); DeclareScratch(); DeclareLds(); - DeclareWave64Scratch(); + _waveLowering.DeclareResources(); DeclareStageInterface(); return; } @@ -707,45 +708,9 @@ public static partial class Gen5SpirvTranslator _module.ConstantNull(arrayType)); _interfaces.Add(_scratch); _module.AddName(_scratch, "scratch"); + DeclarePairedScratch(arrayPointer, arrayType); } - private void DeclareWave64Scratch() - { - if (!UsesWave64Exchange()) - { - return; - } - - // Metal exposes 32 KiB of threadgroup memory on the Apple GPUs we - // target. Some PS5 compute shaders legitimately request all of it, - // so allocating another workgroup variable for the wave64 bridge - // makes pipeline creation fail. Reuse the final dwords of the - // existing LDS allocation in that case. The translator already - // bounds guest LDS accesses to this fixed allocation; keeping the - // bridge inside it preserves the host limit and still provides the - // cross-subgroup rendezvous needed to model one 64-lane guest wave. - if (_lds != 0) - { - _wave64Exchange = _lds; - _wave64ExchangeElementPointer = _ldsElementPointer; - _wave64ExchangeOffset = _ldsDwordMask + 1 < LdsDwordCount - ? _ldsDwordMask + 1 - : LdsDwordCount - Wave64ExchangeDwordCount; - return; - } - - var exchangeArrayType = _module.TypeArray(_uintType, Wave64ExchangeDwordCount); - _wave64ExchangeElementPointer = _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); - _wave64Exchange = _module.AddGlobalVariable( - _module.TypePointer(SpirvStorageClass.Workgroup, exchangeArrayType), - SpirvStorageClass.Workgroup); - _wave64ExchangeOffset = 0; - _module.AddName(_wave64Exchange, "wave64Exchange"); - _interfaces.Add(_wave64Exchange); - } - - private bool UsesWave64Exchange() => _emulateWave64 && UsesSubgroupOperations(); - private uint ComputeLdsGuestDwordCount() { var declared = _request.LocalDataShareDwords; @@ -776,9 +741,7 @@ public static partial class Gen5SpirvTranslator ? ComputeLdsGuestDwordCount() : PrivateLdsDwordCount; _ldsDwordMask = dwordCount - 1; - var arrayDwordCount = UsesWave64Exchange() && dwordCount < LdsDwordCount - ? dwordCount + Wave64ExchangeDwordCount - : dwordCount; + var arrayDwordCount = _waveLowering.ReserveLdsDwords(dwordCount); var ldsArrayType = _module.TypeArray(_uintType, arrayDwordCount); var ldsPointer = _module.TypePointer(storageClass, ldsArrayType); @@ -878,6 +841,8 @@ public static partial class Gen5SpirvTranslator (uint)SpirvBuiltIn.LocalInvocationIndex); _interfaces.Add(_localInvocationIndexInput); } + + DeclarePairedLaneInputs(); } if (_stage == Gen5SpirvStage.Vertex) @@ -1176,31 +1141,19 @@ public static partial class Gen5SpirvTranslator } Store(_scc, _module.ConstantBool(false)); - Store(_reachedPixelExport, _module.ConstantBool(false)); + if (_reachedPixelExport != 0) + { + Store(_reachedPixelExport, _module.ConstantBool(false)); + } + if (_pixelValidMaskActive != 0) { Store(_pixelValidMaskActive, _module.ConstantBool(true)); } - if (_subgroupInvocationIdInput != 0 && _emulateWave64) + _waveLowering.InitializeMasks(); + if (_pixelExecExcludesHelpers) { - StoreS64(106, BothHalvesOfOwnBallot(_module.ConstantBool(false))); - StoreS64(126, BothHalvesOfOwnBallot(_module.ConstantBool(true))); - } - else if (_subgroupInvocationIdInput != 0) - { - StoreWaveMask(106, _module.ConstantBool(false)); - StoreWaveMask(126, _module.ConstantBool(true)); - } - else - { - // Graphics stages emulate one logical wave lane. Keep the - // guest-visible VCC/EXEC scalar pairs synchronized with the - // internal booleans: shaders commonly save EXEC from s126:s127 - // and restore it after divergent work. Initializing only _exec - // left those registers at zero, so the first restore disabled - // every fragment before its color export. - StoreS64(106, _module.Constant64(_ulongType, 0)); - StoreS64(126, _module.Constant64(_ulongType, 1)); + _waveLowering.StoreWaveMask(126, _module.AddInstruction(SpirvOp.LogicalNot, _boolType, Load(_boolType, HelperInvocationInput()))); } // Primitive (NGG) vertex programs rebuild EXEC from the merged wave info in s3: @@ -1266,6 +1219,10 @@ public static partial class Gen5SpirvTranslator Store(output.Variable, _module.ConstantNull(output.Type)); } } + else if (_pairedLanes) + { + EmitPairedComputeInitialState(); + } else { var localId = Load(_uvec3Type, _localInvocationIdInput); @@ -1440,8 +1397,6 @@ public static partial class Gen5SpirvTranslator StoreS(register.Value, value); } - private enum SharedMemoryPhase { None, Read, Write } - private bool TryEmitBlock( IReadOnlyList blocks, int blockIndex, @@ -1463,52 +1418,23 @@ public static partial class Gen5SpirvTranslator { error = string.Empty; var block = blocks[blockIndex]; - var halfMaskPlan = HalfMaskPlan(); - // One guest wave can span two host subgroups. Keep its shared-memory phases ordered. - // The half-mask plan inserts its own barriers; other wave64 programs also - // need LDS phase ordering when reads and writes span basic blocks. - var synchronizeSharedMemory = _emulateWave64 && halfMaskPlan is null; - var sharedMemoryPhase = SharedMemoryPhase.None; + _waveLowering.BeginBlock(); for (var index = block.StartIndex; index < block.EndIndex; index++) { var instruction = _request.Program.Instructions[index]; - if (halfMaskPlan is not null) - { - if (halfMaskPlan.ExactPairsBefore.TryGetValue(instruction.Pc, out var exactPairs)) - { - EmitExactWaveMasks(exactPairs); - } - - if (halfMaskPlan.SharedMemoryBarriersBefore.Contains(instruction.Pc)) - { - EmitWave64Barrier(); - } - } - + _waveLowering.BeforeInstruction(instruction); if (IsBranch(instruction.Opcode) || instruction.Opcode == "SEndpgm") { continue; } - if (synchronizeSharedMemory) - { - var nextPhase = instruction.Control is Gen5DataShareControl { Gds: false } - ? instruction.Opcode.StartsWith("DsRead", StringComparison.Ordinal) ? SharedMemoryPhase.Read - : instruction.Opcode.StartsWith("DsWrite", StringComparison.Ordinal) ? SharedMemoryPhase.Write : SharedMemoryPhase.None - : SharedMemoryPhase.None; - if (instruction.Opcode == "SBarrier") sharedMemoryPhase = SharedMemoryPhase.None; - if (nextPhase != SharedMemoryPhase.None) - { - if (sharedMemoryPhase != SharedMemoryPhase.None && sharedMemoryPhase != nextPhase) EmitWave64Barrier(); - sharedMemoryPhase = nextPhase; - } - } - _execKnownFull = IsExecKnownFull(instruction.Pc); - _emittingPc = instruction.Pc; - var emitted = TryEmitInstruction(instruction, out error); - _emittingPc = null; + _unobservedWrites = ObservationPlan()?.UnobservedWrites(instruction.Pc) ?? default; + var emitted = _pairedLanes && IsPairedLaneInstruction(instruction) + ? TryEmitPairedLaneInstruction(instruction, out error) + : TryEmitInstruction(instruction, out error); _execKnownFull = false; + _unobservedWrites = default; if (!emitted) { error = $"pc=0x{instruction.Pc:X} {instruction.Opcode}: {error}"; @@ -1521,7 +1447,7 @@ public static partial class Gen5SpirvTranslator CapturePixelExec(instruction); } - if (synchronizeSharedMemory && sharedMemoryPhase != SharedMemoryPhase.None) EmitWave64Barrier(); + _waveLowering.EndBlock(); return true; } @@ -1908,8 +1834,8 @@ public static partial class Gen5SpirvTranslator if (_maxDispatcherSteps > 0) { - var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1)); - Store(_iterationGuard, steps); + var steps = IAdd(Load(_uintType, IterationGuard()), UInt(1)); + Store(IterationGuard(), steps); var withinLimit = _module.AddInstruction(SpirvOp.ULessThan, _boolType, steps, UInt((uint)_maxDispatcherSteps)); again = _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, again, withinLimit); } @@ -2063,8 +1989,8 @@ public static partial class Gen5SpirvTranslator { // The dispatcher's safety valve, counted per loop iteration: a mistranslated // exit ends the invocation instead of wedging the GPU queue. - var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1)); - Store(_iterationGuard, steps); + var steps = IAdd(Load(_uintType, IterationGuard()), UInt(1)); + Store(IterationGuard(), steps); var withinLimit = _module.AddInstruction(SpirvOp.ULessThan, _boolType, steps, UInt((uint)_maxDispatcherSteps)); again = _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, again, withinLimit); } @@ -2079,7 +2005,7 @@ public static partial class Gen5SpirvTranslator // the block was skipped. Preserve that diagnostic safety budget. if (_maxDispatcherSteps > 0) { - Store(_iterationGuard, IAdd(Load(_uintType, _iterationGuard), UInt(1))); + Store(IterationGuard(), IAdd(Load(_uintType, IterationGuard()), UInt(1))); } _module.AddStatement(SpirvOp.Branch, entryMerge); _module.AddLabel(entryMerge); @@ -2583,6 +2509,19 @@ public static partial class Gen5SpirvTranslator sizeBytes); var pointer = LdsPointer(baseAddress, offset); var destination = instruction.Destinations[0].Value; + if (_pairedLanes) + { + EmitPairedWaveCounter(destination, inBounds, count => EmitAtomic( + instruction.Opcode == "DsAppend" ? SpirvOp.AtomicIAdd : SpirvOp.AtomicISub, + _uintType, + pointer, + scope: 2, + semantics: 0x108, + value: () => count, + comparator: () => UInt(0))); + return true; + } + var active = Load(_boolType, _exec); var activeMask = BooleanToWaveMask(active); @@ -2657,15 +2596,7 @@ public static partial class Gen5SpirvTranslator }); var firstValue = LoadV(destination); - uint broadcast; - if (_emulateWave64) - { - broadcast = ExchangeWave64Value(isFirstActive, firstValue); - } - else - { - broadcast = ShuffleLane(firstValue, firstLane); - } + var broadcast = BroadcastWaveValue(isFirstActive, firstValue, firstLane); var validResult = _module.AddInstruction( SpirvOp.LogicalAnd, @@ -5469,6 +5400,13 @@ public static partial class Gen5SpirvTranslator [.. operands]); sampled = UnpackImageTexel(resource, sampled); } + + RecordMipStatistics( + resource, + imageObject, + coordinates, + hasZeroLod || hasLod ? lod : null, + hasBias ? lodOrBias : Float(0)); } else if (instruction.Opcode.StartsWith( "ImageGather4", @@ -6347,7 +6285,10 @@ public static partial class Gen5SpirvTranslator return true; } - Store(_reachedPixelExport, _module.ConstantBool(true)); + if (_reachedPixelExport != 0) + { + Store(_reachedPixelExport, _module.ConstantBool(true)); + } var values = new uint[4]; for (var component = 0; component < 4; component++) @@ -7469,6 +7410,13 @@ public static partial class Gen5SpirvTranslator UInt(0), dwordAddress); + private uint DeclareFunctionUint(uint initializer) + { + var variable = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, initializer); + _interfaces.Add(variable); + return variable; + } + // Without subgroup operations each invocation is a one-lane wave, so a V_WRITELANE to // lane N > 0 used to be dropped and V_READLANE returned lane 0. Compilers spill SGPRs // (EXEC included) to VGPR lanes like this; restoring the wrong EXEC made waterfall @@ -7476,7 +7424,7 @@ public static partial class Gen5SpirvTranslator private List<(uint Register, uint Lane)> FindLaneSpillSlots() { var slots = new List<(uint Register, uint Lane)>(); - if (_emulateWave64) + if (_stage == Gen5SpirvStage.Compute && (_waveLowering is EmulatedWave64Lowering || _pairedLanes)) { return slots; } @@ -7610,7 +7558,7 @@ public static partial class Gen5SpirvTranslator throw new InvalidOperationException($"Vector register v{register} is outside the register file."); } - return RegisterVariable(_vectorRegisterVariables, register, "v"); + return RegisterVariable(CurrentVectorRegisterVariables(), register, CurrentVectorRegisterPrefix()); } // The V_MOVREL* opcodes address the VGPR file with a register number that @@ -7686,6 +7634,24 @@ public static partial class Gen5SpirvTranslator // Declared on first use when Private. AMD's compiler keeps an unused 4 KiB private // array as a named .bss global: two stages then fail to link, and the driver copies // the NOBITS section as file data and reads past the end of the ELF. + private uint IterationGuard() + { + if (_iterationGuard == 0) + { + _iterationGuard = _module.AddGlobalVariable( + _privateUintPointer, + SpirvStorageClass.Private, + _module.Constant(_uintType, 0)); + _interfaces.Add(_iterationGuard); + _module.AddName(_iterationGuard, "pcGuard"); + } + + return _iterationGuard; + } + + private static readonly bool TraceTitleShaderState = + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_TITLE_SHADER_STATE") == "1"; + private uint PackedHalfRegisters() { if (_packedHalfRegisters != 0) @@ -7703,7 +7669,10 @@ public static partial class Gen5SpirvTranslator return _packedHalfRegisters; } - private uint LoadS(uint register) => Load(_uintType, ScalarPointer(register)); + private uint LoadS(uint register) => + _scalarSnapshot is { } snapshot && snapshot.TryGetValue(register, out var value) + ? value + : Load(_uintType, ScalarPointer(register)); private uint LoadV(uint register) => Load(_uintType, VectorPointer(register)); @@ -7714,14 +7683,14 @@ public static partial class Gen5SpirvTranslator { if (_waveLaneCount != 32 || register == 106) { - Store(_vcc, IsLaneSetInMaskRegisters(106)); + UpdateLaneMaskFlag(106); } } else if (register is 126 or 127) { if (_waveLaneCount != 32 || register == 126) { - Store(_exec, IsLaneSetInMaskRegisters(126)); + UpdateLaneMaskFlag(126); } } } @@ -7730,15 +7699,21 @@ public static partial class Gen5SpirvTranslator // so the lane's bit is tested on one dword instead of a composed 64-bit mask. private uint IsLaneSetInMaskRegisters(uint lowRegister) { + var low = Load(_uintType, ScalarPointer(lowRegister)); if (_waveLaneCount != 32) { - return IsWaveMaskActive(LoadS64(lowRegister)); + var high = Load(_uintType, ScalarPointer(lowRegister + 1)); + return IsWaveMaskActive(BitwiseOr64( + _module.AddInstruction(SpirvOp.UConvert, _ulongType, low), + ShiftLeftLogical64( + _module.AddInstruction(SpirvOp.UConvert, _ulongType, high), + _module.Constant64(_ulongType, 32)))); } var laneBit = _subgroupInvocationIdInput == 0 ? UInt(1) : ShiftLeftLogical(UInt(1), GuestWaveLane()); - return IsNotZero(BitwiseAnd(LoadS(lowRegister), laneBit)); + return IsNotZero(BitwiseAnd(low, laneBit)); } // SHARPEMU_EXEC_GUARD_ELISION=0 guards every vector register write with EXEC again. @@ -7753,7 +7728,7 @@ public static partial class Gen5SpirvTranslator private bool IsExecKnownFull(uint pc) { - if (!ExecGuardElision || !_request.EnableExecGuardElision) + if (!ExecGuardElision || !_request.EnableExecGuardElision || _pairedLanes || _pixelExecExcludesHelpers) { return false; } @@ -7762,9 +7737,29 @@ public static partial class Gen5SpirvTranslator return _execFullPcs.Contains(pc); } + private static readonly bool ExecSelectElimination = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_EXEC_SELECT_ELIMINATION"), "0", StringComparison.Ordinal); + + private Ir.VectorRegisterSet _unobservedWrites; + private Ir.Gen5VectorObservationPlan? _observationPlan; + private bool _observationPlanBuilt; + + private Ir.Gen5VectorObservationPlan? ObservationPlan() + { + if (!_observationPlanBuilt) + { + _observationPlanBuilt = true; + _observationPlan = ExecSelectElimination && !_pixelExecExcludesHelpers + ? Ir.Gen5VectorObservationAnalysis.Analyze(_request.Program, wave32: _waveLaneCount == 32) + : null; + } + + return _observationPlan; + } + private void StoreV(uint register, uint value, bool guardWithExec = true) { - if (guardWithExec && !_execKnownFull) + if (guardWithExec && !_execKnownFull && !_unobservedWrites.Contains(register)) { var active = Load(_boolType, _exec); var oldValue = LoadV(register); @@ -7956,18 +7951,24 @@ public static partial class Gen5SpirvTranslator _module.AddInstruction(SpirvOp.LogicalNot, _boolType, value); private uint SubgroupAny(uint condition) => - _subgroupInvocationIdInput == 0 - ? condition - : _emulateWave64 - ? IsNotZero64(BooleanToWaveMask(condition)) - : _module.AddInstruction( - SpirvOp.GroupNonUniformAny, - _boolType, - UInt(3), - condition); + _waveLowering.SubgroupAny(condition); private uint GuestWaveLane() { + if (_pairedLanes) + { + return PairedGuestLane(); + } + + // A native wave64 is one host subgroup: its lane is the subgroup lane, the same one + // ballot and shuffle address, whatever order the driver gave the invocations. + if (_waveLaneCount == 64 && _guestLaneFromSubgroup && _subgroupInvocationIdInput != 0) + { + return BitwiseAnd( + Load(_uintType, _subgroupInvocationIdInput), + UInt(63)); + } + if (_waveLaneCount == 64 && _localInvocationIndexInput != 0) { return BitwiseAnd( @@ -8015,31 +8016,17 @@ public static partial class Gen5SpirvTranslator private uint ShuffleLane(uint value, uint lane) => _subgroupInvocationIdInput == 0 ? value - : _module.AddInstruction(SpirvOp.GroupNonUniformShuffle, _uintType, UInt(3), value, lane); - - private uint CurrentLaneBit() - { - if (_subgroupInvocationIdInput == 0) - { - return _module.Constant64(_ulongType, 1); - } - - var maskedLane = GuestWaveLane(); - var shifted = ShiftLeftLogical64( - _module.Constant64(_ulongType, 1), - _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - maskedLane)); - return _emulateWave64 - ? shifted : _module.AddInstruction( - SpirvOp.Select, - _ulongType, - IsCurrentLaneInRdnaWave(), - shifted, - _module.Constant64(_ulongType, 0)); - } + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + value, + _pairedLanes || _waveLowering is HalfWave64Lowering + ? BitwiseAnd(lane, UInt(PairedHalfLaneCount - 1)) + : lane); + + private uint CurrentLaneBit() => + _waveLowering.CurrentLaneBit(); private uint IsCurrentLaneInRdnaWave() => _module.AddInstruction( @@ -8056,193 +8043,11 @@ public static partial class Gen5SpirvTranslator CurrentLaneBit(), _module.Constant64(_ulongType, 0)); - private uint BooleanToWaveMask(uint condition) - { - if (_subgroupInvocationIdInput == 0) - { - return BooleanToLaneMask(condition); - } - - if (_emulateWave64) - { - var lane = GuestWaveLane(); - var owned = OwnHalfBallot(condition); - var exchange = BeginWave64Exchange(); - EmitConditional(IsHalfWaveLeader(lane), () => - Store(Wave64ExchangePointer(exchange, ShiftRightLogical(lane, UInt(5))), owned)); - EmitWave64Barrier(); - return Pair64( - Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))), - Load(_uintType, Wave64ExchangePointer(exchange, UInt(1)))); - } - - var ballot = _module.AddInstruction( - SpirvOp.GroupNonUniformBallot, - _uvec4Type, - UInt(3), - condition); - var low = _module.AddInstruction( - SpirvOp.CompositeExtract, - _uintType, - ballot, - 0); - var widened = _module.AddInstruction(SpirvOp.UConvert, _ulongType, low); - if (_waveLaneCount != 64) - { - return widened; - } - - return _module.AddInstruction( - SpirvOp.Select, - _ulongType, - _module.AddInstruction( - SpirvOp.UGreaterThanEqual, - _boolType, - GuestWaveLane(), - UInt(32)), - ShiftLeftLogical64( - widened, - _module.Constant64(_ulongType, 32)), - widened); - } - - private uint OwnHalfBallot(uint condition) - { - var ballot = _module.AddInstruction( - SpirvOp.GroupNonUniformBallot, - _uvec4Type, - UInt(3), - condition); - var component = ShiftRightLogical(Load(_uintType, _subgroupInvocationIdInput), UInt(5)); - // Avoid dynamic extraction of subgroup ballot words: on the NVIDIA - // device tests it produced zero masks. Select the same word explicitly. - var result = _module.AddInstruction(SpirvOp.CompositeExtract, _uintType, ballot, 0); - for (uint index = 1; index < 4; index++) - { - var word = _module.AddInstruction(SpirvOp.CompositeExtract, _uintType, ballot, index); - result = _module.AddInstruction(SpirvOp.Select, _uintType, - _module.AddInstruction(SpirvOp.IEqual, _boolType, component, UInt(index)), word, result); - } - return result; - } - - private uint BooleanToHalfWaveMask(uint condition) - { - var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition)); - return _module.AddInstruction( - SpirvOp.Select, - _ulongType, - _module.AddInstruction(SpirvOp.UGreaterThanEqual, _boolType, GuestWaveLane(), UInt(32)), - ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32)), - owned); - } - - private uint BothHalvesOfOwnBallot(uint condition) - { - var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition)); - return BitwiseOr64(owned, ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32))); - } - - private uint IsHalfWaveLeader(uint lane) => - _module.AddInstruction(SpirvOp.IEqual, _boolType, BitwiseAnd(lane, UInt(31)), UInt(0)); - - private uint BeginWave64Exchange() - { - var parity = Load(_uintType, _wave64ExchangeParity); - Store(_wave64ExchangeParity, BitwiseXor(parity, UInt(1))); - return IAdd( - UInt(_wave64ExchangeOffset), - _module.AddInstruction(SpirvOp.IMul, _uintType, parity, UInt(Wave64ExchangeSlotCount))); - } - - private uint Wave64ExchangePointer(uint exchange, uint slot) => - _module.AddInstruction( - SpirvOp.AccessChain, - _wave64ExchangeElementPointer, - _wave64Exchange, - IAdd(exchange, slot)); - - private uint ExchangeWave64Value(uint writer, uint value) - { - var exchange = BeginWave64Exchange(); - EmitConditional(writer, () => Store(Wave64ExchangePointer(exchange, UInt(0)), value)); - EmitWave64Barrier(); - return Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))); - } - - private void EmitExactWaveMasks(IReadOnlyList pairs) - { - var lane = GuestWaveLane(); - var upperHalf = ShiftRightLogical(lane, UInt(5)); - var lowerHalf = _module.AddInstruction(SpirvOp.IEqual, _boolType, upperHalf, UInt(0)); - const int pairsPerExchange = (int)Wave64ExchangeSlotCount / 2; - for (var start = 0; start < pairs.Count; start += pairsPerExchange) - { - var count = Math.Min(pairsPerExchange, pairs.Count - start); - var owned = new uint[count]; - for (var index = 0; index < count; index++) - { - var pair = pairs[start + index]; - owned[index] = _module.AddInstruction(SpirvOp.Select, _uintType, lowerHalf, LoadS(pair), LoadS(pair + 1)); - } - - var exchange = BeginWave64Exchange(); - EmitConditional(IsHalfWaveLeader(lane), () => - { - for (var index = 0; index < count; index++) - { - Store(Wave64ExchangePointer(exchange, IAdd(UInt((uint)index * 2), upperHalf)), owned[index]); - } - }); - EmitWave64Barrier(); - for (var index = 0; index < count; index++) - { - var pair = pairs[start + index]; - StoreS(pair, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2)))); - StoreS(pair + 1, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2 + 1)))); - } - } - } + private uint BooleanToWaveMask(uint condition) => + _waveLowering.BooleanToWaveMask(condition); private uint WaveMaskAny(uint register, uint laneActive) => - _emulateWave64 && _subgroupInvocationIdInput != 0 - ? IsNotZero64(LoadS64(register)) - : SubgroupAny(Load(_boolType, laneActive)); - - private static readonly bool HalfWaveMasks = !string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_HALF_MASKS"), "0", StringComparison.Ordinal); - - private bool _halfMaskPlanBuilt; - - private Ir.Gen5Wave64HalfMaskPlan? HalfMaskPlan() - { - if (_halfMaskPlanBuilt) - { - return _halfMaskPlan; - } - - _halfMaskPlanBuilt = true; - if (HalfWaveMasks && UsesWave64Exchange() && _subgroupInvocationIdInput != 0) - { - var sharedFlatPcs = _request.Memory.Entries - .Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) - .Select(static memory => memory.Pc) - .ToHashSet(); - _halfMaskPlan = Ir.Gen5Wave64HalfMaskAnalysis.Analyze(_request.Program, sharedFlatPcs); - } - - return _halfMaskPlan; - } - - private void EmitWave64Barrier() - { - var workgroup = UInt(2); - _module.AddStatement( - SpirvOp.ControlBarrier, - workgroup, - workgroup, - UInt(0x108)); - } + _waveLowering.WaveMaskAny(register, laneActive); // A wave-mask SGPR (VCC/EXEC) consumed as a per-lane predicate — the // condition of VCndmask, a VCC/EXEC branch, or the derived _vcc/_exec @@ -8269,27 +8074,8 @@ public static partial class Gen5SpirvTranslator // In wave32 a lane mask fills its register alone and the next one keeps its value. // That includes VCC and EXEC: compilers use VCC_HI (s107) as an ordinary SGPR. - private void StoreWaveMask(uint register, uint condition) - { - if (_waveLaneCount == 32) - { - StoreS(register, Narrow(BooleanToWaveMask(condition))); - return; - } - - if (_emulateWave64 && - _subgroupInvocationIdInput != 0 && - _emittingPc is { } pc && - HalfMaskPlan() is { } plan && - plan.HalfMaskWrites.TryGetValue(pc, out var halfRegister) && - halfRegister == register) - { - StoreS64(register, BooleanToHalfWaveMask(condition)); - return; - } - - StoreS64(register, BooleanToWaveMask(condition)); - } + private void StoreWaveMask(uint register, uint condition) => + _waveLowering.StoreWaveMask(register, condition); private void EmitExecConditional(Action emit) { @@ -8360,6 +8146,7 @@ public static partial class Gen5SpirvTranslator instruction.Destinations.Any(IsWaveMaskOperand)); private bool UsesSubgroupOperations() => + _pairedLanes || _enableGraphicsSubgroupOperations && (UsesSubgroupShuffle() || UsesSubgroupBroadcast() || diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs index 0768ec7..4c56d88 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs @@ -176,10 +176,12 @@ public enum SpirvOp : ushort GroupNonUniformBroadcast = 337, GroupNonUniformBroadcastFirst = 338, GroupNonUniformBallot = 339, + GroupNonUniformBallotBitCount = 342, GroupNonUniformShuffle = 345, GroupNonUniformShuffleXor = 346, GroupNonUniformShuffleUp = 347, GroupNonUniformShuffleDown = 348, + GroupNonUniformUMin = 354, } public enum SpirvCapability : uint @@ -206,6 +208,7 @@ public enum SpirvCapability : uint StorageImageWriteWithoutFormat = 56, GroupNonUniform = 61, GroupNonUniformVote = 62, + GroupNonUniformArithmetic = 63, GroupNonUniformBallot = 64, GroupNonUniformShuffle = 65, ShaderLayer = 5253, @@ -276,11 +279,13 @@ public enum SpirvBuiltIn : uint BaryCoordKhr = 5286, BaryCoordNoPerspKhr = 5287, FrontFacing = 17, + HelperInvocation = 23, WorkgroupId = 26, LocalInvocationId = 27, GlobalInvocationId = 28, LocalInvocationIndex = 29, SubgroupSize = 36, + SubgroupId = 40, SubgroupLocalInvocationId = 41, SampleId = 18, SampleMask = 20, diff --git a/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs b/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs index fc2260b..1605aa7 100644 --- a/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs +++ b/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs @@ -9,8 +9,18 @@ public static class Gen5PixelInputMapping ReadOnlySpan controls, ReadOnlySpan activeInputs) { - Span usedLocations = stackalloc bool[32]; var locations = new uint[activeInputs.Length]; + ResolveLocations(controls, activeInputs, locations); + return locations; + } + + // Fills `locations` (one slot per active input) without allocating. + public static void ResolveLocations( + ReadOnlySpan controls, + ReadOnlySpan activeInputs, + Span locations) + { + Span usedLocations = stackalloc bool[32]; for (var index = 0; index < activeInputs.Length; index++) { @@ -39,7 +49,5 @@ public static class Gen5PixelInputMapping usedLocations[(int)location] = true; } } - - return locations; } } diff --git a/src/SharpEmu.ShaderCompiler/Ir/Gen5VectorObservationAnalysis.cs b/src/SharpEmu.ShaderCompiler/Ir/Gen5VectorObservationAnalysis.cs new file mode 100644 index 0000000..37eba4e --- /dev/null +++ b/src/SharpEmu.ShaderCompiler/Ir/Gen5VectorObservationAnalysis.cs @@ -0,0 +1,454 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Numerics; + +namespace SharpEmu.ShaderCompiler.Ir; + +public readonly record struct VectorRegisterSet(UInt128 Low, UInt128 High) +{ + public bool IsEmpty => Low == UInt128.Zero && High == UInt128.Zero; + + public bool Contains(uint register) => + register < 128 + ? ((Low >> (int)register) & UInt128.One) != 0 + : register < 256 && ((High >> (int)(register - 128)) & UInt128.One) != 0; + + public VectorRegisterSet With(uint register) => + register < 128 ? this with { Low = Low | (UInt128.One << (int)register) } + : register < 256 ? this with { High = High | (UInt128.One << (int)(register - 128)) } + : this; + + public VectorRegisterSet Union(VectorRegisterSet other) => new(Low | other.Low, High | other.High); + + public VectorRegisterSet Except(VectorRegisterSet other) => new(Low & ~other.Low, High & ~other.High); + + public int Count => PopCount(Low) + PopCount(High); + + private static int PopCount(UInt128 value) => + BitOperations.PopCount((ulong)value) + BitOperations.PopCount((ulong)(value >> 64)); +} + +public sealed class Gen5VectorObservationPlan(IReadOnlyDictionary unobserved) +{ + public VectorRegisterSet UnobservedWrites(uint pc) => unobserved.TryGetValue(pc, out var set) ? set : default; + + public int WriteCount => unobserved.Values.Sum(static set => set.Count); +} + +public static class Gen5VectorObservationAnalysis +{ + private const uint ExecLow = 126; + private const uint ExecHigh = 127; + + private enum ExecEffect : byte + { + None, + Narrowing, + Widening, + } + + private readonly record struct Effect( + VectorRegisterSet Reads, + VectorRegisterSet CrossReads, + VectorRegisterSet Writes, + VectorRegisterSet FullWrites, + ExecEffect Exec); + + public static Gen5VectorObservationPlan? Analyze(Gen5ShaderProgram program, bool wave32) + { + var instructions = program.Instructions; + if (instructions.Count == 0) + { + return null; + } + + var effects = new Effect[instructions.Count]; + for (var index = 0; index < instructions.Count; index++) + { + if (!TryDescribe(instructions[index], out effects[index])) + { + return null; + } + } + + if (BuildSuccessors(instructions) is not { } successors) + { + return null; + } + + var execFull = Gen5ExecFullAnalysis.Analyze(program, wave32); + var predecessors = new List[instructions.Count]; + for (var index = 0; index < instructions.Count; index++) + { + predecessors[index] = []; + } + + for (var index = 0; index < instructions.Count; index++) + { + foreach (var successor in successors[index]) + { + predecessors[successor].Add(index); + } + } + + var liveIn = new VectorRegisterSet[instructions.Count]; + var observedIn = new VectorRegisterSet[instructions.Count]; + var queued = new bool[instructions.Count]; + var work = new Queue(); + for (var index = instructions.Count - 1; index >= 0; index--) + { + work.Enqueue(index); + queued[index] = true; + } + + while (work.TryDequeue(out var index)) + { + queued[index] = false; + var (liveOut, observedOut) = Out(index, successors, liveIn, observedIn); + var (live, observed) = Transfer(effects[index], execFull.Contains(instructions[index].Pc), liveOut, observedOut); + if (live == liveIn[index] && observed == observedIn[index]) + { + continue; + } + + liveIn[index] = live; + observedIn[index] = observed; + foreach (var predecessor in predecessors[index]) + { + if (!queued[predecessor]) + { + queued[predecessor] = true; + work.Enqueue(predecessor); + } + } + } + + var unobserved = new Dictionary(); + for (var index = 0; index < instructions.Count; index++) + { + var writes = effects[index].Writes; + if (writes.IsEmpty) + { + continue; + } + + var (_, observedOut) = Out(index, successors, liveIn, observedIn); + var dead = execFull.Contains(instructions[index].Pc) ? writes : writes.Except(observedOut); + if (!dead.IsEmpty) + { + unobserved[instructions[index].Pc] = dead; + } + } + + return new Gen5VectorObservationPlan(unobserved); + } + + private static (VectorRegisterSet Live, VectorRegisterSet Observed) Out( + int index, int[][] successors, VectorRegisterSet[] liveIn, VectorRegisterSet[] observedIn) + { + var live = default(VectorRegisterSet); + var observed = default(VectorRegisterSet); + foreach (var successor in successors[index]) + { + live = live.Union(liveIn[successor]); + observed = observed.Union(observedIn[successor]); + } + + return (live, observed); + } + + private static (VectorRegisterSet Live, VectorRegisterSet Observed) Transfer( + in Effect effect, bool execFull, VectorRegisterSet liveOut, VectorRegisterSet observedOut) + { + var observed = effect.Exec == ExecEffect.Widening ? observedOut.Union(liveOut) : observedOut; + var kills = execFull ? effect.FullWrites : default; + var live = liveOut.Except(kills).Union(effect.Reads).Union(effect.CrossReads); + observed = observed.Except(kills).Union(effect.CrossReads); + return (live, observed); + } + + private static bool TryDescribe(Gen5ShaderInstruction instruction, out Effect effect) + { + effect = default; + var opcode = instruction.Opcode; + if (opcode.StartsWith("VMovrel", StringComparison.Ordinal) || + opcode.Contains("Raw", StringComparison.Ordinal) || + opcode is "SSetpcB64" or "SSwappcB64" or "SRfeB64" or "SCbranchJoin" or "SCbranchGFork" or "SCbranchIFork") + { + return false; + } + + var reads = default(VectorRegisterSet); + var cross = default(VectorRegisterSet); + var writes = default(VectorRegisterSet); + var full = default(VectorRegisterSet); + var wideName = opcode.Contains("64", StringComparison.Ordinal); + + foreach (var source in instruction.Sources) + { + if (source.Kind != Gen5OperandKind.VectorRegister) + { + continue; + } + + reads = reads.With(source.Value); + if (wideName) + { + reads = reads.With(source.Value + 1); + } + } + + switch (instruction.Encoding) + { + case Gen5ShaderEncoding.Vop1: + case Gen5ShaderEncoding.Vop2: + case Gen5ShaderEncoding.Vop3: + case Gen5ShaderEncoding.Vop3p: + case Gen5ShaderEncoding.Vopc: + case Gen5ShaderEncoding.Vintrp: + DescribeVector(instruction, wideName, ref reads, ref cross, ref writes, ref full); + break; + case Gen5ShaderEncoding.Mubuf: + case Gen5ShaderEncoding.Mtbuf: + DescribeBuffer(instruction, ref reads, ref writes); + break; + case Gen5ShaderEncoding.Flat: + foreach (var destination in instruction.Destinations) + { + if (destination.Kind == Gen5OperandKind.VectorRegister) + { + writes = writes.With(destination.Value); + reads = reads.With(destination.Value); + } + } + + break; + case Gen5ShaderEncoding.Mimg: + cross = cross.Union(reads); + DescribeImage(instruction, ref reads, ref writes); + break; + case Gen5ShaderEncoding.Ds: + if (opcode is "DsSwizzleB32" or "DsBpermuteB32" or "DsPermuteB32") + { + cross = cross.Union(reads); + } + + foreach (var destination in instruction.Destinations) + { + if (destination.Kind == Gen5OperandKind.VectorRegister) + { + writes = writes.With(destination.Value); + reads = reads.With(destination.Value); + } + } + + break; + } + + effect = new Effect(reads, cross, writes, full, DescribeExec(instruction)); + return true; + } + + private static void DescribeVector(Gen5ShaderInstruction instruction, bool wideName, ref VectorRegisterSet reads, + ref VectorRegisterSet cross, ref VectorRegisterSet writes, ref VectorRegisterSet full) + { + var opcode = instruction.Opcode; + if (instruction.Control is Gen5DppControl or Gen5Dpp8Control || + opcode is "VReadlaneB32" or "VReadfirstlaneB32" or "VPermlane16B32" or "VPermlanex16B32") + { + if (instruction.Sources.Count > 0 && instruction.Sources[0].Kind == Gen5OperandKind.VectorRegister) + { + cross = cross.With(instruction.Sources[0].Value); + if (wideName) + { + cross = cross.With(instruction.Sources[0].Value + 1); + } + } + } + + if (instruction.Destinations.Count == 0 || instruction.Destinations[0].Kind != Gen5OperandKind.VectorRegister) + { + return; + } + + var destination = instruction.Destinations[0].Value; + writes = writes.With(destination); + if (wideName) + { + writes = writes.With(destination + 1); + } + + var partial = + opcode.Contains("Mac", StringComparison.Ordinal) || + opcode.StartsWith("VDot", StringComparison.Ordinal) || + opcode is "VWritelaneB32" || + opcode.Contains("16", StringComparison.Ordinal) || + opcode.Contains("Pk", StringComparison.Ordinal) || + instruction.Encoding is Gen5ShaderEncoding.Vintrp || + instruction.Control is Gen5SdwaControl { DestinationSelect: not 6 } or Gen5SdwaControl { DestinationUnused: not 0 } || + instruction.Control is Gen5Vop3Control { OperandSelect: not 0 }; + if (partial) + { + reads = reads.With(destination); + if (wideName) + { + reads = reads.With(destination + 1); + } + + return; + } + + full = full.With(destination); + } + + private static void DescribeBuffer(Gen5ShaderInstruction instruction, ref VectorRegisterSet reads, ref VectorRegisterSet writes) + { + var opcode = instruction.Opcode; + if (instruction.Control is Gen5BufferMemoryControl buffer) + { + reads = reads.With(buffer.VectorAddress).With(buffer.VectorAddress + 1); + } + + var load = opcode.Contains("Load", StringComparison.Ordinal); + foreach (var destination in instruction.Destinations) + { + if (destination.Kind != Gen5OperandKind.VectorRegister) + { + continue; + } + + reads = reads.With(destination.Value); + if (load || opcode.Contains("Atomic", StringComparison.Ordinal)) + { + writes = writes.With(destination.Value); + } + } + } + + private static void DescribeImage(Gen5ShaderInstruction instruction, ref VectorRegisterSet reads, ref VectorRegisterSet writes) + { + var (data, count) = instruction.Control switch + { + Gen5ImageControl image => (image.VectorData, 5u), + Gen5RayIntersectControl ray => (ray.VectorData, Gen5RayIntersectControl.ResultDwords), + _ => (0u, 0u), + }; + var store = instruction.Opcode.StartsWith("ImageStore", StringComparison.Ordinal); + for (var offset = 0u; offset < count; offset++) + { + reads = reads.With(data + offset); + if (!store) + { + writes = writes.With(data + offset); + } + } + } + + private static ExecEffect DescribeExec(Gen5ShaderInstruction instruction) + { + var opcode = instruction.Opcode; + var scalarDestination = instruction.Control switch + { + Gen5Vop3Control control => control.ScalarDestination, + Gen5SdwaControl control => control.ScalarDestination, + _ => null, + }; + if (opcode.StartsWith("VCmpx", StringComparison.Ordinal)) + { + return ExecEffect.Narrowing; + } + + if (opcode.StartsWith("VCmp", StringComparison.Ordinal)) + { + return scalarDestination is ExecLow ? ExecEffect.Narrowing : ExecEffect.None; + } + + var writesExec = + scalarDestination is ExecLow or ExecHigh || + opcode.Contains("Saveexec", StringComparison.Ordinal) || + opcode.Contains("Wrexec", StringComparison.Ordinal) || + instruction.Destinations.Any(static destination => + destination is { Kind: Gen5OperandKind.ScalarRegister, Value: ExecLow or ExecHigh }); + if (!writesExec) + { + return ExecEffect.None; + } + + if (opcode is "SAndSaveexecB64" or "SAndSaveexecB32" or "SAndn1SaveexecB64" or "SAndn1SaveexecB32") + { + return ExecEffect.Narrowing; + } + + var toExec = instruction.Destinations is [{ Kind: Gen5OperandKind.ScalarRegister, Value: ExecLow }]; + var sources = instruction.Sources; + if (toExec && opcode is "SAndB64" or "SAndB32" && sources.Count == 2 && (IsExec(sources[0]) || IsExec(sources[1]))) + { + return ExecEffect.Narrowing; + } + + if (toExec && opcode is "SAndn2B64" or "SAndn2B32" && sources.Count == 2 && IsExec(sources[0])) + { + return ExecEffect.Narrowing; + } + + return ExecEffect.Widening; + } + + private static bool IsExec(Gen5Operand operand) => operand is { Kind: Gen5OperandKind.ScalarRegister, Value: ExecLow }; + + private static int[][]? BuildSuccessors(IReadOnlyList instructions) + { + var indexByPc = new Dictionary(instructions.Count); + for (var index = 0; index < instructions.Count; index++) + { + indexByPc[instructions[index].Pc] = index; + } + + var last = instructions[^1]; + var endPc = last.Pc + (uint)(last.Words.Count * sizeof(uint)); + var successors = new int[instructions.Count][]; + var resolver = Gen5IrBranchResolver.Instance; + for (var index = 0; index < instructions.Count; index++) + { + var instruction = instructions[index]; + var next = index + 1 < instructions.Count ? index + 1 : -1; + if (Gen5IrBranchResolver.IsTerminator(instruction)) + { + successors[index] = []; + continue; + } + + var unconditional = Gen5IrBranchResolver.IsUnconditionalBranch(instruction); + if (!unconditional && !resolver.IsConditional(instruction)) + { + successors[index] = next < 0 ? [] : [next]; + continue; + } + + if (!resolver.TryGetBranchTarget(instruction, out var targetPc)) + { + return null; + } + + var targets = new List(2); + if (indexByPc.TryGetValue(targetPc, out var target)) + { + targets.Add(target); + } + else if (targetPc < endPc) + { + return null; + } + + if (!unconditional && next >= 0 && !targets.Contains(next)) + { + targets.Add(next); + } + + successors[index] = [.. targets]; + } + + return successors; + } +} diff --git a/src/SharpEmu.ShaderCompiler/Ir/Gen5WaveLaneLocalAnalysis.cs b/src/SharpEmu.ShaderCompiler/Ir/Gen5WaveLaneLocalAnalysis.cs new file mode 100644 index 0000000..c44e27a --- /dev/null +++ b/src/SharpEmu.ShaderCompiler/Ir/Gen5WaveLaneLocalAnalysis.cs @@ -0,0 +1,2244 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Numerics; + +namespace SharpEmu.ShaderCompiler.Ir; + +public static class Gen5WaveLaneLocalAnalysis +{ + private const int VccLo = 106; + private const int VccHi = 107; + private const int M0 = 124; + private const int Scc = 125; + private const int ExecLo = 126; + private const int ExecHi = 127; + private const int ScalarCount = 128; + private const int VectorCount = 256; + private const int MaxSteps = 100_000; + private const int NoBlock = -1; + + private enum Family + { + Sop1, + Sop2, + Sopk, + Sopc, + Sopp, + Smem, + Vop1, + Vop2, + Vop3, + Vop3p, + Vopc, + Mubuf, + Mtbuf, + Flat, + Mimg, + Ds, + Other, + } + + private enum Kind : byte + { + None, + Scalar, + Vector, + Zero, + AllOnes, + Constant, + Scc, + VccZ, + ExecZ, + } + + private enum Condition : byte + { + None, + ExecZero, + ExecNonZero, + VccZero, + VccNonZero, + Other, + } + + private enum Sym : byte + { + Original, + Zero, + Ones, + Unknown, + } + + private readonly record struct Operand(Kind Kind, int Register = -1) + { + public static Operand None => new(Kind.None); + + public static Operand Scalar(int register) => new(Kind.Scalar, register); + + public int ScalarIndex => Kind == Kind.Scalar ? Register : -1; + } + + private readonly record struct MaskFacts(bool Covers, bool Within, bool Nonzero); + + private sealed class Op + { + public required uint Pc { get; init; } + + public required string Opcode { get; init; } + + public required Family Family { get; init; } + + public required Operand[] Sources { get; init; } + + public required int[] Widths { get; init; } + + public Operand Destination { get; init; } + + public int DestinationCount { get; init; } = 1; + + public Operand Carry { get; init; } + + public int Select { get; init; } = -1; + + public bool LaneFetch { get; init; } + + public bool CrossRowFetch { get; init; } + + public bool PartialVector { get; init; } + + public int WrittenVectorStart { get; init; } = -1; + + public int WrittenVectorCount { get; init; } + + public bool Wide { get; init; } + + public string? Refused { get; init; } + + public bool IsBranch { get; init; } + + public bool IsConditional { get; init; } + + public bool IsTerminator { get; init; } + + public Condition Condition { get; init; } + + public Operand Source(int index) => index < Sources.Length ? Sources[index] : Operand.None; + } + + private struct State : IEquatable + { + public UInt128 Mask; + public UInt128 Covers; + public UInt128 Within; + public UInt128 Nonzero; + public UInt128 Lanes; + public UInt128 UniformLow; + public UInt128 UniformHigh; + public bool SccMask; + public bool Active; + + public readonly bool Uniform(int register) => + register < 128 + ? ((UniformLow >> register) & UInt128.One) != 0 + : register < VectorCount && ((UniformHigh >> (register - 128)) & UInt128.One) != 0; + + public void SetUniform(int register, bool value) + { + if (register < 128) + { + UniformLow = value ? UniformLow | (UInt128.One << register) : UniformLow & ~(UInt128.One << register); + } + else if (register < VectorCount) + { + var bit = UInt128.One << (register - 128); + UniformHigh = value ? UniformHigh | bit : UniformHigh & ~bit; + } + } + + public void Join(in State other) + { + Mask |= other.Mask; + Covers &= other.Covers; + Within &= other.Within; + Nonzero &= other.Nonzero; + Lanes &= other.Lanes; + UniformLow &= other.UniformLow; + UniformHigh &= other.UniformHigh; + SccMask |= other.SccMask; + Active &= other.Active; + } + + public readonly bool Equals(State other) => + Mask == other.Mask && + Covers == other.Covers && + Within == other.Within && + Nonzero == other.Nonzero && + Lanes == other.Lanes && + UniformLow == other.UniformLow && + UniformHigh == other.UniformHigh && + SccMask == other.SccMask && + Active == other.Active; + + public override readonly bool Equals(object? obj) => obj is State other && Equals(other); + + public override readonly int GetHashCode() => HashCode.Combine(Mask, Covers, Within, Lanes, UniformLow, UniformHigh); + } + + private sealed class Block + { + public required int Start { get; init; } + + public required int End { get; init; } + + public List Successors { get; } = []; + + public Condition Condition { get; set; } + + public int TrueBlock { get; set; } = NoBlock; + + public int FalseBlock { get; set; } = NoBlock; + } + + private readonly record struct Loop(int Header, int Latch, bool[] Body); + + private static readonly HashSet ScalarDestination64Opcodes = new(StringComparer.Ordinal) + { + "SMovB64", "SNotB64", "SWqmB64", "SQuadmaskB64", "SBitreplicateB64B32", "SGetpcB64", + "SAndB64", "SAndn2B64", "SOrB64", "SOrn2B64", "SXorB64", "SNandB64", "SNorB64", "SXnorB64", + "SLshlB64", "SLshrB64", "SAshrI64", "SBfeU64", "SBfeI64", "SBfmB64", "SCselectB64", + "SBitset0B64", "SBitset1B64", + }; + + private static readonly HashSet PartialWriteOpcodes = new(StringComparer.Ordinal) + { + "SBitset0B32", "SBitset1B32", "SBitset0B64", "SBitset1B64", + }; + + private static readonly string[] SccWriterPrefixes = + [ + "SCmp", "SBitcmp", "SAdd", "SSub", "SMin", "SMax", "SAbs", "SAnd", "SOr", "SXor", "SNand", "SNor", + "SXnor", "SNot", "SWqm", "SLshl", "SLshr", "SAshr", "SBfe", "SBcnt", + ]; + + private static readonly HashSet SccKeepers = new(StringComparer.Ordinal) + { + "SMovB32", "SMovB64", "SMovkI32", "SCselectB32", "SCselectB64", "SBrevB32", "SFF1I32B32", "SFF1I32B64", + "SFlbitI32B32", "SFlbitI32B64", "SBitset0B32", "SBitset1B32", "SBitset0B64", "SBitset1B64", + "SMulI32", "SMulHiU32", "SMulHiI32", "SMulkI32", "SBfmB32", "SBfmB64", + "SPackLlB32B16", "SPackLhB32B16", "SPackHhB32B16", "SGetpcB64", + }; + + private static readonly HashSet SccReaders = new(StringComparer.Ordinal) + { + "SCselectB32", "SCselectB64", "SAddcU32", "SSubbU32", "SCbranchScc0", "SCbranchScc1", + }; + + private static readonly HashSet MaskOperations = new(StringComparer.Ordinal) + { + "SMovB32", "SMovB64", "SNotB32", "SNotB64", "SWqmB32", "SWqmB64", + "SAndB32", "SAndB64", "SAndn2B32", "SAndn2B64", "SOrB32", "SOrB64", "SOrn2B32", "SOrn2B64", + "SXorB32", "SXorB64", "SNandB32", "SNandB64", "SNorB32", "SNorB64", "SXnorB32", "SXnorB64", + "SCselectB32", "SCselectB64", + }; + + private static readonly HashSet LaneSelectOpcodes = new(StringComparer.Ordinal) + { + "VCndmaskB32", "VAddcU32", "VAddCoCiU32", "VSubbU32", "VSubCoCiU32", "VSubbrevU32", "VSubrevCoCiU32", + }; + + private static readonly HashSet CarryOutOpcodes = new(StringComparer.Ordinal) + { + "VAddcU32", "VAddCoCiU32", "VSubbU32", "VSubCoCiU32", "VSubbrevU32", "VSubrevCoCiU32", + "VAddCoU32", "VSubCoU32", "VSubrevCoU32", + }; + + public static bool IsLaneLocal(Gen5ShaderProgram program, IReadOnlySet sharedFlatPcs, out string reason) + { + var instructions = program.Instructions; + if (instructions.Count == 0) + { + reason = "control flow"; + return false; + } + + var ops = new Op[instructions.Count]; + for (var index = 0; index < ops.Length; index++) + { + ops[index] = Normalize(instructions[index], sharedFlatPcs); + } + + if (!TryBuildBlocks(instructions, ops, out var blocks)) + { + reason = "control flow"; + return false; + } + + return new Analysis(ops, blocks).Run(out reason); + } + + private static Op Normalize(Gen5ShaderInstruction instruction, IReadOnlySet sharedFlatPcs) + { + var opcode = instruction.Opcode; + var family = instruction.Encoding switch + { + Gen5ShaderEncoding.Sop1 => Family.Sop1, + Gen5ShaderEncoding.Sop2 => Family.Sop2, + Gen5ShaderEncoding.Sopk => Family.Sopk, + Gen5ShaderEncoding.Sopc => Family.Sopc, + Gen5ShaderEncoding.Sopp => Family.Sopp, + Gen5ShaderEncoding.Smem or Gen5ShaderEncoding.Smrd => Family.Smem, + Gen5ShaderEncoding.Vop1 => Family.Vop1, + Gen5ShaderEncoding.Vop2 => Family.Vop2, + Gen5ShaderEncoding.Vop3 => Family.Vop3, + Gen5ShaderEncoding.Vop3p => Family.Vop3p, + Gen5ShaderEncoding.Vopc => Family.Vopc, + Gen5ShaderEncoding.Mubuf => Family.Mubuf, + Gen5ShaderEncoding.Mtbuf => Family.Mtbuf, + Gen5ShaderEncoding.Flat => Family.Flat, + Gen5ShaderEncoding.Mimg => Family.Mimg, + Gen5ShaderEncoding.Ds => Family.Ds, + _ => Family.Other, + }; + var wide = opcode.Contains("64", StringComparison.Ordinal); + var mapped = instruction.Sources.Select(Map).ToList(); + var destination = instruction.Destinations.Count > 0 ? Map(instruction.Destinations[0]) : Operand.None; + var destinationCount = 1; + var carry = Operand.None; + var select = -1; + var writtenStart = -1; + var writtenCount = 0; + var refused = RefusedOpcode(instruction, family, sharedFlatPcs); + List? widths = null; + + switch (family) + { + case Family.Sop1: + case Family.Sop2: + if (PartialWriteOpcodes.Contains(opcode) && destination.Kind == Kind.Scalar) + { + mapped.Add(destination); + } + + break; + case Family.Sopc: + destination = Operand.None; + break; + case Family.Sopk: + { + var register = destination; + if (opcode == "SSetregB32") + { + destination = Operand.None; + mapped = [instruction.Sources.Count > 0 ? Map(instruction.Sources[0]) : Operand.None]; + } + else if (opcode.StartsWith("SWaitcnt", StringComparison.Ordinal)) + { + destination = Operand.None; + mapped = instruction.Sources.Skip(1).Select(Map).ToList(); + } + else if (opcode.StartsWith("SCmpk", StringComparison.Ordinal)) + { + destination = Operand.None; + mapped = [register, new Operand(Kind.Constant)]; + } + else if (opcode == "SMovkI32") + { + mapped = [new Operand(Kind.Constant)]; + } + else + { + mapped = [register, new Operand(Kind.Constant)]; + } + + break; + } + case Family.Sopp: + destination = Operand.None; + mapped = []; + break; + case Family.Smem: + if (instruction.Control is Gen5GlobalMemoryControl { SourceIsScalar: true }) + { + destination = Operand.None; + widths = mapped.Select((_, index) => index == 0 ? 2 : 1).ToList(); + } + else + { + destinationCount = Math.Max(instruction.Destinations.Count, 1); + var baseWidth = opcode.StartsWith("SBuffer", StringComparison.Ordinal) ? 4 : 2; + widths = mapped.Select((_, index) => index == 0 ? baseWidth : 1).ToList(); + } + + break; + case Family.Vop1: + break; + case Family.Vop2: + if (LaneSelectOpcodes.Contains(opcode)) + { + select = 2; + if (mapped.Count < 3) + { + mapped.Add(Operand.Scalar(VccLo)); + } + } + + if (CarryOutOpcodes.Contains(opcode)) + { + carry = Operand.Scalar(VccLo); + } + + if (Accumulates(opcode) && destination.Kind == Kind.Vector) + { + mapped.Add(destination); + } + + break; + case Family.Vopc: + mapped = mapped.Take(2).ToList(); + destination = CompareDestination(instruction); + break; + case Family.Vop3: + if (opcode.StartsWith("VCmp", StringComparison.Ordinal)) + { + mapped = mapped.Take(2).ToList(); + destination = CompareDestination(instruction); + break; + } + + mapped = UsedSources(instruction, Vop3SourceCount(instruction)); + if (LaneSelectOpcodes.Contains(opcode)) + { + select = 2; + } + + if (instruction.Control is Gen5Vop3Control { ScalarDestination: { } scalarDestination }) + { + carry = Map(Gen5Operand.Scalar(scalarDestination)); + } + + if (Accumulates(opcode) && destination.Kind == Kind.Vector) + { + mapped.Add(destination); + } + + break; + case Family.Vop3p: + mapped = UsedSources(instruction, Vop3pSourceCount(instruction)); + if (Accumulates(opcode) && destination.Kind == Kind.Vector) + { + mapped.Add(destination); + } + + break; + case Family.Mubuf: + case Family.Mtbuf: + widths = mapped.Select((_, index) => index == 1 ? 4 : 1).ToList(); + if (instruction.Control is Gen5BufferMemoryControl buffer && + !opcode.Contains("Store", StringComparison.Ordinal)) + { + writtenStart = (int)buffer.VectorData; + writtenCount = (int)Math.Max(buffer.DwordCount, 1); + } + + destination = Operand.None; + break; + case Family.Flat: + widths = mapped.Select(_ => 2).ToList(); + if (instruction.Destinations.Count > 0) + { + writtenStart = (int)instruction.Destinations[0].Value; + writtenCount = instruction.Destinations.Count; + } + + destination = Operand.None; + break; + case Family.Mimg: + widths = mapped.Select(_ => 1).ToList(); + if (instruction.Control is Gen5RayIntersectControl ray) + { + widths[^1] = 4; + writtenStart = (int)ray.VectorData; + writtenCount = (int)Gen5RayIntersectControl.ResultDwords; + } + else if (instruction.Control is Gen5ImageControl image && widths.Count >= 2) + { + widths[^2] = 8; + widths[^1] = UsesSampler(opcode) ? 4 : 0; + if (!opcode.StartsWith("ImageStore", StringComparison.Ordinal)) + { + var channels = opcode.StartsWith("ImageGather", StringComparison.Ordinal) + ? 4 + : Math.Max(BitOperations.PopCount(image.Dmask), 1); + writtenStart = (int)image.VectorData; + writtenCount = channels + 1; + } + } + + destination = Operand.None; + break; + default: + if (destination.Kind == Kind.Vector) + { + writtenStart = destination.Register; + writtenCount = instruction.Destinations.Count; + } + + destination = Operand.None; + break; + } + + widths ??= mapped + .Select((_, index) => index == select ? 2 : SourceWidth(opcode, family, index, wide)) + .ToList(); + var sdwaPartial = instruction.Control is Gen5SdwaControl { DestinationSelect: not 6 } && + family is Family.Vop1 or Family.Vop2; + var resolver = Gen5IrBranchResolver.Instance; + return new Op + { + Pc = instruction.Pc, + Opcode = opcode, + Family = family, + Sources = [.. mapped], + Widths = [.. widths], + Destination = destination, + DestinationCount = destinationCount, + Carry = carry, + Select = select, + LaneFetch = instruction.Control is Gen5DppControl or Gen5Dpp8Control, + CrossRowFetch = instruction.Control is Gen5DppControl dpp && !LaneLocalDpp(dpp.Control), + PartialVector = sdwaPartial || + (opcode.Contains("16", StringComparison.Ordinal) && !opcode.StartsWith("VPk", StringComparison.Ordinal)), + WrittenVectorStart = writtenStart, + WrittenVectorCount = writtenCount, + Wide = wide, + Refused = refused, + IsBranch = resolver.IsBranch(instruction), + IsConditional = resolver.IsConditional(instruction), + IsTerminator = Gen5IrBranchResolver.IsTerminator(instruction), + Condition = opcode switch + { + "SCbranchExecz" => Condition.ExecZero, + "SCbranchExecnz" => Condition.ExecNonZero, + "SCbranchVccz" => Condition.VccZero, + "SCbranchVccnz" => Condition.VccNonZero, + _ when resolver.IsConditional(instruction) => Condition.Other, + _ => Condition.None, + }, + }; + } + + private static int SourceWidth(string opcode, Family family, int index, bool wide) + { + if (family is not (Family.Vop1 or Family.Vop2 or Family.Vop3 or Family.Vopc)) + { + return wide ? 2 : 1; + } + + return opcode switch + { + "VMadU64U32" or "VMadI64I32" => index == 2 ? 2 : 1, + "VLshlrevB64" or "VLshrrevB64" or "VAshrrevI64" => index == 1 ? 2 : 1, + "VLdexpF64" or "VTrigPreopF64" => index == 0 ? 2 : 1, + _ when opcode.StartsWith("VCvtF64", StringComparison.Ordinal) => 1, + _ => wide ? 2 : 1, + }; + } + + private static Operand Map(Gen5Operand operand) => operand.Kind switch + { + Gen5OperandKind.ScalarRegister => operand.Value is < ScalarCount and not Scc + ? Operand.Scalar((int)operand.Value) + : Operand.None, + Gen5OperandKind.VectorRegister => new Operand(Kind.Vector, (int)operand.Value), + Gen5OperandKind.EncodedConstant => operand.Value switch + { + 128 => new Operand(Kind.Zero), + 193 => new Operand(Kind.AllOnes), + 251 => new Operand(Kind.VccZ), + 252 => new Operand(Kind.ExecZ), + 253 => new Operand(Kind.Scc), + 125 => Operand.None, + _ => new Operand(Kind.Constant), + }, + _ => new Operand(Kind.Constant), + }; + + private static Operand CompareDestination(Gen5ShaderInstruction instruction) + { + if (instruction.Opcode.StartsWith("VCmpx", StringComparison.Ordinal)) + { + return Operand.Scalar(ExecLo); + } + + var register = instruction.Control switch + { + Gen5SdwaControl { ScalarDestination: { } sdwa } => sdwa, + Gen5Vop3Control { ScalarDestination: { } vop3 } => vop3, + _ => (uint)VccLo, + }; + return register is < ScalarCount and not Scc ? Operand.Scalar((int)register) : Operand.None; + } + + private static List UsedSources(Gen5ShaderInstruction instruction, int count) + { + var used = new List(instruction.Sources.Count); + for (var index = 0; index < instruction.Sources.Count; index++) + { + var source = instruction.Sources[index]; + if (index < count || source is not { Kind: Gen5OperandKind.ScalarRegister, Value: 0 }) + { + used.Add(Map(source)); + } + } + + return used; + } + + private static int Vop3SourceCount(Gen5ShaderInstruction instruction) + { + var opcode = (instruction.Words[0] >> 16) & 0x3FF; + return opcode switch + { + < 0x100 => 2, + 0x101 or 0x128 or 0x129 or 0x12A => 3, + < 0x140 => 2, + >= 0x164 and <= 0x16C => 2, + 0x174 => 2, + < 0x180 => 3, + < 0x200 => 1, + >= 0x2FF and <= 0x31F => 2, + >= 0x360 and <= 0x36C => 2, + 0x376 or 0x37F => 2, + _ => 3, + }; + } + + private static int Vop3pSourceCount(Gen5ShaderInstruction instruction) + { + var opcode = (instruction.Words[0] >> 16) & 0x7F; + return opcode is 0x00 or 0x09 or 0x0E or (>= 0x13 and <= 0x19) or (>= 0x20 and <= 0x22) ? 3 : 2; + } + + private static bool Accumulates(string opcode) => + opcode.Contains("Mac", StringComparison.Ordinal) || + opcode.StartsWith("VDot2c", StringComparison.Ordinal) || + opcode.StartsWith("VDot4c", StringComparison.Ordinal) || + opcode.StartsWith("VDot8c", StringComparison.Ordinal); + + private static bool UsesSampler(string opcode) => + opcode.StartsWith("ImageSample", StringComparison.Ordinal) || + opcode.StartsWith("ImageGather", StringComparison.Ordinal) || + opcode == "ImageGetLod"; + + private static string? RefusedOpcode(Gen5ShaderInstruction instruction, Family family, IReadOnlySet sharedFlatPcs) + { + var opcode = instruction.Opcode; + switch (opcode) + { + case "VReadlaneB32": + return "V_READLANE"; + case "VWritelaneB32": + return "V_WRITELANE"; + case "VMbcntLoU32B32": + case "VMbcntHiU32B32": + return "V_MBCNT"; + case "SQuadmaskB64": + case "SBitreplicateB64B32": + return "lane-mask layout"; + case "SGetpcB64": + case "SSetpcB64": + case "SSwappcB64": + case "SCbranchJoin": + case "SCbranchGFork": + case "SCbranchIFork": + return "program counter"; + case "SSendmsg": + case "STrap": + return "wave message"; + case "ImageGetLod": + return "compute derivatives"; + } + + if (opcode.StartsWith("VMovrel", StringComparison.Ordinal) || opcode.StartsWith("SMovrel", StringComparison.Ordinal)) + { + return "relative register"; + } + + if (opcode.StartsWith("SSetreg", StringComparison.Ordinal)) + { + return "wave message"; + } + + if (opcode.StartsWith("SRfe", StringComparison.Ordinal) || + opcode.StartsWith("SCall", StringComparison.Ordinal) || + opcode.StartsWith("SSubvector", StringComparison.Ordinal)) + { + return "program counter"; + } + + if (opcode.Contains("Wrexec", StringComparison.Ordinal)) + { + return "EXEC set from the wave's layout"; + } + + if (opcode.Contains("Raw", StringComparison.Ordinal)) + { + return "unknown instruction"; + } + + if (family == Family.Smem && opcode.Contains("Atomic", StringComparison.Ordinal)) + { + return "scalar atomic"; + } + + if (family == Family.Ds || (family == Family.Flat && sharedFlatPcs.Contains(instruction.Pc))) + { + return "LDS"; + } + + return null; + } + + private static bool LaneLocalDpp(uint control) => + control <= 0xFF || + control is >= 0x101 and <= 0x12F || + control is 0x140 or 0x141 || + control is >= 0x150 and <= 0x16F; + + private static bool TryBuildBlocks(IReadOnlyList instructions, Op[] ops, out Block[] blocks) + { + blocks = []; + var count = ops.Length; + var indexByPc = new Dictionary(count); + for (var index = 0; index < count; index++) + { + indexByPc[ops[index].Pc] = index; + } + + var last = instructions[^1]; + var endPc = last.Pc + (uint)(last.Words.Count * sizeof(uint)); + var leaders = new bool[count + 1]; + var targets = new int[count]; + leaders[0] = true; + for (var index = 0; index < count; index++) + { + targets[index] = -1; + var op = ops[index]; + if (op.IsTerminator) + { + leaders[index + 1] = true; + continue; + } + + if (!op.IsBranch) + { + continue; + } + + if (!Gen5IrBranchResolver.Instance.TryGetBranchTarget(instructions[index], out var targetPc)) + { + return false; + } + + if (indexByPc.TryGetValue(targetPc, out var target)) + { + targets[index] = target; + leaders[target] = true; + } + else if (targetPc < endPc) + { + return false; + } + + leaders[index + 1] = true; + } + + var starts = new List(); + for (var index = 0; index < count; index++) + { + if (leaders[index]) + { + starts.Add(index); + } + } + + var blockOf = new int[count]; + var built = new Block[starts.Count]; + for (var block = 0; block < starts.Count; block++) + { + var start = starts[block]; + var end = block + 1 < starts.Count ? starts[block + 1] : count; + built[block] = new Block { Start = start, End = end }; + for (var index = start; index < end; index++) + { + blockOf[index] = block; + } + } + + for (var block = 0; block < built.Length; block++) + { + var current = built[block]; + var lastIndex = current.End - 1; + var op = ops[lastIndex]; + var next = block + 1 < built.Length ? block + 1 : NoBlock; + if (op.IsTerminator) + { + continue; + } + + if (!op.IsBranch) + { + if (next != NoBlock) + { + current.Successors.Add(next); + } + + continue; + } + + var targetBlock = targets[lastIndex] >= 0 ? blockOf[targets[lastIndex]] : NoBlock; + if (!op.IsConditional) + { + if (targetBlock != NoBlock) + { + current.Successors.Add(targetBlock); + } + + continue; + } + + current.Condition = op.Condition; + current.TrueBlock = targetBlock; + current.FalseBlock = next; + if (targetBlock != NoBlock) + { + current.Successors.Add(targetBlock); + } + + if (next != NoBlock && !current.Successors.Contains(next)) + { + current.Successors.Add(next); + } + } + + blocks = built; + return true; + } + + private static bool PairBase(int register) => + register >= 0 && register % 2 == 0 && (register < M0 || register == ExecLo); + + private static bool Test(UInt128 set, int register) => + register is >= 0 and < ScalarCount && ((set >> register) & UInt128.One) != 0; + + private static UInt128 Bit(int register) => + register is >= 0 and < ScalarCount ? UInt128.One << register : UInt128.Zero; + + private static void SetPair(ref UInt128 set, int register, bool value) + { + if (!PairBase(register) || register == ExecLo) + { + return; + } + + var pair = Bit(register) | Bit(register + 1); + set = value ? set | pair : set & ~pair; + } + + private static bool IsSaveExec(string opcode) => + opcode.EndsWith("SaveexecB64", StringComparison.Ordinal) || + opcode.EndsWith("SaveexecB32", StringComparison.Ordinal); + + private static bool ScalarDestination64(string opcode) => + ScalarDestination64Opcodes.Contains(opcode) || opcode.EndsWith("SaveexecB64", StringComparison.Ordinal); + + private static bool IsMaskOperation(string opcode) => + MaskOperations.Contains(opcode) || IsSaveExec(opcode); + + private static bool WritesScc(string opcode) + { + if (opcode.EndsWith("F32", StringComparison.Ordinal) || opcode.EndsWith("F16", StringComparison.Ordinal)) + { + return false; + } + + foreach (var prefix in SccWriterPrefixes) + { + if (opcode.StartsWith(prefix, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + private static bool IsCompare(Op op) => + op.Family == Family.Vopc || + (op.Family == Family.Vop3 && op.Opcode.StartsWith("VCmp", StringComparison.Ordinal)); + + private static bool WritesExec(Op op) + { + var index = op.Destination.ScalarIndex; + return index is ExecLo or ExecHi || IsSaveExec(op.Opcode); + } + + private static UInt128 ScalarRange(Operand operand, int count) + { + var register = operand.ScalarIndex; + var set = UInt128.Zero; + for (var offset = 0; register >= 0 && offset < count && register + offset < ScalarCount; offset++) + { + set |= Bit(register + offset); + } + + return set; + } + + private static UInt128 ScalarUses(Op op) + { + var uses = UInt128.Zero; + for (var index = 0; index < op.Sources.Length; index++) + { + var source = op.Sources[index]; + if (source.Kind == Kind.Scc) + { + uses |= Bit(Scc); + } + + uses |= ScalarRange(source, op.Widths[index]); + } + + if (SccReaders.Contains(op.Opcode)) + { + uses |= Bit(Scc); + } + + if (op.Opcode is "SCbranchVccz" or "SCbranchVccnz") + { + uses |= Bit(VccLo) | Bit(VccHi); + } + + return uses; + } + + private static UInt128 ScalarDefs(Op op, bool sure) + { + var defs = UInt128.Zero; + switch (op.Family) + { + case Family.Sop1: + case Family.Sop2: + case Family.Sopk: + case Family.Sopc: + { + var wide = ScalarDestination64(op.Opcode) || (!sure && op.Wide); + if (!sure || !PartialWriteOpcodes.Contains(op.Opcode)) + { + defs |= ScalarRange(op.Destination, wide ? 2 : 1); + } + + if (IsSaveExec(op.Opcode)) + { + defs |= Bit(ExecLo); + if (op.Wide) + { + defs |= Bit(ExecHi); + } + } + + if (!sure || WritesScc(op.Opcode)) + { + defs |= Bit(Scc); + } + + break; + } + case Family.Smem: + defs |= ScalarRange(op.Destination, op.DestinationCount); + break; + case Family.Vopc: + defs |= ScalarRange(op.Destination, 2); + break; + case Family.Vop1: + case Family.Vop2: + case Family.Vop3: + case Family.Vop3p: + if (op.Opcode is "VReadfirstlaneB32" or "VReadlaneB32") + { + defs |= ScalarRange(op.Destination, 1); + } + else if (op.Destination.Kind == Kind.Scalar) + { + defs |= ScalarRange(op.Destination, 2); + } + + defs |= ScalarRange(op.Carry, 2); + break; + } + + return defs; + } + + private sealed class Analysis(Op[] ops, Block[] blocks) + { + private readonly Op[] _ops = ops; + private readonly Block[] _blocks = blocks; + private readonly List _loops = []; + + public bool Run(out string reason) + { + if (!FindLoops()) + { + reason = "control flow"; + return false; + } + + var count = _blocks.Length; + var input = new State[count]; + var reached = new bool[count]; + var queued = new bool[count]; + input[0].Mask = Bit(ExecLo) | Bit(ExecHi); + input[0].Active = true; + reached[0] = true; + queued[0] = true; + var work = new Queue(); + work.Enqueue(0); + for (var steps = 0; work.TryDequeue(out var block); steps++) + { + if (steps > MaxSteps) + { + reason = "analysis did not converge"; + return false; + } + + queued[block] = false; + var state = input[block]; + for (var index = _blocks[block].Start; index < _blocks[block].End; index++) + { + Transfer(_ops[index], ref state); + } + + foreach (var successor in _blocks[block].Successors) + { + var edge = OnEdge(_blocks[block], successor, state); + if (reached[successor]) + { + var merged = input[successor]; + merged.Join(edge); + if (merged.Equals(input[successor])) + { + continue; + } + + edge = merged; + } + + reached[successor] = true; + input[successor] = edge; + if (!queued[successor]) + { + queued[successor] = true; + work.Enqueue(successor); + } + } + } + + var output = new State[count]; + for (var block = 0; block < count; block++) + { + if (!reached[block]) + { + continue; + } + + var state = input[block]; + for (var index = _blocks[block].Start; index < _blocks[block].End; index++) + { + if (Check(block, index, state) is { } refused) + { + reason = $"{refused} (0x{_ops[index].Pc:X})"; + return false; + } + + Transfer(_ops[index], ref state); + } + + output[block] = state; + } + + return WaveBranches(output, reached, out reason); + } + + private bool FindLoops() + { + var count = _blocks.Length; + var reachable = new bool[count]; + var order = new List(); + var onStack = new bool[count]; + var retreating = new List<(int From, int To)>(); + var stack = new Stack<(int Block, int Next)>(); + stack.Push((0, 0)); + reachable[0] = true; + onStack[0] = true; + while (stack.Count > 0) + { + var (block, next) = stack.Pop(); + var successors = _blocks[block].Successors; + if (next < successors.Count) + { + stack.Push((block, next + 1)); + var successor = successors[next]; + if (onStack[successor]) + { + retreating.Add((block, successor)); + } + else if (!reachable[successor]) + { + reachable[successor] = true; + onStack[successor] = true; + stack.Push((successor, 0)); + } + + continue; + } + + onStack[block] = false; + order.Add(block); + } + + var predecessors = new List[count]; + for (var block = 0; block < count; block++) + { + predecessors[block] = []; + } + + for (var block = 0; block < count; block++) + { + foreach (var successor in _blocks[block].Successors) + { + predecessors[successor].Add(block); + } + } + + var dominators = new bool[count][]; + for (var block = 0; block < count; block++) + { + dominators[block] = new bool[count]; + if (block == 0) + { + dominators[block][0] = true; + } + else + { + Array.Fill(dominators[block], true); + } + } + + order.Reverse(); + for (var changed = true; changed;) + { + changed = false; + foreach (var block in order) + { + if (block == 0) + { + continue; + } + + var next = new bool[count]; + Array.Fill(next, true); + var any = false; + foreach (var predecessor in predecessors[block]) + { + if (!reachable[predecessor]) + { + continue; + } + + any = true; + for (var other = 0; other < count; other++) + { + next[other] &= dominators[predecessor][other]; + } + } + + if (!any) + { + Array.Fill(next, false); + } + + next[block] = true; + if (!next.AsSpan().SequenceEqual(dominators[block])) + { + dominators[block] = next; + changed = true; + } + } + } + + foreach (var (from, to) in retreating) + { + if (!dominators[from][to]) + { + return false; + } + + var body = new bool[count]; + body[to] = true; + var pending = new Stack(); + if (!body[from]) + { + body[from] = true; + pending.Push(from); + } + + while (pending.TryPop(out var block)) + { + foreach (var predecessor in predecessors[block]) + { + if (!body[predecessor]) + { + body[predecessor] = true; + pending.Push(predecessor); + } + } + } + + _loops.Add(new Loop(to, from, body)); + } + + return true; + } + + private static bool IsMask(Op op, int source, in State state) => + (ScalarRange(op.Sources[source], op.Widths[source]) & state.Mask) != 0; + + private static void WriteScalar(Operand operand, int count, bool mask, ref State state) + { + var index = operand.ScalarIndex; + for (var offset = 0; index >= 0 && offset < count && index + offset < ScalarCount; offset++) + { + var register = index + offset; + state.Mask = mask || register is ExecLo or ExecHi + ? state.Mask | Bit(register) + : state.Mask & ~Bit(register); + } + } + + private static bool MaskIn(Operand operand, int count, in State state) => + (ScalarRange(operand, count) & state.Mask) != 0; + + private static bool UniformSources(Op op, in State state) + { + if (op.LaneFetch) + { + return false; + } + + for (var index = 0; index < op.Sources.Length; index++) + { + var source = op.Sources[index]; + switch (source.Kind) + { + case Kind.Vector: + if (!state.Uniform(source.Register) || (op.Widths[index] > 1 && !state.Uniform(source.Register + 1))) + { + return false; + } + + break; + case Kind.Zero: + case Kind.AllOnes: + case Kind.Constant: + break; + default: + if (source.ScalarIndex < 0 || IsMask(op, index, state)) + { + return false; + } + + break; + } + } + + return true; + } + + private static MaskFacts PairFacts(int register, in State state) + { + if (register == ExecLo) + { + return new MaskFacts(true, true, state.Active); + } + + if (!PairBase(register)) + { + return default; + } + + return new MaskFacts( + Test(state.Covers, register) && Test(state.Covers, register + 1), + Test(state.Within, register) && Test(state.Within, register + 1), + Test(state.Nonzero, register) && Test(state.Nonzero, register + 1)); + } + + private static MaskFacts Facts(Operand operand, in State state) => operand.Kind switch + { + Kind.Zero => new MaskFacts(true, true, false), + Kind.AllOnes => new MaskFacts(true, false, true), + Kind.Scalar => PairFacts(operand.Register, state), + _ => default, + }; + + private static bool LaneMask(Operand operand, in State state) + { + if (operand.Kind is Kind.Zero or Kind.AllOnes) + { + return true; + } + + var register = operand.ScalarIndex; + if (register == ExecLo) + { + return true; + } + + return PairBase(register) && Test(state.Lanes, register) && Test(state.Lanes, register + 1); + } + + private static bool UpdateMaskFacts(Op op, in State before, ref State state) + { + var written = ScalarDefs(op, false); + var writesExec = Test(written, ExecLo) || Test(written, ExecHi); + var a = Facts(op.Source(0), before); + var b = Facts(op.Source(1), before); + var destination = op.Destination.ScalarIndex; + var value = default(MaskFacts); + var pair = true; + switch (op.Opcode) + { + case "SMovB64": + value = a; + break; + case "SCselectB64": + value = new MaskFacts(a.Covers && b.Covers, a.Within && b.Within, a.Nonzero && b.Nonzero); + break; + case "SAndB64": + value = new MaskFacts(a.Covers && b.Covers, a.Within || b.Within, false); + break; + case "SOrB64": + value = new MaskFacts(a.Covers && b.Covers, a.Within && b.Within, a.Nonzero || b.Nonzero); + break; + case "SAndn2B64": + value = new MaskFacts(false, a.Within, false); + break; + default: + pair = IsCompare(op); + if (pair) + { + value = new MaskFacts(UniformSources(op, before), true, false); + } + + break; + } + + var save = IsSaveExec(op.Opcode); + var narrowing = save + ? op.Opcode is "SAndSaveexecB64" or "SAndn1SaveexecB64" + : destination == ExecLo && pair && value.Within; + state.Covers &= ~written; + state.Within &= ~written; + state.Nonzero &= ~written; + if (writesExec) + { + if (!narrowing) + { + state.Covers = UInt128.Zero; + } + + state.Within = UInt128.Zero; + state.Active = !save && destination == ExecLo && pair && value.Nonzero; + if (!save && destination == ExecLo) + { + switch (op.Opcode) + { + case "SMovB64": + SetPair(ref state.Covers, op.Source(0).ScalarIndex, true); + SetPair(ref state.Within, op.Source(0).ScalarIndex, true); + break; + case "SAndB64": + SetPair(ref state.Covers, op.Source(0).ScalarIndex, true); + SetPair(ref state.Covers, op.Source(1).ScalarIndex, true); + break; + case "SAndn2B64": + SetPair(ref state.Covers, op.Source(0).ScalarIndex, true); + break; + case "SOrB64": + SetPair(ref state.Within, op.Source(0).ScalarIndex, true); + SetPair(ref state.Within, op.Source(1).ScalarIndex, true); + break; + } + } + } + + if (save) + { + SetPair(ref state.Covers, destination, narrowing); + SetPair(ref state.Nonzero, destination, before.Active); + } + else if (pair && destination != ExecLo) + { + SetPair(ref state.Covers, destination, value.Covers); + SetPair(ref state.Within, destination, value.Within); + SetPair(ref state.Nonzero, destination, value.Nonzero); + } + + return writesExec && narrowing; + } + + private static void Transfer(Op op, ref State state) + { + var before = state; + var narrowed = UpdateMaskFacts(op, before, ref state); + if (WritesExec(op) && !narrowed) + { + state.UniformLow = UInt128.Zero; + state.UniformHigh = UInt128.Zero; + } + + state.Lanes &= ~ScalarDefs(op, false); + switch (op.Family) + { + case Family.Sop1: + case Family.Sop2: + case Family.Sopk: + case Family.Sopc: + { + var mask = false; + for (var source = 0; source < op.Sources.Length; source++) + { + mask = mask || IsMask(op, source, before); + } + + if (SccReaders.Contains(op.Opcode)) + { + mask = mask || before.SccMask; + } + + var resultMask = mask && IsMaskOperation(op.Opcode); + if (IsSaveExec(op.Opcode)) + { + WriteScalar(op.Destination, 2, true, ref state); + SetPair(ref state.Lanes, op.Destination.ScalarIndex, op.Wide); + state.SccMask = true; + break; + } + + if (op.Family != Family.Sopc && op.Destination.Kind == Kind.Scalar) + { + WriteScalar( + op.Destination, + ScalarDestination64(op.Opcode) ? 2 : 1, + resultMask, + ref state); + var lanes = IsMaskOperation(op.Opcode) && op.Wide; + foreach (var source in op.Sources) + { + lanes = lanes && LaneMask(source, before); + } + + SetPair(ref state.Lanes, op.Destination.ScalarIndex, lanes); + } + + if (WritesScc(op.Opcode)) + { + state.SccMask = mask; + } + else if (!SccKeepers.Contains(op.Opcode)) + { + state.SccMask = before.SccMask || mask; + } + + break; + } + case Family.Smem: + WriteScalar(op.Destination, op.DestinationCount, false, ref state); + break; + case Family.Vopc: + WriteScalar(op.Destination, 2, true, ref state); + SetPair(ref state.Lanes, op.Destination.ScalarIndex, true); + break; + case Family.Vop1: + case Family.Vop2: + case Family.Vop3: + case Family.Vop3p: + { + if (op.Opcode is "VReadfirstlaneB32" or "VReadlaneB32") + { + WriteScalar(op.Destination, 1, false, ref state); + break; + } + + if (op.Family == Family.Vop3 && op.Destination.Kind == Kind.Scalar) + { + WriteScalar(op.Destination, 2, true, ref state); + SetPair(ref state.Lanes, op.Destination.ScalarIndex, IsCompare(op)); + break; + } + + if (op.Carry.Kind == Kind.Scalar) + { + WriteScalar(op.Carry, 2, true, ref state); + SetPair(ref state.Lanes, op.Carry.ScalarIndex, true); + } + + if (op.Destination.Kind == Kind.Vector) + { + var uniform = op.Select < 0 && + op.Opcode is not ("VPermlane16B32" or "VPermlanex16B32") && + UniformSources(op, before); + var register = op.Destination.Register; + state.SetUniform(register, uniform && (!op.PartialVector || before.Uniform(register))); + if (op.Wide) + { + state.SetUniform(register + 1, uniform && before.Uniform(register + 1)); + } + } + + break; + } + default: + for (var offset = 0; op.WrittenVectorStart >= 0 && offset < op.WrittenVectorCount; offset++) + { + state.SetUniform(op.WrittenVectorStart + offset, false); + } + + break; + } + } + + private static State OnEdge(Block block, int successor, State state) + { + if (block.Condition == Condition.None || block.TrueBlock == block.FalseBlock) + { + return state; + } + + var vccWithin = PairFacts(VccLo, state).Within; + switch (block.Condition) + { + case Condition.ExecZero: + state.Active = state.Active || successor == block.FalseBlock; + break; + case Condition.ExecNonZero: + state.Active = state.Active || successor == block.TrueBlock; + break; + case Condition.VccZero: + state.Active = state.Active || (vccWithin && successor == block.FalseBlock); + break; + case Condition.VccNonZero: + state.Active = state.Active || (vccWithin && successor == block.TrueBlock); + break; + } + + return state; + } + + private string? Check(int block, int index, in State state) + { + var op = _ops[index]; + if (op.Refused is { } refused) + { + return refused; + } + + var written = ScalarDefs(op, false); + var sop = op.Family is Family.Sop1 or Family.Sop2; + if ((Test(written, ExecLo) || Test(written, ExecHi)) && + !IsCompare(op) && + !(sop && IsMaskOperation(op.Opcode) && op.Wide)) + { + return "EXEC set from the wave's layout"; + } + + if (op.CrossRowFetch) + { + return "cross-row DPP"; + } + + foreach (var source in op.Sources) + { + if (source.Kind is Kind.VccZ or Kind.ExecZ) + { + return "VCCZ/EXECZ as data"; + } + + if (source.Kind == Kind.Scc && state.SccMask) + { + return "SCC made from a lane mask"; + } + } + + if (state.SccMask && SccReaders.Contains(op.Opcode)) + { + return "SCC made from a lane mask"; + } + + switch (op.Family) + { + case Family.Sop1: + case Family.Sop2: + case Family.Sopk: + case Family.Sopc: + { + var mask = false; + for (var source = 0; source < op.Sources.Length; source++) + { + mask = mask || IsMask(op, source, state); + } + + if (mask && !IsMaskOperation(op.Opcode)) + { + return "a lane mask used as a scalar value"; + } + + if (op.Destination.ScalarIndex is ExecLo or ExecHi || IsSaveExec(op.Opcode)) + { + var layout = !op.Wide; + foreach (var source in op.Sources) + { + layout = layout || !LaneMask(source, state); + } + + if (layout) + { + return "EXEC set from the wave's layout"; + } + } + + break; + } + case Family.Sopp: + if (op.Opcode is "SCbranchVccz" or "SCbranchVccnz" && + (!PairFacts(VccLo, state).Covers || !Test(state.Lanes, VccLo) || !Test(state.Lanes, VccHi))) + { + return "a branch on VCC without EXEC's lanes"; + } + + break; + case Family.Smem: + case Family.Mubuf: + case Family.Mtbuf: + case Family.Flat: + case Family.Mimg: + for (var source = 0; source < op.Sources.Length; source++) + { + if (MaskIn(op.Sources[source], op.Widths[source], state)) + { + return "a lane mask used as an address or resource"; + } + } + + break; + case Family.Vop1: + case Family.Vop2: + case Family.Vop3: + case Family.Vop3p: + case Family.Vopc: + { + for (var source = 0; source < op.Sources.Length; source++) + { + if (source != op.Select && IsMask(op, source, state)) + { + return "a lane mask broadcast to the lanes"; + } + } + + if (op.Select >= 0 && !LaneMask(op.Source(op.Select), state)) + { + return "a lane select from the wave's layout"; + } + + if (op.Opcode == "VReadfirstlaneB32") + { + if (!state.Active) + { + return "V_READFIRSTLANE where a half may have no active lane"; + } + + var source = op.Source(0); + var uniform = source.Kind == Kind.Vector + ? state.Uniform(source.Register) + : op.Sources.Length == 0 || !IsMask(op, 0, state); + if (!uniform && !Waterfall(block, index)) + { + return "V_READFIRSTLANE of a lane value outside a waterfall loop"; + } + } + + break; + } + } + + return null; + } + + private bool Waterfall(int block, int index) + { + var head = _ops[index]; + if (head.Source(0).Kind != Kind.Vector) + { + return false; + } + + var value = head.Source(0).Register; + var scalar = head.Destination.ScalarIndex; + if (scalar < 0) + { + return false; + } + + var compareMask = -1; + for (var position = index + 1; position < _blocks[block].End; position++) + { + var op = _ops[position]; + if (compareMask < 0) + { + if (!op.Opcode.StartsWith("VCmpEq", StringComparison.Ordinal) && + !op.Opcode.StartsWith("VCmpxEq", StringComparison.Ordinal)) + { + continue; + } + + var wide = op.Wide; + bool Matches(Operand vector, Operand uniform) + { + if (vector.Kind != Kind.Vector || uniform.ScalarIndex < 0) + { + return false; + } + + var vectorDelta = value - vector.Register; + var scalarDelta = scalar - uniform.ScalarIndex; + return vectorDelta == scalarDelta && vectorDelta >= 0 && vectorDelta <= (wide ? 1 : 0); + } + + if (!Matches(op.Source(0), op.Source(1)) && !Matches(op.Source(1), op.Source(0))) + { + continue; + } + + compareMask = op.Destination.ScalarIndex; + if (compareMask == ExecLo) + { + break; + } + + if (compareMask < 0) + { + return false; + } + + continue; + } + + if (IsSaveExec(op.Opcode) && op.Source(0).ScalarIndex == compareMask) + { + compareMask = ExecLo; + break; + } + } + + if (compareMask != ExecLo) + { + return false; + } + + foreach (var loop in _loops) + { + if (!loop.Body[block] && loop.Header != block) + { + continue; + } + + var latch = _blocks[loop.Latch]; + if (latch.End > latch.Start && _ops[latch.End - 1].Opcode == "SCbranchExecnz") + { + return true; + } + } + + return false; + } + + private static string? SymTransfer(Op op, Sym[] syms) + { + if (op.Opcode == "SBarrier") + { + return "a barrier"; + } + + var written = ScalarDefs(op, false); + if (written == UInt128.Zero) + { + return null; + } + + var destination = op.Destination.ScalarIndex; + Sym Pair(Operand operand) + { + if (operand.Kind == Kind.Zero) + { + return Sym.Zero; + } + + if (operand.Kind == Kind.AllOnes) + { + return Sym.Ones; + } + + var register = operand.ScalarIndex; + if (!PairBase(register) || syms[register] != syms[register + 1]) + { + return Sym.Unknown; + } + + return syms[register] == Sym.Original && register != destination ? Sym.Unknown : syms[register]; + } + + var a = Pair(op.Source(0)); + var b = Pair(op.Source(1)); + var e = syms[ExecLo] == syms[ExecHi] ? syms[ExecLo] : Sym.Unknown; + var value = Sym.Unknown; + var exec = Sym.Unknown; + switch (op.Opcode) + { + case "SMovB64": + value = a; + break; + case "SCselectB64": + value = a == b ? a : Sym.Unknown; + break; + case "SNotB64": + value = Not(a); + break; + case "SWqmB64": + value = a is Sym.Zero or Sym.Ones ? a : Sym.Unknown; + break; + case "SAndB64": + value = And(a, b); + break; + case "SOrB64": + value = Or(a, b); + break; + case "SXorB64": + value = Xor(a, b); + break; + case "SAndn2B64": + value = And(a, Not(b)); + break; + case "SOrn2B64": + value = Or(a, Not(b)); + break; + case "SNandB64": + value = Not(And(a, b)); + break; + case "SNorB64": + value = Not(Or(a, b)); + break; + case "SXnorB64": + value = Not(Xor(a, b)); + break; + case "SAndSaveexecB64": + value = e; + exec = And(a, e); + break; + case "SAndn1SaveexecB64": + value = e; + exec = And(Not(a), e); + break; + case "SAndn2SaveexecB64": + value = e; + exec = And(a, Not(e)); + break; + case "SOrSaveexecB64": + value = e; + exec = Or(a, e); + break; + case "SOrn1SaveexecB64": + value = e; + exec = Or(Not(a), e); + break; + case "SOrn2SaveexecB64": + value = e; + exec = Or(a, Not(e)); + break; + case "SXorSaveexecB64": + value = e; + exec = Xor(a, e); + break; + case "SNandSaveexecB64": + value = e; + exec = Not(And(a, e)); + break; + case "SNorSaveexecB64": + value = e; + exec = Not(Or(a, e)); + break; + case "SXnorSaveexecB64": + value = e; + exec = Not(Xor(a, e)); + break; + default: + if (IsCompare(op)) + { + value = e == Sym.Zero ? Sym.Zero : Sym.Unknown; + } + + break; + } + + for (var register = 0; register < ScalarCount; register++) + { + if (Test(written, register)) + { + syms[register] = Sym.Unknown; + } + } + + if (IsSaveExec(op.Opcode)) + { + syms[ExecLo] = exec; + syms[ExecHi] = exec; + } + + if (PairBase(destination) && (ScalarDestination64(op.Opcode) || IsCompare(op))) + { + syms[destination] = value; + syms[destination + 1] = value; + } + + return syms[ExecLo] != Sym.Zero || syms[ExecHi] != Sym.Zero ? "EXEC gaining a half's lanes" : null; + } + + private static Sym Not(Sym value) => value switch + { + Sym.Zero => Sym.Ones, + Sym.Ones => Sym.Zero, + _ => Sym.Unknown, + }; + + private static Sym And(Sym a, Sym b) => + a == Sym.Zero || b == Sym.Zero ? Sym.Zero + : a == Sym.Ones ? b + : b == Sym.Ones ? a + : a == b ? a + : Sym.Unknown; + + private static Sym Or(Sym a, Sym b) => + a == Sym.Ones || b == Sym.Ones ? Sym.Ones + : a == Sym.Zero ? b + : b == Sym.Zero ? a + : a == b ? a + : Sym.Unknown; + + private static Sym Xor(Sym a, Sym b) => + a == Sym.Zero ? b + : b == Sym.Zero ? a + : a == Sym.Ones ? Not(b) + : b == Sym.Ones ? Not(a) + : Sym.Unknown; + + private bool Ways(int seed, int merge, Sym[] entry, Sym[] atMerge, out string? reason) + { + reason = null; + Array.Fill(atMerge, Sym.Unknown); + if (seed == merge) + { + entry.CopyTo(atMerge, 0); + return true; + } + + static bool JoinInto(Sym[] into, Sym[] from) + { + var changed = false; + for (var register = 0; register < into.Length; register++) + { + if (into[register] != from[register] && into[register] != Sym.Unknown) + { + into[register] = Sym.Unknown; + changed = true; + } + } + + return changed; + } + + var count = _blocks.Length; + var merged = false; + var input = new Sym[count][]; + var queued = new bool[count]; + var work = new Queue(); + input[seed] = (Sym[])entry.Clone(); + queued[seed] = true; + work.Enqueue(seed); + while (work.TryDequeue(out var block)) + { + queued[block] = false; + var syms = (Sym[])input[block].Clone(); + for (var index = _blocks[block].Start; index < _blocks[block].End; index++) + { + if (SymTransfer(_ops[index], syms) is { } refused) + { + reason = $"{refused} between a lane-mask branch and its merge (0x{_ops[index].Pc:X})"; + return false; + } + } + + foreach (var successor in _blocks[block].Successors) + { + if (successor == merge) + { + if (!merged) + { + syms.CopyTo(atMerge, 0); + } + else + { + JoinInto(atMerge, syms); + } + + merged = true; + continue; + } + + if (input[successor] is null) + { + input[successor] = (Sym[])syms.Clone(); + } + else if (!JoinInto(input[successor], syms)) + { + continue; + } + + if (!queued[successor]) + { + queued[successor] = true; + work.Enqueue(successor); + } + } + } + + return true; + } + + private bool WaveBranches(State[] output, bool[] reached, out string reason) + { + var count = _blocks.Length; + var use = new UInt128[count]; + var def = new UInt128[count]; + var liveIn = new UInt128[count]; + for (var block = 0; block < count; block++) + { + for (var index = _blocks[block].Start; index < _blocks[block].End; index++) + { + use[block] |= ScalarUses(_ops[index]) & ~def[block]; + def[block] |= ScalarDefs(_ops[index], true); + } + } + + for (var changed = true; changed;) + { + changed = false; + for (var block = count - 1; block >= 0; block--) + { + var liveOut = UInt128.Zero; + foreach (var successor in _blocks[block].Successors) + { + liveOut |= liveIn[successor]; + } + + var live = use[block] | (liveOut & ~def[block]); + if (live != liveIn[block]) + { + liveIn[block] = live; + changed = true; + } + } + } + + var post = new bool[count][]; + for (var block = 0; block < count; block++) + { + post[block] = new bool[count]; + if (_blocks[block].Successors.Count == 0) + { + post[block][block] = true; + } + else + { + Array.Fill(post[block], true); + } + } + + for (var changed = true; changed;) + { + changed = false; + for (var block = count - 1; block >= 0; block--) + { + if (_blocks[block].Successors.Count == 0) + { + continue; + } + + var next = new bool[count]; + Array.Fill(next, true); + foreach (var successor in _blocks[block].Successors) + { + for (var other = 0; other < count; other++) + { + next[other] &= post[successor][other]; + } + } + + next[block] = true; + if (!next.AsSpan().SequenceEqual(post[block])) + { + post[block] = next; + changed = true; + } + } + } + + var postCounts = post.Select(set => set.Count(member => member)).ToArray(); + var half = new Sym[ScalarCount]; + var wave = new Sym[ScalarCount]; + for (var block = 0; block < count; block++) + { + var current = _blocks[block]; + if (!reached[block] || current.Condition == Condition.None || current.TrueBlock == current.FalseBlock) + { + continue; + } + + bool vcc; + bool zeroTaken; + switch (current.Condition) + { + case Condition.ExecZero: + vcc = false; + zeroTaken = true; + break; + case Condition.ExecNonZero: + vcc = false; + zeroTaken = false; + break; + case Condition.VccZero: + vcc = true; + zeroTaken = true; + break; + case Condition.VccNonZero: + vcc = true; + zeroTaken = false; + break; + default: + continue; + } + + var halfWay = zeroTaken ? current.TrueBlock : current.FalseBlock; + var waveWay = zeroTaken ? current.FalseBlock : current.TrueBlock; + if (halfWay == NoBlock || waveWay == NoBlock) + { + reason = "control flow"; + return false; + } + + var members = 0; + var common = new bool[count]; + for (var other = 0; other < count; other++) + { + common[other] = post[halfWay][other] && post[waveWay][other]; + if (common[other]) + { + members++; + } + } + + var merge = count; + for (var other = 0; other < count && merge == count; other++) + { + if (common[other] && postCounts[other] == members) + { + merge = other; + } + } + + var entry = new Sym[ScalarCount]; + entry[ExecLo] = Sym.Zero; + entry[ExecHi] = Sym.Zero; + if (vcc) + { + entry[VccLo] = Sym.Zero; + entry[VccHi] = Sym.Zero; + } + + for (var register = 0; register < M0; register += 2) + { + if (PairFacts(register, output[block]).Within) + { + entry[register] = Sym.Zero; + entry[register + 1] = Sym.Zero; + } + } + + if (!Ways(halfWay, merge, entry, half, out var halfReason)) + { + reason = halfReason!; + return false; + } + + if (!Ways(waveWay, merge, entry, wave, out var waveReason)) + { + reason = waveReason!; + return false; + } + + if (merge == count) + { + continue; + } + + var escaping = new List(); + for (var register = 0; register < ScalarCount; register++) + { + if (Test(liveIn[merge], register) && (half[register] == Sym.Unknown || half[register] != wave[register])) + { + escaping.Add(ScalarName(register)); + } + } + + if (escaping.Count != 0) + { + reason = $"scalar state escapes a lane-mask branch (0x{_ops[current.End - 1].Pc:X}: {string.Join(' ', escaping)})"; + return false; + } + } + + reason = string.Empty; + return true; + } + + private static string ScalarName(int register) => register switch + { + VccLo => "vcc_lo", + VccHi => "vcc_hi", + M0 => "m0", + Scc => "scc", + ExecLo => "exec_lo", + ExecHi => "exec_hi", + _ => $"s{register}", + }; + } +} diff --git a/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs b/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs index 9829b63..5263f83 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs @@ -18,6 +18,7 @@ public enum DescriptorBindingKind : uint FaultBuffer, FlattenedResourceTable, ShaderData, + MipStatistics, Count, } @@ -342,7 +343,11 @@ public sealed class BindingLayout : IEquatable public uint DispatchThreadLimitsDword => MemoryOffsetDword + (MemoryOffsetCount + 3) / 4; - public uint ShaderDataDwordCount => DispatchThreadLimitsDword + (UsesDispatchThreadLimits ? 3u : 0u); + public uint MipStatisticsCount { get; init; } + + public uint MipStatisticsDword => DispatchThreadLimitsDword + (UsesDispatchThreadLimits ? 3u : 0u); + + public uint ShaderDataDwordCount => MipStatisticsDword + MipStatisticsCount; public bool UsesPushData => PushDataStartDword != PushData.NoStart; @@ -359,7 +364,20 @@ public sealed class BindingLayout : IEquatable public static uint NativeBindingIndex(ShaderStage stage, DescriptorBindingKind kind) => (uint)kind + (stage == ShaderStage.Pixel ? (uint)DescriptorBindingKind.Count : 0); - public DescriptorBinding? Find(DescriptorBindingKind kind) => Descriptors.FirstOrDefault(binding => binding.Kind == kind); + // Called for every bound stage of every draw: an indexed loop allocates no closure or enumerator. + public DescriptorBinding? Find(DescriptorBindingKind kind) + { + var descriptors = Descriptors; + for (var index = 0; index < descriptors.Count; index++) + { + if (descriptors[index].Kind == kind) + { + return descriptors[index]; + } + } + + return null; + } // The user-data registers live at program entry: those some path reads before it // writes them, found by liveness over the control-flow graph, in ascending order. @@ -566,7 +584,8 @@ public sealed class BindingLayout : IEquatable var shaderBaseDword = usesShaderBase ? (uint)userDataRegisters.Count : NoShaderBase; var memoryOffsetDword = (uint)userDataRegisters.Count + (usesShaderBase ? ShaderBaseDwordCount : 0); var memoryOffsetCount = (uint)info.Buffers.Count; - var shaderDataDwords = memoryOffsetDword + (memoryOffsetCount + 3) / 4 + (usesDispatchThreadLimits ? 3u : 0u); + var mipStatisticsCount = info.Images.Any(static image => image.MipStatistics) ? (uint)info.Images.Count : 0u; + var shaderDataDwords = memoryOffsetDword + (memoryOffsetCount + 3) / 4 + (usesDispatchThreadLimits ? 3u : 0u) + mipStatisticsCount; var pushStart = PushData.StartFor(pushDataStartDword, shaderDataDwords); var descriptors = new List(); if (info.Buffers.Count != 0) @@ -636,6 +655,11 @@ public sealed class BindingLayout : IEquatable descriptors.Add(new DescriptorBinding(DescriptorBindingKind.ShaderData, [])); } + if (mipStatisticsCount != 0) + { + descriptors.Add(new DescriptorBinding(DescriptorBindingKind.MipStatistics, [])); + } + return new BindingLayout { PushDataStartDword = pushStart, @@ -644,6 +668,7 @@ public sealed class BindingLayout : IEquatable MemoryOffsetDword = memoryOffsetDword, MemoryOffsetCount = memoryOffsetCount, UsesDispatchThreadLimits = usesDispatchThreadLimits, + MipStatisticsCount = mipStatisticsCount, UserDataRegisters = userDataRegisters, Descriptors = descriptors, }; @@ -657,6 +682,7 @@ public sealed class BindingLayout : IEquatable MemoryOffsetDword == other.MemoryOffsetDword && MemoryOffsetCount == other.MemoryOffsetCount && UsesDispatchThreadLimits == other.UsesDispatchThreadLimits && + MipStatisticsCount == other.MipStatisticsCount && UserDataRegisters.SequenceEqual(other.UserDataRegisters) && Descriptors.Count == other.Descriptors.Count && Descriptors.Zip(other.Descriptors).All(pair => pair.First.Kind == pair.Second.Kind && pair.First.Resources.SequenceEqual(pair.Second.Resources)); diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs index 759e5bc..85fdec6 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs @@ -10,11 +10,14 @@ namespace SharpEmu.ShaderCompiler.Resources; // returns false so the caller falls back to a full materialization. public delegate bool ResidentGuestBytesReader(ulong address, Span destination, bool clean); -// Materialization is a pure function of the plan, the draw's user data, the shader -// base, the compute state and the guest words it reads. This cache records exactly -// those words and reuses the result while every one of them is unchanged, so a draw -// that re-binds the same resources skips the descriptor walk. Any failed read, any -// GPU-owned range and any byte difference make the entry miss; nothing is guessed. +// Materialization is a pure function of the plan, the user data words the evaluator +// reads, the shader base, the compute state and the guest words it reads. This cache +// records exactly those words and reuses the result while every one of them is +// unchanged, so a draw that re-binds the same resources skips the descriptor walk. A user +// data word the evaluator never read (a per-draw constant for the shader's own ALU, say) +// does not distinguish two draws; the snapshot a hit returns carries the current user +// data. Any failed read, any GPU-owned range and any byte difference make the entry +// miss; nothing is guessed. public sealed class ResourceMaterializationCache { // Two generations approximate LRU: a full young generation becomes the old one, @@ -24,6 +27,761 @@ public sealed class ResourceMaterializationCache private Dictionary _old = new(); private byte[] _scratch = new byte[256]; private ReadRecorder? _spareRecorder; + // The user data words each (plan, base, compute state) has read so far. They only index the + // entries; Entry.Matches is what decides a hit. + private readonly Dictionary _learnedUserData = new(); + private readonly Dictionary _rebases = new(); + + private AddressRebase RebaseOf(ulong drawKey, ShaderResourcePlan plan) + { + if (!_rebases.TryGetValue(drawKey, out var rebase)) + _rebases.Add(drawKey, rebase = new AddressRebase(plan)); + return rebase; + } + + // Titles allocate per-draw constants linearly, so a buffer's base address changes on every + // draw while everything else the draw binds stays the same; keyed on that address, the cache + // misses nearly every draw. A user data word the plan uses verbatim as a buffer's base dword + // is a candidate. It is rebased only once misses have shown it repeatedly that two draws + // differing in nothing but candidate words materialize identically, except for those base + // dwords, which equal the words: the same descriptors, table, ranges, specialization and + // recorded guest bytes. One counterexample rejects the word for good. A rebased word leaves + // the key and the match, and a hit patches the base dwords from the draw's own user data. + private sealed class AddressRebase + { + private const int Confirmations = 3; + + // A word that is a base address. A buffer candidate is the base dword of a buffer descriptor + // (Buffer >= 0): the hit patches that dword, and reads inside the descriptor's range move. + // A raw candidate is the low dword of the handle of scalar loads; its reads lie at constant + // byte offsets [MinOffset, MaxOffset] from the base whose high dword is user data word High + // (or HighConstant when High < 0). + private readonly record struct Candidate(int UserData, int Buffer, int High = -1, uint HighConstant = 0, + long MinOffset = 0, long MaxOffset = 0); + + private const ulong RawAddressMask = 0x0000_FFFF_FFFF_FFFFul; + + private readonly Candidate[] _candidates; + private readonly ulong[] _candidateMask; + private ulong[] _rejected = []; + private readonly Dictionary _confirmed = new(); + private const int RecentCapacity = 512; + private readonly Dictionary _recent = new(); + + private ulong StructureOf(Entry entry) + { + var hash = new HashCode(); + hash.Add(entry.UserData.Length); + for (var index = 0; index < entry.UserData.Length; index++) + { + if (!IsUsed(entry.UsedUserData, index) || IsUsed(_candidateMask, index)) + continue; + hash.Add(index); + hash.Add(entry.UserData[index]); + } + + return (uint)hash.ToHashCode(); + } + + private readonly ShaderResourcePlan _plan; + + public AddressRebase(ShaderResourcePlan plan) + { + _plan = plan; + var candidates = new List(); + + for (var buffer = 0; buffer < plan.Info.Buffers.Count; buffer++) + { + var source = plan.DescriptorSources[(int)plan.Info.Buffers[buffer].Source]; + if (source.Dwords.Length != 0 && UserDataIndex(source.Dwords[0]) is { } index) + candidates.Add(new Candidate(index, buffer)); + } + + var values = PlanValues(plan); + var buffered = candidates.Select(candidate => candidate.UserData).ToHashSet(); + foreach (var raw in RawCandidates(plan, values)) + if (!buffered.Contains(raw.UserData)) + candidates.Add(raw); + + _candidates = candidates.ToArray(); + _candidateMask = MaskOf(_candidates.Select(candidate => candidate.UserData)); + var memoryBases = MemoryBaseCandidates(plan); + MemoryBases = memoryBases.Select(candidate => candidate.Buffer).ToArray(); + _memoryState = new int[plan.Info.Buffers.Count]; + _memorySlots = new int[plan.Info.Buffers.Count]; + _memorySlots.AsSpan().Fill(-1); + foreach (var (buffer, slot) in memoryBases) + _memorySlots[buffer] = slot; + _memoryLearnable = memoryBases.Count; + (TableSlotBuffers, TableSlotOffsets, TableSlotsThroughBases) = TableSlotSources(plan, memoryBases); + DirectSlots = DirectSlotsOf(plan, TableSlotBuffers); + } + + // A table slot that is one dword loaded through a user data pointer at constant offsets: its + // word is that dword. High is the user data index of the pointer's high dword, or -1 for + // the constant HighConstant. + public readonly record struct DirectLoad(int Low, int High, ulong HighConstant, long Immediate, uint Offset); + + // Per table slot: its direct load (default for a slot through a memory-based buffer), or + // null when some other slot is not a direct load. + public DirectLoad[]? DirectSlots { get; } + + private DirectLoad[]? DirectSlotsOf(ShaderResourcePlan plan, int[] slotBuffers) + { + var loads = new DirectLoad[plan.TableReads.Count]; + for (var slot = 0; slot < loads.Length; slot++) + { + if (slotBuffers[slot] >= 0) + continue; + var value = plan.TableReads[slot].Value; + if (value.Kind != ScalarValueKind.ScalarAddressWord || value.Operands.Length < 2 || + value.Operands[1].Kind != ScalarValueKind.Constant || (uint)value.MemoryIndex >= (uint)plan.Memory.Count) + return null; + var handle = value.Operands[0]; + if (handle.Operands.Length < 2 || UserDataIndex(handle.Operands[0]) is not { } low) + return null; + var high = UserDataIndex(handle.Operands[1]); + if (high is null && handle.Operands[1].Kind != ScalarValueKind.Constant) + return null; + loads[slot] = new DirectLoad(low, high ?? -1, high is null ? handle.Operands[1].Payload : 0, + (int)plan.Memory[value.MemoryIndex].Offset, (uint)value.Operands[1].Payload); + } + + return loads; + } + + // Per table slot: the memory-based buffer it reads a dword of at a constant byte offset from + // the base, or -1. TableSlotsThroughBases: some slot reads through such a buffer otherwise. + public int[] TableSlotBuffers { get; } = []; + public long[] TableSlotOffsets { get; } = []; + public bool TableSlotsThroughBases { get; } + + private static (int[] Buffers, long[] Offsets, bool Through) TableSlotSources(ShaderResourcePlan plan, + List<(int Buffer, int Slot)> memoryBases) + { + var buffers = new int[plan.TableReads.Count]; + var offsets = new long[plan.TableReads.Count]; + buffers.AsSpan().Fill(-1); + if (memoryBases.Count == 0) + return (buffers, offsets, false); + + int BufferOf(ScalarValue handleWord) + { + foreach (var (buffer, slot) in memoryBases) + { + var baseWord = plan.DescriptorSources[(int)plan.Info.Buffers[buffer].Source].Dwords[0]; + if (ReferenceEquals(handleWord, baseWord) || + (slot >= 0 && handleWord.Kind == ScalarValueKind.ResourceTableWord && (int)handleWord.Payload == slot)) + return buffer; + } + + return -1; + } + + var through = false; + for (var slot = 0; slot < plan.TableReads.Count; slot++) + { + var value = plan.TableReads[slot].Value; + if (value.Kind == ScalarValueKind.ScalarBufferWord && value.Operands.Length >= 2 && + value.Operands[0].Operands.Length != 0 && value.Operands[1].Kind == ScalarValueKind.Constant && + value.MemoryIndex >= 0 && value.MemoryIndex < plan.Memory.Count && + BufferOf(value.Operands[0].Operands[0]) is var direct and >= 0) + { + buffers[slot] = direct; + offsets[slot] = ((long)plan.Memory[value.MemoryIndex].Offset + (long)(uint)value.Operands[1].Payload) & ~3L; + continue; + } + + var visited = new HashSet(); + var pending = new Stack(); + pending.Push(value); + while (!through && pending.Count != 0) + { + var current = pending.Pop(); + if (!visited.Add(current)) + continue; + if (current.Kind == ScalarValueKind.ScalarBufferWord && current.Operands.Length != 0 && + current.Operands[0].Operands.Length != 0 && BufferOf(current.Operands[0].Operands[0]) >= 0) + through = true; + foreach (var operand in current.Operands) + pending.Push(operand); + } + } + + return (buffers, offsets, through); + } + + private readonly int[] _memorySlots; + + // Memory-loaded bases, learned per buffer like the user data words: confirmations count up + // to MemoryActive; MemoryRejected is final. + private const int MemoryActive = Confirmations; + private const int MemoryRejected = -1; + private readonly int[] _memoryState; + private int _memoryLearnable; + + public IReadOnlyList MemoryBases { get; } + + public bool HasLearnableMemoryBases => _memoryLearnable != 0; + + + public bool IsRejectedMemoryBase(int buffer) => _memoryState[buffer] == MemoryRejected; + + public bool AreActiveMemoryBases(List<(int Buffer, uint Word)> patches) + { + foreach (var (buffer, _) in patches) + if (_memoryState[buffer] < MemoryActive) + return false; + return true; + } + + // A rebased refresh computed for a stale entry, compared with the full materialization of + // the same draw: agreement counts towards activating its bases, one disagreement rejects + // them for good. + public void ConfirmMemoryBases(List<(int Buffer, uint Word)> patches, bool agrees) + { + foreach (var (buffer, _) in patches) + { + if (_memoryState[buffer] == MemoryRejected) + continue; + if (!agrees) + { + _memoryState[buffer] = MemoryRejected; + _memoryLearnable--; + } + else if (_memoryState[buffer] < MemoryActive) + { + _memoryState[buffer]++; + } + } + } + + // The window of buffer reads a patched base covers for this draw. + public static bool TryMovedWindow(uint[] descriptor, uint baseWord, out ulong start, out ulong size) + { + start = size = 0; + if (descriptor.Length < 3) + return false; + start = baseWord | ((ulong)(descriptor[1] & 0xFFFF) << 32); + var stride = (descriptor[1] >> 16) & 0x3FFF; + size = stride == 0 ? descriptor[2] : (ulong)descriptor[2] * stride; + return true; + } + + // The base dword moves, and so does the table slot that holds the same word. + public ResourceSnapshot PatchMemoryBases(ResourceSnapshot snapshot, List<(int Buffer, uint Word)> patches) + { + var buffers = (uint[][])snapshot.Buffers.Clone(); + uint[]? table = null; + foreach (var (buffer, word) in patches) + { + if (ReferenceEquals(buffers[buffer], snapshot.Buffers[buffer])) + buffers[buffer] = (uint[])buffers[buffer].Clone(); + var previous = buffers[buffer][0]; + buffers[buffer][0] = word; + var slot = _memorySlots[buffer]; + var flat = slot >= 0 && slot < _plan.TableReads.Count ? (int)_plan.TableReads[slot].FlatOffset : -1; + if (flat >= 0 && flat < snapshot.FlattenedResourceTable.Length && snapshot.FlattenedResourceTable[flat] == previous) + { + table ??= (uint[])snapshot.FlattenedResourceTable.Clone(); + table[flat] = word; + } + } + + return new ResourceSnapshot + { + Buffers = buffers, + Images = snapshot.Images, + Samplers = snapshot.Samplers, + FlattenedResourceTable = table ?? snapshot.FlattenedResourceTable, + UserData = snapshot.UserData, + DeviceAddressRanges = snapshot.DeviceAddressRanges, + }; + } + + private int? UserDataIndex(ScalarValue value) + { + if (value.Kind != ScalarValueKind.UserData) + return null; + var register = value.UserDataRegister; + return register >= _plan.UserDataBase && register - _plan.UserDataBase < _plan.UserDataCount + ? (int)(register - _plan.UserDataBase) + : null; + } + + // Every scalar load of the plan whose handle takes its low dword verbatim from user data. + // A word qualifies only when all its loads use constant offsets and one high dword. + // Every value the plan evaluates, operands included. + private static HashSet PlanValues(ShaderResourcePlan plan) + { + var roots = new List(); + foreach (var source in plan.DescriptorSources) + roots.AddRange(source.Dwords); + roots.AddRange(plan.TableReads.Select(read => read.Value)); + roots.AddRange(plan.DynamicReads); + foreach (var access in plan.Accesses) + { + if (access is null) + continue; + foreach (var value in new[] { access.Handle, access.SamplerHandle, access.Read, access.Offset, access.Active }) + if (value is not null) + roots.Add(value); + } + + roots.AddRange(plan.Graph.Values); + var visited = new HashSet(); + var pending = new Stack(roots); + while (pending.Count != 0) + { + var value = pending.Pop(); + if (!visited.Add(value)) + continue; + foreach (var operand in value.Operands) + pending.Push(operand); + } + + return visited; + } + + // Buffers whose base dword is read from guest memory, directly or through a slot of the + // flattened table: descriptor tables the title rewrites per frame or per draw point them at + // freshly allocated constants. The slot (or -1) is where the table holds the same word. + // Recorded reads inside such a buffer's window move with its base. + private static List<(int Buffer, int Slot)> MemoryBaseCandidates(ShaderResourcePlan plan) + { + var bases = new List<(int Buffer, int Slot)>(); + for (var buffer = 0; buffer < plan.Info.Buffers.Count; buffer++) + { + var source = plan.DescriptorSources[(int)plan.Info.Buffers[buffer].Source]; + if (source.Dwords.Length < 2) + continue; + var baseWord = source.Dwords[0]; + var slot = baseWord.Kind == ScalarValueKind.ResourceTableWord ? (int)baseWord.Payload : -1; + if (slot < 0 && baseWord.Kind is not (ScalarValueKind.ScalarAddressWord or ScalarValueKind.ScalarBufferWord)) + continue; + bases.Add((buffer, slot)); + } + + return bases; + } + + private IEnumerable RawCandidates(ShaderResourcePlan plan, HashSet values) + { + var found = new Dictionary(); + var disqualified = new HashSet(); + foreach (var value in values) + { + if (value.Kind is not (ScalarValueKind.ScalarAddressWord or ScalarValueKind.ScalarBufferWord) || + value.Operands.Length < 2 || value.Operands[0].Operands.Length < 2 || + UserDataIndex(value.Operands[0].Operands[0]) is not { } index) + continue; + + var high = value.Operands[0].Operands[1]; + var highIndex = UserDataIndex(high); + var offset = value.Operands[1]; + if ((highIndex is null && high.Kind != ScalarValueKind.Constant) || offset.Kind != ScalarValueKind.Constant || + value.MemoryIndex >= plan.Memory.Count) + { + disqualified.Add(index); + continue; + } + + var immediate = (long)plan.Memory[value.MemoryIndex].Offset; + var relative = value.Kind == ScalarValueKind.ScalarBufferWord + ? (immediate + (long)(uint)offset.Payload) & ~3L + : (immediate & ~3L) + (long)((uint)offset.Payload & ~3u); + var candidate = new Candidate(index, -1, highIndex ?? -1, highIndex is null ? (uint)high.Payload : 0, relative, relative); + if (found.TryGetValue(index, out var existing)) + { + if (existing.High != candidate.High || existing.HighConstant != candidate.HighConstant) + { + disqualified.Add(index); + continue; + } + + candidate = existing with + { + MinOffset = Math.Min(existing.MinOffset, relative), + MaxOffset = Math.Max(existing.MaxOffset, relative), + }; + } + + found[index] = candidate; + } + + return found.Values.Where(candidate => !disqualified.Contains(candidate.UserData)); + } + + // The window [start, start + size) of reads a candidate's base covers in this entry. + private bool TryWindow(Entry entry, in Candidate candidate, out ulong start, out ulong size, out ulong addressBase) + { + start = size = addressBase = 0; + if (candidate.Buffer >= 0) + { + if (candidate.Buffer >= entry.Snapshot.Buffers.Length) + return false; + var descriptor = entry.Snapshot.Buffers[candidate.Buffer]; + if (descriptor.Length < 3) + return false; + start = addressBase = descriptor[0] | ((ulong)(descriptor[1] & 0xFFFF) << 32); + var stride = (descriptor[1] >> 16) & 0x3FFF; + size = stride == 0 ? descriptor[2] : (ulong)descriptor[2] * stride; + return true; + } + + // The window starts at the base itself: the read prefetch fetches a pointer's block from + // its base, ahead of the lowest offset a load uses. + addressBase = RawBase(entry.UserData, candidate, entry.UserData[candidate.UserData]); + var lowest = Math.Min(0, candidate.MinOffset); + var low = (long)addressBase + lowest; + if (low < 0) + return false; + start = (ulong)low; + size = (ulong)(candidate.MaxOffset - lowest) + sizeof(uint); + return true; + } + + private static ulong RawBase(IReadOnlyList userData, in Candidate candidate, uint low) + { + var high = candidate.High >= 0 ? userData[candidate.High] : candidate.HighConstant; + return (((ulong)high << 32) | low) & RawAddressMask & ~3UL; + } + + // The rebased words; replaced, never mutated, so a caller can compare references. + public ulong[] Active { get; private set; } = []; + + // Every rebased base dword of the entry equals the word it came from. + public bool AppliesTo(Entry entry) + { + if (Active.Length == 0) + return true; + foreach (var (userData, buffer, _, _, _, _) in _candidates) + { + if (buffer >= 0 && IsUsed(Active, userData) && + (buffer >= entry.Snapshot.Buffers.Length || entry.Snapshot.Buffers[buffer].Length == 0 || + userData >= entry.UserData.Length || entry.Snapshot.Buffers[buffer][0] != entry.UserData[userData])) + return false; + } + + return true; + } + + public ResourceSnapshot Apply(ResourceSnapshot snapshot, IReadOnlyList userData) + { + if (Active.Length == 0) + return snapshot; + uint[][]? buffers = null; + foreach (var (index, buffer, _, _, _, _) in _candidates) + { + if (buffer < 0 || !IsUsed(Active, index) || snapshot.Buffers[buffer][0] == userData[index]) + continue; + buffers ??= (uint[][])snapshot.Buffers.Clone(); + if (ReferenceEquals(buffers[buffer], snapshot.Buffers[buffer])) + buffers[buffer] = (uint[])buffers[buffer].Clone(); + buffers[buffer][0] = userData[index]; + } + + return buffers is null + ? snapshot + : new ResourceSnapshot + { + Buffers = buffers, + Images = snapshot.Images, + Samplers = snapshot.Samplers, + FlattenedResourceTable = snapshot.FlattenedResourceTable, + UserData = snapshot.UserData, + DeviceAddressRanges = snapshot.DeviceAddressRanges, + }; + } + + // Compares a fresh materialization with the latest one that agrees on every word but the + // candidates: the same object drawn again, usually a frame earlier, with new constants. + // Comparing with the plan's previous draw instead finds another object's textures in + // the way and never concludes. + public void Learn(Entry built, + Func? Patches)?>? predict = null) + { + if (_candidates.Length == 0) + return; + var structure = StructureOf(built); + _recent.TryGetValue(structure, out var previous); + if (_recent.Count >= RecentCapacity) + _recent.Clear(); + _recent[structure] = built; + if (previous is null || previous.UserData.Length != built.UserData.Length || + !previous.UsedUserData.AsSpan().SequenceEqual(built.UsedUserData)) + return; + + var differing = new List(); + for (var index = 0; index < built.UserData.Length; index++) + { + if (!IsUsed(built.UsedUserData, index) || previous.UserData[index] == built.UserData[index]) + continue; + if (!IsUsed(_candidateMask, index)) + return; + differing.Add(index); + } + + // A word already rejected explains any mismatch; blaming the others with it would reject + // sound bases that merely change together with it. + if (differing.Count == 0 || differing.Any(index => IsUsed(_rejected, index))) + return; + + // Reads must land where the previous draw's did, moved with a rebased buffer when they + // fall inside its window. Guest bytes that changed between the two draws make the + // comparison say nothing about the words. + var reads = CompareMovedReads(this, previous, built, differing); + if (reads == ReadComparison.Inconclusive) + { + // Changed guest bytes hide whether the words are bases. Predict this draw from the + // previous one as a hit with the words rebased would, moved memory-loaded bases + // included, and judge the words by the prediction instead. + if (predict?.Invoke(previous, Or(Active, MaskOf(differing))) is not { } prediction) + return; + + var agrees = Difference(prediction.Snapshot, previous.Specialization, built.Snapshot, built.Specialization) is null; + if (prediction.Patches is { } patches) + { + // A wrong prediction that moved bases blames those bases, not the words. + ConfirmMemoryBases(patches, agrees); + if (!agrees) + return; + } + else if (!agrees) + { + _rejected = Or(_rejected, MaskOf(differing)); + if (Overlaps(Active, _rejected)) + Active = AndNot(Active, _rejected); + return; + } + + Confirm(differing); + return; + } + + if (reads == ReadComparison.Mismatch || + !SameExceptBases(previous, built, differing, compareTable: reads == ReadComparison.SameBytes)) + { + _rejected = Or(_rejected, MaskOf(differing)); + if (Overlaps(Active, _rejected)) + Active = AndNot(Active, _rejected); + return; + } + + Confirm(differing); + } + + private void Confirm(List differing) + { + var activated = new List(); + foreach (var index in differing) + { + var count = _confirmed.GetValueOrDefault(index) + 1; + _confirmed[index] = count; + if (count >= Confirmations && !IsUsed(_rejected, index) && !IsUsed(Active, index)) + activated.Add(index); + } + + if (activated.Count != 0) + Active = Or(Active, MaskOf(activated)); + } + + // Runs a prediction as though the words in probe were rebased already. + public T WithActive(ulong[] probe, Func run) + { + var saved = Active; + Active = probe; + try + { + return run(); + } + finally + { + Active = saved; + } + } + + // The flattened table is left out when only table-only bytes changed: it holds the words read + // from memory, whose addresses the read comparison already checked. + private bool SameExceptBases(Entry previous, Entry built, List differing, bool compareTable) + { + if (!previous.Specialization.Equals(built.Specialization)) + return false; + + var left = previous.Snapshot; + var right = built.Snapshot; + if (!SameWords(left.Images, right.Images) || !SameWords(left.Samplers, right.Samplers) || + (compareTable + ? !left.FlattenedResourceTable.AsSpan().SequenceEqual(right.FlattenedResourceTable) + : left.FlattenedResourceTable.Length != right.FlattenedResourceTable.Length) || + !left.DeviceAddressRanges.AsSpan().SequenceEqual(right.DeviceAddressRanges) || + left.Buffers.Length != right.Buffers.Length) + return false; + + for (var buffer = 0; buffer < left.Buffers.Length; buffer++) + { + var before = left.Buffers[buffer]; + var after = right.Buffers[buffer]; + if (before.Length != after.Length) + return false; + for (var dword = 0; dword < before.Length; dword++) + { + if (dword == 0 && BaseWord(buffer, differing) is { } index) + { + if (before[0] != previous.UserData[index] || after[0] != built.UserData[index]) + return false; + } + else if (before[dword] != after[dword]) + { + return false; + } + } + } + + return true; + } + + // Every read of the built draw is the previous draw's read, moved by the rebase when it lies + // in a moved buffer's window; bytesDiffer tells whether the guest bytes read there changed. + private enum ReadComparison + { + // A read moved where the words cannot explain it while every byte read alike matched. + Mismatch, + // Guest bytes changed where the descriptors depend on them, or a changed pointer in + // guest memory redirected later reads: the pair says nothing about the words. + Inconclusive, + SameBytes, + // Only words the flattened table alone reads changed: per-draw constants. + TableBytesDiffer, + } + + private static ReadComparison CompareMovedReads(AddressRebase rebase, Entry previous, Entry built, List differing) + { + var matched = previous.RangeAddresses.Length == built.RangeAddresses.Length; + var bytesDiffer = false; + var descriptorBytesDiffer = false; + var common = Math.Min(previous.RangeAddresses.Length, built.RangeAddresses.Length); + for (var range = 0; range < common; range++) + { + var length = previous.RangeLengths[range]; + if (rebase.Move(previous, built.UserData, previous.RangeAddresses[range], length, differing) != built.RangeAddresses[range] || + length != built.RangeLengths[range] || previous.RangeClean[range] != built.RangeClean[range]) + { + matched = false; + continue; + } + + for (var offset = 0; offset < length; offset += sizeof(uint)) + { + var before = previous.RangeOffsets[range] + offset; + var after = built.RangeOffsets[range] + offset; + if (previous.Bytes.AsSpan(before, sizeof(uint)).SequenceEqual(built.Bytes.AsSpan(after, sizeof(uint)))) + continue; + bytesDiffer = true; + descriptorBytesDiffer |= !previous.WordTableOnly[before / sizeof(uint)] || !built.WordTableOnly[after / sizeof(uint)]; + } + } + + if (!matched) + return bytesDiffer ? ReadComparison.Inconclusive : ReadComparison.Mismatch; + if (descriptorBytesDiffer || (bytesDiffer && !(previous.TableRefreshable && built.TableRefreshable))) + return ReadComparison.Inconclusive; + return bytesDiffer ? ReadComparison.TableBytesDiffer : ReadComparison.SameBytes; + } + + // Where a read the entry recorded lies for a draw with this user data: a read inside the + // window of a candidate whose base word moved moves with it. Only the words in moving count + // (the differing words, or the rebased ones on a hit). + private ulong Move(Entry entry, IReadOnlyList userData, ulong address, int length, IReadOnlyCollection? moving) + { + foreach (var candidate in _candidates) + { + var index = candidate.UserData; + if (moving is null ? !IsUsed(Active, index) : !moving.Contains(index)) + continue; + if (userData[index] == entry.UserData[index] || !TryWindow(entry, candidate, out var start, out var size, out var addressBase)) + continue; + // Only a range wholly inside the window moves: a range that merely overlaps it can hold + // another block the title allocated next to it, which must not move. + if (address < start || address - start + (ulong)length > size) + continue; + var movedBase = candidate.Buffer >= 0 + ? (addressBase & ~0xFFFF_FFFFUL) | userData[index] + : RawBase(entry.UserData, candidate, userData[index]); + return address - addressBase + movedBase; + } + + return address; + } + + // A hit reads the entry's recorded ranges where they lie for this draw. + public ulong MoveRead(Entry entry, IReadOnlyList userData, ulong address, int length) => + Active.Length == 0 ? address : Move(entry, userData, address, length, null); + + // True when a rebased base of the entry differs from this draw's word. + public bool Moves(Entry entry, IReadOnlyList userData) + { + foreach (var candidate in _candidates) + if (IsUsed(Active, candidate.UserData) && entry.UserData[candidate.UserData] != userData[candidate.UserData]) + return true; + return false; + } + + private int? BaseWord(int buffer, List differing) + { + foreach (var candidate in _candidates) + if (candidate.Buffer == buffer && differing.Contains(candidate.UserData)) + return candidate.UserData; + return null; + } + + private static bool SameWords(uint[][] left, uint[][] right) + { + if (left.Length != right.Length) + return false; + for (var index = 0; index < left.Length; index++) + if (!left[index].AsSpan().SequenceEqual(right[index])) + return false; + return true; + } + + private static ulong[] MaskOf(IEnumerable indices) + { + var mask = Array.Empty(); + foreach (var index in indices) + { + if (mask.Length <= index >> 6) + Array.Resize(ref mask, (index >> 6) + 1); + mask[index >> 6] |= 1UL << (index & 63); + } + + return mask; + } + + private static ulong[] Or(ulong[] left, ulong[] right) + { + var result = new ulong[Math.Max(left.Length, right.Length)]; + for (var index = 0; index < result.Length; index++) + result[index] = (index < left.Length ? left[index] : 0) | (index < right.Length ? right[index] : 0); + return result; + } + + private static ulong[] AndNot(ulong[] left, ulong[] right) + { + var result = (ulong[])left.Clone(); + for (var index = 0; index < result.Length && index < right.Length; index++) + result[index] &= ~right[index]; + return result.All(word => word == 0) ? [] : result; + } + + private static bool Overlaps(ulong[] left, ulong[] right) + { + for (var index = 0; index < left.Length && index < right.Length; index++) + if ((left[index] & right[index]) != 0) + return true; + return false; + } + } public ResourceMaterializationCache(int generationCapacity = 16384) { @@ -33,10 +791,15 @@ public sealed class ResourceMaterializationCache private static long _totalHits; private static long _totalMisses; private static long _totalUncacheable; - private const int MaxVariants = 4; + private const int MaxVariants = 8; private static long _totalStale; private static long _totalStaleUnreadable; private static long _totalRefreshes; + private static long _totalNotFound; + private static readonly long[] _refreshRefusals = new long[6]; + private static readonly string[] RefreshRefusalNames = ["not_refreshable", "unreadable", "descriptor_words", "unchanged", "table_eval", "rebased"]; + private int _lastRefreshRefusal; + [ThreadStatic] private static bool _readingTable; @@ -61,9 +824,105 @@ public sealed class ResourceMaterializationCache var stale = Interlocked.Exchange(ref _totalStale, 0); var staleUnreadable = Interlocked.Exchange(ref _totalStaleUnreadable, 0); var refreshes = Interlocked.Exchange(ref _totalRefreshes, 0); + var notFound = Interlocked.Exchange(ref _totalNotFound, 0); + var refusals = string.Join(",", RefreshRefusalNames.Select((name, index) => FormattableString.Invariant($"{name}:{Interlocked.Exchange(ref _refreshRefusals[index], 0)}"))); var total = hits + misses; return FormattableString.Invariant( - $"[PERF][RESOURCE_CACHE] hits={hits} misses={misses} stale={stale} stale_unreadable={staleUnreadable} refreshes={refreshes} uncacheable={uncacheable} hit_rate={(total == 0 ? 0 : hits * 100.0 / total):F1}% {RawReadPrefetch.TakeReport()}"); + $"[PERF][RESOURCE_CACHE] hits={hits} misses={misses} not_found={notFound} stale={stale} stale_unreadable={staleUnreadable} refreshes={refreshes} refused={refusals} uncacheable={uncacheable} hit_rate={(total == 0 ? 0 : hits * 100.0 / total):F1}% {RawReadPrefetch.TakeReport()}") + + TakeStaleDifferenceReport(); + } + + private static readonly bool StaleDiffEnabled = Environment.GetEnvironmentVariable("SHARPEMU_PROFILE_PERFORMANCE") == "1"; + private static readonly System.Collections.Concurrent.ConcurrentDictionary _staleDifferences = new(StringComparer.Ordinal); + + private static void NoteStaleDifference(Entry stale, Entry built) + { + var parts = new SortedSet(StringComparer.Ordinal); + if (!stale.Specialization.Equals(built.Specialization)) + parts.Add("spec"); + DifferingWords("buf", stale.Snapshot.Buffers, built.Snapshot.Buffers, parts); + DifferingWords("img", stale.Snapshot.Images, built.Snapshot.Images, parts); + DifferingWords("smp", stale.Snapshot.Samplers, built.Snapshot.Samplers, parts); + if (!stale.Snapshot.FlattenedResourceTable.AsSpan().SequenceEqual(built.Snapshot.FlattenedResourceTable)) + parts.Add("table"); + if (!stale.Snapshot.DeviceAddressRanges.AsSpan().SequenceEqual(built.Snapshot.DeviceAddressRanges)) + parts.Add("ranges"); + if (stale.Bytes.Length != built.Bytes.Length || !stale.RangeAddresses.AsSpan().SequenceEqual(built.RangeAddresses)) + parts.Add("reads_moved"); + var signature = parts.Count == 0 ? "none" : string.Join("+", parts); + _staleDifferences.AddOrUpdate(signature, 1, static (_, count) => count + 1); + } + + private static void DifferingWords(string name, uint[][] left, uint[][] right, SortedSet parts) + { + if (left.Length != right.Length) + { + parts.Add($"{name}.count"); + return; + } + + for (var index = 0; index < left.Length; index++) + { + if (left[index].Length != right[index].Length) + { + parts.Add($"{name}.length"); + continue; + } + + for (var word = 0; word < left[index].Length; word++) + if (left[index][word] != right[index][word]) + parts.Add($"{name}.w{word}"); + } + } + + private static string TakeStaleDifferenceReport() + { + if (_staleDifferences.IsEmpty) + return string.Empty; + var ranked = _staleDifferences.ToArray().OrderByDescending(pair => pair.Value).Take(10).ToArray(); + _staleDifferences.Clear(); + return Environment.NewLine + string.Join(Environment.NewLine, + ranked.Select(pair => FormattableString.Invariant($"[PERF][RESOURCE_STALE_DIFF] n={pair.Value} changed={pair.Key}"))); + } + + // The first part a cached result differs from a fresh one in, or null when they are the same. + private static string? Difference(ResourceSnapshot cached, ResourceSpecialization cachedSpecialization, + ResourceSnapshot fresh, ResourceSpecialization freshSpecialization) + { + static string? Words(string name, uint[][] left, uint[][] right) + { + if (left.Length != right.Length) + return $"{name}.count {left.Length}!={right.Length}"; + for (var index = 0; index < left.Length; index++) + { + if (left[index].Length != right[index].Length) + return $"{name}[{index}].length"; + for (var word = 0; word < left[index].Length; word++) + if (left[index][word] != right[index][word]) + return $"{name}[{index}][{word}] cached=0x{left[index][word]:X} fresh=0x{right[index][word]:X}"; + } + + return null; + } + + if (!cachedSpecialization.Equals(freshSpecialization)) + return "specialization"; + if (Words("buffer", cached.Buffers, fresh.Buffers) is { } buffers) + return buffers; + if (Words("image", cached.Images, fresh.Images) is { } images) + return images; + if (Words("sampler", cached.Samplers, fresh.Samplers) is { } samplers) + return samplers; + if (cached.FlattenedResourceTable.Length != fresh.FlattenedResourceTable.Length) + return "table.length"; + for (var slot = 0; slot < cached.FlattenedResourceTable.Length; slot++) + if (cached.FlattenedResourceTable[slot] != fresh.FlattenedResourceTable[slot]) + return $"table[{slot}] cached=0x{cached.FlattenedResourceTable[slot]:X} fresh=0x{fresh.FlattenedResourceTable[slot]:X}"; + if (!cached.DeviceAddressRanges.AsSpan().SequenceEqual(fresh.DeviceAddressRanges)) + return "ranges"; + if (!cached.UserData.AsSpan().SequenceEqual(fresh.UserData)) + return "user_data"; + return null; } public bool Materialize( @@ -74,17 +933,23 @@ public sealed class ResourceMaterializationCache ref ResourceSpecialization specialization, out ResourceMaterializationFailure failure) { - var key = KeyOf(plan, inputs); - var found = TryFind(key, plan, inputs, out var cached); + var drawKey = DrawKeyOf(plan, inputs); + _learnedUserData.TryGetValue(drawKey, out var learned); + var rebase = RebaseOf(drawKey, plan); + var key = KeyOf(drawKey, inputs.UserData, learned, rebase.Active); + var found = TryFind(key, plan, inputs, rebase.Active, out var cached); + Entry? stale = null; + (List<(int Buffer, uint Word)> Patches, ResourceSnapshot Snapshot, Entry From)? learning = null; if (found) { var unreadable = false; Entry? previous = null; for (var variant = cached; variant is not null; previous = variant, variant = variant.Next) { - if (!variant.Matches(plan, inputs)) + if (!variant.Matches(plan, inputs, rebase.Active) || !rebase.AppliesTo(variant)) continue; - if (Validate(variant, residentReader, out var variantUnreadable)) + stale ??= variant; + if (Validate(variant, residentReader, rebase, inputs.UserData, out var variantUnreadable)) { if (previous is not null) { @@ -95,7 +960,7 @@ public sealed class ResourceMaterializationCache Hits++; Interlocked.Increment(ref _totalHits); - snapshot = variant.Snapshot; + snapshot = rebase.Apply(WithCurrentUserData(variant.Snapshot, inputs.UserData), inputs.UserData); specialization = variant.Specialization; failure = default; return true; @@ -104,20 +969,70 @@ public sealed class ResourceMaterializationCache unreadable |= variantUnreadable; } - if (TryRefreshTable(key, cached, plan, inputs, residentReader, out var refreshed)) + // Moved bases first: a changed base word is a descriptor word, which a table refresh + // would read in full only to refuse. + ResourceSnapshot rebased = null!; + List<(int Buffer, uint Word)> patches = null!; + // Each variant is tried: a key's variants are often the same draw with different + // descriptors (materials), and only the one this draw binds can be rebased. + Entry? refreshedFrom = null; + _lastRefreshRefusal = 0; + Entry? beforeRefreshed = null; + if (stale is not null && rebase.HasLearnableMemoryBases) + { + for (Entry? previousVariant = null, variant = cached; variant is not null; previousVariant = variant, variant = variant.Next) + { + if (!variant.Matches(plan, inputs, rebase.Active) || !rebase.AppliesTo(variant)) + continue; + if (TryRebaseRefresh(variant, plan, inputs, residentReader, rebase, out rebased, out patches)) + { + refreshedFrom = variant; + beforeRefreshed = previousVariant; + break; + } + } + } + + if (refreshedFrom is not null) + { + if (rebase.AreActiveMemoryBases(patches)) + { + if (beforeRefreshed is not null) + { + beforeRefreshed.Next = refreshedFrom.Next; + refreshedFrom.Next = cached; + Store(key, refreshedFrom); + } + + Hits++; + Interlocked.Increment(ref _totalHits); + snapshot = rebased; + specialization = refreshedFrom.Specialization; + failure = default; + return true; + } + + learning = (patches, rebased, refreshedFrom); + } + else if (rebase.AppliesTo(cached) && TryRefreshTable(key, cached, plan, inputs, residentReader, rebase, out var refreshed)) { TableRefreshes++; Interlocked.Increment(ref _totalRefreshes); - snapshot = refreshed.Snapshot; + snapshot = rebase.Apply(WithCurrentUserData(refreshed.Snapshot, inputs.UserData), inputs.UserData); specialization = refreshed.Specialization; failure = default; return true; } Interlocked.Increment(ref _totalStale); + Interlocked.Increment(ref _refreshRefusals[refreshedFrom is not null ? 5 : _lastRefreshRefusal]); if (unreadable) Interlocked.Increment(ref _totalStaleUnreadable); } + else + { + Interlocked.Increment(ref _totalNotFound); + } Misses++; Interlocked.Increment(ref _totalMisses); @@ -126,7 +1041,7 @@ public sealed class ResourceMaterializationCache { var recording = new ResourceRuntimeInputs { - UserData = inputs.UserData, + UserData = recorder.WrapUserData(inputs.UserData), ShaderBase = inputs.ShaderBase, ReadMemory = recorder.Wrap(inputs.ReadMemory, clean: false), ReadCleanMemory = recorder.Wrap(inputs.ReadCleanMemory, clean: true), @@ -145,7 +1060,25 @@ public sealed class ResourceMaterializationCache return true; } - var built = recorder.Build(plan, inputs, snapshot, specialization); + var built = recorder.Build(plan, inputs, snapshot, specialization, rebase.MemoryBases); + if (StaleDiffEnabled && stale is not null && learning is null) + NoteStaleDifference(stale, built); + if (learning is { } pending) + { + var agrees = Difference(pending.Snapshot, pending.From.Specialization, built.Snapshot, built.Specialization) is null; + rebase.ConfirmMemoryBases(pending.Patches, agrees); + } + + var activeBefore = rebase.Active; + Learn(rebase, built, plan, inputs, residentReader); + var widened = Union(learned, built.UsedUserData); + if (!ReferenceEquals(widened, learned) || !ReferenceEquals(activeBefore, rebase.Active)) + { + _learnedUserData[drawKey] = widened; + key = KeyOf(drawKey, inputs.UserData, widened, rebase.Active); + found = false; + } + if (found) { built.Next = cached; @@ -175,18 +1108,34 @@ public sealed class ResourceMaterializationCache // flattened table, only the table is evaluated again. It is refused when the plan's // specialization reads or extends the table (indirect or candidate tables), when the table // now reads a word the entry did not validate, or when a word moved between the two reads. + // + // With rebased words the entry's ranges are read where they lie for this draw. The refreshed + // entry keeps the entry's own addresses and base words with the bytes read there, so a later + // draw moves its reads the same way and compares against those bytes. private bool TryRefreshTable(ulong key, Entry cached, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, - ResidentGuestBytesReader residentReader, out Entry refreshed) + ResidentGuestBytesReader residentReader, AddressRebase rebase, out Entry refreshed) { refreshed = null!; + _lastRefreshRefusal = 0; if (!cached.TableRefreshable) return false; var current = new byte[cached.Bytes.Length]; + var moved = cached.RangeAddresses; + if (rebase.Moves(cached, inputs.UserData)) + { + moved = new ulong[cached.RangeAddresses.Length]; + for (var index = 0; index < moved.Length; index++) + moved[index] = rebase.MoveRead(cached, inputs.UserData, cached.RangeAddresses[index], cached.RangeLengths[index]); + } + for (var index = 0; index < cached.RangeAddresses.Length; index++) { - if (!residentReader(cached.RangeAddresses[index], current.AsSpan(cached.RangeOffsets[index], cached.RangeLengths[index]), cached.RangeClean[index])) + if (!residentReader(moved[index], current.AsSpan(cached.RangeOffsets[index], cached.RangeLengths[index]), cached.RangeClean[index])) + { + _lastRefreshRefusal = 1; return false; + } } var changed = false; @@ -195,12 +1144,21 @@ public sealed class ResourceMaterializationCache if (current.AsSpan(offset, sizeof(uint)).SequenceEqual(cached.Bytes.AsSpan(offset, sizeof(uint)))) continue; if (!cached.WordTableOnly[offset / sizeof(uint)]) + { + _lastRefreshRefusal = 2; return false; + } + changed = true; } if (!changed) + { + _lastRefreshRefusal = 3; return false; + } + + _lastRefreshRefusal = 4; var recorder = TakeRecorder(); try @@ -233,8 +1191,10 @@ public sealed class ResourceMaterializationCache foreach (var (address, word, _, _) in recorder.Reads) { - if (!TryFindWord(cached, address, out var offset) || - System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset, sizeof(uint))) != word) + var found = ReferenceEquals(moved, cached.RangeAddresses) + ? TryFindWord(cached, address, out var offset) + : TryFindMovedWord(cached, moved, address, out offset); + if (!found || System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset, sizeof(uint))) != word) return false; } @@ -247,6 +1207,7 @@ public sealed class ResourceMaterializationCache { Plan = cached.Plan, UserData = cached.UserData, + UsedUserData = cached.UsedUserData, ShaderBase = cached.ShaderBase, ComputeState = cached.ComputeState, RangeAddresses = cached.RangeAddresses, @@ -254,6 +1215,7 @@ public sealed class ResourceMaterializationCache RangeLengths = cached.RangeLengths, RangeClean = cached.RangeClean, WordTableOnly = cached.WordTableOnly, + BaseWordBuffers = cached.BaseWordBuffers, TableRefreshable = true, Bytes = current, Snapshot = new ResourceSnapshot @@ -290,6 +1252,26 @@ public sealed class ResourceMaterializationCache _spareRecorder = recorder; } + // The byte offset of a dword read at address when the entry's ranges lie at moved addresses. + private static bool TryFindMovedWord(Entry entry, ulong[] moved, ulong address, out int offset) + { + // moved may be a reused buffer longer than the entry's ranges. + for (var index = 0; index < entry.RangeAddresses.Length; index++) + { + if (address < moved[index]) + continue; + var delta = address - moved[index]; + if (delta % sizeof(uint) == 0 && delta < (ulong)entry.RangeLengths[index]) + { + offset = entry.RangeOffsets[index] + (int)delta; + return true; + } + } + + offset = 0; + return false; + } + // The byte offset of a recorded dword in the entry's bytes, found by its address. private static bool TryFindWord(Entry entry, ulong address, out int offset) { @@ -319,29 +1301,85 @@ public sealed class ResourceMaterializationCache return true; } - private static ulong KeyOf(ShaderResourcePlan plan, ResourceRuntimeInputs inputs) + private static ulong DrawKeyOf(ShaderResourcePlan plan, ResourceRuntimeInputs inputs) { var hash = new HashCode(); hash.Add(RuntimeHelpers.GetHashCode(plan)); hash.Add(inputs.ShaderBase); hash.Add(inputs.ComputeState); - var userData = inputs.UserData; - hash.Add(userData.Count); - for (var index = 0; index < userData.Count; index++) - hash.Add(userData[index]); - var low = (uint)hash.ToHashCode(); - // A second, independent mix keeps accidental collisions out of the 64-bit key. - var high = 0x9E3779B9u; - for (var index = 0; index < userData.Count; index++) - high = (high ^ userData[index]) * 0x01000193u; + hash.Add(inputs.UserData.Count); + return (ulong)(uint)hash.ToHashCode() | ((ulong)(uint)RuntimeHelpers.GetHashCode(plan) << 32); + } + + // Only the words the draw's shader has been seen to read take part, less the rebased ones: the + // key is an index, and Entry.Matches decides whether an entry is reusable. + private static ulong KeyOf(ulong drawKey, IReadOnlyList userData, ulong[]? learned, ulong[]? rebased = null) + { + var low = (uint)drawKey; + var high = (uint)(drawKey >> 32) ^ 0x9E3779B9u; + if (learned is not null) + { + for (var index = 0; index < userData.Count; index++) + { + if (!IsUsed(learned, index) || (rebased is not null && IsUsed(rebased, index))) + continue; + var word = userData[index]; + low = (low ^ word) * 0x01000193u; + high = ((high + word) * 0x85EBCA6Bu) ^ (high >> 13); + } + } + return ((ulong)high << 32) | low; } - private bool TryFind(ulong key, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, out Entry entry) + private static bool IsUsed(ulong[] mask, int index) => + (uint)(index >> 6) < (uint)mask.Length && (mask[index >> 6] & (1UL << (index & 63))) != 0; + + // The same array when nothing new was learned, so the caller can tell. + private static ulong[] Union(ulong[]? learned, ulong[] used) + { + if (learned is null) + return used; + var changed = learned.Length < used.Length; + for (var index = 0; !changed && index < used.Length; index++) + changed = (used[index] & ~learned[index]) != 0; + if (!changed) + return learned; + var merged = new ulong[Math.Max(learned.Length, used.Length)]; + for (var index = 0; index < merged.Length; index++) + merged[index] = (index < learned.Length ? learned[index] : 0) | (index < used.Length ? used[index] : 0); + return merged; + } + + // A cached snapshot is shared by every draw that hits it, so it keeps the user data of the draw + // that built it; a draw whose unread words differ gets its own copy that carries its own. + private static ResourceSnapshot WithCurrentUserData(ResourceSnapshot cached, IReadOnlyList userData) + { + var words = cached.UserData; + var same = words.Length == userData.Count; + for (var index = 0; same && index < words.Length; index++) + same = words[index] == userData[index]; + if (same) + return cached; + var current = new uint[userData.Count]; + for (var index = 0; index < current.Length; index++) + current[index] = userData[index]; + return new ResourceSnapshot + { + Buffers = cached.Buffers, + Images = cached.Images, + Samplers = cached.Samplers, + FlattenedResourceTable = cached.FlattenedResourceTable, + UserData = current, + DeviceAddressRanges = cached.DeviceAddressRanges, + }; + } + + private bool TryFind(ulong key, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, ulong[] rebased, out Entry entry) { if (_young.TryGetValue(key, out entry!)) - return entry.Matches(plan, inputs); - if (!_old.TryGetValue(key, out entry!) || !entry.Matches(plan, inputs)) + return entry.Matches(plan, inputs, rebased); + if (!_old.TryGetValue(key, out entry!) || !entry.Matches(plan, inputs, rebased)) return false; _old.Remove(key); Store(key, entry); @@ -359,16 +1397,390 @@ public sealed class ResourceMaterializationCache _young[key] = entry; } - private bool Validate(Entry entry, ResidentGuestBytesReader residentReader, out bool unreadable) + // What a hit on the previous entry would give this draw with the words in probe rebased: its + // reads moved and its bytes unchanged, or its memory-loaded bases moved as well. Null when + // neither path applies, which says nothing about the words. + private (ResourceSnapshot Snapshot, List<(int Buffer, uint Word)>? Patches)? Predict(Entry previous, ShaderResourcePlan plan, + ResourceRuntimeInputs inputs, ResidentGuestBytesReader residentReader, AddressRebase rebase, ulong[] probe) => + rebase.WithActive<(ResourceSnapshot, List<(int Buffer, uint Word)>?)?>(probe, () => + { + if (!previous.Matches(plan, inputs, probe) || !rebase.AppliesTo(previous)) + return null; + if (Validate(previous, residentReader, rebase, inputs.UserData, out _)) + return (rebase.Apply(WithCurrentUserData(previous.Snapshot, inputs.UserData), inputs.UserData), null); + if (rebase.HasLearnableMemoryBases && + TryRebaseRefresh(previous, plan, inputs, residentReader, rebase, out var snapshot, out var patches)) + return (snapshot, patches); + return null; + }); + + // Its own method: a lambda in MaterializeCore that captured the parameters made every call, + // hits included, allocate the closure on entry. + private void Learn(AddressRebase rebase, Entry built, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, + ResidentGuestBytesReader residentReader) => + rebase.Learn(built, (previous, probe) => Predict(previous, plan, inputs, residentReader, rebase, probe)); + + // The table for this draw without evaluating it: a slot that reads a memory-based buffer at a + // constant offset takes the word now at that offset, which the moved window has just read; the + // other slots read nothing that changed, so they keep the entry's words, except a base slot, + // which the patch has already moved. current holds this draw's words in the entry's layout. + // The table is patched in place: the caller passes a copy it owns. + private static bool TryCopyTable(Entry entry, ShaderResourcePlan plan, AddressRebase rebase, byte[] current, uint[] table) + { + var sources = entry.TableCopySources ??= TableCopySourcesOf(entry, plan, rebase, table.Length); + if (sources.Length == 0) + return false; + for (var slot = 0; slot < sources.Length; slot++) + { + var offset = sources[slot]; + if (offset >= 0) + table[(int)plan.TableReads[slot].FlatOffset] = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset)); + } + + return true; + } + + // A table whose slots are all direct loads or reads through memory-based buffers. + private static bool CanCopyDirect(AddressRebase rebase) + { + if (rebase.DirectSlots is not null) + return true; + return false; + } + + + // Every slot takes the dword it loads for this draw. A direct slot's address is evaluated from + // the draw's own user data, as the table evaluation would, and its dword is taken from the range + // this draw read around it; a slot through a memory-based buffer copies as in TryCopyTable. + // Nothing is written unless every slot is placed. + private static bool TryCopyDirectTable(Entry entry, ShaderResourcePlan plan, AddressRebase rebase, byte[] current, + ulong[] moved, IReadOnlyList userData, uint[] table) + { + var through = entry.TableCopySources ??= TableCopySourcesOf(entry, plan, rebase, table.Length); + var direct = rebase.DirectSlots!; + if (through.Length != direct.Length) + return Unplaced(); + Span sources = direct.Length <= 256 ? stackalloc int[direct.Length] : new int[direct.Length]; + for (var slot = 0; slot < direct.Length; slot++) + { + if (plan.TableReads[slot].FlatOffset >= table.Length) + return Unplaced(); + if (rebase.TableSlotBuffers[slot] >= 0) + { + if (through[slot] < 0) + return Unplaced(); + sources[slot] = through[slot]; + continue; + } + + var load = direct[slot]; + if (load.Low >= userData.Count || load.High >= userData.Count) + return Unplaced(); + var high = load.High >= 0 ? userData[load.High] : load.HighConstant; + var baseAddress = ((high << 32) | userData[load.Low]) & 0x0000_FFFF_FFFF_FFFFul; + if (RuntimeValueEvaluator.ResolveRawAddress(ScalarValueKind.ScalarAddressWord, baseAddress, high, 0, + load.Immediate, load.Offset, out var address) != RuntimeValueEvaluator.RawAddress.Read || + !TryFindMovedWord(entry, moved, address, out sources[slot])) + return Unplaced(); + } + + for (var slot = 0; slot < sources.Length; slot++) + table[(int)plan.TableReads[slot].FlatOffset] = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(sources[slot])); + return true; + + static bool Unplaced() + { + return false; + } + } + + // Per table slot: the byte offset in the entry's words of the word the slot copies, or -1 for a + // slot that keeps its word. Empty when some slot cannot be placed. + private static int[] TableCopySourcesOf(Entry entry, ShaderResourcePlan plan, AddressRebase rebase, int tableLength) + { + var sources = new int[plan.TableReads.Count]; + for (var slot = 0; slot < sources.Length; slot++) + { + sources[slot] = -1; + var buffer = rebase.TableSlotBuffers[slot]; + if (buffer < 0) + continue; + var descriptor = entry.Snapshot.Buffers[buffer]; + if (!AddressRebase.TryMovedWindow(descriptor, descriptor[0], out var start, out var size) || + rebase.TableSlotOffsets[slot] < 0 || (ulong)rebase.TableSlotOffsets[slot] + sizeof(uint) > size || + !TryFindWord(entry, start + (ulong)rebase.TableSlotOffsets[slot], out var offset) || + plan.TableReads[slot].FlatOffset >= tableLength) + return []; + sources[slot] = offset; + } + + return sources; + } + + // Where each memory-loaded base word of the entry lies: its range and byte offset in it. + private static (int Range, int Offset)[] BaseWordProbesOf(Entry entry) + { + var probes = new List<(int Range, int Offset)>(); + for (var range = 0; range < entry.RangeAddresses.Length; range++) + { + for (var offset = 0; offset < entry.RangeLengths[range]; offset += sizeof(uint)) + { + if (entry.BaseWordBuffers![(entry.RangeOffsets[range] + offset) / sizeof(uint)] >= 0) + probes.Add((range, offset)); + } + } + + return probes.ToArray(); + } + + private byte[] _rebaseCurrent = []; + private ulong[] _rebaseMoved = []; + private readonly List<(int Buffer, uint Word)> _rebaseBases = new(); + + // Per recorded range: the memory-based buffer whose window wholly holds it, or NoBase. Such a + // range is read only once the buffer's base is known: it moves with the base. + private static int[] RangeWindowsOf(Entry entry, AddressRebase rebase) + { + var windows = new int[entry.RangeAddresses.Length]; + windows.AsSpan().Fill(NoBase); + for (var index = 0; index < windows.Length; index++) + { + foreach (var buffer in rebase.MemoryBases) + { + var descriptor = entry.Snapshot.Buffers[buffer]; + if (descriptor.Length >= 3 && AddressRebase.TryMovedWindow(descriptor, descriptor[0], out var start, out var size) && + entry.RangeAddresses[index] >= start && entry.RangeAddresses[index] - start + (ulong)entry.RangeLengths[index] <= size) + { + windows[index] = buffer; + break; + } + } + } + + return windows; + } + + // A stale entry whose only changed words are memory-loaded buffer bases, and constants only the + // table reads: the bases are patched into a copy of the snapshot and the table is evaluated + // again for this draw, which reads the constants where the moved bases now point. Every read of + // that evaluation is a word the entry validated unchanged, or lies in the window a patched base + // now covers. The descriptors and specialization are the entry's; learning checks that. + private bool TryRebaseRefresh(Entry entry, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, + ResidentGuestBytesReader residentReader, AddressRebase rebase, out ResourceSnapshot snapshot, + out List<(int Buffer, uint Word)> patches) + { + snapshot = null!; + patches = null!; + if (entry.BaseWordBuffers is not { } map) + return false; + var userData = inputs.UserData; + // The words read here are compared and copied, never kept, so the buffers are reused. + if (_rebaseCurrent.Length < entry.Bytes.Length) + _rebaseCurrent = new byte[Math.Max(entry.Bytes.Length, _rebaseCurrent.Length * 2)]; + if (_rebaseMoved.Length < entry.RangeAddresses.Length) + _rebaseMoved = new ulong[Math.Max(entry.RangeAddresses.Length, _rebaseMoved.Length * 2)]; + var current = _rebaseCurrent; + var moved = _rebaseMoved; + var windowed = entry.RangeWindows ??= RangeWindowsOf(entry, rebase); + for (var index = 0; index < entry.RangeAddresses.Length; index++) + { + if (windowed[index] != NoBase) + continue; + moved[index] = rebase.MoveRead(entry, userData, entry.RangeAddresses[index], entry.RangeLengths[index]); + if (!residentReader(moved[index], current.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]), entry.RangeClean[index])) + return false; + } + + // The new base of each buffer whose base word changed, from the ranges read so far. A base + // word inside a moving window would need its own base first; such an entry is not rebased. + var rangeCount = entry.RangeAddresses.Length; + var bases = _rebaseBases; + bases.Clear(); + foreach (var (range, rangeOffset) in entry.BaseWordProbes ??= BaseWordProbesOf(entry)) + { + if (windowed[range] != NoBase) + return false; + var offset = entry.RangeOffsets[range] + rangeOffset; + var now = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset)); + if (now != System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(entry.Bytes.AsSpan(offset))) + bases.Add((map[offset / sizeof(uint)], now)); + } + + for (var index = 0; index < rangeCount; index++) + { + var buffer = windowed[index]; + if (buffer == NoBase) + continue; + var oldBase = entry.Snapshot.Buffers[buffer][0]; + var newBase = oldBase; + foreach (var (changed, word) in bases) + if (changed == buffer) + newBase = word; + moved[index] = entry.RangeAddresses[index] - oldBase + newBase; + if (!residentReader(moved[index], current.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]), entry.RangeClean[index])) + return false; + } + + List<(int Buffer, uint Word)>? found = null; + // A changed constant outside the moved windows is read by a slot the copy cannot place. + var constantsOutsideWindows = false; + for (var index = 0; index < rangeCount; index++) + { + // Most ranges did not change at all; only a changed one is looked at word by word. + if (current.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]) + .SequenceEqual(entry.Bytes.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]))) + continue; + for (var offset = entry.RangeOffsets[index]; offset < entry.RangeOffsets[index] + entry.RangeLengths[index]; offset += sizeof(uint)) + { + var word = offset / sizeof(uint); + var now = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset)); + if (now == System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(entry.Bytes.AsSpan(offset))) + continue; + var buffer = map[word]; + if (buffer >= 0) + { + if (rebase.IsRejectedMemoryBase(buffer)) + return false; + (found ??= new()).Add((buffer, now)); + } + else if (!entry.WordTableOnly[word]) + return false; + else + constantsOutsideWindows |= windowed[index] == NoBase; + } + } + + if (found is null) + return false; + var patched = rebase.PatchMemoryBases(rebase.Apply(WithCurrentUserData(entry.Snapshot, userData), userData), found); + // A changed constant outside the windows is still copied when every slot is a direct load: + // each slot's word is then the dword at its own address, which this draw has just read. + var copyDirect = constantsOutsideWindows && CanCopyDirect(rebase); + if (plan.TableReads.Count != 0 && (!constantsOutsideWindows || copyDirect) && !rebase.TableSlotsThroughBases) + { + // The patch cloned the table when it moved a base slot; otherwise it is the entry's. + var table = ReferenceEquals(patched.FlattenedResourceTable, entry.Snapshot.FlattenedResourceTable) + ? (uint[])patched.FlattenedResourceTable.Clone() + : patched.FlattenedResourceTable; + // The copy writes nothing unless every slot can be placed. + if (copyDirect + ? TryCopyDirectTable(entry, plan, rebase, current, moved, userData, table) + : TryCopyTable(entry, plan, rebase, current, table)) + { + snapshot = new ResourceSnapshot + { + Buffers = patched.Buffers, + Images = patched.Images, + Samplers = patched.Samplers, + FlattenedResourceTable = table, + UserData = patched.UserData, + DeviceAddressRanges = patched.DeviceAddressRanges, + }; + patches = found; + return true; + } + } + + if (plan.TableReads.Count != 0) + { + if (!entry.TableRefreshable) + return false; + var windows = new List<(ulong Start, ulong Size)>(found.Count); + foreach (var (buffer, baseWord) in found) + if (AddressRebase.TryMovedWindow(patched.Buffers[buffer], baseWord, out var start, out var size)) + windows.Add((start, size)); + + var recorder = TakeRecorder(); + try + { + var recording = new ResourceRuntimeInputs + { + UserData = userData, + ShaderBase = inputs.ShaderBase, + ReadMemory = recorder.Wrap(inputs.ReadMemory, clean: false), + ReadCleanMemory = recorder.Wrap(inputs.ReadCleanMemory, clean: true), + ReadResidentMemory = recorder.WrapResident(inputs.ReadResidentMemory), + ReadsClean = inputs.ReadsClean, + ComputeState = inputs.ComputeState, + }; + var cachedTable = entry.Snapshot.FlattenedResourceTable; + ReadingTable = true; + bool evaluated; + uint[] table; + try + { + evaluated = ResourceMaterializer.TryEvaluateTable(plan, recording, out table); + } + finally + { + ReadingTable = false; + } + + if (!evaluated || recorder.Failed || table.Length != cachedTable.Length) + return false; + + foreach (var (address, word, _, _) in recorder.Reads) + { + var inWindow = false; + foreach (var (start, size) in windows) + inWindow |= address >= start && address - start < size; + if (inWindow) + continue; + if (!TryFindMovedWord(entry, moved, address, out var offset) || + System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset, sizeof(uint))) != word) + return false; + } + + // The written device-address slots come from reads validated unchanged above. + foreach (var slot in plan.WrittenRangeSlotByHandle.Values) + cachedTable.AsSpan((int)slot, ShaderResourcePlan.WrittenRangeDwordCount).CopyTo(table.AsSpan((int)slot)); + patched = new ResourceSnapshot + { + Buffers = patched.Buffers, + Images = patched.Images, + Samplers = patched.Samplers, + FlattenedResourceTable = table, + UserData = patched.UserData, + DeviceAddressRanges = patched.DeviceAddressRanges, + }; + } + finally + { + ReturnRecorder(recorder); + } + } + + snapshot = patched; + patches = found; + return true; + } + + private bool Validate(Entry entry, ResidentGuestBytesReader residentReader, AddressRebase rebase, IReadOnlyList userData, + out bool unreadable) { unreadable = false; + // A memory-loaded base usually moved when anything did: its few bytes are checked before + // ranges that can span kilobytes. The full comparison below still decides a match. + if (entry.BaseWordBuffers is not null) + { + Span word = stackalloc byte[sizeof(uint)]; + foreach (var (range, offset) in entry.BaseWordProbes ??= BaseWordProbesOf(entry)) + { + var address = rebase.MoveRead(entry, userData, entry.RangeAddresses[range], entry.RangeLengths[range]) + (ulong)offset; + if (residentReader(address, word, entry.RangeClean[range]) && + !word.SequenceEqual(entry.Bytes.AsSpan(entry.RangeOffsets[range] + offset, sizeof(uint)))) + return false; + } + } + for (var index = 0; index < entry.RangeAddresses.Length; index++) { var length = entry.RangeLengths[index]; if (_scratch.Length < length) _scratch = new byte[Math.Max(length, _scratch.Length * 2)]; var current = _scratch.AsSpan(0, length); - if (!residentReader(entry.RangeAddresses[index], current, entry.RangeClean[index])) + var address = rebase.MoveRead(entry, userData, entry.RangeAddresses[index], length); + if (!residentReader(address, current, entry.RangeClean[index])) { unreadable = true; return false; @@ -385,6 +1797,8 @@ public sealed class ResourceMaterializationCache { public required ShaderResourcePlan Plan { get; init; } public required uint[] UserData { get; init; } + // The user data words the evaluator read, one bit per word. + public required ulong[] UsedUserData { get; init; } public required ulong ShaderBase { get; init; } public required ComputeSelectorState? ComputeState { get; init; } public required ulong[] RangeAddresses { get; init; } @@ -398,20 +1812,59 @@ public sealed class ResourceMaterializationCache public required byte[] Bytes { get; init; } public required ResourceSnapshot Snapshot { get; init; } public required ResourceSpecialization Specialization { get; init; } + // Per recorded dword: the buffer whose memory-loaded base dword it is, or a negative value. + public int[]? BaseWordBuffers { get; init; } + // Filled on the first rebased refresh of the entry; see RangeWindowsOf and TableCopySourcesOf. + public int[]? RangeWindows { get; set; } + public int[]? TableCopySources { get; set; } + public (int Range, int Offset)[]? BaseWordProbes { get; set; } public Entry? Next { get; set; } - public bool Matches(ShaderResourcePlan plan, ResourceRuntimeInputs inputs) + // A rebased word is an address the hit patches in, so it does not distinguish entries. + public bool Matches(ShaderResourcePlan plan, ResourceRuntimeInputs inputs, ulong[] rebased) { if (!ReferenceEquals(Plan, plan) || ShaderBase != inputs.ShaderBase || !Nullable.Equals(ComputeState, inputs.ComputeState) || UserData.Length != inputs.UserData.Count) return false; for (var index = 0; index < UserData.Length; index++) - if (UserData[index] != inputs.UserData[index]) + if (IsUsed(UsedUserData, index) && !IsUsed(rebased, index) && UserData[index] != inputs.UserData[index]) return false; return true; } } + // A user data list that notes which words are read by index. Copying it out (the snapshot's + // own copy) goes through the enumerator and is not a read of the evaluator. + private sealed class UserDataRecorder : IReadOnlyList + { + private IReadOnlyList _inner = []; + private ulong[] _used = []; + + public void Begin(IReadOnlyList inner) + { + _inner = inner; + _used = new ulong[(inner.Count + 63) >> 6]; + } + + // The recorded set; the next Begin allocates a fresh one, so the entry keeps this array. + public ulong[] TakeUsed() => _used; + + public int Count => _inner.Count; + + public uint this[int index] + { + get + { + _used[index >> 6] |= 1UL << (index & 63); + return _inner[index]; + } + } + + public IEnumerator GetEnumerator() => _inner.GetEnumerator(); + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + private sealed class ReadRecorder { private readonly List<(ulong Address, uint Word, bool Clean, bool Table)> _reads = new(); @@ -451,6 +1904,14 @@ public sealed class ResourceMaterializationCache public bool Failed { get; private set; } + private readonly UserDataRecorder _userData = new(); + + public IReadOnlyList WrapUserData(IReadOnlyList inner) + { + _userData.Begin(inner); + return _userData; + } + public List<(ulong Address, uint Word, bool Clean, bool Table)> Reads => _reads; public Action TablePhase { get; } @@ -499,9 +1960,14 @@ public sealed class ResourceMaterializationCache } public Entry Build(ShaderResourcePlan plan, ResourceRuntimeInputs inputs, ResourceSnapshot snapshot, - ResourceSpecialization specialization) + ResourceSpecialization specialization, IReadOnlyList memoryBases) { - _reads.Sort((left, right) => left.Address.CompareTo(right.Address)); + // Reads mostly arrive in address order; sorting a sorted list was ~5% of a draw-heavy render thread. + var ordered = true; + for (var index = 1; ordered && index < _reads.Count; index++) + ordered = _reads[index - 1].Address <= _reads[index].Address; + if (!ordered) + _reads.Sort((left, right) => left.Address.CompareTo(right.Address)); // Count unique words and contiguous ranges first, then fill the // retained arrays directly. Temporary lists previously duplicated // every recorded byte and range on each cache miss. @@ -558,8 +2024,10 @@ public sealed class ResourceMaterializationCache userData[index] = inputs.UserData[index]; return new Entry { + BaseWordBuffers = memoryBases.Count == 0 ? null : MapBaseWords(snapshot, memoryBases, bytes), Plan = plan, UserData = userData, + UsedUserData = _userData.TakeUsed(), ShaderBase = inputs.ShaderBase, ComputeState = inputs.ComputeState, RangeAddresses = addresses, @@ -574,5 +2042,46 @@ public sealed class ResourceMaterializationCache Specialization = specialization, }; } + + // The recorded word each memory-loaded base dword came from: the only recorded word holding + // that value. A word two bases claim names neither. Learning checks the guess. + private static int[]? MapBaseWords(ResourceSnapshot snapshot, IReadOnlyList memoryBases, byte[] bytes) + { + int[]? map = null; + var wordCount = bytes.Length / sizeof(uint); + foreach (var buffer in memoryBases) + { + if (buffer >= snapshot.Buffers.Length || snapshot.Buffers[buffer].Length < 2) + continue; + var value = snapshot.Buffers[buffer][0]; + var match = -1; + for (var word = 0; word < wordCount; word++) + { + if (System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(word * sizeof(uint))) != value) + continue; + if (match >= 0) + { + match = -1; + break; + } + match = word; + } + + if (match < 0) + continue; + if (map is null) + { + map = new int[wordCount]; + map.AsSpan().Fill(NoBase); + } + + map[match] = map[match] == NoBase ? buffer : SharedBase; + } + + return map; + } } + + private const int NoBase = -1; + private const int SharedBase = -2; } diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs index b65eb7e..376c019 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs @@ -1050,6 +1050,10 @@ public static class ResourceMaterializer } var storage = baseImage.ResourceClass == ImageResourceClass.Storage; + image = image with + { + MipStatistics = RecordMipStatistics && plan.Stage == ShaderStage.Pixel && !storage && ((descriptor[5] >> 25) & 1) != 0, + }; var conversionFormat = ImageConversionFormat(format); var shaderSwizzle = storage || conversionFormat != GuestImageFormat.Invalid ? descriptor[3] & 0xFFF : image.ShaderSwizzle; var rawSintStorage = storage && format == GuestImageFormat.Format32Sint && baseImage.Written && !baseImage.Read && !baseImage.Atomic; @@ -1363,6 +1367,8 @@ public static class ResourceMaterializer // Applies a specialization to the plan's tables: buffer strides and formats, image // classes and indirect candidates, point samplers and the sampler each access uses. + public static bool RecordMipStatistics { get; set; } + public static SpecializedResourceInfo ApplyTo(ShaderResourcePlan plan, ResourceSpecialization specialization) { var source = plan.Info; @@ -1431,6 +1437,7 @@ public static class ResourceMaterializer image.IndirectSearchIterations = specialized.IndirectSearchIterations; image.Cube = specialized.Cube; image.EmulatedCompareFunction = specialized.EmulatedCompareFunction; + image.MipStatistics = specialized.MipStatistics; if (specialized.EmulatedCompareFunction >= 0) { // The shader compares instead; the host keeps a color view and a plain sampler. diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs index f1fbcd9..0165cd2 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs @@ -27,7 +27,8 @@ public readonly record struct ImageSpecialization( uint IndirectMappingOffset, uint IndirectSearchIterations, bool Cube, - int EmulatedCompareFunction = -1); + int EmulatedCompareFunction = -1, + bool MipStatistics = false); // The module-affecting resource state of one draw. Addresses and descriptor payloads // stay in the snapshot, so they never create a permutation. diff --git a/src/SharpEmu.ShaderCompiler/Resources/ShaderResourceInfo.cs b/src/SharpEmu.ShaderCompiler/Resources/ShaderResourceInfo.cs index 9154e70..cd782f6 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ShaderResourceInfo.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ShaderResourceInfo.cs @@ -82,6 +82,8 @@ public sealed class ImageResource public uint IndirectSearchIterations { get; set; } public List IndirectResources { get; set; } = []; + public bool MipStatistics { get; set; } + public ImageResource Clone() { var clone = (ImageResource)MemberwiseClone(); diff --git a/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs b/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs index 2e64c02..e513e5e 100644 --- a/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs +++ b/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs @@ -162,10 +162,15 @@ public sealed class ShaderCompileRequest public IReadOnlyDictionary WrittenRangeSlotByMemoryIndex { get; } public uint WaveSize { get; init; } = 32; + // True when the pipeline runs on a 64-lane host subgroup, so one subgroup is one guest Wave64. + public bool HostWave64Supported { get; init; } + public bool HostComputeSubgroup32 { get; init; } public bool EnableExecGuardElision { get; init; } = true; public uint ScratchDwords { get; init; } public bool EnableGraphicsSubgroupOperations { get; init; } = true; + public bool MipStatisticsSubgroupReduction { get; init; } + // The device supports 64-bit integer atomics on workgroup memory // (VkPhysicalDeviceFeatures.shaderSharedInt64Atomics). When set, the LDS // 64-bit atomics are emitted as real 64-bit atomics instead of a pair of diff --git a/tests/SharpEmu.Libs.Tests/Agc/AgcSetShRegisterRangeTests.cs b/tests/SharpEmu.Libs.Tests/Agc/AgcSetShRegisterRangeTests.cs index 642d9b7..417452d 100644 --- a/tests/SharpEmu.Libs.Tests/Agc/AgcSetShRegisterRangeTests.cs +++ b/tests/SharpEmu.Libs.Tests/Agc/AgcSetShRegisterRangeTests.cs @@ -91,8 +91,8 @@ public sealed class AgcSetShRegisterRangeTests Assert.Equal(PacketAddress + 8, context[CpuRegister.Rax]); Assert.Equal(1, memory.CopyCalls); Assert.Equal((PacketAddress + 16, sourceAddress, (ulong)valueCount * 4), memory.LastCopy); - Assert.Equal(10, memory.ReadCalls); - Assert.Equal(6, memory.WriteCalls); + Assert.Equal(2, memory.ReadCalls); + Assert.Equal(4, memory.WriteCalls); Assert.Equal(0xC0001000u, ReadUInt32(memory, PacketAddress)); Assert.Equal(0x6875000Du, ReadUInt32(memory, PacketAddress + 4)); Assert.Equal(0xC0007600u | (valueCount << 16), ReadUInt32(memory, PacketAddress + 8)); @@ -136,8 +136,8 @@ public sealed class AgcSetShRegisterRangeTests Assert.Equal(PacketAddress + 8, context[CpuRegister.Rax]); Assert.Equal(0, memory.CopyCalls); - Assert.Equal(10, memory.ReadCalls); - Assert.Equal(6, memory.WriteCalls); + Assert.Equal(2, memory.ReadCalls); + Assert.Equal(4, memory.WriteCalls); Assert.Equal(0xC0001000u, ReadUInt32(memory, PacketAddress)); Assert.Equal(0x6875000Du, ReadUInt32(memory, PacketAddress + 4)); Assert.Equal(0xC0007600u | (valueCount << 16), ReadUInt32(memory, PacketAddress + 8)); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs index 015ec32..dfb1ab9 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs @@ -110,6 +110,162 @@ public sealed class GuestBufferAsyncReadbackTests(HeadlessVulkanFixture fixture) harness.Shutdown(); } + [Fact] + public void AWindowWhoseReadAheadsKeepGoingStaleStopsBeingReadAhead() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + Attach(harness, _vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + GpuWrite(harness, address, 0x01010101); + QueueRead(harness, address); + + GpuWrite(harness, address, 0x02020202); + Assert.True(harness.Cache.HotWritePending); + var value = 0x03030303u; + var deadline = DateTime.UtcNow.AddSeconds(20); + while (harness.Cache.EagerReadbacksWasted < 3 && DateTime.UtcNow < deadline) + { + SubmitBatch(harness); + GpuWrite(harness, address, value++); + } + + Assert.Equal(3, harness.Cache.EagerReadbacksWasted); + SubmitBatch(harness); + var started = harness.Cache.EagerReadbacksStarted; + GpuWrite(harness, address, 0x07070707); + Assert.False(harness.Cache.HotWritePending); + SubmitBatch(harness); + SubmitBatch(harness); + + Assert.Equal(started, harness.Cache.EagerReadbacksStarted); + QueueRead(harness, address); + Assert.Equal(0x07070707u, BitConverter.ToUInt32(harness.Read(address + 0x40, 4))); + Detach(harness); + harness.Shutdown(); + } + + private static void CoverWithOneBuffer(CacheHarness harness, ulong address, ulong size) => + harness.Worker.Run(() => _ = harness.Cache.ObtainBuffer(address, size, isWritten: false)); + + [Fact] + public void AWriteElsewhereInTheBufferKeepsAReadAheadUsable() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + Attach(harness, _vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + CoverWithOneBuffer(harness, address, 0x10000); + GpuWrite(harness, address, 0x01010101); + QueueRead(harness, address); + GpuWrite(harness, address, 0x02020202); + SubmitBatch(harness); + Assert.Equal(1, harness.Cache.EagerReadbacksStarted); + + GpuWrite(harness, address + 0x8000, 0x07070707); + QueueRead(harness, address); + + Assert.Equal(1, harness.Cache.EagerReadbacksUsed); + Assert.Equal(0x02020202u, BitConverter.ToUInt32(harness.Read(address + 0x40, 4))); + Assert.True(harness.Cache.HasGpuDirtyBytes(address + 0x8000, 0x100)); + Assert.True(harness.Cache.DownloadToCpu(address + 0x8000, 4)); + Assert.Equal(0x07070707u, BitConverter.ToUInt32(harness.Read(address + 0x8010, 4))); + Detach(harness); + harness.Shutdown(); + } + + [Fact] + public void ANewerWriteKeepsOnlyItsOwnPieceProtected() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + Attach(harness, _vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + CoverWithOneBuffer(harness, address, 0x10000); + GpuWrite(harness, address, 0x01010101); + GpuWrite(harness, address + 0x8000, 0x01010101); + QueueRead(harness, address); + GpuWrite(harness, address, 0x02020202); + GpuWrite(harness, address + 0x8000, 0x02020202); + SubmitBatch(harness); + Assert.Equal(1, harness.Cache.EagerReadbacksStarted); + + GpuWrite(harness, address + 0x8000, 0x03030303); + QueueRead(harness, address); + + Assert.Equal(1, harness.Cache.EagerReadbacksUsed); + Assert.Equal(1, harness.Cache.StalePiecesKept); + Assert.Equal(0x02020202u, BitConverter.ToUInt32(harness.Read(address + 0x40, 4))); + Assert.True(harness.Cache.HasGpuDirtyBytes(address + 0x8000, 0x100)); + Assert.True(harness.Cache.MayHaveGpuDirtyPages(address + 0x8000, 0x100)); + Assert.True(harness.Cache.DownloadToCpu(address + 0x8000, 4)); + Assert.Equal(0x03030303u, BitConverter.ToUInt32(harness.Read(address + 0x8040, 4))); + Detach(harness); + harness.Shutdown(); + } + + [Fact] + public void AGuestReadOffTheQueueIsReadAheadToo() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + Attach(harness, _vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + GpuWrite(harness, address, 0x01010101); + Assert.True(harness.Cache.DownloadToCpu(address, 4)); + + GpuWrite(harness, address, 0x02020202); + SubmitBatch(harness); + Assert.Equal(1, harness.Cache.EagerReadbacksStarted); + Assert.True(harness.Cache.DownloadToCpu(address, 4)); + + Assert.Equal(1, harness.Cache.EagerReadbacksUsed); + Assert.Equal(0x02020202u, BitConverter.ToUInt32(harness.Read(address + 0x40, 4))); + Detach(harness); + harness.Shutdown(); + } + + [Fact] + public void AReadAheadDeferredForASlotStartsLater() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + Attach(harness, _vulkan); + const ulong window = 0x80000; + const int windows = 5; + var address = harness.MapBacked(window * windows, ReadWrite); + for (var index = 0; index < windows; index++) + { + GpuWrite(harness, address + (ulong)index * window, 0x01010101); + QueueRead(harness, address + (ulong)index * window); + } + + for (var index = 0; index < windows; index++) + { + GpuWrite(harness, address + (ulong)index * window, 0x02020202); + } + + SubmitBatch(harness); + Assert.Equal(4, harness.Cache.EagerReadbacksStarted); + + var deadline = DateTime.UtcNow.AddSeconds(10); + while (harness.Cache.EagerReadbacksStarted < windows && DateTime.UtcNow < deadline) + { + harness.Worker.Run(() => harness.Scheduler.Finish()); + Thread.Sleep(10); + SubmitBatch(harness); + } + + Assert.Equal(windows, harness.Cache.EagerReadbacksStarted); + Detach(harness); + harness.Shutdown(); + } + [Fact] public void RepeatedGuestReadsReuseTheReadbackSlots() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/WriteTickMapTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/WriteTickMapTests.cs new file mode 100644 index 0000000..7fb9679 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/WriteTickMapTests.cs @@ -0,0 +1,62 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Buffers; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.Buffers; + +public sealed class WriteTickMapTests +{ + [Fact] + public void ARangeReportsOnlyTheWritesThatReachIt() + { + var map = new WriteTickMap(); + map.Assign(0x000, 0x100, 5); + map.Assign(0x800, 0x100, 9); + + Assert.Equal(5UL, map.MaxTick(0x000, 0x100)); + Assert.Equal(9UL, map.MaxTick(0x880, 0x10)); + Assert.Equal(0UL, map.MaxTick(0x200, 0x400)); + Assert.Equal(9UL, map.MaxTick(0x000, 0x1000)); + } + + [Fact] + public void ANewerWriteSplitsTheOlderRange() + { + var map = new WriteTickMap(); + map.Assign(0x000, 0x300, 3); + map.Assign(0x100, 0x100, 7); + + Assert.Equal(3, map.Count); + Assert.Equal(3UL, map.MaxTick(0x000, 0x100)); + Assert.Equal(7UL, map.MaxTick(0x180, 0x8)); + Assert.Equal(3UL, map.MaxTick(0x200, 0x100)); + } + + [Fact] + public void AdjacentWritesOfOneTickMerge() + { + var map = new WriteTickMap(); + map.Assign(0x000, 0x100, 4); + map.Assign(0x200, 0x100, 4); + map.Assign(0x100, 0x100, 4); + + Assert.Equal(1, map.Count); + Assert.Equal(4UL, map.MaxTick(0x2FF, 1)); + } + + [Fact] + public void PrunedWritesReadAsTheFloor() + { + var map = new WriteTickMap(); + map.Assign(0x000, 0x100, 2); + map.Assign(0x100, 0x100, 6); + map.Prune(5); + + Assert.Equal(1, map.Count); + Assert.Equal(2UL, map.Floor); + Assert.Equal(2UL, map.MaxTick(0x400, 0x100)); + Assert.Equal(6UL, map.MaxTick(0x100, 0x100)); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs new file mode 100644 index 0000000..18ccd44 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs @@ -0,0 +1,236 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using Xunit; +using static SharpEmu.Libs.Gpu.GpuCommands.Registers.ContextRegisterOffset; +using static SharpEmu.Libs.Gpu.GpuCommands.Registers.ShaderRegisterOffset; +using static SharpEmu.Libs.Gpu.GpuCommands.Registers.UserConfigRegisterOffset; + +namespace SharpEmu.Libs.Tests.Gpu.GpuCommands.Registers; + +// A snapshot shares the live banks' parts until the live banks write them; it never changes. +public sealed class RegisterBanksSnapshotTests +{ + private const ulong PacketAddress = 0x1_0000_1000; + + private static RegisterBanks NewBanks() => new(static message => new InvalidOperationException(message)); + + private static PacketContext Packet(uint opcode, int valueCount) => + new(PacketHeader.Make((uint)valueCount + 2, opcode), PacketAddress, 0, 100, 100); + + private static void WriteContext(RegisterBanks banks, uint offset, params uint[] values) => + RegisterWriteTable.WriteContextPacket(banks, Packet(PacketOpcode.SetContextRegister, values.Length), offset, values); + + private static void WriteShader(RegisterBanks banks, uint offset, params uint[] values) => + RegisterWriteTable.WriteShaderPacket(banks, Packet(PacketOpcode.SetShaderRegister, values.Length), offset, values); + + private static void WriteUserConfig(RegisterBanks banks, uint offset, params uint[] values) => + RegisterWriteTable.WriteUserConfigPacket(banks, Packet(PacketOpcode.SetUserConfigRegister, values.Length), offset, values); + + [Fact] + public void WritesAfterASnapshotLeaveItUnchanged() + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + WriteShader(banks, SpiShaderPgmLoPs, 0x1000); + WriteShader(banks, SpiShaderUserDataPs0, 11, 12); + WriteShader(banks, SpiShaderPgmLoEs, 0x2000); + WriteShader(banks, ComputePgmLo, 0x3000); + WriteUserConfig(banks, VgtPrimitiveType, 4); + var snapshot = banks.Snapshot(); + + WriteContext(banks, DbDepthControl, 0x0000_0000); + WriteShader(banks, SpiShaderPgmLoPs, 0x5000); + WriteShader(banks, SpiShaderUserDataPs0, 21, 22); + WriteShader(banks, SpiShaderPgmLoEs, 0x6000); + WriteShader(banks, ComputePgmLo, 0x7000); + WriteUserConfig(banks, VgtPrimitiveType, 6); + + Assert.True(snapshot.Context.DepthTarget.DepthTestEnabled); + Assert.False(banks.Context.DepthTarget.DepthTestEnabled); + Assert.Equal(0x1000ul << 8, snapshot.Shader.Pixel.Address); + Assert.Equal(0x5000ul << 8, banks.Shader.Pixel.Address); + Assert.Equal([11u, 12u], snapshot.Shader.Pixel.UserScalars.Values[..2]); + Assert.Equal([21u, 22u], banks.Shader.Pixel.UserScalars.Values[..2]); + Assert.Equal(0x2000ul << 8, snapshot.Shader.Vertex.ExportAddress); + Assert.Equal(0x6000ul << 8, banks.Shader.Vertex.ExportAddress); + Assert.Equal(0x3000ul << 8, snapshot.Shader.Compute.Address); + Assert.Equal(0x7000ul << 8, banks.Shader.Compute.Address); + Assert.Equal(4u, snapshot.UserConfig.PrimitiveType); + Assert.Equal(6u, banks.UserConfig.PrimitiveType); + } + + [Fact] + public void AWriteCopiesOnlyThePartItStoresInto() + { + var banks = NewBanks(); + var snapshot = banks.Snapshot(); + + WriteShader(banks, SpiShaderUserDataPs0, 7); + + Assert.NotSame(snapshot.Shader.Pixel, banks.Shader.Pixel); + Assert.Same(snapshot.Shader.Vertex, banks.Shader.Vertex); + Assert.Same(snapshot.Shader.Compute, banks.Shader.Compute); + Assert.Same(snapshot.Context, banks.Context); + Assert.Same(snapshot.UserConfig, banks.UserConfig); + } + + [Fact] + public void AUserScalarWriteCopiesOnlyItsSet() + { + var banks = NewBanks(); + WriteShader(banks, SpiShaderUserDataGs0, 1); + WriteShader(banks, SpiShaderUserDataHs0, 2); + var snapshot = banks.Snapshot(); + + WriteShader(banks, SpiShaderUserDataGs0, 3); + WriteShader(banks, SpiShaderPgmLoPs, 0x4000); + + Assert.NotSame(snapshot.Shader.Vertex, banks.Shader.Vertex); + Assert.NotSame(snapshot.Shader.Vertex.GeometryUserScalars, banks.Shader.Vertex.GeometryUserScalars); + Assert.Same(snapshot.Shader.Vertex.HullUserScalars, banks.Shader.Vertex.HullUserScalars); + Assert.Equal(1u, snapshot.Shader.Vertex.GeometryUserScalars.Values[0]); + Assert.Equal(3u, banks.Shader.Vertex.GeometryUserScalars.Values[0]); + Assert.NotSame(snapshot.Shader.Pixel, banks.Shader.Pixel); + Assert.Same(snapshot.Shader.Pixel.UserScalars, banks.Shader.Pixel.UserScalars); + + // The set copied after the snapshot is the live banks' own: a later write does not copy it again. + var geometry = banks.Shader.Vertex.GeometryUserScalars; + WriteShader(banks, SpiShaderUserDataGs0, 4); + Assert.Same(geometry, banks.Shader.Vertex.GeometryUserScalars); + Assert.Equal(1u, snapshot.Shader.Vertex.GeometryUserScalars.Values[0]); + } + + [Fact] + public void ContextStateOperationsAfterASnapshotLeaveItUnchanged() + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + var snapshot = banks.Snapshot(); + + banks.ApplyContextState(ContextStateOperation.PushClear); + WriteContext(banks, DbDepthControl, 0x0000_0000); + banks.ApplyContextState(ContextStateOperation.Pop); + WriteContext(banks, DbDepthControl, 0x0000_0000); + + Assert.True(snapshot.Context.DepthTarget.DepthTestEnabled); + Assert.False(banks.Context.DepthTarget.DepthTestEnabled); + } + + // Every context register, through the table path and the packet path, against a snapshot + // taken after a first write: the context's nested parts are copied only by their writers. + [Fact] + public void EveryContextRegisterWriteLeavesASnapshotUnchanged() + { + var changed = 0; + for (var offset = 0u; offset < RegisterBankLayout.ContextRegisterCount; offset++) + { + foreach (var packet in new[] { false, true }) + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + var snapshot = banks.Snapshot(); + var before = Describe(snapshot.Context); + foreach (var value in new[] { 0xFFFF_FFFFu, 0x1234_5678u }) + { + try + { + if (packet) + WriteContext(banks, offset, value); + else + RegisterWriteTable.WriteContextEntry(banks, offset, value, PacketAddress); + } + catch (InvalidOperationException) + { + // A register whose writer refuses a single value. + } + } + + Assert.True(before == Describe(snapshot.Context), $"offset=0x{offset:X} packet={packet}"); + if (Describe(banks.Context) != before) + changed++; + } + } + + // The description sees the writes: most registers store something. + Assert.True(changed > 400, $"changed={changed}"); + } + + private static string Describe(object? value) + { + var text = new System.Text.StringBuilder(); + Describe(value, text); + return text.ToString(); + } + + private static void Describe(object? value, System.Text.StringBuilder text) + { + switch (value) + { + case null: + text.Append("null;"); + return; + case System.Collections.IDictionary dictionary: + foreach (var key in dictionary.Keys.Cast().OrderBy(key => key)) + text.Append(key).Append('=').Append(dictionary[key]).Append(','); + text.Append(';'); + return; + case Array array: + text.Append('['); + foreach (var element in array) + Describe(element, text); + text.Append(']'); + return; + } + + var type = value.GetType(); + if (type.IsPrimitive || type.IsEnum || value is string) + { + text.Append(value).Append(';'); + return; + } + + text.Append('{'); + foreach (var field in type.GetFields(System.Reflection.BindingFlags.Instance | + System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic)) + { + text.Append(field.Name).Append(':'); + Describe(field.GetValue(value), text); + } + + text.Append('}'); + } + + [Fact] + public void AContextWriteCopiesOnlyTheNestedPartsItStoresInto() + { + var banks = NewBanks(); + var snapshot = banks.Snapshot(); + + WriteContext(banks, DbDepthControl, 0x0000_0006); + WriteContext(banks, PaClVteCntl, 0x43F); + + Assert.NotSame(snapshot.Context, banks.Context); + Assert.NotSame(snapshot.Context.ScreenViewport, banks.Context.ScreenViewport); + Assert.Same(snapshot.Context.ScreenViewport.Viewports, banks.Context.ScreenViewport.Viewports); + Assert.Same(snapshot.Context.ShaderInterface, banks.Context.ShaderInterface); + Assert.Same(snapshot.Context.ColorTargets, banks.Context.ColorTargets); + Assert.Same(snapshot.Context.UnmodeledTableRegisters, banks.Context.UnmodeledTableRegisters); + } + + [Fact] + public void ASecondSnapshotSharesAgainUntilTheNextWrite() + { + var banks = NewBanks(); + var first = banks.Snapshot(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + var second = banks.Snapshot(); + WriteContext(banks, DbDepthControl, 0x0000_0000); + + Assert.False(first.Context.DepthTarget.DepthTestEnabled); + Assert.True(second.Context.DepthTarget.DepthTestEnabled); + Assert.False(banks.Context.DepthTarget.DepthTestEnabled); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs index 2eb5a41..233a83e 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs @@ -99,6 +99,24 @@ public sealed partial class GuestImageCacheTests harness.Shutdown(); } + // A fill seen before its target leaves its range known as DCC metadata, so the next fill of it + // runs on the host and the range stays CPU-readable. + [Fact] + public void APendingDccFill_MarksItsRangeAsDccMetadata() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var fatal = new FatalScope(); + using var harness = new CacheHarness(_vulkan); + var address = harness.MapBacked(0x20000, ReadWrite); + var metadata = address + 0x10000; + Assert.False(harness.Images.OverlapsDccMetadata(metadata, 0x100)); + + Assert.False(harness.Images.TryAbsorbDccFill(metadata, 0x100, 0x00000000u)); + + Assert.True(harness.Images.OverlapsDccMetadata(metadata + 0x80, 0x10)); + Assert.False(harness.Images.OverlapsDccMetadata(metadata + 0x100, 0x10)); + } + [Fact] public void DccFills_RemainGuestProducersUntilDescriptorDiscovery() { @@ -451,6 +469,35 @@ public sealed partial class GuestImageCacheTests harness.Shutdown(); } + // The pages inside a filled range are clean after it; a page it shares with another GPU-written + // range keeps that range's state. + [Fact] + public void DirtyDccFill_KeepsASharedBoundaryPageDirty() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + const ulong Page = SharpEmu.HLE.GpuMemory.TrackerLayout.PageBytes; + var memory = harness.MapBacked(8 * Page, ReadWrite); + var metadata = memory + Page + Page / 2; + var size = 3 * Page; + var neighbour = metadata - 0x100; + harness.Images.RegisterDccMetadataForTest(metadata, size); + harness.Worker.Run(() => + { + _ = harness.Cache.ObtainBuffer(neighbour, 0x100, isWritten: true); + _ = harness.Cache.ObtainBuffer(metadata, size, isWritten: true); + }); + + harness.Worker.Run(() => harness.Cache.FillBuffer(metadata, size, 0x00000000, isGds: false)); + + Assert.False(harness.Cache.HasGpuDirtyBytes(metadata, size)); + Assert.False(harness.Cache.HasGpuDirtyPages(memory + 2 * Page, 2 * Page)); + Assert.False(harness.Cache.HasGpuDirtyPages(memory + 4 * Page, Page)); + Assert.True(harness.Cache.HasGpuDirtyPages(memory + Page, Page)); + Assert.True(harness.Cache.HasGpuDirtyBytes(neighbour, 0x100)); + harness.Shutdown(); + } + [Fact] public void CleanDccFill_WritesBothCopiesWithoutMarkingTheGpuCopyNewer() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Ownership.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Ownership.cs index 21492f8..0751dfb 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Ownership.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Ownership.cs @@ -138,6 +138,58 @@ public sealed partial class GuestImageCacheTests } } + [Theory] + [InlineData(Format.R16G16Sfloat, GuestPixelFormat.Bits16_16Float, 4u, 0x3c00bc00u)] + [InlineData(Format.R16G16B16A16Sfloat, GuestPixelFormat.Bits16_16_16_16Float, 8u, 0x3c00bc00u)] + [InlineData(Format.R16G16B16A16Sfloat, GuestPixelFormat.Bits16_16_16_16Float, 8u, 0x80000001u)] + [InlineData(Format.R32G32Uint, GuestPixelFormat.Bits32_32UInt, 8u, 0xdeadbeefu)] + [InlineData(Format.R32G32B32A32Uint, GuestPixelFormat.Bits32_32_32_32UInt, 16u, 0xdeadbeefu)] + public void FullColorBufferFillPreservesRawWords(Format format, GuestPixelFormat guestFormat, uint bytesPerTexel, uint word) + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + var request = LinearRequest(address, bytesPerTexel, format, guestFormat, GuestImageType.Color2D, + new Extent3D(1, 1, 1), 1, bytesPerTexel, 1); + var image = harness.Find(ref request); + Assert.True(harness.Worker.Run(() => harness.Images.TryClearImageFromBuffer(address, bytesPerTexel, word))); + Assert.Equal(Bytes(Enumerable.Repeat(word, (int)bytesPerTexel / 4).ToArray()), harness.ReadImageBytes(harness.Image(image))); + Assert.True(harness.Image(image).IsGpuModified); + harness.Shutdown(); + } + + [Fact] + public void HalfFloatNaNBufferFillFallsBackWithoutChangingImageOwnership() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + var request = LinearRequest(address, 8, Format.R16G16B16A16Sfloat, GuestPixelFormat.Bits16_16_16_16Float, + GuestImageType.Color2D, new Extent3D(1, 1, 1), 1, 8, 1); + var image = harness.Find(ref request); + Assert.True(harness.Image(image).IsCpuDirty); + Assert.False(harness.Worker.Run(() => harness.Images.TryClearImageFromBuffer(address, 8, 0x7e003c00u))); + Assert.True(harness.Image(image).IsCpuDirty); + Assert.False(harness.Image(image).IsGpuModified); + harness.Shutdown(); + } + + [Fact] + public void HalfFloatBufferFillStillRequiresAnExactRange() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + var request = LinearRequest(address, 16, Format.R16G16B16A16Sfloat, GuestPixelFormat.Bits16_16_16_16Float, + GuestImageType.Color2D, new Extent3D(2, 1, 1), 1, 8, 1); + var image = harness.Find(ref request); + Assert.False(harness.Worker.Run(() => harness.Images.TryClearImageFromBuffer(address + 8, 8, 0x3c00bc00u))); + Assert.False(harness.Worker.Run(() => harness.Images.TryClearImageFromBuffer(address, 20, 0x3c00bc00u))); + Assert.True(harness.Image(image).IsCpuDirty); + Assert.False(harness.Image(image).IsGpuModified); + harness.Shutdown(); + } + [Fact] public void PartialPage_BufferAndImageShareOneTrackerPage() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs index 192a2e4..d06354f 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs @@ -212,6 +212,47 @@ public sealed class ImageDescriptionTests Assert.All(fatal.Messages, message => Assert.Contains("overlap range is invalid", message)); } + [Theory] + [InlineData(Format.R16G16Sfloat)] + [InlineData(Format.R16G16B16A16Sfloat)] + public void BufferFillDecodesRepeatedHalfWordsWithoutChangingDccClears(Format format) + { + Assert.True(PackedClearValue.TryDecodeBufferColor(format, 0x3c00bc00u, out var clear)); + Assert.Equal((-1f, 1f, -1f, 1f), (clear.Float32_0, clear.Float32_1, clear.Float32_2, clear.Float32_3)); + Assert.False(PackedClearValue.TryDecodeColor(format, 0x3c00bc00u, out _)); + Assert.False(PackedClearValue.TryDecodeBufferColor(format, 0x7e003c00u, out _)); + Assert.False(PackedClearValue.TryDecodeBufferColor(format, 0x3c007e00u, out _)); + } + + [Theory] + [InlineData(Format.R32G32Uint)] + [InlineData(Format.R32G32B32A32Uint)] + public void BufferFillRepeatsTheEntireUintWordInEveryComponent(Format format) + { + Assert.True(PackedClearValue.TryDecodeBufferColor(format, 0xdeadbeefu, out var clear)); + Assert.Equal((0xdeadbeefu, 0xdeadbeefu, 0xdeadbeefu, 0xdeadbeefu), + (clear.Uint32_0, clear.Uint32_1, clear.Uint32_2, clear.Uint32_3)); + } + + [Fact] + public void BufferFillPreservesEveryRepresentableHalfWord() + { + for (var bits = 0; bits <= ushort.MaxValue; bits++) + { + var packed = (uint)bits | ((uint)bits << 16); + var decoded = PackedClearValue.TryDecodeBufferColor(Format.R16G16B16A16Sfloat, packed, out var clear); + if (Half.IsNaN(BitConverter.UInt16BitsToHalf((ushort)bits))) + { + Assert.False(decoded); + continue; + } + + Assert.True(decoded); + Assert.Equal((ushort)bits, BitConverter.HalfToUInt16Bits((Half)clear.Float32_0)); + Assert.Equal((ushort)bits, BitConverter.HalfToUInt16Bits((Half)clear.Float32_1)); + } + } + [Fact] public void Validate_RejectsInconsistentDescriptions() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/MipStatisticsTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/MipStatisticsTests.cs index 28b5d32..51bdbed 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/MipStatisticsTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/MipStatisticsTests.cs @@ -113,4 +113,57 @@ public sealed class MipStatisticsTests Assert.All(report, static value => Assert.Equal(0, value)); } + + [Fact] + public void WrittenReport_PointsAFullReportAtItsCounters() + { + var statistics = new MipStatistics(); + statistics.NoteSampled(Texture(slot: 3)); + var report = new byte[MipStatistics.ReportBytes]; + + statistics.WriteReport(report); + + Assert.Equal(MipStatistics.FirstCounterOffset, BinaryPrimitives.ReadUInt32LittleEndian(report)); + Assert.Equal((0x10000u, 3u, 0x10000u, 0u), Counter(report, 3)); + } + + [Fact] + public void WrittenReport_MarksAShortBufferValidWithoutCounters() + { + var statistics = new MipStatistics(); + var report = new byte[16]; + + statistics.WriteReport(report); + + Assert.Equal(MipStatistics.ReportWithoutCounters, BinaryPrimitives.ReadUInt32LittleEndian(report)); + Assert.All(report[sizeof(uint)..], value => Assert.Equal(0, value)); + } + + [Fact] + public void ImageField_CarriesTheCounterBaseLevelAndMinimumLod() + { + uint[] texture = [0, 0x280u << 8, 0, 3u << 12, 0, CounterEnabled, 0x2A, 0]; + + Assert.Equal(0x2Au | (3u << 8) | (0x280u << 16), MipStatistics.ImageField(texture)); + texture[5] = 0; + Assert.Equal(MipStatistics.NoCounterField, MipStatistics.ImageField(texture)); + } + + [Fact] + public void PackedGpuReport_ReportsTheFinestMipAndTheClampedSamples() + { + var words = new uint[2 * MipStatistics.CounterWords]; + Array.Fill(words, uint.MaxValue, 0, (int)MipStatistics.CounterWords); + words[5] = 2; + words[MipStatistics.CounterWords + 5] = 40; + words[9] = 0; + var report = new byte[MipStatistics.ReportBytes]; + + MipStatistics.PackReport(words, report); + + Assert.Equal(MipStatistics.FirstCounterOffset, BinaryPrimitives.ReadUInt32LittleEndian(report)); + Assert.Equal((40u, 5u, 0u, 2u), Counter(report, 5)); + Assert.Equal((0u, 9u, 0u, 0u), Counter(report, 9)); + Assert.Equal((0u, 6u, 0u, MipStatistics.NoMipSampled), Counter(report, 6)); + } } diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs index 4d2a98b..e16ad15 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs @@ -266,6 +266,9 @@ public sealed class ShaderPipelineCacheTests : IDisposable executor.DrawAuto(1, Banks(), Auto(3)); executor.DrawAuto(2, Banks(), Auto(3)); executor.DrawAuto(3, Banks(PrimitiveTriangleStrip), Auto(4)); + // The lookup key is reused across draws: after another pipeline, the first is found again. + executor.DrawAuto(4, Banks(), Auto(3)); + executor.DrawAuto(5, Banks(PrimitiveTriangleStrip), Auto(4)); Assert.Equal(2, guest.Host.GraphicsPipelines.Count); Assert.Equal(2, cache.GraphicsPipelineCount); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs index b05928c..9d09711 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs @@ -279,9 +279,25 @@ public sealed class RenderExecutorComputeTests : IDisposable Assert.Contains($"clear_image {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); Assert.Contains($"absorb_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); + Assert.Contains($"fill_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); AssertDispatched(4, 1, 1); } + // A pending surface's metadata the host knows is filled there: as a dispatch it would be GPU-written + // and read back at every later bind. + [Fact] + public void PendingMetadataTheHostKnows_IsFilledOnTheHost() + { + _host.HostFillableDcc.Add(MetadataAddress); + ConfigureClearKernel(256); + _executor.Dispatch(1, Banks(), 256, 1, 1, ClearInitiator); + + Assert.Contains($"absorb_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); + Assert.Contains($"fill_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); + AssertNotDispatched(); + Assert.Contains("reset_bindings", _host.Calls); + } + [Fact] public void RegisteredDccAddress_ConsumesTheFillAndRecordsNothing() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs index aa772f4..66e3d5d 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs @@ -22,9 +22,21 @@ public sealed class RenderExecutorDrawTests : IDisposable private readonly RenderExecutor _executor; private readonly FatalScope _fatal = new(); - public RenderExecutorDrawTests() => _executor = new RenderExecutor(_host, _pipelines); + private readonly InputsCheckingPrefetch _prefetch; - public void Dispose() => _fatal.Dispose(); + public RenderExecutorDrawTests() + { + _executor = new RenderExecutor(_host, _pipelines); + _prefetch = new InputsCheckingPrefetch(_host); + _executor.Prefetch = _prefetch; + } + + // Every draw of these tests also checks that its program inputs follow from its registers. + public void Dispose() + { + _fatal.Dispose(); + Assert.Empty(_prefetch.Mismatches); + } private void AssertOrder(params string[] prefixes) { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs new file mode 100644 index 0000000..28377d1 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs @@ -0,0 +1,87 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Tests.Gpu.Scheduling; +using Xunit; +using static SharpEmu.Libs.Tests.Gpu.Rendering.RenderExecutorFixtures; + +namespace SharpEmu.Libs.Tests.Gpu.Rendering; + +// A draw takes the programs a prefetch resolved from its banks, or resolves them itself. +[Collection(SchedulingStateCollection.Name)] +public sealed class RenderExecutorPrefetchTests : IDisposable +{ + private readonly RecordingRenderHost _host = new(); + private readonly FakePipelineProvider _pipelines = new(); + private readonly RenderExecutor _executor; + private readonly FatalScope _fatal = new(); + + public RenderExecutorPrefetchTests() => _executor = new RenderExecutor(_host, _pipelines); + + public void Dispose() => _fatal.Dispose(); + + // Resolves ahead with the executor, as a worker would, and hands the result back once. + private sealed class AheadPrefetch(RenderExecutor executor, IRenderHost host) : IProgramPrefetch + { + private RegisterBanks? _banks; + private GraphicsProgramInputs _inputs; + private GraphicsPrograms? _programs; + + public int Taken { get; private set; } + + public int Declined { get; private set; } + + public void Resolve(RegisterBanks banks) + { + _banks = banks; + _inputs = RenderExecutor.ProgramInputsOf(banks, host.FormatSupport, host.Fatal); + _programs = executor.ResolveGraphicsPrograms(banks, in _inputs); + } + + public bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs) + { + if (_programs is not null && ReferenceEquals(banks, _banks) && inputs.Equals(_inputs)) + { + programs = _programs; + _programs = null; + Taken++; + return true; + } + + programs = null!; + Declined++; + return false; + } + } + + [Fact] + public void ADrawTakesTheProgramsResolvedAheadFromItsBanks() + { + var prefetch = new AheadPrefetch(_executor, _host); + _executor.Prefetch = prefetch; + var banks = Banks(); + prefetch.Resolve(banks); + var resolutions = _pipelines.Calls.Count(call => call.StartsWith("get_graphics_programs", StringComparison.Ordinal)); + + _executor.DrawAuto(1, banks, Auto(3)); + + Assert.Equal(1, prefetch.Taken); + Assert.Equal(resolutions, _pipelines.Calls.Count(call => call.StartsWith("get_graphics_programs", StringComparison.Ordinal))); + Assert.Contains(_host.Calls, call => call.StartsWith("draw ", StringComparison.Ordinal)); + } + + [Fact] + public void ADrawWithOtherBanksResolvesItsOwnPrograms() + { + var prefetch = new AheadPrefetch(_executor, _host); + _executor.Prefetch = prefetch; + prefetch.Resolve(Banks()); + + _executor.DrawAuto(1, Banks(), Auto(3)); + + Assert.Equal((0, 1), (prefetch.Taken, prefetch.Declined)); + Assert.Equal(2, _pipelines.Calls.Count(call => call.StartsWith("get_graphics_programs", StringComparison.Ordinal))); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs index c50241a..cfd6432 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs @@ -108,6 +108,9 @@ internal sealed class RecordingRenderHost : IRenderHost public HashSet RegisteredDcc { get; } = new(); + // Metadata the host knows and fills itself although its surface is not registered yet. + public HashSet HostFillableDcc { get; } = new(); + public HashSet ClearableImages { get; } = new(); public Func? ClampOverride { get; set; } @@ -391,7 +394,7 @@ internal sealed class RecordingRenderHost : IRenderHost public bool TryFillDccMetadata(ulong address, ulong size, uint fillValue) { Calls.Add($"fill_dcc {address:X} {size:X} {fillValue:X8}"); - return RegisteredDcc.Contains(address); + return RegisteredDcc.Contains(address) || HostFillableDcc.Contains(address); } public bool HostCopyAccepted { get; set; } = true; @@ -405,6 +408,33 @@ internal sealed class RecordingRenderHost : IRenderHost public Exception Fatal(string message) => new RenderExecutorFatalException(message); } +// Takes nothing, but checks that the inputs a draw resolves its programs with are the ones its +// register banks alone give: a prefetch computed ahead of the draw would use those. +internal sealed class InputsCheckingPrefetch(IRenderHost host) : IProgramPrefetch +{ + public List Mismatches { get; } = new(); + + public int Checks { get; private set; } + + public bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs) + { + Checks++; + var expected = RenderExecutor.ProgramInputsOf(banks, host.FormatSupport, host.Fatal); + if (!expected.Equals(inputs)) + { + Mismatches.Add( + $"draw pixelActive={inputs.PixelActive}/{expected.PixelActive} depthBound={inputs.DepthBound}/{expected.DepthBound} " + + $"mapping=[{Describe(inputs.Mapping)}]/[{Describe(expected.Mapping)}]"); + } + + programs = null!; + return false; + } + + private static string Describe(ReadOnlySpan mapping) => + string.Join(",", mapping.ToArray().Select(map => map.Packed)); +} + // Hands back the configured programs and records every pipeline request. internal sealed class FakePipelineProvider : IShaderPipelineProvider { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Scheduling/RecordingBufferBarrierTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Scheduling/RecordingBufferBarrierTests.cs new file mode 100644 index 0000000..2e85c3e --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Scheduling/RecordingBufferBarrierTests.cs @@ -0,0 +1,72 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Scheduling; +using Silk.NET.Vulkan; +using Xunit; +using static SharpEmu.Libs.Tests.Gpu.Scheduling.SchedulingTestSupport; + +namespace SharpEmu.Libs.Tests.Gpu.Scheduling; + +[Collection(SchedulingStateCollection.Name)] +public sealed class RecordingBufferBarrierTests +{ + private static readonly MemoryBarrierRequest ComputeToAll = new( + PipelineStageFlags.ComputeShaderBit, PipelineStageFlags.AllCommandsBit, + AccessFlags.ShaderWriteBit, AccessFlags.MemoryReadBit | AccessFlags.MemoryWriteBit); + + private static readonly MemoryBarrierRequest AllToCompute = new( + PipelineStageFlags.AllCommandsBit, PipelineStageFlags.ComputeShaderBit, + AccessFlags.MemoryReadBit | AccessFlags.MemoryWriteBit, AccessFlags.ShaderReadBit | AccessFlags.ShaderWriteBit); + + private readonly FakeTickDevice _device = new(); + private readonly RecordingRenderingState _rendering = new(); + + [Fact] + public void RequestsBetweenCommandsAreRecordedAsOneUnion() + { + using var scheduler = NewActiveScheduler(_device, _rendering); + var recorded = new List(); + scheduler.MemoryBarrierRecorder = (_, request) => recorded.Add(request); + + scheduler.Current.RequestMemoryBarrier(ComputeToAll); + _ = scheduler.Current.StateHandle; + scheduler.Current.RequestMemoryBarrier(AllToCompute); + Assert.Empty(recorded); + _ = scheduler.Current.Handle; + _ = scheduler.Current.Handle; + + Assert.Equal([ComputeToAll.Union(AllToCompute)], recorded); + Assert.Equal((1L, 1L), (scheduler.Current.MergedBarriers, scheduler.Current.RecordedBarriers)); + _device.CompleteOnSubmit = true; + } + + [Fact] + public void APendingRequestIsRecordedBeforeTheBufferEnds() + { + using var scheduler = NewActiveScheduler(_device, _rendering); + var recorded = new List(); + scheduler.MemoryBarrierRecorder = (buffer, _) => recorded.Add($"barrier {_device.Name(buffer)} after {_device.Log[^1]}"); + scheduler.Current.RequestMemoryBarrier(ComputeToAll); + _device.CompleteOnSubmit = true; + + scheduler.Submit(); + + Assert.Equal(["barrier b0 after begin b0"], recorded); + Assert.Contains("end b0", _device.Log); + } + + [Fact] + public void ARequestInsideRenderingIsRefused() + { + using var fatal = new FatalScope(); + using var scheduler = NewActiveScheduler(_device, _rendering); + scheduler.MemoryBarrierRecorder = (_, _) => { }; + _rendering.IsRendering = true; + + Assert.Throws(() => scheduler.Current.RequestMemoryBarrier(ComputeToAll)); + + _rendering.IsRendering = false; + _device.CompleteOnSubmit = true; + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/HeadlessVulkan.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/HeadlessVulkan.cs index 5c3f36f..60068f5 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/HeadlessVulkan.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/HeadlessVulkan.cs @@ -161,6 +161,7 @@ internal sealed unsafe class HeadlessVulkan : IDisposable public void Dispose() { Vk.DeviceWaitIdle(Device); + _deviceInfo?.Slabs.Destroy(); Vk.DestroyDevice(Device, null); if (_debugUtils is { } debugUtils) { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs index fb9f391..141bca6 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs @@ -5,6 +5,7 @@ using System.Buffers.Binary; using SharpEmu.Libs.Gpu.Buffers; using SharpEmu.Libs.Tests.Gpu.Images; using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Ir; using SharpEmu.ShaderCompiler.Resources; using SharpEmu.ShaderCompiler.Vulkan; using Xunit; @@ -12,16 +13,18 @@ using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; namespace SharpEmu.Libs.Tests.Gpu.Vulkan; -public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : IClassFixture +public class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : IClassFixture { - private const uint Vcc = 106; - private const uint Exec = 126; - private static readonly Gen5Operand AllLanes = Gen5Operand.Source(193); + protected virtual bool HostSubgroup32 => false; + + protected const uint Vcc = 106; + protected const uint Exec = 126; + protected static readonly Gen5Operand AllLanes = Gen5Operand.Source(193); private readonly List _instructions = []; private uint _pc; - private uint Add(Gen5ShaderInstruction instruction) + protected uint Add(Gen5ShaderInstruction instruction) { var pc = _pc; _instructions.Add(instruction with { Pc = pc }); @@ -29,46 +32,60 @@ public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : I return pc; } - private void Point(uint branchPc, uint targetPc) + protected void Point(uint branchPc, uint targetPc) { var index = _instructions.FindIndex(instruction => instruction.Pc == branchPc); var offset = (short)(((int)targetPc - (int)branchPc - 4) / 4); _instructions[index] = _instructions[index] with { Words = [unchecked((uint)(ushort)offset)] }; } - private void Lane() + protected void Lane(uint rowShift = 3) { - Add(Vop2(0, "VLshlrevB32", 2, Operand(3), Gen5Operand.Vector(1))); + Add(Vop2(0, "VLshlrevB32", 2, Operand(rowShift), Gen5Operand.Vector(1))); Add(Vop2(0, "VAddI32", 2, Gen5Operand.Vector(0), Gen5Operand.Vector(2))); } - private uint CompareTo(string opcode, uint destination, Gen5Operand left, Gen5Operand right) => + protected uint CompareTo(string opcode, uint destination, Gen5Operand left, Gen5Operand right) => Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop3, opcode, [0u, 0u], [left, right], [Gen5Operand.Scalar(destination)], new Gen5Vop3Control(0, 0, 0, false, 0, destination))); - private uint[] Run(uint localDataShareDwords = 0) + protected uint[] Run(uint localDataShareDwords = 0, uint sizeX = 8, uint sizeY = 8, bool? laneLocal = null) { + var threads = sizeX * sizeY; Add(Vop2(0, "VLshlrevB32", 5, Operand(2), Gen5Operand.Vector(2))); Add(BufferAccess(0, "BufferStoreDword", 4, vectorData: 4, offsetEnabled: true, vectorAddress: 5)); Add(EndProgram(0)); - var (plan, resources, layout) = Prepare(Program([.. _instructions])); + var program = Program([.. _instructions]); + if (laneLocal is { } expected) + { + Assert.Equal(expected, Gen5WaveLaneLocalAnalysis.IsLaneLocal(program, new HashSet(), out var reason)); + Assert.True(expected || reason.Length != 0); + } + + var (plan, resources, layout) = Prepare(program); var request = new ShaderCompileRequest(plan, resources, layout) { - LocalSizeX = 8, LocalSizeY = 8, ThreadCountX = 8, ThreadCountY = 8, WaveSize = 64, - LocalDataShareDwords = localDataShareDwords, + LocalSizeX = sizeX, LocalSizeY = sizeY, ThreadCountX = sizeX, ThreadCountY = sizeY, WaveSize = 64, + LocalDataShareDwords = localDataShareDwords, HostComputeSubgroup32 = HostSubgroup32, }; Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); + if (HostSubgroup32 && laneLocal is { } oneLanePerInvocation) + { + var halfWavesEnabled = Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_LANE_LOCAL") == "1"; + Assert.Equal(oneLanePerInvocation && halfWavesEnabled ? sizeX : (threads + 63) / 64 * 32, LocalSizeX(shader.Spirv)); + } + Assert.NotNull(fixture.Vulkan); using var harness = new ImageTestHarness(fixture.Vulkan); using var runner = new LayoutComputeRunner(harness, request, shader.Spirv); - var output = runner.CreateBuffer(256); + var output = runner.CreateBuffer(threads * 4); var registers = new uint[256]; - registers[6] = 256; + registers[6] = threads * 4; harness.Run(() => runner.Dispatch(registers, new Dictionary { [DescriptorBindingKind.Buffers] = [output] }, 1)); - var bytes = runner.ReadBack(output, 0, 256); + var bytes = runner.ReadBack(output, 0, threads * 4); harness.AssertNoValidationMessages(); - var lanes = new uint[64]; + var lanes = new uint[threads]; for (var lane = 0; lane < lanes.Length; lane++) { lanes[lane] = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(lane * sizeof(uint))); @@ -87,6 +104,25 @@ public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : I Assert.All(Run(), x => Assert.Equal(0xFFFF_FFF8u, x)); } + [Fact] + public void ANarrowedRegionKeepsTheValuesItsInactiveLanesShowLater() + { + if (!Ready()) return; + Lane(); + Add(MoveVector(0, 4, 7)); + Add(Sop1(0, "SMovB64", 20, Gen5Operand.Scalar(Exec))); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + Add(Vop2(0, "VAddU32", 3, Operand(64), Gen5Operand.Vector(2))); + Add(Vop2(0, "VAddU32", 4, Operand(1), Gen5Operand.Vector(3))); + Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(20))); + + var lanes = Run(); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal(lane >= 40 ? lane + 65 : 7u, lanes[lane]); + } + } + [Fact] public void InitialExecContainsEveryLane() { @@ -96,7 +132,27 @@ public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : I Assert.All(Run(), x => Assert.Equal(uint.MaxValue, x)); } - private bool Ready() => GatePrerequisites.Ready(fixture.Vulkan, shaderInt64: true); + private static uint LocalSizeX(byte[] spirv) + { + const uint ExecutionMode = 16; + const uint LocalSize = 17; + for (var offset = 5 * sizeof(uint); offset < spirv.Length;) + { + var word = BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset)); + var count = (int)(word >> 16); + if ((word & 0xFFFF) == ExecutionMode && + BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset + 2 * sizeof(uint))) == LocalSize) + { + return BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset + 3 * sizeof(uint))); + } + + offset += Math.Max(count, 1) * sizeof(uint); + } + + throw new InvalidOperationException("no LocalSize execution mode"); + } + + protected bool Ready() => GatePrerequisites.Ready(fixture.Vulkan, shaderInt64: true); [Fact] public void MasksAcrossTheHalfBoundary_StayExactPerLaneAndWhenCounted() @@ -271,7 +327,7 @@ public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : I var (plan, resources, layout) = Prepare(program, userDataCount: 12); var request = new ShaderCompileRequest(plan, resources, layout) { - LocalSizeX = 64, WaveSize = 64, EnableExecGuardElision = false, + LocalSizeX = 64, WaveSize = 64, EnableExecGuardElision = false, HostComputeSubgroup32 = HostSubgroup32, ComputeSystemRegisters = new Gen5ComputeSystemRegisters(12, null, null, null), }; Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64Subgroup32DeviceTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64Subgroup32DeviceTests.cs new file mode 100644 index 0000000..e6f9096 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64Subgroup32DeviceTests.cs @@ -0,0 +1,183 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.Libs.Tests.Gpu.Vulkan; + +public sealed class Wave64Subgroup32DeviceTests(HeadlessVulkanFixture fixture) : Wave64HalfMaskDeviceTests(fixture) +{ + protected override bool HostSubgroup32 => true; + + [Fact] + public void LaneLocalArithmetic_UsesPairedLanesUnlessHalfWavesAreEnabled() + { + if (!Ready()) return; + Lane(rowShift: 4); + Add(Vop2(0, "VAddU32", 4, Operand(17), Gen5Operand.Vector(2))); + + var lanes = Run(sizeX: 16, sizeY: 16, laneLocal: true); + for (uint index = 0; index < 256; index++) + { + Assert.Equal(index + 17, lanes[index]); + } + } + + [Theory] + [InlineData(45u)] + [InlineData(7u)] + public void ReadLane_ReadsEitherHalfOfTheWave(uint lane) + { + if (!Ready()) return; + Lane(); + Add(Vop2(0, "VAddU32", 3, Operand(100), Gen5Operand.Vector(2))); + Add(ReadLane(0, 20, 3, lane)); + Add(MoveVectorFromScalar(0, 4, 20)); + + Assert.All(Run(laneLocal: false), value => Assert.Equal(100 + lane, value)); + } + + [Fact] + public void WriteLane_WritesOneLaneOfTheUpperHalf() + { + if (!Ready()) return; + Lane(); + Add(MoveVector(0, 4, 1)); + Add(MoveScalar(0, 20, 9)); + Add(WriteLane(0, 4, 20, 50)); + + var lanes = Run(laneLocal: false); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal(lane == 50 ? 9u : 1u, lanes[lane]); + } + } + + [Fact] + public void EveryWaveOfALargeGroup_CountsItsOwnLanes() + { + if (!Ready()) return; + Lane(rowShift: 4); + CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(100)); + Add(Sop1(0, "SBcnt1I32B64", 14, Gen5Operand.Scalar(10))); + Add(MoveVectorFromScalar(0, 4, 14)); + + var lanes = Run(sizeX: 16, sizeY: 16, laneLocal: false); + for (uint index = 0; index < 256; index++) + { + var wave = index / 64; + Assert.Equal(wave == 0 ? 64u : wave == 1 ? 36u : 0u, lanes[index]); + } + } + + [Fact] + public void APartialWave_HasOnlyItsThreadsInExec() + { + if (!Ready()) return; + Lane(); + Add(Sop1(0, "SBcnt1I32B64", 14, Gen5Operand.Scalar(Exec))); + Add(MoveVectorFromScalar(0, 4, 14)); + + Assert.All(Run(sizeX: 8, sizeY: 6, laneLocal: false), value => Assert.Equal(48u, value)); + } + + [Fact] + public void TwoWaves_ExchangeThroughSharedMemoryAfterABarrier() + { + if (!Ready()) return; + Lane(rowShift: 4); + Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(2))); + Add(Vop2(0, "VLshlrevB32", 6, Operand(2), Gen5Operand.Vector(2))); + Add(DataShare(0, "DsWriteB32", false, [Gen5Operand.Vector(6), Gen5Operand.Vector(3)], [])); + Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Sopp, "SBarrier", [0u], [], [], null)); + Add(Vop2(0, "VSubU32", 7, Operand(127), Gen5Operand.Vector(2))); + Add(Vop2(0, "VLshlrevB32", 7, Operand(2), Gen5Operand.Vector(7))); + Add(DataShare(0, "DsReadB32", false, [Gen5Operand.Vector(7)], [4])); + + var lanes = Run(localDataShareDwords: 128, sizeX: 16, sizeY: 8, laneLocal: false); + for (uint index = 0; index < 128; index++) + { + Assert.Equal(128 - index, lanes[index]); + } + } + + [Fact] + public void EveryHalfOfALargeGroup_SelectsItsOwnLanes() + { + if (!Ready()) return; + Lane(rowShift: 4); + CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(100)); + Add(Vop3(0, "VCndmaskB32", 4, Operand(0), Operand(1000), Gen5Operand.Scalar(10))); + Add(Vop2(0, "VAddU32", 4, Gen5Operand.Vector(2), Gen5Operand.Vector(4))); + + var lanes = Run(sizeX: 16, sizeY: 16, laneLocal: true); + for (uint index = 0; index < 256; index++) + { + Assert.Equal(index < 100 ? 1000 + index : index, lanes[index]); + } + } + + [Fact] + public void APartialLastWave_SelectsOnlyItsThreads() + { + if (!Ready()) return; + Lane(rowShift: 4); + CompareTo("VCmpGeU32", 10, Gen5Operand.Vector(2), Operand(40)); + Add(Vop3(0, "VCndmaskB32", 4, Operand(0), Operand(2000), Gen5Operand.Scalar(10))); + Add(Vop2(0, "VAddU32", 4, Gen5Operand.Vector(2), Gen5Operand.Vector(4))); + + var lanes = Run(sizeX: 16, sizeY: 5, laneLocal: true); + for (uint index = 0; index < 80; index++) + { + Assert.Equal(index >= 40 ? 2000 + index : index, lanes[index]); + } + } + + [Theory] + [InlineData(48u)] + [InlineData(8u)] + [InlineData(64u)] + public void AnExecSkip_LeavesTheInactiveLanesOfEachHalf(uint firstActiveLane) + { + if (!Ready()) return; + Lane(); + Add(MoveVector(0, 4, 5)); + Add(Vopc(0, "VCmpxLeU32", Operand(firstActiveLane), 2)); + var skip = Add(Branch(0, "SCbranchExecz", 0)); + Add(MoveVector(0, 4, 77)); + var join = Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Point(skip, join); + + var lanes = Run(laneLocal: true); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal(lane >= firstActiveLane ? 77u : 5u, lanes[lane]); + } + } + + [Fact] + public void AWaterfallLoop_VisitsEveryValueOfEveryHalf() + { + if (!Ready()) return; + Lane(rowShift: 4); + Add(Vop2(0, "VAndB32", 3, Operand(3), Gen5Operand.Vector(2))); + Add(Sop1(0, "SMovB64", 30, Gen5Operand.Scalar(Exec))); + var loop = Add(ReadFirstLane(0, 20, 3)); + CompareTo("VCmpEqU32", 22, Gen5Operand.Vector(3), Gen5Operand.Scalar(20)); + Add(Sop1(0, "SAndSaveexecB64", 24, Gen5Operand.Scalar(22))); + Add(MoveVectorFromScalar(0, 4, 20)); + Add(Vop2(0, "VAddU32", 4, Operand(100), Gen5Operand.Vector(4))); + Add(Sop2(0, "SXorB64", Exec, Gen5Operand.Scalar(24), Gen5Operand.Scalar(Exec))); + var back = Add(Branch(0, "SCbranchExecnz", 0)); + Point(back, loop); + Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(30))); + + var lanes = Run(sizeX: 16, sizeY: 16, laneLocal: true); + for (uint index = 0; index < 256; index++) + { + Assert.Equal((index & 3) + 100, lanes[index]); + } + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveMaxReductionDeviceTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveMaxReductionDeviceTests.cs index b97025d..e537c18 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveMaxReductionDeviceTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveMaxReductionDeviceTests.cs @@ -32,9 +32,11 @@ public sealed class WaveMaxReductionDeviceTests(HeadlessVulkanFixture fixture) : ]; [Theory] - [InlineData(0x7E040300u, 63u)] // v_mov_b32 v2, v0: the maximum is in the last lane - [InlineData(0x4C0400BFu, 63u)] // v_sub_nc_u32 v2, 63, v0: the maximum is in lane 0 - public void ReductionYieldsTheWaveMaximum(uint prepareValue, uint expected) + [InlineData(0x7E040300u, 63u, false)] // v_mov_b32 v2, v0: the maximum is in the last lane + [InlineData(0x4C0400BFu, 63u, false)] // v_sub_nc_u32 v2, 63, v0: the maximum is in lane 0 + [InlineData(0x7E040300u, 63u, true)] + [InlineData(0x4C0400BFu, 63u, true)] + public void ReductionYieldsTheWaveMaximum(uint prepareValue, uint expected, bool hostSubgroup32) { var vulkan = fixture.Vulkan; if (!GatePrerequisites.Ready(vulkan, shaderInt64: true)) return; @@ -58,7 +60,7 @@ public sealed class WaveMaxReductionDeviceTests(HeadlessVulkanFixture fixture) : var (plan, resources, layout) = Prepare(program); var request = new ShaderCompileRequest(plan, resources, layout) { - LocalSizeX = 64, ThreadCountX = 64, WaveSize = 64, + LocalSizeX = 64, ThreadCountX = 64, WaveSize = 64, HostComputeSubgroup32 = hostSubgroup32, }; Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); using var harness = new ImageTestHarness(vulkan); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveShuffleDeviceTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveShuffleDeviceTests.cs index bd09267..f7d2bf7 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveShuffleDeviceTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveShuffleDeviceTests.cs @@ -17,12 +17,17 @@ public sealed class WaveShuffleDeviceTests(HeadlessVulkanFixture fixture, ITestO : IClassFixture { [Theory] - [InlineData("Dpp8")] - [InlineData("Dpp16")] - [InlineData("Permlane16")] - [InlineData("Permlanex16")] - [InlineData("Reduction")] - public void Wave64Shuffle_PreservesUpperHalf(string operation) + [InlineData("Dpp8", false)] + [InlineData("Dpp16", false)] + [InlineData("Permlane16", false)] + [InlineData("Permlanex16", false)] + [InlineData("Reduction", false)] + [InlineData("Dpp8", true)] + [InlineData("Dpp16", true)] + [InlineData("Permlane16", true)] + [InlineData("Permlanex16", true)] + [InlineData("Reduction", true)] + public void Wave64Shuffle_PreservesUpperHalf(string operation, bool hostSubgroup32) { var vulkan = fixture.Vulkan; if (!GatePrerequisites.Ready(vulkan, shaderInt64: true)) return; @@ -80,6 +85,7 @@ public sealed class WaveShuffleDeviceTests(HeadlessVulkanFixture fixture, ITestO var request = new ShaderCompileRequest(plan, resources, layout) { LocalSizeX = 8, LocalSizeY = 8, ThreadCountX = 8, ThreadCountY = 8, WaveSize = 64, + HostComputeSubgroup32 = hostSubgroup32, }; Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); using var harness = new ImageTestHarness(vulkan); diff --git a/tests/SharpEmu.Libs.Tests/VideoOut/ConcurrentResolveTests.cs b/tests/SharpEmu.Libs.Tests/VideoOut/ConcurrentResolveTests.cs new file mode 100644 index 0000000..84652a3 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/VideoOut/ConcurrentResolveTests.cs @@ -0,0 +1,75 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.VideoOut; +using Xunit; +using static SharpEmu.Libs.VideoOut.VulkanVideoPresenter; + +namespace SharpEmu.Libs.Tests.VideoOut; + +public sealed class ConcurrentResolveTests +{ + [Fact] + public void ConsecutiveAndRepeatedReadsKeepOneProofPerQuery() + { + var assumptions = new ConcurrentAssumptions(1, 2); + assumptions.Record(ConcurrentQuery.ResidentClean, 0x1000, 4, false); + assumptions.Record(ConcurrentQuery.CleanWord, 0x1000, 4, false); + var allocated = GC.GetAllocatedBytesForCurrentThread(); + for (var index = 0; index < 100000; index++) + { + var address = 0x1000UL + (ulong)index * 4; + assumptions.Record(ConcurrentQuery.ResidentClean, address, 4, false); + assumptions.Record(ConcurrentQuery.CleanWord, address, 4, false); + assumptions.Record(ConcurrentQuery.ResidentClean, address, 4, false); + } + + Assert.Equal(allocated, GC.GetAllocatedBytesForCurrentThread()); + Assert.Equal(2, assumptions.Answers.Count); + Assert.All(assumptions.Answers, proof => + { + Assert.Equal(0x1000UL, proof.Address); + Assert.Equal(400000UL, proof.Size); + Assert.False(proof.Answer); + }); + Assert.True(assumptions.ReadsImageOwnership); + } + + [Fact] + public void OverlappingReadsMergeWithoutAddingUnreadGaps() + { + var assumptions = new ConcurrentAssumptions(1, 2); + assumptions.Record(ConcurrentQuery.DirtyBytes, 0x1020, 0x20, false); + assumptions.Record(ConcurrentQuery.DirtyBytes, 0x1000, 0x30, false); + assumptions.Record(ConcurrentQuery.DirtyBytes, 0x1050, 0x10, false); + + Assert.Equal((ConcurrentQuery.DirtyBytes, 0x1000UL, 0x40UL, false), assumptions.Answers[0]); + Assert.Equal((ConcurrentQuery.DirtyBytes, 0x1050UL, 0x10UL, false), assumptions.Answers[1]); + Assert.False(assumptions.ReadsImageOwnership); + } + + [Fact] + public void OwnedReadsAndDifferentOwnershipRulesRemainSeparate() + { + var assumptions = new ConcurrentAssumptions(1, 2); + assumptions.Record(ConcurrentQuery.Resident, 0x1000, 4, false); + assumptions.Record(ConcurrentQuery.ResidentClean, 0x1000, 4, false); + assumptions.Record(ConcurrentQuery.Resident, 0x1004, 4, true); + assumptions.Record(ConcurrentQuery.Resident, 0x1008, 4, true); + assumptions.Record(ConcurrentQuery.Resident, 0x1004, 4, true); + + Assert.Equal(4, assumptions.Answers.Count); + Assert.Equal(2, assumptions.Answers.Count(proof => proof.Answer)); + Assert.True(assumptions.ReadsImageOwnership); + } + + [Fact] + public void WrappingRangesDoNotMergeWithLowAddresses() + { + var assumptions = new ConcurrentAssumptions(1, 2); + assumptions.Record(ConcurrentQuery.Resident, ulong.MaxValue - 1, 4, false); + assumptions.Record(ConcurrentQuery.Resident, 0, 4, false); + + Assert.Equal(2, assumptions.Answers.Count); + } +} diff --git a/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs b/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs index 5592b49..c604ec1 100644 --- a/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs +++ b/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs @@ -92,6 +92,41 @@ public sealed class PresenterImageBindingTests : IClassFixture + { + var resource = new ImageResource + { + ResourceClass = PlanImageResourceClass.Sampled, + NumericClass = ImageNumericClass.Float, + Dimension = ImageDimension.Dim2D, + Read = true, + }; + var program = new ShaderProgramInfo(); + var words = RegisterWords.Texture(address, GuestPixelFormat.Bits32Float, 48, 24); + var first = (ResourceSlotIdentifier)GetFieldValue(presenter.InvokeMethod("ResolveImageBinding", resource, words, program, 0)!, "ImageIdentifier"); + var second = (ResourceSlotIdentifier)GetFieldValue(presenter.InvokeMethod("ResolveImageBinding", resource, words, program, 0)!, "ImageIdentifier"); + Assert.Equal(first, second); + + // A removed image is not served again: the binding looks the descriptor up afresh. + var generation = harness.Images.LookupGeneration; + typeof(GuestImageCache).GetMethod("DeleteImage", InstanceMembers)!.Invoke(harness.Images, [first]); + Assert.NotEqual(generation, harness.Images.LookupGeneration); + var third = (ResourceSlotIdentifier)GetFieldValue(presenter.InvokeMethod("ResolveImageBinding", resource, words, program, 0)!, "ImageIdentifier"); + Assert.True(harness.Images.TryGetImage(third, out var image) && image.Registered); + presenter.RenderHost.ResetBindings(); + }); + harness.Shutdown(); + } + [Theory] [InlineData(true, 1u, false)] [InlineData(true, 2u, true)] diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5MipStatisticsTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5MipStatisticsTests.cs new file mode 100644 index 0000000..ba14e13 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5MipStatisticsTests.cs @@ -0,0 +1,118 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using SharpEmu.ShaderCompiler.Resources; +using SharpEmu.ShaderCompiler.Tests.Resources; +using SharpEmu.ShaderCompiler.Vulkan; +using Xunit; + +namespace SharpEmu.ShaderCompiler.Tests; + +public sealed class Gen5MipStatisticsTests +{ + private const ulong ShaderAddress = 0x1_0000_C000; + + private static (ShaderCompileRequest Request, BindingLayout Layout) PixelSample(string opcode, bool recordStatistics, bool reduce) + { + var addressCount = opcode.Contains("SampleL", StringComparison.Ordinal) && !opcode.Contains("Lz", StringComparison.Ordinal) ? 3 : 2; + var control = new Gen5ImageControl(0xF, 0, Enumerable.Range(0, addressCount).Select(static value => (uint)value).ToArray(), 4, 8, 16, 1, + false, false, false, false, false); + var program = new Gen5ShaderProgram(ShaderAddress, + [ + new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Mimg, opcode, [], [], [], control), + new Gen5ShaderInstruction(8, Gen5ShaderEncoding.Sopp, "SEndpgm", [0xBF810000], [], [], null), + ]); + var plan = ShaderResourcePlan.Extract(program, ShaderStage.Pixel, ResourceTestProgram.Hash, 0, 20); + var userData = new uint[20]; + userData[8] = 0x20; + userData[9] = 71u << 20; + userData[11] = 9u << 28; + userData[13] = 1u << 25; + userData[14] = 7; + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, ResourceTestProgram.Inputs(userData), ref snapshot, ref specialization)); + specialization.Images[0] = specialization.Images[0] with { MipStatistics = recordStatistics }; + var resources = ResourceMaterializer.ApplyTo(plan, specialization); + var layout = BindingLayout.Allocate(resources.Info, BindingLayout.CollectUserDataRegisters(program, 0, 20), false, + ShaderCompileRequest.RequiresFlattenedTable(plan, resources), false); + var request = new ShaderCompileRequest(plan, resources, layout) + { + PixelOutputs = [new Gen5PixelOutputBinding(0, 0, Gen5PixelOutputKind.Float)], + MipStatisticsSubgroupReduction = reduce, + }; + return (request, layout); + } + + private static List Compile(ShaderCompileRequest request) + { + Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); + ValidateWhenAvailable(shader.Spirv); + var opcodes = new List(); + for (var index = 20; index + 4 <= shader.Spirv.Length;) + { + var word = BitConverter.ToUInt32(shader.Spirv, index); + opcodes.Add((SpirvOp)(word & 0xFFFF)); + index += (int)(word >> 16) * sizeof(uint); + } + + return opcodes; + } + + [Theory] + [InlineData("ImageSample", false)] + [InlineData("ImageSample", true)] + [InlineData("ImageSampleL", false)] + [InlineData("ImageSampleLz", true)] + public void ASampledImageWithACounterRecordsItsFinestMip(string opcode, bool reduce) + { + var (request, layout) = PixelSample(opcode, recordStatistics: true, reduce); + + Assert.Equal(1u, layout.MipStatisticsCount); + Assert.NotNull(layout.Find(DescriptorBindingKind.MipStatistics)); + var opcodes = Compile(request); + Assert.Contains(SpirvOp.AtomicUMin, opcodes); + Assert.Equal(opcode == "ImageSample", opcodes.Contains(SpirvOp.ImageQueryLod)); + Assert.Equal(reduce, opcodes.Contains(SpirvOp.GroupNonUniformUMin)); + } + + [Fact] + public void ADrawWithoutCountersCompilesThePlainProgram() + { + var (request, layout) = PixelSample("ImageSample", recordStatistics: false, reduce: true); + + Assert.Equal(0u, layout.MipStatisticsCount); + Assert.Null(layout.Find(DescriptorBindingKind.MipStatistics)); + Assert.DoesNotContain(SpirvOp.AtomicUMin, Compile(request)); + } + + internal static void ValidateWhenAvailable(byte[] code) + { + var sdk = Environment.GetEnvironmentVariable("VULKAN_SDK"); + var executable = string.IsNullOrWhiteSpace(sdk) ? null + : Path.Combine(sdk, OperatingSystem.IsWindows() ? "Bin/spirv-val.exe" : "bin/spirv-val"); + if (executable is null || !File.Exists(executable)) + { + return; + } + + var path = Path.GetTempFileName(); + try + { + File.WriteAllBytes(path, code); + var start = new ProcessStartInfo(executable) { UseShellExecute = false, RedirectStandardError = true, RedirectStandardOutput = true, CreateNoWindow = true }; + start.ArgumentList.Add("--target-env"); + start.ArgumentList.Add("vulkan1.2"); + start.ArgumentList.Add(path); + using var process = Process.Start(start)!; + var error = process.StandardError.ReadToEnd() + process.StandardOutput.ReadToEnd(); + process.WaitForExit(); + Assert.True(process.ExitCode == 0, error); + } + finally + { + File.Delete(path); + } + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5VectorObservationAnalysisTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5VectorObservationAnalysisTests.cs new file mode 100644 index 0000000..e49dc6e --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5VectorObservationAnalysisTests.cs @@ -0,0 +1,98 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Ir; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.ShaderCompiler.Tests; + +public sealed class Gen5VectorObservationAnalysisTests +{ + private const uint Exec = 126; + + private readonly List _instructions = []; + private uint _pc; + + private uint Add(Gen5ShaderInstruction instruction) + { + var pc = _pc; + _instructions.Add(instruction with { Pc = pc }); + _pc += 8; + return pc; + } + + private Gen5VectorObservationPlan Analyze() + { + Add(EndProgram(0)); + var plan = Gen5VectorObservationAnalysis.Analyze(Program([.. _instructions]), wave32: false); + Assert.NotNull(plan); + return plan; + } + + private void Narrow() => Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + + private void SaveExec() => Add(Sop1(0, "SMovB64", 20, Gen5Operand.Scalar(Exec))); + + private void RestoreExec() => Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(20))); + + [Fact] + public void ATemporaryOfANarrowedRegion_IsNotObserved() + { + SaveExec(); + Narrow(); + var temporary = Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(2))); + var result = Add(Vop2(0, "VAddU32", 4, Operand(1), Gen5Operand.Vector(3))); + RestoreExec(); + Add(Vop2(0, "VAddU32", 5, Operand(1), Gen5Operand.Vector(4))); + + var plan = Analyze(); + Assert.True(plan.UnobservedWrites(temporary).Contains(3)); + Assert.False(plan.UnobservedWrites(result).Contains(4)); + } + + [Fact] + public void ACrossLaneReadObservesInactiveLanes() + { + Narrow(); + var write = Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(2))); + Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop1, "VMovB32", [0u, 0u], [Gen5Operand.Vector(3)], [Gen5Operand.Vector(4)], + new Gen5DppControl(0x111, false, false, 0, 0, 0xF, 0xF))); + + Assert.False(Analyze().UnobservedWrites(write).Contains(3)); + } + + [Fact] + public void AFullWriteUnderAFullExecEndsTheObservation() + { + SaveExec(); + Narrow(); + var write = Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(2))); + RestoreExec(); + Add(MoveVector(0, 3, 9)); + Add(Vop2(0, "VAddU32", 4, Operand(1), Gen5Operand.Vector(3))); + + Assert.True(Analyze().UnobservedWrites(write).Contains(3)); + } + + [Fact] + public void AnAccumulatingWriteReadsItsDestination() + { + SaveExec(); + Narrow(); + var write = Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(2))); + RestoreExec(); + Add(Vop2(0, "VMacF32", 3, Gen5Operand.Vector(1), Gen5Operand.Vector(2))); + + Assert.False(Analyze().UnobservedWrites(write).Contains(3)); + } + + [Fact] + public void ARelativeMoveDisablesThePlan() + { + Add(Vop1(0, "VMovrelsB32", 3, Gen5Operand.Vector(4))); + Add(EndProgram(0)); + + Assert.Null(Gen5VectorObservationAnalysis.Analyze(Program([.. _instructions]), wave32: false)); + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLaneLocalAnalysisTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLaneLocalAnalysisTests.cs new file mode 100644 index 0000000..959eb32 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLaneLocalAnalysisTests.cs @@ -0,0 +1,262 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Ir; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.ShaderCompiler.Tests; + +public sealed class Gen5WaveLaneLocalAnalysisTests +{ + private const uint Exec = 126; + private static readonly Gen5Operand AllLanes = Gen5Operand.Source(193); + + private readonly List _instructions = []; + private uint _pc; + + private uint Add(Gen5ShaderInstruction instruction) + { + var pc = _pc; + _instructions.Add(instruction with { Pc = pc }); + _pc += 8; + return pc; + } + + private void Point(uint branchPc, uint targetPc) + { + var index = _instructions.FindIndex(instruction => instruction.Pc == branchPc); + var offset = (short)(((int)targetPc - (int)branchPc - 4) / 4); + _instructions[index] = _instructions[index] with { Words = [unchecked((uint)(ushort)offset)] }; + } + + private void Lane() + { + Add(Vop2(0, "VLshlrevB32", 2, Operand(3), Gen5Operand.Vector(1))); + Add(Vop2(0, "VAddI32", 2, Gen5Operand.Vector(0), Gen5Operand.Vector(2))); + } + + private void CompareTo(string opcode, uint destination, Gen5Operand left, Gen5Operand right) => + Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop3, opcode, [0u, 0u], [left, right], [Gen5Operand.Scalar(destination)], + new Gen5Vop3Control(0, 0, 0, false, 0, destination))); + + private void SkipUnlessActive(Action body) + { + var skip = Add(Branch(0, "SCbranchExecz", 0)); + body(); + var join = Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Point(skip, join); + } + + private bool Analyze(out string reason, params uint[] sharedFlatPcs) + { + Add(Vop2(0, "VLshlrevB32", 5, Operand(2), Gen5Operand.Vector(2))); + Add(BufferAccess(0, "BufferStoreDword", 4, vectorData: 4, offsetEnabled: true, vectorAddress: 5)); + Add(EndProgram(0)); + return Gen5WaveLaneLocalAnalysis.IsLaneLocal(Program([.. _instructions]), sharedFlatPcs.ToHashSet(), out reason); + } + + private void AssertRefused(string expected) + { + Assert.False(Analyze(out var reason)); + Assert.Contains(expected, reason, StringComparison.Ordinal); + } + + [Fact] + public void LaneArithmetic_IsLaneLocal() + { + Lane(); + Add(Vop2(0, "VAddU32", 4, Operand(100), Gen5Operand.Vector(2))); + + Assert.True(Analyze(out _)); + } + + [Fact] + public void ACompareSelectingLanes_IsLaneLocal() + { + Lane(); + CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(40)); + Add(Vop3(0, "VCndmaskB32", 4, Operand(0), Operand(1000), Gen5Operand.Scalar(10))); + + Assert.True(Analyze(out _)); + } + + [Fact] + public void ReadingAnotherLane_IsRefused() + { + Lane(); + Add(ReadLane(0, 20, 2, 45)); + Add(MoveVectorFromScalar(0, 4, 20)); + + AssertRefused("V_READLANE"); + } + + [Fact] + public void CountingTheLanesOfAMask_IsRefused() + { + Lane(); + Add(Sop1(0, "SBcnt1I32B64", 14, Gen5Operand.Scalar(Exec))); + Add(MoveVectorFromScalar(0, 4, 14)); + + AssertRefused("a lane mask used as a scalar value"); + } + + [Fact] + public void BroadcastingAMaskToTheLanes_IsRefused() + { + Lane(); + CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(40)); + Add(MoveVectorFromScalar(0, 4, 10)); + + AssertRefused("a lane mask broadcast to the lanes"); + } + + [Fact] + public void SharedMemory_IsRefused() + { + Lane(); + Add(DataShare(0, "DsWriteB32", false, [Gen5Operand.Vector(2), Gen5Operand.Vector(2)], [])); + + AssertRefused("LDS"); + } + + [Fact] + public void ExecFromALiteralMask_IsRefused() + { + Lane(); + Add(Sop1(0, "SMovB64", Exec, Operand(0xFFFF))); + + AssertRefused("EXEC set from the wave's layout"); + } + + [Fact] + public void AnExecSkipAroundLaneWork_IsLaneLocal() + { + Lane(); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + SkipUnlessActive(() => Add(MoveVector(0, 4, 77))); + + Assert.True(Analyze(out _)); + } + + [Fact] + public void ScalarStateWrittenInAnExecSkip_IsRefused() + { + Lane(); + Add(MoveScalar(0, 20, 5)); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + SkipUnlessActive(() => Add(MoveScalar(0, 20, 77))); + Add(MoveVectorFromScalar(0, 4, 20)); + + AssertRefused("scalar state escapes a lane-mask branch"); + } + + [Fact] + public void AMaskKeptThroughItsComplementInAnExecSkip_IsLaneLocal() + { + Lane(); + Add(Sop1(0, "SMovB64", 2, Gen5Operand.Scalar(Exec))); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + var skip = Add(Branch(0, "SCbranchExecz", 0)); + Add(Sop1(0, "SMovB64", 8, Gen5Operand.Scalar(Exec))); + Add(Vopc(0, "VCmpNeU32", Operand(0), 2)); + Add(Sop2(0, "SOrn2B64", 106, Gen5Operand.Scalar(106), Gen5Operand.Scalar(8))); + Add(Sop2(0, "SAndB64", 2, Gen5Operand.Scalar(2), Gen5Operand.Scalar(106))); + var join = Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(2))); + Point(skip, join); + + Assert.True(Analyze(out var reason), reason); + } + + [Fact] + public void AMaskChangedInAnExecSkip_IsRefused() + { + Lane(); + Add(Sop1(0, "SMovB64", 2, Gen5Operand.Scalar(Exec))); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + var skip = Add(Branch(0, "SCbranchExecz", 0)); + Add(Sop1(0, "SMovB64", 8, Gen5Operand.Scalar(Exec))); + Add(Sop2(0, "SOrB64", 2, Gen5Operand.Scalar(2), Gen5Operand.Scalar(8))); + Add(Sop1(0, "SNotB64", 2, Gen5Operand.Scalar(2))); + var join = Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Point(skip, join); + Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(2))); + + AssertRefused("scalar state escapes a lane-mask branch"); + } + + [Fact] + public void AWideMultiplyOfAScalarBesideAMask_IsLaneLocal() + { + Lane(); + Add(Sop2(0, "SCselectB64", 10, Gen5Operand.Scalar(Exec), Operand(0))); + Add(Vop3(0, "VMadU64U32", 4, Gen5Operand.Scalar(9), Gen5Operand.Vector(2), Operand(0))); + + Assert.True(Analyze(out var reason), reason); + } + + [Fact] + public void ABarrierInAnExecSkip_IsRefused() + { + Lane(); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + SkipUnlessActive(() => Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Sopp, "SBarrier", [0u], [], [], null))); + + AssertRefused("a barrier"); + } + + [Fact] + public void ReadFirstLaneOfAUniformValue_IsLaneLocal() + { + Lane(); + Add(MoveScalar(0, 20, 9)); + Add(MoveVectorFromScalar(0, 3, 20)); + Add(ReadFirstLane(0, 21, 3)); + Add(MoveVectorFromScalar(0, 4, 21)); + + Assert.True(Analyze(out _)); + } + + [Fact] + public void ReadFirstLaneOfALaneValue_IsRefused() + { + Lane(); + Add(ReadFirstLane(0, 21, 2)); + Add(MoveVectorFromScalar(0, 4, 21)); + + AssertRefused("outside a waterfall loop"); + } + + [Fact] + public void AWaterfallLoop_IsLaneLocal() + { + Lane(); + Add(Vop2(0, "VAndB32", 3, Operand(3), Gen5Operand.Vector(2))); + Add(Sop1(0, "SMovB64", 30, Gen5Operand.Scalar(Exec))); + var loop = Add(ReadFirstLane(0, 20, 3)); + CompareTo("VCmpEqU32", 22, Gen5Operand.Vector(3), Gen5Operand.Scalar(20)); + Add(Sop1(0, "SAndSaveexecB64", 24, Gen5Operand.Scalar(22))); + Add(MoveVectorFromScalar(0, 4, 20)); + Add(Sop2(0, "SXorB64", Exec, Gen5Operand.Scalar(24), Gen5Operand.Scalar(Exec))); + var back = Add(Branch(0, "SCbranchExecnz", 0)); + Point(back, loop); + Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(30))); + + Assert.True(Analyze(out var reason), reason); + } + + [Theory] + [InlineData(0x111u, true)] + [InlineData(0x141u, true)] + [InlineData(0x142u, false)] + [InlineData(0x143u, false)] + public void Dpp_IsLaneLocalOnlyWithinARow(uint control, bool laneLocal) + { + Lane(); + Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop1, "VMovB32", [0u, 0u], [Gen5Operand.Vector(2)], [Gen5Operand.Vector(4)], + new Gen5DppControl(control, false, false, 0, 0, 0xF, 0xF))); + + Assert.Equal(laneLocal, Analyze(out var reason)); + Assert.True(laneLocal || reason.Contains("cross-row DPP", StringComparison.Ordinal)); + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLoweringTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLoweringTests.cs new file mode 100644 index 0000000..d1f598e --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLoweringTests.cs @@ -0,0 +1,250 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using System.Text; +using SharpEmu.ShaderCompiler.Vulkan; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.ShaderCompiler.Tests; + +// Checks the SPIR-V each wave lowering (NativeWave32, NativeWave64, EmulatedWave64) produces for the +// same guest programs: which subgroup instructions appear, how a ballot is split into a 64-bit mask, +// and whether the workgroup exchange / rendezvous machinery exists. +public sealed class Gen5WaveLoweringTests +{ + public enum Lowering { Wave32, NativeWave64, EmulatedWave64 } + + private sealed record Module( + IReadOnlyList<(SpirvOp Op, uint[] Operands)> Instructions, + IReadOnlyList Names) + { + public int Count(SpirvOp op) => Instructions.Count(instruction => instruction.Op == op); + + // CompositeExtract of a ballot result: its literal index is the last operand. + public IReadOnlyList BallotExtractIndices() + { + var ballots = Instructions.Where(i => i.Op == SpirvOp.GroupNonUniformBallot).Select(i => i.Operands[1]).ToHashSet(); + return Instructions + .Where(i => i.Op == SpirvOp.CompositeExtract && ballots.Contains(i.Operands[2])) + .Select(i => i.Operands[^1]) + .ToList(); + } + } + + private static Module Parse(byte[] spirv) + { + var instructions = new List<(SpirvOp, uint[])>(); + var names = new List(); + for (var offset = 20; offset < spirv.Length;) + { + var word = BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset)); + var length = (int)(word >> 16); + var op = (SpirvOp)(word & 0xFFFF); + var operands = new uint[length - 1]; + for (var index = 0; index < operands.Length; index++) + { + operands[index] = BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset + 4 + index * 4)); + } + + instructions.Add((op, operands)); + if (op == SpirvOp.Name) + { + var text = new List(); + for (var index = 1; index < operands.Length; index++) + { + text.AddRange(BitConverter.GetBytes(operands[index])); + } + + names.Add(Encoding.UTF8.GetString(text.ToArray()).TrimEnd('\0')); + } + + offset += length * 4; + } + + return new Module(instructions, names); + } + + private static Module Compile(Gen5ShaderProgram program, Lowering lowering, uint threads = 64) + { + var (plan, resources, layout) = Prepare(program); + var configured = new ShaderCompileRequest(plan, resources, layout) + { + WaveSize = lowering == Lowering.Wave32 ? 32u : 64u, + HostWave64Supported = lowering == Lowering.NativeWave64, + LocalSizeX = threads, + }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(configured, out var shader, out var error), error); + return Parse(shader.Spirv); + } + + // v_cmp_lt_u32 vcc, v0, v1 : a per-lane compare whose result becomes a wave mask. + private static Gen5ShaderProgram CompareProgram() => Program( + Vopc(0, "VCmpLtU32", Operand(5), 1), + EndProgram(4)); + + private static Gen5ShaderProgram ReadFirstLaneProgram() => Program( + ReadFirstLane(0, 4, 0), + EndProgram(4)); + + private static Gen5ShaderProgram ReadLaneProgram(uint lane) => Program( + ReadLane(0, 4, 0, lane), + EndProgram(4)); + + private static Gen5ShaderProgram DataShareProgram(bool barrierBetween) => + Program( + new[] + { + DataShare(0, "DsWriteB32", false, [Gen5Operand.Vector(0), Gen5Operand.Vector(1)], [], 0, 0), + } + .Concat(barrierBetween + ? new[] { new Gen5ShaderInstruction(4, Gen5ShaderEncoding.Sopp, "SBarrier", [0u], [], [], null) } + : []) + .Concat(new[] + { + DataShare(8, "DsReadB32", false, [Gen5Operand.Vector(0)], [2u], 0, 0), + EndProgram(16), + }) + .ToArray()); + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + [InlineData(Lowering.EmulatedWave64)] + public void EveryLoweringCompilesTheBasicWavePrograms(Lowering lowering) + { + foreach (var program in new[] { CompareProgram(), ReadFirstLaneProgram(), ReadLaneProgram(0), ReadLaneProgram(33), DataShareProgram(false), DataShareProgram(true) }) + { + Compile(program, lowering); + } + } + + [Fact] + public void Wave32_BallotUsesOnlyTheLowWord() + { + var module = Compile(CompareProgram(), Lowering.Wave32); + Assert.True(module.Count(SpirvOp.GroupNonUniformBallot) > 0); + Assert.All(module.BallotExtractIndices(), index => Assert.Equal(0u, index)); + } + + [Fact] + public void NativeWave64_BallotCombinesBothWords() + { + var module = Compile(CompareProgram(), Lowering.NativeWave64); + var indices = module.BallotExtractIndices(); + Assert.Contains(0u, indices); + Assert.Contains(1u, indices); + } + + [Fact] + public void OnlyTheEmulatedLoweringDeclaresTheExchange() + { + foreach (var lowering in new[] { Lowering.Wave32, Lowering.NativeWave64 }) + { + var module = Compile(ReadFirstLaneProgram(), lowering); + Assert.DoesNotContain("wave64Exchange", module.Names); + Assert.DoesNotContain("wave64ExchangeParity", module.Names); + } + + var emulated = Compile(ReadFirstLaneProgram(), Lowering.EmulatedWave64); + Assert.Contains("wave64ExchangeParity", emulated.Names); + Assert.Contains("wave64Exchange", emulated.Names); + } + + [Fact] + public void EmulatedWave64_NeedsNoExchangeWithoutSubgroupOperations() + { + var module = Compile(Program(Vop2(0, "VAddI32", 1, Operand(1), Gen5Operand.Vector(0)), EndProgram(4)), Lowering.EmulatedWave64); + Assert.DoesNotContain("wave64ExchangeParity", module.Names); + } + + [Theory] + [InlineData(Lowering.Wave32, 64u)] + [InlineData(Lowering.NativeWave64, 64u)] + [InlineData(Lowering.NativeWave64, 256u)] + public void NativeLoweringsAddNoBarriersAroundSharedMemory(Lowering lowering, uint threads) + { + var module = Compile(DataShareProgram(false), lowering, threads); + Assert.Equal(0, module.Count(SpirvOp.ControlBarrier)); + } + + [Theory] + [InlineData(false, 2)] + [InlineData(true, 2)] + public void EmulatedWave64_OrdersSharedMemoryPhasesWithinOneWave(bool explicitBarrier, int expectedBarriers) + { + var module = Compile(DataShareProgram(explicitBarrier), Lowering.EmulatedWave64, threads: 64); + Assert.Equal(expectedBarriers, module.Count(SpirvOp.ControlBarrier)); + } + + [Fact] + public void EmulatedWave64_MultiWaveGroupsUseNoWorkgroupBarrierForSharedMemoryPhases() + { + var module = Compile(DataShareProgram(false), Lowering.EmulatedWave64, threads: 128); + Assert.Equal(0, module.Count(SpirvOp.ControlBarrier)); + } + + [Fact] + public void EmulatedWave64_MultiWaveExchangeMeetsAtAnArrivalCounter() + { + var single = Compile(ReadFirstLaneProgram(), Lowering.EmulatedWave64, threads: 64); + var multi = Compile(ReadFirstLaneProgram(), Lowering.EmulatedWave64, threads: 256); + Assert.DoesNotContain("wave64RendezvousEpoch", single.Names); + Assert.Contains("wave64RendezvousEpoch", multi.Names); + Assert.True(multi.Count(SpirvOp.AtomicIAdd) > single.Count(SpirvOp.AtomicIAdd)); + } + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + public void NativeLoweringsNeverDeclareARendezvous(Lowering lowering) + { + var module = Compile(ReadFirstLaneProgram(), lowering, threads: 256); + Assert.DoesNotContain("wave64RendezvousEpoch", module.Names); + Assert.Equal(0, module.Count(SpirvOp.AtomicIAdd)); + } + + [Fact] + public void HostWave64SupportDoesNotChangeAWave32Shader() + { + var program = ReadFirstLaneProgram(); + var (plan, resources, layout) = Prepare(program); + byte[] Compile32(bool hostWave64) + { + var configured = new ShaderCompileRequest(plan, resources, layout) + { + WaveSize = 32, + HostWave64Supported = hostWave64, + LocalSizeX = 64, + }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(configured, out var shader, out var error), error); + return shader.Spirv.ToArray(); + } + + Assert.Equal(Compile32(false), Compile32(true)); + } + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + [InlineData(Lowering.EmulatedWave64)] + public void ReadLaneBroadcastsFromTheRequestedLane(Lowering lowering) + { + var module = Compile(ReadLaneProgram(33), lowering); + Assert.True( + module.Count(SpirvOp.GroupNonUniformBroadcast) > 0 || module.Count(SpirvOp.GroupNonUniformShuffle) > 0 || + module.Count(SpirvOp.Store) > 0); + } + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + [InlineData(Lowering.EmulatedWave64)] + public void LoweringsAreDeterministic(Lowering lowering) + { + var first = Compile(CompareProgram(), lowering); + var second = Compile(CompareProgram(), lowering); + Assert.Equal(first.Instructions.Count, second.Instructions.Count); + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/BindingLayoutTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/BindingLayoutTests.cs index 165bb08..300f151 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/BindingLayoutTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/BindingLayoutTests.cs @@ -158,7 +158,8 @@ public sealed class BindingLayoutTests Assert.Equal(49u, (uint)DescriptorBindingKind.FaultBuffer); Assert.Equal(50u, (uint)DescriptorBindingKind.FlattenedResourceTable); Assert.Equal(51u, (uint)DescriptorBindingKind.ShaderData); - Assert.Equal(52u, (uint)DescriptorBindingKind.Count); + Assert.Equal(52u, (uint)DescriptorBindingKind.MipStatistics); + Assert.Equal(53u, (uint)DescriptorBindingKind.Count); ImageDimension[] sampledDimensions = [ diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs index e1f7354..a7b71ae 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs @@ -153,6 +153,26 @@ public sealed class ResourceMaterializationCacheTests Assert.Equal(0, cache.Hits); } + private static ShaderResourcePlan PlanWithSpareUserData() => + ShaderResourcePlan.Extract(DirectImageTableTests.CreateWaveIndexedDescriptorProgram(), ShaderStage.Compute, Hash, 0, 4); + + [Fact] + public void AUserDataWordTheProgramNeverReadsDoesNotSplitTheEntry() + { + var plan = PlanWithSpareUserData(); + var heap = new Heap(); + var cache = new ResourceMaterializationCache(); + Assert.True(Run(cache, plan, heap, [0x1000, 0, 5, 6], out var first, out _)); + Assert.True(Run(cache, plan, heap, [0x1000, 0, 7, 8], out var second, out _)); + Assert.Equal((1, 1), (cache.Hits, cache.Misses)); + // The hit carries the second draw's user data and leaves the first draw's snapshot alone. + Assert.Equal(new uint[] { 0x1000, 0, 7, 8 }, second.UserData); + Assert.Equal(new uint[] { 0x1000, 0, 5, 6 }, first.UserData); + Assert.Same(first.Images, second.Images); + Assert.True(Run(cache, plan, heap, [0x1000, 0, 5, 6], out var third, out _)); + Assert.Same(first, third); + } + [Fact] public void ADifferentDrawKeyIsADifferentEntry() { @@ -264,6 +284,276 @@ public sealed class ResourceMaterializationCacheTests Assert.Equal(0, cache.TableRefreshes); } + // A V# taken verbatim from s[4:7], the way titles pass per-draw constants. + private static ShaderResourcePlan DirectBufferPlan() => + Extract(Program(BufferLoad(0, 4), EndProgram(8)), userDataCount: 8); + + // The same V#, whose base pair s[4:5] is also the address of a scalar load. + private static ShaderResourcePlan SteeringBufferPlan() => + Extract(Program(ScalarLoad(0, 4, 16), Vop1(8, "VMovB32", 0, Gen5Operand.Scalar(16)), BufferStore(12, 4), EndProgram(20)), + userDataCount: 8); + + private static bool RunWords(ResourceMaterializationCache cache, ShaderResourcePlan plan, TestWordMemory memory, uint[] userData, + out ResourceSnapshot snapshot) + { + snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + return cache.Materialize(plan, Inputs(userData, memory.Read, memory.Read), (address, destination, _) => + { + for (var offset = 0; offset < destination.Length; offset += 4) + { + if (!memory.Read(address + (ulong)offset, out var word)) + return false; + BitConverter.TryWriteBytes(destination[offset..], word); + } + + return true; + }, ref snapshot, ref specialization, out _); + } + + private static uint[] Reference(ShaderResourcePlan plan, TestWordMemory memory, uint[] userData) + { + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref snapshot, ref specialization)); + return snapshot.Buffers[0]; + } + + private static uint[] LinearConstants(int draw) => [0, 0, 0, 0, 0x10000u + (uint)draw * 0x100u, 0, 256, 0x2_0000]; + + [Fact] + public void ALinearlyAllocatedBufferBaseStopsMissingOnceConfirmed() + { + var plan = DirectBufferPlan(); + var memory = new TestWordMemory(); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = LinearConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + Assert.Equal(Reference(plan, memory, userData), snapshot.Buffers[0]); + Assert.Equal(userData, snapshot.UserData); + } + + // One miss to build, three to confirm the word; every later draw is a hit. + Assert.Equal((12, 4), (cache.Hits, cache.Misses)); + } + + [Fact] + public void InterleavedObjectsConfirmTheirOwnBases() + { + // Constants V# in s[4:7], a second buffer in s[8:11] whose size tells the two objects apart. + var plan = Extract(Program(BufferLoad(0, 4), BufferLoad(8, 8), EndProgram(16)), userDataCount: 12); + var memory = new TestWordMemory(); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 32; draw++) + { + uint[] userData = [0, 0, 0, 0, 0x10000u + (uint)draw * 0x100u, 0, 256, 0x2_0000, 0x40000, 0, draw % 2 == 0 ? 256u : 512u, 0x2_0000]; + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + } + + Assert.True(cache.Hits >= 20, $"hits={cache.Hits} misses={cache.Misses}"); + } + + // Pointer s[0:1] moves on every draw; the program loads four constants through it. + private static uint[] PointerConstants(int draw) => [0x10000u + (uint)draw * 0x100u, 0, 0, 0, 0x30000, 0, 256, 0x2_0000, 0]; + + [Fact] + public void AMovingScalarLoadPointerStopsMissingOnceConfirmed() + { + var (plan, _, _) = Prepare(FlattenedReadReuseTests.RepeatedReadProgram(4), userDataCount: 9); + // The same four words repeat every 0x100 bytes, so every pointer finds the same constants. + var memory = new TestWordMemory { Base = 0x10000, Words = Enumerable.Range(0, 0x1000).Select(index => (uint)(index % 64) * 3 + 1).ToArray() }; + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = PointerConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + } + + Assert.Equal((12, 4), (cache.Hits, cache.Misses)); + + // Other constants behind the next pointer: the draw reads them instead of reusing the table. + var moved = PointerConstants(30); + memory.At(moved[0] + 8) = 0xBEEF; + Assert.True(RunWords(cache, plan, memory, moved, out var reread)); + Assert.Equal(0xBEEFu, reread.FlattenedResourceTable[2]); + Assert.Equal(12, cache.Hits); + } + + [Fact] + public void ARebasedHitLeavesTheCachedSnapshotAlone() + { + var plan = DirectBufferPlan(); + var memory = new TestWordMemory(); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 4; draw++) + Assert.True(RunWords(cache, plan, memory, LinearConstants(draw), out _)); + + Assert.True(RunWords(cache, plan, memory, LinearConstants(3), out var built)); + Assert.True(RunWords(cache, plan, memory, LinearConstants(9), out var patched)); + Assert.Equal(0x10900u, patched.Buffers[0][0]); + Assert.Equal(0x10300u, built.Buffers[0][0]); + Assert.NotSame(built.Buffers, patched.Buffers); + Assert.True(RunWords(cache, plan, memory, LinearConstants(3), out var again)); + Assert.Equal(0x10300u, again.Buffers[0][0]); + } + + [Fact] + public void AReadThatMovesWithTheBaseAndKeepsItsBytesIsReused() + { + var plan = SteeringBufferPlan(); + var memory = new TestWordMemory { Base = 0x10000, Words = Enumerable.Repeat(0x1234u, 0x1000).ToArray() }; + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = LinearConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers[0], snapshot.Buffers[0]); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + Assert.Equal((12, 4), (cache.Hits, cache.Misses)); + + // The moved read now finds other bytes at the new base: the draw must not reuse the table. + var changed = LinearConstants(20); + memory.At(changed[4]) = 0x5678; + Assert.True(RunWords(cache, plan, memory, changed, out var refreshed)); + Assert.Equal(0x5678u, refreshed.FlattenedResourceTable[0]); + Assert.Equal(12, cache.Hits); + } + + [Fact] + public void MovedConstantsThatChangeEveryDrawAreReadAgain() + { + var plan = SteeringBufferPlan(); + var memory = new TestWordMemory { Base = 0x10000, Words = Enumerable.Range(0, 0x1000).Select(index => (uint)index * 7).ToArray() }; + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = LinearConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + Assert.Equal(Reference(plan, memory, userData), snapshot.Buffers[0]); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + // The constants differ at every base, so nothing is reused as is; once the base is + // confirmed, the moved table is evaluated again instead of the whole walk. + Assert.Equal(0, cache.Hits); + Assert.True(cache.TableRefreshes >= 10, $"refreshes={cache.TableRefreshes} misses={cache.Misses}"); + } + + // The V# comes from a descriptor table at s[0:1]; its constants are read through it. + private static ShaderResourcePlan MemoryDescriptorPlan() => + Extract(Program(ScalarLoad(0, 0, 8, 4), ScalarBufferLoad(4, 8, 12, 4), Vop1(8, "VMovB32", 0, Gen5Operand.Scalar(12)), + BufferLoad(12, 8), EndProgram(16)), userDataCount: 2); + + // The title rewrites the table every draw: a freshly allocated base, and new constants there. + private static void WriteDraw(TestWordMemory memory, int draw) + { + var bufferBase = 0x10000u + (uint)draw * 0x100u; + memory.At(0x1000) = bufferBase; + memory.At(0x1004) = 0; + memory.At(0x1008) = 256; + memory.At(0x100C) = 0x2_0000; + for (var word = 0u; word < 4; word++) + memory.At(bufferBase + word * 4) = (uint)draw * 10 + word; + } + + [Fact] + public void ABaseRewrittenInGuestMemoryStopsMissingOnceConfirmed() + { + var plan = MemoryDescriptorPlan(); + var memory = new TestWordMemory { Base = 0x1000, Words = new uint[0x8000] }; + var cache = new ResourceMaterializationCache(); + uint[] userData = [0x1000, 0]; + for (var draw = 0; draw < 16; draw++) + { + WriteDraw(memory, draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + // One miss to build, three to confirm the base; the later draws patch it. + Assert.True(cache.Hits >= 10, $"hits={cache.Hits} misses={cache.Misses}"); + + // Another descriptor word changed with the base: the draw is materialized again. + WriteDraw(memory, 20); + memory.At(0x1008) = 512; + var hits = cache.Hits; + Assert.True(RunWords(cache, plan, memory, userData, out var resized)); + Assert.Equal(hits, cache.Hits); + Assert.Equal(512u, resized.Buffers[0][2]); + } + + private static void AssertMatchesFullWalk(ShaderResourcePlan plan, TestWordMemory memory, uint[] userData, ResourceSnapshot snapshot) + { + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + [Fact] + public void ATableConstantBesideARewrittenBaseIsCopiedForTheDraw() + { + // The V# and a constant both come from the table at s[0:1]; the constant changes every draw. + var plan = Extract(Program(ScalarLoad(0, 0, 8, 4), ScalarLoad(4, 0, 16, 1, immediateOffset: 16), + ScalarBufferLoad(8, 8, 12, 4), Vop1(12, "VMovB32", 0, Gen5Operand.Scalar(12)), + Vop1(16, "VMovB32", 1, Gen5Operand.Scalar(16)), BufferLoad(20, 8), EndProgram(24)), userDataCount: 2); + var memory = new TestWordMemory { Base = 0x1000, Words = new uint[0x8000] }; + var cache = new ResourceMaterializationCache(); + uint[] userData = [0x1000, 0]; + for (var draw = 0; draw < 16; draw++) + { + WriteDraw(memory, draw); + memory.At(0x1010) = (uint)draw * 3 + 1; + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + AssertMatchesFullWalk(plan, memory, userData, snapshot); + } + + Assert.True(cache.Hits >= 10, $"hits={cache.Hits} misses={cache.Misses}"); + } + + [Fact] + public void AlternatingObjectsWithRewrittenBasesRebaseTheirOwnVariant() + { + // Two objects share the draw key and differ in their buffer size; both move their base. + var plan = MemoryDescriptorPlan(); + var memory = new TestWordMemory { Base = 0x1000, Words = new uint[0x8000] }; + var cache = new ResourceMaterializationCache(); + uint[] userData = [0x1000, 0]; + for (var draw = 0; draw < 32; draw++) + { + WriteDraw(memory, draw); + memory.At(0x1008) = draw % 2 == 0 ? 256u : 512u; + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + AssertMatchesFullWalk(plan, memory, userData, snapshot); + } + + Assert.True(cache.Hits >= 20, $"hits={cache.Hits} misses={cache.Misses}"); + } + [Fact] public void AFullGenerationKeepsRecentEntries() { diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/SpirvBindingDeclarationTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/SpirvBindingDeclarationTests.cs index d493b24..469a517 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/SpirvBindingDeclarationTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/SpirvBindingDeclarationTests.cs @@ -408,6 +408,25 @@ public sealed class SpirvBindingDeclarationTests Assert.Contains((uint)SpirvCapability.GroupNonUniformShuffle, module.Capabilities); } + [Fact] + public void PixelAppendStartsWithoutHelperLanes() + { + var appending = Program( + MoveScalar(0, 124, 64), + DataShare(8, "DsAppend", gds: true, [Gen5Operand.Scalar(124)], [1]), + EndProgram(16)); + var plain = Program( + MoveScalar(0, 124, 64), + EndProgram(8)); + + Assert.True(Gen5SpirvTranslator.TryCompileProgram(Request(appending, ShaderStage.Pixel), out var appendShader, out var error), error); + Assert.True(Gen5SpirvTranslator.TryCompileProgram(Request(plain, ShaderStage.Pixel), out var plainShader, out error), error); + + Gen5MipStatisticsTests.ValidateWhenAvailable(appendShader.Spirv); + Assert.Contains("helperInvocation", new SpirvModuleInspector(appendShader.Spirv).Names.Values); + Assert.DoesNotContain("helperInvocation", new SpirvModuleInspector(plainShader.Spirv).Names.Values); + } + [Fact] public void DataShareBpermute_DeclaresTheShuffleCapability() {