diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs index 331b974..442a67b 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs @@ -5604,6 +5604,11 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return false; } + // Map does not reject a range that is already mapped: it reuses and zero-fills it. Two threads + // creating guest threads at once could both find the same slot free and share one stack, so + // choosing and mapping a slot happen under one lock. + private static readonly object GuestThreadRegionGate = new(); + private static bool TryMapGuestThreadRegion( IVirtualMemory virtualMemory, ulong baseAddress, @@ -5611,6 +5616,20 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ProgramHeaderFlags protection, out ulong mappedBase, out string? error) + { + lock (GuestThreadRegionGate) + { + return TryMapGuestThreadRegionLocked(virtualMemory, baseAddress, size, protection, out mappedBase, out error); + } + } + + private static bool TryMapGuestThreadRegionLocked( + IVirtualMemory virtualMemory, + ulong baseAddress, + ulong size, + ProgramHeaderFlags protection, + out ulong mappedBase, + out string? error) { for (int i = 0; i < GuestThreadRegionSlots; i++) { @@ -5645,6 +5664,17 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I IVirtualMemory virtualMemory, out ulong tlsBase, out string? error) + { + lock (GuestThreadRegionGate) + { + return TryMapGuestThreadTlsRegionLocked(virtualMemory, out tlsBase, out error); + } + } + + private static bool TryMapGuestThreadTlsRegionLocked( + IVirtualMemory virtualMemory, + out ulong tlsBase, + out string? error) { for (int i = 0; i < GuestThreadRegionSlots; i++) { @@ -5727,18 +5757,21 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return true; } + private static readonly bool LegacyGuestPriorities = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_GUEST_PRIORITY"), "legacy", StringComparison.OrdinalIgnoreCase); + private static ThreadPriority MapGuestThreadPriority(int priority) { if (priority <= 478) { - return ThreadPriority.Highest; + return LegacyGuestPriorities ? ThreadPriority.Highest : ThreadPriority.Normal; } if (priority >= 733) { return ThreadPriority.Lowest; } - return ThreadPriority.Normal; + return LegacyGuestPriorities ? ThreadPriority.Normal : ThreadPriority.BelowNormal; } private void ApplyGuestThreadAffinity(ulong guestAffinityMask) diff --git a/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs b/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs index fe3b026..ea343fd 100644 --- a/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs +++ b/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs @@ -2070,6 +2070,13 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA GuestImageWriteTracker.NotifyManagedWrite(virtualAddress, (ulong)source.Length); GuestGpuMemoryHook.MarkCpuWrite(virtualAddress, (ulong)source.Length); + if (!source.IsEmpty && _backedSpace is { } backed && !_disposed && + backed.TryWriteSingleView(virtualAddress, source)) + { + NotifyGuestWriteWatch(virtualAddress, source); + return true; + } + var requiresExclusiveAccess = false; _gate.EnterReadLock(); try diff --git a/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs b/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs index a99f4b5..c46e01c 100644 --- a/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs +++ b/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs @@ -30,6 +30,7 @@ public sealed class GuestGpuMemory : IDisposable private readonly PageGuard _pages; private readonly ReaderWriterLockSlim _spansLock = new(); private readonly SpanSet _spans = new(); + private SpanSnapshot _spanSnapshot = SpanSnapshot.Empty; private long _spanVersion; public long SpanVersion => Interlocked.Read(ref _spanVersion); @@ -153,15 +154,7 @@ public sealed class GuestGpuMemory : IDisposable return false; } - _spansLock.EnterReadLock(); - try - { - return _spans.Contains(address, size); - } - finally - { - _spansLock.ExitReadLock(); - } + return Volatile.Read(ref _spanSnapshot).Contains(address, size); } public void ForEachSpan(Action visit) @@ -207,6 +200,7 @@ public sealed class GuestGpuMemory : IDisposable try { _spans.Add(address, size); + Volatile.Write(ref _spanSnapshot, _spans.Snapshot()); Interlocked.Increment(ref _spanVersion); } finally @@ -280,6 +274,7 @@ public sealed class GuestGpuMemory : IDisposable try { _spans.Remove(address, size); + Volatile.Write(ref _spanSnapshot, _spans.Snapshot()); Interlocked.Increment(ref _spanVersion); } finally diff --git a/src/SharpEmu.HLE/GpuMemory/GuestPageTracker.cs b/src/SharpEmu.HLE/GpuMemory/GuestPageTracker.cs index 0ac893a..2032867 100644 --- a/src/SharpEmu.HLE/GpuMemory/GuestPageTracker.cs +++ b/src/SharpEmu.HLE/GpuMemory/GuestPageTracker.cs @@ -198,6 +198,14 @@ public sealed class GuestPageTracker if (Volatile.Read(ref _regions[index]) is { } region) { tracked = true; + if (region.IsCpuDirtyThroughout(offset, bytes)) + { + remaining -= bytes; + offset = 0; + index++; + continue; + } + using (region.Lock.Hold()) { if (region.IsModified(WriteOrigin.Gpu, offset, bytes)) diff --git a/src/SharpEmu.HLE/GpuMemory/SpanSet.cs b/src/SharpEmu.HLE/GpuMemory/SpanSet.cs index fcf861f..291417f 100644 --- a/src/SharpEmu.HLE/GpuMemory/SpanSet.cs +++ b/src/SharpEmu.HLE/GpuMemory/SpanSet.cs @@ -3,6 +3,22 @@ namespace SharpEmu.HLE.GpuMemory; +public sealed class SpanSnapshot(ulong[] starts, ulong[] ends) +{ + public static readonly SpanSnapshot Empty = new([], []); + + public bool Contains(ulong address, ulong size) + { + var index = Array.BinarySearch(starts, address); + if (index < 0) + { + index = ~index - 1; + } + + return index >= 0 && ends[index] >= address + size; + } +} + public sealed class SpanSet { private readonly SortedList _spans = new(); @@ -79,6 +95,19 @@ public sealed class SpanSet return index > 0 && _spans.GetValueAtIndex(index - 1) > address; } + public SpanSnapshot Snapshot() + { + var starts = new ulong[_spans.Count]; + var ends = new ulong[_spans.Count]; + for (var index = 0; index < starts.Length; index++) + { + starts[index] = _spans.GetKeyAtIndex(index); + ends[index] = _spans.GetValueAtIndex(index); + } + + return new SpanSnapshot(starts, ends); + } + public void ForEach(Action visit) { foreach (var (start, end) in _spans) diff --git a/src/SharpEmu.HLE/GpuMemory/TrackedRegion.cs b/src/SharpEmu.HLE/GpuMemory/TrackedRegion.cs index 7dfe95c..9519cf3 100644 --- a/src/SharpEmu.HLE/GpuMemory/TrackedRegion.cs +++ b/src/SharpEmu.HLE/GpuMemory/TrackedRegion.cs @@ -46,6 +46,20 @@ public sealed class TrackedRegion return GetDirtyMask(side).AnyInRange(start, end); } + public bool IsCpuDirtyThroughout(ulong offset, ulong size) + { + var (start, end) = GetPageRange(BaseAddress + offset, size); + for (var page = start; page < end; page++) + { + if (!_cpuDirty.Get(page)) + { + return false; + } + } + + return true; + } + public bool IsCpuWriteHot(ulong offset, ulong size) { var (start, end) = GetPageRange(BaseAddress + offset, size); diff --git a/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs b/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs index 2b44b28..1387921 100644 --- a/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs +++ b/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs @@ -67,6 +67,8 @@ public sealed class GuestSpaceOwner : IDisposable // Reads through a view that covers the whole range without any lock; false otherwise. public bool TryReadSingleView(ulong address, Span data) => _views.TryReadSingleView(address, data); + public bool TryWriteSingleView(ulong address, ReadOnlySpan data) => _views.TryWriteSingleView(address, data); + public object AliasSnapshot => _views.AliasSnapshot; public bool TryEnterAliasAccess() => _views.TryEnterAliasAccess(); diff --git a/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs b/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs index 43be043..1698426 100644 --- a/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs +++ b/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs @@ -118,6 +118,24 @@ public sealed unsafe class SharedBackingViews : IDisposable } } + public bool TryWriteSingleView(ulong address, ReadOnlySpan data) + { + if (!TryAccessSingleView(address, (ulong)data.Length, out var target)) + { + return false; + } + + try + { + data.CopyTo(new Span((void*)target, data.Length)); + return true; + } + finally + { + Interlocked.Decrement(ref _activeAccesses); + } + } + public bool TryReadBacking(ulong address, Span data) { if (TryAccessSingleView(address, (ulong)data.Length, out var source)) diff --git a/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs index 75f378a..906ed65 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs @@ -136,12 +136,7 @@ public static partial class AgcExports var groupCountZ = (uint)ctx[CpuRegister.Rcx]; var modifier = (uint)ctx[CpuRegister.R8]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItDispatchDirect, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, groupCountX) || - !TryWriteUInt32(ctx, commandAddress + 8, groupCountY) || - !TryWriteUInt32(ctx, commandAddress + 12, groupCountZ) || - !TryWriteUInt32(ctx, commandAddress + 16, DirectDispatchInitiator(modifier))) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(5, ItDispatchDirect, 0), groupCountX, groupCountY, groupCountZ, DirectDispatchInitiator(modifier))) { return ReturnPointer(ctx, 0); } @@ -166,9 +161,7 @@ public static partial class AgcExports { var commandBufferAddress = ctx[CpuRegister.Rdi]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RAcbReset)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNop, RAcbReset), 0)) { return ReturnPointer(ctx, 0); } @@ -198,15 +191,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RAcquireMem)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0x8000_0000u) || - !TryWriteUInt32(ctx, commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || - !TryWriteUInt32(ctx, commandAddress + 12, 0) || - !TryWriteUInt32(ctx, commandAddress + 16, (uint)(baseAddress >> 8)) || - !TryWriteUInt32(ctx, commandAddress + 20, 0) || - !TryWriteUInt32(ctx, commandAddress + 24, pollCycles / 40) || - !TryWriteUInt32(ctx, commandAddress + 28, gcrControl)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(8, ItNop, RAcquireMem), 0x8000_0000u, noSize ? 0 : (uint)(sizeBytes >> 8), 0, (uint)(baseAddress >> 8), 0, pollCycles / 40, gcrControl)) { return ReturnPointer(ctx, 0); } @@ -301,11 +286,7 @@ public static partial class AgcExports var argumentsAddress = ctx[CpuRegister.Rsi]; var modifier = (uint)ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItDispatchIndirect, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, (uint)argumentsAddress) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)(argumentsAddress >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 12, (modifier & 0xA038u) | 0x41u)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(4, ItDispatchIndirect, 0), (uint)argumentsAddress, (uint)(argumentsAddress >> 32), (modifier & 0xA038u) | 0x41u)) { return ReturnPointer(ctx, 0); } @@ -360,18 +341,7 @@ public static partial class AgcExports var packetAddressValue = interrupt == 4 ? 0UL : destinationAddress; var packetData = interrupt == 4 ? 0UL : data; - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RReleaseMem)) || - !TryWriteUInt32(ctx, commandAddress + 4, action | (cachePolicy << 8)) || - !TryWriteUInt32( - ctx, - commandAddress + 8, - gcrControl | (dataSelection << 16) | (interrupt << 24)) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)packetAddressValue) || - !TryWriteUInt32(ctx, commandAddress + 16, (uint)(packetAddressValue >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 20, (uint)packetData) || - !TryWriteUInt32(ctx, commandAddress + 24, (uint)(packetData >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 28, interruptContextId & 0x07FF_FFFFu)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(8, ItNop, RReleaseMem), action | (cachePolicy << 8), gcrControl | (dataSelection << 16) | (interrupt << 24), (uint)packetAddressValue, (uint)(packetAddressValue >> 32), (uint)packetData, (uint)(packetData >> 32), interruptContextId & 0x07FF_FFFFu)) { return ReturnPointer(ctx, 0); } @@ -397,9 +367,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RDrawReset)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNop, RDrawReset), 0)) { return ReturnPointer(ctx, 0); } @@ -448,12 +416,7 @@ public static partial class AgcExports (resetCounters << 18) | (counterMask << 10) | (counterSelect << 2); - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItGetLodStats, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, control) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)destinationAddress & ~0x3Fu) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)(destinationAddress >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 16, packetControl)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(5, ItGetLodStats, 0), control, (uint)destinationAddress & ~0x3Fu, (uint)(destinationAddress >> 32), packetControl)) { return ReturnPointer(ctx, 0); } @@ -493,15 +456,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RAcquireMem)) || - !TryWriteUInt32(ctx, commandAddress + 4, (engine << 31) | cbDbOp) || - !TryWriteUInt32(ctx, commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || - !TryWriteUInt32(ctx, commandAddress + 12, 0) || - !TryWriteUInt32(ctx, commandAddress + 16, (uint)(baseAddress >> 8)) || - !TryWriteUInt32(ctx, commandAddress + 20, 0) || - !TryWriteUInt32(ctx, commandAddress + 24, pollCycles / 40) || - !TryWriteUInt32(ctx, commandAddress + 28, gcrControl)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(8, ItNop, RAcquireMem), (engine << 31) | cbDbOp, noSize ? 0 : (uint)(sizeBytes >> 8), 0, (uint)(baseAddress >> 8), 0, pollCycles / 40, gcrControl)) { return ReturnPointer(ctx, 0); } @@ -631,16 +586,7 @@ public static partial class AgcExports var control = 0x10u | compareFunction | (writeSpace == 1 ? 0x100u : 0u); - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 9, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(9, ItCondWrite, RZero)) || - !TryWriteUInt32(ctx, commandAddress + 4, control) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)readAddress & ~0x3u) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)(readAddress >> 32) & 0xFFFFu) || - !TryWriteUInt32(ctx, commandAddress + 16, reference) || - !TryWriteUInt32(ctx, commandAddress + 20, mask) || - !TryWriteUInt32(ctx, commandAddress + 24, (uint)writeAddress & ~0x3u) || - !TryWriteUInt32(ctx, commandAddress + 28, (uint)(writeAddress >> 32) & 0xFFFFu) || - !TryWriteUInt32(ctx, commandAddress + 32, writeValue)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(9, ItCondWrite, RZero), control, (uint)readAddress & ~0x3u, (uint)(readAddress >> 32) & 0xFFFFu, reference, mask, (uint)writeAddress & ~0x3u, (uint)(writeAddress >> 32) & 0xFFFFu, writeValue)) { return ReturnPointer(ctx, 0); } @@ -752,9 +698,7 @@ public static partial class AgcExports var address = ctx[CpuRegister.Rdx]; var reference = ctx[CpuRegister.Rcx]; if (commandBufferAddress == 0 || size > 1 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RZero)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNop, RZero), 0)) { return ReturnPointer(ctx, 0); } @@ -790,11 +734,7 @@ public static partial class AgcExports var baseIndex = (uint)ctx[CpuRegister.Rsi]; var address = ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItSetBase, 0) | (baseIndex << 1)) || - !TryWriteUInt32(ctx, commandAddress + 4, 1) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)address & ~7u) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)(address >> 32))) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(4, ItSetBase, 0) | (baseIndex << 1), 1, (uint)address & ~7u, (uint)(address >> 32))) { return ReturnPointer(ctx, 0); } @@ -813,10 +753,7 @@ public static partial class AgcExports var dataOffset = (uint)ctx[CpuRegister.Rsi]; var modifier = (uint)ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 3, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(3, ItDispatchIndirect, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, dataOffset) || - !TryWriteUInt32(ctx, commandAddress + 8, (modifier & 0xA038u) | 0x41u)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(3, ItDispatchIndirect, 0), dataOffset, (modifier & 0xA038u) | 0x41u)) { return ReturnPointer(ctx, 0); } @@ -884,9 +821,7 @@ public static partial class AgcExports { var commandBufferAddress = ctx[CpuRegister.Rdi]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RPopMarker)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNop, RPopMarker), 0)) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs index 73abc74..ec78bd1 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs @@ -23,6 +23,21 @@ public static partial class AgcExports internal sealed record RetainedTargetlessDraw(RegisterBanks Banks, TargetlessDrawArguments Arguments); + // A draw or dispatch the translation deferred, with the banks it was issued under. + internal abstract record DeferredWork; + + // Takes the deferred work of a command stream interpreted on its own thread. + internal interface IDeferredWorkSink + { + // True on the thread that interprets the stream; elsewhere the translation runs work itself. + bool IsProducerThread { get; } + + // The non-draw items handed over so far; a draw's resolution waits until they ran. + long Gate { get; } + + void EnqueueDeferred(DeferredWork work, bool isDraw); + } + // The snapshots the submit-time prepass captured for one submission. private sealed record SubmittedGeometrySnapshots( Dictionary? IndexSnapshots, @@ -160,9 +175,18 @@ public static partial class AgcExports { _pipelines = new ShaderPipelineCache(_context, pipelineHost, GuestGpu.Current, CreateShaderHeaderRegistry(_context)); _executor = new RenderExecutor(renderHost, _pipelines); + if (host is IConcurrentResolveHost concurrentHost) + { + _parallel = new ParallelProgramPrefetch(_executor, renderHost, concurrentHost); + _executor.Prefetch = _parallel; + } } } + // Resolves the programs of deferred draws ahead of them, for a host that answers reads from + // another thread; draws are deferred whenever it exists. + private readonly ParallelProgramPrefetch? _parallel; + // The context bank of the queue whose slice runs now; the Metal host rebuilds its records from it. internal ContextRegisters? CurrentContextRegisters => _current?.TypedRegisters?.Context; @@ -196,17 +220,194 @@ public static partial class AgcExports } } + private sealed record PendingDraw( + ulong SubmitId, + SubmittedDcbState State, + RegisterBanks Banks, + bool Indexed, + DrawIndexedArguments IndexedArguments, + DrawAutoArguments AutoArguments, + SubmittedVertexSnapshot? VertexSnapshot, + SubmittedIndexSnapshot? IndexSnapshot, + ParallelProgramPrefetch.Job? Job) : DeferredWork; + + private sealed record PendingDispatch( + ulong SubmitId, + SubmittedDcbState State, + RegisterBanks Banks, + uint GroupsX, + uint GroupsY, + uint GroupsZ, + uint DispatchInitiator, + ulong IndirectArgumentsAddress) : DeferredWork; + + // With the command stream on its own thread, its draws and dispatches reach the render + // thread through the sink, which runs them with RunDeferred in stream order. + internal IDeferredWorkSink? Sink { get; set; } + + private bool Producing => Sink is { IsProducerThread: true }; + + // The queue state of the work the render thread runs now; _current belongs to the thread + // interpreting the stream. + private SubmittedDcbState? _executing; + + public void RunDeferred(DeferredWork work) + { + switch (work) + { + case PendingDraw draw: + { + var executing = _executing; + _executing = draw.State; + RecordKnownColorTargets(draw.State, draw.Banks); + if (_parallel is not null) + { + _parallel.Current = draw.Job; + } + + try + { + RunExecutorDraw(draw.SubmitId, draw.State, draw.Banks, draw.Indexed, draw.IndexedArguments, draw.AutoArguments, + clearSnapshots: false); + } + finally + { + _executing = executing; + if (_parallel is not null) + { + _parallel.Current = null; + } + } + + break; + } + case PendingDispatch dispatch: + { + var executing = _executing; + _executing = dispatch.State; + try + { + ExecuteDispatch(_executor!, dispatch.SubmitId, dispatch.Banks, dispatch.GroupsX, dispatch.GroupsY, dispatch.GroupsZ, + dispatch.DispatchInitiator, dispatch.IndirectArgumentsAddress); + } + finally + { + _executing = executing; + } + + break; + } + default: + throw _host.Fatal($"The deferred work is unknown: {work.GetType().Name}."); + } + } + + // The resolution worker, so the render thread can order it with the stream's other work. + internal ParallelProgramPrefetch? Prefetch => _parallel; + + private readonly Queue _pendingDraws = new(); + + public bool HasPendingDraws => _pendingDraws.Count != 0; + + private bool _draining; + + // Runs every deferred draw in issue order with the banks and snapshots it was issued with. + // A draw calls back into the host, whose entry points drain too: those calls run inside + // the draw and must not start the next one. + public void DrainPendingDraws() + { + if (_draining) + { + return; + } + + _draining = true; + try + { + DrainPendingDrawsCore(); + } + finally + { + _draining = false; + } + } + + private void DrainPendingDrawsCore() + { + while (_pendingDraws.TryDequeue(out var pending)) + { + var current = _current; + var state = pending.State; + _current = state; + state.CurrentVertexSnapshot = pending.VertexSnapshot; + state.CurrentIndexSnapshot = pending.IndexSnapshot; + if (_parallel is not null) + { + _parallel.Current = pending.Job; + } + + try + { + RunExecutorDraw(pending.SubmitId, state, pending.Banks, pending.Indexed, pending.IndexedArguments, pending.AutoArguments); + } + finally + { + _current = current; + if (_parallel is not null) + { + _parallel.Current = null; + } + } + } + + // What follows a drain may change guest memory and cache state the worker reads. + _parallel?.WaitIdle(); + } + private bool TryRunExecutorDraw(ulong submitId, SubmittedDcbState state, bool indexed, in DrawIndexedArguments indexedArguments, in DrawAutoArguments autoArguments) { - if (_executor is not { } executor) + if (_executor is null) { return false; } var banks = RequireTypedRegisters(state); - RecordKnownColorTargets(state, banks); + var producing = Producing; + if (!producing) + { + // A draw run by the render thread records its targets there. + RecordKnownColorTargets(state, banks); + } + state.FrameDrawCount++; state.SawIndexedDraw |= indexed; + if (_parallel is not null || producing) + { + var snapshot = banks.Snapshot(); + var pending = new PendingDraw(submitId, state, snapshot, indexed, indexedArguments, autoArguments, + state.CurrentVertexSnapshot, state.CurrentIndexSnapshot, _parallel?.Submit(snapshot, producing ? Sink!.Gate : 0)); + if (producing) + { + Sink!.EnqueueDeferred(pending, isDraw: true); + } + else + { + _pendingDraws.Enqueue(pending); + } + + state.CurrentIndexSnapshot = null; + state.CurrentVertexSnapshot = null; + return true; + } + + RunExecutorDraw(submitId, state, banks, indexed, indexedArguments, autoArguments); + return true; + } + + private void RunExecutorDraw(ulong submitId, SubmittedDcbState state, RegisterBanks banks, bool indexed, in DrawIndexedArguments indexedArguments, in DrawAutoArguments autoArguments, + bool clearSnapshots = true) + { + var executor = _executor!; var drawStarted = DcbParseProfile.Begin(); try { @@ -222,17 +423,19 @@ public static partial class AgcExports finally { DcbParseProfile.RecordDraw(drawStarted); - state.CurrentIndexSnapshot = null; - state.CurrentVertexSnapshot = null; + // The snapshot fields belong to the interpreting thread when it is another one. + if (clearSnapshots) + { + state.CurrentIndexSnapshot = null; + state.CurrentVertexSnapshot = null; + } } - - return true; } // Keeps the banks the draw was issued under; the flip replays it into the display buffer. internal void RetainTargetlessDraw(RegisterBanks banks, in TargetlessDrawArguments arguments) { - var state = RequireCurrent(); + var state = _executing ?? RequireCurrent(); if (RenderTrace.Enabled) { RenderTrace.Write($"TargetlessRetention submit={arguments.SubmitId} replaced={state.RetainedTargetlessDraw is not null} previousSubmit={state.RetainedTargetlessDraw?.Arguments.SubmitId} export=0x{banks.Shader.Vertex.ExportAddress:X16} pixel=0x{banks.Shader.Pixel.Address:X16}"); @@ -363,22 +566,35 @@ public static partial class AgcExports { var banks = RequireTypedRegisters(state); state.FrameDispatchCount++; - var executorStarted = DcbParseProfile.Begin(); - try + if (Producing) { - executor.Dispatch(submitId, banks, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress); - } - finally - { - DcbParseProfile.RecordDispatch(executorStarted); + var snapshot = banks.Snapshot(); + Sink!.EnqueueDeferred(new PendingDispatch(submitId, state, snapshot, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress), + isDraw: false); + return; } + ExecuteDispatch(executor, submitId, banks, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress); return; } throw _host.Fatal("The command stream has no render executor."); } + private static void ExecuteDispatch(RenderExecutor executor, ulong submitId, RegisterBanks banks, uint endX, uint endY, uint endZ, + uint dispatchInitiator, ulong indirectArgumentsAddress) + { + var executorStarted = DcbParseProfile.Begin(); + try + { + executor.Dispatch(submitId, banks, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress); + } + finally + { + DcbParseProfile.RecordDispatch(executorStarted); + } + } + // The retained draw runs into the display buffer with the target words it was last bound with. private void ReplayRetainedTargetlessDraw(RenderExecutor executor, SubmittedDcbState state, int handle, int displayBufferIndex) { diff --git a/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs b/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs index 125c2c8..b5bcf3f 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs @@ -249,11 +249,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItIndirectBuffer, RZero)) || - !TryWriteUInt32(ctx, commandAddress + 4, (uint)(target & 0xFFFF_FFFFUL)) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)((target >> 32) & 0xFFFFUL)) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)targetDwords & 0xFFFFFu)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(4, ItIndirectBuffer, RZero), (uint)(target & 0xFFFF_FFFFUL), (uint)((target >> 32) & 0xFFFFUL), (uint)targetDwords & 0xFFFFFu)) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs index dfb8c60..62d31f2 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs @@ -46,16 +46,7 @@ public static partial class AgcExports uint perInstanceObjectIdSupport) { if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 3, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(3, ItSetUconfigRegIndex, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, 0x2000_0000u | VgtIndexType) || - !TryWriteUInt32( - ctx, - commandAddress + 8, - 0x400u | - (indexSize & 0x3u) | - ((cachePolicy & 0x3u) << 6) | - ((perInstanceObjectIdSupport & 0x1u) << 14))) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(3, ItSetUconfigRegIndex, 0), 0x2000_0000u | VgtIndexType, 0x400u | (indexSize & 0x3u) | ((cachePolicy & 0x3u) << 6) | ((perInstanceObjectIdSupport & 0x1u) << 14))) { return ReturnPointer(ctx, 0); } @@ -77,9 +68,7 @@ public static partial class AgcExports var commandBufferAddress = ctx[CpuRegister.Rdi]; var indexCount = (uint)ctx[CpuRegister.Rsi]; if (commandBufferAddress == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RIndexCount)) || - !TryWriteUInt32(ctx, commandAddress + 4, indexCount)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNop, RIndexCount), indexCount)) { return ReturnPointer(ctx, 0); } @@ -112,9 +101,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNumInstances, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, instanceCount)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItNumInstances, 0), instanceCount)) { return ReturnPointer(ctx, 0); } @@ -140,12 +127,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var baseCommand) || - !TryWriteUInt32(ctx, baseCommand, Pm4(3, ItIndexBase, 0)) || - !TryWriteUInt32(ctx, baseCommand + 4, (uint)indexAddress) || - !TryWriteUInt32(ctx, baseCommand + 8, (uint)(indexAddress >> 32)) || - !TryWriteUInt32(ctx, baseCommand + 12, Pm4(2, ItIndexBufferSize, 0)) || - !TryWriteUInt32(ctx, baseCommand + 16, indexCount)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var baseCommand, Pm4(3, ItIndexBase, 0), (uint)indexAddress, (uint)(indexAddress >> 32), Pm4(2, ItIndexBufferSize, 0), indexCount)) { return ReturnPointer(ctx, 0); } @@ -155,13 +137,7 @@ public static partial class AgcExports // former five-dword packet omitted both the real base and the count // field, so every call made by Unity looked like a zero-count draw to // the submitted-command parser and the complete scene was discarded. - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 6, out var drawCommand) || - !TryWriteUInt32(ctx, drawCommand, Pm4(6, ItDrawIndex2, 0)) || - !TryWriteUInt32(ctx, drawCommand + 4, indexCount) || - !TryWriteUInt32(ctx, drawCommand + 8, (uint)indexAddress) || - !TryWriteUInt32(ctx, drawCommand + 12, (uint)(indexAddress >> 32)) || - !TryWriteUInt32(ctx, drawCommand + 16, indexCount) || - !TryWriteUInt32(ctx, drawCommand + 20, 0)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var drawCommand, Pm4(6, ItDrawIndex2, 0), indexCount, (uint)indexAddress, (uint)(indexAddress >> 32), indexCount, 0)) { return ReturnPointer(ctx, 0); } @@ -204,12 +180,7 @@ public static partial class AgcExports ? 2u : ((modifierBits >> 3) & 0x20u) | 2u; - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var drawCommand) || - !TryWriteUInt32(ctx, drawCommand, Pm4(5, ItDrawIndirect, 0)) || - !TryWriteUInt32(ctx, drawCommand + 4, dataOffset) || - !TryWriteUInt32(ctx, drawCommand + 8, firstVertexRegister) || - !TryWriteUInt32(ctx, drawCommand + 12, firstInstanceRegister) || - !TryWriteUInt32(ctx, drawCommand + 16, drawInitiator)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var drawCommand, Pm4(5, ItDrawIndirect, 0), dataOffset, firstVertexRegister, firstInstanceRegister, drawInitiator)) { var rejects = Interlocked.Increment(ref _indirectDrawEmitRejectCount); if (rejects <= 8 || rejects % 250 == 0) @@ -244,14 +215,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 7, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(7, ItNop, RDrawIndexAuto)) || - !TryWriteUInt32(ctx, commandAddress + 4, indexCount) || - !TryWriteUInt32(ctx, commandAddress + 8, 0) || - !TryWriteUInt32(ctx, commandAddress + 12, 0) || - !TryWriteUInt32(ctx, commandAddress + 16, 0) || - !TryWriteUInt32(ctx, commandAddress + 20, 0) || - !TryWriteUInt32(ctx, commandAddress + 24, 0)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(7, ItNop, RDrawIndexAuto), indexCount, 0, 0, 0, 0, 0)) { return ReturnPointer(ctx, 0); } @@ -390,12 +354,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(3, ItIndexBase, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, (uint)(indexBufferAddress & 0xFFFF_FFFFUL)) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)(indexBufferAddress >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 12, Pm4(2, ItIndexBufferSize, 0)) || - !TryWriteUInt32(ctx, commandAddress + 16, indexCount)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(3, ItIndexBase, 0), (uint)(indexBufferAddress & 0xFFFF_FFFFUL), (uint)(indexBufferAddress >> 32), Pm4(2, ItIndexBufferSize, 0), indexCount)) { return ReturnPointer(ctx, 0); } @@ -424,12 +383,7 @@ public static partial class AgcExports var drawInitiator = (drawModifier & (1UL << 32)) != 0 ? 0u : ((uint)drawModifier >> 3) & 0x20u; - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItDrawIndexOffset2, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, Math.Max(indexCount, 1u)) || - !TryWriteUInt32(ctx, commandAddress + 8, indexOffset) || - !TryWriteUInt32(ctx, commandAddress + 12, indexCount) || - !TryWriteUInt32(ctx, commandAddress + 16, drawInitiator)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(5, ItDrawIndexOffset2, 0), Math.Max(indexCount, 1u), indexOffset, indexCount, drawInitiator)) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.Events.cs b/src/SharpEmu.Libs/Agc/AgcExports.Events.cs index d21a8b6..432a2ec 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.Events.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.Events.cs @@ -69,9 +69,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItEventWrite, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, eventType)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(2, ItEventWrite, 0), eventType)) { return ReturnPointer(ctx, 0); } @@ -95,14 +93,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 7, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(7, ItNop, RWaitFlipDone)) || - !TryWriteUInt32(ctx, commandAddress + 4, videoOutHandle) || - !TryWriteUInt32(ctx, commandAddress + 8, displayBufferIndex) || - !TryWriteUInt32(ctx, commandAddress + 12, 0) || - !TryWriteUInt32(ctx, commandAddress + 16, 0) || - !TryWriteUInt32(ctx, commandAddress + 20, 0) || - !TryWriteUInt32(ctx, commandAddress + 24, 0)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(7, ItNop, RWaitFlipDone), videoOutHandle, displayBufferIndex, 0, 0, 0, 0)) { return ReturnPointer(ctx, 0); } @@ -128,13 +119,7 @@ public static partial class AgcExports return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 6, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(6, ItNop, RFlip)) || - !TryWriteUInt32(ctx, commandAddress + 4, videoOutHandle) || - !TryWriteUInt32(ctx, commandAddress + 8, unchecked((uint)displayBufferIndex)) || - !TryWriteUInt32(ctx, commandAddress + 12, flipMode) || - !TryWriteUInt32(ctx, commandAddress + 16, (uint)(flipArg & 0xFFFF_FFFFUL)) || - !TryWriteUInt32(ctx, commandAddress + 20, (uint)(flipArg >> 32))) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(6, ItNop, RFlip), videoOutHandle, unchecked((uint)displayBufferIndex), flipMode, (uint)(flipArg & 0xFFFF_FFFFUL), (uint)(flipArg >> 32))) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs index 06f2b71..9aeb373 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs @@ -109,12 +109,9 @@ public static partial class AgcExports ulong commandAddress; using (AgcRegisterPacketProfile.Measure(AgcRegisterPacketProfile.Phase.PacketSetup)) { - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var markerAddress) || - !TryWriteUInt32(ctx, markerAddress, Pm4(2, ItNop, RZero)) || - !TryWriteUInt32(ctx, markerAddress + 4, CbSetShRegisterRangeMarker) || + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out _, Pm4(2, ItNop, RZero), CbSetShRegisterRangeMarker) || !TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItSetShReg, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, offset)) + !TryWriteCommandDwords(ctx, commandAddress, Pm4(packetDwords, ItSetShReg, 0), offset)) { AgcRegisterPacketProfile.RecordResult(false); return ReturnPointer(ctx, 0); @@ -288,12 +285,7 @@ public static partial class AgcExports _ => 0u, }; if (op == 0 || - !TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(5, op, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, (uint)(registersAddress & 0xFFFF_FFFCUL)) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)(registersAddress >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 12, 0x8000_0000u) || - !TryWriteUInt32(ctx, commandAddress + 16, registerCount & 0x3FFFu)) + !TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(5, op, 0), (uint)(registersAddress & 0xFFFF_FFFCUL), (uint)(registersAddress >> 32), 0x8000_0000u, registerCount & 0x3FFFu)) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.cs b/src/SharpEmu.Libs/Agc/AgcExports.cs index 89fb747..b32f20a 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.cs @@ -491,15 +491,40 @@ public static partial class AgcExports private static bool TryAllocateCommandDwords(CpuContext ctx, ulong commandBufferAddress, uint sizeDwords, out ulong commandAddress) => TryPrepareCommandDwords(ctx, commandBufferAddress, sizeDwords, true, out commandAddress); + private static bool TryEmitCommandDwords(CpuContext ctx, ulong commandBufferAddress, out ulong commandAddress, params ReadOnlySpan dwords) + { + return TryAllocateCommandDwords(ctx, commandBufferAddress, (uint)dwords.Length, out commandAddress) && + TryWriteCommandDwords(ctx, commandAddress, dwords); + } + + private static bool TryWriteCommandDwords(CpuContext ctx, ulong address, params ReadOnlySpan dwords) => + ctx.Memory.TryWrite(address, System.Runtime.InteropServices.MemoryMarshal.AsBytes(dwords)); + + private const int CommandBufferStateBytes = (int)(CommandBufferReservedDwOffset + sizeof(uint) - CommandBufferCursorUpOffset); + + private static bool TryReadCommandBufferState(CpuContext ctx, ulong commandBufferAddress, out ulong cursorUp, out ulong cursorDown, out ulong callback, out ulong userData, out uint reservedDwords) + { + Span state = stackalloc byte[CommandBufferStateBytes]; + if (!ctx.Memory.TryRead(commandBufferAddress + CommandBufferCursorUpOffset, state)) + { + cursorUp = cursorDown = callback = userData = 0; + reservedDwords = 0; + return false; + } + + cursorUp = BinaryPrimitives.ReadUInt64LittleEndian(state); + cursorDown = BinaryPrimitives.ReadUInt64LittleEndian(state[(int)(CommandBufferCursorDownOffset - CommandBufferCursorUpOffset)..]); + callback = BinaryPrimitives.ReadUInt64LittleEndian(state[(int)(CommandBufferCallbackOffset - CommandBufferCursorUpOffset)..]); + userData = BinaryPrimitives.ReadUInt64LittleEndian(state[(int)(CommandBufferUserDataOffset - CommandBufferCursorUpOffset)..]); + reservedDwords = BinaryPrimitives.ReadUInt32LittleEndian(state[(int)(CommandBufferReservedDwOffset - CommandBufferCursorUpOffset)..]); + return true; + } + private static bool TryPrepareCommandDwords(CpuContext ctx, ulong commandBufferAddress, uint sizeDwords, bool advanceCursor, out ulong commandAddress) { commandAddress = 0; if (sizeDwords == 0 || - !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorUpOffset, out var cursorUp) || - !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorDownOffset, out var cursorDown) || - !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCallbackOffset, out var callback) || - !TryReadUInt64(ctx, commandBufferAddress + CommandBufferUserDataOffset, out var userData) || - !TryReadUInt32(ctx, commandBufferAddress + CommandBufferReservedDwOffset, out var reservedDwords)) + !TryReadCommandBufferState(ctx, commandBufferAddress, out var cursorUp, out var cursorDown, out var callback, out var userData, out var reservedDwords)) { return false; } diff --git a/src/SharpEmu.Libs/Diagnostics/DccWriterTrace.cs b/src/SharpEmu.Libs/Diagnostics/DccWriterTrace.cs index ed00daf..322c317 100644 --- a/src/SharpEmu.Libs/Diagnostics/DccWriterTrace.cs +++ b/src/SharpEmu.Libs/Diagnostics/DccWriterTrace.cs @@ -28,6 +28,42 @@ internal static class DccWriterTrace private static readonly HashSet<(ulong Address, ulong Size, ulong Program)> _reported = new(); private static readonly HashSet<(ulong Hash, string Reason)> _refusals = new(); + private static readonly System.Collections.Concurrent.ConcurrentDictionary _writes = new(); + private static readonly HashSet _syncReported = new(); + + [ThreadStatic] + public static string? CurrentKind; + + public static void RecordWrite(ulong address, ulong size) + { + if (DccEnabled) + { + _writes[address] = (size, CurrentProgram, CurrentKind ?? "?"); + } + } + + public static void ReportSync(ulong metadataAddress, ulong size) + { + if (!DccEnabled) + { + return; + } + + lock (_syncReported) + { + if (_syncReported.Count >= 64 || !_syncReported.Add(metadataAddress)) + { + return; + } + } + + var writers = _writes + .Where(pair => pair.Key < metadataAddress + size && metadataAddress < pair.Key + pair.Value.Size) + .Take(8) + .Select(pair => $"0x{pair.Key:X}+0x{pair.Value.Size:X}:{pair.Value.Kind}:0x{pair.Value.Program:X16}"); + Console.Error.WriteLine( + $"[GPU][DCC_SYNC_WRITER] metadata=0x{metadataAddress:X16} size=0x{size:X} writers={string.Join(" ", writers)}"); + } private static ulong ParseAddress(string? value) { diff --git a/src/SharpEmu.Libs/Gpu/Buffers/BdaFaultProcessor.cs b/src/SharpEmu.Libs/Gpu/Buffers/BdaFaultProcessor.cs index febe4ce..5b9458b 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/BdaFaultProcessor.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/BdaFaultProcessor.cs @@ -216,6 +216,7 @@ public sealed unsafe class BdaFaultProcessor : IDisposable vk.CmdBindPipeline(command, PipelineBindPoint.Compute, _pipeline); var set = _sets[_currentArea]; vk.CmdBindDescriptorSets(command, PipelineBindPoint.Compute, _pipelineLayout, 0, 1, &set, 0, null); + NativeBindingEpoch.Advance(); var threads = _pageCount / 32; vk.CmdDispatch(command, (uint)((threads + 63) / 64), 1, 1); VulkanSynchronization.PipelineBarrier(vk, diff --git a/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs b/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs index fec9cf7..6250279 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs @@ -172,16 +172,39 @@ public unsafe class GpuBuffer : IDisposable // A readback of it only has to wait for that tick, not for all queued work. public ulong LastGpuWriteTick { get; private set; } + private readonly WriteTickMap _writeTicks = new(); + private int _writeTickPruneCount = 64; + // Called by every path that records a GPU write into this buffer. - public void NoteGpuWrite() + public void NoteGpuWrite() => NoteGpuWrite(0, Size); + + public void NoteGpuWrite(ulong offset, ulong size) { var tick = _scheduler.CurrentTick; if (tick > LastGpuWriteTick) { LastGpuWriteTick = tick; } + + if (size == 0 || offset >= Size) + { + return; + } + + _writeTicks.Assign(offset, Math.Min(size, Size - offset), tick); + if (_writeTicks.Count >= _writeTickPruneCount) + { + if (Volatile.Read(ref _foreignReads) == 0) + { + _writeTicks.Prune(_scheduler.Timeline.CompletedTick + 1); + } + + _writeTickPruneCount = Math.Max(64, _writeTicks.Count * 2); + } } + public ulong LastGpuWriteTickIn(ulong offset, ulong size) => _writeTicks.MaxTick(offset, size); + public void Write(ulong offset, ReadOnlySpan source) { if (_mapped == null || offset > Size || (ulong)source.Length > Size - offset) @@ -243,7 +266,7 @@ public unsafe class GpuBuffer : IDisposable } command.EndRendering(); - NoteGpuWrite(); + NoteGpuWrite(destinationOffset, size); var vk = _device.Vk; var native = new CommandBuffer(command.Handle); var before = stackalloc BufferMemoryBarrier2[2]; @@ -271,7 +294,7 @@ public unsafe class GpuBuffer : IDisposable var command = _scheduler.Current; command.EndRendering(); - NoteGpuWrite(); + NoteGpuWrite(offset, size); var vk = _device.Vk; var native = new CommandBuffer(command.Handle); var before = CreateBarrier(offset, size, MemoryAccess, AccessFlags.TransferWriteBit); diff --git a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs index 183f607..1a77b94 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs @@ -240,10 +240,11 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable _ = SynchronizeBuffer(buffer, guestAddress, size, isWritten, isTexelBuffer, preserveCpuWriteHotPages: false); if (isWritten) { - buffer.NoteGpuWrite(); + buffer.NoteGpuWrite(buffer.Offset(guestAddress), size); if (Diagnostics.DccWriterTrace.Enabled) { Diagnostics.DccWriterTrace.Record(RequireImageCache(), guestAddress, size); + Diagnostics.DccWriterTrace.RecordWrite(guestAddress, size); } if (!_gpuModifiedRanges.Contains(guestAddress, size)) @@ -333,6 +334,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable throw SubmissionScheduler.Fatal("The host DMA write range is invalid."); } + GpuWritebackEpoch.Advance(); if (!_backing.TryWriteBacking(guestAddress, data)) { throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{guestAddress:X16} size=0x{data.Length:X16}"); @@ -359,7 +361,10 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } } - public void FillBuffer(ulong guestAddress, ulong size, uint value, bool isGds) + public void FillBuffer(ulong guestAddress, ulong size, uint value, bool isGds) => + FillBuffer(guestAddress, size, value, isGds, publishFill: false); + + private void FillBuffer(ulong guestAddress, ulong size, uint value, bool isGds, bool publishFill) { if ((guestAddress & 3) != 0 || size == 0 || (size & 3) != 0 || size > ulong.MaxValue - guestAddress) { @@ -408,56 +413,116 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable images.InvalidateMemoryFromGpu(guestAddress, size); var bufferIdentifier = FindBuffer(guestAddress, size); + if (Diagnostics.DccWriterTrace.Enabled) + { + Diagnostics.DccWriterTrace.CurrentKind = "fill"; + } + var (destination, destinationOffset) = ObtainBuffer(guestAddress, size, true, true, bufferIdentifier); destination.Fill(destinationOffset, size, value); - if (images.OverlapsDccMetadata(guestAddress, size) && TryWriteFillToBacking(guestAddress, size, value)) + if ((publishFill || images.OverlapsDccMetadata(guestAddress, size)) && TryWriteFillToBacking(guestAddress, size, value)) { _gpuModifiedRanges.Remove(guestAddress, size); - var firstPage = guestAddress & ~(TrackerLayout.PageBytes - 1); - for (var page = firstPage; page < guestAddress + size; page += TrackerLayout.PageBytes) - { - if (!_gpuModifiedRanges.Overlaps(page, TrackerLayout.PageBytes)) - { - _tracker.ClearGpuDirtyPages(page, TrackerLayout.PageBytes); - } - } + Interlocked.Increment(ref _gpuModifiedVersion); + ClearGpuDirtyPagesOfRemovedRange(guestAddress, size); + } + } + + // After a range left the GPU-modified set: the pages inside it are clean in one tracker walk, + // and only a page the range shares at either end can still hold another GPU-modified range. + private void ClearGpuDirtyPagesOfRemovedRange(ulong guestAddress, ulong size) + { + const ulong pageBytes = TrackerLayout.PageBytes; + var end = guestAddress + size; + var firstPage = guestAddress & ~(pageBytes - 1); + var lastPage = (end - 1) & ~(pageBytes - 1); + var startPartial = guestAddress != firstPage; + var endPartial = end != lastPage + pageBytes; + var innerBegin = startPartial ? firstPage + pageBytes : firstPage; + var innerEnd = endPartial ? lastPage : end; + if (innerBegin < innerEnd) + { + _tracker.ClearGpuDirtyPages(innerBegin, innerEnd - innerBegin); + } + + if (startPartial) + { + ClearGpuDirtyPageIfUnmodified(firstPage); + } + + if (endPartial && (lastPage != firstPage || !startPartial)) + { + ClearGpuDirtyPageIfUnmodified(lastPage); + } + } + + private void ClearGpuDirtyPageIfUnmodified(ulong page) + { + if (!_gpuModifiedRanges.Overlaps(page, TrackerLayout.PageBytes)) + { + _tracker.ClearGpuDirtyPages(page, TrackerLayout.PageBytes); } } public void FillDccMetadata(ulong guestAddress, ulong size, uint value) { if (guestAddress == 0 || (guestAddress & 3) != 0 || size == 0 || (size & 3) != 0 || size > ulong.MaxValue - guestAddress || - HasGpuDirtyBytes(guestAddress, size) || RequireImageCache().QueryRegion(guestAddress, size).ImageBytes) + HasGpuDirtyBytes(guestAddress, size)) { - FillBuffer(guestAddress, size, value, false); + FillBuffer(guestAddress, size, value, false, publishFill: true); return; } + // Images over the range read it again; only GPU-owned image bytes need the general fill. + // The buffer copy is filled on the GPU rather than uploaded from the written bytes. + var images = RequireImageCache(); + var region = images.QueryRegion(guestAddress, size); + if (region.GpuImageBytes) + { + FillBuffer(guestAddress, size, value, false, publishFill: true); + return; + } + + if (region.ImageBytes) + { + images.InvalidateMemory(guestAddress, size); + } + var (destination, destinationOffset) = ObtainBuffer(guestAddress, size, false, false, FindBuffer(guestAddress, size)); destination.Fill(destinationOffset, size, value); if (!TryWriteFillToBacking(guestAddress, size, value)) { - FillBuffer(guestAddress, size, value, false); + FillBuffer(guestAddress, size, value, false, publishFill: true); } } private bool TryWriteFillToBacking(ulong guestAddress, ulong size, uint value) { - var values = new uint[(int)Math.Min(size / sizeof(uint), 4096)]; - Array.Fill(values, value); - var bytes = MemoryMarshal.AsBytes(values); - for (ulong offset = 0; offset < size;) + // Metadata fills repeat every frame: the pattern buffer is pooled. + var count = (int)Math.Min(size / sizeof(uint), 4096); + var values = System.Buffers.ArrayPool.Shared.Rent(count); + try { - var chunk = (int)Math.Min(size - offset, (ulong)bytes.Length); - if (!_backing.TryWriteBacking(guestAddress + offset, bytes[..chunk])) + values.AsSpan(0, count).Fill(value); + var bytes = MemoryMarshal.AsBytes(values.AsSpan(0, count)); + for (ulong offset = 0; offset < size;) { - return false; + var chunk = (int)Math.Min(size - offset, (ulong)bytes.Length); + GpuWritebackEpoch.Advance(); + if (!_backing.TryWriteBacking(guestAddress + offset, bytes[..chunk])) + { + return false; + } + + offset += (ulong)chunk; } - offset += (ulong)chunk; + return true; + } + finally + { + System.Buffers.ArrayPool.Shared.Return(values); } - - return true; } public const ulong MaxHostCopyWords = 64 * 1024; @@ -583,6 +648,11 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var srcId = srcMemory ? FindBuffer(srcVaddr, size) : default; var dstId = dstMemory ? FindBuffer(dstVaddr, size) : default; var (src, srcOffset) = srcMemory ? ObtainBuffer(srcVaddr, size, false, true, srcId) : (_gds, srcVaddr); + if (Diagnostics.DccWriterTrace.Enabled) + { + Diagnostics.DccWriterTrace.CurrentKind = $"copy_from_0x{srcVaddr:X}"; + } + var (dst, dstOffset) = dstMemory ? ObtainBuffer(dstVaddr, size, true, true, dstId) : (_gds, dstVaddr); if (ReferenceEquals(src, dst) && srcOffset < dstOffset + size && dstOffset < srcOffset + size) { @@ -898,22 +968,34 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable if (!_relay.IsGpuQueueThread && AsyncReadback is { } readback && GuestReadsAwaitOffQueue) { - PendingDownload? pending = null; - if (_relay.TryRunOnGpuQueue(() => pending = BeginReadMemoryOnGpu(guestAddress, size, isWrite, source, allowPending: true))) + for (var attempt = 0; ; attempt++) { + var allowPending = attempt < MaxAsyncReadbackAttempts; + var markHot = attempt == 0 && !isWrite; + PendingDownload? pending = null; + if (!_relay.TryRunOnGpuQueue(() => + pending = BeginReadMemoryOnGpu(guestAddress, size, isWrite, source, allowPending, markHot))) + { + return AwaitShutdown(); + } + if (pending is null) { return true; } readback.Wait(pending.Ticket); - if (_relay.TryRunOnGpuQueue(() => CompleteReadMemoryOnGpu(pending, retrySynchronously: true))) + var complete = false; + if (!_relay.TryRunOnGpuQueue(() => complete = CompleteReadMemoryOnGpu(pending, consumed: true))) + { + return AwaitShutdown(); + } + + if (complete) { return true; } } - - return AwaitShutdown(); } var onQueue = _relay.IsGpuQueueThread; @@ -934,6 +1016,9 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable private const ulong ReadbackWindowBytes = 512 * 1024; private const long HotWindowLifetime = 512; private const int MaxEagerReadbacks = 4; + private const int MaxAsyncReadbackAttempts = 4; + private const int EagerStrikeLimit = 3; + private const long EagerMuteBatches = 4096; private readonly Dictionary _hotWindows = new(); private readonly HashSet _eagerCandidates = new(); private readonly List _eagerDownloads = new(); @@ -941,8 +1026,9 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable private static long _reportedEagerStarted; private static long _reportedEagerUsed; private static long _reportedEagerStale; + private static long _reportedEagerMuted; - private readonly record struct HotWindow(ulong Address, ulong Size, long LastHit); + private readonly record struct HotWindow(ulong Address, ulong Size, long LastHit, int Strikes = 0, long MutedUntil = 0); internal long EagerReadbacksStarted { get; private set; } @@ -950,7 +1036,44 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable private void NoteHotReadback(ulong guestAddress, ulong size) { - _hotWindows[guestAddress & ~(ReadbackWindowBytes - 1)] = new HotWindow(guestAddress, size, _batchSerial); + var key = guestAddress & ~(ReadbackWindowBytes - 1); + _hotWindows[key] = _hotWindows.TryGetValue(key, out var hot) + ? hot with { Address = guestAddress, Size = size, LastHit = _batchSerial } + : new HotWindow(guestAddress, size, _batchSerial); + } + + internal long EagerReadbacksWasted { get; private set; } + + private void NoteEagerOutcome(ulong key, bool wasted) + { + if (wasted) + { + EagerReadbacksWasted++; + } + + if (!_hotWindows.TryGetValue(key, out var hot)) + { + return; + } + + if (!wasted) + { + if (hot.Strikes != 0) + { + _hotWindows[key] = hot with { Strikes = 0 }; + } + + return; + } + + var strikes = hot.Strikes + 1; + _hotWindows[key] = strikes >= EagerStrikeLimit + ? hot with { Strikes = 0, MutedUntil = _batchSerial + EagerMuteBatches } + : hot with { Strikes = strikes }; + if (strikes >= EagerStrikeLimit) + { + Interlocked.Increment(ref _reportedEagerMuted); + } } private void NoteHotWindowWrite(ulong guestAddress, ulong size) @@ -963,10 +1086,11 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var last = (guestAddress + size - 1) & ~(ReadbackWindowBytes - 1); for (var key = guestAddress & ~(ReadbackWindowBytes - 1); key <= last; key += ReadbackWindowBytes) { - if (_hotWindows.TryGetValue(key, out var hot) && _batchSerial - hot.LastHit <= HotWindowLifetime) + if (_hotWindows.TryGetValue(key, out var hot) && _batchSerial - hot.LastHit <= HotWindowLifetime && + _batchSerial >= hot.MutedUntil) { _eagerCandidates.Add(key); - HotWritePending = true; + HotWritePending |= hot.Strikes == 0; } } } @@ -988,16 +1112,14 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable if (readback.IsComplete(pending.Ticket)) { _eagerDownloads.RemoveAt(index); - if (!CompleteReadMemoryOnGpu(pending, retrySynchronously: false)) - { - Interlocked.Increment(ref _reportedEagerStale); - } + _ = CompleteReadMemoryOnGpu(pending, consumed: false); } } if ((_batchSerial & 1023) == 0) { - foreach (var key in _hotWindows.Where(pair => _batchSerial - pair.Value.LastHit > HotWindowLifetime).Select(pair => pair.Key).ToList()) + foreach (var key in _hotWindows.Where(pair => _batchSerial - pair.Value.LastHit > HotWindowLifetime && _batchSerial >= pair.Value.MutedUntil) + .Select(pair => pair.Key).ToList()) { _hotWindows.Remove(key); } @@ -1008,6 +1130,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable return; } + List? settled = null; foreach (var key in _eagerCandidates) { if (_eagerDownloads.Count >= MaxEagerReadbacks) @@ -1015,26 +1138,52 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable break; } - if (_hotWindows.TryGetValue(key, out var hot)) + if (!_hotWindows.TryGetValue(key, out var hot) || _batchSerial - hot.LastHit > HotWindowLifetime || + _batchSerial < hot.MutedUntil || StartEagerReadback(hot) != EagerStart.Deferred) { - StartEagerReadback(hot); + (settled ??= []).Add(key); } } - _eagerCandidates.Clear(); - } - - private void StartEagerReadback(HotWindow hot) - { - if (FindOwner(hot.Address, hot.Size) is not { } buffer) + if (settled is null) { return; } - var copies = CollectReadbackWindow(buffer, hot.Address, hot.Size, out var windowBegin, out var windowEnd); - if (copies.Count == 0 || !TryBeginDownloadAsync(copies, out var ticket, out var writeTicks)) + foreach (var key in settled) { - return; + _eagerCandidates.Remove(key); + } + } + + private enum EagerStart + { + Started, + Deferred, + Nothing, + } + + private EagerStart StartEagerReadback(HotWindow hot) + { + if (FindOwner(hot.Address, hot.Size) is not { } buffer) + { + return EagerStart.Nothing; + } + + if (OverlapsEagerReadback(hot.Address, hot.Size)) + { + return EagerStart.Deferred; + } + + var copies = CollectReadbackWindow(buffer, hot.Address, hot.Size, out var windowBegin, out var windowEnd); + if (copies.Count == 0) + { + return EagerStart.Nothing; + } + + if (!TryBeginDownloadAsync(copies, out var ticket, out var writeTicks)) + { + return EagerStart.Deferred; } foreach (var copy in copies) @@ -1044,6 +1193,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable _eagerDownloads.Add(new PendingDownload { + HotKey = hot.Address & ~(ReadbackWindowBytes - 1), Ticket = ticket, Copies = copies, WriteTicks = writeTicks, @@ -1052,11 +1202,26 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable GuestAddress = hot.Address, Size = hot.Size, IsWrite = false, + Eager = true, Source = GuestMemoryProfile.ReadbackSource.ShaderResourceRead, Started = System.Diagnostics.Stopwatch.GetTimestamp(), }); EagerReadbacksStarted++; Interlocked.Increment(ref _reportedEagerStarted); + return EagerStart.Started; + } + + private bool OverlapsEagerReadback(ulong guestAddress, ulong size) + { + foreach (var pending in _eagerDownloads) + { + if (guestAddress < pending.WindowEnd && pending.WindowBegin < guestAddress + size) + { + return true; + } + } + + return false; } private PendingDownload? TakeEagerReadback(ulong guestAddress, ulong size) @@ -1068,6 +1233,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable guestAddress - pending.WindowBegin <= pending.WindowEnd - pending.WindowBegin - size) { _eagerDownloads.RemoveAt(index); + NoteHotReadback(guestAddress, size); return pending; } } @@ -1083,15 +1249,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } readback.Wait(pending.Ticket); - if (!CompleteReadMemoryOnGpu(pending, retrySynchronously: false)) - { - Interlocked.Increment(ref _reportedEagerStale); - return false; - } - - EagerReadbacksUsed++; - Interlocked.Increment(ref _reportedEagerUsed); - return !HasGpuDirtyBytes(guestAddress, size); + return CompleteReadMemoryOnGpu(pending with { GuestAddress = guestAddress, Size = size }, consumed: true); } private void AbandonEagerReadbacks() @@ -1131,9 +1289,6 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } } - private void ReadMemoryOnGpu(ulong guestAddress, ulong size, bool isWrite, GuestMemoryProfile.ReadbackSource source) => - _ = BeginReadMemoryOnGpu(guestAddress, size, isWrite, source, allowPending: false); - internal long PendingReadbacksApplied { get; private set; } internal long PendingReadbacksRetried { get; private set; } @@ -1141,8 +1296,13 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable private static long _reportedApplied; private static long _reportedRetried; + private static long _reportedStalePieces; + private static long _reportedPublishedPieces; + + internal long StalePiecesKept { get; private set; } + public static string TakeAsyncReadbackReport() => FormattableString.Invariant( - $"[PERF][ASYNC_READBACK] applied={Interlocked.Exchange(ref _reportedApplied, 0)} retried={Interlocked.Exchange(ref _reportedRetried, 0)} eager_started={Interlocked.Exchange(ref _reportedEagerStarted, 0)} eager_used={Interlocked.Exchange(ref _reportedEagerUsed, 0)} eager_stale={Interlocked.Exchange(ref _reportedEagerStale, 0)}"); + $"[PERF][ASYNC_READBACK] applied={Interlocked.Exchange(ref _reportedApplied, 0)} retried={Interlocked.Exchange(ref _reportedRetried, 0)} published_pieces={Interlocked.Exchange(ref _reportedPublishedPieces, 0)} stale_pieces={Interlocked.Exchange(ref _reportedStalePieces, 0)} eager_started={Interlocked.Exchange(ref _reportedEagerStarted, 0)} eager_used={Interlocked.Exchange(ref _reportedEagerUsed, 0)} eager_stale={Interlocked.Exchange(ref _reportedEagerStale, 0)} eager_muted={Interlocked.Exchange(ref _reportedEagerMuted, 0)}"); private sealed record PendingDownload { @@ -1154,6 +1314,8 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable public required ulong GuestAddress { get; init; } public required ulong Size { get; init; } public required bool IsWrite { get; init; } + public bool Eager { get; init; } + public ulong HotKey { get; init; } public required GuestMemoryProfile.ReadbackSource Source { get; init; } public required long Started { get; init; } } @@ -1169,7 +1331,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable return null; } - if (allowPending && TakeEagerReadback(guestAddress, size) is { } eager) + if (!isWrite && allowPending && TakeEagerReadback(guestAddress, size) is { } eager) { return eager with { GuestAddress = guestAddress, Size = size, IsWrite = isWrite, Source = source }; } @@ -1216,20 +1378,87 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable return null; } - private bool CompleteReadMemoryOnGpu(PendingDownload pending, bool retrySynchronously) + private bool CompleteReadMemoryOnGpu(PendingDownload pending, bool consumed) { using var readbackScope = GuestMemoryProfile.Measure(GuestMemoryProfile.Operation.BufferReadback); var readback = AsyncReadback ?? throw SubmissionScheduler.Fatal("The pending readback lost its queue."); try { - if (!IsPendingDownloadCurrent(pending)) + var copies = pending.Copies; + var current = new bool[copies.Count]; + var published = 0; + for (var index = 0; index < copies.Count; index++) { - readback.Complete(pending.Ticket, null); - if (retrySynchronously) + current[index] = IsPieceCurrent(pending, index); + if (current[index]) { - PendingReadbacksRetried++; - Interlocked.Increment(ref _reportedRetried); - ReadMemoryOnGpu(pending.GuestAddress, pending.Size, pending.IsWrite, pending.Source); + published++; + } + } + + readback.Complete(pending.Ticket, published == 0 ? null : (index, bytes) => + { + if (!current[index]) + { + return; + } + + GpuWritebackEpoch.Advance(); + if (!_backing.TryWriteBacking(copies[index].Address, bytes)) + { + throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copies[index].Address:X16} size=0x{(ulong)bytes.Length:X16}"); + } + }); + for (var index = 0; index < copies.Count; index++) + { + if (current[index]) + { + _gpuModifiedRanges.Remove(copies[index].Address, copies[index].Size); + Interlocked.Increment(ref _gpuModifiedVersion); + } + } + + for (var index = 0; index < copies.Count; index++) + { + if (current[index]) + { + ClearGpuDirtyPagesOfRemovedRange(copies[index].Address, copies[index].Size); + } + } + + var stale = copies.Count - published; + if (pending.Eager) + { + NoteEagerOutcome(pending.HotKey, stale != 0); + } + + if (stale != 0) + { + StalePiecesKept += stale; + Interlocked.Add(ref _reportedStalePieces, stale); + } + + Interlocked.Add(ref _reportedPublishedPieces, published); + var clean = pending.IsWrite + ? !_tracker.HasGpuDirtyPages(pending.GuestAddress, pending.Size) + : !HasGpuDirtyBytes(pending.GuestAddress, pending.Size); + if (!consumed) + { + if (pending.Eager && stale != 0) + { + Interlocked.Increment(ref _reportedEagerStale); + } + + return clean; + } + + if (!clean) + { + PendingReadbacksRetried++; + Interlocked.Increment(ref _reportedRetried); + if (pending.Eager) + { + Interlocked.Increment(ref _reportedEagerStale); } return false; @@ -1237,21 +1466,12 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable PendingReadbacksApplied++; Interlocked.Increment(ref _reportedApplied); - - var copies = pending.Copies; - readback.Complete(pending.Ticket, (index, bytes) => + if (pending.Eager) { - if (!_backing.TryWriteBacking(copies[index].Address, bytes)) - { - throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copies[index].Address:X16} size=0x{(ulong)bytes.Length:X16}"); - } - }); - foreach (var copy in copies) - { - _gpuModifiedRanges.Remove(copy.Address, copy.Size); + EagerReadbacksUsed++; + Interlocked.Increment(ref _reportedEagerUsed); } - _tracker.ClearGpuDirtyPages(pending.WindowBegin, pending.WindowEnd - pending.WindowBegin); if (pending.IsWrite) { _tracker.MarkCpuDirtyPages(pending.GuestAddress, pending.Size); @@ -1269,29 +1489,12 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } } - private bool IsPendingDownloadCurrent(PendingDownload pending) + private bool IsPieceCurrent(PendingDownload pending, int index) { - var total = 0UL; - for (var index = 0; index < pending.Copies.Count; index++) - { - var copy = pending.Copies[index]; - if (copy.Buffer.LastGpuWriteTick != pending.WriteTicks[index] || - !_gpuModifiedRanges.Contains(copy.Address, copy.Size) || - !ReferenceEquals(FindOwner(copy.Address, copy.Size), copy.Buffer)) - { - return false; - } - - total += copy.Size; - } - - var current = 0UL; - foreach (var range in _gpuModifiedRanges.GetOverlappingRanges(pending.WindowBegin, pending.WindowEnd - pending.WindowBegin)) - { - current += range.Size; - } - - return current == total; + var copy = pending.Copies[index]; + return copy.Buffer.LastGpuWriteTickIn(copy.SourceOffset, copy.Size) == pending.WriteTicks[index] && + _gpuModifiedRanges.Contains(copy.Address, copy.Size) && + ReferenceEquals(FindOwner(copy.Address, copy.Size), copy.Buffer); } private static void RecordReadback(ulong windowBegin, ulong windowEnd, bool isWrite, List copies, long started, @@ -1456,6 +1659,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable foreach (var copy in copies) { _gpuModifiedRanges.Remove(copy.Address, copy.Size); + Interlocked.Increment(ref _gpuModifiedVersion); } return; @@ -1492,6 +1696,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable foreach (var copy in copies) { _gpuModifiedRanges.Remove(copy.Address, copy.Size); + Interlocked.Increment(ref _gpuModifiedVersion); } } @@ -1509,7 +1714,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var ticks = new ulong[copies.Count]; for (var index = 0; index < copies.Count; index++) { - var written = copies[index].Buffer.LastGpuWriteTick; + var written = copies[index].Buffer.LastGpuWriteTickIn(copies[index].SourceOffset, copies[index].Size); if (written == 0) { return false; @@ -1559,7 +1764,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var total = 0UL; foreach (var copy in copies) { - var written = copy.Buffer.LastGpuWriteTick; + var written = copy.Buffer.LastGpuWriteTickIn(copy.SourceOffset, copy.Size); // A GPU-modified range with no recorded writer: keep the conservative path. if (written == 0) { @@ -1590,6 +1795,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable readback.Read(pieces, waitTick, (index, bytes) => { + GpuWritebackEpoch.Advance(); if (!_backing.TryWriteBacking(copies[index].Address, bytes)) { throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copies[index].Address:X16} size=0x{(ulong)bytes.Length:X16}"); @@ -1622,6 +1828,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var placement = copy.Placement; var offset = baseOffset + placement.DataOffset; _download.Invalidate(offset, placement.DataSize); + GpuWritebackEpoch.Advance(); if (!_backing.TryWriteBacking(copy.Address, _download.Mapped.Slice((int)offset, (int)placement.DataSize))) { throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copy.Address:X16} size=0x{placement.DataSize:X16}"); @@ -1702,34 +1909,99 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.BufferDirtySynchronization); var startedAt = BufferUploadProfile.Enabled ? System.Diagnostics.Stopwatch.GetTimestamp() : 0; - if ((!preserveCpuWriteHotPages && !isWritten && !isTexelBuffer && !_tracker.MayHaveCpuDirtyPages(guestAddress, size)) || - (isWritten && !isTexelBuffer && _gpuModifiedRanges.Contains(guestAddress, size))) + var nothingToUpload = isWritten + ? _gpuModifiedRanges.Contains(guestAddress, size) + : !_tracker.MayHaveCpuDirtyPages(guestAddress, size); + if (nothingToUpload) { if (BufferUploadProfile.Enabled) BufferUploadProfile.Record(guestAddress, size, 0, 0, 0, System.Diagnostics.Stopwatch.GetTimestamp() - startedAt); - return false; + return isTexelBuffer && SynchronizeTexelFromImage(buffer, guestAddress, size); } profileScope.SwitchPhase(isWritten ? RenderPhaseProfile.Phase.BufferDirtySyncWritten : isTexelBuffer ? RenderPhaseProfile.Phase.BufferDirtySyncTexel : RenderPhaseProfile.Phase.BufferDirtySyncUpload); - var copies = new List(); - var totalSize = 0UL; - GpuBuffer? source = null; - _tracker.ForEachUploadRange( - guestAddress, - size, - isWritten, - (address, bytes) => + var collector = _uploadCollector; + if (collector is null || collector.InUse) + { + // The first call on a thread keeps its collector; a nested one gets its own. + collector = new UploadCollector(); + _uploadCollector ??= collector; + } + + collector.Begin(this, buffer, guestAddress, size); + try + { + return SynchronizeBufferCore(collector, buffer, guestAddress, size, isWritten, isTexelBuffer, preserveCpuWriteHotPages, startedAt); + } + finally + { + collector.End(); + } + } + + // The ranges of one synchronization. The delegates are bound once, so a call that has dirty + // pages to upload allocates no closure and no list. + private sealed class UploadCollector + { + private GuestBufferCache _cache = null!; + private GpuBuffer _buffer = null!; + private ulong _guestAddress; + private ulong _size; + public readonly List Copies = new(); + public ulong TotalSize; + public GpuBuffer? Source; + public bool InUse; + public readonly Action Collect; + public readonly Action Upload; + + public UploadCollector() + { + Collect = (address, bytes) => { - copies.Add(new BufferCopy(totalSize, buffer.Offset(address), bytes)); - totalSize += bytes; - }, - () => source = _uploader.PrepareSource(buffer.CpuAddress, CollectionsMarshal.AsSpan(copies), totalSize, guestAddress, size), - preserveCpuWriteHotPages); + Copies.Add(new BufferCopy(TotalSize, _buffer.Offset(address), bytes)); + TotalSize += bytes; + }; + Upload = () => Source = _cache._uploader.PrepareSource(_buffer.CpuAddress, CollectionsMarshal.AsSpan(Copies), TotalSize, _guestAddress, _size); + } + + public void Begin(GuestBufferCache cache, GpuBuffer buffer, ulong guestAddress, ulong size) + { + InUse = true; + _cache = cache; + _buffer = buffer; + _guestAddress = guestAddress; + _size = size; + } + + public void End() + { + InUse = false; + _cache = null!; + _buffer = null!; + Copies.Clear(); + TotalSize = 0; + Source = null; + } + } + + [ThreadStatic] private static UploadCollector? _uploadCollector; + + private bool SynchronizeBufferCore(UploadCollector collector, GpuBuffer buffer, ulong guestAddress, ulong size, bool isWritten, + bool isTexelBuffer, bool preserveCpuWriteHotPages, long startedAt) + { + var copies = collector.Copies; + _tracker.ForEachUploadRange(guestAddress, size, isWritten, collector.Collect, collector.Upload, preserveCpuWriteHotPages); + var source = collector.Source; + var totalSize = collector.TotalSize; if (source != null) { - buffer.NoteGpuWrite(); + foreach (var copy in copies) + { + buffer.NoteGpuWrite(copy.DstOffset, copy.Size); + } + var command = _scheduler.Current; command.EndRendering(); var native = new CommandBuffer(command.Handle); @@ -1771,18 +2043,18 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable BufferUploadProfile.Record(guestAddress, size, copies.Count, totalSize, hotBytes, elapsedTicks); } - if (isTexelBuffer) - { - var copiedFromImage = RequireImageCache().TrySynchronizeBufferFromImage(buffer, guestAddress, size); - if (copiedFromImage) - { - buffer.NoteGpuWrite(); - } + return isTexelBuffer && SynchronizeTexelFromImage(buffer, guestAddress, size); + } - return copiedFromImage; + private bool SynchronizeTexelFromImage(GpuBuffer buffer, ulong guestAddress, ulong size) + { + var copiedFromImage = RequireImageCache().TrySynchronizeBufferFromImage(buffer, guestAddress, size); + if (copiedFromImage) + { + buffer.NoteGpuWrite(buffer.Offset(guestAddress), size); } - return false; + return copiedFromImage; } private GpuBuffer? FindOwner(ulong guestAddress, ulong size) diff --git a/src/SharpEmu.Libs/Gpu/Buffers/WriteTickMap.cs b/src/SharpEmu.Libs/Gpu/Buffers/WriteTickMap.cs new file mode 100644 index 0000000..66d8d5d --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Buffers/WriteTickMap.cs @@ -0,0 +1,143 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Gpu.Buffers; + +public sealed class WriteTickMap +{ + private readonly List _entries = []; + private ulong _floor; + + private readonly record struct Entry(ulong Begin, ulong End, ulong Tick); + + public int Count => _entries.Count; + + public ulong Floor => _floor; + + public void Assign(ulong begin, ulong size, ulong tick) + { + if (size == 0) + { + return; + } + + var end = begin + size; + var index = FirstEndingAfter(begin); + var removed = 0; + while (index + removed < _entries.Count && _entries[index + removed].Begin < end) + { + removed++; + } + + Entry? left = null; + Entry? right = null; + if (removed != 0) + { + var first = _entries[index]; + if (first.Begin < begin) + { + left = first with { End = begin }; + } + + var last = _entries[index + removed - 1]; + if (last.End > end) + { + right = last with { Begin = end }; + } + + _entries.RemoveRange(index, removed); + } + + var insert = index; + if (left is { } leftPiece) + { + _entries.Insert(insert++, leftPiece); + } + + _entries.Insert(insert, new Entry(begin, end, tick)); + var middle = insert++; + if (right is { } rightPiece) + { + _entries.Insert(insert, rightPiece); + } + + if (middle + 1 < _entries.Count) + { + _ = TryMerge(middle, middle + 1); + } + + if (middle > 0) + { + _ = TryMerge(middle - 1, middle); + } + } + + public ulong MaxTick(ulong begin, ulong size) + { + var tick = _floor; + if (size == 0) + { + return tick; + } + + var end = begin + size; + for (var index = FirstEndingAfter(begin); index < _entries.Count && _entries[index].Begin < end; index++) + { + tick = Math.Max(tick, _entries[index].Tick); + } + + return tick; + } + + public void Prune(ulong below) + { + var kept = 0; + for (var index = 0; index < _entries.Count; index++) + { + var entry = _entries[index]; + if (entry.Tick < below) + { + _floor = Math.Max(_floor, entry.Tick); + continue; + } + + _entries[kept++] = entry; + } + + _entries.RemoveRange(kept, _entries.Count - kept); + } + + private bool TryMerge(int left, int right) + { + var a = _entries[left]; + var b = _entries[right]; + if (a.End != b.Begin || a.Tick != b.Tick) + { + return false; + } + + _entries[left] = a with { End = b.End }; + _entries.RemoveAt(right); + return true; + } + + private int FirstEndingAfter(ulong address) + { + var low = 0; + var high = _entries.Count; + while (low < high) + { + var middle = low + ((high - low) >> 1); + if (_entries[middle].End <= address) + { + low = middle + 1; + } + else + { + high = middle; + } + } + + return low; + } +} diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs index 1db6ec3..2c6fd44 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs @@ -529,7 +529,7 @@ public sealed class CommandStreamQueue if (submission.Kind == CommandSubmissionKind.FrameBoundary) { SlicesRun++; - _host.Flush(); + _host.FlushFrame(); return true; } diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs index 268275b..23adb2a 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs @@ -26,7 +26,7 @@ public sealed partial class GpuCommandInterpreter } // A compare-swap loop applies once per try and suspends while the compare fails. - if (!atomic.TryApply(_host.TryReadGuest, _host.Memory, out var priorValue, out _, out var comparePassed)) + if (!atomic.TryApply(_host.TryReadGuestOperand, _host.Memory, out var priorValue, out _, out var comparePassed)) { throw _host.Fatal($"The atomic address cannot be accessed: address=0x{atomic.Address:X16} size={atomic.ByteCount}."); } @@ -68,7 +68,7 @@ public sealed partial class GpuCommandInterpreter if (semaphore.IsSignal) { - if (!Packets.MemorySemaphorePacket.TrySignal(_host.TryReadGuest, _host.Memory, semaphore.Address, semaphore.WriteSignal, out _)) + if (!Packets.MemorySemaphorePacket.TrySignal(_host.TryReadGuestOperand, _host.Memory, semaphore.Address, semaphore.WriteSignal, out _)) { throw _host.Fatal($"The semaphore cannot be signaled: address=0x{semaphore.Address:X16}."); } @@ -77,7 +77,7 @@ public sealed partial class GpuCommandInterpreter } // A wait takes one token, or suspends until a signal adds one. - if (!Packets.MemorySemaphorePacket.TryConsume(_host.TryReadGuest, _host.Memory, semaphore.Address, out var priorValue)) + if (!Packets.MemorySemaphorePacket.TryConsume(_host.TryReadGuestOperand, _host.Memory, semaphore.Address, out var priorValue)) { throw _host.Fatal($"The semaphore cannot be read: address=0x{semaphore.Address:X16}."); } diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs index 3b98876..1fec559 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs @@ -11,7 +11,6 @@ namespace SharpEmu.Libs.Gpu.GpuCommands; public sealed partial class GpuCommandInterpreter { private const uint DmaSkippedDestination = 0x3022C; - private const uint LodStatsWithoutCounters = 1; // Destination selectors 1, 2, 4 and 5 are memory; registers and GDS are not supported. internal void WriteData(ulong destination, ReadOnlySpan source, uint writeControl) @@ -367,15 +366,16 @@ public sealed partial class GpuCommandInterpreter var bufferSize = payload[0]; var destination = (payload[1] & 0xFFFF_FFC0u) | ((ulong)payload[2] << 32); + if (destination != 0 && bufferSize != 0 && _host.TryRecordLodStatistics(destination, bufferSize)) + { + return 4; + } + if (destination != 0 && bufferSize != 0) { var report = new byte[bufferSize]; - var hasCounters = MipStatistics.Shared.TryWriteReport(report); + MipStatistics.Shared.WriteReport(report); WriteBytes(destination, report); - if (bufferSize >= sizeof(uint)) - { - WriteDword(destination, hasCounters ? MipStatistics.FirstCounterOffset : LodStatsWithoutCounters); - } } return 4; diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs index a168128..9b42efc 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs @@ -82,8 +82,8 @@ public sealed partial class GpuCommandInterpreter var next = packet.PacketAddress + (ulong)packet.Length * sizeof(uint); var expectedHeader = PacketHeader.Make(centroidPacketDwords, PacketOpcode.SetContextRegister); if (packet.Remaining < packet.Length + centroidPacketDwords || - ReadDword(next) != expectedHeader || - RegisterBankLayout.Normalize(ReadDword(next + sizeof(uint))) != ContextRegisterOffset.PaScCentroidPriority0) + ReadDword(next, RenderPhaseProfile.CommandReadKind.Payload) != expectedHeader || + RegisterBankLayout.Normalize(ReadDword(next + sizeof(uint), RenderPhaseProfile.CommandReadKind.Payload)) != ContextRegisterOffset.PaScCentroidPriority0) { throw _host.Fatal($"The sample location packet is not followed by the centroid priority packet: header=0x{packet.Header:X8} address=0x{packet.PacketAddress:X16} next=0x{next:X16}."); } @@ -102,16 +102,16 @@ public sealed partial class GpuCommandInterpreter var address = packet.PacketAddress; var registerHeader = PacketHeader.Make(3, PacketOpcode.SetContextRegister); - if (ReadDword(address + 40) != registerHeader || ReadDword(address + 44) != RegisterBankLayout.DepthInfo || - ReadDword(address + 52) != registerHeader || ReadDword(address + 56) != RegisterBankLayout.DepthView || - ReadDword(address + 64) != registerHeader || ReadDword(address + 68) != RegisterBankLayout.HtileDataBase || - ReadDword(address + 76) != registerHeader || ReadDword(address + 80) != RegisterBankLayout.HtileSurface || - ReadDword(address + 88) != PacketHeader.Make(2, PacketOpcode.Nop)) + if (ReadDword(address + 40, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 44, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.DepthInfo || + ReadDword(address + 52, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 56, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.DepthView || + ReadDword(address + 64, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 68, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.HtileDataBase || + ReadDword(address + 76, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 80, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.HtileSurface || + ReadDword(address + 88, RenderPhaseProfile.CommandReadKind.Payload) != PacketHeader.Make(2, PacketOpcode.Nop)) { return; } - var sizeXy = ReadDword(address + 92); + var sizeXy = ReadDword(address + 92, RenderPhaseProfile.CommandReadKind.Payload); if (sizeXy != 0) { TypedRegisters.CompositeDepthSizeXy = sizeXy; diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs index 7e6f8fd..97b7928 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs @@ -28,7 +28,7 @@ public sealed partial class GpuCommandInterpreter return; } - var value = is64Bit ? ReadQword(address) : ReadDword(address); + var value = ReadWaitOperand(address, is64Bit); if (!WaitOperation.TryCompare(value, reference, mask, compareFunction, out var satisfied)) { throw _host.Fatal($"The wait compare function is unknown: function={compareFunction} address=0x{address:X16}."); diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs index ef91e56..d10d372 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs @@ -394,7 +394,12 @@ public sealed partial class GpuCommandInterpreter { RenderPhaseProfile.RecordCommandRead(readKind, sizeof(uint)); Span bytes = stackalloc byte[sizeof(uint)]; - if (!_host.TryReadGuest(address, bytes)) + // The packets themselves are command data; anything else is an operand earlier work may write. + var read = readKind is RenderPhaseProfile.CommandReadKind.Header or RenderPhaseProfile.CommandReadKind.Payload or + RenderPhaseProfile.CommandReadKind.RegisterTable + ? _host.TryReadGuest(address, bytes) + : _host.TryReadGuestOperand(address, bytes); + if (!read) { throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size=4."); } @@ -406,7 +411,7 @@ public sealed partial class GpuCommandInterpreter { RenderPhaseProfile.RecordCommandRead(RenderPhaseProfile.CommandReadKind.Operand64, sizeof(ulong)); Span bytes = stackalloc byte[sizeof(ulong)]; - if (!_host.TryReadGuest(address, bytes)) + if (!_host.TryReadGuestOperand(address, bytes)) { throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size=8."); } @@ -414,10 +419,24 @@ public sealed partial class GpuCommandInterpreter return BinaryPrimitives.ReadUInt64LittleEndian(bytes); } + private ulong ReadWaitOperand(ulong address, bool is64Bit) + { + RenderPhaseProfile.RecordCommandRead(is64Bit ? RenderPhaseProfile.CommandReadKind.Operand64 : RenderPhaseProfile.CommandReadKind.Operand32, + is64Bit ? sizeof(ulong) : sizeof(uint)); + Span bytes = stackalloc byte[sizeof(ulong)]; + var size = is64Bit ? sizeof(ulong) : sizeof(uint); + if (!_host.TryReadGuestWaitOperand(address, bytes[..size])) + { + throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size={size}."); + } + + return is64Bit ? BinaryPrimitives.ReadUInt64LittleEndian(bytes) : BinaryPrimitives.ReadUInt32LittleEndian(bytes); + } + internal void ReadBytes(ulong address, Span destination) { RenderPhaseProfile.RecordCommandRead(RenderPhaseProfile.CommandReadKind.Other, destination.Length); - if (!_host.TryReadGuest(address, destination)) + if (!_host.TryReadGuestOperand(address, destination)) { throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size={destination.Length}."); } diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs index 6196265..f1e0b9d 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs @@ -81,9 +81,19 @@ public interface ICommandStreamHost { ICpuMemory Memory { get; } + bool TryRecordLodStatistics(ulong destination, uint size) => false; + // Reads guest memory the GPU may have written; the host synchronizes GPU-owned pages first. bool TryReadGuest(ulong address, Span destination); + // Reads a packet operand (a label, a predicate, indirect arguments): memory earlier work may + // write, unlike the packets themselves. A host that defers work completes it first. + bool TryReadGuestOperand(ulong address, Span destination) => TryReadGuest(address, destination); + + // Reads the label a wait polls. A stale value only suspends the wait until a retry, so a host + // that runs work behind the stream need not complete it first. + bool TryReadGuestWaitOperand(ulong address, Span destination) => TryReadGuestOperand(address, destination); + // Runs commands other threads posted to this worker. Called before every packet. void RunPendingCommands(); @@ -92,6 +102,10 @@ public interface ICommandStreamHost void Flush(); + // A frame boundary: submits what the frame recorded. The submitter resumes once it returns, so a + // host that runs the stream ahead of its recording returns only after the frame is recorded. + void FlushFrame() => Flush(); + void FlushAndWait(); void SynchronizeGpu(); diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs index 5ad7830..1e6f6a5 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs @@ -338,8 +338,15 @@ public sealed class ScreenViewportRegisters public ScreenViewportRegisters Copy() { - var copy = (ScreenViewportRegisters)MemberwiseClone(); + var copy = CopySharingViewports(); copy.Viewports = (ViewportRegisters[])Viewports.Clone(); + return copy; + } + + // A copy that shares the viewport array, the largest part, until RegisterBanks copies it. + public ScreenViewportRegisters CopySharingViewports() + { + var copy = (ScreenViewportRegisters)MemberwiseClone(); copy.ClipRectangleLeft = (int[])ClipRectangleLeft.Clone(); copy.ClipRectangleTop = (int[])ClipRectangleTop.Clone(); copy.ClipRectangleRight = (int[])ClipRectangleRight.Clone(); @@ -388,6 +395,9 @@ public sealed class ContextRegisters public EnhancedQualityAntialiasingRegisters EnhancedQualityAntialiasing; public ShaderInterfaceRegisters ShaderInterface = new(); + // A copy that shares every nested part; RegisterBanks copies a part before writing it. + public ContextRegisters ShallowCopy() => (ContextRegisters)MemberwiseClone(); + public uint RenderTargetMaskForSlot(uint slot) => (RenderTargetMask >> (int)(slot * 4)) & 0xFu; public ContextRegisters Copy() diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs index 8715653..7c20201 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs @@ -1,6 +1,8 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using SharpEmu.Libs.Gpu.Images; + namespace SharpEmu.Libs.Gpu.GpuCommands.Registers; // The three typed register banks of one queue, with the state the register writers carry between packets. @@ -9,15 +11,31 @@ public sealed class RegisterBanks private ContextRegisters? _savedContext; private uint? _savedCompositeDepthSizeXy; - public RegisterBanks(Func fatal) => Fatal = fatal; + public RegisterBanks(Func fatal) + : this(fatal, new ContextRegisters(), new ShaderProgramRegisters(), new UserConfigRegisters()) + { + } + + // Banks over given parts, so a copy or a snapshot does not first build empty ones. + private RegisterBanks( + Func fatal, + ContextRegisters context, + ShaderProgramRegisters shader, + UserConfigRegisters userConfig) + { + Fatal = fatal; + Context = context; + Shader = shader; + UserConfig = userConfig; + } public Func Fatal { get; } - public ContextRegisters Context { get; private set; } = new(); + public ContextRegisters Context { get; private set; } - public ShaderProgramRegisters Shader { get; private set; } = new(); + public ShaderProgramRegisters Shader { get; private set; } - public UserConfigRegisters UserConfig { get; private set; } = new(); + public UserConfigRegisters UserConfig { get; private set; } public bool ContextPushed => _savedContext is not null; @@ -31,11 +49,8 @@ public sealed class RegisterBanks // A copy of the live banks; the saved context and the draw-time markers are not part of it. public RegisterBanks Clone() { - var clone = new RegisterBanks(Fatal) + var clone = new RegisterBanks(Fatal, Context.Copy(), Shader.Copy(), UserConfig.Copy()) { - Context = Context.Copy(), - Shader = Shader.Copy(), - UserConfig = UserConfig.Copy(), CompositeDepthSizeXy = CompositeDepthSizeXy, UserDataMarker = UserDataMarker, IndexTypeAndSize = IndexTypeAndSize, @@ -43,9 +58,281 @@ public sealed class RegisterBanks return clone; } + // The parts a snapshot shares with these banks; each is copied before its next write here. + private bool _contextShared; + private bool _vertexShared; + private bool _pixelShared; + private bool _computeShared; + private bool _userConfigShared; + + // A read-only copy that shares every part with the live banks until they next write it: a + // deferred draw takes one per draw, and between draws only a few parts change. + public RegisterBanks Snapshot() + { + var snapshot = new RegisterBanks(Fatal, Context, Shader.ShareStages(), UserConfig) + { + CompositeDepthSizeXy = CompositeDepthSizeXy, + UserDataMarker = UserDataMarker, + IndexTypeAndSize = IndexTypeAndSize, + }; + _contextShared = _vertexShared = _pixelShared = _computeShared = _userConfigShared = true; + _pixelScalarsShared = _hullScalarsShared = _geometryScalarsShared = _legacyVertexScalarsShared = _exportScalarsShared = _computeScalarsShared = true; + return snapshot; + } + + // Called by the register writers before they store into a part. + public void PrepareContextWrite() + { + if (_contextShared) + { + Context = Context.ShallowCopy(); + _contextShared = false; + SetContextPartsShared(true); + } + } + + // The context is copied without its nested parts, and a part only when a writer asks for it + // through these accessors: a draw usually rewrites a few plain fields. + private bool _screenViewportShared; + private bool _viewportsShared; + private bool _sampleLocationsShared; + private bool _shaderInterfaceShared; + private bool _blendControlsShared; + private bool _colorTargetsShared; + private bool _colorClearWord1Shared; + private bool _unmodeledShared; + + private void SetContextPartsShared(bool shared) => + _screenViewportShared = _viewportsShared = _sampleLocationsShared = _shaderInterfaceShared = + _blendControlsShared = _colorTargetsShared = _colorClearWord1Shared = _unmodeledShared = shared; + + private void OwnContext(ContextRegisters context) + { + Context = context; + _contextShared = false; + SetContextPartsShared(false); + } + + public ScreenViewportRegisters ScreenViewportForWrite + { + get + { + PrepareContextWrite(); + if (_screenViewportShared) + { + Context.ScreenViewport = Context.ScreenViewport.CopySharingViewports(); + _screenViewportShared = false; + } + + return Context.ScreenViewport; + } + } + + public ViewportRegisters[] ViewportsForWrite + { + get + { + var screenViewport = ScreenViewportForWrite; + if (_viewportsShared) + { + screenViewport.Viewports = (ViewportRegisters[])screenViewport.Viewports.Clone(); + _viewportsShared = false; + } + + return screenViewport.Viewports; + } + } + + public SampleLocationRegisters SampleLocationsForWrite + { + get + { + PrepareContextWrite(); + if (_sampleLocationsShared) + { + Context.SampleLocations = Context.SampleLocations.Copy(); + _sampleLocationsShared = false; + } + + return Context.SampleLocations; + } + } + + public ShaderInterfaceRegisters ShaderInterfaceForWrite + { + get + { + PrepareContextWrite(); + if (_shaderInterfaceShared) + { + Context.ShaderInterface = Context.ShaderInterface.Copy(); + _shaderInterfaceShared = false; + } + + return Context.ShaderInterface; + } + } + + public BlendRegisters[] BlendControlsForWrite + { + get + { + PrepareContextWrite(); + if (_blendControlsShared) + { + Context.BlendControls = (BlendRegisters[])Context.BlendControls.Clone(); + _blendControlsShared = false; + } + + return Context.BlendControls; + } + } + + public ColorTargetWords[] ColorTargetsForWrite + { + get + { + PrepareContextWrite(); + if (_colorTargetsShared) + { + Context.ColorTargets = (ColorTargetWords[])Context.ColorTargets.Clone(); + _colorTargetsShared = false; + } + + return Context.ColorTargets; + } + } + + public uint[] ColorClearWord1ForWrite + { + get + { + PrepareContextWrite(); + if (_colorClearWord1Shared) + { + Context.ColorClearWord1 = (uint[])Context.ColorClearWord1.Clone(); + _colorClearWord1Shared = false; + } + + return Context.ColorClearWord1; + } + } + + public Dictionary UnmodeledTableRegistersForWrite + { + get + { + PrepareContextWrite(); + if (_unmodeledShared) + { + Context.UnmodeledTableRegisters = new Dictionary(Context.UnmodeledTableRegisters); + _unmodeledShared = false; + } + + return Context.UnmodeledTableRegisters; + } + } + + public void PrepareUserConfigWrite() + { + if (_userConfigShared) + { + UserConfig = UserConfig.Copy(); + _userConfigShared = false; + } + } + + // COMPUTE_DISPATCH_INITIATOR, the first register of the compute range. + private const uint ComputeRangeStart = 0x200; + + // A stage is copied without its user scalar sets, and a set only when its window is written: + // a draw usually rewrites one set of one or two stages. + private bool _pixelScalarsShared; + private bool _hullScalarsShared; + private bool _geometryScalarsShared; + private bool _legacyVertexScalarsShared; + private bool _exportScalarsShared; + private bool _computeScalarsShared; + private const uint UserScalarWindow = UserScalarRegisters.Capacity; + + private static bool Overlaps(uint first, uint last, uint windowStart) => + first < windowStart + UserScalarWindow && last >= windowStart; + + // The shader bank's offsets: pixel below SPI_SHADER_PGM_LO_VS, the geometry stages up to the + // compute range, compute from there. + public void PrepareShaderWrite(uint offset, uint count) + { + var last = offset + Math.Max(count, 1) - 1; + if (offset < ShaderRegisterOffset.SpiShaderPgmLoVs) + { + if (_pixelShared) + { + Shader.Pixel = Shader.Pixel.ShallowCopy(); + _pixelShared = false; + } + + if (_pixelScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataPs0)) + { + Shader.Pixel.UserScalars = Shader.Pixel.UserScalars.Copy(); + _pixelScalarsShared = false; + } + } + + if (offset < ComputeRangeStart && last >= ShaderRegisterOffset.SpiShaderPgmLoVs) + { + if (_vertexShared) + { + Shader.Vertex = Shader.Vertex.ShallowCopy(); + _vertexShared = false; + } + + var vertex = Shader.Vertex; + if (_hullScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataHs0)) + { + vertex.HullUserScalars = vertex.HullUserScalars.Copy(); + _hullScalarsShared = false; + } + + if (_geometryScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataGs0)) + { + vertex.GeometryUserScalars = vertex.GeometryUserScalars.Copy(); + _geometryScalarsShared = false; + } + + if (_legacyVertexScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataVs0)) + { + vertex.LegacyVertexUserScalars = vertex.LegacyVertexUserScalars.Copy(); + _legacyVertexScalarsShared = false; + } + + if (_exportScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataEs0)) + { + vertex.ExportUserScalars = vertex.ExportUserScalars.Copy(); + _exportScalarsShared = false; + } + } + + if (last >= ComputeRangeStart) + { + if (_computeShared) + { + Shader.Compute = Shader.Compute.ShallowCopy(); + _computeShared = false; + } + + if (_computeScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.ComputeUserData0)) + { + Shader.Compute.UserScalars = Shader.Compute.UserScalars.Copy(); + _computeScalarsShared = false; + } + } + } + public void Reset() { - Context = new ContextRegisters(); + _contextShared = _vertexShared = _pixelShared = _computeShared = _userConfigShared = false; + _pixelScalarsShared = _hullScalarsShared = _geometryScalarsShared = _legacyVertexScalarsShared = _exportScalarsShared = _computeScalarsShared = false; + OwnContext(new ContextRegisters()); Shader = new ShaderProgramRegisters(); UserConfig = new UserConfigRegisters(); _savedContext = null; @@ -61,7 +348,7 @@ public sealed class RegisterBanks switch (operation) { case ContextStateOperation.Clear: - Context = new ContextRegisters(); + OwnContext(new ContextRegisters()); CompositeDepthSizeXy = null; break; case ContextStateOperation.Push: @@ -73,14 +360,14 @@ public sealed class RegisterBanks throw Fatal("The context state is not pushed."); } - Context = _savedContext; + OwnContext(_savedContext); CompositeDepthSizeXy = _savedCompositeDepthSizeXy; _savedContext = null; _savedCompositeDepthSizeXy = null; break; case ContextStateOperation.PushClear: Push(); - Context = new ContextRegisters(); + OwnContext(new ContextRegisters()); CompositeDepthSizeXy = null; break; default: diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs index 3592016..a7258f8 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs @@ -28,6 +28,7 @@ public static class RegisterWriteTable public static uint WriteContextPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) { + banks.PrepareContextWrite(); var consumed = WritePacket(banks, in packet, offset, values, Context, ContextIndirect, "context", emptyIsHandled: true); if (Rendering.RenderTrace.Enabled) { @@ -43,6 +44,7 @@ public static class RegisterWriteTable // A shader packet without values has no writer; the other banks accept it. public static uint WriteShaderPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) { + banks.PrepareShaderWrite(offset, (uint)values.Length); var consumed = WritePacket(banks, in packet, offset, values, Shader, ShaderIndirect, "shader", emptyIsHandled: false); if (Rendering.RenderTrace.Enabled) { @@ -52,15 +54,19 @@ public static class RegisterWriteTable return consumed; } - public static uint WriteUserConfigPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) => - WritePacket(banks, in packet, offset, values, UserConfig, UserConfigIndirect, "user-config", emptyIsHandled: true); + public static uint WriteUserConfigPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) + { + banks.PrepareUserConfigWrite(); + return WritePacket(banks, in packet, offset, values, UserConfig, UserConfigIndirect, "user-config", emptyIsHandled: true); + } public static void WriteContextEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) { + banks.PrepareContextWrite(); if (offset < ContextIndirect.Length && ContextIndirect[offset] is null) { // Keep unknown in-bank table values so a later renderer implementation can decode them. - banks.Context.UnmodeledTableRegisters[offset] = value; + banks.UnmodeledTableRegistersForWrite[offset] = value; } else { @@ -88,6 +94,7 @@ public static class RegisterWriteTable public static void WriteShaderEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) { + banks.PrepareShaderWrite(offset, 1); WriteEntry(banks, offset, value, ShaderIndirect, "shader", tableAddress); if (Rendering.RenderTrace.Enabled) TracePixelRegisterWrite(banks, offset, value, tableAddress, "table"); @@ -104,8 +111,11 @@ public static class RegisterWriteTable $"value=0x{value:X8} shader=0x{banks.Shader.Pixel.Address:X16}"); } - public static void WriteUserConfigEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) => + public static void WriteUserConfigEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) + { + banks.PrepareUserConfigWrite(); WriteEntry(banks, offset, value, UserConfigIndirect, "user-config", tableAddress); + } private static uint WritePacket( RegisterBanks banks, diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs index ea46226..706e224 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs @@ -98,8 +98,8 @@ internal static partial class RegisterWriters { indirect[offset] = static (banks, register, value) => { - banks.Context.ShaderInterface.PixelInterpolatorSettings[register - SpiPsInputCntl0] = value; - banks.Context.ShaderInterface.PixelInterpolatorWritten |= 1u << (int)(register - SpiPsInputCntl0); + banks.ShaderInterfaceForWrite.PixelInterpolatorSettings[register - SpiPsInputCntl0] = value; + banks.ShaderInterfaceForWrite.PixelInterpolatorWritten |= 1u << (int)(register - SpiPsInputCntl0); }; } @@ -160,7 +160,7 @@ internal static partial class RegisterWriters for (var offset = PaScAaSampleLocations0; offset < PaScAaSampleLocations0 + SampleLocationRegisters.LocationCount; offset++) { - indirect[offset] = static (banks, register, value) => banks.Context.SampleLocations.Locations[register - PaScAaSampleLocations0] = value; + indirect[offset] = static (banks, register, value) => banks.SampleLocationsForWrite.Locations[register - PaScAaSampleLocations0] = value; } foreach (var offset in new[] @@ -175,36 +175,36 @@ internal static partial class RegisterWriters indirect[offset] = IgnoreEntry; } - indirect[SpiVsOutConfig] = static (banks, _, value) => banks.Context.ShaderInterface.VertexOutputConfiguration = value; + indirect[SpiVsOutConfig] = static (banks, _, value) => banks.ShaderInterfaceForWrite.VertexOutputConfiguration = value; indirect[DbRenderOverride] = static (banks, _, value) => banks.Context.DepthRenderOverride = DepthRenderOverrideRegisters.Decode(value); - indirect[SpiShaderPosFormat] = static (banks, _, value) => banks.Context.ShaderInterface.PositionExportFormat = value; - indirect[SpiShaderIdxFormat] = static (banks, _, value) => banks.Context.ShaderInterface.IndexExportFormat = value; - indirect[PaClVsOutCntl] = static (banks, _, value) => banks.Context.ShaderInterface.VertexOutputControl = value; - indirect[GeNggSubgroupCntl] = static (banks, _, value) => banks.Context.ShaderInterface.PrimitiveShaderSubgroupControl = value; - indirect[VgtGsInstanceCnt] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryInstanceCount = value; - indirect[VgtGsOnchipCntl] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryOnChipControl = value; - indirect[VgtHosMaxTessLevel] = static (banks, _, value) => banks.Context.ShaderInterface.MaxTessellationLevel = value; - indirect[VgtHosMinTessLevel] = static (banks, _, value) => banks.Context.ShaderInterface.MinTessellationLevel = value; - indirect[GeMaxOutputPerSubgroup] = static (banks, _, value) => banks.Context.ShaderInterface.MaxOutputPerSubgroup = value; - indirect[VgtEsgsRingItemSize] = static (banks, _, value) => banks.Context.ShaderInterface.ExportRingItemSize = value; - indirect[VgtGsMaxVertOut] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryMaxVerticesOut = value; - indirect[VgtPrimitiveIdEn] = static (banks, _, value) => banks.Context.ShaderInterface.PrimitiveIdEnable = value; - indirect[VgtReuseOff] = static (banks, _, value) => banks.Context.ShaderInterface.VertexReuseOff = value; + indirect[SpiShaderPosFormat] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PositionExportFormat = value; + indirect[SpiShaderIdxFormat] = static (banks, _, value) => banks.ShaderInterfaceForWrite.IndexExportFormat = value; + indirect[PaClVsOutCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.VertexOutputControl = value; + indirect[GeNggSubgroupCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PrimitiveShaderSubgroupControl = value; + indirect[VgtGsInstanceCnt] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryInstanceCount = value; + indirect[VgtGsOnchipCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryOnChipControl = value; + indirect[VgtHosMaxTessLevel] = static (banks, _, value) => banks.ShaderInterfaceForWrite.MaxTessellationLevel = value; + indirect[VgtHosMinTessLevel] = static (banks, _, value) => banks.ShaderInterfaceForWrite.MinTessellationLevel = value; + indirect[GeMaxOutputPerSubgroup] = static (banks, _, value) => banks.ShaderInterfaceForWrite.MaxOutputPerSubgroup = value; + indirect[VgtEsgsRingItemSize] = static (banks, _, value) => banks.ShaderInterfaceForWrite.ExportRingItemSize = value; + indirect[VgtGsMaxVertOut] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryMaxVerticesOut = value; + indirect[VgtPrimitiveIdEn] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PrimitiveIdEnable = value; + indirect[VgtReuseOff] = static (banks, _, value) => banks.ShaderInterfaceForWrite.VertexReuseOff = value; indirect[VgtMultiPrimIbResetIndex] = static (banks, _, value) => banks.Context.PrimitiveResetIndex = value; - indirect[VgtTessDistribution] = static (banks, _, value) => banks.Context.ShaderInterface.TessellationDistribution = value; + indirect[VgtTessDistribution] = static (banks, _, value) => banks.ShaderInterfaceForWrite.TessellationDistribution = value; indirect[VgtShaderStagesEn] = static (banks, _, value) => banks.Context.ShaderStages = value; - indirect[VgtLsHsConfig] = static (banks, _, value) => banks.Context.ShaderInterface.LocalHullConfiguration = value; - indirect[VgtGsOutPrimType] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryOutputPrimitiveType = value; - indirect[VgtTfParam] = static (banks, _, value) => banks.Context.ShaderInterface.TessellationFactorParameter = value; - indirect[SpiShaderZFormat] = static (banks, _, value) => banks.Context.ShaderInterface.DepthExportFormat = value; + indirect[VgtLsHsConfig] = static (banks, _, value) => banks.ShaderInterfaceForWrite.LocalHullConfiguration = value; + indirect[VgtGsOutPrimType] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryOutputPrimitiveType = value; + indirect[VgtTfParam] = static (banks, _, value) => banks.ShaderInterfaceForWrite.TessellationFactorParameter = value; + indirect[SpiShaderZFormat] = static (banks, _, value) => banks.ShaderInterfaceForWrite.DepthExportFormat = value; indirect[SpiShaderColFormat] = ShaderColorFormatEntry; - indirect[SpiPsInputEna] = static (banks, _, value) => banks.Context.ShaderInterface.PixelInputEnable = value; - indirect[SpiPsInputAddr] = static (banks, _, value) => banks.Context.ShaderInterface.PixelInputAddress = value; - indirect[SpiPsInControl] = static (banks, _, value) => banks.Context.ShaderInterface.PixelInputControl = value; - indirect[SpiBarycCntl] = static (banks, _, value) => banks.Context.ShaderInterface.BarycentricControl = value; - indirect[DbShaderControl] = static (banks, _, value) => banks.Context.ShaderInterface.DepthShaderControl = DepthShaderControlRegisters.Decode(value); - indirect[CbShaderMask] = static (banks, _, value) => banks.Context.ShaderInterface.ColorShaderMask = value; - indirect[PaScShaderControl] = static (banks, _, value) => banks.Context.ShaderInterface.ScanShaderControl = value; + indirect[SpiPsInputEna] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PixelInputEnable = value; + indirect[SpiPsInputAddr] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PixelInputAddress = value; + indirect[SpiPsInControl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PixelInputControl = value; + indirect[SpiBarycCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.BarycentricControl = value; + indirect[DbShaderControl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.DepthShaderControl = DepthShaderControlRegisters.Decode(value); + indirect[CbShaderMask] = static (banks, _, value) => banks.ShaderInterfaceForWrite.ColorShaderMask = value; + indirect[PaScShaderControl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.ScanShaderControl = value; indirect[CbTargetMask] = static (banks, _, value) => banks.Context.RenderTargetMask = value; indirect[PaScScreenScissorTl] = ScreenScissorTopLeftEntry; indirect[PaScScreenScissorBr] = ScreenScissorBottomRightEntry; @@ -248,7 +248,7 @@ internal static partial class RegisterWriters indirect[DbStencilRefMaskBack] = StencilMaskBackEntry; indirect[PaClClipCntl] = static (banks, _, value) => banks.Context.Clip = ClipControlRegisters.Decode(value); indirect[PaSuLineCntl] = LineControlEntry; - indirect[PaClVteCntl] = static (banks, _, value) => banks.Context.ScreenViewport.TransformControl = value; + indirect[PaClVteCntl] = static (banks, _, value) => banks.ScreenViewportForWrite.TransformControl = value; indirect[PaScModeCntl0] = static (banks, _, value) => banks.Context.ScanMode = ScanModeRegisters.Decode(value); indirect[PaScAaConfig] = static (banks, _, value) => banks.Context.AntialiasingConfig = AntialiasingConfigRegisters.Decode(value); indirect[PaScCentroidPriority0] = CentroidPriorityEntry; @@ -369,8 +369,8 @@ internal static partial class RegisterWriters throw PacketError(banks, "The pixel interpolator packet has an invalid count", in packet, offset, values.Length); } - values.CopyTo(banks.Context.ShaderInterface.PixelInterpolatorSettings); - banks.Context.ShaderInterface.PixelInterpolatorWritten |= (uint)((1ul << values.Length) - 1); + values.CopyTo(banks.ShaderInterfaceForWrite.PixelInterpolatorSettings); + banks.ShaderInterfaceForWrite.PixelInterpolatorWritten |= (uint)((1ul << values.Length) - 1); return (uint)values.Length; } @@ -410,7 +410,7 @@ internal static partial class RegisterWriters // One location at any slot, or all sixteen from the first slot. private static uint SampleLocationsPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) { - var locations = banks.Context.SampleLocations.Locations; + var locations = banks.SampleLocationsForWrite.Locations; if (values.Length == 1) { locations[offset - PaScAaSampleLocations0] = values[0]; @@ -506,19 +506,19 @@ internal static partial class RegisterWriters private static void HardwareScreenOffsetEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.HardwareOffsetX = RegisterField.Get(value, 0, 0x1FF); - banks.Context.ScreenViewport.HardwareOffsetY = RegisterField.Get(value, 16, 0x1FF); + banks.ScreenViewportForWrite.HardwareOffsetX = RegisterField.Get(value, 0, 0x1FF); + banks.ScreenViewportForWrite.HardwareOffsetY = RegisterField.Get(value, 16, 0x1FF); } private static void WindowOffsetEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.WindowOffsetX = RegisterField.Coordinate16(value); - banks.Context.ScreenViewport.WindowOffsetY = RegisterField.Coordinate16(value >> 16); + banks.ScreenViewportForWrite.WindowOffsetX = RegisterField.Coordinate16(value); + banks.ScreenViewportForWrite.WindowOffsetY = RegisterField.Coordinate16(value >> 16); } private static void ClipRectangleEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; if (offset == PaScClipRectRule) { viewport.ClipRectangleRule = (ushort)(value & 0xFFFFu); @@ -598,7 +598,7 @@ internal static partial class RegisterWriters private static void CentroidPriorityEntry(RegisterBanks banks, uint offset, uint value) { - var locations = banks.Context.SampleLocations; + var locations = banks.SampleLocationsForWrite; switch (offset) { case PaScCentroidPriority0: @@ -612,7 +612,7 @@ internal static partial class RegisterWriters private static void ShaderColorFormatEntry(RegisterBanks banks, uint offset, uint value) { - var modes = banks.Context.ShaderInterface.TargetOutputModes; + var modes = banks.ShaderInterfaceForWrite.TargetOutputModes; for (var index = 0; index < modes.Length; index++) { modes[index] = (byte)((value >> (index * 4)) & 0xFu); @@ -621,19 +621,19 @@ internal static partial class RegisterWriters private static void ScreenScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.ScreenScissorLeft = RegisterField.Coordinate16(value); - banks.Context.ScreenViewport.ScreenScissorTop = RegisterField.Coordinate16(value >> 16); + banks.ScreenViewportForWrite.ScreenScissorLeft = RegisterField.Coordinate16(value); + banks.ScreenViewportForWrite.ScreenScissorTop = RegisterField.Coordinate16(value >> 16); } private static void ScreenScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.ScreenScissorRight = RegisterField.Coordinate16(value); - banks.Context.ScreenViewport.ScreenScissorBottom = RegisterField.Coordinate16(value >> 16); + banks.ScreenViewportForWrite.ScreenScissorRight = RegisterField.Coordinate16(value); + banks.ScreenViewportForWrite.ScreenScissorBottom = RegisterField.Coordinate16(value >> 16); } private static void GenericScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; viewport.GenericScissorLeft = RegisterField.Coordinate15(value); viewport.GenericScissorTop = RegisterField.Coordinate15(value >> 16); viewport.GenericScissorWindowOffsetEnable = !RegisterField.Bit(value, 31); @@ -641,13 +641,13 @@ internal static partial class RegisterWriters private static void GenericScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.GenericScissorRight = RegisterField.Coordinate15(value); - banks.Context.ScreenViewport.GenericScissorBottom = RegisterField.Coordinate15(value >> 16); + banks.ScreenViewportForWrite.GenericScissorRight = RegisterField.Coordinate15(value); + banks.ScreenViewportForWrite.GenericScissorBottom = RegisterField.Coordinate15(value >> 16); } private static void WindowScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; viewport.WindowScissorLeft = RegisterField.Coordinate15(value); viewport.WindowScissorTop = RegisterField.Coordinate15(value >> 16); viewport.WindowScissorWindowOffsetEnable = !RegisterField.Bit(value, 31); @@ -655,13 +655,13 @@ internal static partial class RegisterWriters private static void WindowScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.WindowScissorRight = RegisterField.Coordinate15(value); - banks.Context.ScreenViewport.WindowScissorBottom = RegisterField.Coordinate15(value >> 16); + banks.ScreenViewportForWrite.WindowScissorRight = RegisterField.Coordinate15(value); + banks.ScreenViewportForWrite.WindowScissorBottom = RegisterField.Coordinate15(value >> 16); } private static void ViewportScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - ref var viewport = ref banks.Context.ScreenViewport.Viewports[(offset - PaScViewportScissor0Tl) / 2]; + ref var viewport = ref banks.ViewportsForWrite[(offset - PaScViewportScissor0Tl) / 2]; viewport.ScissorLeft = RegisterField.Coordinate15(value); viewport.ScissorTop = RegisterField.Coordinate15(value >> 16); viewport.ScissorWindowOffsetEnable = !RegisterField.Bit(value, 31); @@ -669,21 +669,21 @@ internal static partial class RegisterWriters private static void ViewportScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - ref var viewport = ref banks.Context.ScreenViewport.Viewports[(offset - PaScViewportScissor0Br) / 2]; + ref var viewport = ref banks.ViewportsForWrite[(offset - PaScViewportScissor0Br) / 2]; viewport.ScissorRight = RegisterField.Coordinate15(value); viewport.ScissorBottom = RegisterField.Coordinate15(value >> 16); } private static void ViewportZMinEntry(RegisterBanks banks, uint offset, uint value) => - banks.Context.ScreenViewport.Viewports[(offset - PaScViewportZMin0) / 2].MinDepth = RegisterField.AsFloat(value); + banks.ViewportsForWrite[(offset - PaScViewportZMin0) / 2].MinDepth = RegisterField.AsFloat(value); private static void ViewportZMaxEntry(RegisterBanks banks, uint offset, uint value) => - banks.Context.ScreenViewport.Viewports[(offset - PaScViewportZMax0) / 2].MaxDepth = RegisterField.AsFloat(value); + banks.ViewportsForWrite[(offset - PaScViewportZMax0) / 2].MaxDepth = RegisterField.AsFloat(value); private static void ViewportScaleOffsetEntry(RegisterBanks banks, uint offset, uint value) { var relative = offset - PaClViewportXScale; - ref var viewport = ref banks.Context.ScreenViewport.Viewports[relative / ViewportFieldCount]; + ref var viewport = ref banks.ViewportsForWrite[relative / ViewportFieldCount]; var number = RegisterField.AsFloat(value); switch (relative % ViewportFieldCount) { @@ -718,7 +718,7 @@ internal static partial class RegisterWriters private static void GuardBandEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; var number = RegisterField.AsFloat(value); switch (offset) { @@ -760,7 +760,7 @@ internal static partial class RegisterWriters } private static void BlendControlEntry(RegisterBanks banks, uint offset, uint value) => - banks.Context.BlendControls[offset - CbBlend0Control] = BlendRegisters.Decode(value); + banks.BlendControlsForWrite[offset - CbBlend0Control] = BlendRegisters.Decode(value); private static void DepthZInfoEntry(RegisterBanks banks, uint offset, uint value) { @@ -804,7 +804,7 @@ internal static partial class RegisterWriters } private static ref ColorTargetWords ColorTarget(RegisterBanks banks, uint offset, uint first, uint stride) => - ref banks.Context.ColorTargets[(offset - first) / stride]; + ref banks.ColorTargetsForWrite[(offset - first) / stride]; private static void ColorBaseLowEntry(RegisterBanks banks, uint offset, uint value) { @@ -878,7 +878,7 @@ internal static partial class RegisterWriters } private static void ColorClearWord1Entry(RegisterBanks banks, uint offset, uint value) => - banks.Context.ColorClearWord1[(offset - CbColor0ClearWord1) / ColorSlotStride] = value; + banks.ColorClearWord1ForWrite[(offset - CbColor0ClearWord1) / ColorSlotStride] = value; private static void ColorDccBaseLowEntry(RegisterBanks banks, uint offset, uint value) { diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs index e2d3e45..b29fe93 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs @@ -199,6 +199,9 @@ public sealed class VertexStageRegisters public UserScalarRegisters LegacyVertexUserScalars = new(); public UserScalarRegisters ExportUserScalars = new(); + // The same registers sharing the user scalar sets, which a writer then copies one by one. + public VertexStageRegisters ShallowCopy() => (VertexStageRegisters)MemberwiseClone(); + public VertexStageRegisters Copy() { var copy = (VertexStageRegisters)MemberwiseClone(); @@ -217,6 +220,8 @@ public sealed class PixelStageRegisters public PixelResource2 Resource2; public UserScalarRegisters UserScalars = new(); + public PixelStageRegisters ShallowCopy() => (PixelStageRegisters)MemberwiseClone(); + public PixelStageRegisters Copy() { var copy = (PixelStageRegisters)MemberwiseClone(); @@ -253,6 +258,8 @@ public sealed class ComputeStageRegisters public byte SharedVectorRegisters; public UserScalarRegisters UserScalars = new(); + public ComputeStageRegisters ShallowCopy() => (ComputeStageRegisters)MemberwiseClone(); + public ComputeStageRegisters Copy() { var copy = (ComputeStageRegisters)MemberwiseClone(); @@ -264,6 +271,9 @@ public sealed class ComputeStageRegisters // The shader bank: the program registers of every stage and their user scalars. public sealed class ShaderProgramRegisters { + // A bank holding the same stage objects, for a snapshot that shares them. + public ShaderProgramRegisters ShareStages() => (ShaderProgramRegisters)MemberwiseClone(); + public VertexStageRegisters Vertex = new(); public PixelStageRegisters Pixel = new(); public ComputeStageRegisters Compute = new(); diff --git a/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs b/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs index f244c77..ae91f2d 100644 --- a/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs +++ b/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs @@ -31,10 +31,13 @@ public sealed unsafe partial class CachedImage SubresourceRange = new ImageSubresourceRange(ViewFormatRules.FullAspects(Backing.Format), baseLevel, levelCount, baseLayer, layerCount), }; + // What a state that needs no barrier returns, so the common case allocates nothing. Callers only read it. + private static readonly List NoBarriers = []; + // Barriers for the requested state; a repeated write always gets a barrier. public (List Barriers, PipelineStageFlags SourceStages) GetBarriers(ImageLayout layout, AccessFlags access, PipelineStageFlags stage, SubresourceRange? range) { - var barriers = new List(); + List? barriers = null; PipelineStageFlags sourceStages = 0; if ((access & (WriteAccess | AccessFlags.ColorAttachmentWriteBit | AccessFlags.DepthStencilAttachmentWriteBit)) != 0) { @@ -83,7 +86,7 @@ public sealed unsafe partial class CachedImage var repeatedWrite = (state.Access & WriteAccess) != 0; if (state.Layout != layout || state.Access != access || repeatedWrite) { - barriers.Add(MakeBarrier(state, layout, access, level, 1, layer, 1)); + (barriers ??= []).Add(MakeBarrier(state, layout, access, level, 1, layer, 1)); sourceStages |= state.Stage; states[index] = new ImageAccessState(stage, access, layout); } @@ -101,15 +104,15 @@ public sealed unsafe partial class CachedImage var repeatedWrite = (state.Access & WriteAccess) != 0; if (state.Layout == layout && state.Access == access && !repeatedWrite) { - return (barriers, sourceStages); + return (NoBarriers, sourceStages); } - barriers.Add(MakeBarrier(state, layout, access, 0, Vk.RemainingMipLevels, 0, Vk.RemainingArrayLayers)); + (barriers ??= []).Add(MakeBarrier(state, layout, access, 0, Vk.RemainingMipLevels, 0, Vk.RemainingArrayLayers)); sourceStages |= state.Stage; } Backing.State = new ImageAccessState(stage, access, layout); - return (barriers, sourceStages); + return (barriers ?? NoBarriers, sourceStages); } public void Transition(ImageLayout layout, AccessFlags access, SubresourceRange? range, CommandBuffer command) diff --git a/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs b/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs index 9648265..64f8fc2 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs @@ -303,6 +303,8 @@ public sealed unsafe class ColorToMultisampleDepthBlit : IDisposable vk.CmdBeginRenderPass(command, &begin, SubpassContents.Inline); vk.CmdBindDescriptorSets(command, PipelineBindPoint.Graphics, _pipelineLayout, 0, 1, &set, 0, null); vk.CmdBindPipeline(command, PipelineBindPoint.Graphics, cached.Pipeline); + Vulkan.GraphicsDynamicStateEpoch.Advance(); + Vulkan.NativeBindingEpoch.Advance(); var viewport = new Viewport(0.0f, 0.0f, extent.Width, extent.Height, 0.0f, 1.0f); var scissor = new Rect2D(new Offset2D(0, 0), extent); vk.CmdSetViewport(command, 0, 1, &viewport); diff --git a/src/SharpEmu.Libs/Gpu/Images/GpuTiler.cs b/src/SharpEmu.Libs/Gpu/Images/GpuTiler.cs index 72a42cd..eca0c58 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GpuTiler.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GpuTiler.cs @@ -540,8 +540,11 @@ public sealed unsafe class GpuTiler : IDisposable return set; } - private void BindDescriptorSet(CommandBuffer command, DescriptorSet set) => + private void BindDescriptorSet(CommandBuffer command, DescriptorSet set) + { _device.Vk.CmdBindDescriptorSets(command, PipelineBindPoint.Compute, _pipelineLayout, 0, 1, &set, 0, null); + Vulkan.NativeBindingEpoch.Advance(); + } private void Record(bool tile, VkBuffer source, ulong sourceOffset, ulong sourceCapacity, VkBuffer target, ulong targetOffset, ulong targetCapacity, List dispatches, bool clearTarget) { @@ -587,6 +590,7 @@ public sealed unsafe class GpuTiler : IDisposable infos[2] = new DescriptorBufferInfo(dispatch.ArgumentsBuffer, dispatch.ArgumentsOffset, TileTransferArguments.Size); BindDescriptorSet(command, WriteDescriptorSet(infos, 2)); vk.CmdBindPipeline(command, PipelineBindPoint.Compute, GetPipeline(dispatch.PipelineSlot)); + Vulkan.NativeBindingEpoch.Advance(); vk.CmdDispatch(command, (dispatch.Arguments.Width + 7) / 8, (dispatch.Arguments.Height + 7) / 8, dispatch.Arguments.Depth); } @@ -697,6 +701,7 @@ public sealed unsafe class GpuTiler : IDisposable fixed (TileTransferArguments* pointer = &arguments) { _device.Vk.CmdPushConstants(command, _pipelineLayout, ShaderStageFlags.ComputeBit, 0, TileTransferArguments.Size, pointer); + Vulkan.NativeBindingEpoch.Advance(); } } @@ -743,6 +748,7 @@ public sealed unsafe class GpuTiler : IDisposable AccessFlags.ShaderReadBit | AccessFlags.ShaderWriteBit); VulkanSynchronization.PipelineBarrier(vk,command, PipelineStageFlags.AllCommandsBit | PipelineStageFlags.HostBit, PipelineStageFlags.ComputeShaderBit, 0, 0, null, 2, barriers, 0, null); vk.CmdBindPipeline(command, PipelineBindPoint.Compute, pipeline); + Vulkan.NativeBindingEpoch.Advance(); var descriptorAlignment = StorageAlignment; var groupLimit = _device.MaxComputeWorkGroupCount; @@ -830,6 +836,7 @@ public sealed unsafe class GpuTiler : IDisposable var command = new CommandBuffer(_scheduler.Current.Handle); VulkanSynchronization.PipelineBarrier(vk,command, PipelineStageFlags.AllCommandsBit | PipelineStageFlags.HostBit, PipelineStageFlags.ComputeShaderBit, 0, 0, null, 2, barriers, 0, null); vk.CmdBindPipeline(command, PipelineBindPoint.Compute, _swapBgra16); + Vulkan.NativeBindingEpoch.Advance(); BindDescriptorSet(command, WriteDescriptorSet(infos, -1)); PushArguments(command, new TileTransferArguments { SourceBase = inputBinding.Base, DestinationBase = outputBinding.Base, Width = pixels }); vk.CmdDispatch(command, (pixels + 63) / 64, 1, 1); diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs index 807c88c..3707edb 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs @@ -35,6 +35,25 @@ public sealed partial class GuestImageCache private void InvalidateLookups() => _lookupGeneration++; + // Changes whenever an image a lookup could find is created, removed or replaced, so a caller + // that remembers a lookup's result can tell when it may have changed. + public ulong LookupGeneration => _lookupGeneration; + + // What a remembered lookup still does on reuse: the image counts as used now. False when the + // image is gone. + public bool TryTouchRemembered(ResourceSlotIdentifier imageIdentifier) + { + using var held = _lock.Hold(); + if (_slots.TryGet(imageIdentifier) is not { Registered: true } image) + { + return false; + } + + image.LastAccessTick = _scheduler.CurrentTick; + TouchImage(image); + return true; + } + private static int LookupSlot(in ImageRequest request, bool exactFormat) { var hash = HashCode.Combine(request.Description.Data.Address, request.Description.Data.Size, request.Description.PixelFormat, diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs index f7aa599..c05998f 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs @@ -11,6 +11,14 @@ namespace SharpEmu.Libs.Gpu.Images; // Surface metadata (HTile, DCC, CMask, FMask) keyed by its guest address. public sealed partial class GuestImageCache { + private static long _dccSyncs; + private static long _dccSlicesUndecodable; + private static long _dccSlicesMixed; + private static long _dccSlicesCleared; + + public static string TakeDccReport() => FormattableString.Invariant( + $"[PERF][DCC_SYNC] gpu_syncs={Interlocked.Exchange(ref _dccSyncs, 0)} cleared={Interlocked.Exchange(ref _dccSlicesCleared, 0)} mixed={Interlocked.Exchange(ref _dccSlicesMixed, 0)} undecodable={Interlocked.Exchange(ref _dccSlicesUndecodable, 0)}"); + private unsafe void MaterializeDccClear( ResourceSlotIdentifier imageIdentifier, in ImageRequest request, @@ -96,6 +104,7 @@ public sealed partial class GuestImageCache allowClearToRegister: request.Role == ImageRole.ColorTarget, out var clear)) { + Interlocked.Increment(ref _dccSlicesUndecodable); continue; } @@ -126,9 +135,12 @@ public sealed partial class GuestImageCache if (!uniform) { + Interlocked.Increment(ref _dccSlicesMixed); continue; } + Interlocked.Increment(ref _dccSlicesCleared); + using (var held = _lock.Hold()) { if (_slots.TryGet(imageIdentifier) == null) @@ -287,13 +299,20 @@ public sealed partial class GuestImageCache public bool IsMetadataCleared(ulong address, uint slice) => IsMetadataCleared(address, slice, out _); + // DCC metadata of a bound target, or the range of a DCC fill seen before its target: both are + // filled on the host, which keeps them CPU-readable instead of GPU-written. public bool OverlapsDccMetadata(ulong address, ulong size) { using var held = _lock.Hold(); foreach (var (start, metadata) in _surfaceMetadata) { - if (metadata.Kind == SurfaceMetadataKind.Dcc && metadata.Size != 0 && - address < start + metadata.Size && start < address + size) + var extent = metadata.Kind switch + { + SurfaceMetadataKind.Dcc => metadata.Size, + SurfaceMetadataKind.PendingDcc => metadata.FillSize, + _ => 0UL, + }; + if (extent != 0 && address < start + extent && start < address + size) { return true; } @@ -416,6 +435,12 @@ public sealed partial class GuestImageCache var size = (ulong)layerCount * sliceSize; if (IsValidRange(address, size) && _bufferCache.HasGpuDirtyBytes(address, size)) { + Interlocked.Increment(ref _dccSyncs); + if (Diagnostics.DccWriterTrace.Enabled) + { + Diagnostics.DccWriterTrace.ReportSync(address, size); + } + _ = _bufferCache.TrySynchronizeCpuRead(address, size); } } diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs index 000513d..1d938f7 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs @@ -761,6 +761,7 @@ public sealed unsafe partial class GuestImageCache _scheduler.QueuePriorityCompletionAction(() => { download.Invalidate(offset, range.Size); + GpuWritebackEpoch.Advance(); if (!backing.TryWriteBacking(range.Address, download.Mapped.Slice((int)offset, (int)range.Size))) { throw SubmissionScheduler.Fatal($"The image readback could not be written to guest memory: address=0x{range.Address:X16} size=0x{range.Size:X}."); @@ -833,7 +834,7 @@ public sealed unsafe partial class GuestImageCache byte stencilClear = 0; if (aspect == ImageAspectFlags.ColorBit) { - if (!PackedClearValue.TryDecodeColor(image.Description.PixelFormat, packedClear, out colorClear)) + if (!PackedClearValue.TryDecodeBufferColor(image.Description.PixelFormat, packedClear, out colorClear)) { return false; } diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.cs index 5568e0b..2050915 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.cs @@ -125,7 +125,12 @@ public sealed unsafe partial class GuestImageCache : IGuestImageCache, IGuestIma var metadataBaseLayer = request.View.BaseLayer; var result = LookUpImage(ref request, exactFormat); + CompleteLookup(result, in request, metadataBaseLayer); + return result; + } + public void CompleteLookup(ResourceSlotIdentifier result, in ImageRequest request, uint metadataBaseLayer) + { if (request.Role is ImageRole.Texture or ImageRole.StorageImage) { ulong metadataAddress; @@ -163,8 +168,6 @@ public sealed unsafe partial class GuestImageCache : IGuestImageCache, IGuestIma $"A compressed display surface can only be read from clean native GPU contents: address=0x{image.Description.Data.Address:X16} bufferModified={image.IsBufferModified} cpuDirty={image.IsCpuDirty} gpuModified={image.IsGpuModified}."); } } - - return result; } private ResourceSlotIdentifier LookUpImage(ref ImageRequest request, bool exactFormat) diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs b/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs index 7c772e6..25f2ed8 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs @@ -597,6 +597,30 @@ public static class PackedClearValue _ => 0, }; + // A buffer fill repeats the word in every texel, unlike a DCC clear register. + public static bool TryDecodeBufferColor(Format format, uint packed, out ClearColorValue clear) + { + clear = default; + switch (format) + { + case Format.R16G16Sfloat: + case Format.R16G16B16A16Sfloat: + var low = BitConverter.UInt16BitsToHalf((ushort)packed); + var high = BitConverter.UInt16BitsToHalf((ushort)(packed >> 16)); + // A float conversion may change a NaN payload; keep the raw fill in that case. + if (Half.IsNaN(low) || Half.IsNaN(high)) return false; + clear.Float32_0 = clear.Float32_2 = (float)low; + clear.Float32_1 = clear.Float32_3 = (float)high; + return true; + case Format.R32G32Uint: + case Format.R32G32B32A32Uint: + clear.Uint32_0 = clear.Uint32_1 = clear.Uint32_2 = clear.Uint32_3 = packed; + return true; + default: + return TryDecodeColor(format, packed, out clear); + } + } + public static bool TryDecodeColor(Format format, uint packed, out ClearColorValue clear) { static float Unorm8(uint value) => (value & 0xff) / 255.0f; diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs index 12bac61..be088e2 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs @@ -76,8 +76,56 @@ public static partial class ImageRequestBuilders public static bool SupportsDccFixedClear(Format format) => PackedClearValue.SupportsDccFixedColor(format); + // Every draw rebuilds its targets' requests (a whole mip layout) from the same register words, + // twice: once for the attachments and once for the pixel outputs. The build reads nothing but + // its arguments, so a result is reused while they are equal. + private struct ColorTargetMemo + { + public bool Valid; + public ColorTargetWords Words; + public uint TargetMask; + public uint DrawLayerOffset; + public bool IgnoreTargetMask; + public ColorTargetResolution? Result; + } + + private const int TargetMemoCount = 8; + + [ThreadStatic] + private static ColorTargetMemo[]? _colorTargetMemos; + + private static int TargetMemoSlot(ulong address) => (int)((address >> 12) ^ (address >> 20)) & (TargetMemoCount - 1); + // Builds the request for a bound color target. Null when the slot carries no target. public static ColorTargetResolution? ColorTarget(in ColorTargetWords words, uint targetMask, uint drawLayerOffset, bool ignoreTargetMask) + { + if (Rendering.RenderTrace.Enabled) + { + return ColorTargetCore(in words, targetMask, drawLayerOffset, ignoreTargetMask); + } + + var memos = _colorTargetMemos ??= new ColorTargetMemo[TargetMemoCount]; + ref var memo = ref memos[TargetMemoSlot(words.BaseAddress)]; + if (memo.Valid && memo.TargetMask == targetMask && memo.DrawLayerOffset == drawLayerOffset && + memo.IgnoreTargetMask == ignoreTargetMask && memo.Words.Equals(words)) + { + return memo.Result; + } + + var result = ColorTargetCore(in words, targetMask, drawLayerOffset, ignoreTargetMask); + memo = new ColorTargetMemo + { + Valid = true, + Words = words, + TargetMask = targetMask, + DrawLayerOffset = drawLayerOffset, + IgnoreTargetMask = ignoreTargetMask, + Result = result, + }; + return result; + } + + private static ColorTargetResolution? ColorTargetCore(in ColorTargetWords words, uint targetMask, uint drawLayerOffset, bool ignoreTargetMask) { var mask = targetMask & 0xF; if (ignoreTargetMask && words.BaseAddress != 0 && mask == 0) @@ -235,8 +283,7 @@ public static partial class ImageRequestBuilders pitch = TileGeometry.TexturePitch(transferFormat, width, GuestTileMode.Linear); } - var mipSpans = new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; - var mipPadded = new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + var (mipSpans, mipPadded) = MipLayoutScratch.Rent(); TiledSurfaceLayout? volumeLayout = null; ulong size; ulong backingSize = 0; diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs index 8a0c357..3fed162 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs @@ -59,8 +59,28 @@ public static partial class ImageRequestBuilders } // Builds the request for the bound depth target. Null when no depth or stencil state is active. - public static DepthTargetResolution? DepthTarget(in DepthTargetWords depthWords, IImageFormatSupport device) => - DepthTargetCore(in depthWords, device, copyMode: false, writeBuffer: false); + // Reused while the words and the device are the same, as ColorTarget's requests are. + public static DepthTargetResolution? DepthTarget(in DepthTargetWords depthWords, IImageFormatSupport device) + { + if (Rendering.RenderTrace.Enabled) + { + return DepthTargetCore(in depthWords, device, copyMode: false, writeBuffer: false); + } + + if (_depthTargetMemo is { } memo && ReferenceEquals(memo.Device, device) && memo.Words.Equals(depthWords)) + { + return memo.Result; + } + + var result = DepthTargetCore(in depthWords, device, copyMode: false, writeBuffer: false); + _depthTargetMemo = new DepthTargetMemo(depthWords, device, result); + return result; + } + + private sealed record DepthTargetMemo(DepthTargetWords Words, IImageFormatSupport Device, DepthTargetResolution? Result); + + [ThreadStatic] + private static DepthTargetMemo? _depthTargetMemo; // Builds the source or destination image used by DB_RENDER_OVERRIDE depth/stencil copies. public static DepthTargetResolution? DepthTargetCopy(in DepthTargetWords depthWords, IImageFormatSupport device, bool writeBuffer) => diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs index 1508dd0..f99fe0a 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs @@ -89,8 +89,7 @@ public static partial class ImageRequestBuilders return; } - var spans = new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; - var padded = new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + var (spans, padded) = MipLayoutScratch.Rent(); TileGeometry.TryGetTextureSize( description.GuestFormat, description.Extent.Width, description.Extent.Height, description.Resources.Levels, description.TileMode, out _, spans, padded); diff --git a/src/SharpEmu.Libs/Gpu/Images/MipStatistics.cs b/src/SharpEmu.Libs/Gpu/Images/MipStatistics.cs index e9b555f..a8f46ed 100644 --- a/src/SharpEmu.Libs/Gpu/Images/MipStatistics.cs +++ b/src/SharpEmu.Libs/Gpu/Images/MipStatistics.cs @@ -11,6 +11,10 @@ public sealed class MipStatistics public const uint FirstCounterOffset = 64; public const int ReportBytes = (int)FirstCounterOffset + SlotCount * CounterBytes; public const uint NoMipSampled = 15; + public const uint CounterWords = 257; + public const ulong CounterBufferBytes = 2 * CounterWords * sizeof(uint); + public const uint NoCounterField = 0x8000; + public const uint ReportWithoutCounters = 1; private const int CounterBytes = 2 * sizeof(uint); private const uint FinestMip = 0; @@ -29,6 +33,53 @@ public sealed class MipStatistics } } + public static uint ImageField(ReadOnlySpan texture) + { + if (((texture[5] >> 25) & 1) == 0) + { + return NoCounterField; + } + + var counter = texture[6] & 0xFF; + var baseLevel = (texture[3] >> 12) & 0xF; + var minLod = (texture[1] >> 8) & 0xFFF; + return counter | (baseLevel << 8) | (minLod << 16); + } + + public static void PackReport(ReadOnlySpan words, Span report) + { + report.Clear(); + if (report.Length >= sizeof(uint)) + { + BinaryPrimitives.WriteUInt32LittleEndian(report, FirstCounterOffset); + } + + for (var slot = 0; slot < SlotCount; slot++) + { + var start = (int)FirstCounterOffset + slot * CounterBytes; + if (start + CounterBytes > report.Length) + { + break; + } + + var finest = words[slot]; + var count = Math.Min(words[(int)CounterWords + slot], MaxSamples); + var level = finest == uint.MaxValue ? NoMipSampled : Math.Min(finest, 14u); + var counter = report.Slice(start, CounterBytes); + BinaryPrimitives.WriteUInt32LittleEndian(counter, count | ((uint)slot << 24)); + BinaryPrimitives.WriteUInt32LittleEndian(counter[sizeof(uint)..], level << 24); + } + } + + public void WriteReport(Span report) + { + var hasCounters = TryWriteReport(report); + if (report.Length >= sizeof(uint)) + { + BinaryPrimitives.WriteUInt32LittleEndian(report, hasCounters ? FirstCounterOffset : ReportWithoutCounters); + } + } + public bool TryWriteReport(Span report) { if (report.Length < ReportBytes) diff --git a/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs b/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs index 466023e..1c23e50 100644 --- a/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs +++ b/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs @@ -48,7 +48,7 @@ public enum SamplerBorderColor : uint public sealed unsafe class SamplerStore : IDisposable { private readonly GpuDeviceInfo _device; - private readonly Dictionary<(uint, uint, uint, uint, bool), Sampler> _samplers = new(); + private readonly System.Collections.Concurrent.ConcurrentDictionary<(uint, uint, uint, uint, bool), Sampler> _samplers = new(); private readonly object _gate = new(); public SamplerStore(GpuDeviceInfo device) => _device = device; @@ -57,16 +57,22 @@ public sealed unsafe class SamplerStore : IDisposable public Sampler GetSampler(in SamplerDescriptorWords words, bool integerView) { + var key = (words[0], words[1], words[2], words[3], integerView); + // Every sampler binding of every draw asks; only the first of each kind creates. + if (_samplers.TryGetValue(key, out var found)) + { + return found; + } + lock (_gate) { - var key = (words[0], words[1], words[2], words[3], integerView); if (_samplers.TryGetValue(key, out var existing)) { return existing; } var sampler = CreateSampler(words, integerView); - _samplers.Add(key, sampler); + _samplers[key] = sampler; return sampler; } } diff --git a/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs b/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs index 989e5e3..fef468d 100644 --- a/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs +++ b/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs @@ -233,8 +233,7 @@ public sealed class TextureTransferLayout layout.Pitch = TileGeometry.TexturePitch(format, width, tile); if (tile == GuestTileMode.Linear) { - var levelSpans = new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; - var paddedSizes = new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + var (levelSpans, paddedSizes) = MipLayoutScratch.Rent(); TileGeometry.TryGetTextureSize(format, width, height, levels, tile, out _, levelSpans, paddedSizes); for (uint level = 0; level < levels; level++) { diff --git a/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs b/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs index 2de8d20..837441e 100644 --- a/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs +++ b/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs @@ -21,6 +21,24 @@ public struct TileLevelSpan public readonly record struct TilePaddedSize(uint Width, uint Height); +// The per-level output arrays of the layout queries, kept per thread: every draw that resolves a +// target or a texture needs them for a moment and reads them before the next query. +internal static class MipLayoutScratch +{ + [ThreadStatic] private static TileLevelSpan[]? _spans; + [ThreadStatic] private static TilePaddedSize[]? _padded; + + // Zeroed, as a new array would be, so a level the query leaves out reads as empty. + public static (TileLevelSpan[] Spans, TilePaddedSize[] Padded) Rent() + { + var spans = _spans ??= new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; + var padded = _padded ??= new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + Array.Clear(spans); + Array.Clear(padded); + return (spans, padded); + } +} + public enum TileBlockKind : uint { Standard256B, diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/DescriptorWriter.cs b/src/SharpEmu.Libs/Gpu/Pipelines/DescriptorWriter.cs index 7a2d378..da19348 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/DescriptorWriter.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/DescriptorWriter.cs @@ -28,7 +28,8 @@ public static class DescriptorWriter { DescriptorBindingKind.Samplers => Silk.NET.Vulkan.DescriptorType.Sampler, DescriptorBindingKind.Buffers or DescriptorBindingKind.GlobalDataShare or DescriptorBindingKind.DeviceAddressPageTable or - DescriptorBindingKind.FaultBuffer or DescriptorBindingKind.FlattenedResourceTable or DescriptorBindingKind.ShaderData => + DescriptorBindingKind.FaultBuffer or DescriptorBindingKind.FlattenedResourceTable or DescriptorBindingKind.ShaderData or + DescriptorBindingKind.MipStatistics => Silk.NET.Vulkan.DescriptorType.StorageBuffer, _ => throw SubmissionScheduler.Fatal($"The descriptor binding kind is invalid: kind={kind}."), }; diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs b/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs index 2eff4e4..2595149 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs @@ -36,6 +36,13 @@ internal interface IShaderPipelineHost bool ComputeWave64Supported { get; } + // True when a 64-invocation wave64 compute workgroup runs as one 64-lane host subgroup. + bool ComputeWave64SubgroupNative => false; + + bool ComputeRunsOn32LaneSubgroups(ComputeInputInfo input) => false; + + bool MipStatisticsSubgroupReduction => false; + bool GraphicsSubgroupOperationsEnabled { get; } // The device supports shaderSharedInt64Atomics, so LDS 64-bit atomics can be diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs b/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs index f4eba84..4394dcd 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs @@ -45,6 +45,13 @@ public sealed class PipelineStaticParameters : IEquatable public Format StencilFormat { get; set; } = Format.Undefined; public uint ColorCount { get; set; } + // The state of a new instance, for a lookup object reused across draws. + public void Reset() + { + Array.Clear(ColorFormats); + DepthFormat = Format.Undefined; + StencilFormat = Format.Undefined; + ColorCount = 0; + } + public bool Equals(PipelineRenderingState? other) => other is not null && ColorCount == other.ColorCount && DepthFormat == other.DepthFormat && StencilFormat == other.StencilFormat && ColorFormats.AsSpan().SequenceEqual(other.ColorFormats); @@ -179,6 +195,15 @@ public sealed class PipelineVertexInputState : IEquatable other is not null && BindingCount == other.BindingCount && AttributeCount == other.AttributeCount && Bindings.AsSpan().SequenceEqual(other.Bindings) && Attributes.AsSpan().SequenceEqual(other.Attributes); @@ -207,8 +232,9 @@ public sealed class PipelineVertexInputState : IEquatable { public required PipelineRenderingState Rendering { get; init; } - public ulong VertexProgramId { get; init; } - public ulong PixelProgramId { get; init; } + // Settable for the lookup key reused across draws; a key in the cache is never changed. + public ulong VertexProgramId { get; set; } + public ulong PixelProgramId { get; set; } public required PipelineVertexInputState VertexInput { get; init; } public required PipelineStaticParameters StaticParameters { get; init; } diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs index 96a8f8d..0aafc52 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs @@ -27,6 +27,9 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider private readonly Dictionary _graphicsPipelines = new(); private readonly Dictionary _computePipelines = new(); private readonly object _gate = new(); + // The pipeline maps: a lookup takes this lock or _gate, an addition both; a cached draw so does + // not wait for the program preparation a resolving thread holds _gate for. + private readonly object _pipelineGate = new(); private readonly bool _strictShaders = Environment.GetEnvironmentVariable("SHARPEMU_STRICT_COMPUTE") != "0"; private readonly HashSet<(ShaderStage Stage, ulong Hash, uint CodeSize)> _reportedShaderSkips = []; @@ -70,6 +73,11 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider return new ShaderSource(registered, hash, userData, userDataBase, stage); } + // Program resolution runs on the render thread and on a thread resolving draws ahead of it; + // the program, decode and materialization caches it updates are used under this lock only. + // A pipeline lookup takes _pipelineGate instead, so it does not wait for a resolution. + private readonly object _resolveGate = new(); + public GraphicsPrograms GetGraphicsPrograms( VertexStageRegisters vertex, PixelStageRegisters pixel, @@ -78,6 +86,21 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider ReadOnlySpan targetExportMapping, bool pixelActive, bool depthBound) + { + lock (_resolveGate) + { + return GetGraphicsProgramsCore(vertex, pixel, shaderInterface, context, targetExportMapping, pixelActive, depthBound); + } + } + + private GraphicsPrograms GetGraphicsProgramsCore( + VertexStageRegisters vertex, + PixelStageRegisters pixel, + ShaderInterfaceRegisters shaderInterface, + ContextRegisters context, + ReadOnlySpan targetExportMapping, + bool pixelActive, + bool depthBound) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.ProgramPreparation); var vertexSource = PrepareSource( @@ -209,44 +232,53 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider shaderInterface.VertexOutputControl, clipSpace); } - // One past the highest parameter location the pixel program reads, resolved as its translator does. - private static uint ReadVertexOutputCount(Gen5ShaderProgram pixelProgram, PixelInputInfo info) - { - var attributes = pixelProgram.Instructions + // The interpolated attributes a pixel program reads, ascending. A program never changes, and this + // was rebuilt with LINQ on every draw. + private static readonly System.Runtime.CompilerServices.ConditionalWeakTable InterpolatedAttributeCache = new(); + + private static uint[] InterpolatedAttributesOf(Gen5ShaderProgram program) => + InterpolatedAttributeCache.GetValue(program, static program => program.Instructions .Select(static instruction => instruction.Control) .OfType() .Select(static control => control.Attribute) .Distinct() .Order() - .ToArray(); + .ToArray()); + + // One past the highest parameter location the pixel program reads, resolved as its translator does. + private static uint ReadVertexOutputCount(Gen5ShaderProgram pixelProgram, PixelInputInfo info) + { + var attributes = InterpolatedAttributesOf(pixelProgram); if (attributes.Length == 0) { return 0; } - var controls = new uint[32]; + Span controls = stackalloc uint[32]; for (var index = 0u; index < (uint)controls.Length; index++) { - controls[index] = index < info.InputCount && index < (uint)info.InterpolatorSettings.Length + controls[(int)index] = index < info.InputCount && index < (uint)info.InterpolatorSettings.Length ? info.InterpolatorSettings[index] : index; } - return Gen5PixelInputMapping.ResolveLocations(controls, attributes).Max() + 1; - } - - private static uint InterpolatedAttributeCount(Gen5ShaderProgram program) - { - var maxAttribute = -1; - foreach (var instruction in program.Instructions) + // Attributes are distinct and below 32, so a program never has more than 32 of them. + Span locations = attributes.Length <= 32 ? stackalloc uint[attributes.Length] : new uint[attributes.Length]; + Gen5PixelInputMapping.ResolveLocations(controls, attributes, locations); + var highest = 0u; + foreach (var location in locations) { - if (instruction.Control is Gen5InterpolationControl interpolation) - { - maxAttribute = Math.Max(maxAttribute, (int)interpolation.Attribute); - } + highest = Math.Max(highest, location); } - return (uint)(maxAttribute + 1); + return highest + 1; + } + + // The attribute list is ascending, so its last entry is the highest. + private static uint InterpolatedAttributeCount(Gen5ShaderProgram program) + { + var attributes = InterpolatedAttributesOf(program); + return attributes.Length == 0 ? 0 : attributes[^1] + 1; } // The bound colour slots in order; each output mode names the kind the pixel program exports. @@ -321,6 +353,20 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider uint dimensionX, uint dimensionY, uint dimensionZ) + { + lock (_resolveGate) + { + return GetComputeProgramCore(compute, shaderInterface, dispatchInitiator, dimensionX, dimensionY, dimensionZ); + } + } + + private ComputeProgram GetComputeProgramCore( + ComputeStageRegisters compute, + ShaderInterfaceRegisters shaderInterface, + uint dispatchInitiator, + uint dimensionX, + uint dimensionY, + uint dimensionZ) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.ProgramPreparation); var source = PrepareSource(compute.Address, ShaderStage.Compute, "compute", compute.UserScalars, compute.UserScalarCount, probeWrittenRegisters: false, userDataBase: 0); @@ -417,6 +463,28 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider ShaderProgram pixelProgram) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.PipelineCreation); + // The lookup fills objects this thread reuses for every draw; only a new pipeline gets its own. + var lookup = _graphicsLookup ??= new GraphicsPipelineKey + { + Rendering = new PipelineRenderingState(), + VertexInput = new PipelineVertexInputState(), + StaticParameters = new PipelineStaticParameters(), + }; + lookup.Rendering.Reset(); + lookup.VertexInput.Reset(); + lookup.StaticParameters.Reset(); + FillGraphicsState(colors, in depth, vertexInput, pixelInput, context, in rendering, topology, primitiveRestartEnabled, disableBlending, + vertexProgram, pixelProgram, _host.NoAttachmentSampleCounts, lookup.Rendering, lookup.StaticParameters, lookup.VertexInput); + lookup.VertexProgramId = vertexProgram.Id; + lookup.PixelProgramId = pixelInput is not null ? pixelProgram.Id : 0; + lock (_pipelineGate) + { + if (_graphicsPipelines.TryGetValue(lookup, out var cached)) + { + return cached; + } + } + var description = BuildGraphicsDescription( colors, in depth, vertexInput, pixelInput, context, in rendering, topology, primitiveRestartEnabled, disableBlending, vertexProgram, pixelProgram, _host.NoAttachmentSampleCounts); @@ -437,12 +505,19 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider } var created = _host.CreateGraphicsPipeline(description); - _graphicsPipelines.Add(key, created); + lock (_pipelineGate) + { + _graphicsPipelines.Add(key, created); + } + ShaderCacheCounters.CountGraphicsPipeline(); return created; } } + [ThreadStatic] + private static GraphicsPipelineKey? _graphicsLookup; + // The full static state of one graphics pipeline from the draw's targets, registers and programs. internal static GraphicsPipelineDescription BuildGraphicsDescription( ReadOnlySpan colors, @@ -457,6 +532,43 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider ShaderProgram vertexProgram, ShaderProgram pixelProgram, SampleCountFlags noAttachmentSampleCounts) + { + var renderingState = new PipelineRenderingState(); + var parameters = new PipelineStaticParameters(); + var vertexState = new PipelineVertexInputState(); + var (vertexStage, pixelStage) = FillGraphicsState(colors, in depth, vertexInput, pixelInput, context, in rendering, topology, + primitiveRestartEnabled, disableBlending, vertexProgram, pixelProgram, noAttachmentSampleCounts, renderingState, parameters, vertexState); + return new GraphicsPipelineDescription + { + Rendering = renderingState, + VertexInput = vertexState, + VertexInfo = vertexInput, + VertexProgram = vertexProgram, + VertexStage = vertexStage, + PixelInfo = pixelInput, + PixelProgram = pixelProgram, + PixelStage = pixelStage, + StaticParameters = parameters, + }; + } + + // Writes the pipeline state into objects in their initial state; returns the stage programs. + private static (ShaderProgramInfo VertexStage, ShaderProgramInfo? PixelStage) FillGraphicsState( + ReadOnlySpan colors, + in DepthAttachmentState depth, + VertexInputInfo vertexInput, + PixelInputInfo? pixelInput, + ContextRegisters context, + in RenderingState rendering, + PrimitiveTopology topology, + bool primitiveRestartEnabled, + bool disableBlending, + ShaderProgram vertexProgram, + ShaderProgram pixelProgram, + SampleCountFlags noAttachmentSampleCounts, + PipelineRenderingState renderingState, + PipelineStaticParameters parameters, + PipelineVertexInputState vertexState) { if (colors.Length > PipelineStaticParameters.ColorAttachmentCount) { @@ -477,8 +589,8 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider var vertexStage = vertexInput.Stage.Program ?? throw SubmissionScheduler.Fatal("The vertex stage has no program."); var pixelStage = pixelActive ? pixelInput!.Stage.Program ?? throw SubmissionScheduler.Fatal("The pixel stage has no program.") : null; var colorCount = (uint)colors.Length; - var parameters = new PipelineStaticParameters { ColorCount = colorCount }; - var renderingState = new PipelineRenderingState { ColorCount = colorCount }; + parameters.ColorCount = colorCount; + renderingState.ColorCount = colorCount; for (var index = 0; index < colors.Length; index++) { ref readonly var color = ref colors[index]; @@ -564,18 +676,8 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider parameters.SetBlendBypass(index, ((context.ColorTargets[slot].Info >> 16) & 0x1) != 0); } - return new GraphicsPipelineDescription - { - Rendering = renderingState, - VertexInput = BuildVertexInputState(vertexInput), - VertexInfo = vertexInput, - VertexProgram = vertexProgram, - VertexStage = vertexStage, - PixelInfo = pixelInput, - PixelProgram = pixelProgram, - PixelStage = pixelStage, - StaticParameters = parameters, - }; + FillVertexInputState(vertexInput, vertexState); + return (vertexStage, pixelStage); } internal static GraphicsPipelineKey KeyOf(GraphicsPipelineDescription description) => new() @@ -587,18 +689,15 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider StaticParameters = description.StaticParameters, }; - private static PipelineVertexInputState BuildVertexInputState(VertexInputInfo info) + private static void FillVertexInputState(VertexInputInfo info, PipelineVertexInputState state) { if (info.Buffers.Length > VertexInputInfo.MaxBuffers || info.Attributes.Length > VertexInputInfo.MaxBuffers) { throw SubmissionScheduler.Fatal($"The vertex input is too large: buffers={info.Buffers.Length} attributes={info.Attributes.Length}."); } - var state = new PipelineVertexInputState - { - BindingCount = (byte)info.Buffers.Length, - AttributeCount = (byte)info.Attributes.Length, - }; + state.BindingCount = (byte)info.Buffers.Length; + state.AttributeCount = (byte)info.Attributes.Length; for (var binding = 0; binding < info.Buffers.Length; binding++) { var buffer = info.Buffers[binding]; @@ -615,8 +714,6 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider state.Attributes[index] = new PipelineVertexAttribute(attribute.OffsetBytes, (byte)attribute.BufferIndex); } - - return state; } public PipelineHandle CreateComputePipeline(ComputeInputInfo input, ShaderProgram program) @@ -629,6 +726,14 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider var stage = input.Stage.Program ?? throw SubmissionScheduler.Fatal("The compute stage has no program."); var key = new ComputePipelineKey(program.Id); + lock (_pipelineGate) + { + if (_computePipelines.TryGetValue(key, out var cached)) + { + return cached; + } + } + lock (_gate) { if (_computePipelines.TryGetValue(key, out var cached)) @@ -642,7 +747,11 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider } var created = _host.CreateComputePipeline(new ComputePipelineDescription { Input = input, Program = program, Stage = stage }); - _computePipelines.Add(key, created); + lock (_pipelineGate) + { + _computePipelines.Add(key, created); + } + ShaderCacheCounters.CountComputePipeline(); return created; } @@ -659,6 +768,15 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider var stage = input.Stage.Program ?? throw SubmissionScheduler.Fatal("The compute stage has no program."); var key = new ComputePipelineKey(program.Id); + lock (_pipelineGate) + { + if (_computePipelines.TryGetValue(key, out var cached)) + { + handle = cached; + return true; + } + } + lock (_gate) { if (_computePipelines.TryGetValue(key, out var cached)) @@ -674,7 +792,11 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider return false; } - _computePipelines.Add(key, created); + lock (_pipelineGate) + { + _computePipelines.Add(key, created); + } + ShaderCacheCounters.CountComputePipeline(); handle = created; return true; diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs index 611b9c8..7607f3d 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs @@ -1,6 +1,7 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Runtime.InteropServices; using SharpEmu.HLE; using SharpEmu.Libs.Gpu.Rendering; using SharpEmu.Libs.Gpu.Scheduling; @@ -68,6 +69,37 @@ public sealed class ProgramKey : IEquatable public override int GetHashCode() => HashCode.Combine(Stage, Hash, UserDataCount, CodeSize, StaticState.Length); } +// A program key as a lookup sees it: the static state words stay in the caller's list, so a draw +// that finds its program does not build a key object or copy the words. +internal readonly ref struct ProgramKeyProbe(ShaderStage stage, ulong hash, uint userDataCount, uint codeSize, ReadOnlySpan staticState) +{ + public readonly ShaderStage Stage = stage; + public readonly ulong Hash = hash; + public readonly uint UserDataCount = userDataCount; + public readonly uint CodeSize = codeSize; + public readonly ReadOnlySpan StaticState = staticState; +} + +internal sealed class ProgramKeyComparer : IEqualityComparer, IAlternateEqualityComparer +{ + public static readonly ProgramKeyComparer Instance = new(); + + public bool Equals(ProgramKey? x, ProgramKey? y) => x is null ? y is null : x.Equals(y); + + public int GetHashCode(ProgramKey key) => key.GetHashCode(); + + public bool Equals(ProgramKeyProbe alternate, ProgramKey key) => + alternate.Stage == key.Stage && alternate.Hash == key.Hash && alternate.UserDataCount == key.UserDataCount && + alternate.CodeSize == key.CodeSize && alternate.StaticState.SequenceEqual(key.StaticState); + + // The same value as ProgramKey.GetHashCode. + public int GetHashCode(ProgramKeyProbe alternate) => + HashCode.Combine(alternate.Stage, alternate.Hash, alternate.UserDataCount, alternate.CodeSize, alternate.StaticState.Length); + + public ProgramKey Create(ProgramKeyProbe alternate) => + new(alternate.Stage, alternate.Hash, alternate.UserDataCount, alternate.CodeSize, alternate.StaticState.ToArray()); +} + // One compiled module of a program entry for one specialization and push-data start. internal sealed class ProgramPermutation { @@ -102,7 +134,7 @@ internal sealed class ShaderProgramCache private readonly CpuContext _context; private readonly IGuestGpuBackend _compiler; private readonly IShaderPipelineHost _host; - private readonly Dictionary _programs = new(); + private readonly Dictionary _programs = new(ProgramKeyComparer.Instance); private readonly Dictionary<(ulong Hash, uint CodeSize), Gen5ShaderProgram> _decoded = new(); private readonly Dictionary<(ulong Hash, uint CodeSize), ShaderCodeCapture> _codeCaptures = new(); private readonly List _staticState = new(StageStaticKey.MaxWords); @@ -216,13 +248,14 @@ internal sealed class ShaderProgramCache private ShaderProgram GetOrCompileCore(ShaderSource source, StageCompileOptions options, ref uint pushDataCursor, out ShaderStageResources stage) { - ProgramKey key; + ProgramKey? key = null; ProgramSourceEntry? entry; using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.ProgramCacheLookup)) { BuildStaticState(source.Stage, options); - key = new ProgramKey(source.Stage, source.Hash, (uint)source.UserData.Length, source.CodeSize, _staticState.ToArray()); - _programs.TryGetValue(key, out entry); + var probe = new ProgramKeyProbe(source.Stage, source.Hash, (uint)source.UserData.Length, source.CodeSize, + CollectionsMarshal.AsSpan(_staticState)); + _programs.GetAlternateLookup().TryGetValue(probe, out entry); } var sourceWasCached = entry is not null; if (RenderTrace.Enabled && RenderTrace.Pipeline()) @@ -246,12 +279,14 @@ internal sealed class ShaderProgramCache if (entry is null) { entry = CreateEntry(source, options); + key = BuildProgramKey(source); _programs.Add(key, entry); ShaderCacheCounters.CountProgram(); } - var snapshot = new ResourceSnapshot(); - var specialization = new ResourceSpecialization(); + // Both are assigned by the materialization that follows; a failed one throws. + ResourceSnapshot snapshot = null!; + ResourceSpecialization specialization = null!; var captureIndirectImageFailure = _spirvDumpEnabled ? ShaderPermutationDump.CreateFailureCapture(source) : null; if (Diagnostics.GpuReadTrace.Enabled) { @@ -295,7 +330,7 @@ internal sealed class ShaderProgramCache } } - var permutation = CompilePermutation(source, options, entry, specialization, pushDataCursor, key, sourceWasCached); + var permutation = CompilePermutation(source, options, entry, specialization, pushDataCursor, key ?? BuildProgramKey(source), sourceWasCached); entry.Permutations.Add(permutation); ShaderCacheCounters.CountPermutation(); stage = CreateStageResources(permutation.Program, snapshot, source, options); @@ -308,6 +343,10 @@ internal sealed class ShaderProgramCache return permutation.Handle; } + // The key of the entry the static state was just built for. + private ProgramKey BuildProgramKey(ShaderSource source) => + new(source.Stage, source.Hash, (uint)source.UserData.Length, source.CodeSize, _staticState.ToArray()); + private static ShaderStageResources CreateStageResources( ShaderProgramInfo program, ResourceSnapshot snapshot, ShaderSource source, StageCompileOptions options) { @@ -641,12 +680,14 @@ internal sealed class ShaderProgramCache SupportsPerVertexPixelInputs = _host.PerVertexPixelInputsSupported, PixelInputAddress = options.PixelInputAddress, PixelInputCntl = interpolators, + MipStatisticsSubgroupReduction = _host.MipStatisticsSubgroupReduction, }; } default: return BuildComputeRequest(entry.Plan, resources, layout, options.ComputeInfo!, options.ComputeSystemRegisters, - sharedInt64Atomics, _host.ExecGuardElisionEnabled, signedZeroInfNanPreserve); + sharedInt64Atomics, _host.ExecGuardElisionEnabled, signedZeroInfNanPreserve, _host.ComputeWave64SubgroupNative, + _host.ComputeRunsOn32LaneSubgroups(options.ComputeInfo!)); } } @@ -663,10 +704,12 @@ internal sealed class ShaderProgramCache private static ShaderCompileRequest BuildComputeRequest(ShaderResourcePlan plan, SpecializedResourceInfo resources, BindingLayout layout, ComputeInputInfo info, Gen5ComputeSystemRegisters? systemRegisters, bool sharedInt64Atomics, bool execGuardElision, - bool signedZeroInfNanPreserve) => + bool signedZeroInfNanPreserve, bool hostWave64Supported = false, bool hostComputeSubgroup32 = false) => new(plan, resources, layout) { WaveSize = info.WaveSize, + HostWave64Supported = info.WaveSize == 64 && hostWave64Supported, + HostComputeSubgroup32 = info.WaveSize == 64 && !hostWave64Supported && hostComputeSubgroup32, EnableExecGuardElision = info.WaveSize != 64 || execGuardElision, TraceDeviceAddressFaults = SharpEmu.HLE.GpuMemory.GuestGpuMemoryHook.TraceEnabled, ScratchDwords = info.ScratchDwords, @@ -681,7 +724,7 @@ internal sealed class ShaderProgramCache internal static bool TryCompilePrewarm(ComputePrewarmRecord record, ShaderCodeCapture code, IGuestGpuBackend compiler, bool sharedInt64Atomics, bool execGuardElision, bool signedZeroInfNanPreserve, out IGuestCompiledShader? compiled, - out BindingLayout? layout, out string error) + out BindingLayout? layout, out string error, bool hostWave64Supported = false, bool hostComputeSubgroup32 = false) { compiled = null; layout = null; @@ -698,7 +741,7 @@ internal sealed class ShaderProgramCache layout = AllocateLayout(program, plan, resources, record.UserDataBase, record.UserDataCount, record.PushDataCursor, record.Info.DispatchThreadDimensions); var request = BuildComputeRequest(plan, resources, layout, record.Info, record.SystemRegisters, - sharedInt64Atomics, execGuardElision, signedZeroInfNanPreserve); + sharedInt64Atomics, execGuardElision, signedZeroInfNanPreserve, hostWave64Supported, hostComputeSubgroup32); return compiler.TryCompileProgram(request, out compiled, out error) && compiled is not null; } catch (Exception exception) diff --git a/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs new file mode 100644 index 0000000..0ea527f --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs @@ -0,0 +1,282 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Scheduling; + +namespace SharpEmu.Libs.Gpu.Rendering; + +// Resolves the programs of deferred draws on a worker thread, in issue order, while the render +// thread interprets and records. The render thread takes a result for the draw it runs only when +// the resolution completed, used the inputs the draw derives and still matches the host's +// GPU-ownership state; otherwise it resolves the draw itself. +public sealed class ParallelProgramPrefetch : IProgramPrefetch, IDisposable +{ + public sealed class Job(RegisterBanks banks, long gate) + { + internal readonly RegisterBanks Banks = banks; + // The count of stream items other than draws that must have run before the job resolves. + internal readonly long Gate = gate; + internal GraphicsProgramInputs Inputs; + internal GraphicsPrograms? Programs; + internal object? Assumptions; + // 0 queued, 1 resolved, 2 abandoned. + internal volatile int State; + } + + private readonly RenderExecutor _executor; + private readonly IRenderHost _host; + private readonly IConcurrentResolveHost _concurrent; + private readonly object _lock = new(); + private readonly Queue _queue = new(); + private readonly Thread _thread; + private int _outstanding; + private long _finished; + private long _executedGate; + // The gate the worker sleeps on; long.MaxValue while it does not. + private long _sleepingOnGate = long.MaxValue; + private bool _paused; + private bool _running; + private bool _stopping; + + public ParallelProgramPrefetch(RenderExecutor executor, IRenderHost host, IConcurrentResolveHost concurrent) + { + _executor = executor; + _host = host; + _concurrent = concurrent; + _thread = new Thread(Run) { IsBackground = true, Name = "SharpEmu program resolve" }; + _thread.Start(); + } + + // The job of the draw the render thread runs now; set around each deferred draw. + public Job? Current { get; set; } + + public long Submitted { get; private set; } + + // A gate above zero holds the job until AdvanceGate reaches it: with the command stream on its own + // thread, draws are submitted ahead of the work before them, which may change what they read. + public Job Submit(RegisterBanks banks, long gate = 0) + { + var job = new Job(banks, gate); + lock (_lock) + { + _queue.Enqueue(job); + Submitted++; + _outstanding++; + Monitor.PulseAll(_lock); + } + + return job; + } + + // Returns once the worker has finished every submitted job: it reads guest memory and cache + // state, which the render thread changes only with the worker idle. + public void WaitIdle() + { + if (Volatile.Read(ref _outstanding) == 0) + { + return; + } + + lock (_lock) + { + while (_outstanding != 0) + { + Monitor.Wait(_lock); + } + } + } + + // The render thread ran this many non-draw items: jobs gated on them may resolve. + public void AdvanceGate(long executed) + { + Interlocked.Exchange(ref _executedGate, executed); + // Only a worker asleep on a gate this reaches needs a wake: one per item costs the render + // thread a kernel call each. + if (Volatile.Read(ref _sleepingOnGate) <= executed) + { + lock (_lock) + { + Monitor.PulseAll(_lock); + } + } + } + + // Returns once the worker finished the first count jobs it was given, so a non-draw item runs + // with no resolution of the draws before it still reading. + public void WaitFinished(long count) + { + if (Volatile.Read(ref _finished) >= count) + { + return; + } + + lock (_lock) + { + while (_finished < count) + { + Monitor.Wait(_lock); + } + } + } + + // Holds the worker between jobs, for host work that changes what a resolution reads. + public void Pause() + { + lock (_lock) + { + _paused = true; + while (_running) + { + Monitor.Wait(_lock); + } + } + } + + public void Resume() + { + lock (_lock) + { + _paused = false; + Monitor.PulseAll(_lock); + } + } + + public bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs) + { + programs = null!; + var job = Current; + if (job is null || !ReferenceEquals(job.Banks, banks)) + { + return false; + } + + Current = null; + WaitFor(job); + if (job.State != 1 || !job.Inputs.Equals(inputs) || !_concurrent.IsStillCurrent(job.Assumptions!)) + { + return false; + } + + programs = job.Programs!; + return true; + } + + private void WaitFor(Job job) + { + var spinner = new SpinWait(); + for (var spin = 0; spin < 64 && job.State == 0; spin++) + { + spinner.SpinOnce(sleep1Threshold: -1); + } + + if (job.State != 0) + { + return; + } + + lock (_lock) + { + while (job.State == 0) + { + Monitor.Wait(_lock); + } + } + } + + private void Run() + { + SubmissionScheduler.ThrowOnFatal = true; + while (true) + { + Job job; + lock (_lock) + { + while (_queue.Count == 0 && !_stopping) + { + Monitor.Wait(_lock); + } + + if (_stopping) + { + return; + } + + job = _queue.Peek(); + if (job.Gate > Volatile.Read(ref _executedGate)) + { + // The gate usually opens within a few items: spin before sleeping. + Monitor.Exit(_lock); + try + { + var spinner = new SpinWait(); + for (var spin = 0; spin < 50 && job.Gate > Volatile.Read(ref _executedGate); spin++) + { + spinner.SpinOnce(sleep1Threshold: -1); + } + } + finally + { + Monitor.Enter(_lock); + } + } + + while ((_paused || job.Gate > Volatile.Read(ref _executedGate)) && !_stopping) + { + Interlocked.Exchange(ref _sleepingOnGate, _paused ? long.MaxValue : job.Gate); + if (!_paused && job.Gate <= Volatile.Read(ref _executedGate)) + { + break; + } + + Monitor.Wait(_lock, 50); + } + + Volatile.Write(ref _sleepingOnGate, long.MaxValue); + if (_stopping) + { + return; + } + + _queue.Dequeue(); + _running = true; + } + + Resolve(job); + lock (_lock) + { + _outstanding--; + _finished++; + _running = false; + Monitor.PulseAll(_lock); + } + } + } + + private void Resolve(Job job) + { + _concurrent.BeginConcurrentResolve(); + try + { + job.Inputs = RenderExecutor.ProgramInputsOf(job.Banks, _host.FormatSupport, _host.Fatal); + job.Programs = _executor.ResolveGraphicsPrograms(job.Banks, in job.Inputs); + job.Assumptions = _concurrent.EndConcurrentResolve(); + job.State = 1; + } + catch (Exception) + { + // The render thread resolves the draw itself, and fails there if the error is real. + job.Assumptions = _concurrent.EndConcurrentResolve(); + job.State = 2; + } + } + + public void Dispose() + { + lock (_lock) + { + _stopping = true; + Monitor.PulseAll(_lock); + } + } +} diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs index bf29ca8..9eafa94 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs @@ -72,6 +72,7 @@ public sealed partial class RenderExecutor if (Diagnostics.DccWriterTrace.Enabled) { Diagnostics.DccWriterTrace.CurrentProgram = program.Hash; + Diagnostics.DccWriterTrace.CurrentKind = "compute"; } if (RenderTrace.Enabled) @@ -714,15 +715,19 @@ public sealed partial class RenderExecutor var hash = input.Stage.Program!.Hash; if (!_host.TryClearImageFromBuffer(address, clear.Size, clear.PackedClear)) { - // A metadata fill may run before its target is bound; the store keeps it pending and the dispatch runs. + // A metadata fill may run before its target is bound; the store keeps it pending. The fill + // itself then runs on the host when the range is known DCC metadata: run as a dispatch it + // leaves the metadata GPU-written, and every later bind of the surface waits for the GPU to + // read it back (Astro Bot fills several surfaces this way each frame). var registered = _host.TryAbsorbDccFill(address, clear.Size, clear.PackedClear); + var filledOnHost = !registered && _host.TryFillDccMetadata(address, clear.Size, clear.PackedClear); if (RenderTrace.Enabled && RenderTrace.MetadataClear()) { RenderTrace.Write( - $"{(registered ? "Tracked" : "Deferred")} a metadata clear: shader=0x{hash:X16} address=0x{address:X16} size=0x{clear.Size:X16} value=0x{clear.PackedClear:X8}"); + $"{(registered ? "Tracked" : filledOnHost ? "Filled on the host" : "Deferred")} a metadata clear: shader=0x{hash:X16} address=0x{address:X16} size=0x{clear.Size:X16} value=0x{clear.PackedClear:X8}"); } - return registered; + return registered || filledOnHost; } if (RenderTrace.Enabled && RenderTrace.ImageClear()) diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs new file mode 100644 index 0000000..31c96ac --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs @@ -0,0 +1,115 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Buffers; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Images; + +namespace SharpEmu.Libs.Gpu.Rendering; + +// What a draw's program resolution reads besides the stage and context registers: the export +// mapping of every bound color slot, whether the pixel stage runs and whether a depth target is +// bound. All three follow from the register banks alone. +public struct GraphicsProgramInputs : IEquatable +{ + [System.Runtime.CompilerServices.InlineArray(RenderingState.ColorAttachmentCapacity)] + public struct ExportMappings + { + private ColorComponentMap _element0; + } + + public ExportMappings Mapping; + public bool PixelActive; + public bool DepthBound; + + public static GraphicsProgramInputs Create() + { + var inputs = new GraphicsProgramInputs(); + ((Span)inputs.Mapping).Fill(ColorComponentMap.Identity); + return inputs; + } + + public readonly bool Equals(GraphicsProgramInputs other) => + PixelActive == other.PixelActive && DepthBound == other.DepthBound && + ((ReadOnlySpan)Mapping).SequenceEqual(other.Mapping); + + public override readonly bool Equals(object? obj) => obj is GraphicsProgramInputs other && Equals(other); + + public override readonly int GetHashCode() + { + var hash = new HashCode(); + foreach (var map in (ReadOnlySpan)Mapping) + { + hash.Add(map.Packed); + } + + hash.Add(PixelActive); + hash.Add(DepthBound); + return hash.ToHashCode(); + } +} + +// The host side of resolving programs on a thread other than the render thread. The resolving +// thread records what it assumed of the host's GPU-ownership state; the render thread checks, +// when it takes the result, that the assumptions still hold. +public interface IConcurrentResolveHost +{ + // On the resolving thread, around one resolution. + void BeginConcurrentResolve(); + + object EndConcurrentResolve(); + + // On the render thread. + bool IsStillCurrent(object assumptions); +} + +// Program resolutions computed ahead of their draws. A draw takes one only when it was resolved +// from the same register banks with the same inputs; otherwise it resolves its programs itself. +public interface IProgramPrefetch +{ + bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs); +} + +public sealed partial class RenderExecutor +{ + public IProgramPrefetch? Prefetch { get; set; } + + // The inputs ResolveShaderPrograms derives from the targets TryResolveDrawTargets binds, from the + // registers alone: the target requests and the pixel and depth activity, without the images. + public static GraphicsProgramInputs ProgramInputsOf(RegisterBanks banks, IImageFormatSupport formats, Func fatal) + { + var inputs = GraphicsProgramInputs.Create(); + var context = banks.Context; + Span mapping = inputs.Mapping; + for (var slot = 0u; slot < ContextRegisters.ColorTargetCount; slot++) + { + if (slot != 0 && (context.RenderTargetMaskForSlot(slot) == 0 || context.ColorTargets[slot].BaseAddress == 0)) + { + continue; + } + + if (ColorTargetResolver.Resolve(context, slot, DrawLayerOffset, ignoreTargetMask: false, out var resolvedSlot) is { } resolution) + { + mapping[(int)resolvedSlot] = resolution.ExportMapping; + } + } + + inputs.DepthBound = DepthTargetResolver.Resolve(context, formats, fatal) is not null; + inputs.PixelActive = HasActivePixelShader(banks); + return inputs; + } + + // The programs of a draw with these banks and inputs, as ResolveShaderPrograms resolves them. + public GraphicsPrograms ResolveGraphicsPrograms(RegisterBanks banks, in GraphicsProgramInputs inputs) + { + var context = banks.Context; + return _pipelines.GetGraphicsPrograms( + banks.Shader.Vertex, + banks.Shader.Pixel, + context.ShaderInterface, + context, + inputs.Mapping, + inputs.PixelActive, + inputs.DepthBound); + } +} diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs index 5831fae..e1569cc 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs @@ -35,7 +35,8 @@ public sealed partial class RenderExecutor throw _host.Fatal($"The vertex input has too many buffers: count={buffers.Length} max={VertexInputInfo.MaxBuffers}."); } - Span ranges = stackalloc VertexBufferRange[VertexInputInfo.MaxBuffers]; + // Sized to this draw's buffers: a MaxBuffers span is over a kilobyte, zeroed on every draw. + Span ranges = stackalloc VertexBufferRange[buffers.Length]; var rangeCount = 0; foreach (ref readonly var vertex in buffers.AsSpan()) { @@ -54,7 +55,7 @@ public sealed partial class RenderExecutor } ranges[..rangeCount].Sort(static (left, right) => left.BaseAddress.CompareTo(right.BaseAddress)); - Span merged = stackalloc VertexBufferRange[VertexInputInfo.MaxBuffers]; + Span merged = stackalloc VertexBufferRange[buffers.Length]; var mergedCount = 0; for (var i = 0; i < rangeCount; i++) { diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs index 38fa148..2a2afcd 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs @@ -17,6 +17,12 @@ public sealed partial class RenderExecutor private bool TryResolveDrawTargets(RegisterBanks banks, in DrawCall draw, ref DrawState state) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.DrawTargetResolution); + if (Diagnostics.DccWriterTrace.Enabled) + { + Diagnostics.DccWriterTrace.CurrentKind = "draw"; + Diagnostics.DccWriterTrace.CurrentProgram = 0; + } + var context = banks.Context; if (TryResolveMultisampleColor(context)) { diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs index 8558cf4..cc6dd14 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs @@ -667,29 +667,22 @@ public sealed partial class RenderExecutor private void ResolveShaderPrograms(RegisterBanks banks, ref DrawState state) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.DrawProgramResolution); - var context = banks.Context; - Span mappingStorage = stackalloc ColorComponentMapArray[1]; - Span targetExportMapping = mappingStorage[0]; - targetExportMapping.Fill(ColorComponentMap.Identity); + var inputs = GraphicsProgramInputs.Create(); + Span targetExportMapping = inputs.Mapping; foreach (ref readonly var color in BoundColors(ref state)) { targetExportMapping[(int)color.Slot] = color.Resolution.ExportMapping; } - state.Programs = _pipelines.GetGraphicsPrograms( - banks.Shader.Vertex, - banks.Shader.Pixel, - context.ShaderInterface, - context, - targetExportMapping, - state.PixelActive, - state.Depth.HasTarget); - } + inputs.PixelActive = state.PixelActive; + inputs.DepthBound = state.Depth.HasTarget; + if (Prefetch is { } prefetch && prefetch.TryTakeGraphics(banks, in inputs, out var prefetched)) + { + state.Programs = prefetched; + return; + } - [System.Runtime.CompilerServices.InlineArray(RenderingState.ColorAttachmentCapacity)] - private struct ColorComponentMapArray - { - private ColorComponentMap _element0; + state.Programs = ResolveGraphicsPrograms(banks, in inputs); } private static void TraceDrawState(ulong submitId, RegisterBanks banks, in DrawCall draw, in DrawState state) diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs index 30b36d8..0f1bd06 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs @@ -19,7 +19,29 @@ public readonly record struct RenderingAttachment( bool HasDepth, bool DepthClear, bool HasStencil, - bool StencilClear); + bool StencilClear) +{ + // Compared by handle value: the generated comparison goes through the object comparer of ImageView + // and boxes both views, which a draw pays for every attachment it re-checks. + public bool Equals(RenderingAttachment other) => + View.Handle == other.View.Handle && + Layout == other.Layout && + Format == other.Format && + ClearWord0 == other.ClearWord0 && + ClearWord1 == other.ClearWord1 && + ClearWord2 == other.ClearWord2 && + ClearWord3 == other.ClearWord3 && + IsClear == other.IsClear && + HasDepth == other.HasDepth && + DepthClear == other.DepthClear && + HasStencil == other.HasStencil && + StencilClear == other.StencilClear; + + public override int GetHashCode() => HashCode.Combine( + View.Handle, + HashCode.Combine(Layout, Format, ClearWord0, ClearWord1, ClearWord2, ClearWord3), + HashCode.Combine(IsClear, HasDepth, DepthClear, HasStencil, StencilClear)); +} [InlineArray(RenderingState.ColorAttachmentCapacity)] public struct RenderingColorAttachments diff --git a/src/SharpEmu.Libs/Gpu/Scheduling/GpuWritebackEpoch.cs b/src/SharpEmu.Libs/Gpu/Scheduling/GpuWritebackEpoch.cs new file mode 100644 index 0000000..5d329cd --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Scheduling/GpuWritebackEpoch.cs @@ -0,0 +1,16 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Gpu.Scheduling; + +// Counts the stores of GPU data into guest memory: buffer and image downloads, and buffer transfers +// the caches complete there. Work done ahead of the render thread from guest memory is still +// current while the count has not moved. +internal static class GpuWritebackEpoch +{ + private static long _value; + + public static long Value => Volatile.Read(ref _value); + + public static void Advance() => Interlocked.Increment(ref _value); +} diff --git a/src/SharpEmu.Libs/Gpu/Scheduling/RecordingBuffer.cs b/src/SharpEmu.Libs/Gpu/Scheduling/RecordingBuffer.cs index ea228a5..6f791f2 100644 --- a/src/SharpEmu.Libs/Gpu/Scheduling/RecordingBuffer.cs +++ b/src/SharpEmu.Libs/Gpu/Scheduling/RecordingBuffer.cs @@ -1,8 +1,23 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using Silk.NET.Vulkan; + namespace SharpEmu.Libs.Gpu.Scheduling; +public readonly record struct MemoryBarrierRequest( + PipelineStageFlags SourceStages, + PipelineStageFlags DestinationStages, + AccessFlags SourceAccess, + AccessFlags DestinationAccess) +{ + public MemoryBarrierRequest Union(in MemoryBarrierRequest other) => new( + SourceStages | other.SourceStages, + DestinationStages | other.DestinationStages, + SourceAccess | other.SourceAccess, + DestinationAccess | other.DestinationAccess); +} + public enum RecordedOperation : uint { DispatchDirect, @@ -23,6 +38,8 @@ public sealed class RecordingBuffer private readonly IRenderingState _rendering; private SubmissionContext? _context; private bool _handleWasRequested; + private MemoryBarrierRequest _pendingBarrier; + private bool _hasPendingBarrier; internal nint Buffer; @@ -51,7 +68,11 @@ public sealed class RecordingBuffer public bool IsInvalid => Buffer == 0; - internal bool HasPendingCommands => _handleWasRequested || _rendering.IsRendering; + internal bool HasPendingCommands => _handleWasRequested || _hasPendingBarrier || _rendering.IsRendering; + + public long MergedBarriers { get; private set; } + + public long RecordedBarriers { get; private set; } // The native handle stays valid only until the next submission. public nint Handle @@ -64,11 +85,68 @@ public sealed class RecordingBuffer } // A caller can record native commands after it obtains the handle. + FlushPendingBarrier(); _handleWasRequested = true; return Buffer; } } + public nint StateHandle + { + get + { + if (IsInvalid) + { + throw SubmissionScheduler.Fatal("The command buffer is not recording."); + } + + _handleWasRequested = true; + return Buffer; + } + } + + public void RequestMemoryBarrier(in MemoryBarrierRequest request) + { + if (IsInvalid || _rendering.IsRendering) + { + throw SubmissionScheduler.Fatal("A memory barrier needs a recording command buffer outside rendering."); + } + + if (_hasPendingBarrier) + { + _pendingBarrier = _pendingBarrier.Union(request); + MergedBarriers++; + if (Vulkan.GpuSyncProfile.Enabled) + { + Vulkan.GpuSyncProfile.CountMergedRequest(); + } + + return; + } + + _pendingBarrier = request; + _hasPendingBarrier = true; + } + + private void FlushPendingBarrier() + { + if (!_hasPendingBarrier) + { + return; + } + + _hasPendingBarrier = false; + if (_rendering.IsRendering) + { + throw SubmissionScheduler.Fatal("A pending memory barrier reached a rendering scope."); + } + + var recorder = Owner.MemoryBarrierRecorder ?? + throw SubmissionScheduler.Fatal("A memory barrier was requested without a recorder."); + recorder(Buffer, _pendingBarrier); + RecordedBarriers++; + } + public SubmissionContext Context => _context ?? throw SubmissionScheduler.Fatal("The command buffer has no submission context."); public void SetDebugInfo(uint op, ulong submitId, uint arg0 = 0, uint arg1 = 0, uint arg2 = 0, uint arg3 = 0, ulong arg4 = 0) @@ -101,6 +179,7 @@ public sealed class RecordingBuffer _device.BeginBuffer(Buffer); _handleWasRequested = false; + _hasPendingBarrier = false; SetDebugInfo((uint)RecordedOperation.Unknown, 0); } diff --git a/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs b/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs index 7999a4d..2a5b2f9 100644 --- a/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs +++ b/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs @@ -84,13 +84,27 @@ public sealed class SubmissionScheduler : IGpuTickScheduler, IDisposable public ulong CurrentTick => _timeline.CurrentTick; + public Action? MemoryBarrierRecorder { get; set; } + public TickTimeline Timeline => _timeline; internal bool PriorityWorkerAlive => _priorityThread.IsAlive; + // Set on a thread that does work ahead of the render thread: there a fatal condition only + // abandons that work, which the render thread then does itself and fails on if it is real. + [ThreadStatic] + internal static bool ThrowOnFatal; + + public sealed class AbandonedWorkException(string message) : Exception(message); + // Report the fatal error, then return an exception to stop the caller. internal static Exception Fatal(string message) { + if (ThrowOnFatal) + { + throw new AbandonedWorkException(message); + } + Console.Error.WriteLine($"[GPU][FATAL] {message}"); OnFatal(message); return new InvalidOperationException(message); diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/GpuSyncProfile.cs b/src/SharpEmu.Libs/Gpu/Vulkan/GpuSyncProfile.cs new file mode 100644 index 0000000..d5fbd62 --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Vulkan/GpuSyncProfile.cs @@ -0,0 +1,79 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using Silk.NET.Vulkan; + +namespace SharpEmu.Libs.Gpu.Vulkan; + +internal enum NativeStateKind +{ + Pipeline, + PushConstants, + Descriptors, + VertexBuffers, + IndexBuffer, + Count, +} + +internal static class GpuSyncProfile +{ + private static readonly long[] _nativeStateRecorded = new long[(int)NativeStateKind.Count]; + private static readonly long[] _nativeStateReused = new long[(int)NativeStateKind.Count]; + private static long _barriers; + private static long _drainingBarriers; + private static long _memoryBarriers; + private static long _bufferBarriers; + private static long _imageBarriers; + private static long _mergedRequests; + private static long _renderingScopes; + private static long _batchFlushes; + + public static bool Enabled => VideoOut.RenderPhaseProfile.Enabled; + + public static void CountBarrier(PipelineStageFlags source, PipelineStageFlags destination, uint memory, uint buffer, uint image) + { + Interlocked.Increment(ref _barriers); + if ((source & PipelineStageFlags.AllCommandsBit) != 0 && (destination & (PipelineStageFlags.AllCommandsBit | PipelineStageFlags.TopOfPipeBit | PipelineStageFlags.DrawIndirectBit)) != 0) + { + Interlocked.Increment(ref _drainingBarriers); + } + + Interlocked.Add(ref _memoryBarriers, memory); + Interlocked.Add(ref _bufferBarriers, buffer); + Interlocked.Add(ref _imageBarriers, image); + } + + public static void CountMergedRequest() => Interlocked.Increment(ref _mergedRequests); + + public static void CountRenderingScope() => Interlocked.Increment(ref _renderingScopes); + + public static void CountBatchFlush() => Interlocked.Increment(ref _batchFlushes); + + public static void CountNativeState(NativeStateKind kind, bool reused) + { + if (reused) + { + Interlocked.Increment(ref _nativeStateReused[(int)kind]); + } + else + { + Interlocked.Increment(ref _nativeStateRecorded[(int)kind]); + } + } + + private static string TakeNativeStateReport() + { + var parts = new string[(int)NativeStateKind.Count]; + for (var index = 0; index < parts.Length; index++) + { + var reused = Interlocked.Exchange(ref _nativeStateReused[index], 0); + var recorded = Interlocked.Exchange(ref _nativeStateRecorded[index], 0); + parts[index] = FormattableString.Invariant($"{(NativeStateKind)index}={reused}/{reused + recorded}"); + } + + return string.Join(" ", parts); + } + + public static string TakeReport() => FormattableString.Invariant( + $"[PERF][GPU_SYNC] barriers={Interlocked.Exchange(ref _barriers, 0)} draining={Interlocked.Exchange(ref _drainingBarriers, 0)} memory={Interlocked.Exchange(ref _memoryBarriers, 0)} buffer={Interlocked.Exchange(ref _bufferBarriers, 0)} image={Interlocked.Exchange(ref _imageBarriers, 0)} merged_requests={Interlocked.Exchange(ref _mergedRequests, 0)} rendering_scopes={Interlocked.Exchange(ref _renderingScopes, 0)} draw_count_flushes={Interlocked.Exchange(ref _batchFlushes, 0)} reused {TakeNativeStateReport()}"); +} diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/GraphicsDynamicStateEpoch.cs b/src/SharpEmu.Libs/Gpu/Vulkan/GraphicsDynamicStateEpoch.cs new file mode 100644 index 0000000..9df63b5 --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Vulkan/GraphicsDynamicStateEpoch.cs @@ -0,0 +1,16 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Gpu.Vulkan; + +// Advanced by every graphics pipeline bind or dynamic state change outside the guest draw path: +// such a pipeline may fix the state the guest pipelines leave dynamic, so the dynamic state a +// guest draw last set no longer holds in the command buffer. +internal static class GraphicsDynamicStateEpoch +{ + private static long _value; + + public static long Value => Volatile.Read(ref _value); + + public static void Advance() => Interlocked.Increment(ref _value); +} diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/NativeBindingEpoch.cs b/src/SharpEmu.Libs/Gpu/Vulkan/NativeBindingEpoch.cs new file mode 100644 index 0000000..91be64c --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Vulkan/NativeBindingEpoch.cs @@ -0,0 +1,13 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Gpu.Vulkan; + +internal static class NativeBindingEpoch +{ + private static long _value; + + public static long Value => Volatile.Read(ref _value); + + public static void Advance() => Interlocked.Increment(ref _value); +} diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanAsyncReadback.cs b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanAsyncReadback.cs index 42d8420..d771e3f 100644 --- a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanAsyncReadback.cs +++ b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanAsyncReadback.cs @@ -172,10 +172,13 @@ internal sealed unsafe class VulkanAsyncReadback : IDisposable PSemaphores = &semaphore, PValues = &signalValue, }; + var waitStart = VideoOut.GpuWaitSiteProfile.Begin(); using (VideoOut.RenderPhaseProfile.MeasureDetail(VideoOut.RenderPhaseProfile.Phase.GpuCompletionWait)) { Require(vk.WaitSemaphores(_device.Device, &waitInfo, ulong.MaxValue), "vkWaitSemaphores(readback)"); } + + VideoOut.GpuWaitSiteProfile.End(waitStart); } finally { diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanSynchronization.cs b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanSynchronization.cs index d275742..93cecea 100644 --- a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanSynchronization.cs +++ b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanSynchronization.cs @@ -22,6 +22,11 @@ internal static unsafe class VulkanSynchronization uint imageMemoryBarrierCount, ImageMemoryBarrier2* imageMemoryBarriers) { + if (GpuSyncProfile.Enabled) + { + GpuSyncProfile.CountBarrier(sourceStages, destinationStages, memoryBarrierCount, bufferMemoryBarrierCount, imageMemoryBarrierCount); + } + var sourceStages2 = (PipelineStageFlags2)(ulong)sourceStages; var destinationStages2 = (PipelineStageFlags2)(ulong)destinationStages; for (var index = 0u; index < memoryBarrierCount; index++) diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs index db3e26f..cae352a 100644 --- a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs +++ b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs @@ -220,10 +220,13 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice PValues = &tick, }; Result result; + var waitStart = GpuWaitSiteProfile.Begin(); using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.GpuCompletionWait)) { result = _vk.WaitSemaphores(_device, &waitInfo, ulong.MaxValue); } + + GpuWaitSiteProfile.End(waitStart); failure = result.ToString(); return result == Result.Success && WaitRetired(tick); } diff --git a/src/SharpEmu.Libs/VideoOut/GpuWaitSiteProfile.cs b/src/SharpEmu.Libs/VideoOut/GpuWaitSiteProfile.cs new file mode 100644 index 0000000..d86cc75 --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/GpuWaitSiteProfile.cs @@ -0,0 +1,96 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; + +namespace SharpEmu.Libs.VideoOut; + +internal static class GpuWaitSiteProfile +{ + private const int SiteFrames = 5; + + private static readonly object _gate = new(); + private static readonly Dictionary _sites = new(StringComparer.Ordinal); + + private static readonly HashSet PlumbingTypes = new(StringComparer.Ordinal) + { + "GpuWaitSiteProfile", + "TickTimeline", + "SubmissionScheduler", + "VulkanTickDevice", + "VulkanAsyncReadback", + "GpuWorkerRelay", + "GpuWaitSpin", + }; + + public static long Begin() => RenderPhaseProfile.DetailMeasurementsEnabled ? Stopwatch.GetTimestamp() : 0; + + public static void End(long start) + { + if (start == 0) + { + return; + } + + var elapsed = Stopwatch.GetTimestamp() - start; + var site = Site(); + lock (_gate) + { + _sites.TryGetValue(site, out var entry); + _sites[site] = (entry.Count + 1, entry.Ticks + elapsed); + } + } + + private static string Site() + { + var names = new List(SiteFrames); + foreach (var frame in new StackTrace(2, false).GetFrames()) + { + if (frame.GetMethod() is not { } method) + { + continue; + } + + var type = method.DeclaringType; + while (type is { DeclaringType: not null } && type.Name.StartsWith('<')) + { + type = type.DeclaringType; + } + + var typeName = type?.Name ?? "?"; + if (PlumbingTypes.Contains(typeName) || method.Name.StartsWith('<')) + { + continue; + } + + names.Add($"{typeName}.{method.Name}"); + if (names.Count == SiteFrames) + { + break; + } + } + + return string.Join(" < ", names); + } + + public static void Report() + { + List> ranked; + lock (_gate) + { + if (_sites.Count == 0) + { + return; + } + + ranked = _sites.OrderByDescending(pair => pair.Value.Ticks).Take(12).ToList(); + _sites.Clear(); + } + + foreach (var (site, (count, ticks)) in ranked) + { + Console.Error.WriteLine( + $"[PERF][GPU_WAIT_SITE] ms={ticks * 1000.0 / Stopwatch.Frequency:F1} n={count} site={site}"); + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs b/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs index 8821b5d..a1f6c38 100644 --- a/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs +++ b/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs @@ -465,11 +465,14 @@ internal static class RenderPhaseProfile $"[PERF][RENDER_MS] window_s={seconds:F1} frames={frames} " + string.Join(" ", parts.Select(part => $"{part.Phase}={part.Milliseconds:F2}ms/n{part.Entries}"))); + GpuWaitSiteProfile.Report(); ReportImageUploads(); BufferUploadProfile.Report(); Console.Error.WriteLine(SharpEmu.ShaderCompiler.Resources.ResourceMaterializationCache.TakeReport()); Console.Error.WriteLine(SharpEmu.Libs.Gpu.Buffers.GuestBufferCache.TakeAsyncReadbackReport()); + Console.Error.WriteLine(SharpEmu.Libs.Gpu.Vulkan.GpuSyncProfile.TakeReport()); Console.Error.WriteLine(SharpEmu.Libs.Gpu.Images.GuestImageCache.TakeLookupReport()); + Console.Error.WriteLine(SharpEmu.Libs.Gpu.Images.GuestImageCache.TakeDccReport()); SharpEmu.ShaderCompiler.Resources.ResourceMaterializationProfile.WriteReport(); SharpEmu.Libs.Diagnostics.AgcRegisterPacketProfile.WriteReport(); SharpEmu.HLE.GpuMemory.GpuMemoryAccessProfile.WriteReport(); diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.BindingShadow.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.BindingShadow.cs new file mode 100644 index 0000000..022640c --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.BindingShadow.cs @@ -0,0 +1,249 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Vulkan; +using SharpEmu.ShaderCompiler.Resources; +using Silk.NET.Vulkan; +using VkBuffer = Silk.NET.Vulkan.Buffer; + +namespace SharpEmu.Libs.VideoOut; + +internal static unsafe partial class VulkanVideoPresenter +{ + private static readonly bool NativeStateReuse = + Environment.GetEnvironmentVariable("SHARPEMU_NATIVE_STATE_REUSE") != "0"; + + private sealed partial class Presenter + { + private const int ShadowVertexBufferCapacity = 32; + + private nint _shadowCommand; + private ulong _shadowTick; + private long _shadowEpoch = -1; + private Pipeline _shadowGraphicsPipeline; + private Pipeline _shadowComputePipeline; + private PipelineLayout _shadowPushLayout; + private ShaderStageFlags _shadowPushStages; + private uint[]? _shadowPushData; + private DescriptorShadow? _graphicsDescriptorShadow; + private DescriptorShadow? _computeDescriptorShadow; + private ulong[]? _shadowVertexBindings; + private int _shadowVertexCount = -1; + private VkBuffer _shadowIndexBuffer; + private ulong _shadowIndexOffset; + private IndexType _shadowIndexType; + private bool _shadowIndexValid; + private ulong[]? _descriptorKeyScratch; + + private sealed class DescriptorShadow + { + public bool Valid; + public PipelineLayout Layout; + public ulong[] Key = new ulong[256]; + public int Length; + } + + private bool SyncNativeShadow(CommandBuffer command) + { + if (!NativeStateReuse) + { + return false; + } + + var tick = _scheduler.CurrentTick; + var epoch = NativeBindingEpoch.Value; + if (command.Handle == _shadowCommand && tick == _shadowTick && epoch == _shadowEpoch) + { + return true; + } + + _shadowCommand = command.Handle; + _shadowTick = tick; + _shadowEpoch = epoch; + _shadowGraphicsPipeline = default; + _shadowComputePipeline = default; + _shadowPushLayout = default; + if (_graphicsDescriptorShadow is not null) + { + _graphicsDescriptorShadow.Valid = false; + } + + if (_computeDescriptorShadow is not null) + { + _computeDescriptorShadow.Valid = false; + } + + _shadowVertexCount = -1; + _shadowIndexValid = false; + return true; + } + + private void BindPipelineShadowed(CommandBuffer command, PipelineBindPoint bindPoint, Pipeline pipeline) + { + if (SyncNativeShadow(command)) + { + ref var bound = ref bindPoint == PipelineBindPoint.Compute ? ref _shadowComputePipeline : ref _shadowGraphicsPipeline; + var reused = bound.Handle == pipeline.Handle; + CountNativeState(NativeStateKind.Pipeline, reused); + if (reused) + { + return; + } + + bound = pipeline; + } + + _vk.CmdBindPipeline(command, bindPoint, pipeline); + } + + private void PushConstantsShadowed(CommandBuffer command, PipelineLayout layout, ShaderStageFlags stages, uint* data) + { + if (SyncNativeShadow(command)) + { + var values = new ReadOnlySpan(data, (int)PushData.DwordCount); + var shadowData = _shadowPushData ??= new uint[PushData.DwordCount]; + var reused = _shadowPushLayout.Handle == layout.Handle && _shadowPushStages == stages && values.SequenceEqual(shadowData); + CountNativeState(NativeStateKind.PushConstants, reused); + if (reused) + { + return; + } + + _shadowPushLayout = layout; + _shadowPushStages = stages; + values.CopyTo(shadowData); + } + + _vk.CmdPushConstants(command, layout, stages, 0, PushData.ByteSize, data); + } + + private bool DescriptorsAlreadyBound(CommandBuffer command, PipelineBindPoint bindPoint, PipelineLayout layout, ReadOnlySpan writes) + { + if (!SyncNativeShadow(command)) + { + return false; + } + + var length = DescriptorKey(writes); + var shadow = bindPoint == PipelineBindPoint.Compute + ? _computeDescriptorShadow ??= new DescriptorShadow() + : _graphicsDescriptorShadow ??= new DescriptorShadow(); + var key = _descriptorKeyScratch.AsSpan(0, length); + var reused = shadow.Valid && shadow.Layout.Handle == layout.Handle && key.SequenceEqual(shadow.Key.AsSpan(0, shadow.Length)); + CountNativeState(NativeStateKind.Descriptors, reused); + if (reused) + { + return true; + } + + if (shadow.Key.Length < length) + { + shadow.Key = new ulong[Math.Max(length, shadow.Key.Length * 2)]; + } + + key.CopyTo(shadow.Key); + shadow.Length = length; + shadow.Layout = layout; + shadow.Valid = true; + return false; + } + + private int DescriptorKey(ReadOnlySpan writes) + { + var length = 0; + _descriptorKeyScratch ??= new ulong[256]; + foreach (ref readonly var write in writes) + { + var needed = length + 2 + (int)write.DescriptorCount * 3; + if (_descriptorKeyScratch.Length < needed) + { + Array.Resize(ref _descriptorKeyScratch, Math.Max(needed, _descriptorKeyScratch.Length * 2)); + } + + var key = _descriptorKeyScratch; + key[length++] = write.DstBinding | ((ulong)write.DescriptorType << 32); + key[length++] = write.DescriptorCount | (write.PBufferInfo != null ? 1UL << 32 : 0) | (write.PImageInfo != null ? 1UL << 33 : 0); + for (var index = 0; index < write.DescriptorCount; index++) + { + if (write.PBufferInfo != null) + { + var info = write.PBufferInfo[index]; + key[length++] = info.Buffer.Handle; + key[length++] = info.Offset; + key[length++] = info.Range; + } + else if (write.PImageInfo != null) + { + var info = write.PImageInfo[index]; + key[length++] = info.Sampler.Handle; + key[length++] = info.ImageView.Handle; + key[length++] = (ulong)info.ImageLayout; + } + } + } + + return length; + } + + private bool VertexBuffersAlreadyBound(CommandBuffer command, ReadOnlySpan bindings) + { + if (!SyncNativeShadow(command) || bindings.Length > ShadowVertexBufferCapacity) + { + _shadowVertexCount = -1; + return false; + } + + var shadow = _shadowVertexBindings ??= new ulong[ShadowVertexBufferCapacity * 2]; + var reused = _shadowVertexCount == bindings.Length; + for (var index = 0; reused && index < bindings.Length; index++) + { + reused = shadow[index * 2] == bindings[index].Handle && shadow[index * 2 + 1] == bindings[index].Offset; + } + + CountNativeState(NativeStateKind.VertexBuffers, reused); + if (reused) + { + return true; + } + + for (var index = 0; index < bindings.Length; index++) + { + shadow[index * 2] = bindings[index].Handle; + shadow[index * 2 + 1] = bindings[index].Offset; + } + + _shadowVertexCount = bindings.Length; + return false; + } + + private bool IndexBufferAlreadyBound(CommandBuffer command, VkBuffer buffer, ulong offset, IndexType type) + { + if (!SyncNativeShadow(command)) + { + return false; + } + + var reused = _shadowIndexValid && _shadowIndexBuffer.Handle == buffer.Handle && _shadowIndexOffset == offset && _shadowIndexType == type; + CountNativeState(NativeStateKind.IndexBuffer, reused); + if (reused) + { + return true; + } + + _shadowIndexBuffer = buffer; + _shadowIndexOffset = offset; + _shadowIndexType = type; + _shadowIndexValid = true; + return false; + } + + private static void CountNativeState(NativeStateKind kind, bool reused) + { + if (GpuSyncProfile.Enabled) + { + GpuSyncProfile.CountNativeState(kind, reused); + } + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs index 8d57211..18d37c3 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs @@ -130,6 +130,37 @@ internal static unsafe partial class VulkanVideoPresenter // A draw-local copy used when the guest samples a depth aspect while the // same image is attached for writes. It retires with the submission. public bool IsFeedbackSnapshot; + // Taken from the presenter's pool; goes back to it when its submission retires. + public bool Pooled; + public bool InPool; + + // Back to the state of a new object, so a reused one never carries a previous binding. + public void Reset() + { + Address = 0; + ImageIdentifier = default; + Request = default; + CachedImage = null; + MipLevel = 0; + Image = default; + View = default; + Layout = ImageLayout.ShaderReadOnlyOptimal; + Sampler = default; + SamplerState = default; + IsStorage = false; + IsHostMovie = false; + HostMoviePlane = -1; + HostMovieFrameSerial = 0; + NeedsUpload = false; + StagingBuffer = default; + StagingMemory = default; + Width = 0; + Height = 0; + RowLength = 0; + DestinationSelect = 0; + MipViews = []; + IsFeedbackSnapshot = false; + } } private sealed partial class Presenter @@ -1017,6 +1048,8 @@ internal static unsafe partial class VulkanVideoPresenter continue; } + _commandBuffer = BeginBatchedGuestCommands(); + var initialized = texture.HostMoviePlane == 0 ? _hostMovieImageInitialized : _hostMovieChromaImageInitialized; var toTransfer = new ImageMemoryBarrier2 { diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs index 36e741d..77b714d 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs @@ -176,9 +176,6 @@ internal static unsafe partial class VulkanVideoPresenter private EndOfPipe _endOfPipe = null!; private AgcExports.CommandStreamTranslation _translation = null!; private ICpuMemory _guestMemory = null!; - private long _lastCommandStreamProgressTicks; - private static readonly long _blockedRetryTicks = - CommandStreamQueue.AllBlockedRetryMilliseconds * System.Diagnostics.Stopwatch.Frequency / 1000L; public CommandStreamQueue CommandStream => _commandStream; @@ -188,7 +185,6 @@ internal static unsafe partial class VulkanVideoPresenter _guestMemory = guest; _endOfPipe = new EndOfPipe(_endOfPipeEvents, this); _translation = AgcExports.CreateCommandStreamTranslation(guest, this); - _lastCommandStreamProgressTicks = System.Diagnostics.Stopwatch.GetTimestamp(); } public void EnqueueCommandStream(uint queue, ulong address, uint dwordCount, ulong submissionId, object? geometrySnapshots) @@ -204,25 +200,42 @@ internal static unsafe partial class VulkanVideoPresenter } } - // Runs slices until the budget ends or nothing is runnable; blocked heads retry every 100 ms. + // The command stream thread interprets the submissions; this thread runs what it hands over + // until the budget ends or nothing is queued. Null until the first render tick. + private CommandThread? _commandThread; + private Action? _betweenCommandItems; + + private bool OnCommandThread => _commandThread is { IsProducerThread: true }; + + // Items run in batches; completions are collected between them so blocked waits retry. + private const int CommandItemBatch = 256; + private void RunCommandStreamSlices(long renderWorkDeadline) { - var slices = 0; - while (slices < _maxGuestWorkPerRender) + if (_commandThread is null) + { + _commandThread = new CommandThread(_commandStream, _guestMemory, RunDeferredWork, _translation.Prefetch); + _betweenCommandItems = RunRelayBetweenCommandItems; + _translation.Sink = _commandThread; + _commandThread.Start(); + } + + var thread = _commandThread; + for (;;) { using (RenderPhaseProfile.Measure(RenderPhaseProfile.Phase.Collect)) { CollectCompletedGuestSubmissions(waitForOldest: false); } - SliceResult result; + int ran; using (RenderPhaseProfile.Measure(RenderPhaseProfile.Phase.CommandStream)) { var aliasAccess = _guestBacking?.TryEnterBackingAliasAccess() == true; _backingAliasAccess = aliasAccess; try { - result = _commandStream.ProcessOne(); + ran = thread.RunBatch(renderWorkDeadline, CommandItemBatch, _betweenCommandItems!); } finally { @@ -234,64 +247,162 @@ internal static unsafe partial class VulkanVideoPresenter } } - switch (result) - { - case SliceResult.NoWork: - return; - case SliceResult.AllBlocked: - RetryBlockedCommandStreamIfDue(); - return; - case SliceResult.BlockedWithoutProgress: - if (!_commandStream.HasUnblockedPending) - { - RetryBlockedCommandStreamIfDue(); - return; - } - - break; - default: - _lastCommandStreamProgressTicks = System.Diagnostics.Stopwatch.GetTimestamp(); - break; - } - - slices++; - if (System.Diagnostics.Stopwatch.GetTimestamp() >= renderWorkDeadline) + if (ran == 0 || System.Diagnostics.Stopwatch.GetTimestamp() >= renderWorkDeadline) { return; } } } - private void RetryBlockedCommandStreamIfDue() + private void RunDeferredWork(AgcExports.DeferredWork work) { - var now = System.Diagnostics.Stopwatch.GetTimestamp(); - if (now - _lastCommandStreamProgressTicks < _blockedRetryTicks) + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + try { - return; + _translation.RunDeferred(work); + } + finally + { + Interlocked.Add(ref _perfDrawTicks, System.Diagnostics.Stopwatch.GetTimestamp() - started); } - - _commandStream.RetryBlocked(); - _lastCommandStreamProgressTicks = now; } - // Milliseconds until the blocked heads are due for a retry; null when none is blocked. - private int? BlockedRetryWaitMilliseconds() + // Commands other threads post to the GPU worker run between items, with the resolution + // worker held: they change the caches it reads. + private void RunRelayBetweenCommandItems() { - if (!_vulkanReady || !_commandStream.HasPending || _commandStream.HasUnblockedPending) + if (_relay.HasPendingCommands) { - return null; + RunRelayCommandsWithPrefetchHeld(); } - - var remaining = _blockedRetryTicks - (System.Diagnostics.Stopwatch.GetTimestamp() - _lastCommandStreamProgressTicks); - return remaining <= 0 ? 0 : (int)Math.Max(1, remaining * 1000 / System.Diagnostics.Stopwatch.Frequency); } - ICpuMemory ICommandStreamHost.Memory => _guestMemory; + private void RunRelayCommandsWithPrefetchHeld() + { + var prefetch = _commandThread is not null ? _translation.Prefetch : null; + prefetch?.Pause(); + try + { + using var relayScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay); + _relay.RunPendingCommands(); + } + finally + { + prefetch?.Resume(); + } + } + + // The interpreter's own stores (labels, atomics, semaphores) follow every deferred draw. + private DrainingWriteMemory? _commandMemory; + + ICpuMemory ICommandStreamHost.Memory => _commandMemory ??= new DrainingWriteMemory(_guestMemory, DrainDraws, this); + + // On the command stream thread a store is queued behind the work before it and reads see + // the queued stores; elsewhere the deferred draws run first. + private sealed class DrainingWriteMemory(ICpuMemory inner, Action drain, Presenter presenter) : ICpuMemory, ICpuMemoryWrapper + { + public ICpuMemory Inner => inner; + + public bool TryRead(ulong virtualAddress, Span destination) => + presenter.OnCommandThread + ? presenter._commandThread!.ReadOverlay(virtualAddress, destination) + : inner.TryRead(virtualAddress, destination); + + public bool TryWrite(ulong virtualAddress, ReadOnlySpan source) + { + if (presenter.OnCommandThread) + { + presenter._commandThread!.PostWrite(virtualAddress, source); + return true; + } + + drain(); + return inner.TryWrite(virtualAddress, source); + } + + public bool TryCompare(ulong virtualAddress, ReadOnlySpan expected, out bool equal) + { + if (presenter.OnCommandThread) + { + var bytes = expected.ToArray(); + var compared = false; + var result = presenter._commandThread!.Run(() => TryCompare(virtualAddress, bytes, out compared)); + equal = compared; + return result; + } + + drain(); + return inner.TryCompare(virtualAddress, expected, out equal); + } + + public bool TryCopy(ulong destinationAddress, ulong sourceAddress, ulong length) + { + if (presenter.OnCommandThread) + { + return presenter._commandThread!.Run(() => TryCopy(destinationAddress, sourceAddress, length)); + } + + drain(); + return inner.TryCopy(destinationAddress, sourceAddress, length); + } + + public bool TryScanCString(ulong address, byte needle, bool findLast, ulong maxLength, out ulong match) => + inner.TryScanCString(address, needle, findLast, maxLength, out match); + + public bool CanRead(ulong address, ulong size) => inner.CanRead(address, size); + + public string DescribeReadRange(ulong address, ulong size) => inner.DescribeReadRange(address, size); + } + + // A read the render thread runs after everything queued before it, for the command thread. + private bool ReadOnRenderThread(ulong address, Span destination, bool operand) + { + var buffer = new byte[destination.Length]; + var read = _commandThread!.Run(() => operand ? TryReadGuestOperand(address, buffer) : TryReadGuest(address, buffer)); + buffer.CopyTo(destination); + return read; + } + + // Command memory and polled labels: GPU-written pages go through the render thread, the + // rest is read here with the stream's queued stores applied. + private bool ReadOnCommandThread(ulong address, Span destination) + { + _commandThread!.WaitForPendingRange(address, (ulong)destination.Length); + return _bufferCache.MayHaveGpuDirtyPages(address, (ulong)destination.Length) + ? ReadOnRenderThread(address, destination, operand: false) + : _commandThread!.ReadOverlay(address, destination); + } + + public bool TryReadGuestWaitOperand(ulong address, Span destination) => + OnCommandThread ? ReadOnCommandThread(address, destination) : TryReadGuestOperand(address, destination); + + public bool TryReadGuestOperand(ulong address, Span destination) + { + if (OnCommandThread) + { + return ReadOnRenderThread(address, destination, operand: true); + } + + DrainDraws(); + return TryReadGuest(address, destination); + } public bool TryReadGuest(ulong address, Span destination) { + if (OnCommandThread) + { + return ReadOnCommandThread(address, destination); + } + using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandMemorySync)) { + // GPU-written command memory is downloaded first: deferred draws are recorded before it. + if (_translation is { HasPendingDraws: true } && _bufferCache.MayHaveGpuDirtyPages(address, (ulong)destination.Length)) + { + DrainDraws(); + } + if (!_bufferCache.TrySynchronizeCpuRead(address, (ulong)destination.Length, SharpEmu.HLE.GuestMemory.GuestMemoryProfile.ReadbackSource.CommandMemoryRead)) { @@ -305,27 +416,79 @@ internal static unsafe partial class VulkanVideoPresenter public void RunPendingCommands() { + // With the stream on its own thread the render thread runs them between its items. + if (OnCommandThread) + { + return; + } + + if (_relay.HasPendingCommands) + { + DrainDraws(); + } + using var relayScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay); _relay.RunPendingCommands(); } public void BeginSubmission(int queueId, ulong submissionId, object? geometrySnapshots) + { + if (OnCommandThread) + { + // The translation state belongs to this thread; the queue context to the render thread. + _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId)); + _commandThread!.Post(() => BeginSubmissionContext(queueId, submissionId)); + return; + } + + DrainDraws(); + BeginSubmissionContext(queueId, submissionId); + _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId)); + } + + private void BeginSubmissionContext(int queueId, ulong submissionId) { using var contextScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueContext); _activeGuestQueue = new VulkanGuestQueueIdentity(_commandQueueNames[queueId], submissionId); BindSubmissionContext(_activeGuestQueue); _ = CurrentRecordingBuffer(); - _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId)); } public void Flush() { + if (OnCommandThread) + { + _commandThread!.Post(Flush); + return; + } + + DrainDraws(); using var flushScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.Flush); FlushBatchedGuestCommands(); } + // The guest reuses the frame's memory (descriptor tables, rings) once the boundary returns, + // so the draws queued before it are recorded first. + public void FlushFrame() + { + if (OnCommandThread) + { + _commandThread!.Run(Flush); + return; + } + + Flush(); + } + public void FlushAndWait() { + if (OnCommandThread) + { + _commandThread!.Run(FlushAndWait); + return; + } + + DrainDraws(); using var waitScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandGpuWait); _ = CurrentRecordingBuffer(); _scheduler.FlushAndWait(); @@ -334,16 +497,40 @@ internal static unsafe partial class VulkanVideoPresenter public void SynchronizeGpu() { + if (OnCommandThread) + { + _commandThread!.Run(SynchronizeGpu); + return; + } + + DrainDraws(); using var waitScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandGpuWait); _ = CurrentRecordingBuffer(); _scheduler.Finish(); CollectCompletedGuestSubmissions(waitForOldest: false); } - public void RunGarbageCollector() => RunGuestCacheCollection(endsFrame: false); + public void RunGarbageCollector() + { + if (OnCommandThread) + { + _commandThread!.Post(RunGarbageCollector); + return; + } + + DrainDraws(); + RunGuestCacheCollection(endsFrame: false); + } public void EmitGlobalBarrier() { + if (OnCommandThread) + { + _commandThread!.Post(EmitGlobalBarrier); + return; + } + + DrainDraws(); // Inside a rendering scope whose draws only wrote attachments, the barrier matters only // to what runs after the scope: everything before the scope is ordered by the barrier // BeginRendering records, and a draw that stores to memory ends a deferred scope first. @@ -354,6 +541,12 @@ internal static unsafe partial class VulkanVideoPresenter } EndRendering(); + if (BatchBarriers) + { + RequestGuestMemoryBarrier(GlobalMemoryBarrier); + return; + } + RecordGlobalBarrier(BeginBatchedGuestCommands()); } @@ -380,6 +573,13 @@ internal static unsafe partial class VulkanVideoPresenter public void FillBuffer(ulong address, ulong size, uint value, bool isGds) { + if (OnCommandThread) + { + _commandThread!.PostRange(() => FillBuffer(address, size, value, isGds), address, isGds ? 0 : size); + return; + } + + DrainDraws(); using var transferScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandMemoryTransfer); // Transfers the buffer cache completes in guest memory leave the rendering scope open; // its GPU paths end the scope before they record. @@ -389,6 +589,13 @@ internal static unsafe partial class VulkanVideoPresenter public void CopyBuffer(ulong destination, ulong source, ulong size, bool destinationIsGds, bool sourceIsGds) { + if (OnCommandThread) + { + _commandThread!.PostRange(() => CopyBuffer(destination, source, size, destinationIsGds, sourceIsGds), destination, destinationIsGds ? 0 : size); + return; + } + + DrainDraws(); using var transferScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandMemoryTransfer); // Transfers the buffer cache completes in guest memory leave the rendering scope open; // its GPU paths end the scope before they record. @@ -396,11 +603,30 @@ internal static unsafe partial class VulkanVideoPresenter _bufferCache.CopyBuffer(destination, source, size, destinationIsGds, sourceIsGds); } - public void ReadGds(Span destination, uint wordOffset, uint wordCount) => + public void ReadGds(Span destination, uint wordOffset, uint wordCount) + { + if (OnCommandThread) + { + var words = new uint[destination.Length]; + _commandThread!.Run(() => ReadGds(words, wordOffset, wordCount)); + words.CopyTo(destination); + return; + } + + DrainDraws(); EndOfPipe.ReadGdsWords(_bufferCache.GdsBuffer.Mapped, destination, wordOffset, wordCount); + } public void RecordEndOfPipe(in EndOfPipeWrite write) { + if (OnCommandThread) + { + var queued = write; + _commandThread!.Post(() => RecordEndOfPipe(in queued)); + return; + } + + DrainDraws(); using var completionScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandEndOfPipe); _ = BeginBatchedGuestCommands(); var buffer = _scheduler.Current; @@ -456,10 +682,26 @@ internal static unsafe partial class VulkanVideoPresenter } } - public void TriggerInterrupt(int eventId, uint contextId) => _endOfPipeEvents.TriggerInterrupt(eventId, contextId); + public void TriggerInterrupt(int eventId, uint contextId) + { + if (OnCommandThread) + { + _commandThread!.Post(() => TriggerInterrupt(eventId, contextId)); + return; + } + + DrainDraws(); + _endOfPipeEvents.TriggerInterrupt(eventId, contextId); + } public bool HasFlipSlot() { + if (OnCommandThread) + { + return _commandThread!.Run(HasFlipSlot); + } + + DrainDraws(); lock (_gate) { return _pendingGuestImagePresentations.Count < MaxPendingGuestFlipVersions; @@ -468,17 +710,47 @@ internal static unsafe partial class VulkanVideoPresenter public ulong PrepareFlip(int handle, int index, int flipMode, long flipArgument) { + if (OnCommandThread) + { + return _commandThread!.Run(() => PrepareFlip(handle, index, flipMode, flipArgument)); + } + + DrainDraws(); _ = BeginBatchedGuestCommands(); return _endOfPipe.PrepareVideoOutFlip(_scheduler.Current, handle, index, flipMode, flipArgument); } - public bool IsFlipDone(int handle, int index) => VideoOutExports.IsFlipDone(handle, index); + public bool IsFlipDone(int handle, int index) + { + if (OnCommandThread) + { + return _commandThread!.Run(() => IsFlipDone(handle, index)); + } - public void PrepareCpuFlip(int handle, int index, ulong requestId) => + DrainDraws(); + return VideoOutExports.IsFlipDone(handle, index); + } + + public void PrepareCpuFlip(int handle, int index, ulong requestId) + { + if (OnCommandThread) + { + _commandThread!.Run(() => PrepareCpuFlip(handle, index, requestId)); + return; + } + + DrainDraws(); CaptureFlip(handle, index, requestId, flipMode: 0, flipArg: 0); + } public void DrawIndexed(ulong submitId, in DrawIndexedArguments arguments) { + if (OnCommandThread) + { + _translation.DrawIndexed(submitId, in arguments); + return; + } + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); var started = System.Diagnostics.Stopwatch.GetTimestamp(); try @@ -493,6 +765,12 @@ internal static unsafe partial class VulkanVideoPresenter public void DrawAuto(ulong submitId, in DrawAutoArguments arguments) { + if (OnCommandThread) + { + _translation.DrawAuto(submitId, in arguments); + return; + } + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); var started = System.Diagnostics.Stopwatch.GetTimestamp(); try @@ -521,6 +799,13 @@ internal static unsafe partial class VulkanVideoPresenter public void DispatchDirect(ulong submitId, uint groupsX, uint groupsY, uint groupsZ, uint dispatchInitiator, ulong indirectArgumentsAddress = 0) { + if (OnCommandThread) + { + _translation.Dispatch(submitId, groupsX, groupsY, groupsZ, dispatchInitiator, indirectArgumentsAddress); + return; + } + + DrainDraws(); using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDispatchTranslation); var started = System.Diagnostics.Stopwatch.GetTimestamp(); try @@ -533,13 +818,48 @@ internal static unsafe partial class VulkanVideoPresenter } } - public void OnQueueReset(int queueId) => _translation.QueueReset(queueId); + public void OnQueueReset(int queueId) + { + if (OnCommandThread) + { + _translation.QueueReset(queueId); + return; + } + + DrainDraws(); + _translation.QueueReset(queueId); + } + + // Deferred draws run before anything else the stream asks of the host: everything but a + // draw may record, read or write what they use. Their time counts as draw time. Completion + // callbacks reach some of the same entry points from the GPU priority worker; they are not + // ordered with the stream and leave the draws to the render thread. + private void DrainDraws() + { + // Hosts built without a command stream have no translation. + if (_translation is not { HasPendingDraws: true } || !_relay.IsGpuQueueThread) + { + return; + } + + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + _translation.DrainPendingDraws(); + } + finally + { + Interlocked.Add(ref _perfDrawTicks, System.Diagnostics.Stopwatch.GetTimestamp() - started); + } + } public Exception Fatal(string message) => SubmissionScheduler.Fatal(message); // The flip packet: translation first, then the request, then the capture in stream order. public int SubmitFlipFromGpu(RecordingBuffer buffer, int handle, int index, int flipMode, long flipArg, out ulong requestId) { + DrainDraws(); _ = buffer; _translation.PrepareFlip(handle, index); var result = VideoOutExports.TryReserveFlipRequest(handle, index, flipMode, flipArg, gpuQueued: true, out requestId); @@ -563,11 +883,16 @@ internal static unsafe partial class VulkanVideoPresenter // The interpreter must suspend before capture so this worker can present queued frames. public void WaitForSubmitSlot() => throw SubmissionScheduler.Fatal("The GPU worker cannot wait for a flip slot. Suspend the command stream before capture."); - public void CompleteFlip(ulong requestId) => VideoOutExports.CompleteFlip(requestId); + public void CompleteFlip(ulong requestId) + { + DrainDraws(); + VideoOutExports.CompleteFlip(requestId); + } // Copies the display surface through the store; presentation shows the copy once its tick retires. internal void CaptureFlip(int handle, int index, ulong requestId, int flipMode, long flipArg) { + DrainDraws(); // A no-buffer flip has no image to capture; its completion still retires in order. if (VideoOutExports.ReleaseNoBufferFlip(index, requestId)) { @@ -586,7 +911,7 @@ internal static unsafe partial class VulkanVideoPresenter FlushBatchedGuestCommands(); RunGuestCacheCollection(endsFrame: true); - EnsureGuestSubmissionCapacity(); + EnsureGuestSubmissionCapacity(collectNow: true); long version; lock (_gate) { diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandThread.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandThread.cs new file mode 100644 index 0000000..0db0cd4 --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandThread.cs @@ -0,0 +1,511 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.VideoOut; + +using System.Diagnostics; +using SharpEmu.HLE; +using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Scheduling; + +internal static partial class VulkanVideoPresenter +{ + // The command stream is interpreted on its own thread, which hands its work to the render thread + // in stream order. Draws, dispatches, barriers, completions and the interpreter's own stores are + // queued; a callback that returns a result waits for the render thread to run everything before it. + private sealed class CommandThread : AgcExports.IDeferredWorkSink + { + // The thread runs this far ahead of the render thread at most: enough to keep it busy, and + // no more, since a GPU round trip (a readback, a label wait) waits for everything queued. + // Measured in Astro Bot: 4096 items halved the frame rate of a crowd scene, 16 starved the + // render thread, 32 to 64 kept it busy. + private const int MaxOutstanding = 64; + + private enum Kind : byte + { + Deferred, + Call, + Write, + } + + private sealed class Item + { + public Kind Kind; + public bool IsDraw; + public AgcExports.DeferredWork? Work; + public Action? Action; + public ulong Address; + public byte[]? Data; + // The guest range a queued fill or copy writes; zero for other items. + public ulong RangeSize; + // Resolution jobs submitted before this non-draw item; they finish before it runs. + public long JobsBefore; + public bool Sync; + public volatile bool Done; + public Exception? Failure; + } + + private readonly CommandStreamQueue _queue; + private readonly ICpuMemory _memory; + private readonly Action _runDeferred; + private readonly ParallelProgramPrefetch? _prefetch; + private readonly Thread _thread; + private readonly object _lock = new(); + private List _incoming = new(); + private List _draining = new(); + private int _drainIndex; + private int _outstanding; + // What the producer sleeps on: room in the queue, or its synchronous item. + private volatile bool _waitingForSpace; + private volatile bool _waitingForSync; + private volatile bool _consumerIdle; + private volatile bool _stopping; + private volatile Exception? _failure; + private long _enqueuedNonDraw; + private long _executedNonDraw; + + // The interpreter's stores the render thread has not applied yet, oldest first; reads on + // this thread see them over guest memory. + private readonly object _writeLock = new(); + private readonly Queue _pendingWrites = new(); + private volatile int _pendingWriteCount; + + // The destinations of queued fills and copies; a read on this thread that overlaps one waits + // for the render thread to run it. + private readonly List _pendingRanges = new(); + private volatile int _pendingRangeCount; + + private static readonly long RetryTicks = CommandStreamQueue.AllBlockedRetryMilliseconds * Stopwatch.Frequency / 1000L; + + public CommandThread(CommandStreamQueue queue, ICpuMemory memory, Action runDeferred, ParallelProgramPrefetch? prefetch) + { + _queue = queue; + _memory = memory; + _runDeferred = runDeferred; + _prefetch = prefetch; + _thread = new Thread(Run) { IsBackground = true, Name = "SharpEmu command stream" }; + } + + public void Start() => _thread.Start(); + + public bool IsProducerThread => Environment.CurrentManagedThreadId == _thread.ManagedThreadId; + + public long Gate => _enqueuedNonDraw; + + public bool HasWork => Volatile.Read(ref _outstanding) != 0 || _failure is not null; + + // Read by the render loop before it waits, so a queued item wakes it. + public bool ConsumerIdle + { + set => _consumerIdle = value; + } + + public bool IsAlive => _thread.IsAlive; + + // The producer side, on the command stream thread. + + public void EnqueueDeferred(AgcExports.DeferredWork work, bool isDraw) => + Enqueue(new Item { Kind = Kind.Deferred, Work = work, IsDraw = isDraw }); + + public void Post(Action action) => Enqueue(new Item { Kind = Kind.Call, Action = action }); + + public void Run(Action action) + { + var item = new Item { Kind = Kind.Call, Action = action, Sync = true }; + Enqueue(item); + WaitFor(item); + } + + public T Run(Func function) + { + T result = default!; + Run(() => { result = function(); }); + return result; + } + + public void PostWrite(ulong address, ReadOnlySpan data) + { + var item = new Item { Kind = Kind.Write, Address = address, Data = data.ToArray() }; + lock (_writeLock) + { + _pendingWrites.Enqueue(item); + _pendingWriteCount = _pendingWrites.Count; + } + + Enqueue(item); + } + + // A fill or copy the render thread runs in order; reads of its destination wait for it. + public void PostRange(Action action, ulong address, ulong size) + { + var item = new Item { Kind = Kind.Call, Action = action, Address = address, RangeSize = size }; + if (size != 0) + { + lock (_writeLock) + { + _pendingRanges.Add(item); + _pendingRangeCount = _pendingRanges.Count; + } + } + + Enqueue(item); + } + + // Runs every queued item when one of them is a fill or copy into this range. + public void WaitForPendingRange(ulong address, ulong size) + { + if (_pendingRangeCount != 0 && OverlapsPendingRange(address, size)) + { + Run(static () => { }); + } + } + + private bool OverlapsPendingRange(ulong address, ulong size) + { + lock (_writeLock) + { + foreach (var pending in _pendingRanges) + { + if (address < pending.Address + pending.RangeSize && pending.Address < address + size) + { + return true; + } + } + + return false; + } + } + + // Guest memory as the stream sees it: with its own stores applied, queued or not. + public bool ReadOverlay(ulong address, Span destination) + { + WaitForPendingRange(address, (ulong)destination.Length); + if (_pendingWriteCount == 0) + { + return _memory.TryRead(address, destination); + } + + lock (_writeLock) + { + if (!_memory.TryRead(address, destination)) + { + return false; + } + + var end = address + (ulong)destination.Length; + foreach (var write in _pendingWrites) + { + var writeEnd = write.Address + (ulong)write.Data!.Length; + if (write.Address >= end || writeEnd <= address) + { + continue; + } + + var start = Math.Max(address, write.Address); + var stop = Math.Min(end, writeEnd); + write.Data.AsSpan((int)(start - write.Address), (int)(stop - start)) + .CopyTo(destination[(int)(start - address)..]); + } + + return true; + } + } + + private void Enqueue(Item item) + { + ThrowIfFailed(); + if (!item.IsDraw) + { + item.JobsBefore = _prefetch?.Submitted ?? 0; + _enqueuedNonDraw++; + } + + if (Volatile.Read(ref _outstanding) >= MaxOutstanding) + { + WaitForSpace(); + } + + lock (_lock) + { + _incoming.Add(item); + } + + Interlocked.Increment(ref _outstanding); + if (_consumerIdle || item.Sync) + { + Presenter.WakeRenderThread(); + } + } + + private void WaitForSpace() + { + lock (_lock) + { + // The flag is set before the check: the render thread reads it after it changes what is checked. + for (;;) + { + _waitingForSpace = true; + Interlocked.MemoryBarrier(); + if (Volatile.Read(ref _outstanding) < MaxOutstanding / 2 || _failure is not null) + { + break; + } + + Monitor.Wait(_lock, 10); + } + + _waitingForSpace = false; + } + + ThrowIfFailed(); + } + + private void WaitFor(Item item) + { + var spinner = new SpinWait(); + for (var spin = 0; spin < 100 && !item.Done; spin++) + { + spinner.SpinOnce(sleep1Threshold: -1); + } + + if (!item.Done) + { + lock (_lock) + { + for (;;) + { + _waitingForSync = true; + Interlocked.MemoryBarrier(); + if (item.Done || _failure is not null) + { + break; + } + + Monitor.Wait(_lock, 10); + } + + _waitingForSync = false; + } + } + + if (item.Failure is { } failure) + { + throw new InvalidOperationException("A command stream callback failed on the render thread.", failure); + } + + if (!item.Done) + { + ThrowIfFailed(); + } + } + + // The consumer side, on the render thread. Runs queued items until the deadline or until + // none is left; returns how many ran. + public int RunBatch(long deadline, int maxItems, Action betweenItems) + { + if (_drainIndex == _draining.Count) + { + _draining.Clear(); + _drainIndex = 0; + lock (_lock) + { + (_incoming, _draining) = (_draining, _incoming); + } + + if (_draining.Count == 0) + { + ThrowIfFailed(); + return 0; + } + } + + var ran = 0; + while (_drainIndex < _draining.Count) + { + var item = _draining[_drainIndex]; + _draining[_drainIndex++] = null!; + Execute(item); + ran++; + var outstanding = Interlocked.Decrement(ref _outstanding); + // One pulse per wait: the producer sets its flag again if it keeps waiting. + var wakeForSync = item.Sync && _waitingForSync; + var wakeForSpace = _waitingForSpace && outstanding < MaxOutstanding / 2; + if (wakeForSync || wakeForSpace) + { + if (wakeForSync) + { + _waitingForSync = false; + } + + if (wakeForSpace) + { + _waitingForSpace = false; + } + + lock (_lock) + { + Monitor.PulseAll(_lock); + } + } + + betweenItems(); + if (ran >= maxItems || ((ran & 31) == 0 && Stopwatch.GetTimestamp() >= deadline)) + { + break; + } + } + + return ran; + } + + private void Execute(Item item) + { + if (item.IsDraw) + { + _runDeferred(item.Work!); + return; + } + + _prefetch?.WaitFinished(item.JobsBefore); + try + { + switch (item.Kind) + { + case Kind.Deferred: + _runDeferred(item.Work!); + break; + case Kind.Call: + item.Action!(); + break; + case Kind.Write: + ApplyWrite(item); + break; + } + } + catch (Exception exception) when (item.Sync) + { + item.Failure = exception; + } + catch (Exception exception) + { + _failure ??= exception; + throw; + } + finally + { + _executedNonDraw++; + if (item.RangeSize != 0) + { + lock (_writeLock) + { + _pendingRanges.Remove(item); + _pendingRangeCount = _pendingRanges.Count; + } + } + + _prefetch?.AdvanceGate(_executedNonDraw); + if (item.Sync) + { + item.Done = true; + } + } + } + + private void ApplyWrite(Item item) + { + lock (_writeLock) + { + var written = _memory.TryWrite(item.Address, item.Data); + if (!ReferenceEquals(_pendingWrites.Peek(), item)) + { + throw SubmissionScheduler.Fatal("The command stream stores ran out of order."); + } + + _pendingWrites.Dequeue(); + _pendingWriteCount = _pendingWrites.Count; + if (!written) + { + throw SubmissionScheduler.Fatal( + $"The command stream cannot write guest memory: address=0x{item.Address:X16} size={item.Data!.Length}."); + } + } + } + + public void ThrowIfFailed() + { + if (_failure is { } failure) + { + throw new InvalidOperationException("The command stream thread failed.", failure); + } + } + + // Ends the thread after its current slice; the render thread keeps running its items + // meanwhile, since the slice may wait for one. + public void Stop(Action runItems) + { + _stopping = true; + _queue.Wake(); + var deadline = Stopwatch.GetTimestamp() + 5 * Stopwatch.Frequency; + while (_thread.IsAlive && Stopwatch.GetTimestamp() < deadline) + { + runItems(); + _thread.Join(1); + } + + runItems(); + } + + private void Run() + { + try + { + var lastProgress = Stopwatch.GetTimestamp(); + while (!_stopping) + { + var result = _queue.ProcessOne(); + switch (result) + { + case SliceResult.NoWork: + _ = _queue.WaitForWork(8); + break; + case SliceResult.AllBlocked: + WaitBlocked(ref lastProgress); + break; + case SliceResult.BlockedWithoutProgress: + if (!_queue.HasUnblockedPending) + { + WaitBlocked(ref lastProgress); + } + + break; + default: + lastProgress = Stopwatch.GetTimestamp(); + break; + } + } + } + catch (Exception exception) + { + _failure ??= exception; + Presenter.WakeRenderThread(); + } + } + + // Blocked heads retry when a GPU completion or a presentation unblocks them, or every + // retry interval without progress. + private void WaitBlocked(ref long lastProgress) + { + var remaining = RetryTicks - (Stopwatch.GetTimestamp() - lastProgress); + if (remaining <= 0) + { + _queue.RetryBlocked(); + lastProgress = Stopwatch.GetTimestamp(); + return; + } + + _ = _queue.WaitForRetryInterval((int)Math.Max(1, remaining * 1000 / Stopwatch.Frequency)); + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ConcurrentResolve.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ConcurrentResolve.cs new file mode 100644 index 0000000..ad0223e --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ConcurrentResolve.cs @@ -0,0 +1,152 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.VideoOut; + +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Scheduling; + +internal static unsafe partial class VulkanVideoPresenter +{ + internal enum ConcurrentQuery : byte + { + DirtyBytes, + CleanWord, + Resident, + ResidentClean, + } + + internal sealed class ConcurrentAssumptions(long bufferVersion, long writebackEpoch) + { + public readonly long BufferVersion = bufferVersion; + public readonly long WritebackEpoch = writebackEpoch; + public readonly List<(ConcurrentQuery Query, ulong Address, ulong Size, bool Answer)> Answers = new(); + public bool ReadsImageOwnership; + + public void Record(ConcurrentQuery query, ulong address, ulong size, bool answer) + { + ReadsImageOwnership |= query is ConcurrentQuery.CleanWord or ConcurrentQuery.ResidentClean; + for (var index = Answers.Count - 1; index >= Math.Max(0, Answers.Count - 8); index--) + { + var previous = Answers[index]; + if (previous.Query != query || previous.Answer != answer) + continue; + if (previous.Address == address && previous.Size == size) + return; + // No owned bytes in either range proves the union, but an owned byte does not. + if (!answer && size <= ulong.MaxValue - address && previous.Size <= ulong.MaxValue - previous.Address && + address <= previous.Address + previous.Size && previous.Address <= address + size) + { + var start = Math.Min(address, previous.Address); + var end = Math.Max(address + size, previous.Address + previous.Size); + Answers[index] = (query, start, end - start, false); + return; + } + } + + Answers.Add((query, address, size, answer)); + } + } + + private sealed partial class Presenter : IConcurrentResolveHost + { + // This partial serves the guest readers of program resolution on a thread resolving draws + // ahead of the render thread. That thread never synchronizes GPU-owned memory: a word the + // GPU owns abandons the resolution, which the render thread then does itself. Every + // ownership answer it relied on is recorded and checked again when the result is taken. + + // Set only on the resolving thread, around one resolution. + [ThreadStatic] + private static ConcurrentAssumptions? _concurrentAssumptions; + + void IConcurrentResolveHost.BeginConcurrentResolve() => + _concurrentAssumptions = new ConcurrentAssumptions(_bufferCache.GpuModifiedVersion, GpuWritebackEpoch.Value); + + object IConcurrentResolveHost.EndConcurrentResolve() + { + var assumptions = _concurrentAssumptions ?? throw new InvalidOperationException("No concurrent resolution is open."); + _concurrentAssumptions = null; + return assumptions; + } + + // GPU data stored into guest memory since the resolution may have changed what it read; + // otherwise the answers stand while the ownership they came from has not changed. + bool IConcurrentResolveHost.IsStillCurrent(object assumptionsObject) + { + var assumptions = (ConcurrentAssumptions)assumptionsObject; + if (assumptions.WritebackEpoch != GpuWritebackEpoch.Value) + { + return false; + } + + if (assumptions.BufferVersion == _bufferCache.GpuModifiedVersion && !assumptions.ReadsImageOwnership) + { + return true; + } + + foreach (var (query, address, size, answer) in assumptions.Answers) + { + var now = query switch + { + ConcurrentQuery.DirtyBytes => _bufferCache.HasGpuDirtyBytes(address, size), + ConcurrentQuery.CleanWord => IsGpuOwnedForCleanRead(address, size), + ConcurrentQuery.Resident => IsGpuOwnedForResidentRead(address, size, clean: false), + _ => IsGpuOwnedForResidentRead(address, size, clean: true), + }; + if (now != answer) + { + return false; + } + } + + return true; + } + + private bool IsGpuOwnedForCleanRead(ulong address, ulong size) => + (_bufferCache.MayHaveGpuDirtyPages(address, size) && _bufferCache.HasGpuDirtyPages(address, size)) || + _bufferCache.HasGpuDirtyBytes(address, size) || + _imageCache.HasGpuModifiedImageBytes(address, size); + + private bool IsGpuOwnedForResidentRead(ulong address, ulong size, bool clean) => + _bufferCache.HasGpuDirtyBytes(address, size) || + (clean && (_bufferCache.HasGpuDirtyPages(address, size) || _imageCache.HasGpuModifiedImageBytes(address, size))); + + private bool TryReadGuestWordConcurrent(ulong address, out uint word, ConcurrentAssumptions assumptions) + { + word = 0; + if (!_guestMemory.CanRead(address, sizeof(uint))) + { + return false; + } + + var dirty = _bufferCache.HasGpuDirtyBytes(address, sizeof(uint)); + assumptions.Record(ConcurrentQuery.DirtyBytes, address, sizeof(uint), dirty); + if (dirty) + { + throw new SubmissionScheduler.AbandonedWorkException($"The resolution reads GPU-owned memory: address=0x{address:X16}."); + } + + Span bytes = stackalloc byte[sizeof(uint)]; + if (!_guestMemory.TryRead(address, bytes)) + { + return false; + } + + word = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes); + return true; + } + + private bool TryReadResidentGuestBytesConcurrent(ulong address, Span destination, bool clean, ConcurrentAssumptions assumptions) + { + var size = (ulong)destination.Length; + if (!_guestMemory.CanRead(address, size)) + { + return false; + } + + var owned = IsGpuOwnedForResidentRead(address, size, clean); + assumptions.Record(clean ? ConcurrentQuery.ResidentClean : ConcurrentQuery.Resident, address, size, owned); + return !owned && _guestMemory.TryRead(address, destination); + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs index cdc3b6d..3200cce 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs @@ -111,21 +111,89 @@ internal static unsafe partial class VulkanVideoPresenter Atomic: image.Atomic); private static readonly System.Collections.Concurrent.ConcurrentDictionary _tracedTextureBindings = new(); + // The last lookup of one image of one program. A draw binds the same descriptor as the + // previous draw of its program most of the time; while no image the lookup could find has + // changed (the cache's lookup generation), the request and the image found are the same. + private struct ImageBindingMemo + { + public uint[]? Words; + public ulong Generation; + public TextureRequestResolution Resolution; + public ResourceSlotIdentifier Found; + public uint MetadataBaseLayer; + public bool CompletesLookup; + + public readonly bool Matches(uint[] words, ulong generation) => + Words is not null && Generation == generation && words.AsSpan().SequenceEqual(Words); + } + + private Dictionary? _imageBindingMemos; + private ShaderProgramInfo? _lastMemoProgram; + private ImageBindingMemo[] _lastMemos = []; + + // A draw's images resolve one after another for one program, so the last program's memos + // are kept at hand. + private ref ImageBindingMemo ImageBindingMemoOf(ShaderProgramInfo program, int index) + { + if (!ReferenceEquals(program, _lastMemoProgram) || index >= _lastMemos.Length) + { + var all = _imageBindingMemos ??= new Dictionary(ReferenceEqualityComparer.Instance); + if (!all.TryGetValue(program, out var memos) || index >= memos.Length) + { + Array.Resize(ref memos, Math.Max(index + 1, program.Resources?.Info.Images.Count ?? 0)); + all[program] = memos; + } + + _lastMemoProgram = program; + _lastMemos = memos; + } + + return ref _lastMemos[index]; + } // Render-state discovery for one shader image; the view is acquired later with the draw. private TextureResource ResolveImageBinding(ImageResource image, uint[] words, ShaderProgramInfo program, int index) { + ref var memo = ref ImageBindingMemoOf(program, index); if (words.Length < 4) { throw SubmissionScheduler.Fatal($"An image descriptor is too short: image={index} words={words.Length} hash=0x{program.Hash:X16}."); } var storage = image.ResourceClass == ShaderCompiler.Resources.ImageResourceClass.Storage; - var resolution = ImageRequestBuilders.Texture(words, ShapeOf(image)); _ = BeginBatchedGuestCommands(); - var request = resolution.Request; - var imageIdentifier = _imageCache.FindImage(ref request, resolution.ExactFormat); - resolution = resolution with { Request = request }; + TextureRequestResolution resolution; + ResourceSlotIdentifier imageIdentifier; + if (memo.Matches(words, _imageCache.LookupGeneration) && _imageCache.TryTouchRemembered(memo.Found)) + { + resolution = memo.Resolution; + imageIdentifier = memo.Found; + if (memo.CompletesLookup) + { + _imageCache.CompleteLookup(imageIdentifier, resolution.Request, memo.MetadataBaseLayer); + } + } + else + { + resolution = ImageRequestBuilders.Texture(words, ShapeOf(image)); + var request = resolution.Request; + var metadataBaseLayer = request.View.BaseLayer; + imageIdentifier = _imageCache.FindImage(ref request, resolution.ExactFormat); + resolution = resolution with { Request = request }; + memo = !ImageDescription.IsEmptyRange(request.Description.Data) + ? new ImageBindingMemo + { + Words = (uint[])words.Clone(), + Generation = _imageCache.LookupGeneration, + Resolution = resolution, + Found = imageIdentifier, + MetadataBaseLayer = metadataBaseLayer, + CompletesLookup = request.Description.Metadata.Kind == MetadataKind.Dcc || + _imageCache.GetImage(imageIdentifier).Description.DccSliceSize != 0, + } + : default; + } + imageIdentifier = ImageRequestBuilders.ValidateTextureOwner(_imageCache, imageIdentifier, resolution); BindImage(imageIdentifier, storage); var descriptor = new TextureDescriptorWords(words); @@ -151,16 +219,47 @@ internal static unsafe partial class VulkanVideoPresenter Console.Error.WriteLine(line); } } - return new TextureResource + var texture = RentTextureResource(); + texture.Address = descriptor.BaseAddress; + texture.ImageIdentifier = imageIdentifier; + texture.Request = resolution.Request; + texture.IsStorage = storage; + texture.DestinationSelect = words[3] & 0xFFFu; + texture.Width = descriptor.Width; + texture.Height = descriptor.Height; + return texture; + } + + // Binding descriptions are large (an image request carries its whole mip layout) and one is made + // per shader image per draw, so they are reused once the submission that read them has retired. + // Only the render thread rents and returns them. + // Sized for the bindings of every draw still in flight (a few frames of a thousand draws each); + // a smaller cap drops returns during busy scenes and the pool allocates again. + private const int MaxPooledTextureResources = 32768; + private Stack? _texturePool; + + private TextureResource RentTextureResource() + { + if (_texturePool is not null && _texturePool.TryPop(out var texture)) { - Address = descriptor.BaseAddress, - ImageIdentifier = imageIdentifier, - Request = request, - IsStorage = storage, - DestinationSelect = words[3] & 0xFFFu, - Width = descriptor.Width, - Height = descriptor.Height, - }; + texture.InPool = false; + return texture; + } + + return new TextureResource { Pooled = true }; + } + + private void ReturnTextureResource(TextureResource texture) + { + _texturePool ??= new Stack(); + if (!texture.Pooled || texture.InPool || _texturePool.Count >= MaxPooledTextureResources) + { + return; + } + + texture.Reset(); + texture.InPool = true; + _texturePool.Push(texture); } // Compare bits stay only on depth-compare samplers; a forced point sampler drops its filters. @@ -572,6 +671,12 @@ internal static unsafe partial class VulkanVideoPresenter } prepared.Descriptors.Buffers = views; + if (layout.MipStatisticsCount != 0) + { + LodStatistics().EnsureInitialized(); + WriteMipStatisticsFields(shaderData, layout, info, snapshot.Images); + } + if (layout.Find(DescriptorBindingKind.FlattenedResourceTable) is not null) { prepared.Descriptors.FlattenedTable = UploadDwords(snapshot.FlattenedResourceTable, "flattened resource table", program); @@ -751,7 +856,7 @@ internal static unsafe partial class VulkanVideoPresenter using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.DescriptorCommit); var preparation = RequirePreparation(); var entry = RequirePipelineEntry(in pipeline); - var command = BeginBatchedGuestCommands(); + var command = BeginBatchedGuestStateCommands(); _commandBuffer = command; var writeCount = 0; var bufferCount = 0; @@ -768,8 +873,11 @@ internal static unsafe partial class VulkanVideoPresenter throw SubmissionScheduler.Fatal($"A stage does not belong to the bind point: stage={stage.Program.Stage} bindPoint={bindPoint}."); } - foreach (var binding in stage.Layout.Descriptors) + // Indexed loops over the layout's lists: a foreach on IReadOnlyList allocates an enumerator per draw. + var countedBindings = stage.Layout.Descriptors; + for (var bindingIndex = 0; bindingIndex < countedBindings.Count; bindingIndex++) { + var binding = countedBindings[bindingIndex]; writeCount++; var count = (int)DescriptorWriter.DescriptorCount(binding); if (ImageDescriptorBinding.ResourceClass(binding.Kind) != ShaderCompiler.Resources.ImageResourceClass.None || binding.Kind == DescriptorBindingKind.Samplers) @@ -862,8 +970,11 @@ internal static unsafe partial class VulkanVideoPresenter var occurrences = occurrenceScratch[..descriptors.Images.Length]; occurrences.Clear(); - foreach (var binding in stage.Layout.Descriptors) + var bindings = stage.Layout.Descriptors; + for (var bindingIndex = 0; bindingIndex < bindings.Count; bindingIndex++) { + var binding = bindings[bindingIndex]; + var resources = binding.Resources; var write = new WriteDescriptorSet { SType = StructureType.WriteDescriptorSet, @@ -875,8 +986,9 @@ internal static unsafe partial class VulkanVideoPresenter var imageStart = imageIndex; if (ImageDescriptorBinding.ResourceClass(binding.Kind) != ShaderCompiler.Resources.ImageResourceClass.None) { - foreach (var resource in binding.Resources) + for (var resourceIndex = 0; resourceIndex < resources.Count; resourceIndex++) { + var resource = resources[resourceIndex]; imageInfos[imageIndex++] = ImageInfo(descriptors.Images[(int)resource], occurrences[(int)resource]++, program, (int)resource); } } @@ -885,8 +997,9 @@ internal static unsafe partial class VulkanVideoPresenter switch (binding.Kind) { case DescriptorBindingKind.Buffers: - foreach (var resource in binding.Resources) + for (var resourceIndex = 0; resourceIndex < resources.Count; resourceIndex++) { + var resource = resources[resourceIndex]; var view = descriptors.Buffers[(int)resource]; if (view.Buffer.Handle == 0) { @@ -896,6 +1009,9 @@ internal static unsafe partial class VulkanVideoPresenter bufferInfos[bufferIndex++] = new DescriptorBufferInfo { Buffer = view.Buffer, Offset = view.Offset, Range = view.Range }; } + break; + case DescriptorBindingKind.MipStatistics: + bufferInfos[bufferIndex++] = MipStatisticsDescriptor(); break; case DescriptorBindingKind.DeviceAddressPageTable: case DescriptorBindingKind.FaultBuffer: @@ -925,8 +1041,9 @@ internal static unsafe partial class VulkanVideoPresenter } case DescriptorBindingKind.Samplers: - foreach (var resource in binding.Resources) + for (var resourceIndex = 0; resourceIndex < resources.Count; resourceIndex++) { + var resource = resources[resourceIndex]; var sampler = descriptors.Samplers[(int)resource]; if (sampler.Handle == 0) { @@ -983,10 +1100,10 @@ internal static unsafe partial class VulkanVideoPresenter if (hasPushData) { var pushStages = bindPoint == PipelineBindPoint.Graphics ? ShaderStageFlags.VertexBit | ShaderStageFlags.FragmentBit : ShaderStageFlags.ComputeBit; - _vk.CmdPushConstants(command, entry.Layout, pushStages, 0, PushData.ByteSize, pushData); + PushConstantsShadowed(command, entry.Layout, pushStages, pushData); } - if (writeIndex != 0) + if (writeIndex != 0 && !DescriptorsAlreadyBound(command, bindPoint, entry.Layout, writes[..writeIndex])) { fixed (WriteDescriptorSet* writePointer = writes) { diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs index 9d0d1d0..08099d2 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs @@ -608,11 +608,16 @@ internal static unsafe partial class VulkanVideoPresenter _supportsShaderSignedZeroInfNanPreserveFloat32 = floatControls.ShaderSignedZeroInfNanPreserveFloat32; SetNativeSubgroupCapabilities(subgroup.SubgroupSize, subgroup.SupportedStages); + SetMipStatisticsCapabilities(true, subgroup.SupportedOperations, subgroup.SupportedStages); _canRequireComputeSubgroup32 = subgroup.SubgroupSize != RdnaSubgroupSize && subgroupSizeControl.MinSubgroupSize <= RdnaSubgroupSize && subgroupSizeControl.MaxSubgroupSize >= RdnaSubgroupSize && (subgroupSizeControl.RequiredSubgroupSizeStages & ShaderStageFlags.ComputeBit) != 0; + _canRequireComputeSubgroup64 = + subgroupSizeControl.MinSubgroupSize <= Wave64SubgroupSize && + subgroupSizeControl.MaxSubgroupSize >= Wave64SubgroupSize && + (subgroupSizeControl.RequiredSubgroupSizeStages & ShaderStageFlags.ComputeBit) != 0; _maxComputeWorkgroupSubgroups = subgroupSizeControl.MaxComputeWorkgroupSubgroups; // MoltenVK reports a zero OpArrayLength for storage buffers bound through push // descriptors, which turns every bounds-checked load into zero and drops every store. @@ -750,6 +755,11 @@ internal static unsafe partial class VulkanVideoPresenter queueInfoCount = 2; } _vk.GetPhysicalDeviceFeatures(_physicalDevice, out var supportedFeatures); + if (!supportedFeatures.FragmentStoresAndAtomics) + { + ShaderCompiler.Resources.ResourceMaterializer.RecordMipStatistics = false; + } + _supportsIndependentBlend = supportedFeatures.IndependentBlend; _supportsDepthBiasClamp = supportedFeatures.DepthBiasClamp; _supportsDepthBounds = supportedFeatures.DepthBounds; @@ -945,6 +955,7 @@ internal static unsafe partial class VulkanVideoPresenter var supportsNullDescriptor = robustness2Features.NullDescriptor; var supportsRobustness2 = supportsRobustImageAccess2 || supportsNullDescriptor; _canRequireComputeSubgroup32 &= vulkan13Features.SubgroupSizeControl; + _canRequireComputeSubgroup64 &= vulkan13Features.SubgroupSizeControl; SetSharedInt64AtomicsCapability(supportsSharedInt64Atomics); if (!supportsSharedInt64Atomics) { @@ -1157,7 +1168,7 @@ internal static unsafe partial class VulkanVideoPresenter SType = StructureType.PhysicalDeviceVulkan13Features, DynamicRendering = true, Synchronization2 = true, - SubgroupSizeControl = _canRequireComputeSubgroup32, + SubgroupSizeControl = _canRequireComputeSubgroup32 || _canRequireComputeSubgroup64, PNext = renderingChain, }; var features2 = new PhysicalDeviceFeatures2 diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs index 08a48bb..552baeb 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs @@ -275,8 +275,12 @@ internal static unsafe partial class VulkanVideoPresenter { RecycleHostBuffer(texture.StagingBuffer, texture.StagingMemory); } + + ReturnTextureResource(texture); } + resources.Textures = []; + foreach (var snapshot in resources.FeedbackSnapshots) { RetireFeedbackSnapshot(snapshot); diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs index 4c56738..4f721ab 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs @@ -47,13 +47,32 @@ internal static unsafe partial class VulkanVideoPresenter private CommandBuffer BeginBatchedGuestCommands() { var commandBuffer = CurrentRecordingBuffer(); + OpenGuestBatch(); + return commandBuffer; + } + + private CommandBuffer BeginBatchedGuestStateCommands() + { + var recording = CurrentRecording(); + OpenGuestBatch(); + return new CommandBuffer(recording.StateHandle); + } + + private void RequestGuestMemoryBarrier(in MemoryBarrierRequest request) + { + var recording = CurrentRecording(); + OpenGuestBatch(); + _scheduler.MemoryBarrierRecorder ??= RecordRequestedMemoryBarrier; + recording.RequestMemoryBarrier(request); + } + + private void OpenGuestBatch() + { if (!_batchOpen) { _batchOpen = true; _batchDrawCount = 0; } - - return commandBuffer; } // Submits the current recording buffer with everything the batch lists own. @@ -108,9 +127,23 @@ internal static unsafe partial class VulkanVideoPresenter retireBuffers)); } - private void EnsureGuestSubmissionCapacity() + // Collecting completed submissions polls the GPU timeline twice and may wait for the priority + // worker. Every draw used to pay that; completed work only frees resources and runs deferred + // callbacks, so a collection a few hundred microseconds later changes nothing but how long + // those resources stay reserved. A full in-flight queue still collects, and waits, at once. + private static readonly long CollectIntervalTicks = System.Diagnostics.Stopwatch.Frequency / 4000; + private long _lastCollectTicks; + + private void EnsureGuestSubmissionCapacity(bool collectNow = false) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.SubmissionCapacity); + var now = System.Diagnostics.Stopwatch.GetTimestamp(); + if (!collectNow && _pendingGuestSubmissions.Count < MaxInFlightGuestSubmissions && now - _lastCollectTicks < CollectIntervalTicks) + { + return; + } + + _lastCollectTicks = now; CollectCompletedGuestSubmissions(waitForOldest: false); if (_pendingGuestSubmissions.Count >= MaxInFlightGuestSubmissions) { @@ -149,6 +182,7 @@ internal static unsafe partial class VulkanVideoPresenter } _commandStream.NotifyCompletedGpuTick(_scheduler.Timeline.CompletedTick); + PublishLodStatistics(); ProcessDeferredTextureDestroys(); } diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.LodStatistics.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.LodStatistics.cs new file mode 100644 index 0000000..6b1d0de --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.LodStatistics.cs @@ -0,0 +1,199 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Collections.Concurrent; +using SharpEmu.Libs.Gpu.Buffers; +using SharpEmu.Libs.Gpu.Images; +using SharpEmu.Libs.Gpu.Scheduling; +using SharpEmu.ShaderCompiler.Resources; +using Silk.NET.Vulkan; + +namespace SharpEmu.Libs.VideoOut; + +internal static unsafe partial class VulkanVideoPresenter +{ + private static readonly bool GpuLodStatistics = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOD_STATS"), "gpu", StringComparison.Ordinal); + + private static int _fragmentSubgroupArithmetic; + + internal static bool FragmentSubgroupArithmetic => Volatile.Read(ref _fragmentSubgroupArithmetic) != 0; + + private static void SetMipStatisticsCapabilities(bool fragmentStoresAndAtomics, SubgroupFeatureFlags operations, ShaderStageFlags stages) + { + ResourceMaterializer.RecordMipStatistics = GpuLodStatistics && fragmentStoresAndAtomics; + var arithmetic = (operations & (SubgroupFeatureFlags.ArithmeticBit | SubgroupFeatureFlags.BallotBit | SubgroupFeatureFlags.BasicBit)) == + (SubgroupFeatureFlags.ArithmeticBit | SubgroupFeatureFlags.BallotBit | SubgroupFeatureFlags.BasicBit) && + (stages & ShaderStageFlags.FragmentBit) != 0; + Volatile.Write(ref _fragmentSubgroupArithmetic, arithmetic ? 1 : 0); + } + + private sealed partial class Presenter + { + private LodStatisticsRecorder? _lodStatistics; + + private LodStatisticsRecorder LodStatistics() => + _lodStatistics ??= new LodStatisticsRecorder(_deviceInfo, _scheduler); + + bool Gpu.Pipelines.IShaderPipelineHost.MipStatisticsSubgroupReduction => FragmentSubgroupArithmetic && GraphicsSubgroupOperationsEnabled; + + bool Gpu.GpuCommands.ICommandStreamHost.TryRecordLodStatistics(ulong destination, uint size) + { + if (!ResourceMaterializer.RecordMipStatistics) + { + if (!OnCommandThread) + { + return false; + } + + _commandThread!.PostRange(() => WriteBindingLodStatistics(destination, size), destination, size); + return true; + } + + if (size < MipStatistics.ReportBytes) + { + return false; + } + + if (OnCommandThread) + { + _commandThread!.Post(() => RecordLodStatistics(destination, size)); + return true; + } + + RecordLodStatistics(destination, size); + return true; + } + + private void RecordLodStatistics(ulong destination, uint size) + { + DrainDraws(); + EndRendering(); + _ = BeginBatchedGuestCommands(); + LodStatistics().Record(destination, size); + } + + private void WriteBindingLodStatistics(ulong destination, uint size) + { + var report = new byte[size]; + MipStatistics.Shared.WriteReport(report); + if (!((Gpu.GpuCommands.ICommandStreamHost)this).Memory.TryWrite(destination, report)) + { + throw SubmissionScheduler.Fatal($"The LOD statistics report could not be written: address=0x{destination:X16} size={report.Length}."); + } + } + + private void PublishLodStatistics() + { + if (_lodStatistics is not { } recorder) + { + return; + } + + while (recorder.TryTakeReport(out var destination, out var report)) + { + if (!((Gpu.GpuCommands.ICommandStreamHost)this).Memory.TryWrite(destination, report)) + { + throw SubmissionScheduler.Fatal($"The LOD statistics report could not be written: address=0x{destination:X16} size={report.Length}."); + } + } + } + + private void WriteMipStatisticsFields(uint[] shaderData, BindingLayout layout, ShaderResourceInfo info, uint[][] images) + { + for (var index = 0; index < layout.MipStatisticsCount; index++) + { + var image = index < info.Images.Count ? info.Images[index] : null; + var words = index < images.Length ? images[index] : null; + shaderData[layout.MipStatisticsDword + (uint)index] = image is { MipStatistics: true } && words is { Length: >= 8 } + ? MipStatistics.ImageField(words) + : MipStatistics.NoCounterField; + } + } + + private DescriptorBufferInfo MipStatisticsDescriptor() + { + var counters = LodStatistics().Counters; + return new DescriptorBufferInfo { Buffer = counters.Handle, Offset = 0, Range = counters.Size }; + } + } + + private sealed class LodStatisticsRecorder + { + private const int Slots = 32; + private const ulong SlotStride = 4096; + + private readonly SubmissionScheduler _scheduler; + private readonly GpuBuffer _publish; + private readonly ulong[] _slotTicks = new ulong[Slots]; + private readonly ConcurrentQueue<(ulong Destination, byte[] Report)> _ready = new(); + private int _nextSlot; + private bool _initialized; + + public LodStatisticsRecorder(GpuDeviceInfo device, SubmissionScheduler scheduler) + { + _scheduler = scheduler; + Counters = new GpuBuffer(device, scheduler, GpuBufferUsage.DeviceLocal, 0, GpuBuffer.AllFlags, MipStatistics.CounterBufferBytes); + _publish = new GpuBuffer(device, scheduler, GpuBufferUsage.Download, 0, GpuBuffer.AllFlags, Slots * SlotStride); + } + + public GpuBuffer Counters { get; } + + public void Reset() + { + Counters.Fill(0, MipStatistics.CounterWords * sizeof(uint), uint.MaxValue); + Counters.Fill(MipStatistics.CounterWords * sizeof(uint), MipStatistics.CounterWords * sizeof(uint), 0); + _initialized = true; + } + + public void EnsureInitialized() + { + if (!_initialized) + { + Reset(); + } + } + + public void Record(ulong destination, uint size) + { + EnsureInitialized(); + var slot = _nextSlot; + _nextSlot = (_nextSlot + 1) % Slots; + if (_slotTicks[slot] != 0 && !_scheduler.IsTickComplete(_slotTicks[slot])) + { + _scheduler.Wait(_slotTicks[slot]); + } + + var offset = (ulong)slot * SlotStride; + _publish.CopyFrom( + _scheduler.Current, Counters, 0, offset, MipStatistics.CounterBufferBytes, + AccessFlags.ShaderWriteBit | AccessFlags.MemoryWriteBit, AccessFlags.None, + AccessFlags.ShaderReadBit | AccessFlags.ShaderWriteBit | AccessFlags.MemoryReadBit | AccessFlags.MemoryWriteBit, AccessFlags.HostReadBit); + Reset(); + _slotTicks[slot] = _scheduler.CurrentTick; + var reportSize = (int)Math.Min(size, (uint)MipStatistics.ReportBytes); + _scheduler.QueueCompletionAction(() => + { + _publish.Invalidate(offset, MipStatistics.CounterBufferBytes); + var words = System.Runtime.InteropServices.MemoryMarshal.Cast( + _publish.Mapped.Slice((int)offset, (int)MipStatistics.CounterBufferBytes)); + var report = new byte[reportSize]; + MipStatistics.PackReport(words, report); + _ready.Enqueue((destination, report)); + }); + } + + public bool TryTakeReport(out ulong destination, out byte[] report) + { + if (_ready.TryDequeue(out var entry)) + { + (destination, report) = entry; + return true; + } + + destination = 0; + report = []; + return false; + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs index 3b0e354..d9cecb5 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs @@ -67,23 +67,47 @@ internal static unsafe partial class VulkanVideoPresenter uint IShaderPipelineHost.MaxPushDescriptors => _maxPushDescriptors; // Wave64 compute runs natively when the device's subgroup is that wide; smaller devices emulate it. + bool IShaderPipelineHost.ComputeWave64SubgroupNative => + Volatile.Read(ref _nativeSubgroupSize) >= 64 && (_canRequireComputeSubgroup64 || Volatile.Read(ref _nativeSubgroupSize) == 64); + bool IShaderPipelineHost.ComputeWave64Supported => Volatile.Read(ref _nativeSubgroupSize) >= 64; - // Only a 64-invocation wave64 workgroup is translated for either host subgroup width. Every - // other compute translation maps a guest wave to 32-lane host subgroups, and a 64-lane host - // subgroup (AMD's default) left lanes 32..63 inactive: a wave64 8x8x8 group lost rows 4..7. + bool IShaderPipelineHost.ComputeRunsOn32LaneSubgroups(ComputeInputInfo input) => + RequiresComputeSubgroup32(input) || + (Volatile.Read(ref _nativeSubgroupSize) == RdnaSubgroupSize && !RequiresComputeSubgroup64(input)); + + // Wave32 compute maps a guest wave to 32-lane host subgroups, and a 64-lane host subgroup + // (AMD's default) left lanes 32..63 inactive, so those pipelines are pinned to 32. Wave64 + // compute is pinned to 64 instead when the host runs it natively. private const uint RdnaSubgroupSize = 32; private bool _canRequireComputeSubgroup32; private uint _maxComputeWorkgroupSubgroups; + private const uint Wave64SubgroupSize = 64; + private bool _canRequireComputeSubgroup64; + + // A native wave64 translation reads one guest wave as one host subgroup, so the driver must + // not pick 32-lane subgroups for it (RDNA drivers may do so for compute). + private bool RequiresComputeSubgroup64(ComputeInputInfo input) + { + return _canRequireComputeSubgroup64 && input.WaveSize == 64 && + ((IShaderPipelineHost)this).ComputeWave64SubgroupNative; + } + private bool RequiresComputeSubgroup32(ComputeInputInfo input) { var invocations = (ulong)Math.Max(input.ThreadsX, 1) * Math.Max(input.ThreadsY, 1) * Math.Max(input.ThreadsZ, 1); return _canRequireComputeSubgroup32 && - !(input.WaveSize == 64 && invocations == 64) && + !(input.WaveSize == 64 && ((IShaderPipelineHost)this).ComputeWave64SubgroupNative) && invocations <= (ulong)_maxComputeWorkgroupSubgroups * RdnaSubgroupSize; } + // The subgroup width a compute pipeline must be pinned to, or 0 to leave it to the driver. + private uint ComputeRequiredSubgroupSize(ComputeInputInfo input) => + RequiresComputeSubgroup64(input) ? Wave64SubgroupSize + : RequiresComputeSubgroup32(input) ? RdnaSubgroupSize + : 0u; + bool IShaderPipelineHost.GraphicsSubgroupOperationsEnabled => GraphicsSubgroupOperationsEnabled; bool IShaderPipelineHost.SharedInt64AtomicsEnabled => SharedInt64AtomicsEnabled; @@ -101,6 +125,11 @@ internal static unsafe partial class VulkanVideoPresenter // A range the GPU wrote is downloaded first, so the word is what the guest CPU would read. public bool TryReadGuestWord(ulong address, out uint word) { + if (_concurrentAssumptions is { } assumptions) + { + return TryReadGuestWordConcurrent(address, out word, assumptions); + } + using var profile = ResourceMaterializationProfile.Measure(ResourceMaterializationProfile.Phase.GuestRead); word = 0; var synchronized = false; @@ -170,9 +199,9 @@ internal static unsafe partial class VulkanVideoPresenter return false; } - if ((_bufferCache.MayHaveGpuDirtyPages(address, sizeof(uint)) && _bufferCache.HasGpuDirtyPages(address, sizeof(uint))) || - _bufferCache.HasGpuDirtyBytes(address, sizeof(uint)) || - _imageCache.HasGpuModifiedImageBytes(address, sizeof(uint))) + var owned = IsGpuOwnedForCleanRead(address, sizeof(uint)); + _concurrentAssumptions?.Record(ConcurrentQuery.CleanWord, address, sizeof(uint), owned); + if (owned) { return false; } @@ -190,6 +219,11 @@ internal static unsafe partial class VulkanVideoPresenter // The same ownership rules as the two word readers, checked once for a whole range. public bool TryReadResidentGuestBytes(ulong address, Span destination, bool clean) { + if (_concurrentAssumptions is { } assumptions) + { + return TryReadResidentGuestBytesConcurrent(address, destination, clean, assumptions); + } + var size = (ulong)destination.Length; if (clean || !IsCleanReadPage(address, size)) { @@ -967,15 +1001,16 @@ internal static unsafe partial class VulkanVideoPresenter Pipeline pipeline; try { + var pin = ComputeRequiredSubgroupSize(description.Input); var requiredSubgroupSize = new PipelineShaderStageRequiredSubgroupSizeCreateInfo { SType = StructureType.PipelineShaderStageRequiredSubgroupSizeCreateInfo, - RequiredSubgroupSize = RdnaSubgroupSize, + RequiredSubgroupSize = pin, }; var stageInfo = new PipelineShaderStageCreateInfo { SType = StructureType.PipelineShaderStageCreateInfo, - PNext = RequiresComputeSubgroup32(description.Input) ? &requiredSubgroupSize : null, + PNext = pin != 0 ? &requiredSubgroupSize : null, Stage = ShaderStageFlags.ComputeBit, Module = computeModule, PName = entryPoint, @@ -1099,6 +1134,7 @@ internal static unsafe partial class VulkanVideoPresenter var device = _device; var cacheSource = GetGuestPipelineCacheSource(ComputeCacheKey(description.Stage.Hash, computeModule.Handle)); var vk = _vk; + var requiredSubgroupSize = ComputeRequiredSubgroupSize(description.Input); var started = new PendingComputePipeline { SetLayout = setLayout, @@ -1122,7 +1158,7 @@ internal static unsafe partial class VulkanVideoPresenter // Importing a MoltenVK cache compiles its MSL libraries. // Keep that work inside the same bounded compiler slot. var cache = ResolveGuestPipelineCache(cacheSource); - return CompileComputePipeline(vk, device, cache, computeModule, layout); + return CompileComputePipeline(vk, device, cache, computeModule, layout, requiredSubgroupSize); } finally { @@ -1146,14 +1182,21 @@ internal static unsafe partial class VulkanVideoPresenter Device device, PipelineCache cache, ShaderModule module, - PipelineLayout layout) + PipelineLayout layout, + uint requiredSubgroupSize) { var entryPoint = (byte*)SilkMarshal.StringToPtr("main"); try { + var subgroupInfo = new PipelineShaderStageRequiredSubgroupSizeCreateInfo + { + SType = StructureType.PipelineShaderStageRequiredSubgroupSizeCreateInfo, + RequiredSubgroupSize = requiredSubgroupSize, + }; var stageInfo = new PipelineShaderStageCreateInfo { SType = StructureType.PipelineShaderStageCreateInfo, + PNext = requiredSubgroupSize != 0 ? &subgroupInfo : null, Stage = ShaderStageFlags.ComputeBit, Module = module, PName = entryPoint, diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs index 025312b..7c8d072 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs @@ -357,6 +357,8 @@ internal static unsafe partial class VulkanVideoPresenter _commandBuffer, PipelineBindPoint.Graphics, isHdr ? _hdrPqPipeline : _hdrPipeline); + SharpEmu.Libs.Gpu.Vulkan.GraphicsDynamicStateEpoch.Advance(); + SharpEmu.Libs.Gpu.Vulkan.NativeBindingEpoch.Advance(); var descriptorSet = isHdr ? _hdrPqDescriptorSets[imageIndex] : _hdrDescriptorSets[imageIndex]; diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs index 307d6f9..f6d0c6d 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs @@ -46,38 +46,50 @@ internal static unsafe partial class VulkanVideoPresenter using var profileScope = RenderPhaseProfile.Measure(RenderPhaseProfile.Phase.Idle); var gpuWorkInFlight = _pendingGuestSubmissions.Count > 0 || Array.Exists(_frameInFlight, static pending => pending); - var blockedRetryWait = BlockedRetryWaitMilliseconds(); - lock (_gate) + // The command stream thread retries its blocked heads itself and wakes this thread for its + // items; before it starts, queued submissions start it on the next tick. + var commandThread = _commandThread; + if (commandThread is not null) { - if (_closed || - Volatile.Read(ref _presenterCloseRequested) || - _relay.HasPendingCommands || - (_vulkanReady && _commandStream.HasUnblockedPending) || - blockedRetryWait == 0 || - HasReadyPresentationLocked()) - { - return; - } + commandThread.ConsumerIdle = true; + } - var waitMilliseconds = gpuWorkInFlight ? 1 : 8; - if (blockedRetryWait is { } retryWait) + try + { + lock (_gate) { - waitMilliseconds = Math.Min(waitMilliseconds, retryWait); - } + if (_closed || + Volatile.Read(ref _presenterCloseRequested) || + _relay.HasPendingCommands || + (commandThread is not null ? commandThread.HasWork : _vulkanReady && _commandStream.HasPending) || + HasReadyPresentationLocked()) + { + return; + } - var waitPhase = _commandStream.HasPending - ? RenderPhaseProfile.Phase.IdleBlockedCommands - : _pendingGuestImagePresentations.Count > 0 || _pendingVideoPresentations.Count > 0 || - (_latestPresentation is { } latest && latest.Sequence != _presentedSequence) - ? RenderPhaseProfile.Phase.IdlePendingPresentation - : RenderPhaseProfile.Phase.IdleNoQueuedWork; - using var waitProfile = RenderPhaseProfile.MeasureDetail(waitPhase); - SubmissionFlowProfile.Record(SubmissionFlowProfile.EventKind.WaitStarted, - detail: waitMilliseconds); - var signaled = System.Threading.Monitor.Wait(_gate, waitMilliseconds); - // The result describes the monitor wait, not the arrival of runnable guest work. - SubmissionFlowProfile.Record(signaled ? SubmissionFlowProfile.EventKind.WaitSignaled - : SubmissionFlowProfile.EventKind.WaitTimedOut, detail: (int)waitPhase); + var waitMilliseconds = gpuWorkInFlight ? 1 : 8; + + var waitPhase = _commandStream.HasPending + ? RenderPhaseProfile.Phase.IdleBlockedCommands + : _pendingGuestImagePresentations.Count > 0 || _pendingVideoPresentations.Count > 0 || + (_latestPresentation is { } latest && latest.Sequence != _presentedSequence) + ? RenderPhaseProfile.Phase.IdlePendingPresentation + : RenderPhaseProfile.Phase.IdleNoQueuedWork; + using var waitProfile = RenderPhaseProfile.MeasureDetail(waitPhase); + SubmissionFlowProfile.Record(SubmissionFlowProfile.EventKind.WaitStarted, + detail: waitMilliseconds); + var signaled = System.Threading.Monitor.Wait(_gate, waitMilliseconds); + // The result describes the monitor wait, not the arrival of runnable guest work. + SubmissionFlowProfile.Record(signaled ? SubmissionFlowProfile.EventKind.WaitSignaled + : SubmissionFlowProfile.EventKind.WaitTimedOut, detail: (int)waitPhase); + } + } + finally + { + if (commandThread is not null) + { + commandThread.ConsumerIdle = false; + } } } @@ -117,9 +129,16 @@ internal static unsafe partial class VulkanVideoPresenter return; } - using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay)) + if (_commandThread is not null) { - _relay.RunPendingCommands(); + RunRelayCommandsWithPrefetchHeld(); + } + else + { + using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay)) + { + _relay.RunPendingCommands(); + } } if (_deviceLost) { @@ -388,6 +407,8 @@ internal static unsafe partial class VulkanVideoPresenter _commandBuffer, PipelineBindPoint.Graphics, _barycentricPipeline); + SharpEmu.Libs.Gpu.Vulkan.GraphicsDynamicStateEpoch.Advance(); + SharpEmu.Libs.Gpu.Vulkan.NativeBindingEpoch.Advance(); _vk.CmdDraw(_commandBuffer, 3, 1, 0, 0); _vk.CmdEndRenderPass(_commandBuffer); waitStage = PipelineStageFlags.ColorAttachmentOutputBit; diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs index 799601f..e602ca2 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs @@ -12,6 +12,7 @@ using SharpEmu.Libs.Gpu.GpuCommands.Registers; using SharpEmu.Libs.Gpu.Images; using SharpEmu.Libs.Gpu.Pipelines; using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Vulkan; using SharpEmu.Libs.Gpu.Scheduling; using SharpEmu.Libs.Gpu.Vulkan; using Silk.NET.Core; @@ -280,14 +281,21 @@ internal static unsafe partial class VulkanVideoPresenter _boundGraphicsPipeline = null; } + // The acquisition below reads the attachment back into its result and keeps no reference to it, + // so one render-thread instance serves every draw. + private ColorAttachment? _colorAttachmentScratch; + ColorAttachmentAcquisition IRenderHost.AcquireColorAttachment(in ColorTargetState target) { - var attachment = new ColorAttachment - { - ImageIdentifier = target.Image, - Request = target.Resolution.Request, - Resolution = target.Resolution, - }; + var attachment = _colorAttachmentScratch ??= new ColorAttachment(); + attachment.ImageIdentifier = target.Image; + attachment.Request = target.Resolution.Request; + attachment.Resolution = target.Resolution; + attachment.Image = null!; + attachment.View = default; + attachment.Layout = default; + attachment.Clear = false; + attachment.ClearValue = default; AcquireColorAttachment(attachment); return new ColorAttachmentAcquisition( attachment.ImageIdentifier, @@ -373,6 +381,11 @@ internal static unsafe partial class VulkanVideoPresenter { if (bindings.IsEmpty) return; var command = BeginBatchedGuestCommands(); + if (VertexBuffersAlreadyBound(command, bindings)) + { + return; + } + var buffers = stackalloc VkBuffer[bindings.Length]; var offsets = stackalloc ulong[bindings.Length]; for (var index = 0; index < bindings.Length; index++) @@ -384,8 +397,14 @@ internal static unsafe partial class VulkanVideoPresenter _vk.CmdBindVertexBuffers(command, 0, (uint)bindings.Length, buffers, offsets); } - public void BindIndexBuffer(BufferBinding binding, IndexType type) => - _vk.CmdBindIndexBuffer(BeginBatchedGuestCommands(), new VkBuffer(binding.Handle), binding.Offset, type); + public void BindIndexBuffer(BufferBinding binding, IndexType type) + { + var command = BeginBatchedGuestCommands(); + if (!IndexBufferAlreadyBound(command, new VkBuffer(binding.Handle), binding.Offset, type)) + { + _vk.CmdBindIndexBuffer(command, new VkBuffer(binding.Handle), binding.Offset, type); + } + } public IResourcePreparation BeginPreparation() { @@ -399,6 +418,11 @@ internal static unsafe partial class VulkanVideoPresenter if (_batchDrawCount >= (_renderingActive ? DrawsPerBatchInRenderPass : DrawsPerBatch)) { + if (GpuSyncProfile.Enabled) + { + GpuSyncProfile.CountBatchFlush(); + } + EndRendering(); FlushBatchedGuestCommands(); } @@ -644,29 +668,80 @@ internal static unsafe partial class VulkanVideoPresenter _vk.CmdClearDepthStencilImage(command, image.Backing.Handle, ImageLayout.TransferDstOptimal, &value, 1, &vkRange); } + // The dynamic state the last guest draw set, and where it holds: the command buffer and its + // tick (a new command buffer starts without state), and no other graphics pipeline since. + private DynamicDrawState _lastDynamicState; + private CommandBuffer _lastDynamicStateBuffer; + private ulong _lastDynamicStateTick; + private long _lastDynamicStateEpoch = -1; + public void SetDynamicState(in DynamicDrawState state) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.DrawDynamicStateRecording); var command = BeginBatchedGuestCommands(); - var viewport = new Viewport(state.ViewportX, state.ViewportY, state.ViewportWidth, state.ViewportHeight, state.ViewportMinDepth, state.ViewportMaxDepth); - _vk.CmdSetViewport(command, 0, 1, &viewport); - var scissor = new Rect2D( - new Offset2D(state.Scissor.Left, state.Scissor.Top), - new Extent2D((uint)(state.Scissor.Right - state.Scissor.Left), (uint)(state.Scissor.Bottom - state.Scissor.Top))); - _vk.CmdSetScissor(command, 0, 1, &scissor); - _vk.CmdSetLineWidth(command, state.LineWidth); - var blendConstants = stackalloc float[4] { state.BlendRed, state.BlendGreen, state.BlendBlue, state.BlendAlpha }; - _vk.CmdSetBlendConstants(command, blendConstants); - _vk.CmdSetDepthTestEnable(command, state.DepthTestEnabled); - _vk.CmdSetDepthWriteEnable(command, state.DepthWriteEnabled); - _vk.CmdSetDepthCompareOp(command, state.DepthCompare); - _vk.CmdSetDepthBiasEnable(command, state.DepthBiasEnabled); - if (state.DepthBiasEnabled) + // Every guest pipeline keeps these states dynamic, so they persist across its binds; + // only what changed since the last draw in this command buffer is set again. + var known = command.Handle == _lastDynamicStateBuffer.Handle && _scheduler.CurrentTick == _lastDynamicStateTick && + GraphicsDynamicStateEpoch.Value == _lastDynamicStateEpoch; + ref readonly var last = ref _lastDynamicState; + if (!known || state.ViewportX != last.ViewportX || state.ViewportY != last.ViewportY || + state.ViewportWidth != last.ViewportWidth || state.ViewportHeight != last.ViewportHeight || + state.ViewportMinDepth != last.ViewportMinDepth || state.ViewportMaxDepth != last.ViewportMaxDepth) + { + var viewport = new Viewport(state.ViewportX, state.ViewportY, state.ViewportWidth, state.ViewportHeight, state.ViewportMinDepth, state.ViewportMaxDepth); + _vk.CmdSetViewport(command, 0, 1, &viewport); + } + + if (!known || state.Scissor != last.Scissor) + { + var scissor = new Rect2D( + new Offset2D(state.Scissor.Left, state.Scissor.Top), + new Extent2D((uint)(state.Scissor.Right - state.Scissor.Left), (uint)(state.Scissor.Bottom - state.Scissor.Top))); + _vk.CmdSetScissor(command, 0, 1, &scissor); + } + + if (!known || state.LineWidth != last.LineWidth) + { + _vk.CmdSetLineWidth(command, state.LineWidth); + } + + if (!known || state.BlendRed != last.BlendRed || state.BlendGreen != last.BlendGreen || + state.BlendBlue != last.BlendBlue || state.BlendAlpha != last.BlendAlpha) + { + var blendConstants = stackalloc float[4] { state.BlendRed, state.BlendGreen, state.BlendBlue, state.BlendAlpha }; + _vk.CmdSetBlendConstants(command, blendConstants); + } + + if (!known || state.DepthTestEnabled != last.DepthTestEnabled) + { + _vk.CmdSetDepthTestEnable(command, state.DepthTestEnabled); + } + + if (!known || state.DepthWriteEnabled != last.DepthWriteEnabled) + { + _vk.CmdSetDepthWriteEnable(command, state.DepthWriteEnabled); + } + + if (!known || state.DepthCompare != last.DepthCompare) + { + _vk.CmdSetDepthCompareOp(command, state.DepthCompare); + } + + if (!known || state.DepthBiasEnabled != last.DepthBiasEnabled) + { + _vk.CmdSetDepthBiasEnable(command, state.DepthBiasEnabled); + } + + // The factors only matter while the bias is on; they are set when it turns on or changes. + if (state.DepthBiasEnabled && (!known || !last.DepthBiasEnabled || + state.DepthBiasConstantFactor != last.DepthBiasConstantFactor || state.DepthBiasClamp != last.DepthBiasClamp || + state.DepthBiasSlopeFactor != last.DepthBiasSlopeFactor)) { _vk.CmdSetDepthBias(command, state.DepthBiasConstantFactor, _supportsDepthBiasClamp ? state.DepthBiasClamp : 0f, state.DepthBiasSlopeFactor); } - if (state.StencilTestEnabled) + if (state.StencilTestEnabled && (!known || !last.StencilTestEnabled || + state.FrontStencil != last.FrontStencil || state.BackStencil != last.BackStencil)) { _vk.CmdSetStencilCompareMask(command, StencilFaceFlags.FaceFrontBit, state.FrontStencil.CompareMask); _vk.CmdSetStencilCompareMask(command, StencilFaceFlags.FaceBackBit, state.BackStencil.CompareMask); @@ -686,6 +761,11 @@ internal static unsafe partial class VulkanVideoPresenter colorWriteEnable.CmdSetColorWriteEnable(command, state.ColorWriteCount, enables); } + + _lastDynamicState = state; + _lastDynamicStateBuffer = command; + _lastDynamicStateTick = _scheduler.CurrentTick; + _lastDynamicStateEpoch = GraphicsDynamicStateEpoch.Value; } @@ -787,12 +867,17 @@ internal static unsafe partial class VulkanVideoPresenter } EndRendering(); - var command = BeginBatchedGuestCommands(); - _commandBuffer = command; - if (DeferGlobalBarriers) + if (DeferGlobalBarriers && BatchBarriers) { // A guest cache flush inside this scope may be deferred to its end; this keeps // everything recorded before the scope ordered before its draws either way. + RequestGuestMemoryBarrier(GlobalMemoryBarrier); + } + + var command = BeginBatchedGuestCommands(); + _commandBuffer = command; + if (DeferGlobalBarriers && !BatchBarriers) + { RecordGlobalBarrier(command); } @@ -863,6 +948,11 @@ internal static unsafe partial class VulkanVideoPresenter }; _vk.CmdBeginRendering(command, &rendering); _renderingScopesBegun++; + if (GpuSyncProfile.Enabled) + { + GpuSyncProfile.CountRenderingScope(); + } + _renderingActive = true; _renderingState = state; } @@ -912,19 +1002,19 @@ internal static unsafe partial class VulkanVideoPresenter public void BindPipeline(PipelineBindPoint bindPoint, in PipelineHandle pipeline) { var entry = RequirePipelineEntry(in pipeline); - var command = BeginBatchedGuestCommands(); + var command = BeginBatchedGuestStateCommands(); if (bindPoint == PipelineBindPoint.Graphics) { _boundGraphicsPipeline = entry; if (!entry.RectangleList) { - _vk.CmdBindPipeline(command, bindPoint, entry.Pipeline); + BindPipelineShadowed(command, bindPoint, entry.Pipeline); } return; } - _vk.CmdBindPipeline(command, bindPoint, entry.Pipeline); + BindPipelineShadowed(command, bindPoint, entry.Pipeline); _profileComputePipeline = entry.Id; } @@ -948,7 +1038,7 @@ internal static unsafe partial class VulkanVideoPresenter variant = CreateRenderPipeline(entry.Description!, strip ? PrimitiveTopology.TriangleStrip : PrimitiveTopology.TriangleList, entry.Layout); } - _vk.CmdBindPipeline(command, PipelineBindPoint.Graphics, variant); + BindPipelineShadowed(command, PipelineBindPoint.Graphics, variant); } private void BindNativeRectangleList(RenderPipelineEntry entry, CommandBuffer command) @@ -959,7 +1049,7 @@ internal static unsafe partial class VulkanVideoPresenter entry.Layout, PolygonMode.FillRectangleNV); } - _vk.CmdBindPipeline(command, PipelineBindPoint.Graphics, entry.RectangleVariant); + BindPipelineShadowed(command, PipelineBindPoint.Graphics, entry.RectangleVariant); } // Rectangle2D consumes three vertices and fills their projected bounding box. @@ -1075,9 +1165,24 @@ internal static unsafe partial class VulkanVideoPresenter return true; } + private static readonly bool BatchBarriers = + Environment.GetEnvironmentVariable("SHARPEMU_BATCH_BARRIERS") != "0"; + + private static readonly MemoryBarrierRequest GlobalMemoryBarrier = new( + PipelineStageFlags.AllCommandsBit, + PipelineStageFlags.AllCommandsBit, + AccessFlags.MemoryWriteBit, + AccessFlags.MemoryReadBit | AccessFlags.MemoryWriteBit); + private void RecordMemoryBarrier(PipelineStageFlags sourceStages, PipelineStageFlags destinationStages, AccessFlags sourceAccess, AccessFlags destinationAccess) { EndRendering(); + if (BatchBarriers) + { + RequestGuestMemoryBarrier(new MemoryBarrierRequest(sourceStages, destinationStages, sourceAccess, destinationAccess)); + return; + } + var command = BeginBatchedGuestCommands(); var barrier = new MemoryBarrier2 { @@ -1194,7 +1299,7 @@ internal static unsafe partial class VulkanVideoPresenter public bool TryFillDccMetadata(ulong address, ulong size, uint fillValue) { - if (!_imageCache.OverlapsDccMetadata(address, size)) + if (!_imageCache.OverlapsDccMetadata(address, size) && !IsMetadataShapedFill(address, size, fillValue)) { return false; } @@ -1203,6 +1308,16 @@ internal static unsafe partial class VulkanVideoPresenter return true; } + private bool IsMetadataShapedFill(ulong address, ulong size, uint fillValue) + { + const ulong metadataBlockSize = 0x1000; + const ulong maxMetadataFill = 0x100000; + return address != 0 && size != 0 && size <= maxMetadataFill && + address % metadataBlockSize == 0 && size % metadataBlockSize == 0 && + fillValue == (fillValue & 0xFF) * 0x01010101u && + !_imageCache.QueryRegion(address, size).ImageBytes; + } + public bool TryCopyWordsOnHost(ulong destination, ulong source, ulong sourceWords, ulong words) => _bufferCache.TryCopyWordsOnHost(destination, source, sourceWords, words); } diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs index dbdff71..a974b1f 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs @@ -74,6 +74,18 @@ internal static unsafe partial class VulkanVideoPresenter _ => WakeRenderThread()); } + private void RecordRequestedMemoryBarrier(nint handle, MemoryBarrierRequest request) + { + var barrier = new MemoryBarrier2 + { + SType = StructureType.MemoryBarrier2, + SrcAccessMask = VulkanSynchronization.Access(request.SourceAccess), + DstAccessMask = VulkanSynchronization.Access(request.DestinationAccess), + }; + VulkanSynchronization.PipelineBarrier(_vk, new CommandBuffer(handle), request.SourceStages, request.DestinationStages, + 0, 1, &barrier, 0, null, 0, null); + } + // Both stores share the manager's page guard and read guest memory through its address space. private static (GuestGpuMemory Memory, ICpuMemory Guest, IGuestBackedSpace Backing) RequireGuestMemory(string store) { @@ -120,6 +132,8 @@ internal static unsafe partial class VulkanVideoPresenter // Drain accepted submissions before closing the relay. // Cancel blocked submissions if a full retry cycle makes no progress. _commandStream.StopAccepting(); + // The command stream thread ends after its slice; the rest drains here. + _commandThread?.Stop(() => RunCommandStreamSlices(long.MaxValue)); var outcome = _commandStream.DrainForShutdown(cancelBlockedOnNoProgress: true); FlushBatchedGuestCommands(); Console.Error.WriteLine( @@ -165,6 +179,16 @@ internal static unsafe partial class VulkanVideoPresenter return new CommandBuffer(_scheduler.Current.Handle); } + private RecordingBuffer CurrentRecording() + { + if (!_scheduler.Active) + { + _scheduler.Begin(_submissionContext); + } + + return _scheduler.Current; + } + private void BindSubmissionContext(VulkanGuestQueueIdentity queue) { _submissionContext.QueueName = queue.Name; diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs index d063eae..06665c1 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs @@ -163,7 +163,9 @@ internal static unsafe partial class VulkanVideoPresenter { if (!ShaderProgramCache.TryCompilePrewarm( record, code, compiler, sharedInt64Atomics, execGuardElision, - _supportsShaderSignedZeroInfNanPreserveFloat32, out var compiled, out var layout, out var error)) + _supportsShaderSignedZeroInfNanPreserveFloat32, out var compiled, out var layout, out var error, + ((IShaderPipelineHost)this).ComputeWave64SubgroupNative, + ((IShaderPipelineHost)this).ComputeRunsOn32LaneSubgroups(record.Info))) { NoteShaderPrewarmFailure(record, error); return; @@ -181,7 +183,7 @@ internal static unsafe partial class VulkanVideoPresenter setLayout = CreateDescriptorSetLayout(bindings, out _, out _); pipelineLayout = CreatePipelineLayout(setLayout, ShaderStageFlags.ComputeBit); module = CreateShaderModule(payload); - var pipeline = CompileComputePipeline(_vk, _device, _pipelineCache, module, pipelineLayout); + var pipeline = CompileComputePipeline(_vk, _device, _pipelineCache, module, pipelineLayout, ComputeRequiredSubgroupSize(record.Info)); _vk.DestroyPipeline(_device, pipeline, null); Interlocked.Increment(ref _shaderPrewarmCompiled); } diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs index bce01f0..cda8ecc 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs @@ -51,40 +51,9 @@ public static partial class Gen5SpirvTranslator return false; } - var value = GetRawSource(instruction, 0); - if (_subgroupInvocationIdInput != 0) - { - if (_emulateWave64) - { - value = BroadcastFirstWave64Active(value); - } - else - { - // SPIR-V's BroadcastFirst uses the first host-active - // invocation. Guest EXEC is modeled as data, so obtain the - // guest-active mask explicitly and broadcast from its first - // set lane instead. This also updates the private SGPR copy - // for lanes that are currently disabled and may be restored - // by a later saveexec sequence. - var activeLanes = _module.AddInstruction( - SpirvOp.GroupNonUniformBallot, - _uvec4Type, - UInt(3), - Load(_boolType, _exec)); - var activeLow = _module.AddInstruction( - SpirvOp.CompositeExtract, - _uintType, - activeLanes, - 0); - var firstActiveLane = Ext(73, _uintType, activeLow); - value = _module.AddInstruction( - SpirvOp.GroupNonUniformBroadcast, - _uintType, - UInt(3), - value, - firstActiveLane); - } - } + var value = _pairedLanes + ? PairedReadFirstLane(instruction) + : BroadcastFirstWaveActive(GetRawSource(instruction, 0)); StoreS(instruction.Destinations[0].Value, value); return true; @@ -2773,7 +2742,7 @@ public static partial class Gen5SpirvTranslator if (instruction.Opcode.EndsWith("SaveexecB32", StringComparison.Ordinal)) { - var oldExec64 = BooleanToWaveMask(Load(_boolType, _exec)); + var oldExec64 = CurrentExecMask(); var oldExec = _module.AddInstruction( SpirvOp.UConvert, _uintType, @@ -3422,9 +3391,7 @@ public static partial class Gen5SpirvTranslator var left = GetRawSource64(instruction, 0); if (instruction.Opcode.EndsWith("SaveexecB64", StringComparison.Ordinal)) { - var oldExec = _emulateWave64 && _subgroupInvocationIdInput != 0 - ? LoadS64(126) - : BooleanToWaveMask(Load(_boolType, _exec)); + var oldExec = CurrentExecMask(); var notLeft = _module.AddInstruction(SpirvOp.Not, _ulongType, left); var newExec = instruction.Opcode switch { @@ -3493,12 +3460,7 @@ public static partial class Gen5SpirvTranslator return false; } - if (!_emulateWave64 && _waveLaneCount != 64) - { - newExec = BitwiseAnd64( - newExec, - _module.Constant64(_ulongType, 0xFFFF_FFFFUL)); - } + newExec = NormalizeExecMask(newExec); StoreS64(destination, oldExec); StoreS64(126, newExec); @@ -4387,6 +4349,12 @@ public static partial class Gen5SpirvTranslator private void StoreS64(uint register, uint value) { + if (_pairedLanes && _inLanePass) + { + StorePairedLaneMaskWord(register, value); + return; + } + StoreS( register, _module.AddInstruction(SpirvOp.UConvert, _uintType, value)); @@ -4958,52 +4926,6 @@ public static partial class Gen5SpirvTranslator private uint GuestLaneSelect(uint selector) => BitwiseAnd(selector, UInt(_waveLaneCount == 64 ? 63u : RdnaWaveLaneCount - 1)); - // An emulated wave64 spans two host subgroups, so a subgroup broadcast cannot reach - // the other half. The selected guest lane publishes through workgroup scratch, like - // BroadcastFirstWave64Active. - private uint BroadcastWave64Lane(uint value, uint lane) => - ExchangeWave64Value(_module.AddInstruction(SpirvOp.IEqual, _boolType, GuestWaveLane(), lane), value); - - private uint BroadcastFirstWave64Active(uint value) - { - var lane = GuestWaveLane(); - var upperHalf = ShiftRightLogical(lane, UInt(5)); - var activeInHalf = OwnHalfBallot(Load(_boolType, _exec)); - var halfHasActive = IsNotZero(activeInHalf); - var firstInHalf = _module.AddInstruction( - SpirvOp.Select, - _uintType, - halfHasActive, - Ext(73, _uintType, activeInHalf), - UInt(0)); - var halfBase = BitwiseAnd(Load(_uintType, _subgroupInvocationIdInput), UInt(~31u)); - var halfValue = _module.AddInstruction( - SpirvOp.GroupNonUniformBroadcast, - _uintType, - UInt(3), - value, - IAdd(halfBase, firstInHalf)); - var exchange = BeginWave64Exchange(); - EmitConditional(IsHalfWaveLeader(lane), () => - { - Store(Wave64ExchangePointer(exchange, upperHalf), halfValue); - Store( - Wave64ExchangePointer(exchange, IAdd(UInt(2), upperHalf)), - _module.AddInstruction(SpirvOp.Select, _uintType, halfHasActive, UInt(1), UInt(0))); - }); - EmitWave64Barrier(); - var lowerValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))); - var upperValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(1))); - var lowerActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(2)))); - var upperActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(3)))); - return _module.AddInstruction( - SpirvOp.Select, - _uintType, - _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, LogicalNot(lowerActive), upperActive), - upperValue, - lowerValue); - } - private void StoreCarryOut( Gen5ShaderInstruction instruction, uint carry) @@ -5035,23 +4957,24 @@ public static partial class Gen5SpirvTranslator } var destination = instruction.Destinations[0].Value; - var sourceValue = GetRawSource(instruction, 0); var selectedLane = BitwiseAnd(GetRawSource(instruction, 1), UInt(LaneSelectMask)); + if (_pairedLanes) + { + StoreS(destination, PairedReadLane(instruction, selectedLane)); + return true; + } - if (_emulateWave64) + var sourceValue = GetRawSource(instruction, 0); + + if (_stage == Gen5SpirvStage.Compute && _waveLowering is EmulatedWave64Lowering) { // The selected guest lane can belong to another host subgroup. // Read it even when the guest execution mask disables that lane. - StoreS(destination, BroadcastWave64Lane(sourceValue, selectedLane)); + StoreS(destination, BroadcastWaveLane(sourceValue, selectedLane)); } else if (_subgroupInvocationIdInput != 0 && _stage == Gen5SpirvStage.Compute) { - var broadcast = _module.AddInstruction( - SpirvOp.GroupNonUniformBroadcast, - _uintType, - UInt(3), - sourceValue, - selectedLane); + var broadcast = BroadcastWaveLane(sourceValue, selectedLane); StoreS(destination, ReadLaneSpillSlot(instruction, selectedLane, broadcast)); } else diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.CommonWave.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.CommonWave.cs new file mode 100644 index 0000000..b843e5c --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.CommonWave.cs @@ -0,0 +1,27 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private uint BroadcastFirstWaveActive(uint value) => + _waveLowering.BroadcastFirstActive(value); + + private uint BroadcastWaveLane(uint value, uint lane) => + _waveLowering.BroadcastLane(value, lane); + + private uint BroadcastWaveValue(uint writer, uint value, uint lane) => + _waveLowering.BroadcastValue(writer, value, lane); + + private uint CurrentExecMask() => + _waveLowering.CurrentExecMask(); + + private uint NormalizeExecMask(uint mask) => + _waveLowering.NormalizeExecMask(mask); + } +} + + diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.EmulatedWave64Lowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.EmulatedWave64Lowering.cs new file mode 100644 index 0000000..7361761 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.EmulatedWave64Lowering.cs @@ -0,0 +1,578 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private sealed class EmulatedWave64Lowering : IWaveLowering + { + private const uint ExchangeSlotCount = 4; + private const uint ExchangeDwordCount = ExchangeSlotCount * 2; + private const uint GuestWaveLaneCount = 64; + private static readonly bool HalfWaveMasks = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_HALF_MASKS"), + "0", + StringComparison.Ordinal); + + private readonly CompilationContext _context; + private uint _exchange; + private uint _exchangeElementPointer; + private uint _exchangeOffset; + private uint _exchangeParity; + // Several guest waves share one workgroup. Each gets its own exchange area plus an + // arrival counter, and its two halves meet at that counter instead of at a workgroup + // barrier, which other waves (possibly in other blocks) would never reach. + private readonly uint _waveCount; + private readonly bool _multiWave; + private readonly uint _waveStride; + private uint _rendezvousEpoch; + private Ir.Gen5Wave64HalfMaskPlan? _halfMaskPlan; + private bool _halfMaskPlanBuilt; + private bool _synchronizeSharedMemory; + private SharedMemoryPhase _sharedMemoryPhase; + private uint? _emittingPc; + + private enum SharedMemoryPhase { None, Read, Write } + + public EmulatedWave64Lowering(CompilationContext context) + { + _context = context; + var threads = (ulong)context._localSizeX * context._localSizeY * context._localSizeZ; + _waveCount = (uint)Math.Max((threads + GuestWaveLaneCount - 1) / GuestWaveLaneCount, 1); + _multiWave = _waveCount > 1; + _waveStride = _multiWave ? ExchangeDwordCount + 1 : ExchangeDwordCount; + } + + private uint ExchangeTotalDwords => _waveCount * _waveStride; + + public bool RequiresCrossSubgroupSynchronization => true; + + private bool UsesExchange => _context.UsesSubgroupOperations(); + + public uint ReserveLdsDwords(uint guestDwordCount) => + UsesExchange && guestDwordCount < LdsDwordCount + ? guestDwordCount + ExchangeTotalDwords + : guestDwordCount; + + public void DeclareResources() + { + var context = _context; + if (!UsesExchange) + { + return; + } + + if (context._lds != 0) + { + // Metal exposes 32 KiB of threadgroup memory on the Apple + // GPUs we target. Reuse the tail of guest LDS so the + // cross-subgroup bridge stays within that limit. + _exchange = context._lds; + _exchangeElementPointer = context._ldsElementPointer; + _exchangeOffset = context._ldsDwordMask + 1 < LdsDwordCount + ? context._ldsDwordMask + 1 + : LdsDwordCount - ExchangeTotalDwords; + return; + } + + var exchangeArrayType = context._module.TypeArray(context._uintType, ExchangeTotalDwords); + _exchangeElementPointer = context._module.TypePointer(SpirvStorageClass.Workgroup, context._uintType); + _exchange = context._module.AddGlobalVariable( + context._module.TypePointer(SpirvStorageClass.Workgroup, exchangeArrayType), + SpirvStorageClass.Workgroup); + _exchangeOffset = 0; + context._module.AddName(_exchange, "wave64Exchange"); + context._interfaces.Add(_exchange); + } + + public void DeclareFunctionResources() + { + if (!UsesExchange) + { + return; + } + + _exchangeParity = _context.DeclareFunctionUint( + _context._module.Constant(_context._uintType, 0)); + _context._module.AddName(_exchangeParity, "wave64ExchangeParity"); + if (_multiWave) + { + _rendezvousEpoch = _context.DeclareFunctionUint( + _context._module.Constant(_context._uintType, 0)); + _context._module.AddName(_rendezvousEpoch, "wave64RendezvousEpoch"); + } + } + + public void InitializeMasks() + { + var context = _context; + if (context._subgroupInvocationIdInput != 0) + { + context.StoreS64(106, BothHalvesOfOwnBallot(context._module.ConstantBool(false))); + context.StoreS64(126, BothHalvesOfOwnBallot(context._module.ConstantBool(true))); + InitializeRendezvous(); + return; + } + + context.StoreS64(106, context._module.Constant64(context._ulongType, 0)); + context.StoreS64(126, context._module.Constant64(context._ulongType, 1)); + } + + public void BeginBlock() + { + _synchronizeSharedMemory = HalfMaskPlan() is null; + _sharedMemoryPhase = SharedMemoryPhase.None; + } + + public void BeforeInstruction(Gen5ShaderInstruction instruction) + { + var context = _context; + _emittingPc = instruction.Pc; + if (HalfMaskPlan() is { } plan) + { + if (plan.ExactPairsBefore.TryGetValue(instruction.Pc, out var exactPairs)) + { + EmitExactWaveMasks(exactPairs); + } + + if (plan.SharedMemoryBarriersBefore.Contains(instruction.Pc)) + { + Barrier(); + } + } + + if (!_synchronizeSharedMemory) + { + return; + } + + var nextPhase = instruction.Control is Gen5DataShareControl { Gds: false } + ? instruction.Opcode.StartsWith("DsRead", StringComparison.Ordinal) ? SharedMemoryPhase.Read + : instruction.Opcode.StartsWith("DsWrite", StringComparison.Ordinal) ? SharedMemoryPhase.Write + : SharedMemoryPhase.None + : SharedMemoryPhase.None; + if (instruction.Opcode == "SBarrier") + { + _sharedMemoryPhase = SharedMemoryPhase.None; + } + if (nextPhase != SharedMemoryPhase.None) + { + if (_sharedMemoryPhase != SharedMemoryPhase.None && _sharedMemoryPhase != nextPhase) + { + Barrier(); + } + _sharedMemoryPhase = nextPhase; + } + } + + public void EndBlock() + { + if (_synchronizeSharedMemory && _sharedMemoryPhase != SharedMemoryPhase.None) + { + Barrier(); + } + } + + public uint SubgroupAny(uint condition) => + _context._subgroupInvocationIdInput == 0 + ? condition + : _context.IsNotZero64(BooleanToWaveMask(condition)); + + public uint CurrentLaneBit() + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.Constant64(context._ulongType, 1); + } + + return context.ShiftLeftLogical64( + context._module.Constant64(context._ulongType, 1), + context._module.AddInstruction( + SpirvOp.UConvert, + context._ulongType, + context.GuestWaveLane())); + } + + public uint BooleanToWaveMask(uint condition) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context.BooleanToLaneMask(condition); + } + + var lane = context.GuestWaveLane(); + var owned = OwnHalfBallot(condition); + var exchange = BeginExchange(); + context.EmitConditional(IsHalfWaveLeader(lane), () => + context.Store(ExchangePointer(exchange, context.ShiftRightLogical(lane, context.UInt(5))), owned)); + Barrier(); + return context.Pair64( + context.Load(context._uintType, ExchangePointer(exchange, context.UInt(0))), + context.Load(context._uintType, ExchangePointer(exchange, context.UInt(1)))); + } + + public uint BroadcastFirstActive(uint value) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return value; + } + + var lane = context.GuestWaveLane(); + var upperHalf = context.ShiftRightLogical(lane, context.UInt(5)); + var activeInHalf = OwnHalfBallot(context.Load(context._boolType, context._exec)); + var halfHasActive = context.IsNotZero(activeInHalf); + var firstInHalf = context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + halfHasActive, + context.Ext(73, context._uintType, activeInHalf), + context.UInt(0)); + var halfBase = context.BitwiseAnd( + context.Load(context._uintType, context._subgroupInvocationIdInput), + context.UInt(~31u)); + var halfValue = context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + context._uintType, + context.UInt(3), + value, + context.IAdd(halfBase, firstInHalf)); + var exchange = BeginExchange(); + context.EmitConditional(IsHalfWaveLeader(lane), () => + { + context.Store(ExchangePointer(exchange, upperHalf), halfValue); + context.Store( + ExchangePointer(exchange, context.IAdd(context.UInt(2), upperHalf)), + context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + halfHasActive, + context.UInt(1), + context.UInt(0))); + }); + Barrier(); + var lowerValue = context.Load(context._uintType, ExchangePointer(exchange, context.UInt(0))); + var upperValue = context.Load(context._uintType, ExchangePointer(exchange, context.UInt(1))); + var lowerActive = context.IsNotZero(context.Load(context._uintType, ExchangePointer(exchange, context.UInt(2)))); + var upperActive = context.IsNotZero(context.Load(context._uintType, ExchangePointer(exchange, context.UInt(3)))); + return context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + context._module.AddInstruction( + SpirvOp.LogicalAnd, + context._boolType, + context.LogicalNot(lowerActive), + upperActive), + upperValue, + lowerValue); + } + + public uint BroadcastLane(uint value, uint lane) => + ExchangeValue( + _context._module.AddInstruction( + SpirvOp.IEqual, + _context._boolType, + _context.GuestWaveLane(), + lane), + value); + + public uint BroadcastValue(uint writer, uint value, uint lane) => ExchangeValue(writer, value); + + public uint WaveMaskAny(uint register, uint laneActive) => + _context._subgroupInvocationIdInput != 0 + ? _context.IsNotZero64(_context.LoadS64(register)) + : SubgroupAny(_context.Load(_context._boolType, laneActive)); + + public uint CurrentExecMask() => + _context._subgroupInvocationIdInput != 0 + ? _context.LoadS64(126) + : BooleanToWaveMask(_context.Load(_context._boolType, _context._exec)); + + public uint NormalizeExecMask(uint mask) => mask; + + public void StoreWaveMask(uint register, uint condition) + { + var context = _context; + if (context._waveLaneCount == 32) + { + context.StoreS(register, context.Narrow(BooleanToWaveMask(condition))); + return; + } + + if (context._subgroupInvocationIdInput != 0 && + _emittingPc is { } pc && + HalfMaskPlan() is { } plan && + plan.HalfMaskWrites.TryGetValue(pc, out var halfRegister) && + halfRegister == register) + { + context.StoreS64(register, BooleanToHalfWaveMask(condition)); + return; + } + + context.StoreS64(register, BooleanToWaveMask(condition)); + } + + private uint OwnHalfBallot(uint condition) + { + var context = _context; + var ballot = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + condition); + return context._module.AddInstruction( + SpirvOp.VectorExtractDynamic, + context._uintType, + ballot, + context.ShiftRightLogical( + context.Load(context._uintType, context._subgroupInvocationIdInput), + context.UInt(5))); + } + + private uint BooleanToHalfWaveMask(uint condition) + { + var context = _context; + var owned = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, OwnHalfBallot(condition)); + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + context._module.AddInstruction( + SpirvOp.UGreaterThanEqual, + context._boolType, + context.GuestWaveLane(), + context.UInt(32)), + context.ShiftLeftLogical64(owned, context._module.Constant64(context._ulongType, 32)), + owned); + } + + private uint BothHalvesOfOwnBallot(uint condition) + { + var context = _context; + var owned = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, OwnHalfBallot(condition)); + return context.BitwiseOr64( + owned, + context.ShiftLeftLogical64(owned, context._module.Constant64(context._ulongType, 32))); + } + + private uint IsHalfWaveLeader(uint lane) => + _context._module.AddInstruction( + SpirvOp.IEqual, + _context._boolType, + _context.BitwiseAnd(lane, _context.UInt(31)), + _context.UInt(0)); + + private uint BeginExchange() + { + var context = _context; + var parity = context.Load(context._uintType, _exchangeParity); + context.Store(_exchangeParity, context.BitwiseXor(parity, context.UInt(1))); + return context.IAdd( + WaveExchangeBase(), + context._module.AddInstruction( + SpirvOp.IMul, + context._uintType, + parity, + context.UInt(ExchangeSlotCount))); + } + + private uint WaveExchangeBase() + { + var context = _context; + if (!_multiWave) + { + return context.UInt(_exchangeOffset); + } + + var wave = context.ShiftRightLogical( + context.Load(context._uintType, context._localInvocationIndexInput), + context.UInt(6)); + return context.IAdd( + context.UInt(_exchangeOffset), + context._module.AddInstruction(SpirvOp.IMul, context._uintType, wave, context.UInt(_waveStride))); + } + + // Zeroes each wave's arrival counter. Runs once at entry, where the whole workgroup is + // converged, so the workgroup barrier is safe here. + private void InitializeRendezvous() + { + if (!_multiWave || !UsesExchange) + { + return; + } + + var context = _context; + context.EmitConditional( + context._module.AddInstruction( + SpirvOp.IEqual, + context._boolType, + context.GuestWaveLane(), + context.UInt(0)), + () => context.Store( + ExchangePointer(WaveExchangeBase(), context.UInt(ExchangeDwordCount)), + context.UInt(0))); + WorkgroupBarrier(); + } + + // Both halves of the wave arrive, then wait until the counter shows two arrivals per + // rendezvous so far. The halves of one wave are co-resident in the workgroup, so the + // wait always ends. + private void WaveRendezvous() + { + var context = _context; + var counter = ExchangePointer(WaveExchangeBase(), context.UInt(ExchangeDwordCount)); + context.EmitConditional( + IsHalfWaveLeader(context.GuestWaveLane()), + () => context._module.AddInstruction( + SpirvOp.AtomicIAdd, + context._uintType, + counter, + context.UInt(2), + context.UInt(0x108), + context.UInt(1))); + var epoch = context.IAdd(context.Load(context._uintType, _rendezvousEpoch), context.UInt(1)); + context.Store(_rendezvousEpoch, epoch); + var target = context._module.AddInstruction(SpirvOp.IMul, context._uintType, epoch, context.UInt(2)); + + var header = context._module.AllocateId(); + var continueLabel = context._module.AllocateId(); + var mergeLabel = context._module.AllocateId(); + context._module.AddStatement(SpirvOp.Branch, header); + context._module.AddLabel(header); + var arrived = context._module.AddInstruction( + SpirvOp.AtomicLoad, + context._uintType, + counter, + context.UInt(2), + context.UInt(0x102)); + var done = context._module.AddInstruction( + SpirvOp.UGreaterThanEqual, + context._boolType, + arrived, + target); + context._module.AddStatement(SpirvOp.LoopMerge, mergeLabel, continueLabel, 0); + context._module.AddStatement(SpirvOp.BranchConditional, done, mergeLabel, continueLabel); + context._module.AddLabel(continueLabel); + context._module.AddStatement(SpirvOp.Branch, header); + context._module.AddLabel(mergeLabel); + } + + private uint ExchangePointer(uint exchange, uint slot) => + _context._module.AddInstruction( + SpirvOp.AccessChain, + _exchangeElementPointer, + _exchange, + _context.IAdd(exchange, slot)); + + private uint ExchangeValue(uint writer, uint value) + { + var context = _context; + var exchange = BeginExchange(); + context.EmitConditional(writer, () => context.Store(ExchangePointer(exchange, context.UInt(0)), value)); + Barrier(); + return context.Load(context._uintType, ExchangePointer(exchange, context.UInt(0))); + } + + private void EmitExactWaveMasks(IReadOnlyList pairs) + { + var context = _context; + var lane = context.GuestWaveLane(); + var upperHalf = context.ShiftRightLogical(lane, context.UInt(5)); + var lowerHalf = context._module.AddInstruction(SpirvOp.IEqual, context._boolType, upperHalf, context.UInt(0)); + const int pairsPerExchange = (int)ExchangeSlotCount / 2; + for (var start = 0; start < pairs.Count; start += pairsPerExchange) + { + var count = Math.Min(pairsPerExchange, pairs.Count - start); + var owned = new uint[count]; + for (var index = 0; index < count; index++) + { + var pair = pairs[start + index]; + owned[index] = context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + lowerHalf, + context.LoadS(pair), + context.LoadS(pair + 1)); + } + + var exchange = BeginExchange(); + context.EmitConditional(IsHalfWaveLeader(lane), () => + { + for (var index = 0; index < count; index++) + { + context.Store( + ExchangePointer(exchange, context.IAdd(context.UInt((uint)index * 2), upperHalf)), + owned[index]); + } + }); + Barrier(); + for (var index = 0; index < count; index++) + { + var pair = pairs[start + index]; + context.StoreS(pair, context.Load(context._uintType, ExchangePointer(exchange, context.UInt((uint)index * 2)))); + context.StoreS(pair + 1, context.Load(context._uintType, ExchangePointer(exchange, context.UInt((uint)index * 2 + 1)))); + } + } + } + + private Ir.Gen5Wave64HalfMaskPlan? HalfMaskPlan() + { + if (_halfMaskPlanBuilt) + { + return _halfMaskPlan; + } + + _halfMaskPlanBuilt = true; + var context = _context; + if (HalfWaveMasks && UsesExchange && context._subgroupInvocationIdInput != 0) + { + var sharedFlatPcs = context._request.Memory.Entries + .Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) + .Select(static memory => memory.Pc) + .ToHashSet(); + _halfMaskPlan = Ir.Gen5Wave64HalfMaskAnalysis.Analyze(context._request.Program, sharedFlatPcs); + } + + return _halfMaskPlan; + } + + private void Barrier() + { + if (_multiWave) + { + // The rendezvous needs the exchange area, declared only with subgroup operations. + if (UsesExchange) + { + WaveRendezvous(); + } + + return; + } + + WorkgroupBarrier(); + } + + private void WorkgroupBarrier() + { + var workgroup = _context.UInt(2); + _context._module.AddStatement( + SpirvOp.ControlBarrier, + workgroup, + workgroup, + _context.UInt(0x108)); + } + } + } +} + + + + + diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.HalfWave64Lowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.HalfWave64Lowering.cs new file mode 100644 index 0000000..95fdc62 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.HalfWave64Lowering.cs @@ -0,0 +1,158 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + // Independent halves duplicate scalar work; a lane-local proof is not a performance proof. + private static readonly bool LaneLocalWave64Enabled = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_LANE_LOCAL"), "1", StringComparison.Ordinal); + + private static bool CanRunHalfWaves(ShaderCompileRequest request, int[] physicalAxisOfLogical) + { + if (!LaneLocalWave64Enabled || + physicalAxisOfLogical is not [0, 1, 2]) + { + return false; + } + + var sharedFlatPcs = request.Memory.Entries + .Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) + .Select(static memory => memory.Pc) + .ToHashSet(); + return Ir.Gen5WaveLaneLocalAnalysis.IsLaneLocal(request.Program, sharedFlatPcs, out _); + } + + private sealed class HalfWave64Lowering : IWaveLowering + { + private readonly CompilationContext _context; + + public HalfWave64Lowering(CompilationContext context) + { + _context = context; + } + + public bool RequiresCrossSubgroupSynchronization => false; + + public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount; + + public void DeclareResources() { } + public void DeclareFunctionResources() { } + public void BeginBlock() { } + public void BeforeInstruction(Gen5ShaderInstruction instruction) { } + public void EndBlock() { } + + public void InitializeMasks() + { + var context = _context; + if (context._subgroupInvocationIdInput != 0) + { + StoreWaveMask(106, context._module.ConstantBool(false)); + StoreWaveMask(126, context._module.ConstantBool(true)); + return; + } + + context.StoreS64(106, context._module.Constant64(context._ulongType, 0)); + context.StoreS64(126, context._module.Constant64(context._ulongType, 1)); + } + + public uint BooleanToWaveMask(uint condition) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + condition, + CurrentLaneBit(), + context._module.Constant64(context._ulongType, 0)); + } + + var ballot = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + condition); + var low = context._module.AddInstruction(SpirvOp.CompositeExtract, context._uintType, ballot, 0); + var widened = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, low); + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + context._module.AddInstruction( + SpirvOp.UGreaterThanEqual, + context._boolType, + context.GuestWaveLane(), + context.UInt(32)), + context.ShiftLeftLogical64(widened, context._module.Constant64(context._ulongType, 32)), + widened); + } + + public uint SubgroupAny(uint condition) => + _context._subgroupInvocationIdInput == 0 + ? condition + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformAny, + _context._boolType, + _context.UInt(3), + condition); + + public uint CurrentLaneBit() + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.Constant64(context._ulongType, 1); + } + + return context.ShiftLeftLogical64( + context._module.Constant64(context._ulongType, 1), + context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, context.GuestWaveLane())); + } + + public uint BroadcastFirstActive(uint value) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return value; + } + + var activeLanes = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + context.Load(context._boolType, context._exec)); + var activeLow = context._module.AddInstruction(SpirvOp.CompositeExtract, context._uintType, activeLanes, 0); + return context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + context._uintType, + context.UInt(3), + value, + context.Ext(73, context._uintType, activeLow)); + } + + public uint BroadcastLane(uint value, uint lane) => + throw new InvalidOperationException("a half-wave program reads no other lane"); + + public uint BroadcastValue(uint writer, uint value, uint lane) => + throw new InvalidOperationException("a half-wave program writes no other lane"); + + public uint WaveMaskAny(uint register, uint laneActive) => + SubgroupAny(_context.Load(_context._boolType, laneActive)); + + public uint CurrentExecMask() => + BooleanToWaveMask(_context.Load(_context._boolType, _context._exec)); + + public uint NormalizeExecMask(uint mask) => mask; + + public void StoreWaveMask(uint register, uint condition) => + _context.StoreS64(register, BooleanToWaveMask(condition)); + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave32Lowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave32Lowering.cs new file mode 100644 index 0000000..3491e4f --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave32Lowering.cs @@ -0,0 +1,156 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + // A guest Wave32 maps directly to one 32-lane host subgroup. This is the + // former implicit/default lowering, made explicit so every guest wave size + // has a concrete lowering strategy. + private sealed class NativeWave32Lowering : IWaveLowering + { + private readonly CompilationContext _context; + + public NativeWave32Lowering(CompilationContext context) + { + _context = context; + } + + public bool RequiresCrossSubgroupSynchronization => false; + + public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount; + + public void DeclareResources() { } + public void DeclareFunctionResources() { } + public void BeginBlock() { } + public void BeforeInstruction(Gen5ShaderInstruction instruction) { } + public void EndBlock() { } + + public void InitializeMasks() + { + var context = _context; + if (context._subgroupInvocationIdInput != 0) + { + StoreWaveMask(106, context._module.ConstantBool(false)); + StoreWaveMask(126, context._module.ConstantBool(true)); + return; + } + + context.StoreS64(106, context._module.Constant64(context._ulongType, 0)); + context.StoreS64(126, context._module.Constant64(context._ulongType, 1)); + } + + public uint BooleanToWaveMask(uint condition) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + condition, + CurrentLaneBit(), + context._module.Constant64(context._ulongType, 0)); + } + + var ballot = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + condition); + var low = context._module.AddInstruction( + SpirvOp.CompositeExtract, + context._uintType, + ballot, + 0); + return context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, low); + } + + public uint SubgroupAny(uint condition) => + _context._subgroupInvocationIdInput == 0 + ? condition + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformAny, + _context._boolType, + _context.UInt(3), + condition); + + public uint CurrentLaneBit() + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.Constant64(context._ulongType, 1); + } + + var shifted = context.ShiftLeftLogical64( + context._module.Constant64(context._ulongType, 1), + context._module.AddInstruction( + SpirvOp.UConvert, + context._ulongType, + context.GuestWaveLane())); + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + context.IsCurrentLaneInRdnaWave(), + shifted, + context._module.Constant64(context._ulongType, 0)); + } + + public uint BroadcastFirstActive(uint value) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return value; + } + + var activeLanes = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + context.Load(context._boolType, context._exec)); + var activeLow = context._module.AddInstruction( + SpirvOp.CompositeExtract, + context._uintType, + activeLanes, + 0); + var firstActiveLane = context.Ext(73, context._uintType, activeLow); + return context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + context._uintType, + context.UInt(3), + value, + firstActiveLane); + } + + public uint BroadcastLane(uint value, uint lane) => + _context._subgroupInvocationIdInput == 0 + ? value + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + _context._uintType, + _context.UInt(3), + value, + lane); + + public uint BroadcastValue(uint writer, uint value, uint lane) => + _context.ShuffleLane(value, lane); + + public uint WaveMaskAny(uint register, uint laneActive) => + SubgroupAny(_context.Load(_context._boolType, laneActive)); + + public uint CurrentExecMask() => + BooleanToWaveMask(_context.Load(_context._boolType, _context._exec)); + + public uint NormalizeExecMask(uint mask) => + _context.BitwiseAnd64(mask, _context._module.Constant64(_context._ulongType, 0xFFFF_FFFFUL)); + + public void StoreWaveMask(uint register, uint condition) => + _context.StoreS(register, _context.Narrow(BooleanToWaveMask(condition))); + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave64Lowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave64Lowering.cs new file mode 100644 index 0000000..d4daa6e --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave64Lowering.cs @@ -0,0 +1,160 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + // One guest wave is one 64-lane host subgroup (ballot.x = lanes 0..31, ballot.y = lanes + // 32..63). It has no workgroup exchange, LDS reservation, half-mask plan or + // cross-subgroup barrier state. + private sealed class NativeWave64Lowering : IWaveLowering + { + private readonly CompilationContext _context; + + public NativeWave64Lowering(CompilationContext context) + { + _context = context; + } + + public bool RequiresCrossSubgroupSynchronization => false; + + public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount; + + public void DeclareResources() { } + public void DeclareFunctionResources() { } + public void BeginBlock() { } + public void BeforeInstruction(Gen5ShaderInstruction instruction) { } + public void EndBlock() { } + + public void InitializeMasks() + { + var context = _context; + if (context._subgroupInvocationIdInput != 0) + { + StoreWaveMask(106, context._module.ConstantBool(false)); + StoreWaveMask(126, context._module.ConstantBool(true)); + } + else + { + context.StoreS64(106, context._module.Constant64(context._ulongType, 0)); + context.StoreS64(126, context._module.Constant64(context._ulongType, 1)); + } + } + + public uint BooleanToWaveMask(uint condition) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + condition, + CurrentLaneBit(), + context._module.Constant64(context._ulongType, 0)); + } + + return BallotToMask(condition); + } + + // The host subgroup is at least 64 wide: lanes 0..31 are ballot.x and lanes 32..63 ballot.y. + private uint BallotToMask(uint condition) + { + var context = _context; + var ballot = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + condition); + var low = context._module.AddInstruction(SpirvOp.CompositeExtract, context._uintType, ballot, 0); + var high = context._module.AddInstruction(SpirvOp.CompositeExtract, context._uintType, ballot, 1); + return context.BitwiseOr64( + context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, low), + context.ShiftLeftLogical64( + context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, high), + context._module.Constant64(context._ulongType, 32))); + } + + public uint SubgroupAny(uint condition) => + _context._subgroupInvocationIdInput == 0 + ? condition + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformAny, + _context._boolType, + _context.UInt(3), + condition); + + public uint CurrentLaneBit() + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.Constant64(context._ulongType, 1); + } + + var shifted = context.ShiftLeftLogical64( + context._module.Constant64(context._ulongType, 1), + context._module.AddInstruction( + SpirvOp.UConvert, + context._ulongType, + context.GuestWaveLane())); + return shifted; + } + + public uint BroadcastFirstActive(uint value) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return value; + } + + var activeLanes = BallotToMask(context.Load(context._boolType, context._exec)); + var activeLow = context._module.AddInstruction(SpirvOp.UConvert, context._uintType, activeLanes); + var activeHigh = context._module.AddInstruction( + SpirvOp.UConvert, + context._uintType, + context.ShiftRightLogical64(activeLanes, context._module.Constant64(context._ulongType, 32))); + var firstActiveLane = context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + context._module.AddInstruction(SpirvOp.IEqual, context._boolType, activeLow, context.UInt(0)), + context.IAdd(context.Ext(73, context._uintType, activeHigh), context.UInt(32)), + context.Ext(73, context._uintType, activeLow)); + return context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + context._uintType, + context.UInt(3), + value, + firstActiveLane); + } + + public uint BroadcastLane(uint value, uint lane) => + _context._subgroupInvocationIdInput == 0 + ? value + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + _context._uintType, + _context.UInt(3), + value, + lane); + + public uint BroadcastValue(uint writer, uint value, uint lane) => + _context.ShuffleLane(value, lane); + + public uint WaveMaskAny(uint register, uint laneActive) => + SubgroupAny(_context.Load(_context._boolType, laneActive)); + + public uint CurrentExecMask() => + BooleanToWaveMask(_context.Load(_context._boolType, _context._exec)); + + public uint NormalizeExecMask(uint mask) => mask; + + public void StoreWaveMask(uint register, uint condition) => + _context.StoreS64(register, BooleanToWaveMask(condition)); + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.PairedLanes.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.PairedLanes.cs new file mode 100644 index 0000000..6e325ac --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.PairedLanes.cs @@ -0,0 +1,464 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private const uint PairedHalfLaneCount = 32; + + private static readonly bool PairedWave64Enabled = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_PAIRED"), "0", StringComparison.Ordinal); + + private readonly bool _pairedLanes; + private int _laneHalf; + private bool _inLanePass; + private readonly Dictionary _upperVectorRegisterVariables = new(); + private readonly uint[] _packedHalfRegistersByHalf = new uint[2]; + private uint _lowerExec; + private uint _upperExec; + private uint _lowerVcc; + private uint _upperVcc; + private uint _lowerScratch; + private uint _upperScratch; + private uint _subgroupIdInput; + private uint _pairedCounterResult; + private Dictionary? _scalarSnapshot; + + private void SelectLaneHalf(int half) + { + if (!_pairedLanes) + { + return; + } + + _packedHalfRegistersByHalf[_laneHalf] = _packedHalfRegisters; + _laneHalf = half; + _packedHalfRegisters = _packedHalfRegistersByHalf[half]; + _exec = half == 0 ? _lowerExec : _upperExec; + _vcc = half == 0 ? _lowerVcc : _upperVcc; + if (_lowerScratch != 0) + { + _scratch = half == 0 ? _lowerScratch : _upperScratch; + } + } + + private void DeclarePairedLaneFlags(uint lowerVcc, uint lowerExec, Func flag) + { + _lowerVcc = lowerVcc; + _lowerExec = lowerExec; + if (!_pairedLanes) + { + return; + } + + _upperVcc = flag(false); + _upperExec = flag(true); + _interfaces.Add(_upperVcc); + _interfaces.Add(_upperExec); + _module.AddName(_upperVcc, "vccUpperHalf"); + _module.AddName(_upperExec, "execUpperHalf"); + } + + private void DeclarePairedScratch(uint arrayPointer, uint arrayType) + { + _lowerScratch = _scratch; + if (!_pairedLanes) + { + return; + } + + _upperScratch = _module.AddGlobalVariable( + arrayPointer, + SpirvStorageClass.Private, + _module.ConstantNull(arrayType)); + _interfaces.Add(_upperScratch); + _module.AddName(_upperScratch, "scratchUpperHalf"); + } + + private void DeclarePairedLaneInputs() + { + if (!_pairedLanes) + { + return; + } + + _subgroupIdInput = _module.AddGlobalVariable( + _module.TypePointer(SpirvStorageClass.Input, _uintType), + SpirvStorageClass.Input); + _module.AddDecoration(_subgroupIdInput, SpirvDecoration.BuiltIn, (uint)SpirvBuiltIn.SubgroupId); + _interfaces.Add(_subgroupIdInput); + } + + private uint PairedHostThreadCount() + { + var guestThreads = checked(_localSizeX * _localSizeY * _localSizeZ); + return (guestThreads + 63) / 64 * PairedHalfLaneCount; + } + + private Dictionary CurrentVectorRegisterVariables() => + _pairedLanes && _laneHalf == 1 ? _upperVectorRegisterVariables : _vectorRegisterVariables; + + private string CurrentVectorRegisterPrefix() => + _pairedLanes && _laneHalf == 1 ? "vu" : "v"; + + private uint PairedSubgroupLane() => Load(_uintType, _subgroupInvocationIdInput); + + private uint PairedGuestLane() => + _laneHalf == 0 ? PairedSubgroupLane() : IAdd(PairedSubgroupLane(), UInt(PairedHalfLaneCount)); + + private uint HalfBallot(uint condition) => + _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + _module.AddInstruction(SpirvOp.GroupNonUniformBallot, _uvec4Type, UInt(3), condition), + 0); + + private uint SubgroupBroadcast(uint value, uint lane) => + _module.AddInstruction(SpirvOp.GroupNonUniformBroadcast, _uintType, UInt(3), value, lane); + + private uint ExecHalfBallot(int half) + { + var current = _laneHalf; + SelectLaneHalf(half); + var ballot = HalfBallot(Load(_boolType, _exec)); + SelectLaneHalf(current); + return ballot; + } + + private uint VectorSourceOfHalf(Gen5ShaderInstruction instruction, int sourceIndex, int half) + { + var current = _laneHalf; + SelectLaneHalf(half); + var value = GetRawSource(instruction, sourceIndex); + SelectLaneHalf(current); + return value; + } + + private static bool IsPairedLaneInstruction(Gen5ShaderInstruction instruction) => + instruction.Opcode is not ("VReadlaneB32" or "VReadfirstlaneB32" or "DsAppend" or "DsConsume") && + instruction.Encoding is + Gen5ShaderEncoding.Mubuf or + Gen5ShaderEncoding.Mtbuf or + Gen5ShaderEncoding.Vop1 or + Gen5ShaderEncoding.Vop2 or + Gen5ShaderEncoding.Vopc or + Gen5ShaderEncoding.Vop3 or + Gen5ShaderEncoding.Vop3p or + Gen5ShaderEncoding.Vintrp or + Gen5ShaderEncoding.Ds or + Gen5ShaderEncoding.Flat or + Gen5ShaderEncoding.Mimg or + Gen5ShaderEncoding.Exp; + + private bool TryEmitPairedLaneInstruction(Gen5ShaderInstruction instruction, out string error) + { + _scalarSnapshot = SnapshotScalarSources(instruction); + _inLanePass = true; + try + { + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + if (!TryEmitInstruction(instruction, out error)) + { + return false; + } + } + + error = string.Empty; + return true; + } + finally + { + _inLanePass = false; + _scalarSnapshot = null; + SelectLaneHalf(0); + } + } + + private Dictionary? SnapshotScalarSources(Gen5ShaderInstruction instruction) + { + Dictionary? snapshot = null; + foreach (var operand in instruction.Sources) + { + if (operand.Kind != Gen5OperandKind.ScalarRegister) + { + continue; + } + + for (var register = operand.Value; register <= operand.Value + 1 && register < ScalarRegisterCount; register++) + { + if (register == 125) + { + continue; + } + + snapshot ??= new Dictionary(); + if (!snapshot.ContainsKey(register)) + { + snapshot[register] = Load(_uintType, ScalarPointer(register)); + } + } + } + + return snapshot; + } + + private void UpdateLaneMaskFlag(uint lowRegister) + { + if (!_pairedLanes || _inLanePass) + { + Store(lowRegister == 106 ? _vcc : _exec, IsLaneSetInMaskRegisters(lowRegister)); + return; + } + + var current = _laneHalf; + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + Store(lowRegister == 106 ? _vcc : _exec, IsLaneSetInMaskRegisters(lowRegister)); + } + + SelectLaneHalf(current); + } + + private void StorePairedLaneMaskWord(uint register, uint value) + { + var word = _laneHalf == 0 + ? Narrow(value) + : Narrow(ShiftRightLogical64(value, _module.Constant64(_ulongType, 32))); + StoreS(register + (uint)_laneHalf, word); + } + + private void EmitPairedComputeInitialState() + { + var workGroupId = Load(_uvec3Type, _workGroupIdInput); + var waveBase = _module.AddInstruction( + SpirvOp.IMul, + _uintType, + Load(_uintType, _subgroupIdInput), + UInt(2 * PairedHalfLaneCount)); + var guestThreads = checked(_localSizeX * _localSizeY * _localSizeZ); + var limits = new uint[3]; + var groups = new uint[3]; + for (uint component = 0; component < 3; component++) + { + limits[component] = ComputeThreadLimit(component); + groups[component] = _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + workGroupId, + (uint)_physicalAxisOfLogical[component]); + } + + var active = new uint[2]; + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + var index = IAdd(waveBase, PairedGuestLane()); + var inBounds = _module.AddInstruction(SpirvOp.ULessThan, _boolType, index, UInt(guestThreads)); + var row = _module.AddInstruction(SpirvOp.UDiv, _uintType, index, UInt(_localSizeX)); + var local = new[] + { + _module.AddInstruction(SpirvOp.UMod, _uintType, index, UInt(_localSizeX)), + _module.AddInstruction(SpirvOp.UMod, _uintType, row, UInt(_localSizeY)), + _module.AddInstruction(SpirvOp.UDiv, _uintType, row, UInt(_localSizeY)), + }; + var sizes = new[] { _localSizeX, _localSizeY, _localSizeZ }; + for (uint component = 0; component < 3; component++) + { + StoreV(component, local[component], guardWithExec: false); + var global = IAdd( + _module.AddInstruction(SpirvOp.IMul, _uintType, groups[component], UInt(sizes[component])), + local[component]); + inBounds = LogicalAnd( + inBounds, + _module.AddInstruction(SpirvOp.ULessThan, _boolType, global, limits[component])); + } + + active[half] = inBounds; + } + + SelectLaneHalf(0); + var lowerActive = HalfBallot(active[0]); + var upperActive = HalfBallot(active[1]); + StoreS(126, lowerActive); + StoreS(127, upperActive); + Store(_programActive, _module.AddInstruction(SpirvOp.LogicalOr, _boolType, active[0], active[1])); + + if (_request.ComputeSystemRegisters is { } registers) + { + StoreComputeSystemRegister(registers.WorkGroupXRegister, workGroupId, (uint)_physicalAxisOfLogical[0]); + StoreComputeSystemRegister(registers.WorkGroupYRegister, workGroupId, (uint)_physicalAxisOfLogical[1]); + StoreComputeSystemRegister(registers.WorkGroupZRegister, workGroupId, (uint)_physicalAxisOfLogical[2]); + if (registers.ThreadGroupSizeRegister is { } sizeRegister) + { + StoreS(sizeRegister, UInt(guestThreads)); + } + } + } + + private uint PairedReadFirstLane(Gen5ShaderInstruction instruction) + { + var lowerActive = ExecHalfBallot(0); + var upperActive = ExecHalfBallot(1); + var lowerValue = VectorSourceOfHalf(instruction, 0, 0); + var upperValue = VectorSourceOfHalf(instruction, 0, 1); + var lowerHasActive = IsNotZero(lowerActive); + var useLower = LogicalOr(lowerHasActive, LogicalNot(IsNotZero(upperActive))); + var lowerLane = _module.AddInstruction( + SpirvOp.Select, + _uintType, + lowerHasActive, + Ext(73, _uintType, lowerActive), + UInt(0)); + var lane = BitwiseAnd( + _module.AddInstruction( + SpirvOp.Select, + _uintType, + useLower, + lowerLane, + Ext(73, _uintType, upperActive)), + UInt(PairedHalfLaneCount - 1)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + useLower, + SubgroupBroadcast(lowerValue, lane), + SubgroupBroadcast(upperValue, lane)); + } + + private uint PairedReadLane(Gen5ShaderInstruction instruction, uint selectedLane) + { + var lowerValue = VectorSourceOfHalf(instruction, 0, 0); + var upperValue = VectorSourceOfHalf(instruction, 0, 1); + var lane = BitwiseAnd(selectedLane, UInt(PairedHalfLaneCount - 1)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction(SpirvOp.ULessThan, _boolType, selectedLane, UInt(PairedHalfLaneCount)), + SubgroupBroadcast(lowerValue, lane), + SubgroupBroadcast(upperValue, lane)); + } + + private uint LogicalOr(uint left, uint right) => + _module.AddInstruction(SpirvOp.LogicalOr, _boolType, left, right); + + private void EmitPairedWaveCounter(uint destination, uint inBounds, Func update) + { + var lowerActive = ExecHalfBallot(0); + var upperActive = ExecHalfBallot(1); + var activeCount = IAdd( + _module.AddInstruction(SpirvOp.BitCount, _uintType, lowerActive), + _module.AddInstruction(SpirvOp.BitCount, _uintType, upperActive)); + var anyActive = IsNotZero(BitwiseOr(lowerActive, upperActive)); + var leaderLane = BitwiseAnd( + _module.AddInstruction( + SpirvOp.Select, + _uintType, + IsNotZero(lowerActive), + Ext(73, _uintType, lowerActive), + Ext(73, _uintType, upperActive)), + UInt(PairedHalfLaneCount - 1)); + if (_pairedCounterResult == 0) + { + _pairedCounterResult = DeclareFunctionUint(UInt(0)); + _module.AddName(_pairedCounterResult, "pairedCounterResult"); + } + + var valid = LogicalAnd(inBounds, anyActive); + var leader = LogicalAnd( + valid, + _module.AddInstruction(SpirvOp.IEqual, _boolType, PairedSubgroupLane(), leaderLane)); + EmitConditional(leader, () => Store(_pairedCounterResult, update(activeCount))); + var original = SubgroupBroadcast(Load(_uintType, _pairedCounterResult), leaderLane); + var result = _module.AddInstruction(SpirvOp.Select, _uintType, valid, original, UInt(0)); + var current = _laneHalf; + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + StoreV(destination, result); + } + + SelectLaneHalf(current); + } + + private sealed class PairedWave64Lowering : IWaveLowering + { + private readonly CompilationContext _context; + + public PairedWave64Lowering(CompilationContext context) + { + _context = context; + } + + public bool RequiresCrossSubgroupSynchronization => false; + + public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount; + + public void DeclareResources() { } + public void DeclareFunctionResources() { } + public void BeginBlock() { } + public void BeforeInstruction(Gen5ShaderInstruction instruction) { } + public void EndBlock() { } + + public void InitializeMasks() => + _context.StoreS64(106, _context._module.Constant64(_context._ulongType, 0)); + + public uint BooleanToWaveMask(uint condition) + { + var context = RequireLanePass(nameof(BooleanToWaveMask)); + var wide = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, context.HalfBallot(condition)); + return context._laneHalf == 0 + ? wide + : context.ShiftLeftLogical64(wide, context._module.Constant64(context._ulongType, 32)); + } + + public uint SubgroupAny(uint condition) + { + var context = RequireLanePass(nameof(SubgroupAny)); + return context._module.AddInstruction(SpirvOp.GroupNonUniformAny, context._boolType, context.UInt(3), condition); + } + + public uint CurrentLaneBit() => + _context.ShiftLeftLogical64( + _context._module.Constant64(_context._ulongType, 1), + _context._module.AddInstruction(SpirvOp.UConvert, _context._ulongType, _context.PairedGuestLane())); + + public uint BroadcastFirstActive(uint value) => + throw new InvalidOperationException("paired wave64 reads the first active lane from both halves"); + + public uint BroadcastLane(uint value, uint lane) => + throw new InvalidOperationException("paired wave64 reads a lane from both halves"); + + public uint BroadcastValue(uint writer, uint value, uint lane) => + throw new InvalidOperationException("paired wave64 broadcasts a wave counter from both halves"); + + public uint WaveMaskAny(uint register, uint laneActive) => + _context.IsNotZero64(_context.LoadS64(register)); + + public uint CurrentExecMask() => _context.LoadS64(126); + + public uint NormalizeExecMask(uint mask) => mask; + + public void StoreWaveMask(uint register, uint condition) + { + var context = RequireLanePass(nameof(StoreWaveMask)); + context.StoreS(register + (uint)context._laneHalf, context.HalfBallot(condition)); + } + + private CompilationContext RequireLanePass(string operation) + { + if (!_context._inLanePass) + { + throw new InvalidOperationException($"paired wave64 {operation} needs a lane half"); + } + + return _context; + } + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs index 9d682a8..ce3d1f4 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs @@ -68,6 +68,8 @@ public static partial class Gen5SpirvTranslator private readonly Dictionary _indirectKeyScratch = []; private uint _pushData; private uint _shaderData; + private uint _mipStatistics; + private uint _helperInvocationInput; private uint _flattenedTable; private uint _pageTable; private uint _faultBuffer; @@ -110,13 +112,43 @@ public static partial class Gen5SpirvTranslator _usesPixelValidMask = _stage == Gen5SpirvStage.Pixel && request.Program.Instructions.Any(static instruction => instruction.Control is Gen5ExportControl { ValidMask: true }); + _pixelExecExcludesHelpers = + _stage == Gen5SpirvStage.Pixel && PixelLiveExec && + request.Program.Instructions.Any(static instruction => instruction.Opcode is "DsAppend" or "DsConsume"); _enableGraphicsSubgroupOperations = _stage == Gen5SpirvStage.Compute || request.EnableGraphicsSubgroupOperations; _waveLaneCount = request.WaveSize == 64 ? 64u : 32u; _localSizeX = Math.Max(request.LocalSizeX, 1); _localSizeY = Math.Max(request.LocalSizeY, 1); _localSizeZ = Math.Max(request.LocalSizeZ, 1); _physicalAxisOfLogical = ComputeWorkgroupAxisOrder(_localSizeX, _localSizeY, _localSizeZ); - _emulateWave64 = _stage == Gen5SpirvStage.Compute && _waveLaneCount == 64 && (ulong)_localSizeX * _localSizeY * _localSizeZ == 64; + // A wave64 runs as one 64-lane host subgroup when the host pins it to one + // (HostWave64Supported). Hosts without 64-lane subgroups bridge the two 32-lane + // halves through workgroup memory. + if (_waveLaneCount == 32) + { + _waveLowering = new NativeWave32Lowering(this); + } + else if (request.HostWave64Supported) + { + _waveLowering = new NativeWave64Lowering(this); + _guestLaneFromSubgroup = true; + } + else if (_stage == Gen5SpirvStage.Compute && + request.HostComputeSubgroup32 && + CanRunHalfWaves(request, _physicalAxisOfLogical)) + { + _waveLowering = new HalfWave64Lowering(this); + } + else if (_stage == Gen5SpirvStage.Compute && request.HostComputeSubgroup32 && PairedWave64Enabled) + { + _pairedLanes = true; + _waveLowering = new PairedWave64Lowering(this); + } + else + { + _waveLowering = new EmulatedWave64Lowering(this); + } + _requiredVertexOutputCount = request.RequiredVertexOutputCount; _pixelInputEnable = request.PixelInputEnable; _pixelInputAddress = request.PixelInputAddress; @@ -174,6 +206,9 @@ public static partial class Gen5SpirvTranslator case DescriptorBindingKind.ShaderData: _shaderData = DeclareWordBlock("shaderData", number); break; + case DescriptorBindingKind.MipStatistics: + _mipStatistics = DeclareWordBlock("mipStatistics", number); + break; default: DeclareImageClass(binding, number); break; @@ -1590,11 +1625,127 @@ public static partial class Gen5SpirvTranslator imageClass.Dimension, imageInfo.ConversionFormat, imageInfo.ShaderSwizzle, - imageInfo.EmulatedCompareFunction); + imageInfo.EmulatedCompareFunction, + resourceIndex); dstSelect = imageInfo.ShaderSwizzle; return true; } + private const uint MipStatisticsEntries = 257; + private const uint MipStatisticsNoCounter = 0x8000; + + private void RecordMipStatistics(SpirvImageResource resource, uint imageObject, uint coordinates, uint? explicitLod, uint bias) + { + if (_mipStatistics == 0 || _stage != Gen5SpirvStage.Pixel || resource.ResourceIndex < 0 || + resource.IsStorage || resource.Multisampled || resource.Cube) + { + return; + } + + var field = LoadShaderDataDword(UInt(_request.Bindings.MipStatisticsDword + (uint)resource.ResourceIndex)); + uint lod; + if (explicitLod is { } value) + { + lod = value; + } + else + { + var queried = _module.AddInstruction( + SpirvOp.ImageQueryLod, + _vec2Type, + imageObject, + SpatialCoordinates(resource, coordinates)); + lod = _module.AddInstruction( + SpirvOp.FAdd, + _floatType, + _module.AddInstruction(SpirvOp.CompositeExtract, _floatType, queried, 1u), + bias); + } + + var counter = BitwiseAnd(field, UInt(0xFF)); + var baseLevel = _module.AddInstruction(SpirvOp.ConvertUToF, _floatType, BitwiseAnd(ShiftRightLogical(field, UInt(8)), UInt(0xF))); + var threshold = _module.AddInstruction( + SpirvOp.FMul, + _floatType, + _module.AddInstruction(SpirvOp.ConvertUToF, _floatType, BitwiseAnd(ShiftRightLogical(field, UInt(16)), UInt(0xFFF))), + Float(1f / 256f)); + var absolute = Ext(43, _floatType, _module.AddInstruction(SpirvOp.FAdd, _floatType, lod, baseLevel), Float(0f), Float(14f)); + var level = _module.AddInstruction(SpirvOp.ConvertFToU, _uintType, absolute); + var clamped = _module.AddInstruction(SpirvOp.FOrdLessThan, _boolType, absolute, threshold); + var record = LogicalAnd( + Load(_boolType, _exec), + _module.AddInstruction(SpirvOp.IEqual, _boolType, BitwiseAnd(field, UInt(MipStatisticsNoCounter)), UInt(0))); + record = LogicalAnd(record, _module.AddInstruction(SpirvOp.LogicalNot, _boolType, Load(_boolType, HelperInvocationInput()))); + var finestPointer = MipStatisticsPointer(counter); + var countPointer = MipStatisticsPointer(IAdd(counter, UInt(MipStatisticsEntries))); + EmitConditional(record, () => + { + if (!_request.MipStatisticsSubgroupReduction) + { + _module.AddInstruction(SpirvOp.AtomicUMin, _uintType, finestPointer, UInt(1), UInt(0), level); + EmitConditional(clamped, () => + _module.AddInstruction(SpirvOp.AtomicIAdd, _uintType, countPointer, UInt(1), UInt(0), UInt(1))); + return; + } + + _module.AddCapability(SpirvCapability.GroupNonUniform); + _module.AddCapability(SpirvCapability.GroupNonUniformBallot); + _module.AddCapability(SpirvCapability.GroupNonUniformArithmetic); + var finest = _module.AddInstruction(SpirvOp.GroupNonUniformUMin, _uintType, UInt(3), 0u, level); + var ballot = _module.AddInstruction(SpirvOp.GroupNonUniformBallot, _uvec4Type, UInt(3), clamped); + var count = _module.AddInstruction(SpirvOp.GroupNonUniformBallotBitCount, _uintType, UInt(3), 0u, ballot); + var leader = _module.AddInstruction(SpirvOp.GroupNonUniformElect, _boolType, UInt(3)); + EmitConditional(leader, () => + { + _module.AddInstruction(SpirvOp.AtomicUMin, _uintType, finestPointer, UInt(1), UInt(0), finest); + EmitConditional(IsNotZero(count), () => + _module.AddInstruction(SpirvOp.AtomicIAdd, _uintType, countPointer, UInt(1), UInt(0), count)); + }); + }); + } + + private uint MipStatisticsPointer(uint index) => + BlockWordPointer(_mipStatistics, index); + + private uint HelperInvocationInput() + { + if (_helperInvocationInput == 0) + { + _helperInvocationInput = _module.AddGlobalVariable( + _module.TypePointer(SpirvStorageClass.Input, _boolType), + SpirvStorageClass.Input); + _module.AddDecoration(_helperInvocationInput, SpirvDecoration.BuiltIn, (uint)SpirvBuiltIn.HelperInvocation); + _module.AddName(_helperInvocationInput, "helperInvocation"); + _interfaces.Add(_helperInvocationInput); + } + + return _helperInvocationInput; + } + + private uint SpatialCoordinates(SpirvImageResource resource, uint coordinates) + { + var spatial = ImageSpatialComponentCountOf(resource.Dimension); + if (!resource.Arrayed) + { + return coordinates; + } + + if (spatial == 1) + { + return _module.AddInstruction(SpirvOp.CompositeExtract, _floatType, coordinates, 0u); + } + + var components = new uint[spatial + 2]; + components[0] = coordinates; + components[1] = coordinates; + for (var index = 0u; index < spatial; index++) + { + components[index + 2] = index; + } + + return _module.AddInstruction(SpirvOp.VectorShuffle, _module.TypeVector(_floatType, spatial), components); + } + private static uint ImageSpatialComponentCountOf(SpirvImageDim dimension) => dimension switch { SpirvImageDim.Dim1D => 1u, @@ -1881,7 +2032,25 @@ public static partial class Gen5SpirvTranslator var index = GlobalDataShareIndex(baseAddress, offset); var inBounds = LogicalAnd(IsNotZero(sizeBytes), IsBlockWordInRange(_globalDataShare, index)); var destination = instruction.Destinations[0].Value; + if (_pairedLanes) + { + EmitPairedWaveCounter(destination, inBounds, count => EmitAtomic( + instruction.Opcode == "DsAppend" ? SpirvOp.AtomicIAdd : SpirvOp.AtomicISub, + _uintType, + BlockWordPointer(_globalDataShare, index), + scope: 1, + semantics: 0x48, + value: () => count, + comparator: () => UInt(0))); + return true; + } + var active = Load(_boolType, _exec); + if (_stage == Gen5SpirvStage.Pixel) + { + active = LogicalAnd(active, _module.AddInstruction(SpirvOp.LogicalNot, _boolType, Load(_boolType, HelperInvocationInput()))); + } + var activeMask = BooleanToWaveMask(active); var activeLow = Narrow(activeMask); var activeHigh = Narrow(ShiftRightLogical64(activeMask, ULong(32))); @@ -1909,21 +2078,7 @@ public static partial class Gen5SpirvTranslator }); var firstValue = LoadV(destination); - uint broadcast; - if (_emulateWave64) - { - broadcast = ExchangeWave64Value(isFirstActive, firstValue); - } - else if (_stage == Gen5SpirvStage.Compute || _enableGraphicsSubgroupOperations) - { - // Every active lane receives the first lane's old counter, in graphics stages too. - broadcast = _module.AddInstruction(SpirvOp.GroupNonUniformShuffle, _uintType, UInt(3), firstValue, firstLane); - } - else - { - // Without subgroup operations a graphics invocation is its own wave of one lane. - broadcast = firstValue; - } + var broadcast = BroadcastWaveValue(isFirstActive, firstValue, firstLane); var validResult = LogicalAnd(inBounds, IsNotZero64(activeMask)); StoreV(destination, _module.AddInstruction(SpirvOp.Select, _uintType, validResult, broadcast, UInt(0))); diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.WaveLowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.WaveLowering.cs new file mode 100644 index 0000000..a0d7296 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.WaveLowering.cs @@ -0,0 +1,34 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private interface IWaveLowering + { + bool RequiresCrossSubgroupSynchronization { get; } + + uint ReserveLdsDwords(uint guestDwordCount); + void DeclareResources(); + void DeclareFunctionResources(); + void InitializeMasks(); + void BeginBlock(); + void BeforeInstruction(Gen5ShaderInstruction instruction); + void EndBlock(); + uint BooleanToWaveMask(uint condition); + uint SubgroupAny(uint condition); + uint CurrentLaneBit(); + uint BroadcastFirstActive(uint value); + uint BroadcastLane(uint value, uint lane); + uint BroadcastValue(uint writer, uint value, uint lane); + uint WaveMaskAny(uint register, uint laneActive); + uint CurrentExecMask(); + uint NormalizeExecMask(uint mask); + void StoreWaveMask(uint register, uint condition); + } + } +} + diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs index 73a8a16..da1c589 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs @@ -17,8 +17,6 @@ public static partial class Gen5SpirvTranslator // so a smaller array is safe. private const uint PrivateLdsDwordCount = 2048; private const uint RdnaWaveLaneCount = 32; - private const uint Wave64ExchangeSlotCount = 4; - private const uint Wave64ExchangeDwordCount = Wave64ExchangeSlotCount * 2; internal static SpirvImageFormat DecodeStorageImageFormat( uint dataFormat, @@ -69,9 +67,14 @@ public static partial class Gen5SpirvTranslator private readonly Gen5SpirvStage _stage; private readonly IReadOnlyList _pixelOutputBindings; private readonly bool _usesPixelValidMask; + private readonly bool _pixelExecExcludesHelpers; + + private static readonly bool PixelLiveExec = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_PS_LIVE_EXEC"), "0", StringComparison.Ordinal); private readonly bool _enableGraphicsSubgroupOperations; private readonly uint _waveLaneCount; - private readonly bool _emulateWave64; + private readonly IWaveLowering _waveLowering; + private bool _guestLaneFromSubgroup; // Safety valve for the PC-dispatcher loop. Each iteration executes one // GCN basic block; a correctly-translated shader always reaches its @@ -195,12 +198,6 @@ public static partial class Gen5SpirvTranslator private uint _workGroupIdInput; private uint _pushConstantUintPointer; private uint _subgroupInvocationIdInput; - private uint _wave64Exchange; - private uint _wave64ExchangeElementPointer; - private uint _wave64ExchangeOffset; - private uint _wave64ExchangeParity; - private Ir.Gen5Wave64HalfMaskPlan? _halfMaskPlan; - private uint? _emittingPc; private uint _glsl; private enum ImageComponentKind @@ -231,7 +228,8 @@ public static partial class Gen5SpirvTranslator SpirvImageDim Dimension = SpirvImageDim.Dim2D, uint ConversionFormat = 0, uint ShaderSwizzle = 0, - int EmulatedCompareFunction = -1); + int EmulatedCompareFunction = -1, + int ResourceIndex = -1); private readonly record struct SpirvVertexInput( uint Variable, @@ -421,8 +419,8 @@ public static partial class Gen5SpirvTranslator var active = Load(_boolType, _programActive); if (_maxDispatcherSteps > 0) { - var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1)); - Store(_iterationGuard, steps); + var steps = IAdd(Load(_uintType, IterationGuard()), UInt(1)); + Store(IterationGuard(), steps); var withinLimit = _module.AddInstruction( SpirvOp.ULessThan, _boolType, @@ -538,12 +536,19 @@ public static partial class Gen5SpirvTranslator physicalSizes[_physicalAxisOfLogical[logical]] = logicalSizes[logical]; } - _module.AddExecutionMode( - main, - SpirvExecutionMode.LocalSize, - physicalSizes[0], - physicalSizes[1], - physicalSizes[2]); + if (_pairedLanes) + { + _module.AddExecutionMode(main, SpirvExecutionMode.LocalSize, PairedHostThreadCount(), 1, 1); + } + else + { + _module.AddExecutionMode( + main, + SpirvExecutionMode.LocalSize, + physicalSizes[0], + physicalSizes[1], + physicalSizes[2]); + } } var attributeCount = _stage == Gen5SpirvStage.Vertex @@ -625,7 +630,14 @@ public static partial class Gen5SpirvTranslator _scc = Flag(false); _vcc = Flag(false); _exec = Flag(true); - _reachedPixelExport = Flag(false); + DeclarePairedLaneFlags(_vcc, _exec, Flag); + // Only the title-shader trace reads it; elsewhere it would be a write-only Private + // global, and AMD's driver crashes linking stages that carry an unused one. + if (TraceTitleShaderState) + { + _reachedPixelExport = Flag(false); + } + if (_usesPixelValidMask) { _pixelValidMaskActive = Flag(true); @@ -635,22 +647,7 @@ public static partial class Gen5SpirvTranslator SpirvStorageClass.Private, _module.Constant(_uintType, 0)); _programActive = Flag(true); - if (UsesWave64Exchange()) - { - _wave64ExchangeParity = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, UInt(0)); - _interfaces.Add(_wave64ExchangeParity); - _module.AddName(_wave64ExchangeParity, "wave64ExchangeParity"); - } - if (_maxDispatcherSteps > 0) - { - _iterationGuard = _module.AddGlobalVariable( - _privateUintPointer, - SpirvStorageClass.Private, - _module.Constant(_uintType, 0)); - _interfaces.Add(_iterationGuard); - _module.AddName(_iterationGuard, "pcGuard"); - } - + _waveLowering.DeclareFunctionResources(); foreach (var slot in FindLaneSpillSlots()) { var variable = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, _module.Constant(_uintType, 0)); @@ -668,7 +665,11 @@ public static partial class Gen5SpirvTranslator _interfaces.Add(_scc); _interfaces.Add(_vcc); _interfaces.Add(_exec); - _interfaces.Add(_reachedPixelExport); + if (_reachedPixelExport != 0) + { + _interfaces.Add(_reachedPixelExport); + } + if (_pixelValidMaskActive != 0) { _interfaces.Add(_pixelValidMaskActive); @@ -681,7 +682,7 @@ public static partial class Gen5SpirvTranslator DeclareLayoutBindings(); DeclareScratch(); DeclareLds(); - DeclareWave64Scratch(); + _waveLowering.DeclareResources(); DeclareStageInterface(); return; } @@ -707,45 +708,9 @@ public static partial class Gen5SpirvTranslator _module.ConstantNull(arrayType)); _interfaces.Add(_scratch); _module.AddName(_scratch, "scratch"); + DeclarePairedScratch(arrayPointer, arrayType); } - private void DeclareWave64Scratch() - { - if (!UsesWave64Exchange()) - { - return; - } - - // Metal exposes 32 KiB of threadgroup memory on the Apple GPUs we - // target. Some PS5 compute shaders legitimately request all of it, - // so allocating another workgroup variable for the wave64 bridge - // makes pipeline creation fail. Reuse the final dwords of the - // existing LDS allocation in that case. The translator already - // bounds guest LDS accesses to this fixed allocation; keeping the - // bridge inside it preserves the host limit and still provides the - // cross-subgroup rendezvous needed to model one 64-lane guest wave. - if (_lds != 0) - { - _wave64Exchange = _lds; - _wave64ExchangeElementPointer = _ldsElementPointer; - _wave64ExchangeOffset = _ldsDwordMask + 1 < LdsDwordCount - ? _ldsDwordMask + 1 - : LdsDwordCount - Wave64ExchangeDwordCount; - return; - } - - var exchangeArrayType = _module.TypeArray(_uintType, Wave64ExchangeDwordCount); - _wave64ExchangeElementPointer = _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); - _wave64Exchange = _module.AddGlobalVariable( - _module.TypePointer(SpirvStorageClass.Workgroup, exchangeArrayType), - SpirvStorageClass.Workgroup); - _wave64ExchangeOffset = 0; - _module.AddName(_wave64Exchange, "wave64Exchange"); - _interfaces.Add(_wave64Exchange); - } - - private bool UsesWave64Exchange() => _emulateWave64 && UsesSubgroupOperations(); - private uint ComputeLdsGuestDwordCount() { var declared = _request.LocalDataShareDwords; @@ -776,9 +741,7 @@ public static partial class Gen5SpirvTranslator ? ComputeLdsGuestDwordCount() : PrivateLdsDwordCount; _ldsDwordMask = dwordCount - 1; - var arrayDwordCount = UsesWave64Exchange() && dwordCount < LdsDwordCount - ? dwordCount + Wave64ExchangeDwordCount - : dwordCount; + var arrayDwordCount = _waveLowering.ReserveLdsDwords(dwordCount); var ldsArrayType = _module.TypeArray(_uintType, arrayDwordCount); var ldsPointer = _module.TypePointer(storageClass, ldsArrayType); @@ -878,6 +841,8 @@ public static partial class Gen5SpirvTranslator (uint)SpirvBuiltIn.LocalInvocationIndex); _interfaces.Add(_localInvocationIndexInput); } + + DeclarePairedLaneInputs(); } if (_stage == Gen5SpirvStage.Vertex) @@ -1176,31 +1141,19 @@ public static partial class Gen5SpirvTranslator } Store(_scc, _module.ConstantBool(false)); - Store(_reachedPixelExport, _module.ConstantBool(false)); + if (_reachedPixelExport != 0) + { + Store(_reachedPixelExport, _module.ConstantBool(false)); + } + if (_pixelValidMaskActive != 0) { Store(_pixelValidMaskActive, _module.ConstantBool(true)); } - if (_subgroupInvocationIdInput != 0 && _emulateWave64) + _waveLowering.InitializeMasks(); + if (_pixelExecExcludesHelpers) { - StoreS64(106, BothHalvesOfOwnBallot(_module.ConstantBool(false))); - StoreS64(126, BothHalvesOfOwnBallot(_module.ConstantBool(true))); - } - else if (_subgroupInvocationIdInput != 0) - { - StoreWaveMask(106, _module.ConstantBool(false)); - StoreWaveMask(126, _module.ConstantBool(true)); - } - else - { - // Graphics stages emulate one logical wave lane. Keep the - // guest-visible VCC/EXEC scalar pairs synchronized with the - // internal booleans: shaders commonly save EXEC from s126:s127 - // and restore it after divergent work. Initializing only _exec - // left those registers at zero, so the first restore disabled - // every fragment before its color export. - StoreS64(106, _module.Constant64(_ulongType, 0)); - StoreS64(126, _module.Constant64(_ulongType, 1)); + _waveLowering.StoreWaveMask(126, _module.AddInstruction(SpirvOp.LogicalNot, _boolType, Load(_boolType, HelperInvocationInput()))); } // Primitive (NGG) vertex programs rebuild EXEC from the merged wave info in s3: @@ -1266,6 +1219,10 @@ public static partial class Gen5SpirvTranslator Store(output.Variable, _module.ConstantNull(output.Type)); } } + else if (_pairedLanes) + { + EmitPairedComputeInitialState(); + } else { var localId = Load(_uvec3Type, _localInvocationIdInput); @@ -1440,8 +1397,6 @@ public static partial class Gen5SpirvTranslator StoreS(register.Value, value); } - private enum SharedMemoryPhase { None, Read, Write } - private bool TryEmitBlock( IReadOnlyList blocks, int blockIndex, @@ -1463,52 +1418,23 @@ public static partial class Gen5SpirvTranslator { error = string.Empty; var block = blocks[blockIndex]; - var halfMaskPlan = HalfMaskPlan(); - // One guest wave can span two host subgroups. Keep its shared-memory phases ordered. - // The half-mask plan inserts its own barriers; other wave64 programs also - // need LDS phase ordering when reads and writes span basic blocks. - var synchronizeSharedMemory = _emulateWave64 && halfMaskPlan is null; - var sharedMemoryPhase = SharedMemoryPhase.None; + _waveLowering.BeginBlock(); for (var index = block.StartIndex; index < block.EndIndex; index++) { var instruction = _request.Program.Instructions[index]; - if (halfMaskPlan is not null) - { - if (halfMaskPlan.ExactPairsBefore.TryGetValue(instruction.Pc, out var exactPairs)) - { - EmitExactWaveMasks(exactPairs); - } - - if (halfMaskPlan.SharedMemoryBarriersBefore.Contains(instruction.Pc)) - { - EmitWave64Barrier(); - } - } - + _waveLowering.BeforeInstruction(instruction); if (IsBranch(instruction.Opcode) || instruction.Opcode == "SEndpgm") { continue; } - if (synchronizeSharedMemory) - { - var nextPhase = instruction.Control is Gen5DataShareControl { Gds: false } - ? instruction.Opcode.StartsWith("DsRead", StringComparison.Ordinal) ? SharedMemoryPhase.Read - : instruction.Opcode.StartsWith("DsWrite", StringComparison.Ordinal) ? SharedMemoryPhase.Write : SharedMemoryPhase.None - : SharedMemoryPhase.None; - if (instruction.Opcode == "SBarrier") sharedMemoryPhase = SharedMemoryPhase.None; - if (nextPhase != SharedMemoryPhase.None) - { - if (sharedMemoryPhase != SharedMemoryPhase.None && sharedMemoryPhase != nextPhase) EmitWave64Barrier(); - sharedMemoryPhase = nextPhase; - } - } - _execKnownFull = IsExecKnownFull(instruction.Pc); - _emittingPc = instruction.Pc; - var emitted = TryEmitInstruction(instruction, out error); - _emittingPc = null; + _unobservedWrites = ObservationPlan()?.UnobservedWrites(instruction.Pc) ?? default; + var emitted = _pairedLanes && IsPairedLaneInstruction(instruction) + ? TryEmitPairedLaneInstruction(instruction, out error) + : TryEmitInstruction(instruction, out error); _execKnownFull = false; + _unobservedWrites = default; if (!emitted) { error = $"pc=0x{instruction.Pc:X} {instruction.Opcode}: {error}"; @@ -1521,7 +1447,7 @@ public static partial class Gen5SpirvTranslator CapturePixelExec(instruction); } - if (synchronizeSharedMemory && sharedMemoryPhase != SharedMemoryPhase.None) EmitWave64Barrier(); + _waveLowering.EndBlock(); return true; } @@ -1908,8 +1834,8 @@ public static partial class Gen5SpirvTranslator if (_maxDispatcherSteps > 0) { - var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1)); - Store(_iterationGuard, steps); + var steps = IAdd(Load(_uintType, IterationGuard()), UInt(1)); + Store(IterationGuard(), steps); var withinLimit = _module.AddInstruction(SpirvOp.ULessThan, _boolType, steps, UInt((uint)_maxDispatcherSteps)); again = _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, again, withinLimit); } @@ -2063,8 +1989,8 @@ public static partial class Gen5SpirvTranslator { // The dispatcher's safety valve, counted per loop iteration: a mistranslated // exit ends the invocation instead of wedging the GPU queue. - var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1)); - Store(_iterationGuard, steps); + var steps = IAdd(Load(_uintType, IterationGuard()), UInt(1)); + Store(IterationGuard(), steps); var withinLimit = _module.AddInstruction(SpirvOp.ULessThan, _boolType, steps, UInt((uint)_maxDispatcherSteps)); again = _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, again, withinLimit); } @@ -2079,7 +2005,7 @@ public static partial class Gen5SpirvTranslator // the block was skipped. Preserve that diagnostic safety budget. if (_maxDispatcherSteps > 0) { - Store(_iterationGuard, IAdd(Load(_uintType, _iterationGuard), UInt(1))); + Store(IterationGuard(), IAdd(Load(_uintType, IterationGuard()), UInt(1))); } _module.AddStatement(SpirvOp.Branch, entryMerge); _module.AddLabel(entryMerge); @@ -2583,6 +2509,19 @@ public static partial class Gen5SpirvTranslator sizeBytes); var pointer = LdsPointer(baseAddress, offset); var destination = instruction.Destinations[0].Value; + if (_pairedLanes) + { + EmitPairedWaveCounter(destination, inBounds, count => EmitAtomic( + instruction.Opcode == "DsAppend" ? SpirvOp.AtomicIAdd : SpirvOp.AtomicISub, + _uintType, + pointer, + scope: 2, + semantics: 0x108, + value: () => count, + comparator: () => UInt(0))); + return true; + } + var active = Load(_boolType, _exec); var activeMask = BooleanToWaveMask(active); @@ -2657,15 +2596,7 @@ public static partial class Gen5SpirvTranslator }); var firstValue = LoadV(destination); - uint broadcast; - if (_emulateWave64) - { - broadcast = ExchangeWave64Value(isFirstActive, firstValue); - } - else - { - broadcast = ShuffleLane(firstValue, firstLane); - } + var broadcast = BroadcastWaveValue(isFirstActive, firstValue, firstLane); var validResult = _module.AddInstruction( SpirvOp.LogicalAnd, @@ -5469,6 +5400,13 @@ public static partial class Gen5SpirvTranslator [.. operands]); sampled = UnpackImageTexel(resource, sampled); } + + RecordMipStatistics( + resource, + imageObject, + coordinates, + hasZeroLod || hasLod ? lod : null, + hasBias ? lodOrBias : Float(0)); } else if (instruction.Opcode.StartsWith( "ImageGather4", @@ -6347,7 +6285,10 @@ public static partial class Gen5SpirvTranslator return true; } - Store(_reachedPixelExport, _module.ConstantBool(true)); + if (_reachedPixelExport != 0) + { + Store(_reachedPixelExport, _module.ConstantBool(true)); + } var values = new uint[4]; for (var component = 0; component < 4; component++) @@ -7469,6 +7410,13 @@ public static partial class Gen5SpirvTranslator UInt(0), dwordAddress); + private uint DeclareFunctionUint(uint initializer) + { + var variable = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, initializer); + _interfaces.Add(variable); + return variable; + } + // Without subgroup operations each invocation is a one-lane wave, so a V_WRITELANE to // lane N > 0 used to be dropped and V_READLANE returned lane 0. Compilers spill SGPRs // (EXEC included) to VGPR lanes like this; restoring the wrong EXEC made waterfall @@ -7476,7 +7424,7 @@ public static partial class Gen5SpirvTranslator private List<(uint Register, uint Lane)> FindLaneSpillSlots() { var slots = new List<(uint Register, uint Lane)>(); - if (_emulateWave64) + if (_stage == Gen5SpirvStage.Compute && (_waveLowering is EmulatedWave64Lowering || _pairedLanes)) { return slots; } @@ -7610,7 +7558,7 @@ public static partial class Gen5SpirvTranslator throw new InvalidOperationException($"Vector register v{register} is outside the register file."); } - return RegisterVariable(_vectorRegisterVariables, register, "v"); + return RegisterVariable(CurrentVectorRegisterVariables(), register, CurrentVectorRegisterPrefix()); } // The V_MOVREL* opcodes address the VGPR file with a register number that @@ -7686,6 +7634,24 @@ public static partial class Gen5SpirvTranslator // Declared on first use when Private. AMD's compiler keeps an unused 4 KiB private // array as a named .bss global: two stages then fail to link, and the driver copies // the NOBITS section as file data and reads past the end of the ELF. + private uint IterationGuard() + { + if (_iterationGuard == 0) + { + _iterationGuard = _module.AddGlobalVariable( + _privateUintPointer, + SpirvStorageClass.Private, + _module.Constant(_uintType, 0)); + _interfaces.Add(_iterationGuard); + _module.AddName(_iterationGuard, "pcGuard"); + } + + return _iterationGuard; + } + + private static readonly bool TraceTitleShaderState = + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_TITLE_SHADER_STATE") == "1"; + private uint PackedHalfRegisters() { if (_packedHalfRegisters != 0) @@ -7703,7 +7669,10 @@ public static partial class Gen5SpirvTranslator return _packedHalfRegisters; } - private uint LoadS(uint register) => Load(_uintType, ScalarPointer(register)); + private uint LoadS(uint register) => + _scalarSnapshot is { } snapshot && snapshot.TryGetValue(register, out var value) + ? value + : Load(_uintType, ScalarPointer(register)); private uint LoadV(uint register) => Load(_uintType, VectorPointer(register)); @@ -7714,14 +7683,14 @@ public static partial class Gen5SpirvTranslator { if (_waveLaneCount != 32 || register == 106) { - Store(_vcc, IsLaneSetInMaskRegisters(106)); + UpdateLaneMaskFlag(106); } } else if (register is 126 or 127) { if (_waveLaneCount != 32 || register == 126) { - Store(_exec, IsLaneSetInMaskRegisters(126)); + UpdateLaneMaskFlag(126); } } } @@ -7730,15 +7699,21 @@ public static partial class Gen5SpirvTranslator // so the lane's bit is tested on one dword instead of a composed 64-bit mask. private uint IsLaneSetInMaskRegisters(uint lowRegister) { + var low = Load(_uintType, ScalarPointer(lowRegister)); if (_waveLaneCount != 32) { - return IsWaveMaskActive(LoadS64(lowRegister)); + var high = Load(_uintType, ScalarPointer(lowRegister + 1)); + return IsWaveMaskActive(BitwiseOr64( + _module.AddInstruction(SpirvOp.UConvert, _ulongType, low), + ShiftLeftLogical64( + _module.AddInstruction(SpirvOp.UConvert, _ulongType, high), + _module.Constant64(_ulongType, 32)))); } var laneBit = _subgroupInvocationIdInput == 0 ? UInt(1) : ShiftLeftLogical(UInt(1), GuestWaveLane()); - return IsNotZero(BitwiseAnd(LoadS(lowRegister), laneBit)); + return IsNotZero(BitwiseAnd(low, laneBit)); } // SHARPEMU_EXEC_GUARD_ELISION=0 guards every vector register write with EXEC again. @@ -7753,7 +7728,7 @@ public static partial class Gen5SpirvTranslator private bool IsExecKnownFull(uint pc) { - if (!ExecGuardElision || !_request.EnableExecGuardElision) + if (!ExecGuardElision || !_request.EnableExecGuardElision || _pairedLanes || _pixelExecExcludesHelpers) { return false; } @@ -7762,9 +7737,29 @@ public static partial class Gen5SpirvTranslator return _execFullPcs.Contains(pc); } + private static readonly bool ExecSelectElimination = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_EXEC_SELECT_ELIMINATION"), "0", StringComparison.Ordinal); + + private Ir.VectorRegisterSet _unobservedWrites; + private Ir.Gen5VectorObservationPlan? _observationPlan; + private bool _observationPlanBuilt; + + private Ir.Gen5VectorObservationPlan? ObservationPlan() + { + if (!_observationPlanBuilt) + { + _observationPlanBuilt = true; + _observationPlan = ExecSelectElimination && !_pixelExecExcludesHelpers + ? Ir.Gen5VectorObservationAnalysis.Analyze(_request.Program, wave32: _waveLaneCount == 32) + : null; + } + + return _observationPlan; + } + private void StoreV(uint register, uint value, bool guardWithExec = true) { - if (guardWithExec && !_execKnownFull) + if (guardWithExec && !_execKnownFull && !_unobservedWrites.Contains(register)) { var active = Load(_boolType, _exec); var oldValue = LoadV(register); @@ -7956,18 +7951,24 @@ public static partial class Gen5SpirvTranslator _module.AddInstruction(SpirvOp.LogicalNot, _boolType, value); private uint SubgroupAny(uint condition) => - _subgroupInvocationIdInput == 0 - ? condition - : _emulateWave64 - ? IsNotZero64(BooleanToWaveMask(condition)) - : _module.AddInstruction( - SpirvOp.GroupNonUniformAny, - _boolType, - UInt(3), - condition); + _waveLowering.SubgroupAny(condition); private uint GuestWaveLane() { + if (_pairedLanes) + { + return PairedGuestLane(); + } + + // A native wave64 is one host subgroup: its lane is the subgroup lane, the same one + // ballot and shuffle address, whatever order the driver gave the invocations. + if (_waveLaneCount == 64 && _guestLaneFromSubgroup && _subgroupInvocationIdInput != 0) + { + return BitwiseAnd( + Load(_uintType, _subgroupInvocationIdInput), + UInt(63)); + } + if (_waveLaneCount == 64 && _localInvocationIndexInput != 0) { return BitwiseAnd( @@ -8015,31 +8016,17 @@ public static partial class Gen5SpirvTranslator private uint ShuffleLane(uint value, uint lane) => _subgroupInvocationIdInput == 0 ? value - : _module.AddInstruction(SpirvOp.GroupNonUniformShuffle, _uintType, UInt(3), value, lane); - - private uint CurrentLaneBit() - { - if (_subgroupInvocationIdInput == 0) - { - return _module.Constant64(_ulongType, 1); - } - - var maskedLane = GuestWaveLane(); - var shifted = ShiftLeftLogical64( - _module.Constant64(_ulongType, 1), - _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - maskedLane)); - return _emulateWave64 - ? shifted : _module.AddInstruction( - SpirvOp.Select, - _ulongType, - IsCurrentLaneInRdnaWave(), - shifted, - _module.Constant64(_ulongType, 0)); - } + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + value, + _pairedLanes || _waveLowering is HalfWave64Lowering + ? BitwiseAnd(lane, UInt(PairedHalfLaneCount - 1)) + : lane); + + private uint CurrentLaneBit() => + _waveLowering.CurrentLaneBit(); private uint IsCurrentLaneInRdnaWave() => _module.AddInstruction( @@ -8056,193 +8043,11 @@ public static partial class Gen5SpirvTranslator CurrentLaneBit(), _module.Constant64(_ulongType, 0)); - private uint BooleanToWaveMask(uint condition) - { - if (_subgroupInvocationIdInput == 0) - { - return BooleanToLaneMask(condition); - } - - if (_emulateWave64) - { - var lane = GuestWaveLane(); - var owned = OwnHalfBallot(condition); - var exchange = BeginWave64Exchange(); - EmitConditional(IsHalfWaveLeader(lane), () => - Store(Wave64ExchangePointer(exchange, ShiftRightLogical(lane, UInt(5))), owned)); - EmitWave64Barrier(); - return Pair64( - Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))), - Load(_uintType, Wave64ExchangePointer(exchange, UInt(1)))); - } - - var ballot = _module.AddInstruction( - SpirvOp.GroupNonUniformBallot, - _uvec4Type, - UInt(3), - condition); - var low = _module.AddInstruction( - SpirvOp.CompositeExtract, - _uintType, - ballot, - 0); - var widened = _module.AddInstruction(SpirvOp.UConvert, _ulongType, low); - if (_waveLaneCount != 64) - { - return widened; - } - - return _module.AddInstruction( - SpirvOp.Select, - _ulongType, - _module.AddInstruction( - SpirvOp.UGreaterThanEqual, - _boolType, - GuestWaveLane(), - UInt(32)), - ShiftLeftLogical64( - widened, - _module.Constant64(_ulongType, 32)), - widened); - } - - private uint OwnHalfBallot(uint condition) - { - var ballot = _module.AddInstruction( - SpirvOp.GroupNonUniformBallot, - _uvec4Type, - UInt(3), - condition); - var component = ShiftRightLogical(Load(_uintType, _subgroupInvocationIdInput), UInt(5)); - // Avoid dynamic extraction of subgroup ballot words: on the NVIDIA - // device tests it produced zero masks. Select the same word explicitly. - var result = _module.AddInstruction(SpirvOp.CompositeExtract, _uintType, ballot, 0); - for (uint index = 1; index < 4; index++) - { - var word = _module.AddInstruction(SpirvOp.CompositeExtract, _uintType, ballot, index); - result = _module.AddInstruction(SpirvOp.Select, _uintType, - _module.AddInstruction(SpirvOp.IEqual, _boolType, component, UInt(index)), word, result); - } - return result; - } - - private uint BooleanToHalfWaveMask(uint condition) - { - var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition)); - return _module.AddInstruction( - SpirvOp.Select, - _ulongType, - _module.AddInstruction(SpirvOp.UGreaterThanEqual, _boolType, GuestWaveLane(), UInt(32)), - ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32)), - owned); - } - - private uint BothHalvesOfOwnBallot(uint condition) - { - var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition)); - return BitwiseOr64(owned, ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32))); - } - - private uint IsHalfWaveLeader(uint lane) => - _module.AddInstruction(SpirvOp.IEqual, _boolType, BitwiseAnd(lane, UInt(31)), UInt(0)); - - private uint BeginWave64Exchange() - { - var parity = Load(_uintType, _wave64ExchangeParity); - Store(_wave64ExchangeParity, BitwiseXor(parity, UInt(1))); - return IAdd( - UInt(_wave64ExchangeOffset), - _module.AddInstruction(SpirvOp.IMul, _uintType, parity, UInt(Wave64ExchangeSlotCount))); - } - - private uint Wave64ExchangePointer(uint exchange, uint slot) => - _module.AddInstruction( - SpirvOp.AccessChain, - _wave64ExchangeElementPointer, - _wave64Exchange, - IAdd(exchange, slot)); - - private uint ExchangeWave64Value(uint writer, uint value) - { - var exchange = BeginWave64Exchange(); - EmitConditional(writer, () => Store(Wave64ExchangePointer(exchange, UInt(0)), value)); - EmitWave64Barrier(); - return Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))); - } - - private void EmitExactWaveMasks(IReadOnlyList pairs) - { - var lane = GuestWaveLane(); - var upperHalf = ShiftRightLogical(lane, UInt(5)); - var lowerHalf = _module.AddInstruction(SpirvOp.IEqual, _boolType, upperHalf, UInt(0)); - const int pairsPerExchange = (int)Wave64ExchangeSlotCount / 2; - for (var start = 0; start < pairs.Count; start += pairsPerExchange) - { - var count = Math.Min(pairsPerExchange, pairs.Count - start); - var owned = new uint[count]; - for (var index = 0; index < count; index++) - { - var pair = pairs[start + index]; - owned[index] = _module.AddInstruction(SpirvOp.Select, _uintType, lowerHalf, LoadS(pair), LoadS(pair + 1)); - } - - var exchange = BeginWave64Exchange(); - EmitConditional(IsHalfWaveLeader(lane), () => - { - for (var index = 0; index < count; index++) - { - Store(Wave64ExchangePointer(exchange, IAdd(UInt((uint)index * 2), upperHalf)), owned[index]); - } - }); - EmitWave64Barrier(); - for (var index = 0; index < count; index++) - { - var pair = pairs[start + index]; - StoreS(pair, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2)))); - StoreS(pair + 1, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2 + 1)))); - } - } - } + private uint BooleanToWaveMask(uint condition) => + _waveLowering.BooleanToWaveMask(condition); private uint WaveMaskAny(uint register, uint laneActive) => - _emulateWave64 && _subgroupInvocationIdInput != 0 - ? IsNotZero64(LoadS64(register)) - : SubgroupAny(Load(_boolType, laneActive)); - - private static readonly bool HalfWaveMasks = !string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_HALF_MASKS"), "0", StringComparison.Ordinal); - - private bool _halfMaskPlanBuilt; - - private Ir.Gen5Wave64HalfMaskPlan? HalfMaskPlan() - { - if (_halfMaskPlanBuilt) - { - return _halfMaskPlan; - } - - _halfMaskPlanBuilt = true; - if (HalfWaveMasks && UsesWave64Exchange() && _subgroupInvocationIdInput != 0) - { - var sharedFlatPcs = _request.Memory.Entries - .Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) - .Select(static memory => memory.Pc) - .ToHashSet(); - _halfMaskPlan = Ir.Gen5Wave64HalfMaskAnalysis.Analyze(_request.Program, sharedFlatPcs); - } - - return _halfMaskPlan; - } - - private void EmitWave64Barrier() - { - var workgroup = UInt(2); - _module.AddStatement( - SpirvOp.ControlBarrier, - workgroup, - workgroup, - UInt(0x108)); - } + _waveLowering.WaveMaskAny(register, laneActive); // A wave-mask SGPR (VCC/EXEC) consumed as a per-lane predicate — the // condition of VCndmask, a VCC/EXEC branch, or the derived _vcc/_exec @@ -8269,27 +8074,8 @@ public static partial class Gen5SpirvTranslator // In wave32 a lane mask fills its register alone and the next one keeps its value. // That includes VCC and EXEC: compilers use VCC_HI (s107) as an ordinary SGPR. - private void StoreWaveMask(uint register, uint condition) - { - if (_waveLaneCount == 32) - { - StoreS(register, Narrow(BooleanToWaveMask(condition))); - return; - } - - if (_emulateWave64 && - _subgroupInvocationIdInput != 0 && - _emittingPc is { } pc && - HalfMaskPlan() is { } plan && - plan.HalfMaskWrites.TryGetValue(pc, out var halfRegister) && - halfRegister == register) - { - StoreS64(register, BooleanToHalfWaveMask(condition)); - return; - } - - StoreS64(register, BooleanToWaveMask(condition)); - } + private void StoreWaveMask(uint register, uint condition) => + _waveLowering.StoreWaveMask(register, condition); private void EmitExecConditional(Action emit) { @@ -8360,6 +8146,7 @@ public static partial class Gen5SpirvTranslator instruction.Destinations.Any(IsWaveMaskOperand)); private bool UsesSubgroupOperations() => + _pairedLanes || _enableGraphicsSubgroupOperations && (UsesSubgroupShuffle() || UsesSubgroupBroadcast() || diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs index 0768ec7..4c56d88 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs @@ -176,10 +176,12 @@ public enum SpirvOp : ushort GroupNonUniformBroadcast = 337, GroupNonUniformBroadcastFirst = 338, GroupNonUniformBallot = 339, + GroupNonUniformBallotBitCount = 342, GroupNonUniformShuffle = 345, GroupNonUniformShuffleXor = 346, GroupNonUniformShuffleUp = 347, GroupNonUniformShuffleDown = 348, + GroupNonUniformUMin = 354, } public enum SpirvCapability : uint @@ -206,6 +208,7 @@ public enum SpirvCapability : uint StorageImageWriteWithoutFormat = 56, GroupNonUniform = 61, GroupNonUniformVote = 62, + GroupNonUniformArithmetic = 63, GroupNonUniformBallot = 64, GroupNonUniformShuffle = 65, ShaderLayer = 5253, @@ -276,11 +279,13 @@ public enum SpirvBuiltIn : uint BaryCoordKhr = 5286, BaryCoordNoPerspKhr = 5287, FrontFacing = 17, + HelperInvocation = 23, WorkgroupId = 26, LocalInvocationId = 27, GlobalInvocationId = 28, LocalInvocationIndex = 29, SubgroupSize = 36, + SubgroupId = 40, SubgroupLocalInvocationId = 41, SampleId = 18, SampleMask = 20, diff --git a/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs b/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs index fc2260b..1605aa7 100644 --- a/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs +++ b/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs @@ -9,8 +9,18 @@ public static class Gen5PixelInputMapping ReadOnlySpan controls, ReadOnlySpan activeInputs) { - Span usedLocations = stackalloc bool[32]; var locations = new uint[activeInputs.Length]; + ResolveLocations(controls, activeInputs, locations); + return locations; + } + + // Fills `locations` (one slot per active input) without allocating. + public static void ResolveLocations( + ReadOnlySpan controls, + ReadOnlySpan activeInputs, + Span locations) + { + Span usedLocations = stackalloc bool[32]; for (var index = 0; index < activeInputs.Length; index++) { @@ -39,7 +49,5 @@ public static class Gen5PixelInputMapping usedLocations[(int)location] = true; } } - - return locations; } } diff --git a/src/SharpEmu.ShaderCompiler/Ir/Gen5VectorObservationAnalysis.cs b/src/SharpEmu.ShaderCompiler/Ir/Gen5VectorObservationAnalysis.cs new file mode 100644 index 0000000..37eba4e --- /dev/null +++ b/src/SharpEmu.ShaderCompiler/Ir/Gen5VectorObservationAnalysis.cs @@ -0,0 +1,454 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Numerics; + +namespace SharpEmu.ShaderCompiler.Ir; + +public readonly record struct VectorRegisterSet(UInt128 Low, UInt128 High) +{ + public bool IsEmpty => Low == UInt128.Zero && High == UInt128.Zero; + + public bool Contains(uint register) => + register < 128 + ? ((Low >> (int)register) & UInt128.One) != 0 + : register < 256 && ((High >> (int)(register - 128)) & UInt128.One) != 0; + + public VectorRegisterSet With(uint register) => + register < 128 ? this with { Low = Low | (UInt128.One << (int)register) } + : register < 256 ? this with { High = High | (UInt128.One << (int)(register - 128)) } + : this; + + public VectorRegisterSet Union(VectorRegisterSet other) => new(Low | other.Low, High | other.High); + + public VectorRegisterSet Except(VectorRegisterSet other) => new(Low & ~other.Low, High & ~other.High); + + public int Count => PopCount(Low) + PopCount(High); + + private static int PopCount(UInt128 value) => + BitOperations.PopCount((ulong)value) + BitOperations.PopCount((ulong)(value >> 64)); +} + +public sealed class Gen5VectorObservationPlan(IReadOnlyDictionary unobserved) +{ + public VectorRegisterSet UnobservedWrites(uint pc) => unobserved.TryGetValue(pc, out var set) ? set : default; + + public int WriteCount => unobserved.Values.Sum(static set => set.Count); +} + +public static class Gen5VectorObservationAnalysis +{ + private const uint ExecLow = 126; + private const uint ExecHigh = 127; + + private enum ExecEffect : byte + { + None, + Narrowing, + Widening, + } + + private readonly record struct Effect( + VectorRegisterSet Reads, + VectorRegisterSet CrossReads, + VectorRegisterSet Writes, + VectorRegisterSet FullWrites, + ExecEffect Exec); + + public static Gen5VectorObservationPlan? Analyze(Gen5ShaderProgram program, bool wave32) + { + var instructions = program.Instructions; + if (instructions.Count == 0) + { + return null; + } + + var effects = new Effect[instructions.Count]; + for (var index = 0; index < instructions.Count; index++) + { + if (!TryDescribe(instructions[index], out effects[index])) + { + return null; + } + } + + if (BuildSuccessors(instructions) is not { } successors) + { + return null; + } + + var execFull = Gen5ExecFullAnalysis.Analyze(program, wave32); + var predecessors = new List[instructions.Count]; + for (var index = 0; index < instructions.Count; index++) + { + predecessors[index] = []; + } + + for (var index = 0; index < instructions.Count; index++) + { + foreach (var successor in successors[index]) + { + predecessors[successor].Add(index); + } + } + + var liveIn = new VectorRegisterSet[instructions.Count]; + var observedIn = new VectorRegisterSet[instructions.Count]; + var queued = new bool[instructions.Count]; + var work = new Queue(); + for (var index = instructions.Count - 1; index >= 0; index--) + { + work.Enqueue(index); + queued[index] = true; + } + + while (work.TryDequeue(out var index)) + { + queued[index] = false; + var (liveOut, observedOut) = Out(index, successors, liveIn, observedIn); + var (live, observed) = Transfer(effects[index], execFull.Contains(instructions[index].Pc), liveOut, observedOut); + if (live == liveIn[index] && observed == observedIn[index]) + { + continue; + } + + liveIn[index] = live; + observedIn[index] = observed; + foreach (var predecessor in predecessors[index]) + { + if (!queued[predecessor]) + { + queued[predecessor] = true; + work.Enqueue(predecessor); + } + } + } + + var unobserved = new Dictionary(); + for (var index = 0; index < instructions.Count; index++) + { + var writes = effects[index].Writes; + if (writes.IsEmpty) + { + continue; + } + + var (_, observedOut) = Out(index, successors, liveIn, observedIn); + var dead = execFull.Contains(instructions[index].Pc) ? writes : writes.Except(observedOut); + if (!dead.IsEmpty) + { + unobserved[instructions[index].Pc] = dead; + } + } + + return new Gen5VectorObservationPlan(unobserved); + } + + private static (VectorRegisterSet Live, VectorRegisterSet Observed) Out( + int index, int[][] successors, VectorRegisterSet[] liveIn, VectorRegisterSet[] observedIn) + { + var live = default(VectorRegisterSet); + var observed = default(VectorRegisterSet); + foreach (var successor in successors[index]) + { + live = live.Union(liveIn[successor]); + observed = observed.Union(observedIn[successor]); + } + + return (live, observed); + } + + private static (VectorRegisterSet Live, VectorRegisterSet Observed) Transfer( + in Effect effect, bool execFull, VectorRegisterSet liveOut, VectorRegisterSet observedOut) + { + var observed = effect.Exec == ExecEffect.Widening ? observedOut.Union(liveOut) : observedOut; + var kills = execFull ? effect.FullWrites : default; + var live = liveOut.Except(kills).Union(effect.Reads).Union(effect.CrossReads); + observed = observed.Except(kills).Union(effect.CrossReads); + return (live, observed); + } + + private static bool TryDescribe(Gen5ShaderInstruction instruction, out Effect effect) + { + effect = default; + var opcode = instruction.Opcode; + if (opcode.StartsWith("VMovrel", StringComparison.Ordinal) || + opcode.Contains("Raw", StringComparison.Ordinal) || + opcode is "SSetpcB64" or "SSwappcB64" or "SRfeB64" or "SCbranchJoin" or "SCbranchGFork" or "SCbranchIFork") + { + return false; + } + + var reads = default(VectorRegisterSet); + var cross = default(VectorRegisterSet); + var writes = default(VectorRegisterSet); + var full = default(VectorRegisterSet); + var wideName = opcode.Contains("64", StringComparison.Ordinal); + + foreach (var source in instruction.Sources) + { + if (source.Kind != Gen5OperandKind.VectorRegister) + { + continue; + } + + reads = reads.With(source.Value); + if (wideName) + { + reads = reads.With(source.Value + 1); + } + } + + switch (instruction.Encoding) + { + case Gen5ShaderEncoding.Vop1: + case Gen5ShaderEncoding.Vop2: + case Gen5ShaderEncoding.Vop3: + case Gen5ShaderEncoding.Vop3p: + case Gen5ShaderEncoding.Vopc: + case Gen5ShaderEncoding.Vintrp: + DescribeVector(instruction, wideName, ref reads, ref cross, ref writes, ref full); + break; + case Gen5ShaderEncoding.Mubuf: + case Gen5ShaderEncoding.Mtbuf: + DescribeBuffer(instruction, ref reads, ref writes); + break; + case Gen5ShaderEncoding.Flat: + foreach (var destination in instruction.Destinations) + { + if (destination.Kind == Gen5OperandKind.VectorRegister) + { + writes = writes.With(destination.Value); + reads = reads.With(destination.Value); + } + } + + break; + case Gen5ShaderEncoding.Mimg: + cross = cross.Union(reads); + DescribeImage(instruction, ref reads, ref writes); + break; + case Gen5ShaderEncoding.Ds: + if (opcode is "DsSwizzleB32" or "DsBpermuteB32" or "DsPermuteB32") + { + cross = cross.Union(reads); + } + + foreach (var destination in instruction.Destinations) + { + if (destination.Kind == Gen5OperandKind.VectorRegister) + { + writes = writes.With(destination.Value); + reads = reads.With(destination.Value); + } + } + + break; + } + + effect = new Effect(reads, cross, writes, full, DescribeExec(instruction)); + return true; + } + + private static void DescribeVector(Gen5ShaderInstruction instruction, bool wideName, ref VectorRegisterSet reads, + ref VectorRegisterSet cross, ref VectorRegisterSet writes, ref VectorRegisterSet full) + { + var opcode = instruction.Opcode; + if (instruction.Control is Gen5DppControl or Gen5Dpp8Control || + opcode is "VReadlaneB32" or "VReadfirstlaneB32" or "VPermlane16B32" or "VPermlanex16B32") + { + if (instruction.Sources.Count > 0 && instruction.Sources[0].Kind == Gen5OperandKind.VectorRegister) + { + cross = cross.With(instruction.Sources[0].Value); + if (wideName) + { + cross = cross.With(instruction.Sources[0].Value + 1); + } + } + } + + if (instruction.Destinations.Count == 0 || instruction.Destinations[0].Kind != Gen5OperandKind.VectorRegister) + { + return; + } + + var destination = instruction.Destinations[0].Value; + writes = writes.With(destination); + if (wideName) + { + writes = writes.With(destination + 1); + } + + var partial = + opcode.Contains("Mac", StringComparison.Ordinal) || + opcode.StartsWith("VDot", StringComparison.Ordinal) || + opcode is "VWritelaneB32" || + opcode.Contains("16", StringComparison.Ordinal) || + opcode.Contains("Pk", StringComparison.Ordinal) || + instruction.Encoding is Gen5ShaderEncoding.Vintrp || + instruction.Control is Gen5SdwaControl { DestinationSelect: not 6 } or Gen5SdwaControl { DestinationUnused: not 0 } || + instruction.Control is Gen5Vop3Control { OperandSelect: not 0 }; + if (partial) + { + reads = reads.With(destination); + if (wideName) + { + reads = reads.With(destination + 1); + } + + return; + } + + full = full.With(destination); + } + + private static void DescribeBuffer(Gen5ShaderInstruction instruction, ref VectorRegisterSet reads, ref VectorRegisterSet writes) + { + var opcode = instruction.Opcode; + if (instruction.Control is Gen5BufferMemoryControl buffer) + { + reads = reads.With(buffer.VectorAddress).With(buffer.VectorAddress + 1); + } + + var load = opcode.Contains("Load", StringComparison.Ordinal); + foreach (var destination in instruction.Destinations) + { + if (destination.Kind != Gen5OperandKind.VectorRegister) + { + continue; + } + + reads = reads.With(destination.Value); + if (load || opcode.Contains("Atomic", StringComparison.Ordinal)) + { + writes = writes.With(destination.Value); + } + } + } + + private static void DescribeImage(Gen5ShaderInstruction instruction, ref VectorRegisterSet reads, ref VectorRegisterSet writes) + { + var (data, count) = instruction.Control switch + { + Gen5ImageControl image => (image.VectorData, 5u), + Gen5RayIntersectControl ray => (ray.VectorData, Gen5RayIntersectControl.ResultDwords), + _ => (0u, 0u), + }; + var store = instruction.Opcode.StartsWith("ImageStore", StringComparison.Ordinal); + for (var offset = 0u; offset < count; offset++) + { + reads = reads.With(data + offset); + if (!store) + { + writes = writes.With(data + offset); + } + } + } + + private static ExecEffect DescribeExec(Gen5ShaderInstruction instruction) + { + var opcode = instruction.Opcode; + var scalarDestination = instruction.Control switch + { + Gen5Vop3Control control => control.ScalarDestination, + Gen5SdwaControl control => control.ScalarDestination, + _ => null, + }; + if (opcode.StartsWith("VCmpx", StringComparison.Ordinal)) + { + return ExecEffect.Narrowing; + } + + if (opcode.StartsWith("VCmp", StringComparison.Ordinal)) + { + return scalarDestination is ExecLow ? ExecEffect.Narrowing : ExecEffect.None; + } + + var writesExec = + scalarDestination is ExecLow or ExecHigh || + opcode.Contains("Saveexec", StringComparison.Ordinal) || + opcode.Contains("Wrexec", StringComparison.Ordinal) || + instruction.Destinations.Any(static destination => + destination is { Kind: Gen5OperandKind.ScalarRegister, Value: ExecLow or ExecHigh }); + if (!writesExec) + { + return ExecEffect.None; + } + + if (opcode is "SAndSaveexecB64" or "SAndSaveexecB32" or "SAndn1SaveexecB64" or "SAndn1SaveexecB32") + { + return ExecEffect.Narrowing; + } + + var toExec = instruction.Destinations is [{ Kind: Gen5OperandKind.ScalarRegister, Value: ExecLow }]; + var sources = instruction.Sources; + if (toExec && opcode is "SAndB64" or "SAndB32" && sources.Count == 2 && (IsExec(sources[0]) || IsExec(sources[1]))) + { + return ExecEffect.Narrowing; + } + + if (toExec && opcode is "SAndn2B64" or "SAndn2B32" && sources.Count == 2 && IsExec(sources[0])) + { + return ExecEffect.Narrowing; + } + + return ExecEffect.Widening; + } + + private static bool IsExec(Gen5Operand operand) => operand is { Kind: Gen5OperandKind.ScalarRegister, Value: ExecLow }; + + private static int[][]? BuildSuccessors(IReadOnlyList instructions) + { + var indexByPc = new Dictionary(instructions.Count); + for (var index = 0; index < instructions.Count; index++) + { + indexByPc[instructions[index].Pc] = index; + } + + var last = instructions[^1]; + var endPc = last.Pc + (uint)(last.Words.Count * sizeof(uint)); + var successors = new int[instructions.Count][]; + var resolver = Gen5IrBranchResolver.Instance; + for (var index = 0; index < instructions.Count; index++) + { + var instruction = instructions[index]; + var next = index + 1 < instructions.Count ? index + 1 : -1; + if (Gen5IrBranchResolver.IsTerminator(instruction)) + { + successors[index] = []; + continue; + } + + var unconditional = Gen5IrBranchResolver.IsUnconditionalBranch(instruction); + if (!unconditional && !resolver.IsConditional(instruction)) + { + successors[index] = next < 0 ? [] : [next]; + continue; + } + + if (!resolver.TryGetBranchTarget(instruction, out var targetPc)) + { + return null; + } + + var targets = new List(2); + if (indexByPc.TryGetValue(targetPc, out var target)) + { + targets.Add(target); + } + else if (targetPc < endPc) + { + return null; + } + + if (!unconditional && next >= 0 && !targets.Contains(next)) + { + targets.Add(next); + } + + successors[index] = [.. targets]; + } + + return successors; + } +} diff --git a/src/SharpEmu.ShaderCompiler/Ir/Gen5WaveLaneLocalAnalysis.cs b/src/SharpEmu.ShaderCompiler/Ir/Gen5WaveLaneLocalAnalysis.cs new file mode 100644 index 0000000..c44e27a --- /dev/null +++ b/src/SharpEmu.ShaderCompiler/Ir/Gen5WaveLaneLocalAnalysis.cs @@ -0,0 +1,2244 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Numerics; + +namespace SharpEmu.ShaderCompiler.Ir; + +public static class Gen5WaveLaneLocalAnalysis +{ + private const int VccLo = 106; + private const int VccHi = 107; + private const int M0 = 124; + private const int Scc = 125; + private const int ExecLo = 126; + private const int ExecHi = 127; + private const int ScalarCount = 128; + private const int VectorCount = 256; + private const int MaxSteps = 100_000; + private const int NoBlock = -1; + + private enum Family + { + Sop1, + Sop2, + Sopk, + Sopc, + Sopp, + Smem, + Vop1, + Vop2, + Vop3, + Vop3p, + Vopc, + Mubuf, + Mtbuf, + Flat, + Mimg, + Ds, + Other, + } + + private enum Kind : byte + { + None, + Scalar, + Vector, + Zero, + AllOnes, + Constant, + Scc, + VccZ, + ExecZ, + } + + private enum Condition : byte + { + None, + ExecZero, + ExecNonZero, + VccZero, + VccNonZero, + Other, + } + + private enum Sym : byte + { + Original, + Zero, + Ones, + Unknown, + } + + private readonly record struct Operand(Kind Kind, int Register = -1) + { + public static Operand None => new(Kind.None); + + public static Operand Scalar(int register) => new(Kind.Scalar, register); + + public int ScalarIndex => Kind == Kind.Scalar ? Register : -1; + } + + private readonly record struct MaskFacts(bool Covers, bool Within, bool Nonzero); + + private sealed class Op + { + public required uint Pc { get; init; } + + public required string Opcode { get; init; } + + public required Family Family { get; init; } + + public required Operand[] Sources { get; init; } + + public required int[] Widths { get; init; } + + public Operand Destination { get; init; } + + public int DestinationCount { get; init; } = 1; + + public Operand Carry { get; init; } + + public int Select { get; init; } = -1; + + public bool LaneFetch { get; init; } + + public bool CrossRowFetch { get; init; } + + public bool PartialVector { get; init; } + + public int WrittenVectorStart { get; init; } = -1; + + public int WrittenVectorCount { get; init; } + + public bool Wide { get; init; } + + public string? Refused { get; init; } + + public bool IsBranch { get; init; } + + public bool IsConditional { get; init; } + + public bool IsTerminator { get; init; } + + public Condition Condition { get; init; } + + public Operand Source(int index) => index < Sources.Length ? Sources[index] : Operand.None; + } + + private struct State : IEquatable + { + public UInt128 Mask; + public UInt128 Covers; + public UInt128 Within; + public UInt128 Nonzero; + public UInt128 Lanes; + public UInt128 UniformLow; + public UInt128 UniformHigh; + public bool SccMask; + public bool Active; + + public readonly bool Uniform(int register) => + register < 128 + ? ((UniformLow >> register) & UInt128.One) != 0 + : register < VectorCount && ((UniformHigh >> (register - 128)) & UInt128.One) != 0; + + public void SetUniform(int register, bool value) + { + if (register < 128) + { + UniformLow = value ? UniformLow | (UInt128.One << register) : UniformLow & ~(UInt128.One << register); + } + else if (register < VectorCount) + { + var bit = UInt128.One << (register - 128); + UniformHigh = value ? UniformHigh | bit : UniformHigh & ~bit; + } + } + + public void Join(in State other) + { + Mask |= other.Mask; + Covers &= other.Covers; + Within &= other.Within; + Nonzero &= other.Nonzero; + Lanes &= other.Lanes; + UniformLow &= other.UniformLow; + UniformHigh &= other.UniformHigh; + SccMask |= other.SccMask; + Active &= other.Active; + } + + public readonly bool Equals(State other) => + Mask == other.Mask && + Covers == other.Covers && + Within == other.Within && + Nonzero == other.Nonzero && + Lanes == other.Lanes && + UniformLow == other.UniformLow && + UniformHigh == other.UniformHigh && + SccMask == other.SccMask && + Active == other.Active; + + public override readonly bool Equals(object? obj) => obj is State other && Equals(other); + + public override readonly int GetHashCode() => HashCode.Combine(Mask, Covers, Within, Lanes, UniformLow, UniformHigh); + } + + private sealed class Block + { + public required int Start { get; init; } + + public required int End { get; init; } + + public List Successors { get; } = []; + + public Condition Condition { get; set; } + + public int TrueBlock { get; set; } = NoBlock; + + public int FalseBlock { get; set; } = NoBlock; + } + + private readonly record struct Loop(int Header, int Latch, bool[] Body); + + private static readonly HashSet ScalarDestination64Opcodes = new(StringComparer.Ordinal) + { + "SMovB64", "SNotB64", "SWqmB64", "SQuadmaskB64", "SBitreplicateB64B32", "SGetpcB64", + "SAndB64", "SAndn2B64", "SOrB64", "SOrn2B64", "SXorB64", "SNandB64", "SNorB64", "SXnorB64", + "SLshlB64", "SLshrB64", "SAshrI64", "SBfeU64", "SBfeI64", "SBfmB64", "SCselectB64", + "SBitset0B64", "SBitset1B64", + }; + + private static readonly HashSet PartialWriteOpcodes = new(StringComparer.Ordinal) + { + "SBitset0B32", "SBitset1B32", "SBitset0B64", "SBitset1B64", + }; + + private static readonly string[] SccWriterPrefixes = + [ + "SCmp", "SBitcmp", "SAdd", "SSub", "SMin", "SMax", "SAbs", "SAnd", "SOr", "SXor", "SNand", "SNor", + "SXnor", "SNot", "SWqm", "SLshl", "SLshr", "SAshr", "SBfe", "SBcnt", + ]; + + private static readonly HashSet SccKeepers = new(StringComparer.Ordinal) + { + "SMovB32", "SMovB64", "SMovkI32", "SCselectB32", "SCselectB64", "SBrevB32", "SFF1I32B32", "SFF1I32B64", + "SFlbitI32B32", "SFlbitI32B64", "SBitset0B32", "SBitset1B32", "SBitset0B64", "SBitset1B64", + "SMulI32", "SMulHiU32", "SMulHiI32", "SMulkI32", "SBfmB32", "SBfmB64", + "SPackLlB32B16", "SPackLhB32B16", "SPackHhB32B16", "SGetpcB64", + }; + + private static readonly HashSet SccReaders = new(StringComparer.Ordinal) + { + "SCselectB32", "SCselectB64", "SAddcU32", "SSubbU32", "SCbranchScc0", "SCbranchScc1", + }; + + private static readonly HashSet MaskOperations = new(StringComparer.Ordinal) + { + "SMovB32", "SMovB64", "SNotB32", "SNotB64", "SWqmB32", "SWqmB64", + "SAndB32", "SAndB64", "SAndn2B32", "SAndn2B64", "SOrB32", "SOrB64", "SOrn2B32", "SOrn2B64", + "SXorB32", "SXorB64", "SNandB32", "SNandB64", "SNorB32", "SNorB64", "SXnorB32", "SXnorB64", + "SCselectB32", "SCselectB64", + }; + + private static readonly HashSet LaneSelectOpcodes = new(StringComparer.Ordinal) + { + "VCndmaskB32", "VAddcU32", "VAddCoCiU32", "VSubbU32", "VSubCoCiU32", "VSubbrevU32", "VSubrevCoCiU32", + }; + + private static readonly HashSet CarryOutOpcodes = new(StringComparer.Ordinal) + { + "VAddcU32", "VAddCoCiU32", "VSubbU32", "VSubCoCiU32", "VSubbrevU32", "VSubrevCoCiU32", + "VAddCoU32", "VSubCoU32", "VSubrevCoU32", + }; + + public static bool IsLaneLocal(Gen5ShaderProgram program, IReadOnlySet sharedFlatPcs, out string reason) + { + var instructions = program.Instructions; + if (instructions.Count == 0) + { + reason = "control flow"; + return false; + } + + var ops = new Op[instructions.Count]; + for (var index = 0; index < ops.Length; index++) + { + ops[index] = Normalize(instructions[index], sharedFlatPcs); + } + + if (!TryBuildBlocks(instructions, ops, out var blocks)) + { + reason = "control flow"; + return false; + } + + return new Analysis(ops, blocks).Run(out reason); + } + + private static Op Normalize(Gen5ShaderInstruction instruction, IReadOnlySet sharedFlatPcs) + { + var opcode = instruction.Opcode; + var family = instruction.Encoding switch + { + Gen5ShaderEncoding.Sop1 => Family.Sop1, + Gen5ShaderEncoding.Sop2 => Family.Sop2, + Gen5ShaderEncoding.Sopk => Family.Sopk, + Gen5ShaderEncoding.Sopc => Family.Sopc, + Gen5ShaderEncoding.Sopp => Family.Sopp, + Gen5ShaderEncoding.Smem or Gen5ShaderEncoding.Smrd => Family.Smem, + Gen5ShaderEncoding.Vop1 => Family.Vop1, + Gen5ShaderEncoding.Vop2 => Family.Vop2, + Gen5ShaderEncoding.Vop3 => Family.Vop3, + Gen5ShaderEncoding.Vop3p => Family.Vop3p, + Gen5ShaderEncoding.Vopc => Family.Vopc, + Gen5ShaderEncoding.Mubuf => Family.Mubuf, + Gen5ShaderEncoding.Mtbuf => Family.Mtbuf, + Gen5ShaderEncoding.Flat => Family.Flat, + Gen5ShaderEncoding.Mimg => Family.Mimg, + Gen5ShaderEncoding.Ds => Family.Ds, + _ => Family.Other, + }; + var wide = opcode.Contains("64", StringComparison.Ordinal); + var mapped = instruction.Sources.Select(Map).ToList(); + var destination = instruction.Destinations.Count > 0 ? Map(instruction.Destinations[0]) : Operand.None; + var destinationCount = 1; + var carry = Operand.None; + var select = -1; + var writtenStart = -1; + var writtenCount = 0; + var refused = RefusedOpcode(instruction, family, sharedFlatPcs); + List? widths = null; + + switch (family) + { + case Family.Sop1: + case Family.Sop2: + if (PartialWriteOpcodes.Contains(opcode) && destination.Kind == Kind.Scalar) + { + mapped.Add(destination); + } + + break; + case Family.Sopc: + destination = Operand.None; + break; + case Family.Sopk: + { + var register = destination; + if (opcode == "SSetregB32") + { + destination = Operand.None; + mapped = [instruction.Sources.Count > 0 ? Map(instruction.Sources[0]) : Operand.None]; + } + else if (opcode.StartsWith("SWaitcnt", StringComparison.Ordinal)) + { + destination = Operand.None; + mapped = instruction.Sources.Skip(1).Select(Map).ToList(); + } + else if (opcode.StartsWith("SCmpk", StringComparison.Ordinal)) + { + destination = Operand.None; + mapped = [register, new Operand(Kind.Constant)]; + } + else if (opcode == "SMovkI32") + { + mapped = [new Operand(Kind.Constant)]; + } + else + { + mapped = [register, new Operand(Kind.Constant)]; + } + + break; + } + case Family.Sopp: + destination = Operand.None; + mapped = []; + break; + case Family.Smem: + if (instruction.Control is Gen5GlobalMemoryControl { SourceIsScalar: true }) + { + destination = Operand.None; + widths = mapped.Select((_, index) => index == 0 ? 2 : 1).ToList(); + } + else + { + destinationCount = Math.Max(instruction.Destinations.Count, 1); + var baseWidth = opcode.StartsWith("SBuffer", StringComparison.Ordinal) ? 4 : 2; + widths = mapped.Select((_, index) => index == 0 ? baseWidth : 1).ToList(); + } + + break; + case Family.Vop1: + break; + case Family.Vop2: + if (LaneSelectOpcodes.Contains(opcode)) + { + select = 2; + if (mapped.Count < 3) + { + mapped.Add(Operand.Scalar(VccLo)); + } + } + + if (CarryOutOpcodes.Contains(opcode)) + { + carry = Operand.Scalar(VccLo); + } + + if (Accumulates(opcode) && destination.Kind == Kind.Vector) + { + mapped.Add(destination); + } + + break; + case Family.Vopc: + mapped = mapped.Take(2).ToList(); + destination = CompareDestination(instruction); + break; + case Family.Vop3: + if (opcode.StartsWith("VCmp", StringComparison.Ordinal)) + { + mapped = mapped.Take(2).ToList(); + destination = CompareDestination(instruction); + break; + } + + mapped = UsedSources(instruction, Vop3SourceCount(instruction)); + if (LaneSelectOpcodes.Contains(opcode)) + { + select = 2; + } + + if (instruction.Control is Gen5Vop3Control { ScalarDestination: { } scalarDestination }) + { + carry = Map(Gen5Operand.Scalar(scalarDestination)); + } + + if (Accumulates(opcode) && destination.Kind == Kind.Vector) + { + mapped.Add(destination); + } + + break; + case Family.Vop3p: + mapped = UsedSources(instruction, Vop3pSourceCount(instruction)); + if (Accumulates(opcode) && destination.Kind == Kind.Vector) + { + mapped.Add(destination); + } + + break; + case Family.Mubuf: + case Family.Mtbuf: + widths = mapped.Select((_, index) => index == 1 ? 4 : 1).ToList(); + if (instruction.Control is Gen5BufferMemoryControl buffer && + !opcode.Contains("Store", StringComparison.Ordinal)) + { + writtenStart = (int)buffer.VectorData; + writtenCount = (int)Math.Max(buffer.DwordCount, 1); + } + + destination = Operand.None; + break; + case Family.Flat: + widths = mapped.Select(_ => 2).ToList(); + if (instruction.Destinations.Count > 0) + { + writtenStart = (int)instruction.Destinations[0].Value; + writtenCount = instruction.Destinations.Count; + } + + destination = Operand.None; + break; + case Family.Mimg: + widths = mapped.Select(_ => 1).ToList(); + if (instruction.Control is Gen5RayIntersectControl ray) + { + widths[^1] = 4; + writtenStart = (int)ray.VectorData; + writtenCount = (int)Gen5RayIntersectControl.ResultDwords; + } + else if (instruction.Control is Gen5ImageControl image && widths.Count >= 2) + { + widths[^2] = 8; + widths[^1] = UsesSampler(opcode) ? 4 : 0; + if (!opcode.StartsWith("ImageStore", StringComparison.Ordinal)) + { + var channels = opcode.StartsWith("ImageGather", StringComparison.Ordinal) + ? 4 + : Math.Max(BitOperations.PopCount(image.Dmask), 1); + writtenStart = (int)image.VectorData; + writtenCount = channels + 1; + } + } + + destination = Operand.None; + break; + default: + if (destination.Kind == Kind.Vector) + { + writtenStart = destination.Register; + writtenCount = instruction.Destinations.Count; + } + + destination = Operand.None; + break; + } + + widths ??= mapped + .Select((_, index) => index == select ? 2 : SourceWidth(opcode, family, index, wide)) + .ToList(); + var sdwaPartial = instruction.Control is Gen5SdwaControl { DestinationSelect: not 6 } && + family is Family.Vop1 or Family.Vop2; + var resolver = Gen5IrBranchResolver.Instance; + return new Op + { + Pc = instruction.Pc, + Opcode = opcode, + Family = family, + Sources = [.. mapped], + Widths = [.. widths], + Destination = destination, + DestinationCount = destinationCount, + Carry = carry, + Select = select, + LaneFetch = instruction.Control is Gen5DppControl or Gen5Dpp8Control, + CrossRowFetch = instruction.Control is Gen5DppControl dpp && !LaneLocalDpp(dpp.Control), + PartialVector = sdwaPartial || + (opcode.Contains("16", StringComparison.Ordinal) && !opcode.StartsWith("VPk", StringComparison.Ordinal)), + WrittenVectorStart = writtenStart, + WrittenVectorCount = writtenCount, + Wide = wide, + Refused = refused, + IsBranch = resolver.IsBranch(instruction), + IsConditional = resolver.IsConditional(instruction), + IsTerminator = Gen5IrBranchResolver.IsTerminator(instruction), + Condition = opcode switch + { + "SCbranchExecz" => Condition.ExecZero, + "SCbranchExecnz" => Condition.ExecNonZero, + "SCbranchVccz" => Condition.VccZero, + "SCbranchVccnz" => Condition.VccNonZero, + _ when resolver.IsConditional(instruction) => Condition.Other, + _ => Condition.None, + }, + }; + } + + private static int SourceWidth(string opcode, Family family, int index, bool wide) + { + if (family is not (Family.Vop1 or Family.Vop2 or Family.Vop3 or Family.Vopc)) + { + return wide ? 2 : 1; + } + + return opcode switch + { + "VMadU64U32" or "VMadI64I32" => index == 2 ? 2 : 1, + "VLshlrevB64" or "VLshrrevB64" or "VAshrrevI64" => index == 1 ? 2 : 1, + "VLdexpF64" or "VTrigPreopF64" => index == 0 ? 2 : 1, + _ when opcode.StartsWith("VCvtF64", StringComparison.Ordinal) => 1, + _ => wide ? 2 : 1, + }; + } + + private static Operand Map(Gen5Operand operand) => operand.Kind switch + { + Gen5OperandKind.ScalarRegister => operand.Value is < ScalarCount and not Scc + ? Operand.Scalar((int)operand.Value) + : Operand.None, + Gen5OperandKind.VectorRegister => new Operand(Kind.Vector, (int)operand.Value), + Gen5OperandKind.EncodedConstant => operand.Value switch + { + 128 => new Operand(Kind.Zero), + 193 => new Operand(Kind.AllOnes), + 251 => new Operand(Kind.VccZ), + 252 => new Operand(Kind.ExecZ), + 253 => new Operand(Kind.Scc), + 125 => Operand.None, + _ => new Operand(Kind.Constant), + }, + _ => new Operand(Kind.Constant), + }; + + private static Operand CompareDestination(Gen5ShaderInstruction instruction) + { + if (instruction.Opcode.StartsWith("VCmpx", StringComparison.Ordinal)) + { + return Operand.Scalar(ExecLo); + } + + var register = instruction.Control switch + { + Gen5SdwaControl { ScalarDestination: { } sdwa } => sdwa, + Gen5Vop3Control { ScalarDestination: { } vop3 } => vop3, + _ => (uint)VccLo, + }; + return register is < ScalarCount and not Scc ? Operand.Scalar((int)register) : Operand.None; + } + + private static List UsedSources(Gen5ShaderInstruction instruction, int count) + { + var used = new List(instruction.Sources.Count); + for (var index = 0; index < instruction.Sources.Count; index++) + { + var source = instruction.Sources[index]; + if (index < count || source is not { Kind: Gen5OperandKind.ScalarRegister, Value: 0 }) + { + used.Add(Map(source)); + } + } + + return used; + } + + private static int Vop3SourceCount(Gen5ShaderInstruction instruction) + { + var opcode = (instruction.Words[0] >> 16) & 0x3FF; + return opcode switch + { + < 0x100 => 2, + 0x101 or 0x128 or 0x129 or 0x12A => 3, + < 0x140 => 2, + >= 0x164 and <= 0x16C => 2, + 0x174 => 2, + < 0x180 => 3, + < 0x200 => 1, + >= 0x2FF and <= 0x31F => 2, + >= 0x360 and <= 0x36C => 2, + 0x376 or 0x37F => 2, + _ => 3, + }; + } + + private static int Vop3pSourceCount(Gen5ShaderInstruction instruction) + { + var opcode = (instruction.Words[0] >> 16) & 0x7F; + return opcode is 0x00 or 0x09 or 0x0E or (>= 0x13 and <= 0x19) or (>= 0x20 and <= 0x22) ? 3 : 2; + } + + private static bool Accumulates(string opcode) => + opcode.Contains("Mac", StringComparison.Ordinal) || + opcode.StartsWith("VDot2c", StringComparison.Ordinal) || + opcode.StartsWith("VDot4c", StringComparison.Ordinal) || + opcode.StartsWith("VDot8c", StringComparison.Ordinal); + + private static bool UsesSampler(string opcode) => + opcode.StartsWith("ImageSample", StringComparison.Ordinal) || + opcode.StartsWith("ImageGather", StringComparison.Ordinal) || + opcode == "ImageGetLod"; + + private static string? RefusedOpcode(Gen5ShaderInstruction instruction, Family family, IReadOnlySet sharedFlatPcs) + { + var opcode = instruction.Opcode; + switch (opcode) + { + case "VReadlaneB32": + return "V_READLANE"; + case "VWritelaneB32": + return "V_WRITELANE"; + case "VMbcntLoU32B32": + case "VMbcntHiU32B32": + return "V_MBCNT"; + case "SQuadmaskB64": + case "SBitreplicateB64B32": + return "lane-mask layout"; + case "SGetpcB64": + case "SSetpcB64": + case "SSwappcB64": + case "SCbranchJoin": + case "SCbranchGFork": + case "SCbranchIFork": + return "program counter"; + case "SSendmsg": + case "STrap": + return "wave message"; + case "ImageGetLod": + return "compute derivatives"; + } + + if (opcode.StartsWith("VMovrel", StringComparison.Ordinal) || opcode.StartsWith("SMovrel", StringComparison.Ordinal)) + { + return "relative register"; + } + + if (opcode.StartsWith("SSetreg", StringComparison.Ordinal)) + { + return "wave message"; + } + + if (opcode.StartsWith("SRfe", StringComparison.Ordinal) || + opcode.StartsWith("SCall", StringComparison.Ordinal) || + opcode.StartsWith("SSubvector", StringComparison.Ordinal)) + { + return "program counter"; + } + + if (opcode.Contains("Wrexec", StringComparison.Ordinal)) + { + return "EXEC set from the wave's layout"; + } + + if (opcode.Contains("Raw", StringComparison.Ordinal)) + { + return "unknown instruction"; + } + + if (family == Family.Smem && opcode.Contains("Atomic", StringComparison.Ordinal)) + { + return "scalar atomic"; + } + + if (family == Family.Ds || (family == Family.Flat && sharedFlatPcs.Contains(instruction.Pc))) + { + return "LDS"; + } + + return null; + } + + private static bool LaneLocalDpp(uint control) => + control <= 0xFF || + control is >= 0x101 and <= 0x12F || + control is 0x140 or 0x141 || + control is >= 0x150 and <= 0x16F; + + private static bool TryBuildBlocks(IReadOnlyList instructions, Op[] ops, out Block[] blocks) + { + blocks = []; + var count = ops.Length; + var indexByPc = new Dictionary(count); + for (var index = 0; index < count; index++) + { + indexByPc[ops[index].Pc] = index; + } + + var last = instructions[^1]; + var endPc = last.Pc + (uint)(last.Words.Count * sizeof(uint)); + var leaders = new bool[count + 1]; + var targets = new int[count]; + leaders[0] = true; + for (var index = 0; index < count; index++) + { + targets[index] = -1; + var op = ops[index]; + if (op.IsTerminator) + { + leaders[index + 1] = true; + continue; + } + + if (!op.IsBranch) + { + continue; + } + + if (!Gen5IrBranchResolver.Instance.TryGetBranchTarget(instructions[index], out var targetPc)) + { + return false; + } + + if (indexByPc.TryGetValue(targetPc, out var target)) + { + targets[index] = target; + leaders[target] = true; + } + else if (targetPc < endPc) + { + return false; + } + + leaders[index + 1] = true; + } + + var starts = new List(); + for (var index = 0; index < count; index++) + { + if (leaders[index]) + { + starts.Add(index); + } + } + + var blockOf = new int[count]; + var built = new Block[starts.Count]; + for (var block = 0; block < starts.Count; block++) + { + var start = starts[block]; + var end = block + 1 < starts.Count ? starts[block + 1] : count; + built[block] = new Block { Start = start, End = end }; + for (var index = start; index < end; index++) + { + blockOf[index] = block; + } + } + + for (var block = 0; block < built.Length; block++) + { + var current = built[block]; + var lastIndex = current.End - 1; + var op = ops[lastIndex]; + var next = block + 1 < built.Length ? block + 1 : NoBlock; + if (op.IsTerminator) + { + continue; + } + + if (!op.IsBranch) + { + if (next != NoBlock) + { + current.Successors.Add(next); + } + + continue; + } + + var targetBlock = targets[lastIndex] >= 0 ? blockOf[targets[lastIndex]] : NoBlock; + if (!op.IsConditional) + { + if (targetBlock != NoBlock) + { + current.Successors.Add(targetBlock); + } + + continue; + } + + current.Condition = op.Condition; + current.TrueBlock = targetBlock; + current.FalseBlock = next; + if (targetBlock != NoBlock) + { + current.Successors.Add(targetBlock); + } + + if (next != NoBlock && !current.Successors.Contains(next)) + { + current.Successors.Add(next); + } + } + + blocks = built; + return true; + } + + private static bool PairBase(int register) => + register >= 0 && register % 2 == 0 && (register < M0 || register == ExecLo); + + private static bool Test(UInt128 set, int register) => + register is >= 0 and < ScalarCount && ((set >> register) & UInt128.One) != 0; + + private static UInt128 Bit(int register) => + register is >= 0 and < ScalarCount ? UInt128.One << register : UInt128.Zero; + + private static void SetPair(ref UInt128 set, int register, bool value) + { + if (!PairBase(register) || register == ExecLo) + { + return; + } + + var pair = Bit(register) | Bit(register + 1); + set = value ? set | pair : set & ~pair; + } + + private static bool IsSaveExec(string opcode) => + opcode.EndsWith("SaveexecB64", StringComparison.Ordinal) || + opcode.EndsWith("SaveexecB32", StringComparison.Ordinal); + + private static bool ScalarDestination64(string opcode) => + ScalarDestination64Opcodes.Contains(opcode) || opcode.EndsWith("SaveexecB64", StringComparison.Ordinal); + + private static bool IsMaskOperation(string opcode) => + MaskOperations.Contains(opcode) || IsSaveExec(opcode); + + private static bool WritesScc(string opcode) + { + if (opcode.EndsWith("F32", StringComparison.Ordinal) || opcode.EndsWith("F16", StringComparison.Ordinal)) + { + return false; + } + + foreach (var prefix in SccWriterPrefixes) + { + if (opcode.StartsWith(prefix, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + private static bool IsCompare(Op op) => + op.Family == Family.Vopc || + (op.Family == Family.Vop3 && op.Opcode.StartsWith("VCmp", StringComparison.Ordinal)); + + private static bool WritesExec(Op op) + { + var index = op.Destination.ScalarIndex; + return index is ExecLo or ExecHi || IsSaveExec(op.Opcode); + } + + private static UInt128 ScalarRange(Operand operand, int count) + { + var register = operand.ScalarIndex; + var set = UInt128.Zero; + for (var offset = 0; register >= 0 && offset < count && register + offset < ScalarCount; offset++) + { + set |= Bit(register + offset); + } + + return set; + } + + private static UInt128 ScalarUses(Op op) + { + var uses = UInt128.Zero; + for (var index = 0; index < op.Sources.Length; index++) + { + var source = op.Sources[index]; + if (source.Kind == Kind.Scc) + { + uses |= Bit(Scc); + } + + uses |= ScalarRange(source, op.Widths[index]); + } + + if (SccReaders.Contains(op.Opcode)) + { + uses |= Bit(Scc); + } + + if (op.Opcode is "SCbranchVccz" or "SCbranchVccnz") + { + uses |= Bit(VccLo) | Bit(VccHi); + } + + return uses; + } + + private static UInt128 ScalarDefs(Op op, bool sure) + { + var defs = UInt128.Zero; + switch (op.Family) + { + case Family.Sop1: + case Family.Sop2: + case Family.Sopk: + case Family.Sopc: + { + var wide = ScalarDestination64(op.Opcode) || (!sure && op.Wide); + if (!sure || !PartialWriteOpcodes.Contains(op.Opcode)) + { + defs |= ScalarRange(op.Destination, wide ? 2 : 1); + } + + if (IsSaveExec(op.Opcode)) + { + defs |= Bit(ExecLo); + if (op.Wide) + { + defs |= Bit(ExecHi); + } + } + + if (!sure || WritesScc(op.Opcode)) + { + defs |= Bit(Scc); + } + + break; + } + case Family.Smem: + defs |= ScalarRange(op.Destination, op.DestinationCount); + break; + case Family.Vopc: + defs |= ScalarRange(op.Destination, 2); + break; + case Family.Vop1: + case Family.Vop2: + case Family.Vop3: + case Family.Vop3p: + if (op.Opcode is "VReadfirstlaneB32" or "VReadlaneB32") + { + defs |= ScalarRange(op.Destination, 1); + } + else if (op.Destination.Kind == Kind.Scalar) + { + defs |= ScalarRange(op.Destination, 2); + } + + defs |= ScalarRange(op.Carry, 2); + break; + } + + return defs; + } + + private sealed class Analysis(Op[] ops, Block[] blocks) + { + private readonly Op[] _ops = ops; + private readonly Block[] _blocks = blocks; + private readonly List _loops = []; + + public bool Run(out string reason) + { + if (!FindLoops()) + { + reason = "control flow"; + return false; + } + + var count = _blocks.Length; + var input = new State[count]; + var reached = new bool[count]; + var queued = new bool[count]; + input[0].Mask = Bit(ExecLo) | Bit(ExecHi); + input[0].Active = true; + reached[0] = true; + queued[0] = true; + var work = new Queue(); + work.Enqueue(0); + for (var steps = 0; work.TryDequeue(out var block); steps++) + { + if (steps > MaxSteps) + { + reason = "analysis did not converge"; + return false; + } + + queued[block] = false; + var state = input[block]; + for (var index = _blocks[block].Start; index < _blocks[block].End; index++) + { + Transfer(_ops[index], ref state); + } + + foreach (var successor in _blocks[block].Successors) + { + var edge = OnEdge(_blocks[block], successor, state); + if (reached[successor]) + { + var merged = input[successor]; + merged.Join(edge); + if (merged.Equals(input[successor])) + { + continue; + } + + edge = merged; + } + + reached[successor] = true; + input[successor] = edge; + if (!queued[successor]) + { + queued[successor] = true; + work.Enqueue(successor); + } + } + } + + var output = new State[count]; + for (var block = 0; block < count; block++) + { + if (!reached[block]) + { + continue; + } + + var state = input[block]; + for (var index = _blocks[block].Start; index < _blocks[block].End; index++) + { + if (Check(block, index, state) is { } refused) + { + reason = $"{refused} (0x{_ops[index].Pc:X})"; + return false; + } + + Transfer(_ops[index], ref state); + } + + output[block] = state; + } + + return WaveBranches(output, reached, out reason); + } + + private bool FindLoops() + { + var count = _blocks.Length; + var reachable = new bool[count]; + var order = new List(); + var onStack = new bool[count]; + var retreating = new List<(int From, int To)>(); + var stack = new Stack<(int Block, int Next)>(); + stack.Push((0, 0)); + reachable[0] = true; + onStack[0] = true; + while (stack.Count > 0) + { + var (block, next) = stack.Pop(); + var successors = _blocks[block].Successors; + if (next < successors.Count) + { + stack.Push((block, next + 1)); + var successor = successors[next]; + if (onStack[successor]) + { + retreating.Add((block, successor)); + } + else if (!reachable[successor]) + { + reachable[successor] = true; + onStack[successor] = true; + stack.Push((successor, 0)); + } + + continue; + } + + onStack[block] = false; + order.Add(block); + } + + var predecessors = new List[count]; + for (var block = 0; block < count; block++) + { + predecessors[block] = []; + } + + for (var block = 0; block < count; block++) + { + foreach (var successor in _blocks[block].Successors) + { + predecessors[successor].Add(block); + } + } + + var dominators = new bool[count][]; + for (var block = 0; block < count; block++) + { + dominators[block] = new bool[count]; + if (block == 0) + { + dominators[block][0] = true; + } + else + { + Array.Fill(dominators[block], true); + } + } + + order.Reverse(); + for (var changed = true; changed;) + { + changed = false; + foreach (var block in order) + { + if (block == 0) + { + continue; + } + + var next = new bool[count]; + Array.Fill(next, true); + var any = false; + foreach (var predecessor in predecessors[block]) + { + if (!reachable[predecessor]) + { + continue; + } + + any = true; + for (var other = 0; other < count; other++) + { + next[other] &= dominators[predecessor][other]; + } + } + + if (!any) + { + Array.Fill(next, false); + } + + next[block] = true; + if (!next.AsSpan().SequenceEqual(dominators[block])) + { + dominators[block] = next; + changed = true; + } + } + } + + foreach (var (from, to) in retreating) + { + if (!dominators[from][to]) + { + return false; + } + + var body = new bool[count]; + body[to] = true; + var pending = new Stack(); + if (!body[from]) + { + body[from] = true; + pending.Push(from); + } + + while (pending.TryPop(out var block)) + { + foreach (var predecessor in predecessors[block]) + { + if (!body[predecessor]) + { + body[predecessor] = true; + pending.Push(predecessor); + } + } + } + + _loops.Add(new Loop(to, from, body)); + } + + return true; + } + + private static bool IsMask(Op op, int source, in State state) => + (ScalarRange(op.Sources[source], op.Widths[source]) & state.Mask) != 0; + + private static void WriteScalar(Operand operand, int count, bool mask, ref State state) + { + var index = operand.ScalarIndex; + for (var offset = 0; index >= 0 && offset < count && index + offset < ScalarCount; offset++) + { + var register = index + offset; + state.Mask = mask || register is ExecLo or ExecHi + ? state.Mask | Bit(register) + : state.Mask & ~Bit(register); + } + } + + private static bool MaskIn(Operand operand, int count, in State state) => + (ScalarRange(operand, count) & state.Mask) != 0; + + private static bool UniformSources(Op op, in State state) + { + if (op.LaneFetch) + { + return false; + } + + for (var index = 0; index < op.Sources.Length; index++) + { + var source = op.Sources[index]; + switch (source.Kind) + { + case Kind.Vector: + if (!state.Uniform(source.Register) || (op.Widths[index] > 1 && !state.Uniform(source.Register + 1))) + { + return false; + } + + break; + case Kind.Zero: + case Kind.AllOnes: + case Kind.Constant: + break; + default: + if (source.ScalarIndex < 0 || IsMask(op, index, state)) + { + return false; + } + + break; + } + } + + return true; + } + + private static MaskFacts PairFacts(int register, in State state) + { + if (register == ExecLo) + { + return new MaskFacts(true, true, state.Active); + } + + if (!PairBase(register)) + { + return default; + } + + return new MaskFacts( + Test(state.Covers, register) && Test(state.Covers, register + 1), + Test(state.Within, register) && Test(state.Within, register + 1), + Test(state.Nonzero, register) && Test(state.Nonzero, register + 1)); + } + + private static MaskFacts Facts(Operand operand, in State state) => operand.Kind switch + { + Kind.Zero => new MaskFacts(true, true, false), + Kind.AllOnes => new MaskFacts(true, false, true), + Kind.Scalar => PairFacts(operand.Register, state), + _ => default, + }; + + private static bool LaneMask(Operand operand, in State state) + { + if (operand.Kind is Kind.Zero or Kind.AllOnes) + { + return true; + } + + var register = operand.ScalarIndex; + if (register == ExecLo) + { + return true; + } + + return PairBase(register) && Test(state.Lanes, register) && Test(state.Lanes, register + 1); + } + + private static bool UpdateMaskFacts(Op op, in State before, ref State state) + { + var written = ScalarDefs(op, false); + var writesExec = Test(written, ExecLo) || Test(written, ExecHi); + var a = Facts(op.Source(0), before); + var b = Facts(op.Source(1), before); + var destination = op.Destination.ScalarIndex; + var value = default(MaskFacts); + var pair = true; + switch (op.Opcode) + { + case "SMovB64": + value = a; + break; + case "SCselectB64": + value = new MaskFacts(a.Covers && b.Covers, a.Within && b.Within, a.Nonzero && b.Nonzero); + break; + case "SAndB64": + value = new MaskFacts(a.Covers && b.Covers, a.Within || b.Within, false); + break; + case "SOrB64": + value = new MaskFacts(a.Covers && b.Covers, a.Within && b.Within, a.Nonzero || b.Nonzero); + break; + case "SAndn2B64": + value = new MaskFacts(false, a.Within, false); + break; + default: + pair = IsCompare(op); + if (pair) + { + value = new MaskFacts(UniformSources(op, before), true, false); + } + + break; + } + + var save = IsSaveExec(op.Opcode); + var narrowing = save + ? op.Opcode is "SAndSaveexecB64" or "SAndn1SaveexecB64" + : destination == ExecLo && pair && value.Within; + state.Covers &= ~written; + state.Within &= ~written; + state.Nonzero &= ~written; + if (writesExec) + { + if (!narrowing) + { + state.Covers = UInt128.Zero; + } + + state.Within = UInt128.Zero; + state.Active = !save && destination == ExecLo && pair && value.Nonzero; + if (!save && destination == ExecLo) + { + switch (op.Opcode) + { + case "SMovB64": + SetPair(ref state.Covers, op.Source(0).ScalarIndex, true); + SetPair(ref state.Within, op.Source(0).ScalarIndex, true); + break; + case "SAndB64": + SetPair(ref state.Covers, op.Source(0).ScalarIndex, true); + SetPair(ref state.Covers, op.Source(1).ScalarIndex, true); + break; + case "SAndn2B64": + SetPair(ref state.Covers, op.Source(0).ScalarIndex, true); + break; + case "SOrB64": + SetPair(ref state.Within, op.Source(0).ScalarIndex, true); + SetPair(ref state.Within, op.Source(1).ScalarIndex, true); + break; + } + } + } + + if (save) + { + SetPair(ref state.Covers, destination, narrowing); + SetPair(ref state.Nonzero, destination, before.Active); + } + else if (pair && destination != ExecLo) + { + SetPair(ref state.Covers, destination, value.Covers); + SetPair(ref state.Within, destination, value.Within); + SetPair(ref state.Nonzero, destination, value.Nonzero); + } + + return writesExec && narrowing; + } + + private static void Transfer(Op op, ref State state) + { + var before = state; + var narrowed = UpdateMaskFacts(op, before, ref state); + if (WritesExec(op) && !narrowed) + { + state.UniformLow = UInt128.Zero; + state.UniformHigh = UInt128.Zero; + } + + state.Lanes &= ~ScalarDefs(op, false); + switch (op.Family) + { + case Family.Sop1: + case Family.Sop2: + case Family.Sopk: + case Family.Sopc: + { + var mask = false; + for (var source = 0; source < op.Sources.Length; source++) + { + mask = mask || IsMask(op, source, before); + } + + if (SccReaders.Contains(op.Opcode)) + { + mask = mask || before.SccMask; + } + + var resultMask = mask && IsMaskOperation(op.Opcode); + if (IsSaveExec(op.Opcode)) + { + WriteScalar(op.Destination, 2, true, ref state); + SetPair(ref state.Lanes, op.Destination.ScalarIndex, op.Wide); + state.SccMask = true; + break; + } + + if (op.Family != Family.Sopc && op.Destination.Kind == Kind.Scalar) + { + WriteScalar( + op.Destination, + ScalarDestination64(op.Opcode) ? 2 : 1, + resultMask, + ref state); + var lanes = IsMaskOperation(op.Opcode) && op.Wide; + foreach (var source in op.Sources) + { + lanes = lanes && LaneMask(source, before); + } + + SetPair(ref state.Lanes, op.Destination.ScalarIndex, lanes); + } + + if (WritesScc(op.Opcode)) + { + state.SccMask = mask; + } + else if (!SccKeepers.Contains(op.Opcode)) + { + state.SccMask = before.SccMask || mask; + } + + break; + } + case Family.Smem: + WriteScalar(op.Destination, op.DestinationCount, false, ref state); + break; + case Family.Vopc: + WriteScalar(op.Destination, 2, true, ref state); + SetPair(ref state.Lanes, op.Destination.ScalarIndex, true); + break; + case Family.Vop1: + case Family.Vop2: + case Family.Vop3: + case Family.Vop3p: + { + if (op.Opcode is "VReadfirstlaneB32" or "VReadlaneB32") + { + WriteScalar(op.Destination, 1, false, ref state); + break; + } + + if (op.Family == Family.Vop3 && op.Destination.Kind == Kind.Scalar) + { + WriteScalar(op.Destination, 2, true, ref state); + SetPair(ref state.Lanes, op.Destination.ScalarIndex, IsCompare(op)); + break; + } + + if (op.Carry.Kind == Kind.Scalar) + { + WriteScalar(op.Carry, 2, true, ref state); + SetPair(ref state.Lanes, op.Carry.ScalarIndex, true); + } + + if (op.Destination.Kind == Kind.Vector) + { + var uniform = op.Select < 0 && + op.Opcode is not ("VPermlane16B32" or "VPermlanex16B32") && + UniformSources(op, before); + var register = op.Destination.Register; + state.SetUniform(register, uniform && (!op.PartialVector || before.Uniform(register))); + if (op.Wide) + { + state.SetUniform(register + 1, uniform && before.Uniform(register + 1)); + } + } + + break; + } + default: + for (var offset = 0; op.WrittenVectorStart >= 0 && offset < op.WrittenVectorCount; offset++) + { + state.SetUniform(op.WrittenVectorStart + offset, false); + } + + break; + } + } + + private static State OnEdge(Block block, int successor, State state) + { + if (block.Condition == Condition.None || block.TrueBlock == block.FalseBlock) + { + return state; + } + + var vccWithin = PairFacts(VccLo, state).Within; + switch (block.Condition) + { + case Condition.ExecZero: + state.Active = state.Active || successor == block.FalseBlock; + break; + case Condition.ExecNonZero: + state.Active = state.Active || successor == block.TrueBlock; + break; + case Condition.VccZero: + state.Active = state.Active || (vccWithin && successor == block.FalseBlock); + break; + case Condition.VccNonZero: + state.Active = state.Active || (vccWithin && successor == block.TrueBlock); + break; + } + + return state; + } + + private string? Check(int block, int index, in State state) + { + var op = _ops[index]; + if (op.Refused is { } refused) + { + return refused; + } + + var written = ScalarDefs(op, false); + var sop = op.Family is Family.Sop1 or Family.Sop2; + if ((Test(written, ExecLo) || Test(written, ExecHi)) && + !IsCompare(op) && + !(sop && IsMaskOperation(op.Opcode) && op.Wide)) + { + return "EXEC set from the wave's layout"; + } + + if (op.CrossRowFetch) + { + return "cross-row DPP"; + } + + foreach (var source in op.Sources) + { + if (source.Kind is Kind.VccZ or Kind.ExecZ) + { + return "VCCZ/EXECZ as data"; + } + + if (source.Kind == Kind.Scc && state.SccMask) + { + return "SCC made from a lane mask"; + } + } + + if (state.SccMask && SccReaders.Contains(op.Opcode)) + { + return "SCC made from a lane mask"; + } + + switch (op.Family) + { + case Family.Sop1: + case Family.Sop2: + case Family.Sopk: + case Family.Sopc: + { + var mask = false; + for (var source = 0; source < op.Sources.Length; source++) + { + mask = mask || IsMask(op, source, state); + } + + if (mask && !IsMaskOperation(op.Opcode)) + { + return "a lane mask used as a scalar value"; + } + + if (op.Destination.ScalarIndex is ExecLo or ExecHi || IsSaveExec(op.Opcode)) + { + var layout = !op.Wide; + foreach (var source in op.Sources) + { + layout = layout || !LaneMask(source, state); + } + + if (layout) + { + return "EXEC set from the wave's layout"; + } + } + + break; + } + case Family.Sopp: + if (op.Opcode is "SCbranchVccz" or "SCbranchVccnz" && + (!PairFacts(VccLo, state).Covers || !Test(state.Lanes, VccLo) || !Test(state.Lanes, VccHi))) + { + return "a branch on VCC without EXEC's lanes"; + } + + break; + case Family.Smem: + case Family.Mubuf: + case Family.Mtbuf: + case Family.Flat: + case Family.Mimg: + for (var source = 0; source < op.Sources.Length; source++) + { + if (MaskIn(op.Sources[source], op.Widths[source], state)) + { + return "a lane mask used as an address or resource"; + } + } + + break; + case Family.Vop1: + case Family.Vop2: + case Family.Vop3: + case Family.Vop3p: + case Family.Vopc: + { + for (var source = 0; source < op.Sources.Length; source++) + { + if (source != op.Select && IsMask(op, source, state)) + { + return "a lane mask broadcast to the lanes"; + } + } + + if (op.Select >= 0 && !LaneMask(op.Source(op.Select), state)) + { + return "a lane select from the wave's layout"; + } + + if (op.Opcode == "VReadfirstlaneB32") + { + if (!state.Active) + { + return "V_READFIRSTLANE where a half may have no active lane"; + } + + var source = op.Source(0); + var uniform = source.Kind == Kind.Vector + ? state.Uniform(source.Register) + : op.Sources.Length == 0 || !IsMask(op, 0, state); + if (!uniform && !Waterfall(block, index)) + { + return "V_READFIRSTLANE of a lane value outside a waterfall loop"; + } + } + + break; + } + } + + return null; + } + + private bool Waterfall(int block, int index) + { + var head = _ops[index]; + if (head.Source(0).Kind != Kind.Vector) + { + return false; + } + + var value = head.Source(0).Register; + var scalar = head.Destination.ScalarIndex; + if (scalar < 0) + { + return false; + } + + var compareMask = -1; + for (var position = index + 1; position < _blocks[block].End; position++) + { + var op = _ops[position]; + if (compareMask < 0) + { + if (!op.Opcode.StartsWith("VCmpEq", StringComparison.Ordinal) && + !op.Opcode.StartsWith("VCmpxEq", StringComparison.Ordinal)) + { + continue; + } + + var wide = op.Wide; + bool Matches(Operand vector, Operand uniform) + { + if (vector.Kind != Kind.Vector || uniform.ScalarIndex < 0) + { + return false; + } + + var vectorDelta = value - vector.Register; + var scalarDelta = scalar - uniform.ScalarIndex; + return vectorDelta == scalarDelta && vectorDelta >= 0 && vectorDelta <= (wide ? 1 : 0); + } + + if (!Matches(op.Source(0), op.Source(1)) && !Matches(op.Source(1), op.Source(0))) + { + continue; + } + + compareMask = op.Destination.ScalarIndex; + if (compareMask == ExecLo) + { + break; + } + + if (compareMask < 0) + { + return false; + } + + continue; + } + + if (IsSaveExec(op.Opcode) && op.Source(0).ScalarIndex == compareMask) + { + compareMask = ExecLo; + break; + } + } + + if (compareMask != ExecLo) + { + return false; + } + + foreach (var loop in _loops) + { + if (!loop.Body[block] && loop.Header != block) + { + continue; + } + + var latch = _blocks[loop.Latch]; + if (latch.End > latch.Start && _ops[latch.End - 1].Opcode == "SCbranchExecnz") + { + return true; + } + } + + return false; + } + + private static string? SymTransfer(Op op, Sym[] syms) + { + if (op.Opcode == "SBarrier") + { + return "a barrier"; + } + + var written = ScalarDefs(op, false); + if (written == UInt128.Zero) + { + return null; + } + + var destination = op.Destination.ScalarIndex; + Sym Pair(Operand operand) + { + if (operand.Kind == Kind.Zero) + { + return Sym.Zero; + } + + if (operand.Kind == Kind.AllOnes) + { + return Sym.Ones; + } + + var register = operand.ScalarIndex; + if (!PairBase(register) || syms[register] != syms[register + 1]) + { + return Sym.Unknown; + } + + return syms[register] == Sym.Original && register != destination ? Sym.Unknown : syms[register]; + } + + var a = Pair(op.Source(0)); + var b = Pair(op.Source(1)); + var e = syms[ExecLo] == syms[ExecHi] ? syms[ExecLo] : Sym.Unknown; + var value = Sym.Unknown; + var exec = Sym.Unknown; + switch (op.Opcode) + { + case "SMovB64": + value = a; + break; + case "SCselectB64": + value = a == b ? a : Sym.Unknown; + break; + case "SNotB64": + value = Not(a); + break; + case "SWqmB64": + value = a is Sym.Zero or Sym.Ones ? a : Sym.Unknown; + break; + case "SAndB64": + value = And(a, b); + break; + case "SOrB64": + value = Or(a, b); + break; + case "SXorB64": + value = Xor(a, b); + break; + case "SAndn2B64": + value = And(a, Not(b)); + break; + case "SOrn2B64": + value = Or(a, Not(b)); + break; + case "SNandB64": + value = Not(And(a, b)); + break; + case "SNorB64": + value = Not(Or(a, b)); + break; + case "SXnorB64": + value = Not(Xor(a, b)); + break; + case "SAndSaveexecB64": + value = e; + exec = And(a, e); + break; + case "SAndn1SaveexecB64": + value = e; + exec = And(Not(a), e); + break; + case "SAndn2SaveexecB64": + value = e; + exec = And(a, Not(e)); + break; + case "SOrSaveexecB64": + value = e; + exec = Or(a, e); + break; + case "SOrn1SaveexecB64": + value = e; + exec = Or(Not(a), e); + break; + case "SOrn2SaveexecB64": + value = e; + exec = Or(a, Not(e)); + break; + case "SXorSaveexecB64": + value = e; + exec = Xor(a, e); + break; + case "SNandSaveexecB64": + value = e; + exec = Not(And(a, e)); + break; + case "SNorSaveexecB64": + value = e; + exec = Not(Or(a, e)); + break; + case "SXnorSaveexecB64": + value = e; + exec = Not(Xor(a, e)); + break; + default: + if (IsCompare(op)) + { + value = e == Sym.Zero ? Sym.Zero : Sym.Unknown; + } + + break; + } + + for (var register = 0; register < ScalarCount; register++) + { + if (Test(written, register)) + { + syms[register] = Sym.Unknown; + } + } + + if (IsSaveExec(op.Opcode)) + { + syms[ExecLo] = exec; + syms[ExecHi] = exec; + } + + if (PairBase(destination) && (ScalarDestination64(op.Opcode) || IsCompare(op))) + { + syms[destination] = value; + syms[destination + 1] = value; + } + + return syms[ExecLo] != Sym.Zero || syms[ExecHi] != Sym.Zero ? "EXEC gaining a half's lanes" : null; + } + + private static Sym Not(Sym value) => value switch + { + Sym.Zero => Sym.Ones, + Sym.Ones => Sym.Zero, + _ => Sym.Unknown, + }; + + private static Sym And(Sym a, Sym b) => + a == Sym.Zero || b == Sym.Zero ? Sym.Zero + : a == Sym.Ones ? b + : b == Sym.Ones ? a + : a == b ? a + : Sym.Unknown; + + private static Sym Or(Sym a, Sym b) => + a == Sym.Ones || b == Sym.Ones ? Sym.Ones + : a == Sym.Zero ? b + : b == Sym.Zero ? a + : a == b ? a + : Sym.Unknown; + + private static Sym Xor(Sym a, Sym b) => + a == Sym.Zero ? b + : b == Sym.Zero ? a + : a == Sym.Ones ? Not(b) + : b == Sym.Ones ? Not(a) + : Sym.Unknown; + + private bool Ways(int seed, int merge, Sym[] entry, Sym[] atMerge, out string? reason) + { + reason = null; + Array.Fill(atMerge, Sym.Unknown); + if (seed == merge) + { + entry.CopyTo(atMerge, 0); + return true; + } + + static bool JoinInto(Sym[] into, Sym[] from) + { + var changed = false; + for (var register = 0; register < into.Length; register++) + { + if (into[register] != from[register] && into[register] != Sym.Unknown) + { + into[register] = Sym.Unknown; + changed = true; + } + } + + return changed; + } + + var count = _blocks.Length; + var merged = false; + var input = new Sym[count][]; + var queued = new bool[count]; + var work = new Queue(); + input[seed] = (Sym[])entry.Clone(); + queued[seed] = true; + work.Enqueue(seed); + while (work.TryDequeue(out var block)) + { + queued[block] = false; + var syms = (Sym[])input[block].Clone(); + for (var index = _blocks[block].Start; index < _blocks[block].End; index++) + { + if (SymTransfer(_ops[index], syms) is { } refused) + { + reason = $"{refused} between a lane-mask branch and its merge (0x{_ops[index].Pc:X})"; + return false; + } + } + + foreach (var successor in _blocks[block].Successors) + { + if (successor == merge) + { + if (!merged) + { + syms.CopyTo(atMerge, 0); + } + else + { + JoinInto(atMerge, syms); + } + + merged = true; + continue; + } + + if (input[successor] is null) + { + input[successor] = (Sym[])syms.Clone(); + } + else if (!JoinInto(input[successor], syms)) + { + continue; + } + + if (!queued[successor]) + { + queued[successor] = true; + work.Enqueue(successor); + } + } + } + + return true; + } + + private bool WaveBranches(State[] output, bool[] reached, out string reason) + { + var count = _blocks.Length; + var use = new UInt128[count]; + var def = new UInt128[count]; + var liveIn = new UInt128[count]; + for (var block = 0; block < count; block++) + { + for (var index = _blocks[block].Start; index < _blocks[block].End; index++) + { + use[block] |= ScalarUses(_ops[index]) & ~def[block]; + def[block] |= ScalarDefs(_ops[index], true); + } + } + + for (var changed = true; changed;) + { + changed = false; + for (var block = count - 1; block >= 0; block--) + { + var liveOut = UInt128.Zero; + foreach (var successor in _blocks[block].Successors) + { + liveOut |= liveIn[successor]; + } + + var live = use[block] | (liveOut & ~def[block]); + if (live != liveIn[block]) + { + liveIn[block] = live; + changed = true; + } + } + } + + var post = new bool[count][]; + for (var block = 0; block < count; block++) + { + post[block] = new bool[count]; + if (_blocks[block].Successors.Count == 0) + { + post[block][block] = true; + } + else + { + Array.Fill(post[block], true); + } + } + + for (var changed = true; changed;) + { + changed = false; + for (var block = count - 1; block >= 0; block--) + { + if (_blocks[block].Successors.Count == 0) + { + continue; + } + + var next = new bool[count]; + Array.Fill(next, true); + foreach (var successor in _blocks[block].Successors) + { + for (var other = 0; other < count; other++) + { + next[other] &= post[successor][other]; + } + } + + next[block] = true; + if (!next.AsSpan().SequenceEqual(post[block])) + { + post[block] = next; + changed = true; + } + } + } + + var postCounts = post.Select(set => set.Count(member => member)).ToArray(); + var half = new Sym[ScalarCount]; + var wave = new Sym[ScalarCount]; + for (var block = 0; block < count; block++) + { + var current = _blocks[block]; + if (!reached[block] || current.Condition == Condition.None || current.TrueBlock == current.FalseBlock) + { + continue; + } + + bool vcc; + bool zeroTaken; + switch (current.Condition) + { + case Condition.ExecZero: + vcc = false; + zeroTaken = true; + break; + case Condition.ExecNonZero: + vcc = false; + zeroTaken = false; + break; + case Condition.VccZero: + vcc = true; + zeroTaken = true; + break; + case Condition.VccNonZero: + vcc = true; + zeroTaken = false; + break; + default: + continue; + } + + var halfWay = zeroTaken ? current.TrueBlock : current.FalseBlock; + var waveWay = zeroTaken ? current.FalseBlock : current.TrueBlock; + if (halfWay == NoBlock || waveWay == NoBlock) + { + reason = "control flow"; + return false; + } + + var members = 0; + var common = new bool[count]; + for (var other = 0; other < count; other++) + { + common[other] = post[halfWay][other] && post[waveWay][other]; + if (common[other]) + { + members++; + } + } + + var merge = count; + for (var other = 0; other < count && merge == count; other++) + { + if (common[other] && postCounts[other] == members) + { + merge = other; + } + } + + var entry = new Sym[ScalarCount]; + entry[ExecLo] = Sym.Zero; + entry[ExecHi] = Sym.Zero; + if (vcc) + { + entry[VccLo] = Sym.Zero; + entry[VccHi] = Sym.Zero; + } + + for (var register = 0; register < M0; register += 2) + { + if (PairFacts(register, output[block]).Within) + { + entry[register] = Sym.Zero; + entry[register + 1] = Sym.Zero; + } + } + + if (!Ways(halfWay, merge, entry, half, out var halfReason)) + { + reason = halfReason!; + return false; + } + + if (!Ways(waveWay, merge, entry, wave, out var waveReason)) + { + reason = waveReason!; + return false; + } + + if (merge == count) + { + continue; + } + + var escaping = new List(); + for (var register = 0; register < ScalarCount; register++) + { + if (Test(liveIn[merge], register) && (half[register] == Sym.Unknown || half[register] != wave[register])) + { + escaping.Add(ScalarName(register)); + } + } + + if (escaping.Count != 0) + { + reason = $"scalar state escapes a lane-mask branch (0x{_ops[current.End - 1].Pc:X}: {string.Join(' ', escaping)})"; + return false; + } + } + + reason = string.Empty; + return true; + } + + private static string ScalarName(int register) => register switch + { + VccLo => "vcc_lo", + VccHi => "vcc_hi", + M0 => "m0", + Scc => "scc", + ExecLo => "exec_lo", + ExecHi => "exec_hi", + _ => $"s{register}", + }; + } +} diff --git a/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs b/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs index 9829b63..5263f83 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs @@ -18,6 +18,7 @@ public enum DescriptorBindingKind : uint FaultBuffer, FlattenedResourceTable, ShaderData, + MipStatistics, Count, } @@ -342,7 +343,11 @@ public sealed class BindingLayout : IEquatable public uint DispatchThreadLimitsDword => MemoryOffsetDword + (MemoryOffsetCount + 3) / 4; - public uint ShaderDataDwordCount => DispatchThreadLimitsDword + (UsesDispatchThreadLimits ? 3u : 0u); + public uint MipStatisticsCount { get; init; } + + public uint MipStatisticsDword => DispatchThreadLimitsDword + (UsesDispatchThreadLimits ? 3u : 0u); + + public uint ShaderDataDwordCount => MipStatisticsDword + MipStatisticsCount; public bool UsesPushData => PushDataStartDword != PushData.NoStart; @@ -359,7 +364,20 @@ public sealed class BindingLayout : IEquatable public static uint NativeBindingIndex(ShaderStage stage, DescriptorBindingKind kind) => (uint)kind + (stage == ShaderStage.Pixel ? (uint)DescriptorBindingKind.Count : 0); - public DescriptorBinding? Find(DescriptorBindingKind kind) => Descriptors.FirstOrDefault(binding => binding.Kind == kind); + // Called for every bound stage of every draw: an indexed loop allocates no closure or enumerator. + public DescriptorBinding? Find(DescriptorBindingKind kind) + { + var descriptors = Descriptors; + for (var index = 0; index < descriptors.Count; index++) + { + if (descriptors[index].Kind == kind) + { + return descriptors[index]; + } + } + + return null; + } // The user-data registers live at program entry: those some path reads before it // writes them, found by liveness over the control-flow graph, in ascending order. @@ -566,7 +584,8 @@ public sealed class BindingLayout : IEquatable var shaderBaseDword = usesShaderBase ? (uint)userDataRegisters.Count : NoShaderBase; var memoryOffsetDword = (uint)userDataRegisters.Count + (usesShaderBase ? ShaderBaseDwordCount : 0); var memoryOffsetCount = (uint)info.Buffers.Count; - var shaderDataDwords = memoryOffsetDword + (memoryOffsetCount + 3) / 4 + (usesDispatchThreadLimits ? 3u : 0u); + var mipStatisticsCount = info.Images.Any(static image => image.MipStatistics) ? (uint)info.Images.Count : 0u; + var shaderDataDwords = memoryOffsetDword + (memoryOffsetCount + 3) / 4 + (usesDispatchThreadLimits ? 3u : 0u) + mipStatisticsCount; var pushStart = PushData.StartFor(pushDataStartDword, shaderDataDwords); var descriptors = new List(); if (info.Buffers.Count != 0) @@ -636,6 +655,11 @@ public sealed class BindingLayout : IEquatable descriptors.Add(new DescriptorBinding(DescriptorBindingKind.ShaderData, [])); } + if (mipStatisticsCount != 0) + { + descriptors.Add(new DescriptorBinding(DescriptorBindingKind.MipStatistics, [])); + } + return new BindingLayout { PushDataStartDword = pushStart, @@ -644,6 +668,7 @@ public sealed class BindingLayout : IEquatable MemoryOffsetDword = memoryOffsetDword, MemoryOffsetCount = memoryOffsetCount, UsesDispatchThreadLimits = usesDispatchThreadLimits, + MipStatisticsCount = mipStatisticsCount, UserDataRegisters = userDataRegisters, Descriptors = descriptors, }; @@ -657,6 +682,7 @@ public sealed class BindingLayout : IEquatable MemoryOffsetDword == other.MemoryOffsetDword && MemoryOffsetCount == other.MemoryOffsetCount && UsesDispatchThreadLimits == other.UsesDispatchThreadLimits && + MipStatisticsCount == other.MipStatisticsCount && UserDataRegisters.SequenceEqual(other.UserDataRegisters) && Descriptors.Count == other.Descriptors.Count && Descriptors.Zip(other.Descriptors).All(pair => pair.First.Kind == pair.Second.Kind && pair.First.Resources.SequenceEqual(pair.Second.Resources)); diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs index 759e5bc..85fdec6 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs @@ -10,11 +10,14 @@ namespace SharpEmu.ShaderCompiler.Resources; // returns false so the caller falls back to a full materialization. public delegate bool ResidentGuestBytesReader(ulong address, Span destination, bool clean); -// Materialization is a pure function of the plan, the draw's user data, the shader -// base, the compute state and the guest words it reads. This cache records exactly -// those words and reuses the result while every one of them is unchanged, so a draw -// that re-binds the same resources skips the descriptor walk. Any failed read, any -// GPU-owned range and any byte difference make the entry miss; nothing is guessed. +// Materialization is a pure function of the plan, the user data words the evaluator +// reads, the shader base, the compute state and the guest words it reads. This cache +// records exactly those words and reuses the result while every one of them is +// unchanged, so a draw that re-binds the same resources skips the descriptor walk. A user +// data word the evaluator never read (a per-draw constant for the shader's own ALU, say) +// does not distinguish two draws; the snapshot a hit returns carries the current user +// data. Any failed read, any GPU-owned range and any byte difference make the entry +// miss; nothing is guessed. public sealed class ResourceMaterializationCache { // Two generations approximate LRU: a full young generation becomes the old one, @@ -24,6 +27,761 @@ public sealed class ResourceMaterializationCache private Dictionary _old = new(); private byte[] _scratch = new byte[256]; private ReadRecorder? _spareRecorder; + // The user data words each (plan, base, compute state) has read so far. They only index the + // entries; Entry.Matches is what decides a hit. + private readonly Dictionary _learnedUserData = new(); + private readonly Dictionary _rebases = new(); + + private AddressRebase RebaseOf(ulong drawKey, ShaderResourcePlan plan) + { + if (!_rebases.TryGetValue(drawKey, out var rebase)) + _rebases.Add(drawKey, rebase = new AddressRebase(plan)); + return rebase; + } + + // Titles allocate per-draw constants linearly, so a buffer's base address changes on every + // draw while everything else the draw binds stays the same; keyed on that address, the cache + // misses nearly every draw. A user data word the plan uses verbatim as a buffer's base dword + // is a candidate. It is rebased only once misses have shown it repeatedly that two draws + // differing in nothing but candidate words materialize identically, except for those base + // dwords, which equal the words: the same descriptors, table, ranges, specialization and + // recorded guest bytes. One counterexample rejects the word for good. A rebased word leaves + // the key and the match, and a hit patches the base dwords from the draw's own user data. + private sealed class AddressRebase + { + private const int Confirmations = 3; + + // A word that is a base address. A buffer candidate is the base dword of a buffer descriptor + // (Buffer >= 0): the hit patches that dword, and reads inside the descriptor's range move. + // A raw candidate is the low dword of the handle of scalar loads; its reads lie at constant + // byte offsets [MinOffset, MaxOffset] from the base whose high dword is user data word High + // (or HighConstant when High < 0). + private readonly record struct Candidate(int UserData, int Buffer, int High = -1, uint HighConstant = 0, + long MinOffset = 0, long MaxOffset = 0); + + private const ulong RawAddressMask = 0x0000_FFFF_FFFF_FFFFul; + + private readonly Candidate[] _candidates; + private readonly ulong[] _candidateMask; + private ulong[] _rejected = []; + private readonly Dictionary _confirmed = new(); + private const int RecentCapacity = 512; + private readonly Dictionary _recent = new(); + + private ulong StructureOf(Entry entry) + { + var hash = new HashCode(); + hash.Add(entry.UserData.Length); + for (var index = 0; index < entry.UserData.Length; index++) + { + if (!IsUsed(entry.UsedUserData, index) || IsUsed(_candidateMask, index)) + continue; + hash.Add(index); + hash.Add(entry.UserData[index]); + } + + return (uint)hash.ToHashCode(); + } + + private readonly ShaderResourcePlan _plan; + + public AddressRebase(ShaderResourcePlan plan) + { + _plan = plan; + var candidates = new List(); + + for (var buffer = 0; buffer < plan.Info.Buffers.Count; buffer++) + { + var source = plan.DescriptorSources[(int)plan.Info.Buffers[buffer].Source]; + if (source.Dwords.Length != 0 && UserDataIndex(source.Dwords[0]) is { } index) + candidates.Add(new Candidate(index, buffer)); + } + + var values = PlanValues(plan); + var buffered = candidates.Select(candidate => candidate.UserData).ToHashSet(); + foreach (var raw in RawCandidates(plan, values)) + if (!buffered.Contains(raw.UserData)) + candidates.Add(raw); + + _candidates = candidates.ToArray(); + _candidateMask = MaskOf(_candidates.Select(candidate => candidate.UserData)); + var memoryBases = MemoryBaseCandidates(plan); + MemoryBases = memoryBases.Select(candidate => candidate.Buffer).ToArray(); + _memoryState = new int[plan.Info.Buffers.Count]; + _memorySlots = new int[plan.Info.Buffers.Count]; + _memorySlots.AsSpan().Fill(-1); + foreach (var (buffer, slot) in memoryBases) + _memorySlots[buffer] = slot; + _memoryLearnable = memoryBases.Count; + (TableSlotBuffers, TableSlotOffsets, TableSlotsThroughBases) = TableSlotSources(plan, memoryBases); + DirectSlots = DirectSlotsOf(plan, TableSlotBuffers); + } + + // A table slot that is one dword loaded through a user data pointer at constant offsets: its + // word is that dword. High is the user data index of the pointer's high dword, or -1 for + // the constant HighConstant. + public readonly record struct DirectLoad(int Low, int High, ulong HighConstant, long Immediate, uint Offset); + + // Per table slot: its direct load (default for a slot through a memory-based buffer), or + // null when some other slot is not a direct load. + public DirectLoad[]? DirectSlots { get; } + + private DirectLoad[]? DirectSlotsOf(ShaderResourcePlan plan, int[] slotBuffers) + { + var loads = new DirectLoad[plan.TableReads.Count]; + for (var slot = 0; slot < loads.Length; slot++) + { + if (slotBuffers[slot] >= 0) + continue; + var value = plan.TableReads[slot].Value; + if (value.Kind != ScalarValueKind.ScalarAddressWord || value.Operands.Length < 2 || + value.Operands[1].Kind != ScalarValueKind.Constant || (uint)value.MemoryIndex >= (uint)plan.Memory.Count) + return null; + var handle = value.Operands[0]; + if (handle.Operands.Length < 2 || UserDataIndex(handle.Operands[0]) is not { } low) + return null; + var high = UserDataIndex(handle.Operands[1]); + if (high is null && handle.Operands[1].Kind != ScalarValueKind.Constant) + return null; + loads[slot] = new DirectLoad(low, high ?? -1, high is null ? handle.Operands[1].Payload : 0, + (int)plan.Memory[value.MemoryIndex].Offset, (uint)value.Operands[1].Payload); + } + + return loads; + } + + // Per table slot: the memory-based buffer it reads a dword of at a constant byte offset from + // the base, or -1. TableSlotsThroughBases: some slot reads through such a buffer otherwise. + public int[] TableSlotBuffers { get; } = []; + public long[] TableSlotOffsets { get; } = []; + public bool TableSlotsThroughBases { get; } + + private static (int[] Buffers, long[] Offsets, bool Through) TableSlotSources(ShaderResourcePlan plan, + List<(int Buffer, int Slot)> memoryBases) + { + var buffers = new int[plan.TableReads.Count]; + var offsets = new long[plan.TableReads.Count]; + buffers.AsSpan().Fill(-1); + if (memoryBases.Count == 0) + return (buffers, offsets, false); + + int BufferOf(ScalarValue handleWord) + { + foreach (var (buffer, slot) in memoryBases) + { + var baseWord = plan.DescriptorSources[(int)plan.Info.Buffers[buffer].Source].Dwords[0]; + if (ReferenceEquals(handleWord, baseWord) || + (slot >= 0 && handleWord.Kind == ScalarValueKind.ResourceTableWord && (int)handleWord.Payload == slot)) + return buffer; + } + + return -1; + } + + var through = false; + for (var slot = 0; slot < plan.TableReads.Count; slot++) + { + var value = plan.TableReads[slot].Value; + if (value.Kind == ScalarValueKind.ScalarBufferWord && value.Operands.Length >= 2 && + value.Operands[0].Operands.Length != 0 && value.Operands[1].Kind == ScalarValueKind.Constant && + value.MemoryIndex >= 0 && value.MemoryIndex < plan.Memory.Count && + BufferOf(value.Operands[0].Operands[0]) is var direct and >= 0) + { + buffers[slot] = direct; + offsets[slot] = ((long)plan.Memory[value.MemoryIndex].Offset + (long)(uint)value.Operands[1].Payload) & ~3L; + continue; + } + + var visited = new HashSet(); + var pending = new Stack(); + pending.Push(value); + while (!through && pending.Count != 0) + { + var current = pending.Pop(); + if (!visited.Add(current)) + continue; + if (current.Kind == ScalarValueKind.ScalarBufferWord && current.Operands.Length != 0 && + current.Operands[0].Operands.Length != 0 && BufferOf(current.Operands[0].Operands[0]) >= 0) + through = true; + foreach (var operand in current.Operands) + pending.Push(operand); + } + } + + return (buffers, offsets, through); + } + + private readonly int[] _memorySlots; + + // Memory-loaded bases, learned per buffer like the user data words: confirmations count up + // to MemoryActive; MemoryRejected is final. + private const int MemoryActive = Confirmations; + private const int MemoryRejected = -1; + private readonly int[] _memoryState; + private int _memoryLearnable; + + public IReadOnlyList MemoryBases { get; } + + public bool HasLearnableMemoryBases => _memoryLearnable != 0; + + + public bool IsRejectedMemoryBase(int buffer) => _memoryState[buffer] == MemoryRejected; + + public bool AreActiveMemoryBases(List<(int Buffer, uint Word)> patches) + { + foreach (var (buffer, _) in patches) + if (_memoryState[buffer] < MemoryActive) + return false; + return true; + } + + // A rebased refresh computed for a stale entry, compared with the full materialization of + // the same draw: agreement counts towards activating its bases, one disagreement rejects + // them for good. + public void ConfirmMemoryBases(List<(int Buffer, uint Word)> patches, bool agrees) + { + foreach (var (buffer, _) in patches) + { + if (_memoryState[buffer] == MemoryRejected) + continue; + if (!agrees) + { + _memoryState[buffer] = MemoryRejected; + _memoryLearnable--; + } + else if (_memoryState[buffer] < MemoryActive) + { + _memoryState[buffer]++; + } + } + } + + // The window of buffer reads a patched base covers for this draw. + public static bool TryMovedWindow(uint[] descriptor, uint baseWord, out ulong start, out ulong size) + { + start = size = 0; + if (descriptor.Length < 3) + return false; + start = baseWord | ((ulong)(descriptor[1] & 0xFFFF) << 32); + var stride = (descriptor[1] >> 16) & 0x3FFF; + size = stride == 0 ? descriptor[2] : (ulong)descriptor[2] * stride; + return true; + } + + // The base dword moves, and so does the table slot that holds the same word. + public ResourceSnapshot PatchMemoryBases(ResourceSnapshot snapshot, List<(int Buffer, uint Word)> patches) + { + var buffers = (uint[][])snapshot.Buffers.Clone(); + uint[]? table = null; + foreach (var (buffer, word) in patches) + { + if (ReferenceEquals(buffers[buffer], snapshot.Buffers[buffer])) + buffers[buffer] = (uint[])buffers[buffer].Clone(); + var previous = buffers[buffer][0]; + buffers[buffer][0] = word; + var slot = _memorySlots[buffer]; + var flat = slot >= 0 && slot < _plan.TableReads.Count ? (int)_plan.TableReads[slot].FlatOffset : -1; + if (flat >= 0 && flat < snapshot.FlattenedResourceTable.Length && snapshot.FlattenedResourceTable[flat] == previous) + { + table ??= (uint[])snapshot.FlattenedResourceTable.Clone(); + table[flat] = word; + } + } + + return new ResourceSnapshot + { + Buffers = buffers, + Images = snapshot.Images, + Samplers = snapshot.Samplers, + FlattenedResourceTable = table ?? snapshot.FlattenedResourceTable, + UserData = snapshot.UserData, + DeviceAddressRanges = snapshot.DeviceAddressRanges, + }; + } + + private int? UserDataIndex(ScalarValue value) + { + if (value.Kind != ScalarValueKind.UserData) + return null; + var register = value.UserDataRegister; + return register >= _plan.UserDataBase && register - _plan.UserDataBase < _plan.UserDataCount + ? (int)(register - _plan.UserDataBase) + : null; + } + + // Every scalar load of the plan whose handle takes its low dword verbatim from user data. + // A word qualifies only when all its loads use constant offsets and one high dword. + // Every value the plan evaluates, operands included. + private static HashSet PlanValues(ShaderResourcePlan plan) + { + var roots = new List(); + foreach (var source in plan.DescriptorSources) + roots.AddRange(source.Dwords); + roots.AddRange(plan.TableReads.Select(read => read.Value)); + roots.AddRange(plan.DynamicReads); + foreach (var access in plan.Accesses) + { + if (access is null) + continue; + foreach (var value in new[] { access.Handle, access.SamplerHandle, access.Read, access.Offset, access.Active }) + if (value is not null) + roots.Add(value); + } + + roots.AddRange(plan.Graph.Values); + var visited = new HashSet(); + var pending = new Stack(roots); + while (pending.Count != 0) + { + var value = pending.Pop(); + if (!visited.Add(value)) + continue; + foreach (var operand in value.Operands) + pending.Push(operand); + } + + return visited; + } + + // Buffers whose base dword is read from guest memory, directly or through a slot of the + // flattened table: descriptor tables the title rewrites per frame or per draw point them at + // freshly allocated constants. The slot (or -1) is where the table holds the same word. + // Recorded reads inside such a buffer's window move with its base. + private static List<(int Buffer, int Slot)> MemoryBaseCandidates(ShaderResourcePlan plan) + { + var bases = new List<(int Buffer, int Slot)>(); + for (var buffer = 0; buffer < plan.Info.Buffers.Count; buffer++) + { + var source = plan.DescriptorSources[(int)plan.Info.Buffers[buffer].Source]; + if (source.Dwords.Length < 2) + continue; + var baseWord = source.Dwords[0]; + var slot = baseWord.Kind == ScalarValueKind.ResourceTableWord ? (int)baseWord.Payload : -1; + if (slot < 0 && baseWord.Kind is not (ScalarValueKind.ScalarAddressWord or ScalarValueKind.ScalarBufferWord)) + continue; + bases.Add((buffer, slot)); + } + + return bases; + } + + private IEnumerable RawCandidates(ShaderResourcePlan plan, HashSet values) + { + var found = new Dictionary(); + var disqualified = new HashSet(); + foreach (var value in values) + { + if (value.Kind is not (ScalarValueKind.ScalarAddressWord or ScalarValueKind.ScalarBufferWord) || + value.Operands.Length < 2 || value.Operands[0].Operands.Length < 2 || + UserDataIndex(value.Operands[0].Operands[0]) is not { } index) + continue; + + var high = value.Operands[0].Operands[1]; + var highIndex = UserDataIndex(high); + var offset = value.Operands[1]; + if ((highIndex is null && high.Kind != ScalarValueKind.Constant) || offset.Kind != ScalarValueKind.Constant || + value.MemoryIndex >= plan.Memory.Count) + { + disqualified.Add(index); + continue; + } + + var immediate = (long)plan.Memory[value.MemoryIndex].Offset; + var relative = value.Kind == ScalarValueKind.ScalarBufferWord + ? (immediate + (long)(uint)offset.Payload) & ~3L + : (immediate & ~3L) + (long)((uint)offset.Payload & ~3u); + var candidate = new Candidate(index, -1, highIndex ?? -1, highIndex is null ? (uint)high.Payload : 0, relative, relative); + if (found.TryGetValue(index, out var existing)) + { + if (existing.High != candidate.High || existing.HighConstant != candidate.HighConstant) + { + disqualified.Add(index); + continue; + } + + candidate = existing with + { + MinOffset = Math.Min(existing.MinOffset, relative), + MaxOffset = Math.Max(existing.MaxOffset, relative), + }; + } + + found[index] = candidate; + } + + return found.Values.Where(candidate => !disqualified.Contains(candidate.UserData)); + } + + // The window [start, start + size) of reads a candidate's base covers in this entry. + private bool TryWindow(Entry entry, in Candidate candidate, out ulong start, out ulong size, out ulong addressBase) + { + start = size = addressBase = 0; + if (candidate.Buffer >= 0) + { + if (candidate.Buffer >= entry.Snapshot.Buffers.Length) + return false; + var descriptor = entry.Snapshot.Buffers[candidate.Buffer]; + if (descriptor.Length < 3) + return false; + start = addressBase = descriptor[0] | ((ulong)(descriptor[1] & 0xFFFF) << 32); + var stride = (descriptor[1] >> 16) & 0x3FFF; + size = stride == 0 ? descriptor[2] : (ulong)descriptor[2] * stride; + return true; + } + + // The window starts at the base itself: the read prefetch fetches a pointer's block from + // its base, ahead of the lowest offset a load uses. + addressBase = RawBase(entry.UserData, candidate, entry.UserData[candidate.UserData]); + var lowest = Math.Min(0, candidate.MinOffset); + var low = (long)addressBase + lowest; + if (low < 0) + return false; + start = (ulong)low; + size = (ulong)(candidate.MaxOffset - lowest) + sizeof(uint); + return true; + } + + private static ulong RawBase(IReadOnlyList userData, in Candidate candidate, uint low) + { + var high = candidate.High >= 0 ? userData[candidate.High] : candidate.HighConstant; + return (((ulong)high << 32) | low) & RawAddressMask & ~3UL; + } + + // The rebased words; replaced, never mutated, so a caller can compare references. + public ulong[] Active { get; private set; } = []; + + // Every rebased base dword of the entry equals the word it came from. + public bool AppliesTo(Entry entry) + { + if (Active.Length == 0) + return true; + foreach (var (userData, buffer, _, _, _, _) in _candidates) + { + if (buffer >= 0 && IsUsed(Active, userData) && + (buffer >= entry.Snapshot.Buffers.Length || entry.Snapshot.Buffers[buffer].Length == 0 || + userData >= entry.UserData.Length || entry.Snapshot.Buffers[buffer][0] != entry.UserData[userData])) + return false; + } + + return true; + } + + public ResourceSnapshot Apply(ResourceSnapshot snapshot, IReadOnlyList userData) + { + if (Active.Length == 0) + return snapshot; + uint[][]? buffers = null; + foreach (var (index, buffer, _, _, _, _) in _candidates) + { + if (buffer < 0 || !IsUsed(Active, index) || snapshot.Buffers[buffer][0] == userData[index]) + continue; + buffers ??= (uint[][])snapshot.Buffers.Clone(); + if (ReferenceEquals(buffers[buffer], snapshot.Buffers[buffer])) + buffers[buffer] = (uint[])buffers[buffer].Clone(); + buffers[buffer][0] = userData[index]; + } + + return buffers is null + ? snapshot + : new ResourceSnapshot + { + Buffers = buffers, + Images = snapshot.Images, + Samplers = snapshot.Samplers, + FlattenedResourceTable = snapshot.FlattenedResourceTable, + UserData = snapshot.UserData, + DeviceAddressRanges = snapshot.DeviceAddressRanges, + }; + } + + // Compares a fresh materialization with the latest one that agrees on every word but the + // candidates: the same object drawn again, usually a frame earlier, with new constants. + // Comparing with the plan's previous draw instead finds another object's textures in + // the way and never concludes. + public void Learn(Entry built, + Func? Patches)?>? predict = null) + { + if (_candidates.Length == 0) + return; + var structure = StructureOf(built); + _recent.TryGetValue(structure, out var previous); + if (_recent.Count >= RecentCapacity) + _recent.Clear(); + _recent[structure] = built; + if (previous is null || previous.UserData.Length != built.UserData.Length || + !previous.UsedUserData.AsSpan().SequenceEqual(built.UsedUserData)) + return; + + var differing = new List(); + for (var index = 0; index < built.UserData.Length; index++) + { + if (!IsUsed(built.UsedUserData, index) || previous.UserData[index] == built.UserData[index]) + continue; + if (!IsUsed(_candidateMask, index)) + return; + differing.Add(index); + } + + // A word already rejected explains any mismatch; blaming the others with it would reject + // sound bases that merely change together with it. + if (differing.Count == 0 || differing.Any(index => IsUsed(_rejected, index))) + return; + + // Reads must land where the previous draw's did, moved with a rebased buffer when they + // fall inside its window. Guest bytes that changed between the two draws make the + // comparison say nothing about the words. + var reads = CompareMovedReads(this, previous, built, differing); + if (reads == ReadComparison.Inconclusive) + { + // Changed guest bytes hide whether the words are bases. Predict this draw from the + // previous one as a hit with the words rebased would, moved memory-loaded bases + // included, and judge the words by the prediction instead. + if (predict?.Invoke(previous, Or(Active, MaskOf(differing))) is not { } prediction) + return; + + var agrees = Difference(prediction.Snapshot, previous.Specialization, built.Snapshot, built.Specialization) is null; + if (prediction.Patches is { } patches) + { + // A wrong prediction that moved bases blames those bases, not the words. + ConfirmMemoryBases(patches, agrees); + if (!agrees) + return; + } + else if (!agrees) + { + _rejected = Or(_rejected, MaskOf(differing)); + if (Overlaps(Active, _rejected)) + Active = AndNot(Active, _rejected); + return; + } + + Confirm(differing); + return; + } + + if (reads == ReadComparison.Mismatch || + !SameExceptBases(previous, built, differing, compareTable: reads == ReadComparison.SameBytes)) + { + _rejected = Or(_rejected, MaskOf(differing)); + if (Overlaps(Active, _rejected)) + Active = AndNot(Active, _rejected); + return; + } + + Confirm(differing); + } + + private void Confirm(List differing) + { + var activated = new List(); + foreach (var index in differing) + { + var count = _confirmed.GetValueOrDefault(index) + 1; + _confirmed[index] = count; + if (count >= Confirmations && !IsUsed(_rejected, index) && !IsUsed(Active, index)) + activated.Add(index); + } + + if (activated.Count != 0) + Active = Or(Active, MaskOf(activated)); + } + + // Runs a prediction as though the words in probe were rebased already. + public T WithActive(ulong[] probe, Func run) + { + var saved = Active; + Active = probe; + try + { + return run(); + } + finally + { + Active = saved; + } + } + + // The flattened table is left out when only table-only bytes changed: it holds the words read + // from memory, whose addresses the read comparison already checked. + private bool SameExceptBases(Entry previous, Entry built, List differing, bool compareTable) + { + if (!previous.Specialization.Equals(built.Specialization)) + return false; + + var left = previous.Snapshot; + var right = built.Snapshot; + if (!SameWords(left.Images, right.Images) || !SameWords(left.Samplers, right.Samplers) || + (compareTable + ? !left.FlattenedResourceTable.AsSpan().SequenceEqual(right.FlattenedResourceTable) + : left.FlattenedResourceTable.Length != right.FlattenedResourceTable.Length) || + !left.DeviceAddressRanges.AsSpan().SequenceEqual(right.DeviceAddressRanges) || + left.Buffers.Length != right.Buffers.Length) + return false; + + for (var buffer = 0; buffer < left.Buffers.Length; buffer++) + { + var before = left.Buffers[buffer]; + var after = right.Buffers[buffer]; + if (before.Length != after.Length) + return false; + for (var dword = 0; dword < before.Length; dword++) + { + if (dword == 0 && BaseWord(buffer, differing) is { } index) + { + if (before[0] != previous.UserData[index] || after[0] != built.UserData[index]) + return false; + } + else if (before[dword] != after[dword]) + { + return false; + } + } + } + + return true; + } + + // Every read of the built draw is the previous draw's read, moved by the rebase when it lies + // in a moved buffer's window; bytesDiffer tells whether the guest bytes read there changed. + private enum ReadComparison + { + // A read moved where the words cannot explain it while every byte read alike matched. + Mismatch, + // Guest bytes changed where the descriptors depend on them, or a changed pointer in + // guest memory redirected later reads: the pair says nothing about the words. + Inconclusive, + SameBytes, + // Only words the flattened table alone reads changed: per-draw constants. + TableBytesDiffer, + } + + private static ReadComparison CompareMovedReads(AddressRebase rebase, Entry previous, Entry built, List differing) + { + var matched = previous.RangeAddresses.Length == built.RangeAddresses.Length; + var bytesDiffer = false; + var descriptorBytesDiffer = false; + var common = Math.Min(previous.RangeAddresses.Length, built.RangeAddresses.Length); + for (var range = 0; range < common; range++) + { + var length = previous.RangeLengths[range]; + if (rebase.Move(previous, built.UserData, previous.RangeAddresses[range], length, differing) != built.RangeAddresses[range] || + length != built.RangeLengths[range] || previous.RangeClean[range] != built.RangeClean[range]) + { + matched = false; + continue; + } + + for (var offset = 0; offset < length; offset += sizeof(uint)) + { + var before = previous.RangeOffsets[range] + offset; + var after = built.RangeOffsets[range] + offset; + if (previous.Bytes.AsSpan(before, sizeof(uint)).SequenceEqual(built.Bytes.AsSpan(after, sizeof(uint)))) + continue; + bytesDiffer = true; + descriptorBytesDiffer |= !previous.WordTableOnly[before / sizeof(uint)] || !built.WordTableOnly[after / sizeof(uint)]; + } + } + + if (!matched) + return bytesDiffer ? ReadComparison.Inconclusive : ReadComparison.Mismatch; + if (descriptorBytesDiffer || (bytesDiffer && !(previous.TableRefreshable && built.TableRefreshable))) + return ReadComparison.Inconclusive; + return bytesDiffer ? ReadComparison.TableBytesDiffer : ReadComparison.SameBytes; + } + + // Where a read the entry recorded lies for a draw with this user data: a read inside the + // window of a candidate whose base word moved moves with it. Only the words in moving count + // (the differing words, or the rebased ones on a hit). + private ulong Move(Entry entry, IReadOnlyList userData, ulong address, int length, IReadOnlyCollection? moving) + { + foreach (var candidate in _candidates) + { + var index = candidate.UserData; + if (moving is null ? !IsUsed(Active, index) : !moving.Contains(index)) + continue; + if (userData[index] == entry.UserData[index] || !TryWindow(entry, candidate, out var start, out var size, out var addressBase)) + continue; + // Only a range wholly inside the window moves: a range that merely overlaps it can hold + // another block the title allocated next to it, which must not move. + if (address < start || address - start + (ulong)length > size) + continue; + var movedBase = candidate.Buffer >= 0 + ? (addressBase & ~0xFFFF_FFFFUL) | userData[index] + : RawBase(entry.UserData, candidate, userData[index]); + return address - addressBase + movedBase; + } + + return address; + } + + // A hit reads the entry's recorded ranges where they lie for this draw. + public ulong MoveRead(Entry entry, IReadOnlyList userData, ulong address, int length) => + Active.Length == 0 ? address : Move(entry, userData, address, length, null); + + // True when a rebased base of the entry differs from this draw's word. + public bool Moves(Entry entry, IReadOnlyList userData) + { + foreach (var candidate in _candidates) + if (IsUsed(Active, candidate.UserData) && entry.UserData[candidate.UserData] != userData[candidate.UserData]) + return true; + return false; + } + + private int? BaseWord(int buffer, List differing) + { + foreach (var candidate in _candidates) + if (candidate.Buffer == buffer && differing.Contains(candidate.UserData)) + return candidate.UserData; + return null; + } + + private static bool SameWords(uint[][] left, uint[][] right) + { + if (left.Length != right.Length) + return false; + for (var index = 0; index < left.Length; index++) + if (!left[index].AsSpan().SequenceEqual(right[index])) + return false; + return true; + } + + private static ulong[] MaskOf(IEnumerable indices) + { + var mask = Array.Empty(); + foreach (var index in indices) + { + if (mask.Length <= index >> 6) + Array.Resize(ref mask, (index >> 6) + 1); + mask[index >> 6] |= 1UL << (index & 63); + } + + return mask; + } + + private static ulong[] Or(ulong[] left, ulong[] right) + { + var result = new ulong[Math.Max(left.Length, right.Length)]; + for (var index = 0; index < result.Length; index++) + result[index] = (index < left.Length ? left[index] : 0) | (index < right.Length ? right[index] : 0); + return result; + } + + private static ulong[] AndNot(ulong[] left, ulong[] right) + { + var result = (ulong[])left.Clone(); + for (var index = 0; index < result.Length && index < right.Length; index++) + result[index] &= ~right[index]; + return result.All(word => word == 0) ? [] : result; + } + + private static bool Overlaps(ulong[] left, ulong[] right) + { + for (var index = 0; index < left.Length && index < right.Length; index++) + if ((left[index] & right[index]) != 0) + return true; + return false; + } + } public ResourceMaterializationCache(int generationCapacity = 16384) { @@ -33,10 +791,15 @@ public sealed class ResourceMaterializationCache private static long _totalHits; private static long _totalMisses; private static long _totalUncacheable; - private const int MaxVariants = 4; + private const int MaxVariants = 8; private static long _totalStale; private static long _totalStaleUnreadable; private static long _totalRefreshes; + private static long _totalNotFound; + private static readonly long[] _refreshRefusals = new long[6]; + private static readonly string[] RefreshRefusalNames = ["not_refreshable", "unreadable", "descriptor_words", "unchanged", "table_eval", "rebased"]; + private int _lastRefreshRefusal; + [ThreadStatic] private static bool _readingTable; @@ -61,9 +824,105 @@ public sealed class ResourceMaterializationCache var stale = Interlocked.Exchange(ref _totalStale, 0); var staleUnreadable = Interlocked.Exchange(ref _totalStaleUnreadable, 0); var refreshes = Interlocked.Exchange(ref _totalRefreshes, 0); + var notFound = Interlocked.Exchange(ref _totalNotFound, 0); + var refusals = string.Join(",", RefreshRefusalNames.Select((name, index) => FormattableString.Invariant($"{name}:{Interlocked.Exchange(ref _refreshRefusals[index], 0)}"))); var total = hits + misses; return FormattableString.Invariant( - $"[PERF][RESOURCE_CACHE] hits={hits} misses={misses} stale={stale} stale_unreadable={staleUnreadable} refreshes={refreshes} uncacheable={uncacheable} hit_rate={(total == 0 ? 0 : hits * 100.0 / total):F1}% {RawReadPrefetch.TakeReport()}"); + $"[PERF][RESOURCE_CACHE] hits={hits} misses={misses} not_found={notFound} stale={stale} stale_unreadable={staleUnreadable} refreshes={refreshes} refused={refusals} uncacheable={uncacheable} hit_rate={(total == 0 ? 0 : hits * 100.0 / total):F1}% {RawReadPrefetch.TakeReport()}") + + TakeStaleDifferenceReport(); + } + + private static readonly bool StaleDiffEnabled = Environment.GetEnvironmentVariable("SHARPEMU_PROFILE_PERFORMANCE") == "1"; + private static readonly System.Collections.Concurrent.ConcurrentDictionary _staleDifferences = new(StringComparer.Ordinal); + + private static void NoteStaleDifference(Entry stale, Entry built) + { + var parts = new SortedSet(StringComparer.Ordinal); + if (!stale.Specialization.Equals(built.Specialization)) + parts.Add("spec"); + DifferingWords("buf", stale.Snapshot.Buffers, built.Snapshot.Buffers, parts); + DifferingWords("img", stale.Snapshot.Images, built.Snapshot.Images, parts); + DifferingWords("smp", stale.Snapshot.Samplers, built.Snapshot.Samplers, parts); + if (!stale.Snapshot.FlattenedResourceTable.AsSpan().SequenceEqual(built.Snapshot.FlattenedResourceTable)) + parts.Add("table"); + if (!stale.Snapshot.DeviceAddressRanges.AsSpan().SequenceEqual(built.Snapshot.DeviceAddressRanges)) + parts.Add("ranges"); + if (stale.Bytes.Length != built.Bytes.Length || !stale.RangeAddresses.AsSpan().SequenceEqual(built.RangeAddresses)) + parts.Add("reads_moved"); + var signature = parts.Count == 0 ? "none" : string.Join("+", parts); + _staleDifferences.AddOrUpdate(signature, 1, static (_, count) => count + 1); + } + + private static void DifferingWords(string name, uint[][] left, uint[][] right, SortedSet parts) + { + if (left.Length != right.Length) + { + parts.Add($"{name}.count"); + return; + } + + for (var index = 0; index < left.Length; index++) + { + if (left[index].Length != right[index].Length) + { + parts.Add($"{name}.length"); + continue; + } + + for (var word = 0; word < left[index].Length; word++) + if (left[index][word] != right[index][word]) + parts.Add($"{name}.w{word}"); + } + } + + private static string TakeStaleDifferenceReport() + { + if (_staleDifferences.IsEmpty) + return string.Empty; + var ranked = _staleDifferences.ToArray().OrderByDescending(pair => pair.Value).Take(10).ToArray(); + _staleDifferences.Clear(); + return Environment.NewLine + string.Join(Environment.NewLine, + ranked.Select(pair => FormattableString.Invariant($"[PERF][RESOURCE_STALE_DIFF] n={pair.Value} changed={pair.Key}"))); + } + + // The first part a cached result differs from a fresh one in, or null when they are the same. + private static string? Difference(ResourceSnapshot cached, ResourceSpecialization cachedSpecialization, + ResourceSnapshot fresh, ResourceSpecialization freshSpecialization) + { + static string? Words(string name, uint[][] left, uint[][] right) + { + if (left.Length != right.Length) + return $"{name}.count {left.Length}!={right.Length}"; + for (var index = 0; index < left.Length; index++) + { + if (left[index].Length != right[index].Length) + return $"{name}[{index}].length"; + for (var word = 0; word < left[index].Length; word++) + if (left[index][word] != right[index][word]) + return $"{name}[{index}][{word}] cached=0x{left[index][word]:X} fresh=0x{right[index][word]:X}"; + } + + return null; + } + + if (!cachedSpecialization.Equals(freshSpecialization)) + return "specialization"; + if (Words("buffer", cached.Buffers, fresh.Buffers) is { } buffers) + return buffers; + if (Words("image", cached.Images, fresh.Images) is { } images) + return images; + if (Words("sampler", cached.Samplers, fresh.Samplers) is { } samplers) + return samplers; + if (cached.FlattenedResourceTable.Length != fresh.FlattenedResourceTable.Length) + return "table.length"; + for (var slot = 0; slot < cached.FlattenedResourceTable.Length; slot++) + if (cached.FlattenedResourceTable[slot] != fresh.FlattenedResourceTable[slot]) + return $"table[{slot}] cached=0x{cached.FlattenedResourceTable[slot]:X} fresh=0x{fresh.FlattenedResourceTable[slot]:X}"; + if (!cached.DeviceAddressRanges.AsSpan().SequenceEqual(fresh.DeviceAddressRanges)) + return "ranges"; + if (!cached.UserData.AsSpan().SequenceEqual(fresh.UserData)) + return "user_data"; + return null; } public bool Materialize( @@ -74,17 +933,23 @@ public sealed class ResourceMaterializationCache ref ResourceSpecialization specialization, out ResourceMaterializationFailure failure) { - var key = KeyOf(plan, inputs); - var found = TryFind(key, plan, inputs, out var cached); + var drawKey = DrawKeyOf(plan, inputs); + _learnedUserData.TryGetValue(drawKey, out var learned); + var rebase = RebaseOf(drawKey, plan); + var key = KeyOf(drawKey, inputs.UserData, learned, rebase.Active); + var found = TryFind(key, plan, inputs, rebase.Active, out var cached); + Entry? stale = null; + (List<(int Buffer, uint Word)> Patches, ResourceSnapshot Snapshot, Entry From)? learning = null; if (found) { var unreadable = false; Entry? previous = null; for (var variant = cached; variant is not null; previous = variant, variant = variant.Next) { - if (!variant.Matches(plan, inputs)) + if (!variant.Matches(plan, inputs, rebase.Active) || !rebase.AppliesTo(variant)) continue; - if (Validate(variant, residentReader, out var variantUnreadable)) + stale ??= variant; + if (Validate(variant, residentReader, rebase, inputs.UserData, out var variantUnreadable)) { if (previous is not null) { @@ -95,7 +960,7 @@ public sealed class ResourceMaterializationCache Hits++; Interlocked.Increment(ref _totalHits); - snapshot = variant.Snapshot; + snapshot = rebase.Apply(WithCurrentUserData(variant.Snapshot, inputs.UserData), inputs.UserData); specialization = variant.Specialization; failure = default; return true; @@ -104,20 +969,70 @@ public sealed class ResourceMaterializationCache unreadable |= variantUnreadable; } - if (TryRefreshTable(key, cached, plan, inputs, residentReader, out var refreshed)) + // Moved bases first: a changed base word is a descriptor word, which a table refresh + // would read in full only to refuse. + ResourceSnapshot rebased = null!; + List<(int Buffer, uint Word)> patches = null!; + // Each variant is tried: a key's variants are often the same draw with different + // descriptors (materials), and only the one this draw binds can be rebased. + Entry? refreshedFrom = null; + _lastRefreshRefusal = 0; + Entry? beforeRefreshed = null; + if (stale is not null && rebase.HasLearnableMemoryBases) + { + for (Entry? previousVariant = null, variant = cached; variant is not null; previousVariant = variant, variant = variant.Next) + { + if (!variant.Matches(plan, inputs, rebase.Active) || !rebase.AppliesTo(variant)) + continue; + if (TryRebaseRefresh(variant, plan, inputs, residentReader, rebase, out rebased, out patches)) + { + refreshedFrom = variant; + beforeRefreshed = previousVariant; + break; + } + } + } + + if (refreshedFrom is not null) + { + if (rebase.AreActiveMemoryBases(patches)) + { + if (beforeRefreshed is not null) + { + beforeRefreshed.Next = refreshedFrom.Next; + refreshedFrom.Next = cached; + Store(key, refreshedFrom); + } + + Hits++; + Interlocked.Increment(ref _totalHits); + snapshot = rebased; + specialization = refreshedFrom.Specialization; + failure = default; + return true; + } + + learning = (patches, rebased, refreshedFrom); + } + else if (rebase.AppliesTo(cached) && TryRefreshTable(key, cached, plan, inputs, residentReader, rebase, out var refreshed)) { TableRefreshes++; Interlocked.Increment(ref _totalRefreshes); - snapshot = refreshed.Snapshot; + snapshot = rebase.Apply(WithCurrentUserData(refreshed.Snapshot, inputs.UserData), inputs.UserData); specialization = refreshed.Specialization; failure = default; return true; } Interlocked.Increment(ref _totalStale); + Interlocked.Increment(ref _refreshRefusals[refreshedFrom is not null ? 5 : _lastRefreshRefusal]); if (unreadable) Interlocked.Increment(ref _totalStaleUnreadable); } + else + { + Interlocked.Increment(ref _totalNotFound); + } Misses++; Interlocked.Increment(ref _totalMisses); @@ -126,7 +1041,7 @@ public sealed class ResourceMaterializationCache { var recording = new ResourceRuntimeInputs { - UserData = inputs.UserData, + UserData = recorder.WrapUserData(inputs.UserData), ShaderBase = inputs.ShaderBase, ReadMemory = recorder.Wrap(inputs.ReadMemory, clean: false), ReadCleanMemory = recorder.Wrap(inputs.ReadCleanMemory, clean: true), @@ -145,7 +1060,25 @@ public sealed class ResourceMaterializationCache return true; } - var built = recorder.Build(plan, inputs, snapshot, specialization); + var built = recorder.Build(plan, inputs, snapshot, specialization, rebase.MemoryBases); + if (StaleDiffEnabled && stale is not null && learning is null) + NoteStaleDifference(stale, built); + if (learning is { } pending) + { + var agrees = Difference(pending.Snapshot, pending.From.Specialization, built.Snapshot, built.Specialization) is null; + rebase.ConfirmMemoryBases(pending.Patches, agrees); + } + + var activeBefore = rebase.Active; + Learn(rebase, built, plan, inputs, residentReader); + var widened = Union(learned, built.UsedUserData); + if (!ReferenceEquals(widened, learned) || !ReferenceEquals(activeBefore, rebase.Active)) + { + _learnedUserData[drawKey] = widened; + key = KeyOf(drawKey, inputs.UserData, widened, rebase.Active); + found = false; + } + if (found) { built.Next = cached; @@ -175,18 +1108,34 @@ public sealed class ResourceMaterializationCache // flattened table, only the table is evaluated again. It is refused when the plan's // specialization reads or extends the table (indirect or candidate tables), when the table // now reads a word the entry did not validate, or when a word moved between the two reads. + // + // With rebased words the entry's ranges are read where they lie for this draw. The refreshed + // entry keeps the entry's own addresses and base words with the bytes read there, so a later + // draw moves its reads the same way and compares against those bytes. private bool TryRefreshTable(ulong key, Entry cached, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, - ResidentGuestBytesReader residentReader, out Entry refreshed) + ResidentGuestBytesReader residentReader, AddressRebase rebase, out Entry refreshed) { refreshed = null!; + _lastRefreshRefusal = 0; if (!cached.TableRefreshable) return false; var current = new byte[cached.Bytes.Length]; + var moved = cached.RangeAddresses; + if (rebase.Moves(cached, inputs.UserData)) + { + moved = new ulong[cached.RangeAddresses.Length]; + for (var index = 0; index < moved.Length; index++) + moved[index] = rebase.MoveRead(cached, inputs.UserData, cached.RangeAddresses[index], cached.RangeLengths[index]); + } + for (var index = 0; index < cached.RangeAddresses.Length; index++) { - if (!residentReader(cached.RangeAddresses[index], current.AsSpan(cached.RangeOffsets[index], cached.RangeLengths[index]), cached.RangeClean[index])) + if (!residentReader(moved[index], current.AsSpan(cached.RangeOffsets[index], cached.RangeLengths[index]), cached.RangeClean[index])) + { + _lastRefreshRefusal = 1; return false; + } } var changed = false; @@ -195,12 +1144,21 @@ public sealed class ResourceMaterializationCache if (current.AsSpan(offset, sizeof(uint)).SequenceEqual(cached.Bytes.AsSpan(offset, sizeof(uint)))) continue; if (!cached.WordTableOnly[offset / sizeof(uint)]) + { + _lastRefreshRefusal = 2; return false; + } + changed = true; } if (!changed) + { + _lastRefreshRefusal = 3; return false; + } + + _lastRefreshRefusal = 4; var recorder = TakeRecorder(); try @@ -233,8 +1191,10 @@ public sealed class ResourceMaterializationCache foreach (var (address, word, _, _) in recorder.Reads) { - if (!TryFindWord(cached, address, out var offset) || - System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset, sizeof(uint))) != word) + var found = ReferenceEquals(moved, cached.RangeAddresses) + ? TryFindWord(cached, address, out var offset) + : TryFindMovedWord(cached, moved, address, out offset); + if (!found || System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset, sizeof(uint))) != word) return false; } @@ -247,6 +1207,7 @@ public sealed class ResourceMaterializationCache { Plan = cached.Plan, UserData = cached.UserData, + UsedUserData = cached.UsedUserData, ShaderBase = cached.ShaderBase, ComputeState = cached.ComputeState, RangeAddresses = cached.RangeAddresses, @@ -254,6 +1215,7 @@ public sealed class ResourceMaterializationCache RangeLengths = cached.RangeLengths, RangeClean = cached.RangeClean, WordTableOnly = cached.WordTableOnly, + BaseWordBuffers = cached.BaseWordBuffers, TableRefreshable = true, Bytes = current, Snapshot = new ResourceSnapshot @@ -290,6 +1252,26 @@ public sealed class ResourceMaterializationCache _spareRecorder = recorder; } + // The byte offset of a dword read at address when the entry's ranges lie at moved addresses. + private static bool TryFindMovedWord(Entry entry, ulong[] moved, ulong address, out int offset) + { + // moved may be a reused buffer longer than the entry's ranges. + for (var index = 0; index < entry.RangeAddresses.Length; index++) + { + if (address < moved[index]) + continue; + var delta = address - moved[index]; + if (delta % sizeof(uint) == 0 && delta < (ulong)entry.RangeLengths[index]) + { + offset = entry.RangeOffsets[index] + (int)delta; + return true; + } + } + + offset = 0; + return false; + } + // The byte offset of a recorded dword in the entry's bytes, found by its address. private static bool TryFindWord(Entry entry, ulong address, out int offset) { @@ -319,29 +1301,85 @@ public sealed class ResourceMaterializationCache return true; } - private static ulong KeyOf(ShaderResourcePlan plan, ResourceRuntimeInputs inputs) + private static ulong DrawKeyOf(ShaderResourcePlan plan, ResourceRuntimeInputs inputs) { var hash = new HashCode(); hash.Add(RuntimeHelpers.GetHashCode(plan)); hash.Add(inputs.ShaderBase); hash.Add(inputs.ComputeState); - var userData = inputs.UserData; - hash.Add(userData.Count); - for (var index = 0; index < userData.Count; index++) - hash.Add(userData[index]); - var low = (uint)hash.ToHashCode(); - // A second, independent mix keeps accidental collisions out of the 64-bit key. - var high = 0x9E3779B9u; - for (var index = 0; index < userData.Count; index++) - high = (high ^ userData[index]) * 0x01000193u; + hash.Add(inputs.UserData.Count); + return (ulong)(uint)hash.ToHashCode() | ((ulong)(uint)RuntimeHelpers.GetHashCode(plan) << 32); + } + + // Only the words the draw's shader has been seen to read take part, less the rebased ones: the + // key is an index, and Entry.Matches decides whether an entry is reusable. + private static ulong KeyOf(ulong drawKey, IReadOnlyList userData, ulong[]? learned, ulong[]? rebased = null) + { + var low = (uint)drawKey; + var high = (uint)(drawKey >> 32) ^ 0x9E3779B9u; + if (learned is not null) + { + for (var index = 0; index < userData.Count; index++) + { + if (!IsUsed(learned, index) || (rebased is not null && IsUsed(rebased, index))) + continue; + var word = userData[index]; + low = (low ^ word) * 0x01000193u; + high = ((high + word) * 0x85EBCA6Bu) ^ (high >> 13); + } + } + return ((ulong)high << 32) | low; } - private bool TryFind(ulong key, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, out Entry entry) + private static bool IsUsed(ulong[] mask, int index) => + (uint)(index >> 6) < (uint)mask.Length && (mask[index >> 6] & (1UL << (index & 63))) != 0; + + // The same array when nothing new was learned, so the caller can tell. + private static ulong[] Union(ulong[]? learned, ulong[] used) + { + if (learned is null) + return used; + var changed = learned.Length < used.Length; + for (var index = 0; !changed && index < used.Length; index++) + changed = (used[index] & ~learned[index]) != 0; + if (!changed) + return learned; + var merged = new ulong[Math.Max(learned.Length, used.Length)]; + for (var index = 0; index < merged.Length; index++) + merged[index] = (index < learned.Length ? learned[index] : 0) | (index < used.Length ? used[index] : 0); + return merged; + } + + // A cached snapshot is shared by every draw that hits it, so it keeps the user data of the draw + // that built it; a draw whose unread words differ gets its own copy that carries its own. + private static ResourceSnapshot WithCurrentUserData(ResourceSnapshot cached, IReadOnlyList userData) + { + var words = cached.UserData; + var same = words.Length == userData.Count; + for (var index = 0; same && index < words.Length; index++) + same = words[index] == userData[index]; + if (same) + return cached; + var current = new uint[userData.Count]; + for (var index = 0; index < current.Length; index++) + current[index] = userData[index]; + return new ResourceSnapshot + { + Buffers = cached.Buffers, + Images = cached.Images, + Samplers = cached.Samplers, + FlattenedResourceTable = cached.FlattenedResourceTable, + UserData = current, + DeviceAddressRanges = cached.DeviceAddressRanges, + }; + } + + private bool TryFind(ulong key, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, ulong[] rebased, out Entry entry) { if (_young.TryGetValue(key, out entry!)) - return entry.Matches(plan, inputs); - if (!_old.TryGetValue(key, out entry!) || !entry.Matches(plan, inputs)) + return entry.Matches(plan, inputs, rebased); + if (!_old.TryGetValue(key, out entry!) || !entry.Matches(plan, inputs, rebased)) return false; _old.Remove(key); Store(key, entry); @@ -359,16 +1397,390 @@ public sealed class ResourceMaterializationCache _young[key] = entry; } - private bool Validate(Entry entry, ResidentGuestBytesReader residentReader, out bool unreadable) + // What a hit on the previous entry would give this draw with the words in probe rebased: its + // reads moved and its bytes unchanged, or its memory-loaded bases moved as well. Null when + // neither path applies, which says nothing about the words. + private (ResourceSnapshot Snapshot, List<(int Buffer, uint Word)>? Patches)? Predict(Entry previous, ShaderResourcePlan plan, + ResourceRuntimeInputs inputs, ResidentGuestBytesReader residentReader, AddressRebase rebase, ulong[] probe) => + rebase.WithActive<(ResourceSnapshot, List<(int Buffer, uint Word)>?)?>(probe, () => + { + if (!previous.Matches(plan, inputs, probe) || !rebase.AppliesTo(previous)) + return null; + if (Validate(previous, residentReader, rebase, inputs.UserData, out _)) + return (rebase.Apply(WithCurrentUserData(previous.Snapshot, inputs.UserData), inputs.UserData), null); + if (rebase.HasLearnableMemoryBases && + TryRebaseRefresh(previous, plan, inputs, residentReader, rebase, out var snapshot, out var patches)) + return (snapshot, patches); + return null; + }); + + // Its own method: a lambda in MaterializeCore that captured the parameters made every call, + // hits included, allocate the closure on entry. + private void Learn(AddressRebase rebase, Entry built, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, + ResidentGuestBytesReader residentReader) => + rebase.Learn(built, (previous, probe) => Predict(previous, plan, inputs, residentReader, rebase, probe)); + + // The table for this draw without evaluating it: a slot that reads a memory-based buffer at a + // constant offset takes the word now at that offset, which the moved window has just read; the + // other slots read nothing that changed, so they keep the entry's words, except a base slot, + // which the patch has already moved. current holds this draw's words in the entry's layout. + // The table is patched in place: the caller passes a copy it owns. + private static bool TryCopyTable(Entry entry, ShaderResourcePlan plan, AddressRebase rebase, byte[] current, uint[] table) + { + var sources = entry.TableCopySources ??= TableCopySourcesOf(entry, plan, rebase, table.Length); + if (sources.Length == 0) + return false; + for (var slot = 0; slot < sources.Length; slot++) + { + var offset = sources[slot]; + if (offset >= 0) + table[(int)plan.TableReads[slot].FlatOffset] = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset)); + } + + return true; + } + + // A table whose slots are all direct loads or reads through memory-based buffers. + private static bool CanCopyDirect(AddressRebase rebase) + { + if (rebase.DirectSlots is not null) + return true; + return false; + } + + + // Every slot takes the dword it loads for this draw. A direct slot's address is evaluated from + // the draw's own user data, as the table evaluation would, and its dword is taken from the range + // this draw read around it; a slot through a memory-based buffer copies as in TryCopyTable. + // Nothing is written unless every slot is placed. + private static bool TryCopyDirectTable(Entry entry, ShaderResourcePlan plan, AddressRebase rebase, byte[] current, + ulong[] moved, IReadOnlyList userData, uint[] table) + { + var through = entry.TableCopySources ??= TableCopySourcesOf(entry, plan, rebase, table.Length); + var direct = rebase.DirectSlots!; + if (through.Length != direct.Length) + return Unplaced(); + Span sources = direct.Length <= 256 ? stackalloc int[direct.Length] : new int[direct.Length]; + for (var slot = 0; slot < direct.Length; slot++) + { + if (plan.TableReads[slot].FlatOffset >= table.Length) + return Unplaced(); + if (rebase.TableSlotBuffers[slot] >= 0) + { + if (through[slot] < 0) + return Unplaced(); + sources[slot] = through[slot]; + continue; + } + + var load = direct[slot]; + if (load.Low >= userData.Count || load.High >= userData.Count) + return Unplaced(); + var high = load.High >= 0 ? userData[load.High] : load.HighConstant; + var baseAddress = ((high << 32) | userData[load.Low]) & 0x0000_FFFF_FFFF_FFFFul; + if (RuntimeValueEvaluator.ResolveRawAddress(ScalarValueKind.ScalarAddressWord, baseAddress, high, 0, + load.Immediate, load.Offset, out var address) != RuntimeValueEvaluator.RawAddress.Read || + !TryFindMovedWord(entry, moved, address, out sources[slot])) + return Unplaced(); + } + + for (var slot = 0; slot < sources.Length; slot++) + table[(int)plan.TableReads[slot].FlatOffset] = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(sources[slot])); + return true; + + static bool Unplaced() + { + return false; + } + } + + // Per table slot: the byte offset in the entry's words of the word the slot copies, or -1 for a + // slot that keeps its word. Empty when some slot cannot be placed. + private static int[] TableCopySourcesOf(Entry entry, ShaderResourcePlan plan, AddressRebase rebase, int tableLength) + { + var sources = new int[plan.TableReads.Count]; + for (var slot = 0; slot < sources.Length; slot++) + { + sources[slot] = -1; + var buffer = rebase.TableSlotBuffers[slot]; + if (buffer < 0) + continue; + var descriptor = entry.Snapshot.Buffers[buffer]; + if (!AddressRebase.TryMovedWindow(descriptor, descriptor[0], out var start, out var size) || + rebase.TableSlotOffsets[slot] < 0 || (ulong)rebase.TableSlotOffsets[slot] + sizeof(uint) > size || + !TryFindWord(entry, start + (ulong)rebase.TableSlotOffsets[slot], out var offset) || + plan.TableReads[slot].FlatOffset >= tableLength) + return []; + sources[slot] = offset; + } + + return sources; + } + + // Where each memory-loaded base word of the entry lies: its range and byte offset in it. + private static (int Range, int Offset)[] BaseWordProbesOf(Entry entry) + { + var probes = new List<(int Range, int Offset)>(); + for (var range = 0; range < entry.RangeAddresses.Length; range++) + { + for (var offset = 0; offset < entry.RangeLengths[range]; offset += sizeof(uint)) + { + if (entry.BaseWordBuffers![(entry.RangeOffsets[range] + offset) / sizeof(uint)] >= 0) + probes.Add((range, offset)); + } + } + + return probes.ToArray(); + } + + private byte[] _rebaseCurrent = []; + private ulong[] _rebaseMoved = []; + private readonly List<(int Buffer, uint Word)> _rebaseBases = new(); + + // Per recorded range: the memory-based buffer whose window wholly holds it, or NoBase. Such a + // range is read only once the buffer's base is known: it moves with the base. + private static int[] RangeWindowsOf(Entry entry, AddressRebase rebase) + { + var windows = new int[entry.RangeAddresses.Length]; + windows.AsSpan().Fill(NoBase); + for (var index = 0; index < windows.Length; index++) + { + foreach (var buffer in rebase.MemoryBases) + { + var descriptor = entry.Snapshot.Buffers[buffer]; + if (descriptor.Length >= 3 && AddressRebase.TryMovedWindow(descriptor, descriptor[0], out var start, out var size) && + entry.RangeAddresses[index] >= start && entry.RangeAddresses[index] - start + (ulong)entry.RangeLengths[index] <= size) + { + windows[index] = buffer; + break; + } + } + } + + return windows; + } + + // A stale entry whose only changed words are memory-loaded buffer bases, and constants only the + // table reads: the bases are patched into a copy of the snapshot and the table is evaluated + // again for this draw, which reads the constants where the moved bases now point. Every read of + // that evaluation is a word the entry validated unchanged, or lies in the window a patched base + // now covers. The descriptors and specialization are the entry's; learning checks that. + private bool TryRebaseRefresh(Entry entry, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, + ResidentGuestBytesReader residentReader, AddressRebase rebase, out ResourceSnapshot snapshot, + out List<(int Buffer, uint Word)> patches) + { + snapshot = null!; + patches = null!; + if (entry.BaseWordBuffers is not { } map) + return false; + var userData = inputs.UserData; + // The words read here are compared and copied, never kept, so the buffers are reused. + if (_rebaseCurrent.Length < entry.Bytes.Length) + _rebaseCurrent = new byte[Math.Max(entry.Bytes.Length, _rebaseCurrent.Length * 2)]; + if (_rebaseMoved.Length < entry.RangeAddresses.Length) + _rebaseMoved = new ulong[Math.Max(entry.RangeAddresses.Length, _rebaseMoved.Length * 2)]; + var current = _rebaseCurrent; + var moved = _rebaseMoved; + var windowed = entry.RangeWindows ??= RangeWindowsOf(entry, rebase); + for (var index = 0; index < entry.RangeAddresses.Length; index++) + { + if (windowed[index] != NoBase) + continue; + moved[index] = rebase.MoveRead(entry, userData, entry.RangeAddresses[index], entry.RangeLengths[index]); + if (!residentReader(moved[index], current.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]), entry.RangeClean[index])) + return false; + } + + // The new base of each buffer whose base word changed, from the ranges read so far. A base + // word inside a moving window would need its own base first; such an entry is not rebased. + var rangeCount = entry.RangeAddresses.Length; + var bases = _rebaseBases; + bases.Clear(); + foreach (var (range, rangeOffset) in entry.BaseWordProbes ??= BaseWordProbesOf(entry)) + { + if (windowed[range] != NoBase) + return false; + var offset = entry.RangeOffsets[range] + rangeOffset; + var now = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset)); + if (now != System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(entry.Bytes.AsSpan(offset))) + bases.Add((map[offset / sizeof(uint)], now)); + } + + for (var index = 0; index < rangeCount; index++) + { + var buffer = windowed[index]; + if (buffer == NoBase) + continue; + var oldBase = entry.Snapshot.Buffers[buffer][0]; + var newBase = oldBase; + foreach (var (changed, word) in bases) + if (changed == buffer) + newBase = word; + moved[index] = entry.RangeAddresses[index] - oldBase + newBase; + if (!residentReader(moved[index], current.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]), entry.RangeClean[index])) + return false; + } + + List<(int Buffer, uint Word)>? found = null; + // A changed constant outside the moved windows is read by a slot the copy cannot place. + var constantsOutsideWindows = false; + for (var index = 0; index < rangeCount; index++) + { + // Most ranges did not change at all; only a changed one is looked at word by word. + if (current.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]) + .SequenceEqual(entry.Bytes.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]))) + continue; + for (var offset = entry.RangeOffsets[index]; offset < entry.RangeOffsets[index] + entry.RangeLengths[index]; offset += sizeof(uint)) + { + var word = offset / sizeof(uint); + var now = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset)); + if (now == System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(entry.Bytes.AsSpan(offset))) + continue; + var buffer = map[word]; + if (buffer >= 0) + { + if (rebase.IsRejectedMemoryBase(buffer)) + return false; + (found ??= new()).Add((buffer, now)); + } + else if (!entry.WordTableOnly[word]) + return false; + else + constantsOutsideWindows |= windowed[index] == NoBase; + } + } + + if (found is null) + return false; + var patched = rebase.PatchMemoryBases(rebase.Apply(WithCurrentUserData(entry.Snapshot, userData), userData), found); + // A changed constant outside the windows is still copied when every slot is a direct load: + // each slot's word is then the dword at its own address, which this draw has just read. + var copyDirect = constantsOutsideWindows && CanCopyDirect(rebase); + if (plan.TableReads.Count != 0 && (!constantsOutsideWindows || copyDirect) && !rebase.TableSlotsThroughBases) + { + // The patch cloned the table when it moved a base slot; otherwise it is the entry's. + var table = ReferenceEquals(patched.FlattenedResourceTable, entry.Snapshot.FlattenedResourceTable) + ? (uint[])patched.FlattenedResourceTable.Clone() + : patched.FlattenedResourceTable; + // The copy writes nothing unless every slot can be placed. + if (copyDirect + ? TryCopyDirectTable(entry, plan, rebase, current, moved, userData, table) + : TryCopyTable(entry, plan, rebase, current, table)) + { + snapshot = new ResourceSnapshot + { + Buffers = patched.Buffers, + Images = patched.Images, + Samplers = patched.Samplers, + FlattenedResourceTable = table, + UserData = patched.UserData, + DeviceAddressRanges = patched.DeviceAddressRanges, + }; + patches = found; + return true; + } + } + + if (plan.TableReads.Count != 0) + { + if (!entry.TableRefreshable) + return false; + var windows = new List<(ulong Start, ulong Size)>(found.Count); + foreach (var (buffer, baseWord) in found) + if (AddressRebase.TryMovedWindow(patched.Buffers[buffer], baseWord, out var start, out var size)) + windows.Add((start, size)); + + var recorder = TakeRecorder(); + try + { + var recording = new ResourceRuntimeInputs + { + UserData = userData, + ShaderBase = inputs.ShaderBase, + ReadMemory = recorder.Wrap(inputs.ReadMemory, clean: false), + ReadCleanMemory = recorder.Wrap(inputs.ReadCleanMemory, clean: true), + ReadResidentMemory = recorder.WrapResident(inputs.ReadResidentMemory), + ReadsClean = inputs.ReadsClean, + ComputeState = inputs.ComputeState, + }; + var cachedTable = entry.Snapshot.FlattenedResourceTable; + ReadingTable = true; + bool evaluated; + uint[] table; + try + { + evaluated = ResourceMaterializer.TryEvaluateTable(plan, recording, out table); + } + finally + { + ReadingTable = false; + } + + if (!evaluated || recorder.Failed || table.Length != cachedTable.Length) + return false; + + foreach (var (address, word, _, _) in recorder.Reads) + { + var inWindow = false; + foreach (var (start, size) in windows) + inWindow |= address >= start && address - start < size; + if (inWindow) + continue; + if (!TryFindMovedWord(entry, moved, address, out var offset) || + System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset, sizeof(uint))) != word) + return false; + } + + // The written device-address slots come from reads validated unchanged above. + foreach (var slot in plan.WrittenRangeSlotByHandle.Values) + cachedTable.AsSpan((int)slot, ShaderResourcePlan.WrittenRangeDwordCount).CopyTo(table.AsSpan((int)slot)); + patched = new ResourceSnapshot + { + Buffers = patched.Buffers, + Images = patched.Images, + Samplers = patched.Samplers, + FlattenedResourceTable = table, + UserData = patched.UserData, + DeviceAddressRanges = patched.DeviceAddressRanges, + }; + } + finally + { + ReturnRecorder(recorder); + } + } + + snapshot = patched; + patches = found; + return true; + } + + private bool Validate(Entry entry, ResidentGuestBytesReader residentReader, AddressRebase rebase, IReadOnlyList userData, + out bool unreadable) { unreadable = false; + // A memory-loaded base usually moved when anything did: its few bytes are checked before + // ranges that can span kilobytes. The full comparison below still decides a match. + if (entry.BaseWordBuffers is not null) + { + Span word = stackalloc byte[sizeof(uint)]; + foreach (var (range, offset) in entry.BaseWordProbes ??= BaseWordProbesOf(entry)) + { + var address = rebase.MoveRead(entry, userData, entry.RangeAddresses[range], entry.RangeLengths[range]) + (ulong)offset; + if (residentReader(address, word, entry.RangeClean[range]) && + !word.SequenceEqual(entry.Bytes.AsSpan(entry.RangeOffsets[range] + offset, sizeof(uint)))) + return false; + } + } + for (var index = 0; index < entry.RangeAddresses.Length; index++) { var length = entry.RangeLengths[index]; if (_scratch.Length < length) _scratch = new byte[Math.Max(length, _scratch.Length * 2)]; var current = _scratch.AsSpan(0, length); - if (!residentReader(entry.RangeAddresses[index], current, entry.RangeClean[index])) + var address = rebase.MoveRead(entry, userData, entry.RangeAddresses[index], length); + if (!residentReader(address, current, entry.RangeClean[index])) { unreadable = true; return false; @@ -385,6 +1797,8 @@ public sealed class ResourceMaterializationCache { public required ShaderResourcePlan Plan { get; init; } public required uint[] UserData { get; init; } + // The user data words the evaluator read, one bit per word. + public required ulong[] UsedUserData { get; init; } public required ulong ShaderBase { get; init; } public required ComputeSelectorState? ComputeState { get; init; } public required ulong[] RangeAddresses { get; init; } @@ -398,20 +1812,59 @@ public sealed class ResourceMaterializationCache public required byte[] Bytes { get; init; } public required ResourceSnapshot Snapshot { get; init; } public required ResourceSpecialization Specialization { get; init; } + // Per recorded dword: the buffer whose memory-loaded base dword it is, or a negative value. + public int[]? BaseWordBuffers { get; init; } + // Filled on the first rebased refresh of the entry; see RangeWindowsOf and TableCopySourcesOf. + public int[]? RangeWindows { get; set; } + public int[]? TableCopySources { get; set; } + public (int Range, int Offset)[]? BaseWordProbes { get; set; } public Entry? Next { get; set; } - public bool Matches(ShaderResourcePlan plan, ResourceRuntimeInputs inputs) + // A rebased word is an address the hit patches in, so it does not distinguish entries. + public bool Matches(ShaderResourcePlan plan, ResourceRuntimeInputs inputs, ulong[] rebased) { if (!ReferenceEquals(Plan, plan) || ShaderBase != inputs.ShaderBase || !Nullable.Equals(ComputeState, inputs.ComputeState) || UserData.Length != inputs.UserData.Count) return false; for (var index = 0; index < UserData.Length; index++) - if (UserData[index] != inputs.UserData[index]) + if (IsUsed(UsedUserData, index) && !IsUsed(rebased, index) && UserData[index] != inputs.UserData[index]) return false; return true; } } + // A user data list that notes which words are read by index. Copying it out (the snapshot's + // own copy) goes through the enumerator and is not a read of the evaluator. + private sealed class UserDataRecorder : IReadOnlyList + { + private IReadOnlyList _inner = []; + private ulong[] _used = []; + + public void Begin(IReadOnlyList inner) + { + _inner = inner; + _used = new ulong[(inner.Count + 63) >> 6]; + } + + // The recorded set; the next Begin allocates a fresh one, so the entry keeps this array. + public ulong[] TakeUsed() => _used; + + public int Count => _inner.Count; + + public uint this[int index] + { + get + { + _used[index >> 6] |= 1UL << (index & 63); + return _inner[index]; + } + } + + public IEnumerator GetEnumerator() => _inner.GetEnumerator(); + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + private sealed class ReadRecorder { private readonly List<(ulong Address, uint Word, bool Clean, bool Table)> _reads = new(); @@ -451,6 +1904,14 @@ public sealed class ResourceMaterializationCache public bool Failed { get; private set; } + private readonly UserDataRecorder _userData = new(); + + public IReadOnlyList WrapUserData(IReadOnlyList inner) + { + _userData.Begin(inner); + return _userData; + } + public List<(ulong Address, uint Word, bool Clean, bool Table)> Reads => _reads; public Action TablePhase { get; } @@ -499,9 +1960,14 @@ public sealed class ResourceMaterializationCache } public Entry Build(ShaderResourcePlan plan, ResourceRuntimeInputs inputs, ResourceSnapshot snapshot, - ResourceSpecialization specialization) + ResourceSpecialization specialization, IReadOnlyList memoryBases) { - _reads.Sort((left, right) => left.Address.CompareTo(right.Address)); + // Reads mostly arrive in address order; sorting a sorted list was ~5% of a draw-heavy render thread. + var ordered = true; + for (var index = 1; ordered && index < _reads.Count; index++) + ordered = _reads[index - 1].Address <= _reads[index].Address; + if (!ordered) + _reads.Sort((left, right) => left.Address.CompareTo(right.Address)); // Count unique words and contiguous ranges first, then fill the // retained arrays directly. Temporary lists previously duplicated // every recorded byte and range on each cache miss. @@ -558,8 +2024,10 @@ public sealed class ResourceMaterializationCache userData[index] = inputs.UserData[index]; return new Entry { + BaseWordBuffers = memoryBases.Count == 0 ? null : MapBaseWords(snapshot, memoryBases, bytes), Plan = plan, UserData = userData, + UsedUserData = _userData.TakeUsed(), ShaderBase = inputs.ShaderBase, ComputeState = inputs.ComputeState, RangeAddresses = addresses, @@ -574,5 +2042,46 @@ public sealed class ResourceMaterializationCache Specialization = specialization, }; } + + // The recorded word each memory-loaded base dword came from: the only recorded word holding + // that value. A word two bases claim names neither. Learning checks the guess. + private static int[]? MapBaseWords(ResourceSnapshot snapshot, IReadOnlyList memoryBases, byte[] bytes) + { + int[]? map = null; + var wordCount = bytes.Length / sizeof(uint); + foreach (var buffer in memoryBases) + { + if (buffer >= snapshot.Buffers.Length || snapshot.Buffers[buffer].Length < 2) + continue; + var value = snapshot.Buffers[buffer][0]; + var match = -1; + for (var word = 0; word < wordCount; word++) + { + if (System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(word * sizeof(uint))) != value) + continue; + if (match >= 0) + { + match = -1; + break; + } + match = word; + } + + if (match < 0) + continue; + if (map is null) + { + map = new int[wordCount]; + map.AsSpan().Fill(NoBase); + } + + map[match] = map[match] == NoBase ? buffer : SharedBase; + } + + return map; + } } + + private const int NoBase = -1; + private const int SharedBase = -2; } diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs index b65eb7e..376c019 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs @@ -1050,6 +1050,10 @@ public static class ResourceMaterializer } var storage = baseImage.ResourceClass == ImageResourceClass.Storage; + image = image with + { + MipStatistics = RecordMipStatistics && plan.Stage == ShaderStage.Pixel && !storage && ((descriptor[5] >> 25) & 1) != 0, + }; var conversionFormat = ImageConversionFormat(format); var shaderSwizzle = storage || conversionFormat != GuestImageFormat.Invalid ? descriptor[3] & 0xFFF : image.ShaderSwizzle; var rawSintStorage = storage && format == GuestImageFormat.Format32Sint && baseImage.Written && !baseImage.Read && !baseImage.Atomic; @@ -1363,6 +1367,8 @@ public static class ResourceMaterializer // Applies a specialization to the plan's tables: buffer strides and formats, image // classes and indirect candidates, point samplers and the sampler each access uses. + public static bool RecordMipStatistics { get; set; } + public static SpecializedResourceInfo ApplyTo(ShaderResourcePlan plan, ResourceSpecialization specialization) { var source = plan.Info; @@ -1431,6 +1437,7 @@ public static class ResourceMaterializer image.IndirectSearchIterations = specialized.IndirectSearchIterations; image.Cube = specialized.Cube; image.EmulatedCompareFunction = specialized.EmulatedCompareFunction; + image.MipStatistics = specialized.MipStatistics; if (specialized.EmulatedCompareFunction >= 0) { // The shader compares instead; the host keeps a color view and a plain sampler. diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs index f1fbcd9..0165cd2 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs @@ -27,7 +27,8 @@ public readonly record struct ImageSpecialization( uint IndirectMappingOffset, uint IndirectSearchIterations, bool Cube, - int EmulatedCompareFunction = -1); + int EmulatedCompareFunction = -1, + bool MipStatistics = false); // The module-affecting resource state of one draw. Addresses and descriptor payloads // stay in the snapshot, so they never create a permutation. diff --git a/src/SharpEmu.ShaderCompiler/Resources/ShaderResourceInfo.cs b/src/SharpEmu.ShaderCompiler/Resources/ShaderResourceInfo.cs index 9154e70..cd782f6 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ShaderResourceInfo.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ShaderResourceInfo.cs @@ -82,6 +82,8 @@ public sealed class ImageResource public uint IndirectSearchIterations { get; set; } public List IndirectResources { get; set; } = []; + public bool MipStatistics { get; set; } + public ImageResource Clone() { var clone = (ImageResource)MemberwiseClone(); diff --git a/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs b/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs index 2e64c02..e513e5e 100644 --- a/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs +++ b/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs @@ -162,10 +162,15 @@ public sealed class ShaderCompileRequest public IReadOnlyDictionary WrittenRangeSlotByMemoryIndex { get; } public uint WaveSize { get; init; } = 32; + // True when the pipeline runs on a 64-lane host subgroup, so one subgroup is one guest Wave64. + public bool HostWave64Supported { get; init; } + public bool HostComputeSubgroup32 { get; init; } public bool EnableExecGuardElision { get; init; } = true; public uint ScratchDwords { get; init; } public bool EnableGraphicsSubgroupOperations { get; init; } = true; + public bool MipStatisticsSubgroupReduction { get; init; } + // The device supports 64-bit integer atomics on workgroup memory // (VkPhysicalDeviceFeatures.shaderSharedInt64Atomics). When set, the LDS // 64-bit atomics are emitted as real 64-bit atomics instead of a pair of diff --git a/tests/SharpEmu.Libs.Tests/Agc/AgcSetShRegisterRangeTests.cs b/tests/SharpEmu.Libs.Tests/Agc/AgcSetShRegisterRangeTests.cs index 642d9b7..417452d 100644 --- a/tests/SharpEmu.Libs.Tests/Agc/AgcSetShRegisterRangeTests.cs +++ b/tests/SharpEmu.Libs.Tests/Agc/AgcSetShRegisterRangeTests.cs @@ -91,8 +91,8 @@ public sealed class AgcSetShRegisterRangeTests Assert.Equal(PacketAddress + 8, context[CpuRegister.Rax]); Assert.Equal(1, memory.CopyCalls); Assert.Equal((PacketAddress + 16, sourceAddress, (ulong)valueCount * 4), memory.LastCopy); - Assert.Equal(10, memory.ReadCalls); - Assert.Equal(6, memory.WriteCalls); + Assert.Equal(2, memory.ReadCalls); + Assert.Equal(4, memory.WriteCalls); Assert.Equal(0xC0001000u, ReadUInt32(memory, PacketAddress)); Assert.Equal(0x6875000Du, ReadUInt32(memory, PacketAddress + 4)); Assert.Equal(0xC0007600u | (valueCount << 16), ReadUInt32(memory, PacketAddress + 8)); @@ -136,8 +136,8 @@ public sealed class AgcSetShRegisterRangeTests Assert.Equal(PacketAddress + 8, context[CpuRegister.Rax]); Assert.Equal(0, memory.CopyCalls); - Assert.Equal(10, memory.ReadCalls); - Assert.Equal(6, memory.WriteCalls); + Assert.Equal(2, memory.ReadCalls); + Assert.Equal(4, memory.WriteCalls); Assert.Equal(0xC0001000u, ReadUInt32(memory, PacketAddress)); Assert.Equal(0x6875000Du, ReadUInt32(memory, PacketAddress + 4)); Assert.Equal(0xC0007600u | (valueCount << 16), ReadUInt32(memory, PacketAddress + 8)); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs index 015ec32..dfb1ab9 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs @@ -110,6 +110,162 @@ public sealed class GuestBufferAsyncReadbackTests(HeadlessVulkanFixture fixture) harness.Shutdown(); } + [Fact] + public void AWindowWhoseReadAheadsKeepGoingStaleStopsBeingReadAhead() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + Attach(harness, _vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + GpuWrite(harness, address, 0x01010101); + QueueRead(harness, address); + + GpuWrite(harness, address, 0x02020202); + Assert.True(harness.Cache.HotWritePending); + var value = 0x03030303u; + var deadline = DateTime.UtcNow.AddSeconds(20); + while (harness.Cache.EagerReadbacksWasted < 3 && DateTime.UtcNow < deadline) + { + SubmitBatch(harness); + GpuWrite(harness, address, value++); + } + + Assert.Equal(3, harness.Cache.EagerReadbacksWasted); + SubmitBatch(harness); + var started = harness.Cache.EagerReadbacksStarted; + GpuWrite(harness, address, 0x07070707); + Assert.False(harness.Cache.HotWritePending); + SubmitBatch(harness); + SubmitBatch(harness); + + Assert.Equal(started, harness.Cache.EagerReadbacksStarted); + QueueRead(harness, address); + Assert.Equal(0x07070707u, BitConverter.ToUInt32(harness.Read(address + 0x40, 4))); + Detach(harness); + harness.Shutdown(); + } + + private static void CoverWithOneBuffer(CacheHarness harness, ulong address, ulong size) => + harness.Worker.Run(() => _ = harness.Cache.ObtainBuffer(address, size, isWritten: false)); + + [Fact] + public void AWriteElsewhereInTheBufferKeepsAReadAheadUsable() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + Attach(harness, _vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + CoverWithOneBuffer(harness, address, 0x10000); + GpuWrite(harness, address, 0x01010101); + QueueRead(harness, address); + GpuWrite(harness, address, 0x02020202); + SubmitBatch(harness); + Assert.Equal(1, harness.Cache.EagerReadbacksStarted); + + GpuWrite(harness, address + 0x8000, 0x07070707); + QueueRead(harness, address); + + Assert.Equal(1, harness.Cache.EagerReadbacksUsed); + Assert.Equal(0x02020202u, BitConverter.ToUInt32(harness.Read(address + 0x40, 4))); + Assert.True(harness.Cache.HasGpuDirtyBytes(address + 0x8000, 0x100)); + Assert.True(harness.Cache.DownloadToCpu(address + 0x8000, 4)); + Assert.Equal(0x07070707u, BitConverter.ToUInt32(harness.Read(address + 0x8010, 4))); + Detach(harness); + harness.Shutdown(); + } + + [Fact] + public void ANewerWriteKeepsOnlyItsOwnPieceProtected() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + Attach(harness, _vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + CoverWithOneBuffer(harness, address, 0x10000); + GpuWrite(harness, address, 0x01010101); + GpuWrite(harness, address + 0x8000, 0x01010101); + QueueRead(harness, address); + GpuWrite(harness, address, 0x02020202); + GpuWrite(harness, address + 0x8000, 0x02020202); + SubmitBatch(harness); + Assert.Equal(1, harness.Cache.EagerReadbacksStarted); + + GpuWrite(harness, address + 0x8000, 0x03030303); + QueueRead(harness, address); + + Assert.Equal(1, harness.Cache.EagerReadbacksUsed); + Assert.Equal(1, harness.Cache.StalePiecesKept); + Assert.Equal(0x02020202u, BitConverter.ToUInt32(harness.Read(address + 0x40, 4))); + Assert.True(harness.Cache.HasGpuDirtyBytes(address + 0x8000, 0x100)); + Assert.True(harness.Cache.MayHaveGpuDirtyPages(address + 0x8000, 0x100)); + Assert.True(harness.Cache.DownloadToCpu(address + 0x8000, 4)); + Assert.Equal(0x03030303u, BitConverter.ToUInt32(harness.Read(address + 0x8040, 4))); + Detach(harness); + harness.Shutdown(); + } + + [Fact] + public void AGuestReadOffTheQueueIsReadAheadToo() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + Attach(harness, _vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + GpuWrite(harness, address, 0x01010101); + Assert.True(harness.Cache.DownloadToCpu(address, 4)); + + GpuWrite(harness, address, 0x02020202); + SubmitBatch(harness); + Assert.Equal(1, harness.Cache.EagerReadbacksStarted); + Assert.True(harness.Cache.DownloadToCpu(address, 4)); + + Assert.Equal(1, harness.Cache.EagerReadbacksUsed); + Assert.Equal(0x02020202u, BitConverter.ToUInt32(harness.Read(address + 0x40, 4))); + Detach(harness); + harness.Shutdown(); + } + + [Fact] + public void AReadAheadDeferredForASlotStartsLater() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + Attach(harness, _vulkan); + const ulong window = 0x80000; + const int windows = 5; + var address = harness.MapBacked(window * windows, ReadWrite); + for (var index = 0; index < windows; index++) + { + GpuWrite(harness, address + (ulong)index * window, 0x01010101); + QueueRead(harness, address + (ulong)index * window); + } + + for (var index = 0; index < windows; index++) + { + GpuWrite(harness, address + (ulong)index * window, 0x02020202); + } + + SubmitBatch(harness); + Assert.Equal(4, harness.Cache.EagerReadbacksStarted); + + var deadline = DateTime.UtcNow.AddSeconds(10); + while (harness.Cache.EagerReadbacksStarted < windows && DateTime.UtcNow < deadline) + { + harness.Worker.Run(() => harness.Scheduler.Finish()); + Thread.Sleep(10); + SubmitBatch(harness); + } + + Assert.Equal(windows, harness.Cache.EagerReadbacksStarted); + Detach(harness); + harness.Shutdown(); + } + [Fact] public void RepeatedGuestReadsReuseTheReadbackSlots() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/WriteTickMapTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/WriteTickMapTests.cs new file mode 100644 index 0000000..7fb9679 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/WriteTickMapTests.cs @@ -0,0 +1,62 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Buffers; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.Buffers; + +public sealed class WriteTickMapTests +{ + [Fact] + public void ARangeReportsOnlyTheWritesThatReachIt() + { + var map = new WriteTickMap(); + map.Assign(0x000, 0x100, 5); + map.Assign(0x800, 0x100, 9); + + Assert.Equal(5UL, map.MaxTick(0x000, 0x100)); + Assert.Equal(9UL, map.MaxTick(0x880, 0x10)); + Assert.Equal(0UL, map.MaxTick(0x200, 0x400)); + Assert.Equal(9UL, map.MaxTick(0x000, 0x1000)); + } + + [Fact] + public void ANewerWriteSplitsTheOlderRange() + { + var map = new WriteTickMap(); + map.Assign(0x000, 0x300, 3); + map.Assign(0x100, 0x100, 7); + + Assert.Equal(3, map.Count); + Assert.Equal(3UL, map.MaxTick(0x000, 0x100)); + Assert.Equal(7UL, map.MaxTick(0x180, 0x8)); + Assert.Equal(3UL, map.MaxTick(0x200, 0x100)); + } + + [Fact] + public void AdjacentWritesOfOneTickMerge() + { + var map = new WriteTickMap(); + map.Assign(0x000, 0x100, 4); + map.Assign(0x200, 0x100, 4); + map.Assign(0x100, 0x100, 4); + + Assert.Equal(1, map.Count); + Assert.Equal(4UL, map.MaxTick(0x2FF, 1)); + } + + [Fact] + public void PrunedWritesReadAsTheFloor() + { + var map = new WriteTickMap(); + map.Assign(0x000, 0x100, 2); + map.Assign(0x100, 0x100, 6); + map.Prune(5); + + Assert.Equal(1, map.Count); + Assert.Equal(2UL, map.Floor); + Assert.Equal(2UL, map.MaxTick(0x400, 0x100)); + Assert.Equal(6UL, map.MaxTick(0x100, 0x100)); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs new file mode 100644 index 0000000..18ccd44 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs @@ -0,0 +1,236 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using Xunit; +using static SharpEmu.Libs.Gpu.GpuCommands.Registers.ContextRegisterOffset; +using static SharpEmu.Libs.Gpu.GpuCommands.Registers.ShaderRegisterOffset; +using static SharpEmu.Libs.Gpu.GpuCommands.Registers.UserConfigRegisterOffset; + +namespace SharpEmu.Libs.Tests.Gpu.GpuCommands.Registers; + +// A snapshot shares the live banks' parts until the live banks write them; it never changes. +public sealed class RegisterBanksSnapshotTests +{ + private const ulong PacketAddress = 0x1_0000_1000; + + private static RegisterBanks NewBanks() => new(static message => new InvalidOperationException(message)); + + private static PacketContext Packet(uint opcode, int valueCount) => + new(PacketHeader.Make((uint)valueCount + 2, opcode), PacketAddress, 0, 100, 100); + + private static void WriteContext(RegisterBanks banks, uint offset, params uint[] values) => + RegisterWriteTable.WriteContextPacket(banks, Packet(PacketOpcode.SetContextRegister, values.Length), offset, values); + + private static void WriteShader(RegisterBanks banks, uint offset, params uint[] values) => + RegisterWriteTable.WriteShaderPacket(banks, Packet(PacketOpcode.SetShaderRegister, values.Length), offset, values); + + private static void WriteUserConfig(RegisterBanks banks, uint offset, params uint[] values) => + RegisterWriteTable.WriteUserConfigPacket(banks, Packet(PacketOpcode.SetUserConfigRegister, values.Length), offset, values); + + [Fact] + public void WritesAfterASnapshotLeaveItUnchanged() + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + WriteShader(banks, SpiShaderPgmLoPs, 0x1000); + WriteShader(banks, SpiShaderUserDataPs0, 11, 12); + WriteShader(banks, SpiShaderPgmLoEs, 0x2000); + WriteShader(banks, ComputePgmLo, 0x3000); + WriteUserConfig(banks, VgtPrimitiveType, 4); + var snapshot = banks.Snapshot(); + + WriteContext(banks, DbDepthControl, 0x0000_0000); + WriteShader(banks, SpiShaderPgmLoPs, 0x5000); + WriteShader(banks, SpiShaderUserDataPs0, 21, 22); + WriteShader(banks, SpiShaderPgmLoEs, 0x6000); + WriteShader(banks, ComputePgmLo, 0x7000); + WriteUserConfig(banks, VgtPrimitiveType, 6); + + Assert.True(snapshot.Context.DepthTarget.DepthTestEnabled); + Assert.False(banks.Context.DepthTarget.DepthTestEnabled); + Assert.Equal(0x1000ul << 8, snapshot.Shader.Pixel.Address); + Assert.Equal(0x5000ul << 8, banks.Shader.Pixel.Address); + Assert.Equal([11u, 12u], snapshot.Shader.Pixel.UserScalars.Values[..2]); + Assert.Equal([21u, 22u], banks.Shader.Pixel.UserScalars.Values[..2]); + Assert.Equal(0x2000ul << 8, snapshot.Shader.Vertex.ExportAddress); + Assert.Equal(0x6000ul << 8, banks.Shader.Vertex.ExportAddress); + Assert.Equal(0x3000ul << 8, snapshot.Shader.Compute.Address); + Assert.Equal(0x7000ul << 8, banks.Shader.Compute.Address); + Assert.Equal(4u, snapshot.UserConfig.PrimitiveType); + Assert.Equal(6u, banks.UserConfig.PrimitiveType); + } + + [Fact] + public void AWriteCopiesOnlyThePartItStoresInto() + { + var banks = NewBanks(); + var snapshot = banks.Snapshot(); + + WriteShader(banks, SpiShaderUserDataPs0, 7); + + Assert.NotSame(snapshot.Shader.Pixel, banks.Shader.Pixel); + Assert.Same(snapshot.Shader.Vertex, banks.Shader.Vertex); + Assert.Same(snapshot.Shader.Compute, banks.Shader.Compute); + Assert.Same(snapshot.Context, banks.Context); + Assert.Same(snapshot.UserConfig, banks.UserConfig); + } + + [Fact] + public void AUserScalarWriteCopiesOnlyItsSet() + { + var banks = NewBanks(); + WriteShader(banks, SpiShaderUserDataGs0, 1); + WriteShader(banks, SpiShaderUserDataHs0, 2); + var snapshot = banks.Snapshot(); + + WriteShader(banks, SpiShaderUserDataGs0, 3); + WriteShader(banks, SpiShaderPgmLoPs, 0x4000); + + Assert.NotSame(snapshot.Shader.Vertex, banks.Shader.Vertex); + Assert.NotSame(snapshot.Shader.Vertex.GeometryUserScalars, banks.Shader.Vertex.GeometryUserScalars); + Assert.Same(snapshot.Shader.Vertex.HullUserScalars, banks.Shader.Vertex.HullUserScalars); + Assert.Equal(1u, snapshot.Shader.Vertex.GeometryUserScalars.Values[0]); + Assert.Equal(3u, banks.Shader.Vertex.GeometryUserScalars.Values[0]); + Assert.NotSame(snapshot.Shader.Pixel, banks.Shader.Pixel); + Assert.Same(snapshot.Shader.Pixel.UserScalars, banks.Shader.Pixel.UserScalars); + + // The set copied after the snapshot is the live banks' own: a later write does not copy it again. + var geometry = banks.Shader.Vertex.GeometryUserScalars; + WriteShader(banks, SpiShaderUserDataGs0, 4); + Assert.Same(geometry, banks.Shader.Vertex.GeometryUserScalars); + Assert.Equal(1u, snapshot.Shader.Vertex.GeometryUserScalars.Values[0]); + } + + [Fact] + public void ContextStateOperationsAfterASnapshotLeaveItUnchanged() + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + var snapshot = banks.Snapshot(); + + banks.ApplyContextState(ContextStateOperation.PushClear); + WriteContext(banks, DbDepthControl, 0x0000_0000); + banks.ApplyContextState(ContextStateOperation.Pop); + WriteContext(banks, DbDepthControl, 0x0000_0000); + + Assert.True(snapshot.Context.DepthTarget.DepthTestEnabled); + Assert.False(banks.Context.DepthTarget.DepthTestEnabled); + } + + // Every context register, through the table path and the packet path, against a snapshot + // taken after a first write: the context's nested parts are copied only by their writers. + [Fact] + public void EveryContextRegisterWriteLeavesASnapshotUnchanged() + { + var changed = 0; + for (var offset = 0u; offset < RegisterBankLayout.ContextRegisterCount; offset++) + { + foreach (var packet in new[] { false, true }) + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + var snapshot = banks.Snapshot(); + var before = Describe(snapshot.Context); + foreach (var value in new[] { 0xFFFF_FFFFu, 0x1234_5678u }) + { + try + { + if (packet) + WriteContext(banks, offset, value); + else + RegisterWriteTable.WriteContextEntry(banks, offset, value, PacketAddress); + } + catch (InvalidOperationException) + { + // A register whose writer refuses a single value. + } + } + + Assert.True(before == Describe(snapshot.Context), $"offset=0x{offset:X} packet={packet}"); + if (Describe(banks.Context) != before) + changed++; + } + } + + // The description sees the writes: most registers store something. + Assert.True(changed > 400, $"changed={changed}"); + } + + private static string Describe(object? value) + { + var text = new System.Text.StringBuilder(); + Describe(value, text); + return text.ToString(); + } + + private static void Describe(object? value, System.Text.StringBuilder text) + { + switch (value) + { + case null: + text.Append("null;"); + return; + case System.Collections.IDictionary dictionary: + foreach (var key in dictionary.Keys.Cast().OrderBy(key => key)) + text.Append(key).Append('=').Append(dictionary[key]).Append(','); + text.Append(';'); + return; + case Array array: + text.Append('['); + foreach (var element in array) + Describe(element, text); + text.Append(']'); + return; + } + + var type = value.GetType(); + if (type.IsPrimitive || type.IsEnum || value is string) + { + text.Append(value).Append(';'); + return; + } + + text.Append('{'); + foreach (var field in type.GetFields(System.Reflection.BindingFlags.Instance | + System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic)) + { + text.Append(field.Name).Append(':'); + Describe(field.GetValue(value), text); + } + + text.Append('}'); + } + + [Fact] + public void AContextWriteCopiesOnlyTheNestedPartsItStoresInto() + { + var banks = NewBanks(); + var snapshot = banks.Snapshot(); + + WriteContext(banks, DbDepthControl, 0x0000_0006); + WriteContext(banks, PaClVteCntl, 0x43F); + + Assert.NotSame(snapshot.Context, banks.Context); + Assert.NotSame(snapshot.Context.ScreenViewport, banks.Context.ScreenViewport); + Assert.Same(snapshot.Context.ScreenViewport.Viewports, banks.Context.ScreenViewport.Viewports); + Assert.Same(snapshot.Context.ShaderInterface, banks.Context.ShaderInterface); + Assert.Same(snapshot.Context.ColorTargets, banks.Context.ColorTargets); + Assert.Same(snapshot.Context.UnmodeledTableRegisters, banks.Context.UnmodeledTableRegisters); + } + + [Fact] + public void ASecondSnapshotSharesAgainUntilTheNextWrite() + { + var banks = NewBanks(); + var first = banks.Snapshot(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + var second = banks.Snapshot(); + WriteContext(banks, DbDepthControl, 0x0000_0000); + + Assert.False(first.Context.DepthTarget.DepthTestEnabled); + Assert.True(second.Context.DepthTarget.DepthTestEnabled); + Assert.False(banks.Context.DepthTarget.DepthTestEnabled); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs index 2eb5a41..233a83e 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs @@ -99,6 +99,24 @@ public sealed partial class GuestImageCacheTests harness.Shutdown(); } + // A fill seen before its target leaves its range known as DCC metadata, so the next fill of it + // runs on the host and the range stays CPU-readable. + [Fact] + public void APendingDccFill_MarksItsRangeAsDccMetadata() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var fatal = new FatalScope(); + using var harness = new CacheHarness(_vulkan); + var address = harness.MapBacked(0x20000, ReadWrite); + var metadata = address + 0x10000; + Assert.False(harness.Images.OverlapsDccMetadata(metadata, 0x100)); + + Assert.False(harness.Images.TryAbsorbDccFill(metadata, 0x100, 0x00000000u)); + + Assert.True(harness.Images.OverlapsDccMetadata(metadata + 0x80, 0x10)); + Assert.False(harness.Images.OverlapsDccMetadata(metadata + 0x100, 0x10)); + } + [Fact] public void DccFills_RemainGuestProducersUntilDescriptorDiscovery() { @@ -451,6 +469,35 @@ public sealed partial class GuestImageCacheTests harness.Shutdown(); } + // The pages inside a filled range are clean after it; a page it shares with another GPU-written + // range keeps that range's state. + [Fact] + public void DirtyDccFill_KeepsASharedBoundaryPageDirty() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + const ulong Page = SharpEmu.HLE.GpuMemory.TrackerLayout.PageBytes; + var memory = harness.MapBacked(8 * Page, ReadWrite); + var metadata = memory + Page + Page / 2; + var size = 3 * Page; + var neighbour = metadata - 0x100; + harness.Images.RegisterDccMetadataForTest(metadata, size); + harness.Worker.Run(() => + { + _ = harness.Cache.ObtainBuffer(neighbour, 0x100, isWritten: true); + _ = harness.Cache.ObtainBuffer(metadata, size, isWritten: true); + }); + + harness.Worker.Run(() => harness.Cache.FillBuffer(metadata, size, 0x00000000, isGds: false)); + + Assert.False(harness.Cache.HasGpuDirtyBytes(metadata, size)); + Assert.False(harness.Cache.HasGpuDirtyPages(memory + 2 * Page, 2 * Page)); + Assert.False(harness.Cache.HasGpuDirtyPages(memory + 4 * Page, Page)); + Assert.True(harness.Cache.HasGpuDirtyPages(memory + Page, Page)); + Assert.True(harness.Cache.HasGpuDirtyBytes(neighbour, 0x100)); + harness.Shutdown(); + } + [Fact] public void CleanDccFill_WritesBothCopiesWithoutMarkingTheGpuCopyNewer() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Ownership.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Ownership.cs index 21492f8..0751dfb 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Ownership.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Ownership.cs @@ -138,6 +138,58 @@ public sealed partial class GuestImageCacheTests } } + [Theory] + [InlineData(Format.R16G16Sfloat, GuestPixelFormat.Bits16_16Float, 4u, 0x3c00bc00u)] + [InlineData(Format.R16G16B16A16Sfloat, GuestPixelFormat.Bits16_16_16_16Float, 8u, 0x3c00bc00u)] + [InlineData(Format.R16G16B16A16Sfloat, GuestPixelFormat.Bits16_16_16_16Float, 8u, 0x80000001u)] + [InlineData(Format.R32G32Uint, GuestPixelFormat.Bits32_32UInt, 8u, 0xdeadbeefu)] + [InlineData(Format.R32G32B32A32Uint, GuestPixelFormat.Bits32_32_32_32UInt, 16u, 0xdeadbeefu)] + public void FullColorBufferFillPreservesRawWords(Format format, GuestPixelFormat guestFormat, uint bytesPerTexel, uint word) + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + var request = LinearRequest(address, bytesPerTexel, format, guestFormat, GuestImageType.Color2D, + new Extent3D(1, 1, 1), 1, bytesPerTexel, 1); + var image = harness.Find(ref request); + Assert.True(harness.Worker.Run(() => harness.Images.TryClearImageFromBuffer(address, bytesPerTexel, word))); + Assert.Equal(Bytes(Enumerable.Repeat(word, (int)bytesPerTexel / 4).ToArray()), harness.ReadImageBytes(harness.Image(image))); + Assert.True(harness.Image(image).IsGpuModified); + harness.Shutdown(); + } + + [Fact] + public void HalfFloatNaNBufferFillFallsBackWithoutChangingImageOwnership() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + var request = LinearRequest(address, 8, Format.R16G16B16A16Sfloat, GuestPixelFormat.Bits16_16_16_16Float, + GuestImageType.Color2D, new Extent3D(1, 1, 1), 1, 8, 1); + var image = harness.Find(ref request); + Assert.True(harness.Image(image).IsCpuDirty); + Assert.False(harness.Worker.Run(() => harness.Images.TryClearImageFromBuffer(address, 8, 0x7e003c00u))); + Assert.True(harness.Image(image).IsCpuDirty); + Assert.False(harness.Image(image).IsGpuModified); + harness.Shutdown(); + } + + [Fact] + public void HalfFloatBufferFillStillRequiresAnExactRange() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + var address = harness.MapBacked(0x10000, ReadWrite); + var request = LinearRequest(address, 16, Format.R16G16B16A16Sfloat, GuestPixelFormat.Bits16_16_16_16Float, + GuestImageType.Color2D, new Extent3D(2, 1, 1), 1, 8, 1); + var image = harness.Find(ref request); + Assert.False(harness.Worker.Run(() => harness.Images.TryClearImageFromBuffer(address + 8, 8, 0x3c00bc00u))); + Assert.False(harness.Worker.Run(() => harness.Images.TryClearImageFromBuffer(address, 20, 0x3c00bc00u))); + Assert.True(harness.Image(image).IsCpuDirty); + Assert.False(harness.Image(image).IsGpuModified); + harness.Shutdown(); + } + [Fact] public void PartialPage_BufferAndImageShareOneTrackerPage() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs index 192a2e4..d06354f 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs @@ -212,6 +212,47 @@ public sealed class ImageDescriptionTests Assert.All(fatal.Messages, message => Assert.Contains("overlap range is invalid", message)); } + [Theory] + [InlineData(Format.R16G16Sfloat)] + [InlineData(Format.R16G16B16A16Sfloat)] + public void BufferFillDecodesRepeatedHalfWordsWithoutChangingDccClears(Format format) + { + Assert.True(PackedClearValue.TryDecodeBufferColor(format, 0x3c00bc00u, out var clear)); + Assert.Equal((-1f, 1f, -1f, 1f), (clear.Float32_0, clear.Float32_1, clear.Float32_2, clear.Float32_3)); + Assert.False(PackedClearValue.TryDecodeColor(format, 0x3c00bc00u, out _)); + Assert.False(PackedClearValue.TryDecodeBufferColor(format, 0x7e003c00u, out _)); + Assert.False(PackedClearValue.TryDecodeBufferColor(format, 0x3c007e00u, out _)); + } + + [Theory] + [InlineData(Format.R32G32Uint)] + [InlineData(Format.R32G32B32A32Uint)] + public void BufferFillRepeatsTheEntireUintWordInEveryComponent(Format format) + { + Assert.True(PackedClearValue.TryDecodeBufferColor(format, 0xdeadbeefu, out var clear)); + Assert.Equal((0xdeadbeefu, 0xdeadbeefu, 0xdeadbeefu, 0xdeadbeefu), + (clear.Uint32_0, clear.Uint32_1, clear.Uint32_2, clear.Uint32_3)); + } + + [Fact] + public void BufferFillPreservesEveryRepresentableHalfWord() + { + for (var bits = 0; bits <= ushort.MaxValue; bits++) + { + var packed = (uint)bits | ((uint)bits << 16); + var decoded = PackedClearValue.TryDecodeBufferColor(Format.R16G16B16A16Sfloat, packed, out var clear); + if (Half.IsNaN(BitConverter.UInt16BitsToHalf((ushort)bits))) + { + Assert.False(decoded); + continue; + } + + Assert.True(decoded); + Assert.Equal((ushort)bits, BitConverter.HalfToUInt16Bits((Half)clear.Float32_0)); + Assert.Equal((ushort)bits, BitConverter.HalfToUInt16Bits((Half)clear.Float32_1)); + } + } + [Fact] public void Validate_RejectsInconsistentDescriptions() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/MipStatisticsTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/MipStatisticsTests.cs index 28b5d32..51bdbed 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/MipStatisticsTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/MipStatisticsTests.cs @@ -113,4 +113,57 @@ public sealed class MipStatisticsTests Assert.All(report, static value => Assert.Equal(0, value)); } + + [Fact] + public void WrittenReport_PointsAFullReportAtItsCounters() + { + var statistics = new MipStatistics(); + statistics.NoteSampled(Texture(slot: 3)); + var report = new byte[MipStatistics.ReportBytes]; + + statistics.WriteReport(report); + + Assert.Equal(MipStatistics.FirstCounterOffset, BinaryPrimitives.ReadUInt32LittleEndian(report)); + Assert.Equal((0x10000u, 3u, 0x10000u, 0u), Counter(report, 3)); + } + + [Fact] + public void WrittenReport_MarksAShortBufferValidWithoutCounters() + { + var statistics = new MipStatistics(); + var report = new byte[16]; + + statistics.WriteReport(report); + + Assert.Equal(MipStatistics.ReportWithoutCounters, BinaryPrimitives.ReadUInt32LittleEndian(report)); + Assert.All(report[sizeof(uint)..], value => Assert.Equal(0, value)); + } + + [Fact] + public void ImageField_CarriesTheCounterBaseLevelAndMinimumLod() + { + uint[] texture = [0, 0x280u << 8, 0, 3u << 12, 0, CounterEnabled, 0x2A, 0]; + + Assert.Equal(0x2Au | (3u << 8) | (0x280u << 16), MipStatistics.ImageField(texture)); + texture[5] = 0; + Assert.Equal(MipStatistics.NoCounterField, MipStatistics.ImageField(texture)); + } + + [Fact] + public void PackedGpuReport_ReportsTheFinestMipAndTheClampedSamples() + { + var words = new uint[2 * MipStatistics.CounterWords]; + Array.Fill(words, uint.MaxValue, 0, (int)MipStatistics.CounterWords); + words[5] = 2; + words[MipStatistics.CounterWords + 5] = 40; + words[9] = 0; + var report = new byte[MipStatistics.ReportBytes]; + + MipStatistics.PackReport(words, report); + + Assert.Equal(MipStatistics.FirstCounterOffset, BinaryPrimitives.ReadUInt32LittleEndian(report)); + Assert.Equal((40u, 5u, 0u, 2u), Counter(report, 5)); + Assert.Equal((0u, 9u, 0u, 0u), Counter(report, 9)); + Assert.Equal((0u, 6u, 0u, MipStatistics.NoMipSampled), Counter(report, 6)); + } } diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs index 4d2a98b..e16ad15 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs @@ -266,6 +266,9 @@ public sealed class ShaderPipelineCacheTests : IDisposable executor.DrawAuto(1, Banks(), Auto(3)); executor.DrawAuto(2, Banks(), Auto(3)); executor.DrawAuto(3, Banks(PrimitiveTriangleStrip), Auto(4)); + // The lookup key is reused across draws: after another pipeline, the first is found again. + executor.DrawAuto(4, Banks(), Auto(3)); + executor.DrawAuto(5, Banks(PrimitiveTriangleStrip), Auto(4)); Assert.Equal(2, guest.Host.GraphicsPipelines.Count); Assert.Equal(2, cache.GraphicsPipelineCount); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs index b05928c..9d09711 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs @@ -279,9 +279,25 @@ public sealed class RenderExecutorComputeTests : IDisposable Assert.Contains($"clear_image {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); Assert.Contains($"absorb_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); + Assert.Contains($"fill_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); AssertDispatched(4, 1, 1); } + // A pending surface's metadata the host knows is filled there: as a dispatch it would be GPU-written + // and read back at every later bind. + [Fact] + public void PendingMetadataTheHostKnows_IsFilledOnTheHost() + { + _host.HostFillableDcc.Add(MetadataAddress); + ConfigureClearKernel(256); + _executor.Dispatch(1, Banks(), 256, 1, 1, ClearInitiator); + + Assert.Contains($"absorb_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); + Assert.Contains($"fill_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); + AssertNotDispatched(); + Assert.Contains("reset_bindings", _host.Calls); + } + [Fact] public void RegisteredDccAddress_ConsumesTheFillAndRecordsNothing() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs index aa772f4..66e3d5d 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs @@ -22,9 +22,21 @@ public sealed class RenderExecutorDrawTests : IDisposable private readonly RenderExecutor _executor; private readonly FatalScope _fatal = new(); - public RenderExecutorDrawTests() => _executor = new RenderExecutor(_host, _pipelines); + private readonly InputsCheckingPrefetch _prefetch; - public void Dispose() => _fatal.Dispose(); + public RenderExecutorDrawTests() + { + _executor = new RenderExecutor(_host, _pipelines); + _prefetch = new InputsCheckingPrefetch(_host); + _executor.Prefetch = _prefetch; + } + + // Every draw of these tests also checks that its program inputs follow from its registers. + public void Dispose() + { + _fatal.Dispose(); + Assert.Empty(_prefetch.Mismatches); + } private void AssertOrder(params string[] prefixes) { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs new file mode 100644 index 0000000..28377d1 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs @@ -0,0 +1,87 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Tests.Gpu.Scheduling; +using Xunit; +using static SharpEmu.Libs.Tests.Gpu.Rendering.RenderExecutorFixtures; + +namespace SharpEmu.Libs.Tests.Gpu.Rendering; + +// A draw takes the programs a prefetch resolved from its banks, or resolves them itself. +[Collection(SchedulingStateCollection.Name)] +public sealed class RenderExecutorPrefetchTests : IDisposable +{ + private readonly RecordingRenderHost _host = new(); + private readonly FakePipelineProvider _pipelines = new(); + private readonly RenderExecutor _executor; + private readonly FatalScope _fatal = new(); + + public RenderExecutorPrefetchTests() => _executor = new RenderExecutor(_host, _pipelines); + + public void Dispose() => _fatal.Dispose(); + + // Resolves ahead with the executor, as a worker would, and hands the result back once. + private sealed class AheadPrefetch(RenderExecutor executor, IRenderHost host) : IProgramPrefetch + { + private RegisterBanks? _banks; + private GraphicsProgramInputs _inputs; + private GraphicsPrograms? _programs; + + public int Taken { get; private set; } + + public int Declined { get; private set; } + + public void Resolve(RegisterBanks banks) + { + _banks = banks; + _inputs = RenderExecutor.ProgramInputsOf(banks, host.FormatSupport, host.Fatal); + _programs = executor.ResolveGraphicsPrograms(banks, in _inputs); + } + + public bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs) + { + if (_programs is not null && ReferenceEquals(banks, _banks) && inputs.Equals(_inputs)) + { + programs = _programs; + _programs = null; + Taken++; + return true; + } + + programs = null!; + Declined++; + return false; + } + } + + [Fact] + public void ADrawTakesTheProgramsResolvedAheadFromItsBanks() + { + var prefetch = new AheadPrefetch(_executor, _host); + _executor.Prefetch = prefetch; + var banks = Banks(); + prefetch.Resolve(banks); + var resolutions = _pipelines.Calls.Count(call => call.StartsWith("get_graphics_programs", StringComparison.Ordinal)); + + _executor.DrawAuto(1, banks, Auto(3)); + + Assert.Equal(1, prefetch.Taken); + Assert.Equal(resolutions, _pipelines.Calls.Count(call => call.StartsWith("get_graphics_programs", StringComparison.Ordinal))); + Assert.Contains(_host.Calls, call => call.StartsWith("draw ", StringComparison.Ordinal)); + } + + [Fact] + public void ADrawWithOtherBanksResolvesItsOwnPrograms() + { + var prefetch = new AheadPrefetch(_executor, _host); + _executor.Prefetch = prefetch; + prefetch.Resolve(Banks()); + + _executor.DrawAuto(1, Banks(), Auto(3)); + + Assert.Equal((0, 1), (prefetch.Taken, prefetch.Declined)); + Assert.Equal(2, _pipelines.Calls.Count(call => call.StartsWith("get_graphics_programs", StringComparison.Ordinal))); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs index c50241a..cfd6432 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs @@ -108,6 +108,9 @@ internal sealed class RecordingRenderHost : IRenderHost public HashSet RegisteredDcc { get; } = new(); + // Metadata the host knows and fills itself although its surface is not registered yet. + public HashSet HostFillableDcc { get; } = new(); + public HashSet ClearableImages { get; } = new(); public Func? ClampOverride { get; set; } @@ -391,7 +394,7 @@ internal sealed class RecordingRenderHost : IRenderHost public bool TryFillDccMetadata(ulong address, ulong size, uint fillValue) { Calls.Add($"fill_dcc {address:X} {size:X} {fillValue:X8}"); - return RegisteredDcc.Contains(address); + return RegisteredDcc.Contains(address) || HostFillableDcc.Contains(address); } public bool HostCopyAccepted { get; set; } = true; @@ -405,6 +408,33 @@ internal sealed class RecordingRenderHost : IRenderHost public Exception Fatal(string message) => new RenderExecutorFatalException(message); } +// Takes nothing, but checks that the inputs a draw resolves its programs with are the ones its +// register banks alone give: a prefetch computed ahead of the draw would use those. +internal sealed class InputsCheckingPrefetch(IRenderHost host) : IProgramPrefetch +{ + public List Mismatches { get; } = new(); + + public int Checks { get; private set; } + + public bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs) + { + Checks++; + var expected = RenderExecutor.ProgramInputsOf(banks, host.FormatSupport, host.Fatal); + if (!expected.Equals(inputs)) + { + Mismatches.Add( + $"draw pixelActive={inputs.PixelActive}/{expected.PixelActive} depthBound={inputs.DepthBound}/{expected.DepthBound} " + + $"mapping=[{Describe(inputs.Mapping)}]/[{Describe(expected.Mapping)}]"); + } + + programs = null!; + return false; + } + + private static string Describe(ReadOnlySpan mapping) => + string.Join(",", mapping.ToArray().Select(map => map.Packed)); +} + // Hands back the configured programs and records every pipeline request. internal sealed class FakePipelineProvider : IShaderPipelineProvider { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Scheduling/RecordingBufferBarrierTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Scheduling/RecordingBufferBarrierTests.cs new file mode 100644 index 0000000..2e85c3e --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Scheduling/RecordingBufferBarrierTests.cs @@ -0,0 +1,72 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Scheduling; +using Silk.NET.Vulkan; +using Xunit; +using static SharpEmu.Libs.Tests.Gpu.Scheduling.SchedulingTestSupport; + +namespace SharpEmu.Libs.Tests.Gpu.Scheduling; + +[Collection(SchedulingStateCollection.Name)] +public sealed class RecordingBufferBarrierTests +{ + private static readonly MemoryBarrierRequest ComputeToAll = new( + PipelineStageFlags.ComputeShaderBit, PipelineStageFlags.AllCommandsBit, + AccessFlags.ShaderWriteBit, AccessFlags.MemoryReadBit | AccessFlags.MemoryWriteBit); + + private static readonly MemoryBarrierRequest AllToCompute = new( + PipelineStageFlags.AllCommandsBit, PipelineStageFlags.ComputeShaderBit, + AccessFlags.MemoryReadBit | AccessFlags.MemoryWriteBit, AccessFlags.ShaderReadBit | AccessFlags.ShaderWriteBit); + + private readonly FakeTickDevice _device = new(); + private readonly RecordingRenderingState _rendering = new(); + + [Fact] + public void RequestsBetweenCommandsAreRecordedAsOneUnion() + { + using var scheduler = NewActiveScheduler(_device, _rendering); + var recorded = new List(); + scheduler.MemoryBarrierRecorder = (_, request) => recorded.Add(request); + + scheduler.Current.RequestMemoryBarrier(ComputeToAll); + _ = scheduler.Current.StateHandle; + scheduler.Current.RequestMemoryBarrier(AllToCompute); + Assert.Empty(recorded); + _ = scheduler.Current.Handle; + _ = scheduler.Current.Handle; + + Assert.Equal([ComputeToAll.Union(AllToCompute)], recorded); + Assert.Equal((1L, 1L), (scheduler.Current.MergedBarriers, scheduler.Current.RecordedBarriers)); + _device.CompleteOnSubmit = true; + } + + [Fact] + public void APendingRequestIsRecordedBeforeTheBufferEnds() + { + using var scheduler = NewActiveScheduler(_device, _rendering); + var recorded = new List(); + scheduler.MemoryBarrierRecorder = (buffer, _) => recorded.Add($"barrier {_device.Name(buffer)} after {_device.Log[^1]}"); + scheduler.Current.RequestMemoryBarrier(ComputeToAll); + _device.CompleteOnSubmit = true; + + scheduler.Submit(); + + Assert.Equal(["barrier b0 after begin b0"], recorded); + Assert.Contains("end b0", _device.Log); + } + + [Fact] + public void ARequestInsideRenderingIsRefused() + { + using var fatal = new FatalScope(); + using var scheduler = NewActiveScheduler(_device, _rendering); + scheduler.MemoryBarrierRecorder = (_, _) => { }; + _rendering.IsRendering = true; + + Assert.Throws(() => scheduler.Current.RequestMemoryBarrier(ComputeToAll)); + + _rendering.IsRendering = false; + _device.CompleteOnSubmit = true; + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/HeadlessVulkan.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/HeadlessVulkan.cs index 5c3f36f..60068f5 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/HeadlessVulkan.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/HeadlessVulkan.cs @@ -161,6 +161,7 @@ internal sealed unsafe class HeadlessVulkan : IDisposable public void Dispose() { Vk.DeviceWaitIdle(Device); + _deviceInfo?.Slabs.Destroy(); Vk.DestroyDevice(Device, null); if (_debugUtils is { } debugUtils) { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs index fb9f391..141bca6 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs @@ -5,6 +5,7 @@ using System.Buffers.Binary; using SharpEmu.Libs.Gpu.Buffers; using SharpEmu.Libs.Tests.Gpu.Images; using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Ir; using SharpEmu.ShaderCompiler.Resources; using SharpEmu.ShaderCompiler.Vulkan; using Xunit; @@ -12,16 +13,18 @@ using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; namespace SharpEmu.Libs.Tests.Gpu.Vulkan; -public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : IClassFixture +public class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : IClassFixture { - private const uint Vcc = 106; - private const uint Exec = 126; - private static readonly Gen5Operand AllLanes = Gen5Operand.Source(193); + protected virtual bool HostSubgroup32 => false; + + protected const uint Vcc = 106; + protected const uint Exec = 126; + protected static readonly Gen5Operand AllLanes = Gen5Operand.Source(193); private readonly List _instructions = []; private uint _pc; - private uint Add(Gen5ShaderInstruction instruction) + protected uint Add(Gen5ShaderInstruction instruction) { var pc = _pc; _instructions.Add(instruction with { Pc = pc }); @@ -29,46 +32,60 @@ public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : I return pc; } - private void Point(uint branchPc, uint targetPc) + protected void Point(uint branchPc, uint targetPc) { var index = _instructions.FindIndex(instruction => instruction.Pc == branchPc); var offset = (short)(((int)targetPc - (int)branchPc - 4) / 4); _instructions[index] = _instructions[index] with { Words = [unchecked((uint)(ushort)offset)] }; } - private void Lane() + protected void Lane(uint rowShift = 3) { - Add(Vop2(0, "VLshlrevB32", 2, Operand(3), Gen5Operand.Vector(1))); + Add(Vop2(0, "VLshlrevB32", 2, Operand(rowShift), Gen5Operand.Vector(1))); Add(Vop2(0, "VAddI32", 2, Gen5Operand.Vector(0), Gen5Operand.Vector(2))); } - private uint CompareTo(string opcode, uint destination, Gen5Operand left, Gen5Operand right) => + protected uint CompareTo(string opcode, uint destination, Gen5Operand left, Gen5Operand right) => Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop3, opcode, [0u, 0u], [left, right], [Gen5Operand.Scalar(destination)], new Gen5Vop3Control(0, 0, 0, false, 0, destination))); - private uint[] Run(uint localDataShareDwords = 0) + protected uint[] Run(uint localDataShareDwords = 0, uint sizeX = 8, uint sizeY = 8, bool? laneLocal = null) { + var threads = sizeX * sizeY; Add(Vop2(0, "VLshlrevB32", 5, Operand(2), Gen5Operand.Vector(2))); Add(BufferAccess(0, "BufferStoreDword", 4, vectorData: 4, offsetEnabled: true, vectorAddress: 5)); Add(EndProgram(0)); - var (plan, resources, layout) = Prepare(Program([.. _instructions])); + var program = Program([.. _instructions]); + if (laneLocal is { } expected) + { + Assert.Equal(expected, Gen5WaveLaneLocalAnalysis.IsLaneLocal(program, new HashSet(), out var reason)); + Assert.True(expected || reason.Length != 0); + } + + var (plan, resources, layout) = Prepare(program); var request = new ShaderCompileRequest(plan, resources, layout) { - LocalSizeX = 8, LocalSizeY = 8, ThreadCountX = 8, ThreadCountY = 8, WaveSize = 64, - LocalDataShareDwords = localDataShareDwords, + LocalSizeX = sizeX, LocalSizeY = sizeY, ThreadCountX = sizeX, ThreadCountY = sizeY, WaveSize = 64, + LocalDataShareDwords = localDataShareDwords, HostComputeSubgroup32 = HostSubgroup32, }; Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); + if (HostSubgroup32 && laneLocal is { } oneLanePerInvocation) + { + var halfWavesEnabled = Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_LANE_LOCAL") == "1"; + Assert.Equal(oneLanePerInvocation && halfWavesEnabled ? sizeX : (threads + 63) / 64 * 32, LocalSizeX(shader.Spirv)); + } + Assert.NotNull(fixture.Vulkan); using var harness = new ImageTestHarness(fixture.Vulkan); using var runner = new LayoutComputeRunner(harness, request, shader.Spirv); - var output = runner.CreateBuffer(256); + var output = runner.CreateBuffer(threads * 4); var registers = new uint[256]; - registers[6] = 256; + registers[6] = threads * 4; harness.Run(() => runner.Dispatch(registers, new Dictionary { [DescriptorBindingKind.Buffers] = [output] }, 1)); - var bytes = runner.ReadBack(output, 0, 256); + var bytes = runner.ReadBack(output, 0, threads * 4); harness.AssertNoValidationMessages(); - var lanes = new uint[64]; + var lanes = new uint[threads]; for (var lane = 0; lane < lanes.Length; lane++) { lanes[lane] = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(lane * sizeof(uint))); @@ -87,6 +104,25 @@ public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : I Assert.All(Run(), x => Assert.Equal(0xFFFF_FFF8u, x)); } + [Fact] + public void ANarrowedRegionKeepsTheValuesItsInactiveLanesShowLater() + { + if (!Ready()) return; + Lane(); + Add(MoveVector(0, 4, 7)); + Add(Sop1(0, "SMovB64", 20, Gen5Operand.Scalar(Exec))); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + Add(Vop2(0, "VAddU32", 3, Operand(64), Gen5Operand.Vector(2))); + Add(Vop2(0, "VAddU32", 4, Operand(1), Gen5Operand.Vector(3))); + Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(20))); + + var lanes = Run(); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal(lane >= 40 ? lane + 65 : 7u, lanes[lane]); + } + } + [Fact] public void InitialExecContainsEveryLane() { @@ -96,7 +132,27 @@ public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : I Assert.All(Run(), x => Assert.Equal(uint.MaxValue, x)); } - private bool Ready() => GatePrerequisites.Ready(fixture.Vulkan, shaderInt64: true); + private static uint LocalSizeX(byte[] spirv) + { + const uint ExecutionMode = 16; + const uint LocalSize = 17; + for (var offset = 5 * sizeof(uint); offset < spirv.Length;) + { + var word = BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset)); + var count = (int)(word >> 16); + if ((word & 0xFFFF) == ExecutionMode && + BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset + 2 * sizeof(uint))) == LocalSize) + { + return BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset + 3 * sizeof(uint))); + } + + offset += Math.Max(count, 1) * sizeof(uint); + } + + throw new InvalidOperationException("no LocalSize execution mode"); + } + + protected bool Ready() => GatePrerequisites.Ready(fixture.Vulkan, shaderInt64: true); [Fact] public void MasksAcrossTheHalfBoundary_StayExactPerLaneAndWhenCounted() @@ -271,7 +327,7 @@ public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : I var (plan, resources, layout) = Prepare(program, userDataCount: 12); var request = new ShaderCompileRequest(plan, resources, layout) { - LocalSizeX = 64, WaveSize = 64, EnableExecGuardElision = false, + LocalSizeX = 64, WaveSize = 64, EnableExecGuardElision = false, HostComputeSubgroup32 = HostSubgroup32, ComputeSystemRegisters = new Gen5ComputeSystemRegisters(12, null, null, null), }; Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64Subgroup32DeviceTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64Subgroup32DeviceTests.cs new file mode 100644 index 0000000..e6f9096 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64Subgroup32DeviceTests.cs @@ -0,0 +1,183 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.Libs.Tests.Gpu.Vulkan; + +public sealed class Wave64Subgroup32DeviceTests(HeadlessVulkanFixture fixture) : Wave64HalfMaskDeviceTests(fixture) +{ + protected override bool HostSubgroup32 => true; + + [Fact] + public void LaneLocalArithmetic_UsesPairedLanesUnlessHalfWavesAreEnabled() + { + if (!Ready()) return; + Lane(rowShift: 4); + Add(Vop2(0, "VAddU32", 4, Operand(17), Gen5Operand.Vector(2))); + + var lanes = Run(sizeX: 16, sizeY: 16, laneLocal: true); + for (uint index = 0; index < 256; index++) + { + Assert.Equal(index + 17, lanes[index]); + } + } + + [Theory] + [InlineData(45u)] + [InlineData(7u)] + public void ReadLane_ReadsEitherHalfOfTheWave(uint lane) + { + if (!Ready()) return; + Lane(); + Add(Vop2(0, "VAddU32", 3, Operand(100), Gen5Operand.Vector(2))); + Add(ReadLane(0, 20, 3, lane)); + Add(MoveVectorFromScalar(0, 4, 20)); + + Assert.All(Run(laneLocal: false), value => Assert.Equal(100 + lane, value)); + } + + [Fact] + public void WriteLane_WritesOneLaneOfTheUpperHalf() + { + if (!Ready()) return; + Lane(); + Add(MoveVector(0, 4, 1)); + Add(MoveScalar(0, 20, 9)); + Add(WriteLane(0, 4, 20, 50)); + + var lanes = Run(laneLocal: false); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal(lane == 50 ? 9u : 1u, lanes[lane]); + } + } + + [Fact] + public void EveryWaveOfALargeGroup_CountsItsOwnLanes() + { + if (!Ready()) return; + Lane(rowShift: 4); + CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(100)); + Add(Sop1(0, "SBcnt1I32B64", 14, Gen5Operand.Scalar(10))); + Add(MoveVectorFromScalar(0, 4, 14)); + + var lanes = Run(sizeX: 16, sizeY: 16, laneLocal: false); + for (uint index = 0; index < 256; index++) + { + var wave = index / 64; + Assert.Equal(wave == 0 ? 64u : wave == 1 ? 36u : 0u, lanes[index]); + } + } + + [Fact] + public void APartialWave_HasOnlyItsThreadsInExec() + { + if (!Ready()) return; + Lane(); + Add(Sop1(0, "SBcnt1I32B64", 14, Gen5Operand.Scalar(Exec))); + Add(MoveVectorFromScalar(0, 4, 14)); + + Assert.All(Run(sizeX: 8, sizeY: 6, laneLocal: false), value => Assert.Equal(48u, value)); + } + + [Fact] + public void TwoWaves_ExchangeThroughSharedMemoryAfterABarrier() + { + if (!Ready()) return; + Lane(rowShift: 4); + Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(2))); + Add(Vop2(0, "VLshlrevB32", 6, Operand(2), Gen5Operand.Vector(2))); + Add(DataShare(0, "DsWriteB32", false, [Gen5Operand.Vector(6), Gen5Operand.Vector(3)], [])); + Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Sopp, "SBarrier", [0u], [], [], null)); + Add(Vop2(0, "VSubU32", 7, Operand(127), Gen5Operand.Vector(2))); + Add(Vop2(0, "VLshlrevB32", 7, Operand(2), Gen5Operand.Vector(7))); + Add(DataShare(0, "DsReadB32", false, [Gen5Operand.Vector(7)], [4])); + + var lanes = Run(localDataShareDwords: 128, sizeX: 16, sizeY: 8, laneLocal: false); + for (uint index = 0; index < 128; index++) + { + Assert.Equal(128 - index, lanes[index]); + } + } + + [Fact] + public void EveryHalfOfALargeGroup_SelectsItsOwnLanes() + { + if (!Ready()) return; + Lane(rowShift: 4); + CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(100)); + Add(Vop3(0, "VCndmaskB32", 4, Operand(0), Operand(1000), Gen5Operand.Scalar(10))); + Add(Vop2(0, "VAddU32", 4, Gen5Operand.Vector(2), Gen5Operand.Vector(4))); + + var lanes = Run(sizeX: 16, sizeY: 16, laneLocal: true); + for (uint index = 0; index < 256; index++) + { + Assert.Equal(index < 100 ? 1000 + index : index, lanes[index]); + } + } + + [Fact] + public void APartialLastWave_SelectsOnlyItsThreads() + { + if (!Ready()) return; + Lane(rowShift: 4); + CompareTo("VCmpGeU32", 10, Gen5Operand.Vector(2), Operand(40)); + Add(Vop3(0, "VCndmaskB32", 4, Operand(0), Operand(2000), Gen5Operand.Scalar(10))); + Add(Vop2(0, "VAddU32", 4, Gen5Operand.Vector(2), Gen5Operand.Vector(4))); + + var lanes = Run(sizeX: 16, sizeY: 5, laneLocal: true); + for (uint index = 0; index < 80; index++) + { + Assert.Equal(index >= 40 ? 2000 + index : index, lanes[index]); + } + } + + [Theory] + [InlineData(48u)] + [InlineData(8u)] + [InlineData(64u)] + public void AnExecSkip_LeavesTheInactiveLanesOfEachHalf(uint firstActiveLane) + { + if (!Ready()) return; + Lane(); + Add(MoveVector(0, 4, 5)); + Add(Vopc(0, "VCmpxLeU32", Operand(firstActiveLane), 2)); + var skip = Add(Branch(0, "SCbranchExecz", 0)); + Add(MoveVector(0, 4, 77)); + var join = Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Point(skip, join); + + var lanes = Run(laneLocal: true); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal(lane >= firstActiveLane ? 77u : 5u, lanes[lane]); + } + } + + [Fact] + public void AWaterfallLoop_VisitsEveryValueOfEveryHalf() + { + if (!Ready()) return; + Lane(rowShift: 4); + Add(Vop2(0, "VAndB32", 3, Operand(3), Gen5Operand.Vector(2))); + Add(Sop1(0, "SMovB64", 30, Gen5Operand.Scalar(Exec))); + var loop = Add(ReadFirstLane(0, 20, 3)); + CompareTo("VCmpEqU32", 22, Gen5Operand.Vector(3), Gen5Operand.Scalar(20)); + Add(Sop1(0, "SAndSaveexecB64", 24, Gen5Operand.Scalar(22))); + Add(MoveVectorFromScalar(0, 4, 20)); + Add(Vop2(0, "VAddU32", 4, Operand(100), Gen5Operand.Vector(4))); + Add(Sop2(0, "SXorB64", Exec, Gen5Operand.Scalar(24), Gen5Operand.Scalar(Exec))); + var back = Add(Branch(0, "SCbranchExecnz", 0)); + Point(back, loop); + Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(30))); + + var lanes = Run(sizeX: 16, sizeY: 16, laneLocal: true); + for (uint index = 0; index < 256; index++) + { + Assert.Equal((index & 3) + 100, lanes[index]); + } + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveMaxReductionDeviceTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveMaxReductionDeviceTests.cs index b97025d..e537c18 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveMaxReductionDeviceTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveMaxReductionDeviceTests.cs @@ -32,9 +32,11 @@ public sealed class WaveMaxReductionDeviceTests(HeadlessVulkanFixture fixture) : ]; [Theory] - [InlineData(0x7E040300u, 63u)] // v_mov_b32 v2, v0: the maximum is in the last lane - [InlineData(0x4C0400BFu, 63u)] // v_sub_nc_u32 v2, 63, v0: the maximum is in lane 0 - public void ReductionYieldsTheWaveMaximum(uint prepareValue, uint expected) + [InlineData(0x7E040300u, 63u, false)] // v_mov_b32 v2, v0: the maximum is in the last lane + [InlineData(0x4C0400BFu, 63u, false)] // v_sub_nc_u32 v2, 63, v0: the maximum is in lane 0 + [InlineData(0x7E040300u, 63u, true)] + [InlineData(0x4C0400BFu, 63u, true)] + public void ReductionYieldsTheWaveMaximum(uint prepareValue, uint expected, bool hostSubgroup32) { var vulkan = fixture.Vulkan; if (!GatePrerequisites.Ready(vulkan, shaderInt64: true)) return; @@ -58,7 +60,7 @@ public sealed class WaveMaxReductionDeviceTests(HeadlessVulkanFixture fixture) : var (plan, resources, layout) = Prepare(program); var request = new ShaderCompileRequest(plan, resources, layout) { - LocalSizeX = 64, ThreadCountX = 64, WaveSize = 64, + LocalSizeX = 64, ThreadCountX = 64, WaveSize = 64, HostComputeSubgroup32 = hostSubgroup32, }; Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); using var harness = new ImageTestHarness(vulkan); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveShuffleDeviceTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveShuffleDeviceTests.cs index bd09267..f7d2bf7 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveShuffleDeviceTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/WaveShuffleDeviceTests.cs @@ -17,12 +17,17 @@ public sealed class WaveShuffleDeviceTests(HeadlessVulkanFixture fixture, ITestO : IClassFixture { [Theory] - [InlineData("Dpp8")] - [InlineData("Dpp16")] - [InlineData("Permlane16")] - [InlineData("Permlanex16")] - [InlineData("Reduction")] - public void Wave64Shuffle_PreservesUpperHalf(string operation) + [InlineData("Dpp8", false)] + [InlineData("Dpp16", false)] + [InlineData("Permlane16", false)] + [InlineData("Permlanex16", false)] + [InlineData("Reduction", false)] + [InlineData("Dpp8", true)] + [InlineData("Dpp16", true)] + [InlineData("Permlane16", true)] + [InlineData("Permlanex16", true)] + [InlineData("Reduction", true)] + public void Wave64Shuffle_PreservesUpperHalf(string operation, bool hostSubgroup32) { var vulkan = fixture.Vulkan; if (!GatePrerequisites.Ready(vulkan, shaderInt64: true)) return; @@ -80,6 +85,7 @@ public sealed class WaveShuffleDeviceTests(HeadlessVulkanFixture fixture, ITestO var request = new ShaderCompileRequest(plan, resources, layout) { LocalSizeX = 8, LocalSizeY = 8, ThreadCountX = 8, ThreadCountY = 8, WaveSize = 64, + HostComputeSubgroup32 = hostSubgroup32, }; Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); using var harness = new ImageTestHarness(vulkan); diff --git a/tests/SharpEmu.Libs.Tests/VideoOut/ConcurrentResolveTests.cs b/tests/SharpEmu.Libs.Tests/VideoOut/ConcurrentResolveTests.cs new file mode 100644 index 0000000..84652a3 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/VideoOut/ConcurrentResolveTests.cs @@ -0,0 +1,75 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.VideoOut; +using Xunit; +using static SharpEmu.Libs.VideoOut.VulkanVideoPresenter; + +namespace SharpEmu.Libs.Tests.VideoOut; + +public sealed class ConcurrentResolveTests +{ + [Fact] + public void ConsecutiveAndRepeatedReadsKeepOneProofPerQuery() + { + var assumptions = new ConcurrentAssumptions(1, 2); + assumptions.Record(ConcurrentQuery.ResidentClean, 0x1000, 4, false); + assumptions.Record(ConcurrentQuery.CleanWord, 0x1000, 4, false); + var allocated = GC.GetAllocatedBytesForCurrentThread(); + for (var index = 0; index < 100000; index++) + { + var address = 0x1000UL + (ulong)index * 4; + assumptions.Record(ConcurrentQuery.ResidentClean, address, 4, false); + assumptions.Record(ConcurrentQuery.CleanWord, address, 4, false); + assumptions.Record(ConcurrentQuery.ResidentClean, address, 4, false); + } + + Assert.Equal(allocated, GC.GetAllocatedBytesForCurrentThread()); + Assert.Equal(2, assumptions.Answers.Count); + Assert.All(assumptions.Answers, proof => + { + Assert.Equal(0x1000UL, proof.Address); + Assert.Equal(400000UL, proof.Size); + Assert.False(proof.Answer); + }); + Assert.True(assumptions.ReadsImageOwnership); + } + + [Fact] + public void OverlappingReadsMergeWithoutAddingUnreadGaps() + { + var assumptions = new ConcurrentAssumptions(1, 2); + assumptions.Record(ConcurrentQuery.DirtyBytes, 0x1020, 0x20, false); + assumptions.Record(ConcurrentQuery.DirtyBytes, 0x1000, 0x30, false); + assumptions.Record(ConcurrentQuery.DirtyBytes, 0x1050, 0x10, false); + + Assert.Equal((ConcurrentQuery.DirtyBytes, 0x1000UL, 0x40UL, false), assumptions.Answers[0]); + Assert.Equal((ConcurrentQuery.DirtyBytes, 0x1050UL, 0x10UL, false), assumptions.Answers[1]); + Assert.False(assumptions.ReadsImageOwnership); + } + + [Fact] + public void OwnedReadsAndDifferentOwnershipRulesRemainSeparate() + { + var assumptions = new ConcurrentAssumptions(1, 2); + assumptions.Record(ConcurrentQuery.Resident, 0x1000, 4, false); + assumptions.Record(ConcurrentQuery.ResidentClean, 0x1000, 4, false); + assumptions.Record(ConcurrentQuery.Resident, 0x1004, 4, true); + assumptions.Record(ConcurrentQuery.Resident, 0x1008, 4, true); + assumptions.Record(ConcurrentQuery.Resident, 0x1004, 4, true); + + Assert.Equal(4, assumptions.Answers.Count); + Assert.Equal(2, assumptions.Answers.Count(proof => proof.Answer)); + Assert.True(assumptions.ReadsImageOwnership); + } + + [Fact] + public void WrappingRangesDoNotMergeWithLowAddresses() + { + var assumptions = new ConcurrentAssumptions(1, 2); + assumptions.Record(ConcurrentQuery.Resident, ulong.MaxValue - 1, 4, false); + assumptions.Record(ConcurrentQuery.Resident, 0, 4, false); + + Assert.Equal(2, assumptions.Answers.Count); + } +} diff --git a/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs b/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs index 5592b49..c604ec1 100644 --- a/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs +++ b/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs @@ -92,6 +92,41 @@ public sealed class PresenterImageBindingTests : IClassFixture + { + var resource = new ImageResource + { + ResourceClass = PlanImageResourceClass.Sampled, + NumericClass = ImageNumericClass.Float, + Dimension = ImageDimension.Dim2D, + Read = true, + }; + var program = new ShaderProgramInfo(); + var words = RegisterWords.Texture(address, GuestPixelFormat.Bits32Float, 48, 24); + var first = (ResourceSlotIdentifier)GetFieldValue(presenter.InvokeMethod("ResolveImageBinding", resource, words, program, 0)!, "ImageIdentifier"); + var second = (ResourceSlotIdentifier)GetFieldValue(presenter.InvokeMethod("ResolveImageBinding", resource, words, program, 0)!, "ImageIdentifier"); + Assert.Equal(first, second); + + // A removed image is not served again: the binding looks the descriptor up afresh. + var generation = harness.Images.LookupGeneration; + typeof(GuestImageCache).GetMethod("DeleteImage", InstanceMembers)!.Invoke(harness.Images, [first]); + Assert.NotEqual(generation, harness.Images.LookupGeneration); + var third = (ResourceSlotIdentifier)GetFieldValue(presenter.InvokeMethod("ResolveImageBinding", resource, words, program, 0)!, "ImageIdentifier"); + Assert.True(harness.Images.TryGetImage(third, out var image) && image.Registered); + presenter.RenderHost.ResetBindings(); + }); + harness.Shutdown(); + } + [Theory] [InlineData(true, 1u, false)] [InlineData(true, 2u, true)] diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5MipStatisticsTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5MipStatisticsTests.cs new file mode 100644 index 0000000..ba14e13 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5MipStatisticsTests.cs @@ -0,0 +1,118 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using SharpEmu.ShaderCompiler.Resources; +using SharpEmu.ShaderCompiler.Tests.Resources; +using SharpEmu.ShaderCompiler.Vulkan; +using Xunit; + +namespace SharpEmu.ShaderCompiler.Tests; + +public sealed class Gen5MipStatisticsTests +{ + private const ulong ShaderAddress = 0x1_0000_C000; + + private static (ShaderCompileRequest Request, BindingLayout Layout) PixelSample(string opcode, bool recordStatistics, bool reduce) + { + var addressCount = opcode.Contains("SampleL", StringComparison.Ordinal) && !opcode.Contains("Lz", StringComparison.Ordinal) ? 3 : 2; + var control = new Gen5ImageControl(0xF, 0, Enumerable.Range(0, addressCount).Select(static value => (uint)value).ToArray(), 4, 8, 16, 1, + false, false, false, false, false); + var program = new Gen5ShaderProgram(ShaderAddress, + [ + new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Mimg, opcode, [], [], [], control), + new Gen5ShaderInstruction(8, Gen5ShaderEncoding.Sopp, "SEndpgm", [0xBF810000], [], [], null), + ]); + var plan = ShaderResourcePlan.Extract(program, ShaderStage.Pixel, ResourceTestProgram.Hash, 0, 20); + var userData = new uint[20]; + userData[8] = 0x20; + userData[9] = 71u << 20; + userData[11] = 9u << 28; + userData[13] = 1u << 25; + userData[14] = 7; + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, ResourceTestProgram.Inputs(userData), ref snapshot, ref specialization)); + specialization.Images[0] = specialization.Images[0] with { MipStatistics = recordStatistics }; + var resources = ResourceMaterializer.ApplyTo(plan, specialization); + var layout = BindingLayout.Allocate(resources.Info, BindingLayout.CollectUserDataRegisters(program, 0, 20), false, + ShaderCompileRequest.RequiresFlattenedTable(plan, resources), false); + var request = new ShaderCompileRequest(plan, resources, layout) + { + PixelOutputs = [new Gen5PixelOutputBinding(0, 0, Gen5PixelOutputKind.Float)], + MipStatisticsSubgroupReduction = reduce, + }; + return (request, layout); + } + + private static List Compile(ShaderCompileRequest request) + { + Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); + ValidateWhenAvailable(shader.Spirv); + var opcodes = new List(); + for (var index = 20; index + 4 <= shader.Spirv.Length;) + { + var word = BitConverter.ToUInt32(shader.Spirv, index); + opcodes.Add((SpirvOp)(word & 0xFFFF)); + index += (int)(word >> 16) * sizeof(uint); + } + + return opcodes; + } + + [Theory] + [InlineData("ImageSample", false)] + [InlineData("ImageSample", true)] + [InlineData("ImageSampleL", false)] + [InlineData("ImageSampleLz", true)] + public void ASampledImageWithACounterRecordsItsFinestMip(string opcode, bool reduce) + { + var (request, layout) = PixelSample(opcode, recordStatistics: true, reduce); + + Assert.Equal(1u, layout.MipStatisticsCount); + Assert.NotNull(layout.Find(DescriptorBindingKind.MipStatistics)); + var opcodes = Compile(request); + Assert.Contains(SpirvOp.AtomicUMin, opcodes); + Assert.Equal(opcode == "ImageSample", opcodes.Contains(SpirvOp.ImageQueryLod)); + Assert.Equal(reduce, opcodes.Contains(SpirvOp.GroupNonUniformUMin)); + } + + [Fact] + public void ADrawWithoutCountersCompilesThePlainProgram() + { + var (request, layout) = PixelSample("ImageSample", recordStatistics: false, reduce: true); + + Assert.Equal(0u, layout.MipStatisticsCount); + Assert.Null(layout.Find(DescriptorBindingKind.MipStatistics)); + Assert.DoesNotContain(SpirvOp.AtomicUMin, Compile(request)); + } + + internal static void ValidateWhenAvailable(byte[] code) + { + var sdk = Environment.GetEnvironmentVariable("VULKAN_SDK"); + var executable = string.IsNullOrWhiteSpace(sdk) ? null + : Path.Combine(sdk, OperatingSystem.IsWindows() ? "Bin/spirv-val.exe" : "bin/spirv-val"); + if (executable is null || !File.Exists(executable)) + { + return; + } + + var path = Path.GetTempFileName(); + try + { + File.WriteAllBytes(path, code); + var start = new ProcessStartInfo(executable) { UseShellExecute = false, RedirectStandardError = true, RedirectStandardOutput = true, CreateNoWindow = true }; + start.ArgumentList.Add("--target-env"); + start.ArgumentList.Add("vulkan1.2"); + start.ArgumentList.Add(path); + using var process = Process.Start(start)!; + var error = process.StandardError.ReadToEnd() + process.StandardOutput.ReadToEnd(); + process.WaitForExit(); + Assert.True(process.ExitCode == 0, error); + } + finally + { + File.Delete(path); + } + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5VectorObservationAnalysisTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5VectorObservationAnalysisTests.cs new file mode 100644 index 0000000..e49dc6e --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5VectorObservationAnalysisTests.cs @@ -0,0 +1,98 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Ir; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.ShaderCompiler.Tests; + +public sealed class Gen5VectorObservationAnalysisTests +{ + private const uint Exec = 126; + + private readonly List _instructions = []; + private uint _pc; + + private uint Add(Gen5ShaderInstruction instruction) + { + var pc = _pc; + _instructions.Add(instruction with { Pc = pc }); + _pc += 8; + return pc; + } + + private Gen5VectorObservationPlan Analyze() + { + Add(EndProgram(0)); + var plan = Gen5VectorObservationAnalysis.Analyze(Program([.. _instructions]), wave32: false); + Assert.NotNull(plan); + return plan; + } + + private void Narrow() => Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + + private void SaveExec() => Add(Sop1(0, "SMovB64", 20, Gen5Operand.Scalar(Exec))); + + private void RestoreExec() => Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(20))); + + [Fact] + public void ATemporaryOfANarrowedRegion_IsNotObserved() + { + SaveExec(); + Narrow(); + var temporary = Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(2))); + var result = Add(Vop2(0, "VAddU32", 4, Operand(1), Gen5Operand.Vector(3))); + RestoreExec(); + Add(Vop2(0, "VAddU32", 5, Operand(1), Gen5Operand.Vector(4))); + + var plan = Analyze(); + Assert.True(plan.UnobservedWrites(temporary).Contains(3)); + Assert.False(plan.UnobservedWrites(result).Contains(4)); + } + + [Fact] + public void ACrossLaneReadObservesInactiveLanes() + { + Narrow(); + var write = Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(2))); + Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop1, "VMovB32", [0u, 0u], [Gen5Operand.Vector(3)], [Gen5Operand.Vector(4)], + new Gen5DppControl(0x111, false, false, 0, 0, 0xF, 0xF))); + + Assert.False(Analyze().UnobservedWrites(write).Contains(3)); + } + + [Fact] + public void AFullWriteUnderAFullExecEndsTheObservation() + { + SaveExec(); + Narrow(); + var write = Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(2))); + RestoreExec(); + Add(MoveVector(0, 3, 9)); + Add(Vop2(0, "VAddU32", 4, Operand(1), Gen5Operand.Vector(3))); + + Assert.True(Analyze().UnobservedWrites(write).Contains(3)); + } + + [Fact] + public void AnAccumulatingWriteReadsItsDestination() + { + SaveExec(); + Narrow(); + var write = Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(2))); + RestoreExec(); + Add(Vop2(0, "VMacF32", 3, Gen5Operand.Vector(1), Gen5Operand.Vector(2))); + + Assert.False(Analyze().UnobservedWrites(write).Contains(3)); + } + + [Fact] + public void ARelativeMoveDisablesThePlan() + { + Add(Vop1(0, "VMovrelsB32", 3, Gen5Operand.Vector(4))); + Add(EndProgram(0)); + + Assert.Null(Gen5VectorObservationAnalysis.Analyze(Program([.. _instructions]), wave32: false)); + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLaneLocalAnalysisTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLaneLocalAnalysisTests.cs new file mode 100644 index 0000000..959eb32 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLaneLocalAnalysisTests.cs @@ -0,0 +1,262 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Ir; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.ShaderCompiler.Tests; + +public sealed class Gen5WaveLaneLocalAnalysisTests +{ + private const uint Exec = 126; + private static readonly Gen5Operand AllLanes = Gen5Operand.Source(193); + + private readonly List _instructions = []; + private uint _pc; + + private uint Add(Gen5ShaderInstruction instruction) + { + var pc = _pc; + _instructions.Add(instruction with { Pc = pc }); + _pc += 8; + return pc; + } + + private void Point(uint branchPc, uint targetPc) + { + var index = _instructions.FindIndex(instruction => instruction.Pc == branchPc); + var offset = (short)(((int)targetPc - (int)branchPc - 4) / 4); + _instructions[index] = _instructions[index] with { Words = [unchecked((uint)(ushort)offset)] }; + } + + private void Lane() + { + Add(Vop2(0, "VLshlrevB32", 2, Operand(3), Gen5Operand.Vector(1))); + Add(Vop2(0, "VAddI32", 2, Gen5Operand.Vector(0), Gen5Operand.Vector(2))); + } + + private void CompareTo(string opcode, uint destination, Gen5Operand left, Gen5Operand right) => + Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop3, opcode, [0u, 0u], [left, right], [Gen5Operand.Scalar(destination)], + new Gen5Vop3Control(0, 0, 0, false, 0, destination))); + + private void SkipUnlessActive(Action body) + { + var skip = Add(Branch(0, "SCbranchExecz", 0)); + body(); + var join = Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Point(skip, join); + } + + private bool Analyze(out string reason, params uint[] sharedFlatPcs) + { + Add(Vop2(0, "VLshlrevB32", 5, Operand(2), Gen5Operand.Vector(2))); + Add(BufferAccess(0, "BufferStoreDword", 4, vectorData: 4, offsetEnabled: true, vectorAddress: 5)); + Add(EndProgram(0)); + return Gen5WaveLaneLocalAnalysis.IsLaneLocal(Program([.. _instructions]), sharedFlatPcs.ToHashSet(), out reason); + } + + private void AssertRefused(string expected) + { + Assert.False(Analyze(out var reason)); + Assert.Contains(expected, reason, StringComparison.Ordinal); + } + + [Fact] + public void LaneArithmetic_IsLaneLocal() + { + Lane(); + Add(Vop2(0, "VAddU32", 4, Operand(100), Gen5Operand.Vector(2))); + + Assert.True(Analyze(out _)); + } + + [Fact] + public void ACompareSelectingLanes_IsLaneLocal() + { + Lane(); + CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(40)); + Add(Vop3(0, "VCndmaskB32", 4, Operand(0), Operand(1000), Gen5Operand.Scalar(10))); + + Assert.True(Analyze(out _)); + } + + [Fact] + public void ReadingAnotherLane_IsRefused() + { + Lane(); + Add(ReadLane(0, 20, 2, 45)); + Add(MoveVectorFromScalar(0, 4, 20)); + + AssertRefused("V_READLANE"); + } + + [Fact] + public void CountingTheLanesOfAMask_IsRefused() + { + Lane(); + Add(Sop1(0, "SBcnt1I32B64", 14, Gen5Operand.Scalar(Exec))); + Add(MoveVectorFromScalar(0, 4, 14)); + + AssertRefused("a lane mask used as a scalar value"); + } + + [Fact] + public void BroadcastingAMaskToTheLanes_IsRefused() + { + Lane(); + CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(40)); + Add(MoveVectorFromScalar(0, 4, 10)); + + AssertRefused("a lane mask broadcast to the lanes"); + } + + [Fact] + public void SharedMemory_IsRefused() + { + Lane(); + Add(DataShare(0, "DsWriteB32", false, [Gen5Operand.Vector(2), Gen5Operand.Vector(2)], [])); + + AssertRefused("LDS"); + } + + [Fact] + public void ExecFromALiteralMask_IsRefused() + { + Lane(); + Add(Sop1(0, "SMovB64", Exec, Operand(0xFFFF))); + + AssertRefused("EXEC set from the wave's layout"); + } + + [Fact] + public void AnExecSkipAroundLaneWork_IsLaneLocal() + { + Lane(); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + SkipUnlessActive(() => Add(MoveVector(0, 4, 77))); + + Assert.True(Analyze(out _)); + } + + [Fact] + public void ScalarStateWrittenInAnExecSkip_IsRefused() + { + Lane(); + Add(MoveScalar(0, 20, 5)); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + SkipUnlessActive(() => Add(MoveScalar(0, 20, 77))); + Add(MoveVectorFromScalar(0, 4, 20)); + + AssertRefused("scalar state escapes a lane-mask branch"); + } + + [Fact] + public void AMaskKeptThroughItsComplementInAnExecSkip_IsLaneLocal() + { + Lane(); + Add(Sop1(0, "SMovB64", 2, Gen5Operand.Scalar(Exec))); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + var skip = Add(Branch(0, "SCbranchExecz", 0)); + Add(Sop1(0, "SMovB64", 8, Gen5Operand.Scalar(Exec))); + Add(Vopc(0, "VCmpNeU32", Operand(0), 2)); + Add(Sop2(0, "SOrn2B64", 106, Gen5Operand.Scalar(106), Gen5Operand.Scalar(8))); + Add(Sop2(0, "SAndB64", 2, Gen5Operand.Scalar(2), Gen5Operand.Scalar(106))); + var join = Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(2))); + Point(skip, join); + + Assert.True(Analyze(out var reason), reason); + } + + [Fact] + public void AMaskChangedInAnExecSkip_IsRefused() + { + Lane(); + Add(Sop1(0, "SMovB64", 2, Gen5Operand.Scalar(Exec))); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + var skip = Add(Branch(0, "SCbranchExecz", 0)); + Add(Sop1(0, "SMovB64", 8, Gen5Operand.Scalar(Exec))); + Add(Sop2(0, "SOrB64", 2, Gen5Operand.Scalar(2), Gen5Operand.Scalar(8))); + Add(Sop1(0, "SNotB64", 2, Gen5Operand.Scalar(2))); + var join = Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Point(skip, join); + Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(2))); + + AssertRefused("scalar state escapes a lane-mask branch"); + } + + [Fact] + public void AWideMultiplyOfAScalarBesideAMask_IsLaneLocal() + { + Lane(); + Add(Sop2(0, "SCselectB64", 10, Gen5Operand.Scalar(Exec), Operand(0))); + Add(Vop3(0, "VMadU64U32", 4, Gen5Operand.Scalar(9), Gen5Operand.Vector(2), Operand(0))); + + Assert.True(Analyze(out var reason), reason); + } + + [Fact] + public void ABarrierInAnExecSkip_IsRefused() + { + Lane(); + Add(Vopc(0, "VCmpxLeU32", Operand(40), 2)); + SkipUnlessActive(() => Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Sopp, "SBarrier", [0u], [], [], null))); + + AssertRefused("a barrier"); + } + + [Fact] + public void ReadFirstLaneOfAUniformValue_IsLaneLocal() + { + Lane(); + Add(MoveScalar(0, 20, 9)); + Add(MoveVectorFromScalar(0, 3, 20)); + Add(ReadFirstLane(0, 21, 3)); + Add(MoveVectorFromScalar(0, 4, 21)); + + Assert.True(Analyze(out _)); + } + + [Fact] + public void ReadFirstLaneOfALaneValue_IsRefused() + { + Lane(); + Add(ReadFirstLane(0, 21, 2)); + Add(MoveVectorFromScalar(0, 4, 21)); + + AssertRefused("outside a waterfall loop"); + } + + [Fact] + public void AWaterfallLoop_IsLaneLocal() + { + Lane(); + Add(Vop2(0, "VAndB32", 3, Operand(3), Gen5Operand.Vector(2))); + Add(Sop1(0, "SMovB64", 30, Gen5Operand.Scalar(Exec))); + var loop = Add(ReadFirstLane(0, 20, 3)); + CompareTo("VCmpEqU32", 22, Gen5Operand.Vector(3), Gen5Operand.Scalar(20)); + Add(Sop1(0, "SAndSaveexecB64", 24, Gen5Operand.Scalar(22))); + Add(MoveVectorFromScalar(0, 4, 20)); + Add(Sop2(0, "SXorB64", Exec, Gen5Operand.Scalar(24), Gen5Operand.Scalar(Exec))); + var back = Add(Branch(0, "SCbranchExecnz", 0)); + Point(back, loop); + Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(30))); + + Assert.True(Analyze(out var reason), reason); + } + + [Theory] + [InlineData(0x111u, true)] + [InlineData(0x141u, true)] + [InlineData(0x142u, false)] + [InlineData(0x143u, false)] + public void Dpp_IsLaneLocalOnlyWithinARow(uint control, bool laneLocal) + { + Lane(); + Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop1, "VMovB32", [0u, 0u], [Gen5Operand.Vector(2)], [Gen5Operand.Vector(4)], + new Gen5DppControl(control, false, false, 0, 0, 0xF, 0xF))); + + Assert.Equal(laneLocal, Analyze(out var reason)); + Assert.True(laneLocal || reason.Contains("cross-row DPP", StringComparison.Ordinal)); + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLoweringTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLoweringTests.cs new file mode 100644 index 0000000..d1f598e --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLoweringTests.cs @@ -0,0 +1,250 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using System.Text; +using SharpEmu.ShaderCompiler.Vulkan; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.ShaderCompiler.Tests; + +// Checks the SPIR-V each wave lowering (NativeWave32, NativeWave64, EmulatedWave64) produces for the +// same guest programs: which subgroup instructions appear, how a ballot is split into a 64-bit mask, +// and whether the workgroup exchange / rendezvous machinery exists. +public sealed class Gen5WaveLoweringTests +{ + public enum Lowering { Wave32, NativeWave64, EmulatedWave64 } + + private sealed record Module( + IReadOnlyList<(SpirvOp Op, uint[] Operands)> Instructions, + IReadOnlyList Names) + { + public int Count(SpirvOp op) => Instructions.Count(instruction => instruction.Op == op); + + // CompositeExtract of a ballot result: its literal index is the last operand. + public IReadOnlyList BallotExtractIndices() + { + var ballots = Instructions.Where(i => i.Op == SpirvOp.GroupNonUniformBallot).Select(i => i.Operands[1]).ToHashSet(); + return Instructions + .Where(i => i.Op == SpirvOp.CompositeExtract && ballots.Contains(i.Operands[2])) + .Select(i => i.Operands[^1]) + .ToList(); + } + } + + private static Module Parse(byte[] spirv) + { + var instructions = new List<(SpirvOp, uint[])>(); + var names = new List(); + for (var offset = 20; offset < spirv.Length;) + { + var word = BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset)); + var length = (int)(word >> 16); + var op = (SpirvOp)(word & 0xFFFF); + var operands = new uint[length - 1]; + for (var index = 0; index < operands.Length; index++) + { + operands[index] = BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset + 4 + index * 4)); + } + + instructions.Add((op, operands)); + if (op == SpirvOp.Name) + { + var text = new List(); + for (var index = 1; index < operands.Length; index++) + { + text.AddRange(BitConverter.GetBytes(operands[index])); + } + + names.Add(Encoding.UTF8.GetString(text.ToArray()).TrimEnd('\0')); + } + + offset += length * 4; + } + + return new Module(instructions, names); + } + + private static Module Compile(Gen5ShaderProgram program, Lowering lowering, uint threads = 64) + { + var (plan, resources, layout) = Prepare(program); + var configured = new ShaderCompileRequest(plan, resources, layout) + { + WaveSize = lowering == Lowering.Wave32 ? 32u : 64u, + HostWave64Supported = lowering == Lowering.NativeWave64, + LocalSizeX = threads, + }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(configured, out var shader, out var error), error); + return Parse(shader.Spirv); + } + + // v_cmp_lt_u32 vcc, v0, v1 : a per-lane compare whose result becomes a wave mask. + private static Gen5ShaderProgram CompareProgram() => Program( + Vopc(0, "VCmpLtU32", Operand(5), 1), + EndProgram(4)); + + private static Gen5ShaderProgram ReadFirstLaneProgram() => Program( + ReadFirstLane(0, 4, 0), + EndProgram(4)); + + private static Gen5ShaderProgram ReadLaneProgram(uint lane) => Program( + ReadLane(0, 4, 0, lane), + EndProgram(4)); + + private static Gen5ShaderProgram DataShareProgram(bool barrierBetween) => + Program( + new[] + { + DataShare(0, "DsWriteB32", false, [Gen5Operand.Vector(0), Gen5Operand.Vector(1)], [], 0, 0), + } + .Concat(barrierBetween + ? new[] { new Gen5ShaderInstruction(4, Gen5ShaderEncoding.Sopp, "SBarrier", [0u], [], [], null) } + : []) + .Concat(new[] + { + DataShare(8, "DsReadB32", false, [Gen5Operand.Vector(0)], [2u], 0, 0), + EndProgram(16), + }) + .ToArray()); + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + [InlineData(Lowering.EmulatedWave64)] + public void EveryLoweringCompilesTheBasicWavePrograms(Lowering lowering) + { + foreach (var program in new[] { CompareProgram(), ReadFirstLaneProgram(), ReadLaneProgram(0), ReadLaneProgram(33), DataShareProgram(false), DataShareProgram(true) }) + { + Compile(program, lowering); + } + } + + [Fact] + public void Wave32_BallotUsesOnlyTheLowWord() + { + var module = Compile(CompareProgram(), Lowering.Wave32); + Assert.True(module.Count(SpirvOp.GroupNonUniformBallot) > 0); + Assert.All(module.BallotExtractIndices(), index => Assert.Equal(0u, index)); + } + + [Fact] + public void NativeWave64_BallotCombinesBothWords() + { + var module = Compile(CompareProgram(), Lowering.NativeWave64); + var indices = module.BallotExtractIndices(); + Assert.Contains(0u, indices); + Assert.Contains(1u, indices); + } + + [Fact] + public void OnlyTheEmulatedLoweringDeclaresTheExchange() + { + foreach (var lowering in new[] { Lowering.Wave32, Lowering.NativeWave64 }) + { + var module = Compile(ReadFirstLaneProgram(), lowering); + Assert.DoesNotContain("wave64Exchange", module.Names); + Assert.DoesNotContain("wave64ExchangeParity", module.Names); + } + + var emulated = Compile(ReadFirstLaneProgram(), Lowering.EmulatedWave64); + Assert.Contains("wave64ExchangeParity", emulated.Names); + Assert.Contains("wave64Exchange", emulated.Names); + } + + [Fact] + public void EmulatedWave64_NeedsNoExchangeWithoutSubgroupOperations() + { + var module = Compile(Program(Vop2(0, "VAddI32", 1, Operand(1), Gen5Operand.Vector(0)), EndProgram(4)), Lowering.EmulatedWave64); + Assert.DoesNotContain("wave64ExchangeParity", module.Names); + } + + [Theory] + [InlineData(Lowering.Wave32, 64u)] + [InlineData(Lowering.NativeWave64, 64u)] + [InlineData(Lowering.NativeWave64, 256u)] + public void NativeLoweringsAddNoBarriersAroundSharedMemory(Lowering lowering, uint threads) + { + var module = Compile(DataShareProgram(false), lowering, threads); + Assert.Equal(0, module.Count(SpirvOp.ControlBarrier)); + } + + [Theory] + [InlineData(false, 2)] + [InlineData(true, 2)] + public void EmulatedWave64_OrdersSharedMemoryPhasesWithinOneWave(bool explicitBarrier, int expectedBarriers) + { + var module = Compile(DataShareProgram(explicitBarrier), Lowering.EmulatedWave64, threads: 64); + Assert.Equal(expectedBarriers, module.Count(SpirvOp.ControlBarrier)); + } + + [Fact] + public void EmulatedWave64_MultiWaveGroupsUseNoWorkgroupBarrierForSharedMemoryPhases() + { + var module = Compile(DataShareProgram(false), Lowering.EmulatedWave64, threads: 128); + Assert.Equal(0, module.Count(SpirvOp.ControlBarrier)); + } + + [Fact] + public void EmulatedWave64_MultiWaveExchangeMeetsAtAnArrivalCounter() + { + var single = Compile(ReadFirstLaneProgram(), Lowering.EmulatedWave64, threads: 64); + var multi = Compile(ReadFirstLaneProgram(), Lowering.EmulatedWave64, threads: 256); + Assert.DoesNotContain("wave64RendezvousEpoch", single.Names); + Assert.Contains("wave64RendezvousEpoch", multi.Names); + Assert.True(multi.Count(SpirvOp.AtomicIAdd) > single.Count(SpirvOp.AtomicIAdd)); + } + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + public void NativeLoweringsNeverDeclareARendezvous(Lowering lowering) + { + var module = Compile(ReadFirstLaneProgram(), lowering, threads: 256); + Assert.DoesNotContain("wave64RendezvousEpoch", module.Names); + Assert.Equal(0, module.Count(SpirvOp.AtomicIAdd)); + } + + [Fact] + public void HostWave64SupportDoesNotChangeAWave32Shader() + { + var program = ReadFirstLaneProgram(); + var (plan, resources, layout) = Prepare(program); + byte[] Compile32(bool hostWave64) + { + var configured = new ShaderCompileRequest(plan, resources, layout) + { + WaveSize = 32, + HostWave64Supported = hostWave64, + LocalSizeX = 64, + }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(configured, out var shader, out var error), error); + return shader.Spirv.ToArray(); + } + + Assert.Equal(Compile32(false), Compile32(true)); + } + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + [InlineData(Lowering.EmulatedWave64)] + public void ReadLaneBroadcastsFromTheRequestedLane(Lowering lowering) + { + var module = Compile(ReadLaneProgram(33), lowering); + Assert.True( + module.Count(SpirvOp.GroupNonUniformBroadcast) > 0 || module.Count(SpirvOp.GroupNonUniformShuffle) > 0 || + module.Count(SpirvOp.Store) > 0); + } + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + [InlineData(Lowering.EmulatedWave64)] + public void LoweringsAreDeterministic(Lowering lowering) + { + var first = Compile(CompareProgram(), lowering); + var second = Compile(CompareProgram(), lowering); + Assert.Equal(first.Instructions.Count, second.Instructions.Count); + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/BindingLayoutTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/BindingLayoutTests.cs index 165bb08..300f151 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/BindingLayoutTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/BindingLayoutTests.cs @@ -158,7 +158,8 @@ public sealed class BindingLayoutTests Assert.Equal(49u, (uint)DescriptorBindingKind.FaultBuffer); Assert.Equal(50u, (uint)DescriptorBindingKind.FlattenedResourceTable); Assert.Equal(51u, (uint)DescriptorBindingKind.ShaderData); - Assert.Equal(52u, (uint)DescriptorBindingKind.Count); + Assert.Equal(52u, (uint)DescriptorBindingKind.MipStatistics); + Assert.Equal(53u, (uint)DescriptorBindingKind.Count); ImageDimension[] sampledDimensions = [ diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs index e1f7354..a7b71ae 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs @@ -153,6 +153,26 @@ public sealed class ResourceMaterializationCacheTests Assert.Equal(0, cache.Hits); } + private static ShaderResourcePlan PlanWithSpareUserData() => + ShaderResourcePlan.Extract(DirectImageTableTests.CreateWaveIndexedDescriptorProgram(), ShaderStage.Compute, Hash, 0, 4); + + [Fact] + public void AUserDataWordTheProgramNeverReadsDoesNotSplitTheEntry() + { + var plan = PlanWithSpareUserData(); + var heap = new Heap(); + var cache = new ResourceMaterializationCache(); + Assert.True(Run(cache, plan, heap, [0x1000, 0, 5, 6], out var first, out _)); + Assert.True(Run(cache, plan, heap, [0x1000, 0, 7, 8], out var second, out _)); + Assert.Equal((1, 1), (cache.Hits, cache.Misses)); + // The hit carries the second draw's user data and leaves the first draw's snapshot alone. + Assert.Equal(new uint[] { 0x1000, 0, 7, 8 }, second.UserData); + Assert.Equal(new uint[] { 0x1000, 0, 5, 6 }, first.UserData); + Assert.Same(first.Images, second.Images); + Assert.True(Run(cache, plan, heap, [0x1000, 0, 5, 6], out var third, out _)); + Assert.Same(first, third); + } + [Fact] public void ADifferentDrawKeyIsADifferentEntry() { @@ -264,6 +284,276 @@ public sealed class ResourceMaterializationCacheTests Assert.Equal(0, cache.TableRefreshes); } + // A V# taken verbatim from s[4:7], the way titles pass per-draw constants. + private static ShaderResourcePlan DirectBufferPlan() => + Extract(Program(BufferLoad(0, 4), EndProgram(8)), userDataCount: 8); + + // The same V#, whose base pair s[4:5] is also the address of a scalar load. + private static ShaderResourcePlan SteeringBufferPlan() => + Extract(Program(ScalarLoad(0, 4, 16), Vop1(8, "VMovB32", 0, Gen5Operand.Scalar(16)), BufferStore(12, 4), EndProgram(20)), + userDataCount: 8); + + private static bool RunWords(ResourceMaterializationCache cache, ShaderResourcePlan plan, TestWordMemory memory, uint[] userData, + out ResourceSnapshot snapshot) + { + snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + return cache.Materialize(plan, Inputs(userData, memory.Read, memory.Read), (address, destination, _) => + { + for (var offset = 0; offset < destination.Length; offset += 4) + { + if (!memory.Read(address + (ulong)offset, out var word)) + return false; + BitConverter.TryWriteBytes(destination[offset..], word); + } + + return true; + }, ref snapshot, ref specialization, out _); + } + + private static uint[] Reference(ShaderResourcePlan plan, TestWordMemory memory, uint[] userData) + { + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref snapshot, ref specialization)); + return snapshot.Buffers[0]; + } + + private static uint[] LinearConstants(int draw) => [0, 0, 0, 0, 0x10000u + (uint)draw * 0x100u, 0, 256, 0x2_0000]; + + [Fact] + public void ALinearlyAllocatedBufferBaseStopsMissingOnceConfirmed() + { + var plan = DirectBufferPlan(); + var memory = new TestWordMemory(); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = LinearConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + Assert.Equal(Reference(plan, memory, userData), snapshot.Buffers[0]); + Assert.Equal(userData, snapshot.UserData); + } + + // One miss to build, three to confirm the word; every later draw is a hit. + Assert.Equal((12, 4), (cache.Hits, cache.Misses)); + } + + [Fact] + public void InterleavedObjectsConfirmTheirOwnBases() + { + // Constants V# in s[4:7], a second buffer in s[8:11] whose size tells the two objects apart. + var plan = Extract(Program(BufferLoad(0, 4), BufferLoad(8, 8), EndProgram(16)), userDataCount: 12); + var memory = new TestWordMemory(); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 32; draw++) + { + uint[] userData = [0, 0, 0, 0, 0x10000u + (uint)draw * 0x100u, 0, 256, 0x2_0000, 0x40000, 0, draw % 2 == 0 ? 256u : 512u, 0x2_0000]; + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + } + + Assert.True(cache.Hits >= 20, $"hits={cache.Hits} misses={cache.Misses}"); + } + + // Pointer s[0:1] moves on every draw; the program loads four constants through it. + private static uint[] PointerConstants(int draw) => [0x10000u + (uint)draw * 0x100u, 0, 0, 0, 0x30000, 0, 256, 0x2_0000, 0]; + + [Fact] + public void AMovingScalarLoadPointerStopsMissingOnceConfirmed() + { + var (plan, _, _) = Prepare(FlattenedReadReuseTests.RepeatedReadProgram(4), userDataCount: 9); + // The same four words repeat every 0x100 bytes, so every pointer finds the same constants. + var memory = new TestWordMemory { Base = 0x10000, Words = Enumerable.Range(0, 0x1000).Select(index => (uint)(index % 64) * 3 + 1).ToArray() }; + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = PointerConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + } + + Assert.Equal((12, 4), (cache.Hits, cache.Misses)); + + // Other constants behind the next pointer: the draw reads them instead of reusing the table. + var moved = PointerConstants(30); + memory.At(moved[0] + 8) = 0xBEEF; + Assert.True(RunWords(cache, plan, memory, moved, out var reread)); + Assert.Equal(0xBEEFu, reread.FlattenedResourceTable[2]); + Assert.Equal(12, cache.Hits); + } + + [Fact] + public void ARebasedHitLeavesTheCachedSnapshotAlone() + { + var plan = DirectBufferPlan(); + var memory = new TestWordMemory(); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 4; draw++) + Assert.True(RunWords(cache, plan, memory, LinearConstants(draw), out _)); + + Assert.True(RunWords(cache, plan, memory, LinearConstants(3), out var built)); + Assert.True(RunWords(cache, plan, memory, LinearConstants(9), out var patched)); + Assert.Equal(0x10900u, patched.Buffers[0][0]); + Assert.Equal(0x10300u, built.Buffers[0][0]); + Assert.NotSame(built.Buffers, patched.Buffers); + Assert.True(RunWords(cache, plan, memory, LinearConstants(3), out var again)); + Assert.Equal(0x10300u, again.Buffers[0][0]); + } + + [Fact] + public void AReadThatMovesWithTheBaseAndKeepsItsBytesIsReused() + { + var plan = SteeringBufferPlan(); + var memory = new TestWordMemory { Base = 0x10000, Words = Enumerable.Repeat(0x1234u, 0x1000).ToArray() }; + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = LinearConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers[0], snapshot.Buffers[0]); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + Assert.Equal((12, 4), (cache.Hits, cache.Misses)); + + // The moved read now finds other bytes at the new base: the draw must not reuse the table. + var changed = LinearConstants(20); + memory.At(changed[4]) = 0x5678; + Assert.True(RunWords(cache, plan, memory, changed, out var refreshed)); + Assert.Equal(0x5678u, refreshed.FlattenedResourceTable[0]); + Assert.Equal(12, cache.Hits); + } + + [Fact] + public void MovedConstantsThatChangeEveryDrawAreReadAgain() + { + var plan = SteeringBufferPlan(); + var memory = new TestWordMemory { Base = 0x10000, Words = Enumerable.Range(0, 0x1000).Select(index => (uint)index * 7).ToArray() }; + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = LinearConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + Assert.Equal(Reference(plan, memory, userData), snapshot.Buffers[0]); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + // The constants differ at every base, so nothing is reused as is; once the base is + // confirmed, the moved table is evaluated again instead of the whole walk. + Assert.Equal(0, cache.Hits); + Assert.True(cache.TableRefreshes >= 10, $"refreshes={cache.TableRefreshes} misses={cache.Misses}"); + } + + // The V# comes from a descriptor table at s[0:1]; its constants are read through it. + private static ShaderResourcePlan MemoryDescriptorPlan() => + Extract(Program(ScalarLoad(0, 0, 8, 4), ScalarBufferLoad(4, 8, 12, 4), Vop1(8, "VMovB32", 0, Gen5Operand.Scalar(12)), + BufferLoad(12, 8), EndProgram(16)), userDataCount: 2); + + // The title rewrites the table every draw: a freshly allocated base, and new constants there. + private static void WriteDraw(TestWordMemory memory, int draw) + { + var bufferBase = 0x10000u + (uint)draw * 0x100u; + memory.At(0x1000) = bufferBase; + memory.At(0x1004) = 0; + memory.At(0x1008) = 256; + memory.At(0x100C) = 0x2_0000; + for (var word = 0u; word < 4; word++) + memory.At(bufferBase + word * 4) = (uint)draw * 10 + word; + } + + [Fact] + public void ABaseRewrittenInGuestMemoryStopsMissingOnceConfirmed() + { + var plan = MemoryDescriptorPlan(); + var memory = new TestWordMemory { Base = 0x1000, Words = new uint[0x8000] }; + var cache = new ResourceMaterializationCache(); + uint[] userData = [0x1000, 0]; + for (var draw = 0; draw < 16; draw++) + { + WriteDraw(memory, draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + // One miss to build, three to confirm the base; the later draws patch it. + Assert.True(cache.Hits >= 10, $"hits={cache.Hits} misses={cache.Misses}"); + + // Another descriptor word changed with the base: the draw is materialized again. + WriteDraw(memory, 20); + memory.At(0x1008) = 512; + var hits = cache.Hits; + Assert.True(RunWords(cache, plan, memory, userData, out var resized)); + Assert.Equal(hits, cache.Hits); + Assert.Equal(512u, resized.Buffers[0][2]); + } + + private static void AssertMatchesFullWalk(ShaderResourcePlan plan, TestWordMemory memory, uint[] userData, ResourceSnapshot snapshot) + { + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + [Fact] + public void ATableConstantBesideARewrittenBaseIsCopiedForTheDraw() + { + // The V# and a constant both come from the table at s[0:1]; the constant changes every draw. + var plan = Extract(Program(ScalarLoad(0, 0, 8, 4), ScalarLoad(4, 0, 16, 1, immediateOffset: 16), + ScalarBufferLoad(8, 8, 12, 4), Vop1(12, "VMovB32", 0, Gen5Operand.Scalar(12)), + Vop1(16, "VMovB32", 1, Gen5Operand.Scalar(16)), BufferLoad(20, 8), EndProgram(24)), userDataCount: 2); + var memory = new TestWordMemory { Base = 0x1000, Words = new uint[0x8000] }; + var cache = new ResourceMaterializationCache(); + uint[] userData = [0x1000, 0]; + for (var draw = 0; draw < 16; draw++) + { + WriteDraw(memory, draw); + memory.At(0x1010) = (uint)draw * 3 + 1; + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + AssertMatchesFullWalk(plan, memory, userData, snapshot); + } + + Assert.True(cache.Hits >= 10, $"hits={cache.Hits} misses={cache.Misses}"); + } + + [Fact] + public void AlternatingObjectsWithRewrittenBasesRebaseTheirOwnVariant() + { + // Two objects share the draw key and differ in their buffer size; both move their base. + var plan = MemoryDescriptorPlan(); + var memory = new TestWordMemory { Base = 0x1000, Words = new uint[0x8000] }; + var cache = new ResourceMaterializationCache(); + uint[] userData = [0x1000, 0]; + for (var draw = 0; draw < 32; draw++) + { + WriteDraw(memory, draw); + memory.At(0x1008) = draw % 2 == 0 ? 256u : 512u; + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + AssertMatchesFullWalk(plan, memory, userData, snapshot); + } + + Assert.True(cache.Hits >= 20, $"hits={cache.Hits} misses={cache.Misses}"); + } + [Fact] public void AFullGenerationKeepsRecentEntries() { diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/SpirvBindingDeclarationTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/SpirvBindingDeclarationTests.cs index d493b24..469a517 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/SpirvBindingDeclarationTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/SpirvBindingDeclarationTests.cs @@ -408,6 +408,25 @@ public sealed class SpirvBindingDeclarationTests Assert.Contains((uint)SpirvCapability.GroupNonUniformShuffle, module.Capabilities); } + [Fact] + public void PixelAppendStartsWithoutHelperLanes() + { + var appending = Program( + MoveScalar(0, 124, 64), + DataShare(8, "DsAppend", gds: true, [Gen5Operand.Scalar(124)], [1]), + EndProgram(16)); + var plain = Program( + MoveScalar(0, 124, 64), + EndProgram(8)); + + Assert.True(Gen5SpirvTranslator.TryCompileProgram(Request(appending, ShaderStage.Pixel), out var appendShader, out var error), error); + Assert.True(Gen5SpirvTranslator.TryCompileProgram(Request(plain, ShaderStage.Pixel), out var plainShader, out error), error); + + Gen5MipStatisticsTests.ValidateWhenAvailable(appendShader.Spirv); + Assert.Contains("helperInvocation", new SpirvModuleInspector(appendShader.Spirv).Names.Values); + Assert.DoesNotContain("helperInvocation", new SpirvModuleInspector(plainShader.Spirv).Names.Values); + } + [Fact] public void DataShareBpermute_DeclaresTheShuffleCapability() {