[Kernel] Place mappings in untouched address space without waiting for the GPU worker

Every mapping change queued behind the GPU worker so it could not race a pending GPU read.
During loading that worker is busy translating shaders, and Demon's Souls' reservations and
batch maps waited seconds for a queue slot (23.6 s over 132 calls, up to 7.1 s each) while
the change itself took about a millisecond.

A change that only places memory where the GPU was never given anything - a kernel-chosen
address, or a fixed range that is empty or reserved and overlaps no registered span - cannot
race a GPU read and has nothing to unregister, so it now runs on the calling thread. Those
paths also skip the unregister request for free and reserved ranges, which are never
registered; waiting on the worker while holding the mapping lock that GPU reads take would
deadlock, so reaching a registered span there throws instead.

(cherry picked from commit 3fe148a262)

src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs

@@ -174,6 +174,26 @@ public sealed class GuestGpuMemory : IDisposable
         }
     }
 
+    // Whether any GPU-registered span intersects the range. Memory outside every span holds no GPU
+    // cache entries and no tracked pages, so mapping changes there need nothing from the GPU worker.
+    public bool OverlapsRegistered(ulong address, ulong size)
+    {
+        if (!new GuestSpan(address, size).IsValid)
+        {
+            return false;
+        }
+
+        _spansLock.EnterReadLock();
+        try
+        {
+            return _spans.Overlaps(address, size);
+        }
+        finally
+        {
+            _spansLock.ExitReadLock();
+        }
+    }
+
     // The host mapping takes the guest protection here; the views themselves are mapped read-write.
     public void Register(ulong address, ulong size, GuestPageProtection protection)
     {

src/SharpEmu.HLE/GpuMemory/GuestGpuMemoryHook.cs

@@ -102,6 +102,9 @@ public static class GuestGpuMemoryHook
     public static bool NoteProtected(ulong address, ulong size, GuestPageProtection protection) =>
         _current != null && IsWithinGpuAddressSpace(address, size) && _current.NoteProtected(address, size, protection);
 
+    public static bool OverlapsRegistered(ulong address, ulong size) =>
+        _current != null && IsWithinGpuAddressSpace(address, size) && _current.OverlapsRegistered(address, size);
+
     public static void NoteUnmapped(ulong address, ulong size)
     {
         if (_current == null || !IsWithinGpuAddressSpace(address, size))

src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.BackedMappings.cs

@@ -130,6 +130,23 @@ public static partial class KernelMemoryCompatExports
         BackingOffset = region.IsDirect || region.IsFlexible ? region.BackingOffset + start - region.Address : 0,
     };
 
+    // Empty or reserved-only address space that no GPU-registered span reaches.
+    private static bool IsUntouchedByGpu(ulong address, ulong size)
+    {
+        if (size == 0 || ulong.MaxValue - address < size)
+            return false;
+        lock (_memoryGate)
+        {
+            foreach (var region in GetMappingSlices(address, size))
+            {
+                if (!region.IsReserved)
+                    return false;
+            }
+        }
+
+        return !GuestGpuMemoryHook.OverlapsRegistered(address, size);
+    }
+
     private static MappedRegion[] GetMappingSlices(ulong address, ulong size, bool clip = true)
     {
         var lowerIndex = 0;
@@ -231,7 +248,7 @@ public static partial class KernelMemoryCompatExports
         {
             if (!space.TryHoldRange(address, size))
                 return false;
-            GuestGpuMemoryHook.NoteUnmapped(address, size);
+            UnregisterFreeRange(address, size);
             return true;
         }
         if (!TryUnmapViews(space, regions))
@@ -249,6 +266,19 @@ public static partial class KernelMemoryCompatExports
         return true;
     }
 
+    // Free and reserved ranges are never registered with the GPU, and unregistering hands the GPU
+    // worker a request to wait for. A direct transaction holds the mapping lock that GPU reads take,
+    // so waiting there would deadlock; it relies on this range having nothing to unregister.
+    private static void UnregisterFreeRange(ulong address, ulong size)
+    {
+        if (!GuestGpuMemoryHook.OverlapsRegistered(address, size))
+            return;
+        if (_directMappingTransaction)
+            throw new InvalidOperationException(
+                $"A free guest range is still registered with the GPU: address=0x{address:X16} size=0x{size:X}.");
+        GuestGpuMemoryHook.NoteUnmapped(address, size);
+    }
+
     private static bool TrySelectBackingAddress(IGuestBackedSpace space, ulong requested, ulong length,
         ulong alignment, ulong flags, out ulong address, bool reuseReservation = true)
     {
@@ -287,7 +317,7 @@ public static partial class KernelMemoryCompatExports
         }
         if (address == 0)
             return false;
-        GuestGpuMemoryHook.NoteUnmapped(address, length);
+        UnregisterFreeRange(address, length);
         return true;
     }
 
@@ -327,18 +357,58 @@ public static partial class KernelMemoryCompatExports
     }
 
     internal static int ReserveBackingRange(CpuContext ctx, ulong pointer, ulong length, ulong flags, ulong alignment)
-        => RunMappingTransaction(() => ReserveBackingRangeCore(ctx, pointer, length, flags, alignment));
+        => RunMappingTransaction(
+            () => ReserveBackingRangeCore(ctx, pointer, length, flags, alignment),
+            () => (flags & OrbisKernelMapFixed) == 0 ||
+                (ctx.TryReadUInt64(pointer, out var requested) && IsUntouchedByGpu(requested, length)));
 
-    private static int RunMappingTransaction(Func<int> transaction)
+    // A batch wraps its entries in one transaction, so the entries must not hand off again.
+    [ThreadStatic]
+    private static bool _insideMappingTransaction;
+
+    // Set while a transaction runs on the guest thread instead of the GPU worker.
+    [ThreadStatic]
+    private static bool _directMappingTransaction;
+
+    private static int RunMappingTransaction(Func<int> transaction, Func<bool>? onlyAddsOutsideGpuMemory = null)
     {
         // GPU handoff must precede locks needed by image and buffer reads.
         if (Monitor.IsEntered(_memoryGate))
             throw new InvalidOperationException("Cannot start a mapping transaction while holding the mapping lock.");
-        if (GuestGpuMemoryHook.Current is not { } memory)
+        if (_insideMappingTransaction || GuestGpuMemoryHook.Current is not { } memory)
             return transaction();
 
+        // A transaction that only places new mappings in address space the GPU has never been given
+        // (kernel-chosen placements, or empty/reserved ranges outside every registered span) cannot
+        // race a GPU read and unregisters nothing. It runs here instead of queueing behind the GPU
+        // worker, which during loading is busy compiling pipelines: Demon's Souls spent seconds per
+        // reservation waiting for a queue slot while the change itself took a millisecond.
+        if (onlyAddsOutsideGpuMemory?.Invoke() == true)
+        {
+            _insideMappingTransaction = _directMappingTransaction = true;
+            try
+            {
+                return transaction();
+            }
+            finally
+            {
+                _insideMappingTransaction = _directMappingTransaction = false;
+            }
+        }
+
         var result = MemoryFault;
-        memory.RunMappingChange(() => result = transaction());
+        memory.RunMappingChange(() =>
+        {
+            _insideMappingTransaction = true;
+            try
+            {
+                result = transaction();
+            }
+            finally
+            {
+                _insideMappingTransaction = false;
+            }
+        });
         return result;
     }
 

src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs

@@ -3150,8 +3150,14 @@ public static partial class KernelMemoryCompatExports
 
     private static int MapDirectMemoryCore(CpuContext ctx, ulong inOutAddressPointer, ulong length,
         int protection, ulong flags, ulong directMemoryStart, ulong alignment)
-        => RunMappingTransaction(() => MapDirectMemoryTransaction(ctx, inOutAddressPointer, length,
-            protection, flags, directMemoryStart, alignment));
+        => RunMappingTransaction(
+            () => MapDirectMemoryTransaction(ctx, inOutAddressPointer, length, protection, flags, directMemoryStart, alignment),
+            () => IsNewMappingOutsideGpuMemory(ctx, inOutAddressPointer, length, flags));
+
+    // A kernel-placed mapping lands in free space; a fixed one must target untouched space.
+    private static bool IsNewMappingOutsideGpuMemory(CpuContext ctx, ulong inOutAddressPointer, ulong length, ulong flags) =>
+        (flags & OrbisKernelMapFixed) == 0 ||
+        (ctx.TryReadUInt64(inOutAddressPointer, out var requested) && IsUntouchedByGpu(requested, length));
 
     private static int MapDirectMemoryTransaction(CpuContext ctx, ulong inOutAddressPointer, ulong length,
         int protection, ulong flags, ulong directMemoryStart, ulong alignment)
@@ -5777,7 +5783,31 @@ public static partial class KernelMemoryCompatExports
         return TryWriteCompat(ctx, address, bytes);
     }
 
+    // One transaction for the whole batch: every entry would otherwise hand the GPU worker
+    // its own mapping change and wait for the GPU to drain before it could run.
     private static int KernelBatchMapCore(CpuContext ctx, int flags)
+        => RunMappingTransaction(() => KernelBatchMapTransaction(ctx, flags), () => BatchOnlyAddsOutsideGpuMemory(ctx, flags));
+
+    // Only map operations, each placing memory where the GPU has never looked.
+    private static bool BatchOnlyAddsOutsideGpuMemory(CpuContext ctx, int flags)
+    {
+        var entriesAddress = ctx[CpuRegister.Rdi];
+        var entryCount = unchecked((int)ctx[CpuRegister.Rsi]);
+        if (entryCount <= 0 || entryCount > 4096)
+            return false;
+        for (var index = 0; index < entryCount; index++)
+        {
+            var entryAddress = entriesAddress + (ulong)(index * OrbisKernelBatchMapEntrySize);
+            if (!TryReadBatchMapEntry(ctx, entryAddress, out var entry) ||
+                entry.Operation is not (OrbisKernelMapOpMapDirect or OrbisKernelMapOpMapFlexible) ||
+                !IsNewMappingOutsideGpuMemory(ctx, entryAddress + OrbisKernelBatchMapEntryStartOffset, entry.Length, unchecked((uint)flags)))
+                return false;
+        }
+
+        return true;
+    }
+
+    private static int KernelBatchMapTransaction(CpuContext ctx, int flags)
     {
         var entriesAddress = ctx[CpuRegister.Rdi];
         var entryCount = unchecked((int)ctx[CpuRegister.Rsi]);

tests/SharpEmu.Libs.Tests/Kernel/KernelBackedMemoryTests.cs

@@ -107,7 +107,10 @@ public sealed class KernelBackedMemoryTests
             });
             memory.AttachGpuQueue(relay, null);
             var reserved = test.Reserve(0x10000, replaceMapping ? address : 0);
-            Assert.Equal(1, relay.DispatchCount);
+            // Replacing a GPU-registered mapping goes through the GPU queue so the pending read
+            // finishes first. A kernel-placed reservation lands in free address space the GPU
+            // cannot be reading, so it does not wait for the queue.
+            Assert.Equal(replaceMapping ? 1 : 0, relay.DispatchCount);
             if (replaceMapping)
                 Assert.Equal(address, reserved);
             else