fix(agc): import GPU mappings on a host thread when the stack is short

The GPU-map observer imported a new mapping into Vulkan on the thread
that made the mapping, on whatever stack it ran. Guest fibers run on
contexts the title allocates in guest memory, with no guard page;
Demon's Souls packs its 16 KiB job fiber contexts back to back. A GPU
mapping made on a fiber used 10,616 bytes of the fiber's stack on
NVIDIA (Add, NoteGpuMapping, the registry walk in GuestAllocationsEnd,
the import, the fiber's entry and return), about 7.5 KiB of it for the
import, so a fiber that mapped GPU memory deep in a job could overrun
its context into the next one.

The observer now imports inline when the stack pointer lies inside the
thread's stack bounds with at least 256 KiB below it, and otherwise
starts a host thread for the import and joins it. On Windows the bounds
come from GetCurrentThreadStackLimits, which reads the TIB bounds that
libSceFiber sets to the fiber context while a fiber runs; on Linux from
pthread_getattr_np, whose bounds a fiber's stack lies outside of. A
mapping made while the thread holds the GPU mutex still imports inline:
a host thread could wait for that mutex forever. A host thread that
cannot be started is reported like any failed import, which is still
printed to stderr while the mapping carries on.

agc_driver_gpu_map_on_fiber maps GPU memory from a fiber with a 64 KiB
context and requires that the mapping use at most 8 KiB of it (10,616
bytes inline, 2,960 on a host thread) and start one host thread, then
maps the same memory from the main thread and requires that it start
none.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -315,7 +315,7 @@ if(BUILD_TESTING)
 endif()
 
 if(WIN32)
-    set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/GuestMemory.cpp PROPERTIES COMPILE_DEFINITIONS _WIN32_WINNT=0x0602)
+    set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/GuestMemory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Lifecycle.cpp PROPERTIES COMPILE_DEFINITIONS _WIN32_WINNT=0x0602)
 endif()
 
 foreach(runtimeLinkFlag IN ITEMS -static-libgcc -static-libstdc++ -static)
@@ -1065,6 +1065,8 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
     list(APPEND agcExecutionTests ${agcExecutionTest})
 endforeach()
 
+target_link_libraries(agc_driver_gpu_map_on_fiber_tests PRIVATE libSceFiber)
+
 target_include_directories(agc_driver_alpha_to_coverage_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_include_directories(agc_driver_dual_source_blend_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -44,6 +44,7 @@ namespace AgcDriver::DriverDetail {
 class Driver {
 public:
     static Driver& Get();
+    static std::uint64_t GpuMapImportThreads();
     ~Driver();
     void Shutdown();
     void WaitIdle();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp

@@ -6,10 +6,64 @@
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
+#include <atomic>
+#include <cstdint>
 #include <cstdio>
+#include <system_error>
+#include <thread>
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <pthread.h>
+#endif
 
 namespace AgcDriver::DriverDetail {
 
+namespace {
+
+constexpr std::uintptr_t InlineImportStackBytes = 256 * 1024;
+
+std::atomic<std::uint64_t> gpuMapImportThreads{0};
+
+bool StackHoldsImport() {
+#ifdef _WIN32
+    ULONG_PTR low = 0;
+    ULONG_PTR high = 0;
+    GetCurrentThreadStackLimits(&low, &high);
+#else
+    struct Bounds {
+        std::uintptr_t low = 0;
+        std::uintptr_t high = 0;
+        Bounds() {
+            pthread_attr_t attributes;
+            if (pthread_getattr_np(pthread_self(), &attributes) != 0) return;
+            void* base = nullptr;
+            std::size_t size = 0;
+            if (pthread_attr_getstack(&attributes, &base, &size) == 0 && base != nullptr) {
+                low = reinterpret_cast<std::uintptr_t>(base);
+                high = low + size;
+            }
+            pthread_attr_destroy(&attributes);
+        }
+    };
+    thread_local const Bounds bounds;
+    const auto low = bounds.low;
+    const auto high = bounds.high;
+#endif
+    const volatile unsigned char marker = 0;
+    const auto frame = reinterpret_cast<std::uintptr_t>(&marker);
+    return frame >= low && frame < high && frame - low >= InlineImportStackBytes;
+}
+
+}
+
+std::uint64_t Driver::GpuMapImportThreads() {
+    return gpuMapImportThreads.load(std::memory_order_relaxed);
+}
+
 Driver& Driver::Get() {
     static Driver driver;
     return driver;
@@ -64,14 +118,29 @@ Driver::Driver() {
         throw;
     }
     GuestAllocations::GuestAllocationsSetGpuMapObserver_nid_postfix([](const GuestAllocations::Mapped& ranges, std::uint64_t generation) {
-        try {
-            const auto current = Driver::Get().device.Load();
-            if (current == nullptr || current->ImportGuestMemory(ranges, generation, false)) return;
-            std::lock_guard lock(GuestMemory::GpuMutex());
-            if (Driver::Get().device.Load() == current) current->ImportGuestMemory(ranges, generation, true);
-        } catch (const std::exception& error) {
-            std::fprintf(stderr, "[gpu] import of GPU memory at its mapping failed: %s\n", error.what());
+        const auto import = [&] {
+            try {
+                const auto current = Driver::Get().device.Load();
+                if (current == nullptr || current->ImportGuestMemory(ranges, generation, false)) return;
+                std::lock_guard lock(GuestMemory::GpuMutex());
+                if (Driver::Get().device.Load() == current) current->ImportGuestMemory(ranges, generation, true);
+            } catch (const std::exception& error) {
+                std::fprintf(stderr, "[gpu] import of GPU memory at its mapping failed: %s\n", error.what());
+            }
+        };
+        if (GuestMemory::GpuMutex().HeldByThisThread() || StackHoldsImport()) {
+            import();
+            return;
         }
+        std::thread host;
+        try {
+            host = std::thread(import);
+        } catch (const std::system_error& error) {
+            std::fprintf(stderr, "[gpu] import of GPU memory at its mapping failed: no host thread to run it on: %s\n", error.what());
+            return;
+        }
+        gpuMapImportThreads.fetch_add(1, std::memory_order_relaxed);
+        host.join();
     });
 }
 

core/libs/prx/libSceAgcDriver/tests/execution/GpuMapOnFiber.cpp

@@ -0,0 +1,132 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/general/VabiMacros.hpp"
+#include "SceTypes.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <array>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <exception>
+#include <iostream>
+#include <string>
+
+using GuestFiberEntry = void (APS5_VABI*)(std::uint64_t, std::uint64_t);
+
+extern "C" {
+std::int32_t APS5_VABI _sceFiberInitializeImpl_nid_postfix(FiberObject*, const char*, GuestFiberEntry, std::uint64_t, void*, std::uint64_t, const void*, std::uint32_t);
+std::int32_t APS5_VABI sceFiberFinalize(FiberObject*);
+std::int32_t APS5_VABI sceFiberRun_nid_postfix(FiberObject*, std::uint64_t, std::uint64_t*);
+std::int32_t APS5_VABI sceFiberReturnToThread(std::uint64_t, std::uint64_t*);
+}
+
+namespace {
+
+using AgcDriver::DriverDetail::Driver;
+using AgcDriver::Graphics::Require;
+
+constexpr std::size_t ContextBytes = 64 * 1024;
+constexpr std::size_t MappingBytes = 4 * 65536;
+constexpr std::size_t FiberStackLimit = 8 * 1024;
+constexpr std::uint64_t Fill = 0x5aa5c33c5aa5c33cull;
+
+alignas(16) std::array<std::uint64_t, ContextBytes / sizeof(std::uint64_t)> FiberContext;
+struct alignas(8) FiberStorage {
+    unsigned char bytes[0x100];
+};
+FiberStorage FiberObjectStorage;
+
+alignas(256) constexpr std::array<std::uint32_t, 8> VertexCode{0x7e000208u, 0x7e020280u, 0x7e040280u, 0x7e0602f2u, 0xf80008cfu, 0x03020100u, 0xbf810000u, 0xbf810000u};
+
+struct Header {
+    Shader shader{};
+    ShaderUserData userData{};
+};
+alignas(8) Header VertexHeader{};
+
+std::uint8_t* Mapping = nullptr;
+
+void RegisterVertexShader() {
+    VertexHeader.shader.file_header = 0x34333231u;
+    VertexHeader.shader.version = 0x18u;
+    VertexHeader.shader.header_size = sizeof(Header);
+    VertexHeader.shader.shader_size = static_cast<std::uint32_t>(sizeof(VertexCode));
+    VertexHeader.shader.code = VertexCode.data();
+    VertexHeader.shader.type = 2;
+    VertexHeader.shader.user_data = &VertexHeader.userData;
+    AgcDriverRegisterShader_nid_postfix(&VertexHeader.shader);
+}
+
+void MapForGpu() {
+    GuestAllocations::Mutation mutation;
+    mutation.Add(Mapping, MappingBytes, true, true, true);
+    mutation.NoteGpuMapping(Mapping, MappingBytes);
+}
+
+void APS5_VABI MapOnFiber(std::uint64_t, std::uint64_t) {
+    MapForGpu();
+    sceFiberReturnToThread(0, nullptr);
+}
+
+void ReleaseMapping() {
+    if (Mapping == nullptr) return;
+#ifdef _WIN32
+    VirtualFree(Mapping, 0, MEM_RELEASE);
+#else
+    std::free(Mapping);
+#endif
+    Mapping = nullptr;
+}
+
+std::size_t FiberStackUsed() {
+    std::size_t untouched = 0;
+    while (untouched < FiberContext.size() && FiberContext[untouched] == Fill) ++untouched;
+    return (FiberContext.size() - untouched) * sizeof(std::uint64_t);
+}
+
+}
+
+int main() {
+    try {
+        if (!OpenVulkanTestDevice()) return VulkanTestSkipped;
+        RegisterVertexShader();
+#ifdef _WIN32
+        Mapping = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, MappingBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        Mapping = static_cast<std::uint8_t*>(std::aligned_alloc(65536, MappingBytes));
+#endif
+        Require(Mapping != nullptr, "GPU mapping on a fiber: cannot allocate the mapped memory");
+        std::memset(Mapping, 0, MappingBytes);
+        FiberContext.fill(Fill);
+        auto* fiber = reinterpret_cast<FiberObject*>(&FiberObjectStorage);
+        Require(_sceFiberInitializeImpl_nid_postfix(fiber, "gpu map", MapOnFiber, 0, FiberContext.data(), ContextBytes, nullptr, 0) == 0, "GPU mapping on a fiber: the fiber was not initialized");
+        const auto beforeFiber = Driver::GpuMapImportThreads();
+        Require(sceFiberRun_nid_postfix(fiber, 0, nullptr) == 0, "GPU mapping on a fiber: the fiber did not run");
+        const auto onFiber = Driver::GpuMapImportThreads() - beforeFiber;
+        Require(sceFiberFinalize(fiber) == 0, "GPU mapping on a fiber: the fiber was not finalized");
+        const auto used = FiberStackUsed();
+        GuestAllocations::Mutation().Remove(Mapping);
+        const auto beforeThread = Driver::GpuMapImportThreads();
+        MapForGpu();
+        const auto onThread = Driver::GpuMapImportThreads() - beforeThread;
+        GuestAllocations::Mutation().Remove(Mapping);
+        AgcDriverShutdown_nid_postfix();
+        ReleaseMapping();
+        Require(used <= FiberStackLimit, "GPU mapping on a fiber used " + std::to_string(used) + " bytes of the fiber's stack, more than " + std::to_string(FiberStackLimit) + ": the import of the mapped memory ran on it");
+        Require(onFiber == 1, "GPU mapping on a fiber started " + std::to_string(onFiber) + " host threads for its import instead of one");
+        Require(onThread == 0, "GPU mapping on an ordinary thread started " + std::to_string(onThread) + " host threads for its import instead of importing on its own stack");
+        std::cout << "A GPU mapping made on a fiber used " << used << " bytes of its stack\n";
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        AgcDriverShutdown_nid_postfix();
+        ReleaseMapping();
+        return 1;
+    }
+}