Merge pull request #1421 from popcoder565/fix/kernel-raise-exception-ntcontinue

fix(libkernel): stop losing and deadlocking sceKernelRaiseException deliveries on Windows

core/libs/prx/libkernel/System/src/Exception.cpp

@@ -5,6 +5,7 @@
 #include "prx/libkernel/KernelErrors.hpp"
 #include "prx/libkernel/Pthread/include/Pthread.hpp"
 #include <array>
+#include <chrono>
 #include <cstring>
 #include <memory>
 #include <mutex>
@@ -102,6 +103,7 @@ GuestExceptionHandler Handler(int signum) {
 #ifdef _WIN32
 constexpr std::size_t RedZone = 128;
 constexpr std::size_t HomeArea = 32;
+constexpr auto RetryLimit = std::chrono::seconds(1);
 constexpr std::size_t VectorBytes = 16 * 32;
 
 struct Delivery {
@@ -199,6 +201,22 @@ bool Exited(HANDLE native) {
     return WaitForSingleObject(native, 0) == WAIT_OBJECT_0;
 }
 
+bool RestoringContext(DWORD64 rip) {
+    static const std::array<DWORD64, 2> stubs = [] {
+        const HMODULE ntdll = GetModuleHandleW(L"ntdll.dll");
+        return std::array<DWORD64, 2>{reinterpret_cast<DWORD64>(GetProcAddress(ntdll, "NtContinue")), reinterpret_cast<DWORD64>(GetProcAddress(ntdll, "NtContinueEx"))};
+    }();
+    for (const DWORD64 stub : stubs)
+        if (stub != 0 && rip - stub < 0x20) return true;
+    return false;
+}
+
+bool InWinpthread(DWORD64 rip) {
+    static const HMODULE winpthread = GetModuleHandleW(L"libwinpthread-1.dll");
+    MEMORY_BASIC_INFORMATION info{};
+    return winpthread != nullptr && VirtualQuery(reinterpret_cast<const void*>(rip), &info, sizeof(info)) == sizeof(info) && info.AllocationBase == winpthread;
+}
+
 bool RaiseOn(Pthread thread, GuestExceptionHandler handler, int signum) {
     if (thread == scePthreadSelf()) {
         CONTEXT context{};
@@ -208,26 +226,33 @@ bool RaiseOn(Pthread thread, GuestExceptionHandler handler, int signum) {
     }
     const auto native = static_cast<HANDLE>(thread->nativeHandle);
     auto queued = std::make_unique<Delivery>(Delivery{handler, signum, {}, {}, 0});
-    if (SuspendThread(native) == static_cast<DWORD>(-1)) {
-        if (Exited(native)) return false;
-        throw std::runtime_error("sceKernelRaiseException: cannot suspend the target thread");
-    }
-    if (Exited(native)) {
-        ResumeThread(native);
-        return false;
-    }
-    if (thread->waitCount.load(std::memory_order_seq_cst) > 0) {
-        const bool accepted = QueueUserAPC(WaitingEntry, native, reinterpret_cast<ULONG_PTR>(queued.get())) != 0;
-        ResumeThread(native);
-        if (!accepted) throw std::runtime_error("sceKernelRaiseException: cannot queue delivery to the waiting thread");
-        queued.release();
-        return true;
-    }
     alignas(16) Delivery delivery{handler, signum, {}, {}, SaveVectors ? 1u : 0u};
-    delivery.context.ContextFlags = CONTEXT_FULL | CONTEXT_FLOATING_POINT;
-    if (!GetThreadContext(native, &delivery.context)) {
+    const auto retryDeadline = std::chrono::steady_clock::now() + RetryLimit;
+    for (;;) {
+        if (SuspendThread(native) == static_cast<DWORD>(-1)) {
+            if (Exited(native)) return false;
+            throw std::runtime_error("sceKernelRaiseException: cannot suspend the target thread");
+        }
+        if (Exited(native)) {
+            ResumeThread(native);
+            return false;
+        }
+        delivery.context.ContextFlags = CONTEXT_FULL | CONTEXT_SEGMENTS;
+        if (!GetThreadContext(native, &delivery.context)) {
+            ResumeThread(native);
+            throw std::runtime_error("sceKernelRaiseException: cannot read the target thread context");
+        }
+        if (thread->waitCount.load(std::memory_order_seq_cst) > 0) {
+            const bool accepted = QueueUserAPC(WaitingEntry, native, reinterpret_cast<ULONG_PTR>(queued.get())) != 0;
+            ResumeThread(native);
+            if (!accepted) throw std::runtime_error("sceKernelRaiseException: cannot queue delivery to the waiting thread");
+            queued.release();
+            return true;
+        }
+        if (!RestoringContext(delivery.context.Rip) && !InWinpthread(delivery.context.Rip)) break;
         ResumeThread(native);
-        throw std::runtime_error("sceKernelRaiseException: cannot read the target thread context");
+        if (std::chrono::steady_clock::now() >= retryDeadline) throw std::runtime_error("sceKernelRaiseException: the target thread stayed inside NtContinue or winpthreads for 1 s");
+        SwitchToThread();
     }
     const DWORD64 slot = (delivery.context.Rsp - RedZone - sizeof(Delivery)) & ~static_cast<DWORD64>(15);
     if (!StackWritable(slot - HomeArea - 8, delivery.context.Rsp - RedZone)) {

core/libs/tests/GuestRaiseException.cpp

@@ -7,6 +7,7 @@
 #include <mutex>
 #include <stdexcept>
 #include <thread>
+#include <windows.h>
 
 extern "C" {
 int APS5_VABI sceKernelInstallExceptionHandler(int signum, void* handler);
@@ -34,6 +35,7 @@ static std::atomic<int> calls{0};
 static std::atomic<std::thread::id> handlerThread;
 static std::atomic<std::uint64_t> handlerRsp{0};
 static std::atomic<std::uintptr_t> handlerFrame{0};
+static std::atomic<Pthread> handlerSelf{nullptr};
 static std::atomic<bool> clobberVectors{false};
 
 static void APS5_VABI Handler(int signum, void* context) {
@@ -44,6 +46,7 @@ static void APS5_VABI Handler(int signum, void* context) {
     handlerFrame.store(reinterpret_cast<std::uintptr_t>(&local));
     handlerRsp.store(rsp);
     handlerThread.store(std::this_thread::get_id());
+    handlerSelf.store(scePthreadSelf());
     if (clobberVectors.load()) asm volatile("vzeroall" ::: "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15");
     calls.fetch_add(1);
 }
@@ -106,6 +109,57 @@ static void* APS5_VABI Leaving(void* arg) {
     return nullptr;
 }
 
+static constexpr DWORD ContinuedCode = 0xe0000001u;
+static constexpr int ContinuingRounds = 50;
+static std::atomic<int> continued{0};
+
+static LONG CALLBACK ContinueRaised(EXCEPTION_POINTERS* info) {
+    if (info->ExceptionRecord->ExceptionCode != ContinuedCode) return EXCEPTION_CONTINUE_SEARCH;
+    continued.fetch_add(1);
+    return EXCEPTION_CONTINUE_EXECUTION;
+}
+
+static constexpr int SelfingRounds = 100;
+static std::atomic<Pthread> selfSink{nullptr};
+
+static void* APS5_VABI Selfing(void* arg) {
+    auto& worker = *static_cast<Worker*>(arg);
+    worker.id = std::this_thread::get_id();
+    worker.started.store(true);
+    while (!worker.stop.load()) selfSink.store(scePthreadSelf());
+    return nullptr;
+}
+
+static void* APS5_VABI Continuing(void* arg) {
+    auto& worker = *static_cast<Worker*>(arg);
+    worker.id = std::this_thread::get_id();
+    worker.started.store(true);
+    while (!worker.stop.load()) RaiseException(ContinuedCode, 0, 0, nullptr);
+    return nullptr;
+}
+
+using NtContinueFunction = LONG(NTAPI*)(CONTEXT*, BOOLEAN);
+alignas(16) static CONTEXT loopContext;
+alignas(16) static CONTEXT leaveContext;
+static std::atomic<DWORD> loopingId{0};
+
+static void* APS5_VABI Looping(void* arg) {
+    auto& worker = *static_cast<Worker*>(arg);
+    const auto ntContinue = reinterpret_cast<NtContinueFunction>(reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtContinue")));
+    volatile bool left = false;
+    RtlCaptureContext(&leaveContext);
+    if (left) return nullptr;
+    left = true;
+    loopContext = leaveContext;
+    loopContext.Rip = reinterpret_cast<DWORD64>(ntContinue);
+    loopContext.Rcx = reinterpret_cast<DWORD64>(&loopContext);
+    loopContext.Rdx = 0;
+    loopingId.store(GetCurrentThreadId());
+    worker.started.store(true);
+    ntContinue(&loopContext, FALSE);
+    return nullptr;
+}
+
 static constexpr int VectorRounds = 100;
 static constexpr std::size_t VectorBytes = 16 * 32;
 alignas(32) static std::uint8_t vectorPattern[VectorBytes];
@@ -337,6 +391,59 @@ int main() {
     Require(scePthreadJoin(leavingThread, nullptr) == 0);
     Require(sceKernelDeleteSema(leaving.sem) == 0);
 
+    void* const vectored = AddVectoredExceptionHandler(1, ContinueRaised);
+    Require(vectored != nullptr);
+    Worker continuing;
+    Pthread continuingThread = nullptr;
+    Require(scePthreadCreate(&continuingThread, nullptr, Continuing, &continuing, "continuing") == 0);
+    while (!continuing.started.load() || continued.load() == 0) std::this_thread::yield();
+    for (int raised = 0; raised < ContinuingRounds; ++raised) {
+        Require(sceKernelRaiseException(continuingThread, SIGUSR1) == 0);
+        ExpectDelivery(1 + 2 * Repeats + HostRounds + LeavingRounds + raised, continuing.id);
+    }
+    continuing.stop.store(true);
+    Require(scePthreadJoin(continuingThread, nullptr) == 0);
+    Require(RemoveVectoredExceptionHandler(vectored) != 0);
+
+    Worker selfing;
+    Pthread selfingThread = nullptr;
+    Require(scePthreadCreate(&selfingThread, nullptr, Selfing, &selfing, "selfing") == 0);
+    while (!selfing.started.load()) std::this_thread::yield();
+    for (int raised = 0; raised < SelfingRounds; ++raised) {
+        Require(sceKernelRaiseException(selfingThread, SIGUSR1) == 0);
+        ExpectDelivery(1 + 2 * Repeats + HostRounds + LeavingRounds + ContinuingRounds + raised, selfing.id);
+        Require(handlerSelf.load() == selfingThread);
+    }
+    selfing.stop.store(true);
+    Require(scePthreadJoin(selfingThread, nullptr) == 0);
+
+    Worker looping;
+    Pthread loopingThread = nullptr;
+    Require(scePthreadCreate(&loopingThread, nullptr, Looping, &looping, "looping") == 0);
+    while (!looping.started.load()) std::this_thread::yield();
+    std::this_thread::sleep_for(std::chrono::milliseconds(20));
+    const int beforeLooping = calls.load();
+    const auto raisedAt = std::chrono::steady_clock::now();
+    bool refused = false;
+    try {
+        sceKernelRaiseException(loopingThread, SIGUSR1);
+    } catch (const std::runtime_error&) {
+        refused = true;
+    }
+    const auto retried = std::chrono::steady_clock::now() - raisedAt;
+    Require(refused && retried >= std::chrono::milliseconds(900) && retried < std::chrono::seconds(10));
+    Require(calls.load() == beforeLooping);
+    const HANDLE loopingHandle = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT, FALSE, loopingId.load());
+    Require(loopingHandle != nullptr);
+    Require(SuspendThread(loopingHandle) != static_cast<DWORD>(-1));
+    CONTEXT stopped{};
+    stopped.ContextFlags = CONTEXT_CONTROL;
+    Require(GetThreadContext(loopingHandle, &stopped) != 0);
+    loopContext = leaveContext;
+    Require(ResumeThread(loopingHandle) != static_cast<DWORD>(-1));
+    CloseHandle(loopingHandle);
+    Require(scePthreadJoin(loopingThread, nullptr) == 0);
+
     if (__builtin_cpu_supports("avx")) {
         for (std::size_t i = 0; i < VectorBytes; ++i) vectorPattern[i] = static_cast<std::uint8_t>(i * 7 + 1);
         clobberVectors.store(true);
@@ -347,7 +454,7 @@ int main() {
         std::this_thread::sleep_for(std::chrono::milliseconds(20));
         for (int raised = 0; raised < VectorRounds; ++raised) {
             Require(sceKernelRaiseException(vectorsThread, SIGUSR1) == 0);
-            ExpectDelivery(1 + 2 * Repeats + HostRounds + LeavingRounds + raised, vectors.id);
+            ExpectDelivery(1 + 2 * Repeats + HostRounds + LeavingRounds + ContinuingRounds + SelfingRounds + raised, vectors.id);
         }
         clobberVectors.store(false);
         vectorStop = 1;

docs/dev/TechnicalDebt.md

@@ -270,6 +270,7 @@ Throughout the project, every function at every stage either **does exactly what
 - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) builds the handler's `ucontext` with FreeBSD's amd64 layout (16-byte sigmask, `mcontext` at 0x40, `mc_rsp` at 0xf8); the console's layout is not verified beyond the fields PS5Util reads
 - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) does not consult the target thread's signal mask
 - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) reaches a thread that waits in an alertable `TimedWait` wait through an APC. A thread blocked anywhere else, including all waits under `APS5_COARSE_TIMED_WAITS` and host waits inside the emulator, is redirected and runs the handler only when that wait returns
+- [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) on Windows runs the handler wherever the target thread was interrupted. It waits only while the thread is inside winpthreads or restoring a context through `NtContinue`, and throws if that lasts 1 s, so a handler that calls into other host code the thread was interrupted in, such as the C runtime's heap or a lock of the emulator, can deadlock on it or reenter it
 - [sigaltstack](../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) records each thread's alternate signal stack and reports it back, following FreeBSD's `kern_sigaltstack` (`EINVAL` for flags other than `SS_DISABLE`, `ENOMEM` below `MINSIGSTKSZ` 2048), but handlers never run on it, so `SS_ONSTACK` and `EPERM` are never reported
 - [libSceUlt queues](../../core/libs/prx/libSceUlt/Export.cpp) use preallocated host storage instead of the supplied guest work area; its existing size formula and the 512-byte object layouts remain unverified. Waiting-queue pool thread limits and waiter priority ordering are not implemented. Finalization wakes blocked queue callers with `ULT_ERROR_STATE`.
 - [Thread cancellation](../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) is not modelled: `scePthreadCancel` throws, so no cancel request is ever pending and `scePthreadTestcancel` and `pthread_testcancel` return without acting; the cancel state and type set by `scePthreadSetcancelstate` and `scePthreadSetcanceltype` are stored and not used