fix(libkernel): place guest thread stacks inside the application area

On Linux, glibc maps the stack of every guest thread near the top of the
host address space (around 0x7fff00000000). On the console, application
stacks lie inside the application area below 0xFC00000000, and guest code
relies on that: ChakraCore's garbage collector indexes a 4 MiB table with
address >> 18, so it only covers addresses below 1 TiB and reports
Js::OutOfMemoryException for a stack above it.

Map each guest thread's stack with DoMapAnon, which allocates from the
guest arena, behind a PROT_NONE guard page, and pass it to pthread_attr_setstack.
The stack is freed after a join; the stack of a detached thread is freed once its
host thread is gone from /proc/self/task.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/libs/prx/libkernel/Pthread/include/Pthread.hpp

@@ -129,6 +129,9 @@ struct PthreadPrivate {
 #ifndef _WIN32
     clockid_t hostCpuClock{};
     bool hostCpuClockBound = false;
+    void* guestStack = nullptr;
+    std::size_t guestStackBytes = 0;
+    int hostTid = 0;
 #endif
     std::mutex nameLock;
     std::string name;

core/libs/prx/libkernel/Pthread/src/Thread.cpp

@@ -5,6 +5,7 @@
 #include "prx/libc/include/General.hpp"
 #include "prx/libkernel/KernelErrors.hpp"
 #include "prx/libc/include/CpuTopology.hpp"
+#include "prx/libkernel/DirectMemory/DirectMemory.hpp"
 #include <algorithm>
 #include <cstdio>
 #include <cstdlib>
@@ -13,6 +14,7 @@
 #include <limits>
 #include <map>
 #include <utility>
+#include <vector>
 #include <string>
 #include <future>
 #include <memory>
@@ -74,6 +76,75 @@ static std::mutex stackLock;
 static std::map<PthreadPrivate*, std::pair<std::uintptr_t, std::uintptr_t>> liveStacks;
 static std::map<std::uintptr_t, std::size_t> protectedStackPages;
 
+#if defined(__linux__)
+static constexpr std::size_t GUEST_STACK_GUARD = PS5_PAGE_SIZE;
+
+struct RetiredStack {
+    void* address;
+    std::size_t bytes;
+    int hostTid;
+};
+
+static std::mutex retiredStacksLock;
+static std::vector<RetiredStack> retiredStacks;
+
+static void AllocateGuestStack(PthreadPrivate* thread, std::size_t usableBytes) {
+    const std::size_t bytes = (usableBytes + PS5_PAGE_SIZE - 1) / PS5_PAGE_SIZE * PS5_PAGE_SIZE + GUEST_STACK_GUARD;
+    void* stack = nullptr;
+    if (DoMapAnon(&stack, bytes, 3, 0) != 0)
+        throw std::runtime_error("scePthreadCreate: cannot map the guest thread stack");
+    if (DoMprotect(stack, GUEST_STACK_GUARD, 0) != 0) {
+        DoMunmap(stack, bytes);
+        throw std::runtime_error("scePthreadCreate: cannot protect the guest thread stack guard");
+    }
+    thread->guestStack = stack;
+    thread->guestStackBytes = bytes;
+}
+
+static void FreeGuestStack(PthreadPrivate* thread) {
+    if (thread->guestStack == nullptr) return;
+    DoMunmap(thread->guestStack, thread->guestStackBytes);
+    thread->guestStack = nullptr;
+    thread->guestStackBytes = 0;
+}
+
+static bool HostThreadGone(int hostTid) {
+    if (hostTid == 0) return false;
+    char path[48];
+    std::snprintf(path, sizeof(path), "/proc/self/task/%d", hostTid);
+    return access(path, F_OK) != 0;
+}
+
+static void ReapRetiredStacks() {
+    std::vector<RetiredStack> ready;
+    {
+        std::lock_guard lock(retiredStacksLock);
+        for (auto it = retiredStacks.begin(); it != retiredStacks.end();) {
+            if (HostThreadGone(it->hostTid)) {
+                ready.push_back(*it);
+                it = retiredStacks.erase(it);
+            } else {
+                ++it;
+            }
+        }
+    }
+    for (const auto& stack : ready) DoMunmap(stack.address, stack.bytes);
+}
+
+static void RetireGuestStack(PthreadPrivate* thread) {
+    if (thread->guestStack == nullptr) return;
+    {
+        std::lock_guard lock(retiredStacksLock);
+        retiredStacks.push_back({thread->guestStack, thread->guestStackBytes, thread->hostTid});
+    }
+    thread->guestStack = nullptr;
+    thread->guestStackBytes = 0;
+}
+#else
+static void FreeGuestStack(PthreadPrivate*) {}
+static void RetireGuestStack(PthreadPrivate*) {}
+#endif
+
 static bool HostStackLimits(std::uintptr_t* low, std::uintptr_t* high) {
 #ifdef _WIN32
     ULONG_PTR lowLimit = 0;
@@ -518,6 +589,9 @@ static void* StartNativeThread(void* opaque) {
             throw std::runtime_error("Cannot query guest thread stack");
         self->stackAddress = reinterpret_cast<void*>(high - reserve - self->stackSize);
         self->threadId = std::this_thread::get_id();
+#if defined(__linux__)
+        self->hostTid = static_cast<int>(gettid());
+#endif
         args->initialized.set_value();
     } catch (...) {
         args->initialized.set_exception(std::current_exception());
@@ -601,12 +675,22 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread
     sigset_t creatorMask;
     pthread_sigmask(SIG_SETMASK, nullptr, &creatorMask);
     args->signalMask = creatorMask;
+#if defined(__linux__)
+    ReapRetiredStacks();
+    AllocateGuestStack(p.get(), nativeStack);
+#endif
     auto native = std::make_unique<NativeThreadArgs>(NativeThreadArgs{std::move(args), start.get_future(), {}});
     auto initialized = native->initialized.get_future();
     pthread_attr_t nativeAttr;
-    if (const int result = pthread_attr_init(&nativeAttr); result != 0)
+    if (const int result = pthread_attr_init(&nativeAttr); result != 0) {
+        FreeGuestStack(p.get());
         throw std::system_error(result, std::generic_category(), "Creating guest thread");
+    }
+#if defined(__linux__)
+    int created = pthread_attr_setstack(&nativeAttr, static_cast<char*>(p->guestStack) + GUEST_STACK_GUARD, p->guestStackBytes - GUEST_STACK_GUARD);
+#else
     int created = pthread_attr_setstacksize(&nativeAttr, nativeStack);
+#endif
     if (created == 0) {
         sigset_t allSignals;
         sigfillset(&allSignals);
@@ -615,18 +699,22 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread
         pthread_sigmask(SIG_SETMASK, &creatorMask, nullptr);
     }
     pthread_attr_destroy(&nativeAttr);
-    if (created != 0)
+    if (created != 0) {
+        FreeGuestStack(p.get());
         throw std::system_error(created, std::generic_category(), "Creating guest thread");
+    }
     native.release();
     try {
         initialized.get();
         if (detached) {
             if (const int result = pthread_detach(p->hostThread); result != 0)
                 throw std::system_error(result, std::generic_category(), "Detaching guest thread");
+            RetireGuestStack(p.get());
         }
     } catch (...) {
         start.set_value(false);
         pthread_join(p->hostThread, nullptr);
+        FreeGuestStack(p.get());
         throw;
     }
     auto* published = p.release();
@@ -655,6 +743,7 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval) {
 #else
     if (const int result = pthread_join(thread->hostThread, nullptr); result != 0)
         throw std::system_error(result, std::generic_category(), "Joining guest thread");
+    FreeGuestStack(thread);
     if (retval) *retval = thread->_retval;
     ReleaseThread(thread);
 #endif
@@ -670,6 +759,7 @@ int APS5_VABI scePthreadDetach(Pthread thread) {
 #else
     if (const int result = pthread_detach(thread->hostThread); result != 0)
         throw std::system_error(result, std::generic_category(), "Detaching guest thread");
+    RetireGuestStack(thread);
     ReleaseThread(thread);
 #endif
     return SCE_OK;

core/libs/tests/GuestKernelIsStack.cpp

@@ -9,9 +9,11 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval);
 int APS5_VABI sceKernelIsStack(void* addr, void** start, void** end);
 int APS5_VABI sceKernelMapFlexibleMemory(void** addr, std::size_t len, int prot, int flags);
 int APS5_VABI sceKernelMunmap(std::uint64_t addr, std::size_t len);
+int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info, std::uint64_t infoSize);
 }
 
 static constexpr int SCE_OK = 0;
+static constexpr std::uintptr_t APPLICATION_AREA_END = 0x000000FC00000000ull;
 static constexpr int SCE_KERNEL_ERROR_EACCES = static_cast<int>(0x8002000d);
 static int global = 0;
 
@@ -24,6 +26,8 @@ struct WorkerContext {
     void* start = nullptr;
     void* end = nullptr;
     int result = -1;
+    VirtualQueryInfo region{};
+    int regionResult = -1;
 };
 
 static void* APS5_VABI Worker(void* arg) {
@@ -31,6 +35,7 @@ static void* APS5_VABI Worker(void* arg) {
     int local = 0;
     context.local = &local;
     context.result = sceKernelIsStack(&local, &context.start, &context.end);
+    context.regionResult = sceKernelVirtualQuery(&local, 0, &context.region, sizeof(context.region));
     context.queried.store(true);
     while (!context.release.load()) {}
     return nullptr;
@@ -55,6 +60,11 @@ int main() {
     while (!context.queried.load()) {}
     Require(context.result == SCE_OK);
     Require(context.start <= context.local && context.local < context.end);
+    Require(context.regionResult == SCE_OK);
+    Require(context.region.start <= reinterpret_cast<std::uintptr_t>(context.local) && reinterpret_cast<std::uintptr_t>(context.local) < context.region.end);
+#if defined(__linux__)
+    Require(reinterpret_cast<std::uintptr_t>(context.local) < APPLICATION_AREA_END);
+#endif
     Require(sceKernelIsStack(&local, &start, &end) == SCE_OK);
     Require(context.end <= start || context.start >= end);
     void* workerStart = nullptr;