From 7637cac13e6f162bc069525753f15a740aa2a2f5 Mon Sep 17 00:00:00 2001 From: Cesar Rodriguez <11452991+Tamalero@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:01:03 -0700 Subject: [PATCH] fix(libkernel): place guest thread stacks inside the application area On Linux, glibc maps the stack of every guest thread near the top of the host address space (around 0x7fff00000000). On the console, application stacks lie inside the application area below 0xFC00000000, and guest code relies on that: ChakraCore's garbage collector indexes a 4 MiB table with address >> 18, so it only covers addresses below 1 TiB and reports Js::OutOfMemoryException for a stack above it. Map each guest thread's stack with DoMapAnon, which allocates from the guest arena, behind a PROT_NONE guard page, and pass it to pthread_attr_setstack. The stack is freed after a join; the stack of a detached thread is freed once its host thread is gone from /proc/self/task. Co-Authored-By: Claude Opus 5.5 --- .../prx/libkernel/Pthread/include/Pthread.hpp | 3 + .../libs/prx/libkernel/Pthread/src/Thread.cpp | 94 ++++++++++++++++++- core/libs/tests/GuestKernelIsStack.cpp | 10 ++ 3 files changed, 105 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp index a139d7e95..a95394497 100644 --- a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp +++ b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp @@ -129,6 +129,9 @@ struct PthreadPrivate { #ifndef _WIN32 clockid_t hostCpuClock{}; bool hostCpuClockBound = false; + void* guestStack = nullptr; + std::size_t guestStackBytes = 0; + int hostTid = 0; #endif std::mutex nameLock; std::string name; diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index a2ea58f0a..8d0f86e99 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -5,6 +5,7 @@ #include "prx/libc/include/General.hpp" #include "prx/libkernel/KernelErrors.hpp" #include "prx/libc/include/CpuTopology.hpp" +#include "prx/libkernel/DirectMemory/DirectMemory.hpp" #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +76,75 @@ static std::mutex stackLock; static std::map> liveStacks; static std::map protectedStackPages; +#if defined(__linux__) +static constexpr std::size_t GUEST_STACK_GUARD = PS5_PAGE_SIZE; + +struct RetiredStack { + void* address; + std::size_t bytes; + int hostTid; +}; + +static std::mutex retiredStacksLock; +static std::vector retiredStacks; + +static void AllocateGuestStack(PthreadPrivate* thread, std::size_t usableBytes) { + const std::size_t bytes = (usableBytes + PS5_PAGE_SIZE - 1) / PS5_PAGE_SIZE * PS5_PAGE_SIZE + GUEST_STACK_GUARD; + void* stack = nullptr; + if (DoMapAnon(&stack, bytes, 3, 0) != 0) + throw std::runtime_error("scePthreadCreate: cannot map the guest thread stack"); + if (DoMprotect(stack, GUEST_STACK_GUARD, 0) != 0) { + DoMunmap(stack, bytes); + throw std::runtime_error("scePthreadCreate: cannot protect the guest thread stack guard"); + } + thread->guestStack = stack; + thread->guestStackBytes = bytes; +} + +static void FreeGuestStack(PthreadPrivate* thread) { + if (thread->guestStack == nullptr) return; + DoMunmap(thread->guestStack, thread->guestStackBytes); + thread->guestStack = nullptr; + thread->guestStackBytes = 0; +} + +static bool HostThreadGone(int hostTid) { + if (hostTid == 0) return false; + char path[48]; + std::snprintf(path, sizeof(path), "/proc/self/task/%d", hostTid); + return access(path, F_OK) != 0; +} + +static void ReapRetiredStacks() { + std::vector ready; + { + std::lock_guard lock(retiredStacksLock); + for (auto it = retiredStacks.begin(); it != retiredStacks.end();) { + if (HostThreadGone(it->hostTid)) { + ready.push_back(*it); + it = retiredStacks.erase(it); + } else { + ++it; + } + } + } + for (const auto& stack : ready) DoMunmap(stack.address, stack.bytes); +} + +static void RetireGuestStack(PthreadPrivate* thread) { + if (thread->guestStack == nullptr) return; + { + std::lock_guard lock(retiredStacksLock); + retiredStacks.push_back({thread->guestStack, thread->guestStackBytes, thread->hostTid}); + } + thread->guestStack = nullptr; + thread->guestStackBytes = 0; +} +#else +static void FreeGuestStack(PthreadPrivate*) {} +static void RetireGuestStack(PthreadPrivate*) {} +#endif + static bool HostStackLimits(std::uintptr_t* low, std::uintptr_t* high) { #ifdef _WIN32 ULONG_PTR lowLimit = 0; @@ -518,6 +589,9 @@ static void* StartNativeThread(void* opaque) { throw std::runtime_error("Cannot query guest thread stack"); self->stackAddress = reinterpret_cast(high - reserve - self->stackSize); self->threadId = std::this_thread::get_id(); +#if defined(__linux__) + self->hostTid = static_cast(gettid()); +#endif args->initialized.set_value(); } catch (...) { args->initialized.set_exception(std::current_exception()); @@ -601,12 +675,22 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread sigset_t creatorMask; pthread_sigmask(SIG_SETMASK, nullptr, &creatorMask); args->signalMask = creatorMask; +#if defined(__linux__) + ReapRetiredStacks(); + AllocateGuestStack(p.get(), nativeStack); +#endif auto native = std::make_unique(NativeThreadArgs{std::move(args), start.get_future(), {}}); auto initialized = native->initialized.get_future(); pthread_attr_t nativeAttr; - if (const int result = pthread_attr_init(&nativeAttr); result != 0) + if (const int result = pthread_attr_init(&nativeAttr); result != 0) { + FreeGuestStack(p.get()); throw std::system_error(result, std::generic_category(), "Creating guest thread"); + } +#if defined(__linux__) + int created = pthread_attr_setstack(&nativeAttr, static_cast(p->guestStack) + GUEST_STACK_GUARD, p->guestStackBytes - GUEST_STACK_GUARD); +#else int created = pthread_attr_setstacksize(&nativeAttr, nativeStack); +#endif if (created == 0) { sigset_t allSignals; sigfillset(&allSignals); @@ -615,18 +699,22 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread pthread_sigmask(SIG_SETMASK, &creatorMask, nullptr); } pthread_attr_destroy(&nativeAttr); - if (created != 0) + if (created != 0) { + FreeGuestStack(p.get()); throw std::system_error(created, std::generic_category(), "Creating guest thread"); + } native.release(); try { initialized.get(); if (detached) { if (const int result = pthread_detach(p->hostThread); result != 0) throw std::system_error(result, std::generic_category(), "Detaching guest thread"); + RetireGuestStack(p.get()); } } catch (...) { start.set_value(false); pthread_join(p->hostThread, nullptr); + FreeGuestStack(p.get()); throw; } auto* published = p.release(); @@ -655,6 +743,7 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval) { #else if (const int result = pthread_join(thread->hostThread, nullptr); result != 0) throw std::system_error(result, std::generic_category(), "Joining guest thread"); + FreeGuestStack(thread); if (retval) *retval = thread->_retval; ReleaseThread(thread); #endif @@ -670,6 +759,7 @@ int APS5_VABI scePthreadDetach(Pthread thread) { #else if (const int result = pthread_detach(thread->hostThread); result != 0) throw std::system_error(result, std::generic_category(), "Detaching guest thread"); + RetireGuestStack(thread); ReleaseThread(thread); #endif return SCE_OK; diff --git a/core/libs/tests/GuestKernelIsStack.cpp b/core/libs/tests/GuestKernelIsStack.cpp index b2c1097d2..941ffdc97 100644 --- a/core/libs/tests/GuestKernelIsStack.cpp +++ b/core/libs/tests/GuestKernelIsStack.cpp @@ -9,9 +9,11 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval); int APS5_VABI sceKernelIsStack(void* addr, void** start, void** end); int APS5_VABI sceKernelMapFlexibleMemory(void** addr, std::size_t len, int prot, int flags); int APS5_VABI sceKernelMunmap(std::uint64_t addr, std::size_t len); +int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info, std::uint64_t infoSize); } static constexpr int SCE_OK = 0; +static constexpr std::uintptr_t APPLICATION_AREA_END = 0x000000FC00000000ull; static constexpr int SCE_KERNEL_ERROR_EACCES = static_cast(0x8002000d); static int global = 0; @@ -24,6 +26,8 @@ struct WorkerContext { void* start = nullptr; void* end = nullptr; int result = -1; + VirtualQueryInfo region{}; + int regionResult = -1; }; static void* APS5_VABI Worker(void* arg) { @@ -31,6 +35,7 @@ static void* APS5_VABI Worker(void* arg) { int local = 0; context.local = &local; context.result = sceKernelIsStack(&local, &context.start, &context.end); + context.regionResult = sceKernelVirtualQuery(&local, 0, &context.region, sizeof(context.region)); context.queried.store(true); while (!context.release.load()) {} return nullptr; @@ -55,6 +60,11 @@ int main() { while (!context.queried.load()) {} Require(context.result == SCE_OK); Require(context.start <= context.local && context.local < context.end); + Require(context.regionResult == SCE_OK); + Require(context.region.start <= reinterpret_cast(context.local) && reinterpret_cast(context.local) < context.region.end); +#if defined(__linux__) + Require(reinterpret_cast(context.local) < APPLICATION_AREA_END); +#endif Require(sceKernelIsStack(&local, &start, &end) == SCE_OK); Require(context.end <= start || context.start >= end); void* workerStart = nullptr;