From 7637cac13e6f162bc069525753f15a740aa2a2f5 Mon Sep 17 00:00:00 2001 From: Cesar Rodriguez <11452991+Tamalero@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:01:03 -0700 Subject: [PATCH 1/3] fix(libkernel): place guest thread stacks inside the application area On Linux, glibc maps the stack of every guest thread near the top of the host address space (around 0x7fff00000000). On the console, application stacks lie inside the application area below 0xFC00000000, and guest code relies on that: ChakraCore's garbage collector indexes a 4 MiB table with address >> 18, so it only covers addresses below 1 TiB and reports Js::OutOfMemoryException for a stack above it. Map each guest thread's stack with DoMapAnon, which allocates from the guest arena, behind a PROT_NONE guard page, and pass it to pthread_attr_setstack. The stack is freed after a join; the stack of a detached thread is freed once its host thread is gone from /proc/self/task. Co-Authored-By: Claude Opus 5.5 --- .../prx/libkernel/Pthread/include/Pthread.hpp | 3 + .../libs/prx/libkernel/Pthread/src/Thread.cpp | 94 ++++++++++++++++++- core/libs/tests/GuestKernelIsStack.cpp | 10 ++ 3 files changed, 105 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp index a139d7e95..a95394497 100644 --- a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp +++ b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp @@ -129,6 +129,9 @@ struct PthreadPrivate { #ifndef _WIN32 clockid_t hostCpuClock{}; bool hostCpuClockBound = false; + void* guestStack = nullptr; + std::size_t guestStackBytes = 0; + int hostTid = 0; #endif std::mutex nameLock; std::string name; diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index a2ea58f0a..8d0f86e99 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -5,6 +5,7 @@ #include "prx/libc/include/General.hpp" #include "prx/libkernel/KernelErrors.hpp" #include "prx/libc/include/CpuTopology.hpp" +#include "prx/libkernel/DirectMemory/DirectMemory.hpp" #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +76,75 @@ static std::mutex stackLock; static std::map> liveStacks; static std::map protectedStackPages; +#if defined(__linux__) +static constexpr std::size_t GUEST_STACK_GUARD = PS5_PAGE_SIZE; + +struct RetiredStack { + void* address; + std::size_t bytes; + int hostTid; +}; + +static std::mutex retiredStacksLock; +static std::vector retiredStacks; + +static void AllocateGuestStack(PthreadPrivate* thread, std::size_t usableBytes) { + const std::size_t bytes = (usableBytes + PS5_PAGE_SIZE - 1) / PS5_PAGE_SIZE * PS5_PAGE_SIZE + GUEST_STACK_GUARD; + void* stack = nullptr; + if (DoMapAnon(&stack, bytes, 3, 0) != 0) + throw std::runtime_error("scePthreadCreate: cannot map the guest thread stack"); + if (DoMprotect(stack, GUEST_STACK_GUARD, 0) != 0) { + DoMunmap(stack, bytes); + throw std::runtime_error("scePthreadCreate: cannot protect the guest thread stack guard"); + } + thread->guestStack = stack; + thread->guestStackBytes = bytes; +} + +static void FreeGuestStack(PthreadPrivate* thread) { + if (thread->guestStack == nullptr) return; + DoMunmap(thread->guestStack, thread->guestStackBytes); + thread->guestStack = nullptr; + thread->guestStackBytes = 0; +} + +static bool HostThreadGone(int hostTid) { + if (hostTid == 0) return false; + char path[48]; + std::snprintf(path, sizeof(path), "/proc/self/task/%d", hostTid); + return access(path, F_OK) != 0; +} + +static void ReapRetiredStacks() { + std::vector ready; + { + std::lock_guard lock(retiredStacksLock); + for (auto it = retiredStacks.begin(); it != retiredStacks.end();) { + if (HostThreadGone(it->hostTid)) { + ready.push_back(*it); + it = retiredStacks.erase(it); + } else { + ++it; + } + } + } + for (const auto& stack : ready) DoMunmap(stack.address, stack.bytes); +} + +static void RetireGuestStack(PthreadPrivate* thread) { + if (thread->guestStack == nullptr) return; + { + std::lock_guard lock(retiredStacksLock); + retiredStacks.push_back({thread->guestStack, thread->guestStackBytes, thread->hostTid}); + } + thread->guestStack = nullptr; + thread->guestStackBytes = 0; +} +#else +static void FreeGuestStack(PthreadPrivate*) {} +static void RetireGuestStack(PthreadPrivate*) {} +#endif + static bool HostStackLimits(std::uintptr_t* low, std::uintptr_t* high) { #ifdef _WIN32 ULONG_PTR lowLimit = 0; @@ -518,6 +589,9 @@ static void* StartNativeThread(void* opaque) { throw std::runtime_error("Cannot query guest thread stack"); self->stackAddress = reinterpret_cast(high - reserve - self->stackSize); self->threadId = std::this_thread::get_id(); +#if defined(__linux__) + self->hostTid = static_cast(gettid()); +#endif args->initialized.set_value(); } catch (...) { args->initialized.set_exception(std::current_exception()); @@ -601,12 +675,22 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread sigset_t creatorMask; pthread_sigmask(SIG_SETMASK, nullptr, &creatorMask); args->signalMask = creatorMask; +#if defined(__linux__) + ReapRetiredStacks(); + AllocateGuestStack(p.get(), nativeStack); +#endif auto native = std::make_unique(NativeThreadArgs{std::move(args), start.get_future(), {}}); auto initialized = native->initialized.get_future(); pthread_attr_t nativeAttr; - if (const int result = pthread_attr_init(&nativeAttr); result != 0) + if (const int result = pthread_attr_init(&nativeAttr); result != 0) { + FreeGuestStack(p.get()); throw std::system_error(result, std::generic_category(), "Creating guest thread"); + } +#if defined(__linux__) + int created = pthread_attr_setstack(&nativeAttr, static_cast(p->guestStack) + GUEST_STACK_GUARD, p->guestStackBytes - GUEST_STACK_GUARD); +#else int created = pthread_attr_setstacksize(&nativeAttr, nativeStack); +#endif if (created == 0) { sigset_t allSignals; sigfillset(&allSignals); @@ -615,18 +699,22 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread pthread_sigmask(SIG_SETMASK, &creatorMask, nullptr); } pthread_attr_destroy(&nativeAttr); - if (created != 0) + if (created != 0) { + FreeGuestStack(p.get()); throw std::system_error(created, std::generic_category(), "Creating guest thread"); + } native.release(); try { initialized.get(); if (detached) { if (const int result = pthread_detach(p->hostThread); result != 0) throw std::system_error(result, std::generic_category(), "Detaching guest thread"); + RetireGuestStack(p.get()); } } catch (...) { start.set_value(false); pthread_join(p->hostThread, nullptr); + FreeGuestStack(p.get()); throw; } auto* published = p.release(); @@ -655,6 +743,7 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval) { #else if (const int result = pthread_join(thread->hostThread, nullptr); result != 0) throw std::system_error(result, std::generic_category(), "Joining guest thread"); + FreeGuestStack(thread); if (retval) *retval = thread->_retval; ReleaseThread(thread); #endif @@ -670,6 +759,7 @@ int APS5_VABI scePthreadDetach(Pthread thread) { #else if (const int result = pthread_detach(thread->hostThread); result != 0) throw std::system_error(result, std::generic_category(), "Detaching guest thread"); + RetireGuestStack(thread); ReleaseThread(thread); #endif return SCE_OK; diff --git a/core/libs/tests/GuestKernelIsStack.cpp b/core/libs/tests/GuestKernelIsStack.cpp index b2c1097d2..941ffdc97 100644 --- a/core/libs/tests/GuestKernelIsStack.cpp +++ b/core/libs/tests/GuestKernelIsStack.cpp @@ -9,9 +9,11 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval); int APS5_VABI sceKernelIsStack(void* addr, void** start, void** end); int APS5_VABI sceKernelMapFlexibleMemory(void** addr, std::size_t len, int prot, int flags); int APS5_VABI sceKernelMunmap(std::uint64_t addr, std::size_t len); +int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info, std::uint64_t infoSize); } static constexpr int SCE_OK = 0; +static constexpr std::uintptr_t APPLICATION_AREA_END = 0x000000FC00000000ull; static constexpr int SCE_KERNEL_ERROR_EACCES = static_cast(0x8002000d); static int global = 0; @@ -24,6 +26,8 @@ struct WorkerContext { void* start = nullptr; void* end = nullptr; int result = -1; + VirtualQueryInfo region{}; + int regionResult = -1; }; static void* APS5_VABI Worker(void* arg) { @@ -31,6 +35,7 @@ static void* APS5_VABI Worker(void* arg) { int local = 0; context.local = &local; context.result = sceKernelIsStack(&local, &context.start, &context.end); + context.regionResult = sceKernelVirtualQuery(&local, 0, &context.region, sizeof(context.region)); context.queried.store(true); while (!context.release.load()) {} return nullptr; @@ -55,6 +60,11 @@ int main() { while (!context.queried.load()) {} Require(context.result == SCE_OK); Require(context.start <= context.local && context.local < context.end); + Require(context.regionResult == SCE_OK); + Require(context.region.start <= reinterpret_cast(context.local) && reinterpret_cast(context.local) < context.region.end); +#if defined(__linux__) + Require(reinterpret_cast(context.local) < APPLICATION_AREA_END); +#endif Require(sceKernelIsStack(&local, &start, &end) == SCE_OK); Require(context.end <= start || context.start >= end); void* workerStart = nullptr; From 9b6e7f66c4581e44291badeab49a9dc33fc0b1ae Mon Sep 17 00:00:00 2001 From: MarcZX100 Date: Mon, 12 Oct 2026 04:08:31 +0000 Subject: [PATCH 2/3] fix(libkernel): protect host-only guest stack pages --- .../libkernel/DirectMemory/DirectMemory.cpp | 2 + .../prx/libkernel/Pthread/include/Pthread.hpp | 1 + .../libs/prx/libkernel/Pthread/src/Thread.cpp | 47 ++++++++++++++++++- core/libs/tests/GuestKernelStackProtect.cpp | 16 ++++++- 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp index 0f04386b3..a35b55b55 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp @@ -984,6 +984,8 @@ int DoMprotect(const void* addr, size_t len, int prot) { const auto bytes = static_cast(end - first); const auto* pointer = reinterpret_cast(first); const auto nativeProtection = LinuxProtFromSce(prot); + if (GuestThreadStackHostRangeOverlaps(address, address + len)) + throw std::system_error(EACCES, std::generic_category(), "mprotect overlaps host-only guest thread stack pages"); std::uintptr_t stackStart = 0; std::uintptr_t stackEnd = 0; if ((prot & GuestProtGpuReadWrite) == 0 && GuestThreadStack(address, &stackStart, &stackEnd) && address >= stackStart && address + len <= stackEnd) { diff --git a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp index a95394497..b00a839b4 100644 --- a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp +++ b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp @@ -152,6 +152,7 @@ struct PthreadPrivate { }; bool GuestThreadStack(std::uintptr_t address, std::uintptr_t* start, std::uintptr_t* end); +bool GuestThreadStackHostRangeOverlaps(std::uintptr_t start, std::uintptr_t end); void NoteGuestStackProtection(std::uintptr_t start, std::size_t bytes, bool readWrite); int GuestThreadCpuClockId(const PthreadPrivate* thread); bool GuestThreadCpuNanos(int cpuClockThread, std::uint64_t* nanos); diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index 8d0f86e99..b51c5df2f 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -75,6 +75,27 @@ static GuestThreadState& CurrentGuestState() { static std::mutex stackLock; static std::map> liveStacks; static std::map protectedStackPages; +#if defined(__linux__) +struct GuestStackMapping { + std::uintptr_t mappingEnd; + std::uintptr_t stackStart; + std::uintptr_t stackEnd; +}; +static std::mutex guestStackMappingsLock; +static std::map guestStackMappings; + +static void RegisterGuestStackMapping(PthreadPrivate* thread) { + const auto mappingStart = reinterpret_cast(thread->guestStack); + const auto stackStart = reinterpret_cast(thread->stackAddress); + std::lock_guard lock(guestStackMappingsLock); + guestStackMappings[mappingStart] = {mappingStart + thread->guestStackBytes, stackStart, stackStart + thread->stackSize}; +} + +static void UnregisterGuestStackMapping(void* mapping) { + std::lock_guard lock(guestStackMappingsLock); + guestStackMappings.erase(reinterpret_cast(mapping)); +} +#endif #if defined(__linux__) static constexpr std::size_t GUEST_STACK_GUARD = PS5_PAGE_SIZE; @@ -103,7 +124,9 @@ static void AllocateGuestStack(PthreadPrivate* thread, std::size_t usableBytes) static void FreeGuestStack(PthreadPrivate* thread) { if (thread->guestStack == nullptr) return; + void* const mapping = thread->guestStack; DoMunmap(thread->guestStack, thread->guestStackBytes); + UnregisterGuestStackMapping(mapping); thread->guestStack = nullptr; thread->guestStackBytes = 0; } @@ -128,7 +151,10 @@ static void ReapRetiredStacks() { } } } - for (const auto& stack : ready) DoMunmap(stack.address, stack.bytes); + for (const auto& stack : ready) { + DoMunmap(stack.address, stack.bytes); + UnregisterGuestStackMapping(stack.address); + } } static void RetireGuestStack(PthreadPrivate* thread) { @@ -247,6 +273,24 @@ bool GuestThreadStack(std::uintptr_t address, std::uintptr_t* start, std::uintpt return false; } +bool GuestThreadStackHostRangeOverlaps(std::uintptr_t start, std::uintptr_t end) { +#if defined(__linux__) + std::lock_guard lock(guestStackMappingsLock); + auto it = guestStackMappings.upper_bound(start); + if (it != guestStackMappings.begin()) --it; + for (; it != guestStackMappings.end() && it->first < end; ++it) { + const auto mappingStart = it->first; + const auto& stack = it->second; + if (start < stack.stackStart && end > mappingStart) return true; + if (start < stack.mappingEnd && end > stack.stackEnd) return true; + } +#else + (void)start; + (void)end; +#endif + return false; +} + struct CpuClockRegistry { std::mutex lock; std::map threads; @@ -591,6 +635,7 @@ static void* StartNativeThread(void* opaque) { self->threadId = std::this_thread::get_id(); #if defined(__linux__) self->hostTid = static_cast(gettid()); + RegisterGuestStackMapping(self); #endif args->initialized.set_value(); } catch (...) { diff --git a/core/libs/tests/GuestKernelStackProtect.cpp b/core/libs/tests/GuestKernelStackProtect.cpp index 66aed4047..474d0a7d0 100644 --- a/core/libs/tests/GuestKernelStackProtect.cpp +++ b/core/libs/tests/GuestKernelStackProtect.cpp @@ -24,7 +24,7 @@ static constexpr std::uintptr_t Page = 0x4000; static constexpr std::size_t GuardBytes = 0x10000; static constexpr std::size_t HostBytes = 0x40000; -static void Require(bool value) { if (!value) std::abort(); } +#define Require(value) do { if (!(value)) { std::fprintf(stderr, "GuestKernelStackProtect failed at line %d\n", __LINE__); std::abort(); } } while (false) struct PageState { bool mapped; @@ -81,6 +81,16 @@ static void* APS5_VABI Worker(void* arg) { Require(sceKernelIsStack(&local, &start, &end) == SCE_OK); const auto low = (reinterpret_cast(start) + Page - 1) & ~(Page - 1); Require(low + GuardBytes + Page < reinterpret_cast(&local)); + std::uintptr_t hostGuard = 0; + for (auto cursor = low; cursor >= Page && low - cursor <= 4 * 1024 * 1024; cursor -= Page) { + const auto page = QueryPage(cursor); + if (!page.mapped) break; + if (!page.writable) { hostGuard = cursor; break; } + } + Require(hostGuard != 0); + const auto hostGuardBefore = QueryPage(hostGuard).raw; + Require(mprotect_nid_postfix(reinterpret_cast(hostGuard), Page, 3) == -1); + Require(QueryPage(hostGuard).raw == hostGuardBefore); auto* guard = reinterpret_cast(low); Require(mprotect_nid_postfix(const_cast(guard), GuardBytes, 0) == 0); #ifdef _WIN32 @@ -152,7 +162,11 @@ int main() { Require(scePthreadJoin(thread, nullptr) == SCE_OK); for (std::uintptr_t offset = 0; offset < GuardBytes; offset += Page) { const auto left = QueryPage(leftGuard.load() + offset); +#ifdef __linux__ + Require(!left.mapped || left.writable || left.raw == 1); +#else Require(!left.mapped || left.writable); +#endif } Require(scePthreadCreate(&thread, nullptr, WriteLeftGuard, nullptr, nullptr) == SCE_OK); Require(scePthreadJoin(thread, nullptr) == SCE_OK); From 745857448e7111f18fbab9e9b8affa97ab9c8a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Tue, 13 Oct 2026 16:03:39 +0200 Subject: [PATCH 3/3] test(libkernel): check the host-only stack guard on Linux only Windows guest threads keep their normal stacks, so there is no host-only guard page below the guest stack to query. --- core/libs/tests/GuestKernelStackProtect.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/libs/tests/GuestKernelStackProtect.cpp b/core/libs/tests/GuestKernelStackProtect.cpp index 474d0a7d0..f946e321a 100644 --- a/core/libs/tests/GuestKernelStackProtect.cpp +++ b/core/libs/tests/GuestKernelStackProtect.cpp @@ -81,6 +81,7 @@ static void* APS5_VABI Worker(void* arg) { Require(sceKernelIsStack(&local, &start, &end) == SCE_OK); const auto low = (reinterpret_cast(start) + Page - 1) & ~(Page - 1); Require(low + GuardBytes + Page < reinterpret_cast(&local)); +#ifdef __linux__ std::uintptr_t hostGuard = 0; for (auto cursor = low; cursor >= Page && low - cursor <= 4 * 1024 * 1024; cursor -= Page) { const auto page = QueryPage(cursor); @@ -91,6 +92,7 @@ static void* APS5_VABI Worker(void* arg) { const auto hostGuardBefore = QueryPage(hostGuard).raw; Require(mprotect_nid_postfix(reinterpret_cast(hostGuard), Page, 3) == -1); Require(QueryPage(hostGuard).raw == hostGuardBefore); +#endif auto* guard = reinterpret_cast(low); Require(mprotect_nid_postfix(const_cast(guard), GuardBytes, 0) == 0); #ifdef _WIN32