feat(libkernel): implement the rtld thread atexit reference count

A title's libc.prx passes the dso address of __cxa_thread_atexit to
_sceKernelRtldThreadAtexitIncrement and fails the registration when it
returns nonzero. Increment takes a reference on the loaded image that
contains the address, so it stays mapped while thread_local destructors
are pending, and Decrement releases one. An address outside every
loaded image and a Decrement without a reference throw.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/libs/prx/libkernel/System/src/Rtld.cpp

@@ -2,16 +2,78 @@
 #include <cstddef>
 #include <array>
 #include <cstring>
+#include <map>
 #include <mutex>
 #include <stdexcept>
+#include <string>
 #include "SceTypes.hpp"
 #include "prx/libc/include/General.hpp"
 #include "prx/libkernel/Pthread/include/ThreadLifecycle.hpp"
+#ifdef _WIN32
+#define NOMINMAX
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#include <link.h>
+#endif
 
 namespace {
 
 std::mutex rtldHeapMutex;
 std::array<void*, 10> rtldHeapApi{};
+std::mutex threadAtexitMutex;
+std::multimap<const void*, void*> threadAtexitReferences;
+
+const void* imageContaining(const void* address, bool reference, void*& handle, const char* caller) {
+    handle = nullptr;
+#ifdef _WIN32
+    const DWORD flags = GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | (reference ? 0 : GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT);
+    HMODULE module = nullptr;
+    if (!GetModuleHandleExW(flags, reinterpret_cast<LPCWSTR>(address), &module))
+        throw std::runtime_error(std::string(caller) + ": address outside every loaded image");
+    if (reference) handle = module;
+    return module;
+#else
+    Dl_info info{};
+    link_map* image = nullptr;
+    if (!dladdr1(address, &info, reinterpret_cast<void**>(&image), RTLD_DL_LINKMAP) || !image)
+        throw std::runtime_error(std::string(caller) + ": address outside every loaded image");
+    if (reference && image->l_name && *image->l_name) {
+        handle = dlopen(image->l_name, RTLD_LAZY | RTLD_NOLOAD);
+        if (!handle) throw std::runtime_error(std::string(caller) + ": cannot reference " + image->l_name);
+    }
+    return image;
+#endif
+}
+
+int referenceThreadAtexitImage(const void* address, const char* caller) {
+    void* handle = nullptr;
+    const void* image = imageContaining(address, true, handle, caller);
+    std::lock_guard lock(threadAtexitMutex);
+    threadAtexitReferences.emplace(image, handle);
+    return 0;
+}
+
+int releaseThreadAtexitImage(const void* address, const char* caller) {
+    void* handle = nullptr;
+    const void* image = imageContaining(address, false, handle, caller);
+    {
+        std::lock_guard lock(threadAtexitMutex);
+        const auto found = threadAtexitReferences.find(image);
+        if (found == threadAtexitReferences.end())
+            throw std::runtime_error(std::string(caller) + ": image holds no thread atexit reference");
+        handle = found->second;
+        threadAtexitReferences.erase(found);
+    }
+    if (!handle) return 0;
+#ifdef _WIN32
+    if (!FreeLibrary(static_cast<HMODULE>(handle)))
+#else
+    if (dlclose(handle) != 0)
+#endif
+        throw std::runtime_error(std::string(caller) + ": cannot release image");
+    return 0;
+}
 
 void registerApplicationHeapApi(void* const* api) {
     if (api == nullptr)
@@ -34,16 +96,12 @@ void APS5_VABI sceKernelRtldSetApplicationHeapAPI(void* api[]) {
     registerApplicationHeapApi(api);
 }
 
-int APS5_VABI sceKernelRtldThreadAtexitDecrement(uint64_t* c) {
- (void)c;
- NotImplemented_nid_no_patch(__func__);
- return 0;
+int APS5_VABI sceKernelRtldThreadAtexitDecrement(const void* dsoSymbol) {
+    return releaseThreadAtexitImage(dsoSymbol, __func__);
 }
 
-int APS5_VABI sceKernelRtldThreadAtexitIncrement(uint64_t* c) {
- (void)c;
- NotImplemented_nid_no_patch(__func__);
- return 0;
+int APS5_VABI sceKernelRtldThreadAtexitIncrement(const void* dsoSymbol) {
+    return referenceThreadAtexitImage(dsoSymbol, __func__);
 }
 
 void APS5_VABI sceKernelSetThreadAtexitCount(get_thread_atexit_count_func_t func) {
@@ -66,16 +124,12 @@ void APS5_VABI _sceKernelRtldSetApplicationHeapAPI_nid_postfix(void* api[]) {
     registerApplicationHeapApi(api);
 }
 
-int APS5_VABI _sceKernelRtldThreadAtexitDecrement_nid_postfix(std::uint64_t* counter) {
-    (void)counter;
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+int APS5_VABI _sceKernelRtldThreadAtexitDecrement_nid_postfix(const void* dsoSymbol) {
+    return releaseThreadAtexitImage(dsoSymbol, "_sceKernelRtldThreadAtexitDecrement");
 }
 
-int APS5_VABI _sceKernelRtldThreadAtexitIncrement_nid_postfix(std::uint64_t* counter) {
-    (void)counter;
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+int APS5_VABI _sceKernelRtldThreadAtexitIncrement_nid_postfix(const void* dsoSymbol) {
+    return referenceThreadAtexitImage(dsoSymbol, "_sceKernelRtldThreadAtexitIncrement");
 }
 
 void APS5_VABI _sceKernelSetThreadAtexitCount_nid_postfix(get_thread_atexit_count_func_t callback) {

core/libs/tests/GuestDynamicLoader.cpp

@@ -2,14 +2,26 @@
 #include <cstdlib>
 #include <filesystem>
 #include <fstream>
+#include <stdexcept>
 #include <thread>
 extern "C" {
 void* APS5_VABI dlopen_nid_postfix(const char*, int);
 void* APS5_VABI dlsym_nid_postfix(void*, const char*);
 int APS5_VABI dlclose_nid_postfix(void*);
 char* APS5_VABI dlerror_nid_postfix();
+int APS5_VABI _sceKernelRtldThreadAtexitIncrement_nid_postfix(const void*);
+int APS5_VABI _sceKernelRtldThreadAtexitDecrement_nid_postfix(const void*);
 }
 static void Require(bool value) { if (!value) std::abort(); }
+template<typename TFunction>
+static bool ThrowsRuntimeError(TFunction function) {
+    try {
+        function();
+    } catch (const std::runtime_error&) {
+        return true;
+    }
+    return false;
+}
 int main(int argc, char** argv) {
     Require(argc == 2);
     Require(dlerror_nid_postfix() == nullptr);
@@ -61,4 +73,19 @@ int main(int argc, char** argv) {
     Require(dlclose_nid_postfix(redirected) == 0);
     std::filesystem::remove(guest);
     std::filesystem::remove(relinked);
+
+    void* pinned = dlopen_nid_postfix(argv[1], 2);
+    add = reinterpret_cast<Add>(dlsym_nid_postfix(pinned, "GuestModuleAdd"));
+    Require(add && _sceKernelRtldThreadAtexitIncrement_nid_postfix(reinterpret_cast<const void*>(add)) == 0);
+    Require(_sceKernelRtldThreadAtexitIncrement_nid_postfix(reinterpret_cast<const void*>(add)) == 0);
+    Require(dlclose_nid_postfix(pinned) == 0);
+    Require(add(20, 22) == 42);
+    Require(_sceKernelRtldThreadAtexitDecrement_nid_postfix(reinterpret_cast<const void*>(add)) == 0);
+    Require(add(40, 2) == 42);
+    Require(_sceKernelRtldThreadAtexitDecrement_nid_postfix(reinterpret_cast<const void*>(add)) == 0);
+    Require(_sceKernelRtldThreadAtexitIncrement_nid_postfix(reinterpret_cast<const void*>(&ThrowsRuntimeError<void (*)()>)) == 0);
+    Require(_sceKernelRtldThreadAtexitDecrement_nid_postfix(reinterpret_cast<const void*>(&ThrowsRuntimeError<void (*)()>)) == 0);
+    Require(ThrowsRuntimeError([] { _sceKernelRtldThreadAtexitDecrement_nid_postfix(reinterpret_cast<const void*>(&Require)); }));
+    int local = 0;
+    Require(ThrowsRuntimeError([&] { _sceKernelRtldThreadAtexitIncrement_nid_postfix(&local); }));
 }

docs/dev/TechnicalDebt.md

@@ -72,6 +72,7 @@ Throughout the project, every function at every stage either **does exactly what
 - [sceLibcInternalHeapErrorReportForGame](../../core/libs/prx/libc/src/HeapDiagnostics.cpp) (libSceLibcInternal, implemented in libc) - unknown signature
 - [std::_Fiopen](../../core/libs/prx/libc/src/CxxAbiSupport.cpp) (libc) - `_ZSt7_FiopenPKcNSt5_IosbIiE9_OpenmodeEi` (NID `vYWK2Pz8vGE`; the name comes from the public NID list): the values of the `_Openmode` argument and the meaning of the protection argument are unknown, so it throws when called. It is exported so that a title that imports it (PPSA21564 and PPSA21567 do) loads against the host libc when its own `libc.prx` is left out of the guest conversion.
 - [_sceLibcInternalThreadAtexit, _sceLibcInternalThreadDtors](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) - FreeBSD `__cxa_thread_atexit` semantics assumed; a null or non-image destructor and an allocation failure throw
+- [_sceKernelRtldThreadAtexitIncrement, _sceKernelRtldThreadAtexitDecrement](../../core/libs/prx/libkernel/System/src/Rtld.cpp) (libkernel) - the argument is the `dso` address a title's `libc.prx` passes to `__cxa_thread_atexit` (PPSA14632), which fails when Increment returns nonzero. As in the [fpPS4](https://github.com/red-prig/fpPS4) reimplementation, Increment takes a reference on the loaded image containing the address and Decrement releases one; the variants without the leading underscore are assumed to match
 - [_sceLibcInternalForceTlsDestructor](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) - unknown return type and behaviour; a title's `libc.prx` calls it from `__cxa_finalize` with the module handle of a non-null dso
 - [__progname](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - unknown data export
 - [pthread_barrierattr_setpshared](../../core/libs/prx/libkernel/Pthread/Posix/Barrier.cpp) (libkernel) - PTHREAD_PROCESS_SHARED throws: FreeBSD 9.0 libthr rejects it with EINVAL and 11.0 accepts it, and which one the console follows is unknown