fix(libkernel): unregister guest module images when they are unloaded

A .guest.prx image registered by sceKernelMprotect stayed in the guest
allocation registry after dlclose freed it, so its dead ranges blocked
later mappings at that address and could reach GPU builds.

The Windows module destructor now unregisters the image before
FreeLibrary. Unregistering waits for outstanding leases on the image like
unmap and protect do; if the wait fails the module stays loaded.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/libs/prx/libc/include/GuestAllocations.hpp

@@ -27,6 +27,7 @@ void* GuestAllocationsBegin_nid_postfix();
 void GuestAllocationsRegisterMainImage_nid_postfix(void* mutation);
 #ifdef _WIN32
 void GuestAllocationsRegisterImage_nid_postfix(void* mutation, const void* image);
+void GuestAllocationsUnregisterImage_nid_postfix(void* mutation, const void* image);
 #endif
 void GuestAllocationsEnd_nid_postfix(void* mutation) noexcept;
 void GuestAllocationsAdd_nid_postfix(void* mutation, void* pointer, std::size_t bytes, bool readable, bool writable);
@@ -63,6 +64,7 @@ public:
     void RegisterMainImage() { GuestAllocationsRegisterMainImage_nid_postfix(handle); }
 #ifdef _WIN32
     void RegisterImage(const void* image) { GuestAllocationsRegisterImage_nid_postfix(handle, image); }
+    void UnregisterImage(const void* image) { GuestAllocationsUnregisterImage_nid_postfix(handle, image); }
 #endif
     void Add(void* pointer, std::size_t bytes, bool readable, bool writable) { GuestAllocationsAdd_nid_postfix(handle, pointer, bytes, readable, writable); }
     void RequireUnpinned(const void* pointer, std::size_t bytes) const { GuestAllocationsRequireUnpinned_nid_postfix(handle, pointer, bytes); }

core/libs/prx/libc/src/GuestAllocations.cpp

@@ -8,7 +8,6 @@
 #include <limits>
 #include <iterator>
 #include <map>
-#include <set>
 #include <stdexcept>
 #ifdef _WIN32
 #ifndef NOMINMAX
@@ -35,7 +34,7 @@ struct Registry {
     std::mutex mutex;
     std::map<std::uint64_t, std::shared_ptr<const Range>> ranges;
     bool mainImageRegistered = false;
-    std::set<std::uintptr_t> registeredImages;
+    std::map<std::uintptr_t, std::uintptr_t> registeredImages;
 };
 
 Registry& registry() {
@@ -168,7 +167,7 @@ void registerImage(const void* image) {
     }
     require(registered, "guest image has no committed pages");
     state.ranges.swap(replacement);
-    state.registeredImages.insert(reinterpret_cast<std::uintptr_t>(image));
+    state.registeredImages.emplace(reinterpret_cast<std::uintptr_t>(image), cursor);
 }
 }
 
@@ -185,6 +184,17 @@ void GuestAllocationsRegisterImage_nid_postfix(void*, const void* image) {
     require(image != nullptr, "cannot register a guest image without a base");
     registerImage(image);
 }
+
+void GuestAllocationsUnregisterImage_nid_postfix(void* mutation, const void* image) {
+    auto& state = registry();
+    const auto found = state.registeredImages.find(reinterpret_cast<std::uintptr_t>(image));
+    if (found == state.registeredImages.end()) return;
+    const auto [begin, end] = *found;
+    GuestAllocationsRequireUnpinned_nid_postfix(mutation, image, end - begin);
+    recordChange(mutation, image, end - begin);
+    std::erase_if(state.ranges, [&](const auto& entry) { return !entry.second->releasable && entry.second->allocationAddress >= begin && entry.second->allocationAddress < end; });
+    state.registeredImages.erase(found);
+}
 #else
 void GuestAllocationsRegisterMainImage_nid_postfix(void*) {
     auto& state = registry();

core/libs/prx/libkernel/Module/src/DynamicLoader.cpp

@@ -1,4 +1,5 @@
 #include "prx/libc/include/General.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
 #include <nid/NidCompute.hpp>
 #include <array>
 #include <filesystem>
@@ -32,6 +33,13 @@ struct Module {
     ~Module() {
         if (owned && native) {
 #ifdef _WIN32
+            try {
+                GuestAllocations::Mutation mutation;
+                mutation.UnregisterImage(native);
+            } catch (const std::exception& error) {
+                std::fprintf(stderr, "[dlclose] module %p stays loaded: %s\n", native, error.what());
+                return;
+            }
             FreeLibrary(static_cast<HMODULE>(native));
 #else
             ::dlclose(native);

core/libs/tests/GuestMemory.cpp

@@ -7,6 +7,7 @@
 #include "prx/libkernel/KernelErrors.hpp"
 #include "prx/libkernel/DirectMemory/DirectMemory.hpp"
 #include <array>
+#include <atomic>
 #include "SceTypes.hpp"
 #include <chrono>
 #include <cstring>
@@ -71,6 +72,9 @@ int APS5_VABI sceKernelAioDeleteRequest(std::int32_t, std::int32_t*);
 int APS5_VABI sceKernelMlock_nid_postfix(void*, std::uint64_t);
 int APS5_VABI sceKernelGetDirectMemoryType(std::int64_t, int*, std::int64_t*, std::int64_t*);
 int APS5_VABI sceKernelBatchMap2(KernelBatchMapEntry*, int, int*, int);
+void* APS5_VABI dlopen_nid_postfix(const char*, int);
+void* APS5_VABI dlsym_nid_postfix(void*, const char*);
+int APS5_VABI dlclose_nid_postfix(void*);
 }
 
 static void Require(bool condition, std::source_location location = std::source_location::current()) {
@@ -541,15 +545,28 @@ static DWORD ImageProtection(const void* pointer) {
     return memory.Protect & 0xffu;
 }
 
-static void CheckGuestModuleImageProtection() {
-    constexpr std::size_t page = 0x4000;
-    const auto module = LoadLibraryA(GUEST_MEMORY_MODULE);
-    Require(module != nullptr);
-    auto* data = reinterpret_cast<unsigned char*>(GetProcAddress(module, "guestMemoryModuleData"));
+static unsigned char* GuestModulePage(void* module, std::size_t page) {
+    auto* data = static_cast<unsigned char*>(dlsym_nid_postfix(module, "guestMemoryModuleData"));
     Require(data != nullptr);
     auto* target = reinterpret_cast<unsigned char*>((reinterpret_cast<std::uintptr_t>(data) + page - 1) & ~(page - 1));
     Require(target + page <= data + 0x10000);
+    return target;
+}
+
+static bool RegisteredGuestRange(const void* pointer, std::size_t bytes) {
+    GuestAllocations::Mutation mutation;
+    return mutation.Overlaps(pointer, bytes);
+}
+
+static void CheckGuestModuleImageProtection() {
+    constexpr std::size_t page = 0x4000;
+    const auto path = std::filesystem::relative(GUEST_MEMORY_MODULE).generic_string();
+    void* module = dlopen_nid_postfix(path.c_str(), 2);
+    Require(module != nullptr);
+    auto* target = GuestModulePage(module, page);
+    Require(!RegisteredGuestRange(target, page));
     Require(sceKernelMprotect(target, page, 1) == 0);
+    Require(RegisteredGuestRange(target, page));
     Require(ImageProtection(target) == PAGE_READONLY);
     Require(sceKernelMprotect(target, page, 3) == 0);
     Require(ImageProtection(target) == PAGE_READWRITE || ImageProtection(target) == PAGE_WRITECOPY);
@@ -557,6 +574,35 @@ static void CheckGuestModuleImageProtection() {
     Require(target[1] == 7);
     Require(sceKernelMprotect(target, page, 1) == 0);
     Require(ImageProtection(target) == PAGE_READONLY);
+
+    auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
+    std::atomic<bool> closed{false};
+    std::thread closer([&] {
+        Require(dlclose_nid_postfix(module) == 0);
+        closed = true;
+    });
+    std::this_thread::sleep_for(std::chrono::milliseconds(200));
+    Require(!closed);
+    Require(ImageProtection(target) == PAGE_READONLY);
+    lease.clear();
+    closer.join();
+    Require(!RegisteredGuestRange(target, page));
+    {
+        GuestAllocations::Mutation mutation;
+        mutation.Add(target, page, true, true);
+        mutation.Remove(target);
+    }
+
+    module = dlopen_nid_postfix(path.c_str(), 2);
+    Require(module != nullptr);
+    target = GuestModulePage(module, page);
+    Require(!RegisteredGuestRange(target, page));
+    Require(sceKernelMprotect(target, page, 1) == 0);
+    Require(ImageProtection(target) == PAGE_READONLY);
+    Require(RegisteredGuestRange(target, page));
+    Require(dlclose_nid_postfix(module) == 0);
+    Require(!RegisteredGuestRange(target, page));
+
     bool rejected = false;
     try {
         sceKernelMprotect(GetModuleHandleA("kernel32.dll"), page, 1);