fix(libkernel): refuse fixed no-overwrite mappings over live memory with ENOMEM

MAP_FIXED | MAP_NO_OVERWRITE over a range that already holds an
allocation fails with ENOMEM and leaves the mapping in place, as in
fpPS4's TVirtualManager.check_fixed. UFC 6 (PPSA23566) relies on it to
probe for a free address and aborted on the throw.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/libs/prx/libc/include/GuestAllocations.hpp

@@ -30,6 +30,7 @@ void GuestAllocationsAdd_nid_postfix(void* mutation, void* pointer, std::size_t
 void GuestAllocationsRequireUnpinned_nid_postfix(void* mutation, const void* pointer, std::size_t bytes);
 void GuestAllocationsRequireAvailable_nid_postfix(void* mutation, const void* pointer, std::size_t bytes);
 bool GuestAllocationsCovers_nid_postfix(void* mutation, const void* pointer, std::size_t bytes);
+bool GuestAllocationsOverlaps_nid_postfix(void* mutation, const void* pointer, std::size_t bytes);
 Range GuestAllocationsFind_nid_postfix(void* mutation, const void* pointer);
 void GuestAllocationsRemove_nid_postfix(void* mutation, const void* pointer);
 void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, bool readable, bool writable, const std::function<void()>& apply);
@@ -60,6 +61,7 @@ public:
     void RequireUnpinned(const void* pointer, std::size_t bytes) const { GuestAllocationsRequireUnpinned_nid_postfix(handle, pointer, bytes); }
     void RequireAvailable(const void* pointer, std::size_t bytes) const { GuestAllocationsRequireAvailable_nid_postfix(handle, pointer, bytes); }
     bool Covers(const void* pointer, std::size_t bytes) const { return GuestAllocationsCovers_nid_postfix(handle, pointer, bytes); }
+    bool Overlaps(const void* pointer, std::size_t bytes) const { return GuestAllocationsOverlaps_nid_postfix(handle, pointer, bytes); }
     Range Find(const void* pointer) const { return GuestAllocationsFind_nid_postfix(handle, pointer); }
     void Remove(const void* pointer) { GuestAllocationsRemove_nid_postfix(handle, pointer); }
     void Unmap(const void* pointer, std::size_t bytes, const std::function<void(const void*, std::size_t, const void*, bool)>& apply) { GuestAllocationsUnmap_nid_postfix(handle, pointer, bytes, apply); }

core/libs/prx/libc/src/GuestAllocations.cpp

@@ -257,6 +257,16 @@ void GuestAllocationsRequireAvailable_nid_postfix(void*, const void* pointer, st
     }
 }
 
+bool GuestAllocationsOverlaps_nid_postfix(void*, const void* pointer, std::size_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(pointer);
+    require(address != 0 && bytes != 0 && bytes <= std::numeric_limits<std::uint64_t>::max() - address, "invalid guest allocation range");
+    for (const auto& [base, range] : registry().ranges) {
+        if (base >= address + bytes) break;
+        if (base + range->bytes > address) return true;
+    }
+    return false;
+}
+
 bool GuestAllocationsCovers_nid_postfix(void*, const void* pointer, std::size_t bytes) {
     const auto address = reinterpret_cast<std::uintptr_t>(pointer);
     require(address != 0 && bytes != 0 && bytes <= std::numeric_limits<std::uint64_t>::max() - address, "invalid guest allocation range");

core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp

@@ -565,6 +565,26 @@ void ValidateOutput(void** addr) {
 
 }
 
+bool Reserved(const void* addr, size_t len) {
+    const auto start = reinterpret_cast<std::uintptr_t>(addr);
+    const auto end = start + len;
+    std::lock_guard lock(g_reservationLock);
+    auto it = g_reservations.upper_bound(start);
+    if (it == g_reservations.begin()) return false;
+    auto cursor = start;
+    for (--it; it != g_reservations.end() && it->first <= cursor; ++it) {
+        cursor = std::max(cursor, it->second);
+        if (cursor >= end) return true;
+    }
+    return false;
+}
+
+bool FixedNoOverwriteConflict(const GuestAllocations::Mutation& mutation, void* addr, size_t len, int flags) {
+    constexpr int GuestMapNoOverwrite = 0x80;
+    if (addr == nullptr || (flags & GuestMapFixedFlag) == 0 || (flags & GuestMapNoOverwrite) == 0) return false;
+    return mutation.Overlaps(addr, len) && !Reserved(addr, len);
+}
+
 int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart, size_t alignment) {
     ValidateOutput(addr);
     if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0) return SCE_KERNEL_ERROR_EINVAL;
@@ -572,6 +592,7 @@ int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart,
         return SCE_KERNEL_ERROR_EINVAL;
     }
     GuestAllocations::Mutation mutation;
+    if (FixedNoOverwriteConflict(mutation, *addr, len, flags)) return SCE_KERNEL_ERROR_ENOMEM;
     if (RemapFixedIntoRegistered(mutation, *addr, len, prot, flags, physStart)) {
         EraseReservations(*addr, len);
         RecordProtection(*addr, len, prot);
@@ -600,6 +621,7 @@ int DoMapAnon(void** addr, size_t len, int prot, int flags) {
     ValidateOutput(addr);
     if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0) return SCE_KERNEL_ERROR_EINVAL;
     GuestAllocations::Mutation mutation;
+    if (FixedNoOverwriteConflict(mutation, *addr, len, flags)) return SCE_KERNEL_ERROR_ENOMEM;
     if (RemapFixedIntoRegistered(mutation, *addr, len, prot, flags)) {
         EraseReservations(*addr, len);
         RecordProtection(*addr, len, prot);

core/libs/tests/GuestMemory.cpp

@@ -111,10 +111,9 @@ static void CheckNamedAndHintedMappings() {
     Require(sceKernelMapFlexibleMemory(&hinted, length, 3, 0) == 0);
     Require(hinted > first && (reinterpret_cast<std::uintptr_t>(hinted) & 0x3fff) == 0);
     static_cast<volatile unsigned char*>(hinted)[length - 1] = 1;
-    bool rejected = false;
     void* overwrite = first;
-    try { sceKernelMapFlexibleMemory(&overwrite, 0x4000, 3, 0x90); } catch (const std::exception&) { rejected = true; }
-    Require(rejected && overwrite == first);
+    Require(sceKernelMapFlexibleMemory(&overwrite, 0x4000, 3, 0x90) == static_cast<int>(0x8002000cu));
+    Require(overwrite == first);
     Require(sceKernelMunmap(hinted, length) == 0);
     void* exclusive = hinted;
     Require(sceKernelMapFlexibleMemory(&exclusive, length, 3, 0x90) == 0);
@@ -366,6 +365,27 @@ static void CheckReservedRangeIsNotCommitted() {
     Require(sceKernelReleaseDirectMemory(phys, page) == 0);
 }
 
+static void CheckNoOverwriteRefusesLiveMapping() {
+    constexpr std::size_t page = 0x4000;
+    constexpr int outOfMemory = static_cast<int>(0x8002000cu);
+    std::int64_t phys = 0;
+    Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * 2, 0, 0, &phys) == 0);
+    void* mapped = nullptr;
+    Require(sceKernelMapDirectMemory(&mapped, page, 3, 0, phys, 0) == 0);
+    static_cast<unsigned char*>(mapped)[0] = 13;
+    void* again = mapped;
+    Require(sceKernelMapDirectMemory(&again, page, 3, 0x90, phys + page, 0) == outOfMemory);
+    Require(again == mapped);
+    void* flexible = mapped;
+    Require(sceKernelMapFlexibleMemory(&flexible, page, 3, 0x90) == outOfMemory);
+    Require(static_cast<unsigned char*>(mapped)[0] == 13);
+    VirtualQueryInfo info{};
+    Require(sceKernelVirtualQuery(mapped, 0, &info, sizeof(info)) == 0);
+    Require(info.is_direct && info.offset == static_cast<std::uint64_t>(phys));
+    Require(sceKernelMunmap(mapped, page) == 0);
+    Require(sceKernelReleaseDirectMemory(phys, page * 2) == 0);
+}
+
 #ifdef _WIN32
 static void CheckNoOverwriteRejectsHostOccupiedMapping() {
     constexpr std::size_t page = 0x4000;
@@ -857,6 +877,7 @@ int main() {
     CheckReleaseDirectMemoryClearsMappings();
     CheckFixedVirtualReservation();
     CheckReservedRangeIsNotCommitted();
+    CheckNoOverwriteRefusesLiveMapping();
     CheckMlock();
     CheckSharedDirectMemoryLifecycle();
     CheckGetDirectMemoryType();