Merge pull request #2907 from oneandonlydean/carry/libkernel-reserved-range-hint

fix(libkernel): map non-fixed hints inside a reserved range in place

core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp

@@ -762,6 +762,11 @@ bool FixedNoOverwriteConflict(const GuestAllocations::Mutation& mutation, void*
     return mutation.Overlaps(addr, len) && !Reserved(addr, len);
 }
 
+int PlaceHintInReservation(void* addr, size_t len, int flags) {
+    if (addr == nullptr || (flags & GuestMapFixedFlag) != 0 || !Reserved(addr, len)) return flags;
+    return flags | GuestMapFixedFlag;
+}
+
 int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart, size_t alignment) {
     ValidateOutput(addr);
     if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0) return SCE_KERNEL_ERROR_EINVAL;
@@ -769,6 +774,7 @@ int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart,
         return SCE_KERNEL_ERROR_EINVAL;
     }
     GuestAllocations::Mutation mutation;
+    flags = PlaceHintInReservation(*addr, len, flags);
     if (FixedNoOverwriteConflict(mutation, *addr, len, flags)) return SCE_KERNEL_ERROR_ENOMEM;
     if (RemapFixedIntoRegistered(mutation, *addr, len, prot, flags, physStart)) {
         EraseReservations(*addr, len);
@@ -798,6 +804,7 @@ int DoMapAnon(void** addr, size_t len, int prot, int flags, size_t alignment) {
     ValidateOutput(addr);
     if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0) return SCE_KERNEL_ERROR_EINVAL;
     GuestAllocations::Mutation mutation;
+    flags = PlaceHintInReservation(*addr, len, flags);
     if (FixedNoOverwriteConflict(mutation, *addr, len, flags)) return SCE_KERNEL_ERROR_ENOMEM;
     if (RemapFixedIntoRegistered(mutation, *addr, len, prot, flags)) {
         EraseReservations(*addr, len);

core/libs/tests/GuestMemory.cpp

@@ -530,6 +530,37 @@ static void CheckFixedVirtualReservation() {
     Require(sceKernelMunmap(pooled, page * 2) == 0);
 }
 
+static void CheckHintInsideReservation() {
+    constexpr std::size_t page = 0x4000;
+    void* reserved = nullptr;
+    Require(sceKernelReserveVirtualRange(&reserved, page * 4, 0, 0) == 0);
+    auto* base = static_cast<unsigned char*>(reserved);
+    std::int64_t phys = 0;
+    Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * 3, 0, 0, &phys) == 0);
+    void* direct = base + page;
+    Require(sceKernelMapDirectMemory(&direct, page, 3, 0, phys, 0) == 0);
+    Require(direct == base + page);
+    static_cast<unsigned char*>(direct)[0] = 21;
+    void* flexible = base + page * 2;
+    Require(sceKernelMapFlexibleMemory(&flexible, page, 3, 0) == 0);
+    Require(flexible == base + page * 2);
+    static_cast<unsigned char*>(flexible)[0] = 22;
+    VirtualQueryInfo info{};
+    Require(sceKernelVirtualQuery(direct, 0, &info, sizeof(info)) == 0);
+    Require(info.is_direct && info.offset == static_cast<std::uint64_t>(phys));
+    Require(sceKernelVirtualQuery(flexible, 0, &info, sizeof(info)) == 0);
+    Require(info.is_flexible);
+    Require(sceKernelVirtualQuery(base + page * 3, 0, &info, sizeof(info)) == 0);
+    Require(!info.is_committed && info.protection == 0);
+    void* overlapping = base + page;
+    Require(sceKernelMapDirectMemory(&overlapping, page * 2, 3, 0, phys + page, 0) == 0);
+    Require(overlapping != base + page);
+    Require(static_cast<unsigned char*>(direct)[0] == 21 && static_cast<unsigned char*>(flexible)[0] == 22);
+    Require(sceKernelMunmap(overlapping, page * 2) == 0);
+    Require(sceKernelMunmap(reserved, page * 4) == 0);
+    Require(sceKernelReleaseDirectMemory(phys, page * 3) == 0);
+}
+
 static void CheckReservedRangeIsNotCommitted() {
     constexpr std::size_t page = 0x4000;
     void* reserved = nullptr;
@@ -1545,6 +1576,7 @@ int main() {
     CheckHintedVirtualReservation();
     CheckFixedVirtualReservation();
     CheckReservedRangeIsNotCommitted();
+    CheckHintInsideReservation();
     CheckNoOverwriteRefusesLiveMapping();
     CheckMlock();
     CheckSharedDirectMemoryLifecycle();

docs/dev/TechnicalDebt.md

@@ -150,6 +150,7 @@ Throughout the project, every function at every stage either **does exactly what
 - [sceKernelGetCurrentCpu](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - guest affinities are not applied to host threads, so the host CPU index is mapped into the calling thread's affinity clipped to `sceKernelGetAvailableCpumask` (the available mask when they do not overlap): host CPU n reports the (n modulo the CPU count)-th CPU of that mask, so a thread pinned to one CPU reports that CPU, as a FreeBSD `cpuset` guarantees on the console. Which CPU the console reports for a thread allowed on several is not known; shadPS4 returns 0
 - [sceKernelMapNamedFlexibleMemoryInternal](../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - flag 0x8000 is accepted and ignored; its meaning is not established. KytyPS5 (`GUEST_MAP_UNKNOWN_8000`) and sharpemu accept it without reading it. Every known caller is the SDK 9 `libc.prx` heap (Astro Bot PPSA21564, PPSA22520), which passes it with read/write protection and uses the mapping as an older `libc.prx` uses `sceKernelMapNamedFlexibleMemory` with flags 0
 - [sceKernelMtypeprotect](../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - sets the memory type of the direct mappings in the page-rounded range and of the direct memory behind them, as shadPS4 and prosper do (KytyPS5 ignores the type); the type is not validated, flexible pages in the range only get the protection, and a later map of the retyped direct memory reports the type it was allocated with. `sceKernelBatchMap2` operation 4 takes the same path with the entry type
+- [Map hints inside a reservation](../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - `sceKernelMapDirectMemory` and the flexible and anonymous maps place a mapping without `MAP_FIXED` at its hint address when the whole range lies in a range still reserved by `sceKernelReserveVirtualRange`, as if `MAP_FIXED` were set. Taken from PPSA01474, which reserves 64 GiB and then maps its heaps inside it with hints and no `MAP_FIXED`, and uses the hint addresses afterwards; not measured on the console. Hints outside a reservation, or over a range that is partly mapped, stay hints
 - [Guest arena](../../core/libs/prx/libc/src/GuestArena.cpp) (libc) - the application map area is assumed to end at 0xFC_0000_0000, as `SCE_KERNEL_APP_MAP_AREA_END_ADDR` does on the PS4. On Windows, host allocations made in the arena before libc loads stay, and a fixed guest mapping over one throws
 - [sceLibcInternalBacktraceForGame](../../core/libs/prx/libc/src/HeapDiagnostics.cpp) (libSceLibcInternal, implemented in libc) - unknown signature
 - [sceLibcInternalHeapErrorReportForGame](../../core/libs/prx/libc/src/HeapDiagnostics.cpp) (libSceLibcInternal, implemented in libc) - unknown signature