From 1bbdf7daa69d7f588ff9d01ded25317f66c3b40c Mon Sep 17 00:00:00 2001 From: SamG-Coder <13449963+SamG-Coder@users.noreply.github.com> Date: Fri, 25 Sep 2026 06:06:31 +1000 Subject: [PATCH] feat(kernel): support private anonymous guest memory mappings --- core/libs/CMakeLists.txt | 9 +++ core/libs/prx/libkernel/CMakeLists.txt | 1 + .../libkernel/DirectMemory/PosixMemory.cpp | 81 +++++++++++++++++++ core/libs/tests/GuestMemory.cpp | 70 ++++++++++++++++ docs/YAMAGI_QUAKE2.md | 17 ++++ 5 files changed, 178 insertions(+) create mode 100644 core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp create mode 100644 core/libs/tests/GuestMemory.cpp diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index b45e3cbcf..f9970a384 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -104,6 +104,15 @@ endforeach() add_custom_target(libs DEPENDS ${PATCHED_OUTPUTS}) if(BUILD_TESTING) + add_executable(guest_memory_tests tests/GuestMemory.cpp) + target_include_directories(guest_memory_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(guest_memory_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_memory_tests) + add_test(NAME guest_memory COMMAND guest_memory_tests) + if(WIN32) + set_tests_properties(guest_memory PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_executable(guest_string_tests tests/GuestStrings.cpp prx/libc/src/StringExtensions.cpp diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 359346859..598cca734 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -7,6 +7,7 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Semaphore/src/Semaphore.cpp ${CMAKE_CURRENT_SOURCE_DIR}/DirectMemory/Export.cpp ${CMAKE_CURRENT_SOURCE_DIR}/DirectMemory/DirectMemory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/DirectMemory/PosixMemory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/DirectMemory/MemoryPool.cpp ${CMAKE_CURRENT_SOURCE_DIR}/DirectMemory/MemoryPoolExport.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Pthread/src/Mutex.cpp diff --git a/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp b/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp new file mode 100644 index 000000000..e3d2f1796 --- /dev/null +++ b/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp @@ -0,0 +1,81 @@ +#include "DirectMemory.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include + +extern "C" int* APS5_VABI __error_nid_postfix(); + +namespace { +// FreeBSD/PS5 ABI values, independent of the host's errno and mmap constants. +constexpr int GuestInvalid = 22; +constexpr int GuestNoMemory = 12; +constexpr int GuestNotSupported = 45; +constexpr int GuestPrivate = 0x2; +constexpr int GuestAnonymous = 0x1000; + +bool RoundLength(std::size_t length, std::size_t& rounded) { + constexpr auto mask = PS5_PAGE_SIZE - 1; + if (length == 0 || length > std::numeric_limits::max() - mask) + return false; + rounded = (length + mask) & ~mask; + return true; +} + +void SetError(int error) { + *__error_nid_postfix() = error; +} +} + +extern "C" { + +void* APS5_VABI mmap_nid_postfix(void* address, std::size_t length, int protection, + int flags, int descriptor, std::int64_t offset) noexcept { + const auto failed = [](int error) -> void* { + SetError(error); + return reinterpret_cast(static_cast(-1)); + }; + std::size_t rounded; + if (!RoundLength(length, rounded) || (protection & ~7) != 0) + return failed(GuestInvalid); + if (flags != (GuestPrivate | GuestAnonymous)) + return failed(GuestNotSupported); + if (descriptor != -1 || offset != 0) + return failed(GuestInvalid); + // A non-fixed address is a hint; the kernel may choose another address. + (void)address; + void* mapped = nullptr; + try { + const auto result = DoMapAnon(&mapped, rounded, protection, 0); + if (result != 0) return failed(GuestNoMemory); + return mapped; + } catch (const std::bad_alloc&) { + return failed(GuestNoMemory); + } catch (const std::exception&) { + return failed(GuestNoMemory); + } +} + +int APS5_VABI munmap_nid_postfix(void* address, std::size_t length) noexcept { + const auto failed = [](int error) { + SetError(error); + return -1; + }; + std::size_t rounded; + const auto start = reinterpret_cast(address); + if (!RoundLength(length, rounded) || start == 0 || + (start & (PS5_PAGE_SIZE - 1)) != 0 || + rounded > std::numeric_limits::max() - start) + return failed(GuestInvalid); + try { + if (DoMunmap(address, rounded) != 0) return failed(GuestInvalid); + return 0; + } catch (const std::bad_alloc&) { + return failed(GuestNoMemory); + } catch (const std::exception&) { + // The allocation tracker rejects foreign, pinned, and noncontiguous ranges. + return failed(GuestInvalid); + } +} + +} diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp new file mode 100644 index 000000000..e0e906e3d --- /dev/null +++ b/core/libs/tests/GuestMemory.cpp @@ -0,0 +1,70 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include +#include +#include +#include + +extern "C" { +void* APS5_VABI mmap_nid_postfix(void*, std::size_t, int, int, int, std::int64_t) noexcept; +int APS5_VABI munmap_nid_postfix(void*, std::size_t) noexcept; +int* APS5_VABI __error_nid_postfix(); +} + +static void Require(bool condition) { + if (!condition) { + std::fputs("Guest memory check failed\n", stderr); + std::abort(); + } +} + +int main() { + constexpr std::size_t page = 0x4000; + const auto failed = reinterpret_cast(static_cast(-1)); + const auto reject = [&](std::size_t length, int protection, int flags, int fd, + std::int64_t offset, int error) { + *__error_nid_postfix() = 0; + Require(mmap_nid_postfix(nullptr, length, protection, flags, fd, offset) == failed); + Require(*__error_nid_postfix() == error); + }; + reject(0, 3, 0x1002, -1, 0, 22); + reject(std::numeric_limits::max(), 3, 0x1002, -1, 0, 22); + reject(page, 8, 0x1002, -1, 0, 22); + reject(page, 3, 0x1002, 0, 0, 22); + reject(page, 3, 0x1002, -1, 1, 22); + reject(page, 3, 0x1001, -1, 0, 45); // shared + reject(page, 3, 0x1012, -1, 0, 45); // fixed + reject(page, 3, 0x2, 0, 0, 45); // file-backed + reject(page, 3, 0x22, -1, 0, 45); // Linux MAP_ANON is not guest MAP_ANON + + auto* memory = static_cast(mmap_nid_postfix(nullptr, page * 3 - 1, 3, 0x1002, -1, 0)); + Require(memory != failed && (reinterpret_cast(memory) & (page - 1)) == 0); + for (std::size_t i = 0; i < page * 3; ++i) Require(memory[i] == 0); + memory[0] = 42; + memory[page * 2] = 73; + { + GuestAllocations::Mutation mutation; + const auto range = mutation.Find(memory); + Require(range.bytes == page * 3 && range.readable && range.writable); + } + Require(munmap_nid_postfix(memory + 1, page) == -1 && *__error_nid_postfix() == 22); + Require(munmap_nid_postfix(memory, 0) == -1 && *__error_nid_postfix() == 22); + Require(memory[0] == 42); + Require(munmap_nid_postfix(memory + page, 1) == 0); // round to one guest page + Require(memory[0] == 42 && memory[page * 2] == 73); + Require(munmap_nid_postfix(memory, page) == 0); + Require(memory[page * 2] == 73); + Require(munmap_nid_postfix(memory + page * 2, page) == 0); + Require(munmap_nid_postfix(memory, page) == -1); + for (int protection : {0, 1, 3, 5}) { + void* mapped = mmap_nid_postfix(memory, 1, protection, 0x1002, -1, 0); + Require(mapped != failed); + { + GuestAllocations::Mutation mutation; + const auto range = mutation.Find(mapped); + Require(range.readable == ((protection & 3) != 0)); + Require(range.writable == ((protection & 2) != 0)); + } + Require(munmap_nid_postfix(mapped, 1) == 0); + } +} diff --git a/docs/YAMAGI_QUAKE2.md b/docs/YAMAGI_QUAKE2.md index 0c65715fd..2b7f4bcde 100644 --- a/docs/YAMAGI_QUAKE2.md +++ b/docs/YAMAGI_QUAKE2.md @@ -100,3 +100,20 @@ calling convention, bounded unterminated input, concatenation truncation, interleaved tokenizer state, and case-insensitive searches. Windows dependency tests also cover implicit libc visibility, ordering, duplicate avoidance, and unrelated inputs. The complete `libs` target rebuild succeeds. + +## Follow-up: anonymous memory mapping + +The game's `tooling/native/app_heap.c` requests private anonymous read/write +memory for its OpenGL heap. `mmap` and `munmap` now route this use through +AnyPS5's existing tracked guest allocations, with 16 KiB page rounding and +partial unmapping. Guest MAP_ANON is 0x1000, not the Linux host value 0x20. +Failures return MAP_FAILED or -1 and set guest errno through `__error`. +Shared, fixed-address, and file-backed mappings remain explicitly unsupported +(guest EOPNOTSUPP 45). Unmapping currently requires a contiguous tracked range +within one allocation; holes, foreign memory, and pinned ranges are rejected. + +Six CTest suites pass, including real allocations, zero-fill, alignment, +read/write access, tracking, partial unmapping, protection metadata, and invalid +argument/unsupported-mode errors. The unchanged converted game now resolves +the memory-mapping imports and stops at `-hn1tcVHq5Q` (`sceLibcMspaceCreate`). +It still has not reached its entry point or drawn a frame.