Merge pull request #1857 from hikazey/feat/msync

feat(libkernel): implement msync

core/libs/prx/libkernel/CMakeLists.txt

@@ -571,6 +571,16 @@ if(BUILD_TESTING)
         set_tests_properties(guest_mlock PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
     endif()
+    add_test_executable(guest_msync_tests ${LIBS_INCLUDE_DIR}/tests/GuestMsync.cpp)
+    target_include_directories(guest_msync_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_msync_tests PRIVATE libkernel libc)
+    configure_windows_unwind(guest_msync_tests)
+    add_test(NAME guest_msync COMMAND guest_msync_tests)
+    set_tests_properties(guest_msync PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_msync PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
+    endif()
     if(WIN32)
         add_test_executable(guest_directory_buffer_tests ${LIBS_INCLUDE_DIR}/tests/GuestDirectoryBuffer.cpp)
         target_include_directories(guest_directory_buffer_tests PRIVATE ${LIBS_INCLUDE_DIR})

core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp

@@ -1,4 +1,5 @@
 #include "DirectMemory.hpp"
+#include "SceTypes.hpp"
 #include "prx/libc/include/general/VabiMacros.hpp"
 #include <limits>
 #include <new>
@@ -7,6 +8,7 @@
 extern "C" int* APS5_VABI __error_nid_postfix();
 extern "C" int APS5_VABI sceKernelMlock_nid_postfix(void* address, std::uint64_t length);
 extern "C" int APS5_VABI sceKernelMunlock_nid_postfix(void* address, std::uint64_t length);
+extern "C" int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info, uint64_t info_size);
 
 namespace {
 // FreeBSD/PS5 ABI values, independent of the host's errno and mmap constants.
@@ -15,6 +17,8 @@ constexpr int GuestNoMemory = 12;
 constexpr int GuestNotSupported = 45;
 constexpr int GuestPrivate = 0x2;
 constexpr int GuestAnonymous = 0x1000;
+constexpr int GuestSyncAsync = 0x1;
+constexpr int GuestSyncInvalidate = 0x2;
 
 bool RoundLength(std::size_t length, std::size_t& rounded) {
     constexpr auto mask = PS5_PAGE_SIZE - 1;
@@ -27,6 +31,21 @@ bool RoundLength(std::size_t length, std::size_t& rounded) {
 void SetError(int error) {
     *__error_nid_postfix() = error;
 }
+
+bool Mapped(std::uintptr_t address) {
+    VirtualQueryInfo info{};
+    return sceKernelVirtualQuery(reinterpret_cast<const void*>(address), 0, &info, sizeof(info)) == 0;
+}
+
+bool RangeMapped(std::uintptr_t start, std::uintptr_t end) {
+    for (auto cursor = start; cursor < end;) {
+        VirtualQueryInfo info{};
+        if (sceKernelVirtualQuery(reinterpret_cast<const void*>(cursor), 0, &info, sizeof(info)) != 0) return false;
+        if (info.end <= cursor) return false;
+        cursor = info.end;
+    }
+    return true;
+}
 }
 
 extern "C" {
@@ -113,4 +132,21 @@ int APS5_VABI munlock_nid_postfix(const void* address, std::size_t length) {
     return -1;
 }
 
+int APS5_VABI msync_nid_postfix(void* address, std::size_t length, int flags) {
+    const auto failed = [](int error) {
+        SetError(error);
+        return -1;
+    };
+    constexpr std::uintptr_t mask = PS5_PAGE_SIZE - 1;
+    const auto first = reinterpret_cast<std::uintptr_t>(address);
+    const auto start = first & ~mask;
+    const std::uintptr_t size = (length + (first - start) + mask) & ~mask;
+    const auto end = start + size;
+    if (end < start) return failed(GuestInvalid);
+    if ((flags & (GuestSyncAsync | GuestSyncInvalidate)) == (GuestSyncAsync | GuestSyncInvalidate)) return failed(GuestInvalid);
+    if (start == end ? !Mapped(start) : !RangeMapped(start, end)) return failed(GuestNoMemory);
+    if ((flags & GuestSyncInvalidate) != 0) throw std::runtime_error("msync: MS_INVALIDATE is not implemented");
+    return 0;
+}
+
 }

core/libs/tests/GuestMsync.cpp

@@ -0,0 +1,85 @@
+#include "prx/libc/include/general/VabiMacros.hpp"
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <limits>
+#include <source_location>
+#include <stdexcept>
+
+extern "C" {
+void* APS5_VABI mmap_nid_postfix(void*, std::size_t, int, int, int, std::int64_t) noexcept;
+int APS5_VABI munmap_nid_postfix(void*, std::size_t) noexcept;
+int APS5_VABI msync_nid_postfix(void*, std::size_t, int);
+int APS5_VABI sceKernelReserveVirtualRange(void**, std::size_t, int, std::size_t);
+int* APS5_VABI __error_nid_postfix();
+}
+
+namespace {
+constexpr std::size_t Page = 0x4000;
+constexpr int ReadWrite = 3;
+constexpr int PrivateAnonymous = 0x1002;
+constexpr int Sync = 0x0;
+constexpr int Async = 0x1;
+constexpr int Invalidate = 0x2;
+constexpr int Invalid = 22;
+constexpr int NoMemory = 12;
+
+void Require(bool condition, std::source_location location = std::source_location::current()) {
+    if (condition) return;
+    std::fprintf(stderr, "msync check failed at line %u\n", static_cast<unsigned>(location.line()));
+    std::abort();
+}
+
+bool Succeeds(void* address, std::size_t length, int flags) {
+    *__error_nid_postfix() = 77;
+    return msync_nid_postfix(address, length, flags) == 0 && *__error_nid_postfix() == 77;
+}
+
+bool Fails(void* address, std::size_t length, int flags, int error) {
+    *__error_nid_postfix() = 0;
+    return msync_nid_postfix(address, length, flags) == -1 && *__error_nid_postfix() == error;
+}
+
+bool Throws(void* address, std::size_t length, int flags) {
+    try {
+        msync_nid_postfix(address, length, flags);
+    } catch (const std::runtime_error&) {
+        return true;
+    }
+    return false;
+}
+
+char* Map(std::size_t pages) {
+    void* mapped = mmap_nid_postfix(nullptr, pages * Page, ReadWrite, PrivateAnonymous, -1, 0);
+    Require(mapped != reinterpret_cast<void*>(static_cast<std::uintptr_t>(-1)));
+    return static_cast<char*>(mapped);
+}
+}
+
+int main() {
+    char* mapped = Map(3);
+    mapped[0] = 1;
+    Require(Succeeds(mapped, 3 * Page, Sync));
+    Require(Succeeds(mapped, 3 * Page, Async));
+    Require(Succeeds(mapped + 100, 10, Sync));
+    Require(Succeeds(mapped + Page - 1, 2, Sync));
+    Require(Succeeds(mapped + Page, 0, Sync));
+    Require(Succeeds(mapped, Page, 0x4));
+
+    Require(Fails(mapped, Page, Async | Invalidate, Invalid));
+    const auto top = std::numeric_limits<std::uintptr_t>::max() & ~(Page - 1);
+    Require(Fails(reinterpret_cast<void*>(top), 2 * Page, Sync, Invalid));
+    Require(Throws(mapped, Page, Invalidate));
+
+    Require(munmap_nid_postfix(mapped, 3 * Page) == 0);
+    Require(Fails(mapped, Page, Sync, NoMemory));
+    Require(Fails(mapped + Page, 3 * Page, Async, NoMemory));
+    Require(Fails(mapped + Page, 0, Sync, NoMemory));
+    Require(Fails(mapped, Page, Invalidate, NoMemory));
+
+    void* reserved = nullptr;
+    Require(sceKernelReserveVirtualRange(&reserved, 2 * Page, 0, 0) == 0);
+    Require(Succeeds(reserved, 2 * Page, Sync));
+    Require(Succeeds(reserved, 0, Async));
+    return 0;
+}

docs/dev/TechnicalDebt.md

@@ -361,6 +361,7 @@ Throughout the project, every function at every stage either **does exactly what
 - [collate<char>::_Getcat](../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) builds the facet for the C locale only and throws for any other locale name. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size, and for results of 4095 characters or more, whose string storage uses the guest's over-aligned large allocation.
 - [socketpair](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) - the pair is the host's `AF_UNIX` pair on Linux and, on Windows, a loopback TCP connection for `SOCK_STREAM` or two connected UDP sockets for `SOCK_DGRAM`; `SOCK_SEQPACKET` throws on Windows. A datagram sent to a full receiver gets `ENOBUFS` on the console, waits or gets `EAGAIN` on Linux and is dropped on Windows, and on Windows a datagram `recv` after the peer closed reports `ECONNRESET` instead of the console's `ENOTCONN`, and a stream `send` after the peer closed reports `ECONNABORTED` instead of the console's `EPIPE`. `read` and `write` on a pair end go to the host like on every guest socket. `getpeername` on a pair end succeeds after the peer closed, where the console returns `ENOTCONN`, and `connect`, `bind` and `sendto` with an address on a pair end report an address-family error instead of the console's `EISCONN`. On Windows `poll` on a stream end reports `POLLHUP` without `POLLIN` after the peer's `shutdown(SHUT_WR)` or `close`, where the console reports `POLLIN`, with `POLLHUP` only after `close`
 - [poll](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) waits only on guest sockets, through `WSAPoll` on Windows and `poll` on Linux; the `pollfd` layout and flag values are FreeBSD's. A descriptor below the socket range (files, pipes, stdio) throws, as do event bits other than `POLLIN`, `POLLPRI`, `POLLOUT`, `POLLRDNORM`, `POLLRDBAND` and `POLLWRBAND` (the `POLLERR`, `POLLHUP` and `POLLNVAL` bits are ignored in `events`), `POLLPRI` on Windows, which `WSAPoll` rejects, and an infinite timeout with no socket to wait on. A closed socket descriptor reports `POLLNVAL`; the console's limit on the descriptor count is unknown and not checked
+- [msync](../../core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp) (libkernel) follows FreeBSD 11's `kern_msync` and `vm_map_sync`, not measured on the console: the address is rounded down to the page, a range that wraps is `EINVAL`, as is `MS_ASYNC` with `MS_INVALIDATE`, and a range whose start or any page is not mapped (as `sceKernelVirtualQuery` reports it, reservations included) is `ENOMEM`; a page unmapped from inside a larger mapping still counts as mapped there, since `sceKernelVirtualQuery` keeps reporting the whole mapping. The guest has no file mappings, so a successful call writes nothing back. `MS_INVALIDATE` on a mapped range throws: FreeBSD returns `EBUSY` when the range holds `mlock`ed pages, and locked pages are not tracked
 - [Thread exit destructors](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) run when a libkernel thread finishes; on the main or a host thread only `_sceLibcInternalThreadDtors` runs them, as `exit` does, else they are dropped
 - [sceFontOpenFontSet](../../core/libs/prx/libSceFont/src/SystemFont.cpp) (libSceFont) opens only the primary file of a system font set. Characters the console takes from the set's other files (Hangul, Thai and Arabic in Japanese and Chinese sets, Arabic in European sets) are reported as unsupported glyphs. The console file names are those of the PS4 firmware; the names of the PS5-only JG2 sets (`0x1A......`) are unknown, so they use the SSTJpPro files. When a console file is missing, an openly licensed substitute (Noto Sans, Noto Sans Mono, Noto Sans Thai, Noto Sans CJK) is loaded instead. It is not the system font: its metrics differ, so text width, line height and wrapping differ from the console. System font faces are loaded by FreeType in host memory, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for.
 - [Font writing lines](../../core/libs/prx/libSceFont/src/WritingLine.cpp) (libSceFont) - `sceFontCreateWritingLine`, `sceFontWritingLineWritesOrder`, `sceFontWritingLineRefersRenderStep`, `sceFontWritingLineGetRenderMetrics` and `sceFontWritingLineClear` follow [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/main/src/libs/libFont.cpp): runs are placed left to right, each at the line's advance so far, with no spacing or adjustment. Only horizontal and left-to-right lines, writing attribute 0 and non-null run metrics are modelled; other values, the metrics of an empty line and unknown or null handles throw, since no reference gives their result or error code. `sceFontWritingLineGetOrderingSpace` still throws (KytyPS5's values are placeholders). Lines are allocated on the host, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for.