Merge pull request #3821 from semirkabir/feat/coredump-attach-outside-handler

feat(libkernel): answer the coredump attach calls made outside the handler

core/libs/prx/libkernel/CMakeLists.txt

@@ -1193,4 +1193,14 @@ if(BUILD_TESTING)
         set_tests_properties(guest_pthread_rename PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
     endif()
+    add_test_executable(guest_coredump_attach_tests ${LIBS_INCLUDE_DIR}/tests/GuestCoredumpAttach.cpp)
+    target_include_directories(guest_coredump_attach_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_coredump_attach_tests PRIVATE libkernel libc)
+    configure_windows_unwind(guest_coredump_attach_tests)
+    add_test(NAME guest_coredump_attach COMMAND guest_coredump_attach_tests)
+    set_tests_properties(guest_coredump_attach PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_coredump_attach PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
+    endif()
 endif()

core/libs/prx/libkernel/System/src/Coredump.cpp

@@ -2,6 +2,8 @@
 #include <cstddef>
 #include <atomic>
 #include <cstdio>
+#include <stdexcept>
+#include <string>
 #include "SceTypes.hpp"
 #include "prx/libc/include/General.hpp"
 
@@ -9,6 +11,14 @@
 static std::atomic<uint64_t> g_coredumpHandler{0};
 static std::atomic<uint64_t> g_coredumpContext{0};
 
+static constexpr int SCE_COREDUMP_ERROR_NOT_IN_COREDUMP_HANDLER = static_cast<int>(0x81180003u);
+
+static int attachOutsideCoredumpHandler(const char* function, const void* mem, size_t size) {
+    if (mem == nullptr || size == 0 || size > UINTPTR_MAX - reinterpret_cast<uintptr_t>(mem))
+        throw std::runtime_error(std::string(function) + ": null, empty or wrapping region outside the core dump handler");
+    return SCE_COREDUMP_ERROR_NOT_IN_COREDUMP_HANDLER;
+}
+
 extern "C" {
 
 int APS5_VABI sceCoredumpRegisterCoredumpHandler(uint64_t handler, size_t stack_size, uint64_t context) {
@@ -73,13 +83,13 @@ int APS5_VABI sceCoredumpWriteUserString() {
     return 0;
 }
 
-int APS5_VABI sceCoredumpAttachUserMemoryFile(void) {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+int APS5_VABI sceCoredumpAttachUserMemoryFile(uint32_t userValue, const void* mem, size_t size) {
+    (void)userValue;
+    return attachOutsideCoredumpHandler(__func__, mem, size);
 }
 
-int APS5_VABI sceCoredumpAttachMemoryRegion(void) {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+int APS5_VABI sceCoredumpAttachMemoryRegion(uint32_t userValue, const void* mem, size_t size) {
+    (void)userValue;
+    return attachOutsideCoredumpHandler(__func__, mem, size);
 }
 }

core/libs/tests/GuestCoredumpAttach.cpp

@@ -0,0 +1,67 @@
+#include "prx/libc/include/general/VabiMacros.hpp"
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <stdexcept>
+#include <thread>
+
+extern "C" {
+int APS5_VABI sceCoredumpRegisterCoredumpHandler(uint64_t handler, size_t stackSize, uint64_t context);
+int APS5_VABI sceCoredumpUnregisterCoredumpHandler(void);
+int APS5_VABI sceCoredumpAttachMemoryRegion(uint32_t userValue, const void* mem, size_t size);
+int APS5_VABI sceCoredumpAttachUserMemoryFile(uint32_t userValue, const void* mem, size_t size);
+}
+
+namespace {
+constexpr int NotInCoredumpHandler = static_cast<int>(0x81180003u);
+constexpr uint32_t UserFileTruncateIfExceedLimit = 0x80000000u;
+bool handlerRan = false;
+int handlerContext = 0;
+
+int APS5_VABI Handler(void*) {
+    handlerRan = true;
+    return 0;
+}
+
+void Require(bool value) { if (!value) std::abort(); }
+
+template <typename TAttach>
+bool Throws(TAttach attach, const char* function) {
+    try { attach(); }
+    catch (const std::runtime_error& error) { return std::strstr(error.what(), function) != nullptr; }
+    return false;
+}
+}
+
+int main() {
+    std::array<unsigned char, 64> region{};
+    for (size_t index = 0; index < region.size(); ++index) region[index] = static_cast<unsigned char>(index * 7 + 3);
+    const auto original = region;
+
+    Require(sceCoredumpAttachMemoryRegion(1, region.data(), region.size()) == NotInCoredumpHandler);
+    Require(sceCoredumpAttachUserMemoryFile(UserFileTruncateIfExceedLimit | 2, region.data(), region.size()) == NotInCoredumpHandler);
+
+    Require(sceCoredumpRegisterCoredumpHandler(reinterpret_cast<uint64_t>(&Handler), 16 * 1024, reinterpret_cast<uint64_t>(&handlerContext)) == 0);
+    Require(sceCoredumpAttachMemoryRegion(0, region.data(), 1) == NotInCoredumpHandler);
+    Require(sceCoredumpAttachUserMemoryFile(0xFFFFFFFFu, region.data(), region.size()) == NotInCoredumpHandler);
+    int fromThread = 0;
+    std::thread([&] { fromThread = sceCoredumpAttachMemoryRegion(3, region.data(), region.size()); }).join();
+    Require(fromThread == NotInCoredumpHandler);
+    Require(!handlerRan);
+
+    Require(Throws([&] { sceCoredumpAttachMemoryRegion(1, nullptr, region.size()); }, "sceCoredumpAttachMemoryRegion"));
+    Require(Throws([&] { sceCoredumpAttachMemoryRegion(1, region.data(), 0); }, "sceCoredumpAttachMemoryRegion"));
+    Require(Throws([&] { sceCoredumpAttachMemoryRegion(1, region.data(), SIZE_MAX); }, "sceCoredumpAttachMemoryRegion"));
+    Require(Throws([&] { sceCoredumpAttachUserMemoryFile(1, nullptr, region.size()); }, "sceCoredumpAttachUserMemoryFile"));
+    Require(Throws([&] { sceCoredumpAttachUserMemoryFile(1, region.data(), 0); }, "sceCoredumpAttachUserMemoryFile"));
+    Require(Throws([&] { sceCoredumpAttachUserMemoryFile(1, region.data(), SIZE_MAX); }, "sceCoredumpAttachUserMemoryFile"));
+
+    Require(sceCoredumpUnregisterCoredumpHandler() == 0);
+    Require(sceCoredumpAttachMemoryRegion(1, region.data(), region.size()) == NotInCoredumpHandler);
+    Require(sceCoredumpAttachUserMemoryFile(1, region.data(), region.size()) == NotInCoredumpHandler);
+    Require(region == original);
+    Require(!handlerRan);
+    return 0;
+}

docs/dev/debt/libkernel.md

@@ -63,8 +63,7 @@
 - [pthread_barrierattr_setpshared](../../../core/libs/prx/libkernel/Pthread/Posix/Barrier.cpp) (libkernel) - PTHREAD_PROCESS_SHARED is accepted on the assumption that the console follows FreeBSD 11.0 libthr, which accepts it (9.0 rejects it with EINVAL); unverified on hardware. Guest code runs in one process, so a shared barrier behaves as a private one
 - [pthread_getname_np](../../../core/libs/prx/libkernel/Pthread/Posix/Thread.cpp) (libkernel) - a thread that has already been joined returns `ESRCH` (3) on a PS5 (firmware 13.60), but its object is freed after the join here, so the pointer cannot be told apart from a live thread and its name is read as for a live one
 - [sceKernelGetOperationMode](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - the signature `(int* mode, int* submode)` comes from PPSA12544, which logs both values after the call and ignores the result. A homebrew payload on a PS5 (firmware 13.60) reads mode 2 and submode 0, nothing else is written, and the result register holds no status, so the function is taken to return nothing. Whether a title process reads the same values is unverified. Null outputs throw
-- [sceCoredumpAttachUserMemoryFile](../../../core/libs/prx/libkernel/System/src/Coredump.cpp) (libkernel) - unknown signature
-- [sceCoredumpAttachMemoryRegion](../../../core/libs/prx/libkernel/System/src/Coredump.cpp) (libkernel) - unknown signature
+- [sceCoredumpAttachMemoryRegion, sceCoredumpAttachUserMemoryFile](../../../core/libs/prx/libkernel/System/src/Coredump.cpp) (libkernel) - both take `(uint32_t userValue, const void* mem, size_t size)`: for the first, the IDA type database of [Orbital](https://github.com/AlexAltea/orbital/blob/master/tools/ida/db_types.json) and the [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Kernel/KernelSystem.cs) bindings agree; the second has only SharpProspero (the PS4 5.05 firmware does not export it, so Orbital has no entry) and is described there as attaching the region as an unnamed user file. Not confirmed on a title; the RED engine's PS4 crash handler ([Vangaurd](https://github.com/vurimad/Vangaurd/blob/main/source/imported/common/redSystem/src/errorHandlerImplErrorHooksOrbis.cpp)) calls the sibling `sceCoredumpAttachMemoryRegionAsUserFile(999, buffer, bytes, name)` from inside its handler. Attaching belongs inside the core dump handler, which the system runs on its own thread after the process stops (SharpProspero's remark, the RED engine's usage and `SCE_COREDUMP_ERROR_NOT_IN_COREDUMP_HANDLER`, "function was not called from registered core dump handler", in the Vita's SceCoredump error table, [RivetDB](https://github.com/CreepNT/RivetDB/blob/master/rivetdb.json)). The handler is never invoked here, so every call is outside it and returns that error; nothing is recorded, as there is no dump to attach to. Its value `0x81180003` comes from shadPS4 (PS4), not confirmed on a PS5: [fpPS4](https://github.com/red-prig/fpPS4/blob/master/kernel/ps4_libkernel.pas) has `NOT_REGISTERED` 0x81180001 and `ALREADY_REGISTERED` 0x81180002, shadPS4's `coredump_error.h` has `NOT_IN_COREDUMP_HANDLER` = 0x81180003, and the Vita table numbers `PARAM` 0, `NOT_REGISTERED` 1, `ALREADY_REGISTERED` 2 and `NOT_IN_COREDUMP_HANDLER` 3. Whether the console checks the region before the calling context is unknown, so a null, empty or wrapping region throws. `userValue` is not interpreted; SharpProspero names its bits 31 and 30 as truncation flags for the user file variants
 - [sceKernelAddTimerEvent](../../../core/libs/prx/libkernel/Equeue/Equeue.cpp) (libkernel) - `usec` is taken as the period in microseconds; re-adding an existing timer keeps its pending expiration count and restarts the period from the time of the call, where FreeBSD 12+ `filt_timertouch` clears pending expirations on `EV_ADD`; `usec == 0` fires on every wait. None of this is confirmed on a PS5 title
 - [sceKernelWaitEqueue](../../../core/libs/prx/libkernel/Equeue/Equeue.cpp) (libkernel) - a level-triggered user event (`sceKernelAddUserEvent`) is reported at most once per call, as in FreeBSD `kqueue_scan` (`sys/kern/kern_event.c`), which re-queues a reported knote without `EV_CLEAR` or `EV_DISPATCH` behind the scan marker, and kevent(2), where several triggers do not produce several kevents. Not measured on a console. Events are scanned in list order and the list is not rotated, so when `num` is smaller than the number of ready level events the later ones can starve, where FreeBSD moves a reported level knote to the tail. Several completions of one AMPR or VideoOut event within one call are returned as separate entries, where FreeBSD would aggregate them; the Orbis behaviour is unconfirmed
 - [AC1FtjqMCL0](../../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature