fix(libkernel): keep sceKernelDebugOutText off a redirected guest stderr

Guest descriptors are host descriptors, so a guest that moves standard
error with dup2 also moves the stream sceKernelDebugOutText writes to. A
title that forwards its standard error to the kernel log through a pipe
then reads its own debug output back and logs it again without end. Before
the guest's first dup2 onto descriptor 2, the host's standard error is kept,
and sceKernelDebugOutText writes there.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/libs/prx/libkernel/File/include/File.hpp

@@ -25,4 +25,8 @@ std::int64_t NativePositioned_nid_no_patch(int descriptor, void* buf, std::size_
 
 bool DescriptorIsOpen_nid_no_patch(int descriptor);
 
+namespace File {
+int HostStandardError();
+}
+
 #endif

core/libs/prx/libkernel/File/src/Stdio.cpp

@@ -18,6 +18,7 @@
 #include "prx/libkernel/KernelErrors.hpp"
 #include "prx/libkernel/Socket/include/SocketRuntime.hpp"
 #include "prx/libkernel/Socket/include/SocketPoll.hpp"
+#include <atomic>
 #include <cerrno>
 #include <cstring>
 #include <cstdarg>
@@ -513,6 +514,36 @@ int APS5_VABI _close_nid_postfix(int descriptor) {
     return close_nid_postfix(descriptor);
 }
 
+}
+
+namespace {
+std::atomic<int> g_hostStandardError{-1};
+
+void PreserveHostStandardError() {
+    if (g_hostStandardError.load() >= 0) return;
+#ifdef _WIN32
+    const int saved = _dup(2);
+#else
+    const int saved = ::dup(2);
+#endif
+    if (saved < 0) throw std::runtime_error("dup2: cannot keep the host standard error");
+    int expected = -1;
+    if (!g_hostStandardError.compare_exchange_strong(expected, saved)) {
+#ifdef _WIN32
+        _close(saved);
+#else
+        ::close(saved);
+#endif
+    }
+}
+}
+
+int File::HostStandardError() {
+    return g_hostStandardError.load();
+}
+
+extern "C" {
+
 int APS5_VABI dup_nid_postfix(int d) {
     if (d >= GuestSockets::FirstDescriptor) return GuestSockets::Duplicate(d);
     if (d < 0) return PosixFailure(GUEST_EBADF);
@@ -546,6 +577,7 @@ int APS5_VABI dup2_nid_postfix(int from, int to) {
     if (WithoutParameterHandler([from] { return ::_get_osfhandle(from) == -1 ? -1 : 0; }) != 0) return PosixFailure(GUEST_EBADF);
     if (from == to) return to;
     RejectDirectoryDuplicate(from, __func__);
+    if (to == 2) PreserveHostStandardError();
     if (WithoutParameterHandler([from, to] { return ::_dup2(from, to); }) != 0) return PosixFailure(SceErrorFromErrno(errno) & 0xffff);
     File::ForgetDirectoryDescriptor(to);
     File::ForgetFileLock(to);
@@ -558,6 +590,7 @@ int APS5_VABI dup2_nid_postfix(int from, int to) {
         if (::fcntl(from, F_GETFL) < 0) return PosixFailure(GUEST_EBADF);
         return to;
     }
+    if (to == 2) PreserveHostStandardError();
     const int result = ::dup2(from, to);
     if (result < 0) return PosixFailure(SceErrorFromErrno(errno) & 0xffff);
     File::ForgetRandomDevice(to);

core/libs/prx/libkernel/System/src/Notification.cpp

@@ -1,4 +1,6 @@
 #include "prx/libc/include/general/VabiMacros.hpp"
+#include "prx/libkernel/File/include/File.hpp"
+#include <string>
 #include <array>
 #include <cerrno>
 #include <cstddef>
@@ -8,6 +10,7 @@
 #include <mutex>
 #ifdef _WIN32
 #include <windows.h>
+#include <io.h>
 #else
 #include <unistd.h>
 #endif
@@ -17,6 +20,15 @@ extern "C" int APS5_VABI sceKernelDebugOutText(int channel, const char* text) {
     if (!text) return static_cast<int>(0x8002000eu);
     static std::mutex outputMutex;
     const std::lock_guard lock(outputMutex);
+    if (const int host = File::HostStandardError(); host >= 0) {
+        const auto line = "[debug:" + std::to_string(channel) + "] " + text;
+#ifdef _WIN32
+        const auto written = _write(host, line.data(), static_cast<unsigned>(line.size()));
+#else
+        const auto written = ::write(host, line.data(), line.size());
+#endif
+        return written == static_cast<decltype(written)>(line.size()) ? 0 : static_cast<int>(0x80020005u);
+    }
     if (std::fprintf(stderr, "[debug:%d] %s", channel, text) < 0 || std::fflush(stderr) != 0)
         return static_cast<int>(0x80020005u);
     return 0;

core/libs/tests/GuestFilesystem.cpp

@@ -64,6 +64,7 @@ FileStream* APS5_VABI fopen_nid_postfix(const char*, const char*);
 FileStream* APS5_VABI freopen_nid_postfix(const char*, const char*, FileStream*);
 int* APS5_VABI __error_nid_postfix();
 int APS5_VABI pipe_nid_postfix(int*);
+int APS5_VABI dup2_nid_postfix(int, int);
 std::int64_t APS5_VABI read_nid_postfix(int, void*, std::size_t);
 std::int64_t APS5_VABI write_nid_postfix(int, const void*, std::size_t);
 int APS5_VABI sceKernelDebugOutText(int, const char*);
@@ -223,6 +224,29 @@ int main() {
         Require(std::filesystem::remove("creat_probe.bin"));
     }
     Require(close_nid_postfix(descriptors[0]) == 0);
+    auto* hostCapture = std::tmpfile();
+    Require(hostCapture != nullptr && pipe_nid_postfix(descriptors) == 0);
+#ifdef _WIN32
+    const int hostError = ::_dup(2);
+    Require(hostError >= 0 && ::_dup2(::_fileno(hostCapture), 2) == 0);
+#else
+    const int hostError = ::dup(2);
+    Require(hostError >= 0 && ::dup2(::fileno(hostCapture), 2) == 2);
+#endif
+    Require(dup2_nid_postfix(descriptors[1], 2) == 2);
+    Require(sceKernelDebugOutText(1, "kept\n") == 0);
+    Require(write_nid_postfix(2, "P", 1) == 1);
+#ifdef _WIN32
+    Require(::_dup2(hostError, 2) == 0 && ::_close(hostError) == 0);
+#else
+    Require(::dup2(hostError, 2) == 2 && ::close(hostError) == 0);
+#endif
+    Require(close_nid_postfix(descriptors[1]) == 0);
+    Require(read_nid_postfix(descriptors[0], received, sizeof(received)) == 1 && received[0] == 'P');
+    Require(close_nid_postfix(descriptors[0]) == 0);
+    std::rewind(hostCapture);
+    Require(std::fread(debugText, 1, sizeof(debugText), hostCapture) == 15 && std::memcmp(debugText, "[debug:1] kept\n", 15) == 0);
+    Require(std::fclose(hostCapture) == 0);
     const auto root = std::filesystem::path("anyps5-filesystem-test-" +
         std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()));
     Require(mkdir_nid_postfix(root.string().c_str(), 0700) == 0);