Merge pull request #1435 from ewraj/fix/linux-guest-varargs

fix(libc): read every variadic argument of the Linux printf and scanf exports

core/libs/SceTypes.hpp

@@ -1848,12 +1848,6 @@ struct LibcHeapInfo {
 
 using Info = LibcHeapInfo;
 
-#define VA_ARGS \
-    std::uint64_t rdi, std::uint64_t rsi, std::uint64_t rdx, std::uint64_t rcx, \
-    std::uint64_t r8, std::uint64_t r9, std::uint64_t overflow_arg_area, \
-    __m128 xmm0, __m128 xmm1, __m128 xmm2, __m128 xmm3, \
-    __m128 xmm4, __m128 xmm5, __m128 xmm6, __m128 xmm7, ...
-
 struct Packet {
     std::uint32_t* addr;
     std::uint32_t dw_num;

core/libs/prx/libc/CMakeLists.txt

@@ -246,8 +246,8 @@ if(BUILD_TESTING)
     target_include_directories(guest_formatting_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(guest_formatting_tests PRIVATE libc)
     configure_windows_unwind(guest_formatting_tests)
+    add_test(NAME guest_formatting COMMAND guest_formatting_tests)
     if(WIN32)
-        add_test(NAME guest_formatting COMMAND guest_formatting_tests)
         set_tests_properties(guest_formatting PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()

core/libs/prx/libc/include/VarArgsAbi.hpp

@@ -1,59 +0,0 @@
-#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_EXCEPTIONS_VARARGSABI_HPP
-#define CORE_LIBS_PRX_LIBC_INCLUDE_EXCEPTIONS_VARARGSABI_HPP
-
-#include <cstdarg>
-#include <cstdint>
-#include "SceTypes.hpp"
-#include "prx/libc/include/specifics/x86_64/SimdTypes.hpp"
-
-namespace LibcDetail {
-
-struct RegSaveArea {
-    std::uint64_t gp[6];
-    X86_64::Xmm fp[8];
-};
-
-struct VaListLayout {
-    unsigned int gpOffset;
-    unsigned int fpOffset;
-    void* overflowArgArea;
-    void* regSaveArea;
-};
-
-inline void FillRegSaveArea(
-    RegSaveArea& regs,
-    std::uint64_t gp0, std::uint64_t gp1, std::uint64_t gp2,
-    std::uint64_t gp3, std::uint64_t gp4, std::uint64_t gp5,
-    X86_64::Xmm fp0, X86_64::Xmm fp1, X86_64::Xmm fp2, X86_64::Xmm fp3,
-    X86_64::Xmm fp4, X86_64::Xmm fp5, X86_64::Xmm fp6, X86_64::Xmm fp7
-) {
-    regs.gp[0] = gp0;
-    regs.gp[1] = gp1;
-    regs.gp[2] = gp2;
-    regs.gp[3] = gp3;
-    regs.gp[4] = gp4;
-    regs.gp[5] = gp5;
-    regs.fp[0] = fp0;
-    regs.fp[1] = fp1;
-    regs.fp[2] = fp2;
-    regs.fp[3] = fp3;
-    regs.fp[4] = fp4;
-    regs.fp[5] = fp5;
-    regs.fp[6] = fp6;
-    regs.fp[7] = fp7;
-}
-
-inline std::va_list* BuildVaList(
-    VaListLayout& layout, RegSaveArea& regs,
-    unsigned int consumedGpRegisters, void* overflowArgArea
-) {
-    layout.gpOffset = consumedGpRegisters * 8u;
-    layout.fpOffset = 6u * 8u;
-    layout.overflowArgArea = overflowArgArea;
-    layout.regSaveArea = &regs;
-    return reinterpret_cast<std::va_list*>(&layout);
-}
-
-}
-
-#endif

core/libs/prx/libc/include/specifics/x86_64/SimdTypes.hpp

@@ -1,10 +0,0 @@
-#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_X86_64_SIMDTYPES_HPP
-#define CORE_LIBS_PRX_LIBC_INCLUDE_X86_64_SIMDTYPES_HPP
-
-#include <xmmintrin.h>
-
-namespace X86_64 {
-using Xmm = __m128;
-}
-
-#endif

core/libs/prx/libc/src/Formatting.cpp

@@ -12,7 +12,6 @@
 
 #include "prx/libc/include/General.hpp"
 #include "SceTypes.hpp"
-#include "prx/libc/include/VarArgsAbi.hpp"
 #include "prx/libc/include/FileStream.hpp"
 #include "prx/libc/include/ApplicationHeap.hpp"
 
@@ -169,7 +168,10 @@ int APS5_VABI vasprintf_nid_postfix(char** destination, const char* format, VaLi
         const char* source = text.c_str();
 #else
         char* text = nullptr;
-        const int count = ::vasprintf(&text, format, *reinterpret_cast<std::va_list*>(args));
+        std::va_list copy;
+        va_copy(copy, *reinterpret_cast<std::va_list*>(args));
+        const int count = ::vasprintf(&text, format, copy);
+        va_end(copy);
         std::unique_ptr<char, decltype(&std::free)> owner(text, std::free);
         if (count < 0) return -1;
         const char* source = text;
@@ -192,7 +194,10 @@ int APS5_VABI vfprintf_nid_postfix(FileStream* stream, const char* format, VaLis
     const int result = std::fwrite(buffer.data(), 1, static_cast<size_t>(count), native) ==
         static_cast<size_t>(count) ? count : -1;
 #else
-    const int result = std::vfprintf(native, format, *reinterpret_cast<std::va_list*>(args));
+    std::va_list copy;
+    va_copy(copy, *reinterpret_cast<std::va_list*>(args));
+    const int result = std::vfprintf(native, format, copy);
+    va_end(copy);
 #endif
     stream->SyncStatus();
     return result;
@@ -276,44 +281,28 @@ int APS5_VABI printf_nid_postfix(const char* format, ...) {
     return result;
 }
 
-int APS5_VABI libc_printf_nid_postfix(VA_ARGS) {
-    (void)rcx; (void)r8; (void)r9;
-    LibcDetail::RegSaveArea regs;
-    LibcDetail::FillRegSaveArea(regs, rsi, rdx, rcx, r8, r9, 0,
-        xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7);
-    LibcDetail::VaListLayout layout;
-    std::va_list* va = LibcDetail::BuildVaList(layout, regs, 0u,
-        reinterpret_cast<void*>(overflow_arg_area));
-    return std::vprintf(reinterpret_cast<const char*>(rdi), *va);
+int APS5_VABI libc_printf_nid_postfix(const char* format, ...) {
+    std::va_list args;
+    va_start(args, format);
+    const int result = std::vprintf(format, args);
+    va_end(args);
+    return result;
 }
 
-int APS5_VABI snprintf_nid_postfix(VA_ARGS) {
-    LibcDetail::RegSaveArea regs;
-    LibcDetail::FillRegSaveArea(regs, rcx, r8, r9, 0, 0, 0,
-        xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7);
-    LibcDetail::VaListLayout layout;
-    std::va_list* va = LibcDetail::BuildVaList(layout, regs, 0u,
-        reinterpret_cast<void*>(overflow_arg_area));
-    return std::vsnprintf(
-        reinterpret_cast<char*>(rdi),
-        static_cast<size_t>(rsi),
-        reinterpret_cast<const char*>(rdx),
-        *va
-    );
+int APS5_VABI snprintf_nid_postfix(char* buffer, size_t size, const char* format, ...) {
+    std::va_list args;
+    va_start(args, format);
+    const int result = std::vsnprintf(buffer, size, format, args);
+    va_end(args);
+    return result;
 }
 
-int APS5_VABI sprintf_nid_postfix(VA_ARGS) {
-    LibcDetail::RegSaveArea regs;
-    LibcDetail::FillRegSaveArea(regs, rdx, rcx, r8, r9, 0, 0,
-        xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7);
-    LibcDetail::VaListLayout layout;
-    std::va_list* va = LibcDetail::BuildVaList(layout, regs, 0u,
-        reinterpret_cast<void*>(overflow_arg_area));
-    return std::vsprintf(
-        reinterpret_cast<char*>(rdi),
-        reinterpret_cast<const char*>(rsi),
-        *va
-    );
+int APS5_VABI sprintf_nid_postfix(char* buffer, const char* format, ...) {
+    std::va_list args;
+    va_start(args, format);
+    const int result = std::vsprintf(buffer, format, args);
+    va_end(args);
+    return result;
 }
 
 #endif
@@ -322,7 +311,11 @@ int APS5_VABI vsscanf_nid_postfix(const char* input, const char* format, VaList*
 #ifdef _WIN32
     return LibcDetail::ScanWindows(input, format, args);
 #else
-    return std::vsscanf(input, format, *reinterpret_cast<std::va_list*>(args));
+    std::va_list copy;
+    va_copy(copy, *reinterpret_cast<std::va_list*>(args));
+    const int result = std::vsscanf(input, format, copy);
+    va_end(copy);
+    return result;
 #endif
 }
 
@@ -335,18 +328,12 @@ int APS5_VABI sscanf_nid_postfix(const char* input, const char* format, ...) {
     return result;
 }
 #else
-int APS5_VABI sscanf_nid_postfix(VA_ARGS) {
-    LibcDetail::RegSaveArea regs;
-    LibcDetail::FillRegSaveArea(regs, rdx, rcx, r8, r9, 0, 0,
-        xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7);
-    LibcDetail::VaListLayout layout;
-    std::va_list* va = LibcDetail::BuildVaList(layout, regs, 0u,
-        reinterpret_cast<void*>(overflow_arg_area));
-    return std::vsscanf(
-        reinterpret_cast<const char*>(rdi),
-        reinterpret_cast<const char*>(rsi),
-        *va
-    );
+int APS5_VABI sscanf_nid_postfix(const char* input, const char* format, ...) {
+    std::va_list args;
+    va_start(args, format);
+    const int result = std::vsscanf(input, format, args);
+    va_end(args);
+    return result;
 }
 #endif
 
@@ -376,8 +363,11 @@ int APS5_VABI vprintf_nid_postfix(const char* str, VaList* c) {
 #ifdef _WIN32
     return LibcDetail::PrintWindows(str, c);
 #else
-    std::va_list* va = reinterpret_cast<std::va_list*>(c);
-    return std::vprintf(str, *va);
+    std::va_list copy;
+    va_copy(copy, *reinterpret_cast<std::va_list*>(c));
+    const int result = std::vprintf(str, copy);
+    va_end(copy);
+    return result;
 #endif
 }
 
@@ -385,7 +375,11 @@ int APS5_VABI vsprintf_nid_postfix(char* str, const char* format, VaList* args)
 #ifdef _WIN32
     return LibcDetail::FormatWindows(str, SIZE_MAX, format, args);
 #else
-    return std::vsprintf(str, format, *reinterpret_cast<std::va_list*>(args));
+    std::va_list copy;
+    va_copy(copy, *reinterpret_cast<std::va_list*>(args));
+    const int result = std::vsprintf(str, format, copy);
+    va_end(copy);
+    return result;
 #endif
 }
 
@@ -393,8 +387,11 @@ int APS5_VABI vsnprintf_nid_postfix(char* str, size_t size, const char* format,
 #ifdef _WIN32
     return LibcDetail::FormatWindows(str, size, format, c);
 #else
-    std::va_list* va = reinterpret_cast<std::va_list*>(c);
-    return std::vsnprintf(str, size, format, *va);
+    std::va_list copy;
+    va_copy(copy, *reinterpret_cast<std::va_list*>(c));
+    const int result = std::vsnprintf(str, size, format, copy);
+    va_end(copy);
+    return result;
 #endif
 }
 
@@ -419,7 +416,11 @@ int APS5_VABI printf_s_nid_postfix(const char* format, ...) {
 #else
 
 int APS5_VABI vsprintf_s_nid_postfix(char* buffer, size_t size, const char* format, VaList* args) {
-    return std::vsnprintf(buffer, size, format, *reinterpret_cast<std::va_list*>(args));
+    std::va_list copy;
+    va_copy(copy, *reinterpret_cast<std::va_list*>(args));
+    const int result = std::vsnprintf(buffer, size, format, copy);
+    va_end(copy);
+    return result;
 }
 
 int APS5_VABI sprintf_s_nid_postfix(char* buffer, size_t size, const char* format, ...) {

core/libs/prx/libc/src/FormattingWide.cpp

@@ -10,7 +10,6 @@
 
 #include "SceTypes.hpp"
 #include "prx/libc/include/General.hpp"
-#include "prx/libc/include/VarArgsAbi.hpp"
 #include "prx/libc/include/FileStream.hpp"
 #include "prx/libc/include/WindowsFormatting.hpp"
 

core/libs/tests/Formatting.cpp

@@ -8,6 +8,7 @@ int APS5_VABI snprintf_nid_postfix(char*, size_t, const char*, ...);
 int APS5_VABI sprintf_nid_postfix(char*, const char*, ...);
 int APS5_VABI printf_nid_postfix(const char*, ...);
 int APS5_VABI libc_printf_nid_postfix(const char*, ...);
+int APS5_VABI sscanf_nid_postfix(const char*, const char*, ...);
 int APS5_VABI vsnprintf_nid_postfix(char*, size_t, const char*, VaList*);
 int APS5_VABI vprintf_nid_postfix(const char*, VaList*);
 }
@@ -38,6 +39,7 @@ static int APS5_VABI PrintList(const char* format, ...) {
     return result;
 }
 
+#ifdef _WIN32
 static bool CheckWidePrecision() {
     const char16_t input[] = u"A\u00e9\u20ac\U0001f600Z";
     const char* expected[] = {"", "A", "A", "A\xc3\xa9", "A\xc3\xa9", "A\xc3\xa9", "A\xc3\xa9\xe2\x82\xac",
@@ -65,6 +67,7 @@ static bool CheckWidePrecision() {
     Require(snprintf_nid_postfix(output, sizeof(output), "%.1s", "\xc3\xa9") == 1 && static_cast<unsigned char>(output[0]) == 0xc3 && output[1] == 0);
     return correct;
 }
+#endif
 
 __attribute__((noinline)) static void APS5_VABI RunChecks() {
     char buffer[1024];
@@ -101,6 +104,15 @@ __attribute__((noinline)) static void APS5_VABI RunChecks() {
     Require(snprintf_nid_postfix(buffer, 1, "%d", 123) == 3 && buffer[0] == 0);
     Require(sprintf_nid_postfix(buffer, "%hhd %hhu %hd %hu %%", 255, 257, 65535, 65537) == 11);
     Require(std::strcmp(buffer, "-1 1 -1 1 %") == 0);
+    Require(sprintf_nid_postfix(buffer, "%d %d %d %d %d %d", 1, 2, 3, 4, 5, 6) == 11);
+    Require(std::strcmp(buffer, "1 2 3 4 5 6") == 0);
+    int first = 0, second = 0, third = 0, fourth = 0, fifth = 0, sixth = 0;
+    char word[16] = {};
+    double real = 0.0;
+    Require(sscanf_nid_postfix("1 2 3 4 5 6 seven 8.5", "%d %d %d %d %d %d %15s %lf",
+        &first, &second, &third, &fourth, &fifth, &sixth, word, &real) == 8);
+    Require(first == 1 && second == 2 && third == 3 && fourth == 4 && fifth == 5 && sixth == 6);
+    Require(std::strcmp(word, "seven") == 0 && real == 8.5);
     long long count = -1;
     int smallCount = -1;
     Require(snprintf_nid_postfix(buffer, 3, "abcd%lnEF%n", &count, &smallCount) == 6);
@@ -109,8 +121,13 @@ __attribute__((noinline)) static void APS5_VABI RunChecks() {
     Require(buffer[0] == 'a' && buffer[1] == 0 && buffer[2] == 'b' && buffer[3] == 0);
     Require(printf_nid_postfix("printf: %d %.1f\n", 7, 2.5) == 14);
     Require(libc_printf_nid_postfix("libc_printf: %s\n", "OK") == 16);
+    Require(libc_printf_nid_postfix("%d %d %d %d %d %d %d\n", 1, 22, 333, 4444, 55555, 666666, 7777777) == 35);
     Require(PrintList("vprintf: %d\n", 42) == 12);
     std::puts("Formatting checks passed: 10000 iterations");
 }
 
+#ifdef _WIN32
 int main() { RunChecks(); return CheckWidePrecision() ? 0 : 1; }
+#else
+int main() { RunChecks(); }
+#endif

docs/dev/TechnicalDebt.md

@@ -319,6 +319,7 @@ Throughout the project, every function at every stage either **does exactly what
 - [Min/max S#](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver, shader recompiler) - throws with anisotropy, a linear mip filter, depth or color-texture compare, or bilinear filtering of an image the recompiler point-samples (sint, converted, depth-bits).
 - [Raw `buffer_load_dwordx2`](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) is one 64-bit access on the hardware (with or without glc), so it never sees half of a `buffer_atomic_swap_x2`. Only a glc load that is 8-byte aligned and in bounds, in a program that has the 64-bit storage view (it uses 64-bit buffer atomics), is read with one 64-bit atomic load; every other dwordx2 load is two dword loads and can tear against a concurrent 64-bit write: loads without glc, and loads in a program without 64-bit buffer atomics while another shader writes the pair with `swap_x2`.
 - [wcsrtombs_s](../../core/libs/prx/libc/src/Multibyte.cpp) (libc) follows the console's `libc.prx`, but a runtime-constraint violation only returns its error: the default handler there writes the message and a newline to `stderr` before returning, which is not done here. Conversion uses the C locale of `wcrtomb` (`MB_CUR_MAX` 1), so the path where a multibyte character does not fit the remaining space is not reached
+- [The narrow printf family](../../core/libs/prx/libc/src/Formatting.cpp) (libc) passes `%ls` and `%S` to glibc on Linux, which reads them as strings of 32-bit `wchar_t`; the guest's `wchar_t` is 16 bits (UTF-16), so these conversions fail with -1. The Windows path converts them to UTF-8, and `guest_formatting` checks wide conversions on Windows only
 - [collate<char>::_Getcat](../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) builds the facet for the C locale only and throws for any other locale name. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size, and for results of 4095 characters or more, whose string storage uses the guest's over-aligned large allocation.
 - [poll](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) waits only on guest sockets, through `WSAPoll` on Windows and `poll` on Linux; the `pollfd` layout and flag values are FreeBSD's. A descriptor below the socket range (files, pipes, stdio) throws, as do event bits other than `POLLIN`, `POLLPRI`, `POLLOUT`, `POLLRDNORM`, `POLLRDBAND` and `POLLWRBAND` (the `POLLERR`, `POLLHUP` and `POLLNVAL` bits are ignored in `events`), `POLLPRI` on Windows, which `WSAPoll` rejects, and an infinite timeout with no socket to wait on. A closed socket descriptor reports `POLLNVAL`; the console's limit on the descriptor count is unknown and not checked
 - [Thread exit destructors](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) run when a libkernel thread finishes; on the main or a host thread only `_sceLibcInternalThreadDtors` runs them, as `exit` does, else they are dropped