fix(libc): parse fopen and freopen modes as FreeBSD does

FreeBSD's __sflags reads the access character, then b, +, x, e and v, and
stops at the first other character without failing, so "rt" opens for
reading. On Windows fopen inserted b after the first character and passed
the rest on, so "rt" became "rbt", the CRT rejected it with EINVAL, and
fopen threw. vkQuake opens history.txt with "rt" at startup. freopen
accepted only a fixed list of modes.

Both now parse the mode like __sflags. An unknown access character, or x
on a read-only stream, returns NULL with EINVAL; x on an existing file
returns NULL with EEXIST; e and v are accepted and ignored.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/libs/prx/libc/src/FileAndHeap.cpp

@@ -3,6 +3,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <memory>
+#include <optional>
 #include <filesystem>
 #include <limits>
 #include <utility>
@@ -14,16 +15,29 @@
 #include "prx/libc/include/General.hpp"
 #include "prx/libc/include/GuestArena.hpp"
 
-static std::string NativeFileMode(const char* mode) {
-    std::string result(mode);
-#ifdef _WIN32
-    if (!result.empty() && result.find('b') == std::string::npos) result.insert(1, 1, 'b');
-#endif
-    return result;
-}
+struct GuestFileMode {
+    std::string native;
+    bool writes = false;
+    bool exclusive = false;
+};
 
-static bool WritesFile(const char* mode) {
-    return std::strpbrk(mode, "wa+") != nullptr;
+static std::optional<GuestFileMode> ParseFileMode(const char* mode) {
+    const char access = mode[0];
+    if (access != 'r' && access != 'w' && access != 'a') return std::nullopt;
+    bool update = false;
+    bool exclusive = false;
+    for (const char* flag = mode + 1; *flag != '\0' && std::strchr("b+xev", *flag) != nullptr; ++flag) {
+        if (*flag == '+') update = true;
+        if (*flag == 'x') exclusive = true;
+    }
+    if (exclusive && access == 'r' && !update) return std::nullopt;
+    GuestFileMode result{std::string(1, access), access != 'r' || update, exclusive};
+    if (update) result.native += '+';
+#ifdef _WIN32
+    result.native += 'b';
+#endif
+    if (exclusive) result.native += 'x';
+    return result;
 }
 
 extern "C" {
@@ -56,15 +70,12 @@ FileStream* APS5_VABI fdopen_nid_postfix(int descriptor, const char* mode) {
 FileStream* APS5_VABI freopen_nid_postfix(const char* filename, const char* mode, FileStream* stream) {
     if (!stream || !mode) { errno = 22; return nullptr; }
     if (!filename) { errno = 45; return nullptr; } // Mode-only reopening is not supported.
-    const char* supported[] = {"r", "w", "a", "rb", "wb", "ab", "r+", "w+", "a+",
-        "rb+", "wb+", "ab+", "r+b", "w+b", "a+b"};
-    bool valid = false;
-    for (const auto* candidate : supported) if (std::strcmp(mode, candidate) == 0) valid = true;
-    if (!valid) { errno = 22; return nullptr; }
+    const auto parsed = ParseFileMode(mode);
+    if (!parsed) { errno = 22; return nullptr; }
     try {
         const auto path = *filename ? ResolvePath_nid_no_patch(filename).string() : std::string{};
-        if (stream->Reopen(path.c_str(), NativeFileMode(mode).c_str())) {
-            if (!path.empty() && WritesFile(mode)) RecordWrittenPath_nid_no_patch(path);
+        if (stream->Reopen(path.c_str(), parsed->native.c_str())) {
+            if (!path.empty() && parsed->writes) RecordWrittenPath_nid_no_patch(path);
             return stream;
         }
         const int error = errno;
@@ -77,12 +88,17 @@ FileStream* APS5_VABI freopen_nid_postfix(const char* filename, const char* mode
 
 FileStream* APS5_VABI fopen_nid_postfix(const char* filename, const char* mode) {
     if (!filename || !mode) throw std::runtime_error(std::string(__func__) + ": " + FOPEN_MSG_NULL_ARG);
+    const auto parsed = ParseFileMode(mode);
+    if (!parsed) {
+        errno = EINVAL;
+        return nullptr;
+    }
     const std::filesystem::path fpath = ResolvePath_nid_no_patch(filename);
     const auto abs_path = fpath.string();
-    std::unique_ptr<std::FILE, int (*)(std::FILE*)> handle(std::fopen(abs_path.c_str(), NativeFileMode(mode).c_str()), std::fclose);
+    std::unique_ptr<std::FILE, int (*)(std::FILE*)> handle(std::fopen(abs_path.c_str(), parsed->native.c_str()), std::fclose);
     if (!handle) {
         const int error = errno;
-        if (error == ENOENT) {
+        if (error == ENOENT || (parsed->exclusive && error == EEXIST)) {
             errno = error;
             return nullptr;
         }
@@ -103,7 +119,7 @@ FileStream* APS5_VABI fopen_nid_postfix(const char* filename, const char* mode)
         throw std::runtime_error(std::string(__func__) + ": " + FOPEN_MSG_OPEN_FAILED + ": \"" + abs_path + "\": " + reason);
     }
     // APS5_LOG_OUT("success: \"%s\"", abs_path.c_str());
-    if (WritesFile(mode)) RecordWrittenPath_nid_no_patch(fpath);
+    if (parsed->writes) RecordWrittenPath_nid_no_patch(fpath);
     auto stream = std::make_unique<FileStream>(handle.get(), true);
     handle.release();
     return stream.release();

core/libs/tests/GuestStandardStreams.cpp

@@ -95,7 +95,7 @@ static bool CheckBinaryModes() {
     const auto filename = directory + "/bytes";
     const std::string original("A\r\n\x1a" "B\0C", 7);
     const std::string written("D\n\x1a" "E\0F", 6);
-    const char* modes[] = {"r", "r+", "w", "w+", "a", "a+", "rb", "rb+", "r+b", "wb", "wb+", "w+b", "ab", "ab+", "a+b"};
+    const char* modes[] = {"r", "r+", "w", "w+", "a", "a+", "rb", "rb+", "r+b", "wb", "wb+", "w+b", "ab", "ab+", "a+b", "rt", "r+t", "wt", "w+e", "ae", "rbv"};
     bool correct = true;
     for (const auto* mode : modes) {
         for (const bool reopen : {false, true}) {
@@ -124,6 +124,20 @@ static bool CheckBinaryModes() {
             correct &= CheckFileBytes(filename, expected, mode, reopen);
         }
     }
+    for (const auto* mode : {"", "q", "tr", "rx", "rbx"}) {
+        FileStream redirected(std::tmpfile());
+        Require(fopen_nid_postfix(filename.c_str(), mode) == nullptr && *__error_nid_postfix() == 22);
+        Require(freopen_nid_postfix(filename.c_str(), mode, &redirected) == nullptr && *__error_nid_postfix() == 22);
+    }
+    Require(fopen_nid_postfix(filename.c_str(), "wx") == nullptr && *__error_nid_postfix() == 17);
+    FileStream* readOnly = fopen_nid_postfix(filename.c_str(), "rt+");
+    Require(readOnly != nullptr && fputc_nid_postfix('X', readOnly) == EOF);
+    Require(fclose_nid_postfix(readOnly) == 0);
+    Require(std::filesystem::remove(filename));
+    FileStream* created = fopen_nid_postfix(filename.c_str(), "wx");
+    Require(created != nullptr && fputc_nid_postfix('Y', created) == 'Y');
+    Require(fclose_nid_postfix(created) == 0);
+    correct &= CheckFileBytes(filename, "Y", "wx", false);
     Require(std::filesystem::remove(filename));
     Require(std::filesystem::remove(directory));
     return correct;