feat(libc): implement collate<char>::_Getcat

Co-Authored-By: Claude <noreply@anthropic.com>

core/libs/prx/libc/CMakeLists.txt

@@ -232,6 +232,19 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(guest_collate_tests EXCLUDE_FROM_ALL ${LIBS_INCLUDE_DIR}/tests/GuestCollate.cpp)
+target_include_directories(guest_collate_tests PRIVATE ${LIBS_INCLUDE_DIR})
+target_link_libraries(guest_collate_tests PRIVATE libc)
+configure_windows_unwind(guest_collate_tests)
+set_target_properties(guest_collate_tests PROPERTIES CXX_EXTENSIONS OFF)
+if(BUILD_TESTING)
+    set_target_properties(guest_collate_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
+    add_test(NAME guest_collate COMMAND guest_collate_tests)
+    if(WIN32)
+        set_tests_properties(guest_collate PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(application_heap_tests EXCLUDE_FROM_ALL ${LIBS_INCLUDE_DIR}/tests/ApplicationHeap.cpp)
 target_include_directories(application_heap_tests PRIVATE ${LIBS_INCLUDE_DIR})
 target_link_libraries(application_heap_tests PRIVATE libc)

core/libs/prx/libc/include/GuestLocale.hpp

@@ -35,6 +35,29 @@ struct Implementation {
     const char* name;
 };
 
+struct String {
+    std::uint64_t reserved;
+    union {
+        char buffer[16];
+        char* pointer;
+    };
+    std::uint64_t size;
+    std::uint64_t capacity;
+};
+
+struct CollateFacet {
+    Facet base;
+    const void* collation;
+    const void* wideCollation;
+};
+
+struct CollateVtable {
+    FacetVtable facet;
+    int (APS5_VABI *compare)(const CollateFacet* self, const char* first1, const char* last1, const char* first2, const char* last2);
+    String* (APS5_VABI *transform)(String* result, const CollateFacet* self, const char* first, const char* last);
+    std::int64_t (APS5_VABI *hash)(const CollateFacet* self, const char* first, const char* last);
+};
+
 struct IosBase {
     const void* vtable;
     std::int64_t standardStream;
@@ -59,6 +82,13 @@ static_assert(offsetof(IosBase, storage) == 0x28);
 static_assert(offsetof(IosBase, callbacks) == 0x30);
 static_assert(offsetof(IosBase, locale) == 0x38);
 static_assert(sizeof(IosBase) == 0x40);
+static_assert(offsetof(String, buffer) == 0x8);
+static_assert(offsetof(String, size) == 0x18);
+static_assert(sizeof(String) == 0x28);
+static_assert(offsetof(CollateFacet, collation) == 0x10);
+static_assert(sizeof(CollateFacet) == 0x20);
+static_assert(offsetof(CollateVtable, compare) == 0x20);
+static_assert(offsetof(CollateVtable, hash) == 0x30);
 static_assert(alignof(LocinfoStorage) == 8);
 
 }

core/libs/prx/libc/src/LocaleSupport.cpp

@@ -13,12 +13,19 @@
 #include <algorithm>
 #include <array>
 #include <limits>
+#include <stdexcept>
+#include <string>
 
 #include "prx/libc/include/General.hpp"
 #include "prx/libc/include/ApplicationHeap.hpp"
 #include "prx/libc/include/GuestLocale.hpp"
 #include "SceTypes.hpp"
 
+extern "C" {
+void* APS5_VABI _Znwm_nid_postfix(std::size_t size);
+void APS5_VABI _ZdlPv_nid_postfix(void* pointer);
+}
+
 namespace {
 
 std::mutex g_localeInitMutex;
@@ -71,6 +78,71 @@ const GuestLocale::FacetVtable g_copiedLocaleVtable{DestroyCopiedLocale, Destroy
 GuestLocale::Facet* g_classicFacets[1]{};
 GuestLocale::Implementation g_classicLocale{{&g_localeVtable, 1, 0}, g_classicFacets, 1, 0, false, "C"};
 
+constexpr std::size_t GuestStringInlineCapacity = 15;
+constexpr std::size_t GuestStringLargeAllocation = 0x1000;
+
+void ValidateCharacterRange(const char* first, const char* last, const char* function) {
+    if ((first == nullptr) != (last == nullptr) || last < first) throw std::invalid_argument(std::string(function) + ": invalid character range");
+}
+
+void APS5_VABI DestroyCollate(GuestLocale::Facet*) {}
+
+void APS5_VABI DeleteCollate(GuestLocale::Facet* self) {
+    if (self == nullptr) throw std::invalid_argument("collate delete: null facet");
+    _ZdlPv_nid_postfix(self);
+}
+
+void APS5_VABI RetainFacet(GuestLocale::Facet* self) {
+    if (self == nullptr) throw std::invalid_argument("facet retain: null facet");
+    std::atomic_ref<std::uint32_t>(self->references).fetch_add(1);
+}
+
+GuestLocale::Facet* APS5_VABI ReleaseFacet(GuestLocale::Facet* self) {
+    if (self == nullptr) throw std::invalid_argument("facet release: null facet");
+    return std::atomic_ref<std::uint32_t>(self->references).fetch_sub(1) == 1 ? self : nullptr;
+}
+
+int APS5_VABI CollateCompare(const GuestLocale::CollateFacet*, const char* first1, const char* last1, const char* first2, const char* last2) {
+    ValidateCharacterRange(first1, last1, "collate::do_compare");
+    ValidateCharacterRange(first2, last2, "collate::do_compare");
+    const auto length1 = static_cast<std::size_t>(last1 - first1);
+    const auto length2 = static_cast<std::size_t>(last2 - first2);
+    const auto common = std::min(length1, length2);
+    const int order = common == 0 ? 0 : std::memcmp(first1, first2, common);
+    if (order != 0) return order < 0 ? -1 : 1;
+    if (length1 == length2) return 0;
+    return length1 < length2 ? -1 : 1;
+}
+
+GuestLocale::String* APS5_VABI CollateTransform(GuestLocale::String* result, const GuestLocale::CollateFacet*, const char* first, const char* last) {
+    if (result == nullptr) throw std::invalid_argument("collate::do_transform: null result");
+    ValidateCharacterRange(first, last, "collate::do_transform");
+    const auto length = static_cast<std::size_t>(last - first);
+    if (length != 0 && std::memchr(first, 0, length) != nullptr) throw std::runtime_error("collate::do_transform: embedded null characters are not supported");
+    if (length + 1 >= GuestStringLargeAllocation) throw std::runtime_error("collate::do_transform: results of 4095 characters or more are not supported");
+    char* data = result->buffer;
+    std::size_t capacity = GuestStringInlineCapacity;
+    if (length > GuestStringInlineCapacity) {
+        capacity = length;
+        data = static_cast<char*>(_Znwm_nid_postfix(capacity + 1));
+    }
+    if (length != 0) std::memcpy(data, first, length);
+    data[length] = 0;
+    if (data != result->buffer) result->pointer = data;
+    result->size = length;
+    result->capacity = capacity;
+    return result;
+}
+
+std::int64_t APS5_VABI CollateHash(const GuestLocale::CollateFacet*, const char* first, const char* last) {
+    ValidateCharacterRange(first, last, "collate::do_hash");
+    std::uint64_t hash = 0xcbf29ce484222325;
+    for (; first != last; ++first) hash = (hash ^ static_cast<unsigned char>(*first)) * 0x100000001b3;
+    return static_cast<std::int64_t>(hash);
+}
+
+const GuestLocale::CollateVtable g_collateVtable{{DestroyCollate, DeleteCollate, RetainFacet, ReleaseFacet}, CollateCompare, CollateTransform, CollateHash};
+
 constexpr std::array<short, 257> MakeClassificationTable() {
     std::array<short, 257> table{};
     for (unsigned int value = 0; value < 256; ++value) {
@@ -189,8 +261,14 @@ void APS5_VABI _ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(Gu
     NotImplemented_nid_no_patch(__func__);
 }
 
-void APS5_VABI _ZNSt7collateIcE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(GuestLocale::Facet**, const GuestLocale::Implementation*) {
-    NotImplemented_nid_no_patch(__func__);
+std::size_t APS5_VABI _ZNSt7collateIcE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(GuestLocale::Facet** facet, const GuestLocale::Implementation* const* locale) {
+    if (facet != nullptr && *facet == nullptr) {
+        if (locale == nullptr || *locale == nullptr || (*locale)->name == nullptr || std::strcmp((*locale)->name, "C") != 0) throw std::invalid_argument("collate::_Getcat: only the C locale is supported");
+        auto* collate = static_cast<GuestLocale::CollateFacet*>(_Znwm_nid_postfix(sizeof(GuestLocale::CollateFacet)));
+        *collate = {{&g_collateVtable.facet, 0, 0}, nullptr, nullptr};
+        *facet = &collate->base;
+    }
+    return 1;
 }
 
 void APS5_VABI _ZNSt8_LocinfoC1EPKc_nid_postfix(GuestLocale::LocinfoStorage* self, const char* localeName) {

core/libs/tests/GuestCollate.cpp

@@ -0,0 +1,156 @@
+#include "prx/libc/include/ApplicationHeap.hpp"
+#include "prx/libc/include/GuestLocale.hpp"
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <stdexcept>
+#include <string>
+
+extern "C" {
+extern GuestLocale::Implementation* _ZSt21_sceLibcClassicLocale_nid_postfix;
+std::size_t APS5_VABI _ZNSt7collateIcE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(GuestLocale::Facet** facet, const GuestLocale::Implementation* const* locale);
+}
+
+namespace {
+
+std::size_t allocations = 0;
+std::size_t frees = 0;
+std::size_t lastSize = 0;
+void* lastAllocation = nullptr;
+void* lastFree = nullptr;
+
+void require(bool condition) {
+    if (!condition) std::abort();
+}
+
+template<typename TAction>
+void reject(TAction action) {
+    bool rejected = false;
+    try { action(); } catch (const std::exception&) { rejected = true; }
+    require(rejected);
+}
+
+void* APS5_VABI allocate(std::size_t bytes) {
+    ++allocations;
+    lastSize = bytes;
+    lastAllocation = std::malloc(bytes);
+    return lastAllocation;
+}
+
+void APS5_VABI release(void* pointer) {
+    ++frees;
+    lastFree = pointer;
+    std::free(pointer);
+}
+
+void* APS5_VABI allocateZeroed(std::size_t, std::size_t) { std::abort(); }
+void* APS5_VABI reallocate(void*, std::size_t) { std::abort(); }
+void* APS5_VABI align(std::size_t, std::size_t) { std::abort(); }
+void* APS5_VABI realign(void*, std::size_t, std::size_t) { std::abort(); }
+int APS5_VABI posixAlign(void**, std::size_t, std::size_t) { std::abort(); }
+
+const GuestLocale::CollateVtable& Vtable(const GuestLocale::Facet* facet) {
+    return *reinterpret_cast<const GuestLocale::CollateVtable*>(facet->vtable);
+}
+
+int Compare(const GuestLocale::CollateFacet* facet, const std::string& left, const std::string& right) {
+    return Vtable(&facet->base).compare(facet, left.data(), left.data() + left.size(), right.data(), right.data() + right.size());
+}
+
+std::uint64_t Hash(const GuestLocale::CollateFacet* facet, const std::string& text) {
+    return static_cast<std::uint64_t>(Vtable(&facet->base).hash(facet, text.data(), text.data() + text.size()));
+}
+
+}
+
+int main() {
+    const std::array<void*, 10> api{reinterpret_cast<void*>(&allocate), reinterpret_cast<void*>(&release), reinterpret_cast<void*>(&allocateZeroed), reinterpret_cast<void*>(&reallocate), reinterpret_cast<void*>(&align), reinterpret_cast<void*>(&realign), reinterpret_cast<void*>(&posixAlign)};
+    ApplicationHeapRegister_nid_no_patch(api.data());
+
+    const GuestLocale::Implementation* const* classic = &_ZSt21_sceLibcClassicLocale_nid_postfix;
+    require(_ZNSt7collateIcE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(nullptr, classic) == 1);
+    GuestLocale::Facet existing{};
+    GuestLocale::Facet* facet = &existing;
+    require(_ZNSt7collateIcE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(&facet, classic) == 1);
+    require(facet == &existing && allocations == 0);
+
+    GuestLocale::Implementation french = *_ZSt21_sceLibcClassicLocale_nid_postfix;
+    french.name = "fr_FR";
+    const GuestLocale::Implementation* frenchPointer = &french;
+    facet = nullptr;
+    reject([&] { _ZNSt7collateIcE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(&facet, &frenchPointer); });
+    require(facet == nullptr && allocations == 0);
+
+    require(_ZNSt7collateIcE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(&facet, classic) == 1);
+    require(facet != nullptr && allocations == 1 && lastSize == sizeof(GuestLocale::CollateFacet) && lastAllocation == facet);
+    const auto* collate = reinterpret_cast<const GuestLocale::CollateFacet*>(facet);
+    require(facet->references == 0 && collate->collation == nullptr && collate->wideCollation == nullptr);
+
+    require(Compare(collate, "abc", "abd") == -1);
+    require(Compare(collate, "abd", "abc") == 1);
+    require(Compare(collate, "a", "c") == -1);
+    require(Compare(collate, "abc", "abc") == 0);
+    require(Compare(collate, "ab", "abc") == -1);
+    require(Compare(collate, "abc", "ab") == 1);
+    require(Compare(collate, "", "") == 0);
+    require(Compare(collate, "", "a") == -1);
+    require(Compare(collate, "\x80", "a") == 1);
+    require(Compare(collate, std::string("a\0", 2), "a") == 1);
+    require(Compare(collate, std::string("a\0b", 3), std::string("a\0c", 3)) == -1);
+    require(Vtable(facet).compare(collate, nullptr, nullptr, nullptr, nullptr) == 0);
+    const char text[] = "abc";
+    reject([&] { Vtable(facet).compare(collate, text + 2, text, text, text + 1); });
+    reject([&] { Vtable(facet).compare(collate, nullptr, text, text, text + 1); });
+
+    require(Hash(collate, "") == 0xcbf29ce484222325ull);
+    require(Hash(collate, "a") == 0xaf63dc4c8601ec8cull);
+    require(Hash(collate, "foobar") == 0x85944171f73967e8ull);
+    require(Hash(collate, "\xff") == ((0xcbf29ce484222325ull ^ 0xffull) * 0x100000001b3ull));
+    reject([&] { Vtable(facet).hash(collate, text + 1, text); });
+
+    GuestLocale::String result;
+    std::memset(&result, 0xcd, sizeof(result));
+    const std::string shortText = "hello";
+    require(Vtable(facet).transform(&result, collate, shortText.data(), shortText.data() + shortText.size()) == &result);
+    require(result.reserved == 0xcdcdcdcdcdcdcdcdull && result.size == 5 && result.capacity == 15 && std::strcmp(result.buffer, "hello") == 0);
+    require(allocations == 1);
+
+    std::memset(&result, 0xcd, sizeof(result));
+    require(Vtable(facet).transform(&result, collate, nullptr, nullptr) == &result);
+    require(result.size == 0 && result.capacity == 15 && result.buffer[0] == 0);
+
+    const std::string fifteen(15, 'q');
+    require(Vtable(facet).transform(&result, collate, fifteen.data(), fifteen.data() + fifteen.size()) == &result);
+    require(result.size == 15 && result.capacity == 15 && std::string(result.buffer) == fifteen && allocations == 1);
+
+    const std::string longText = "collation of a longer text";
+    require(Vtable(facet).transform(&result, collate, longText.data(), longText.data() + longText.size()) == &result);
+    require(allocations == 2 && lastSize == longText.size() + 1 && result.pointer == lastAllocation);
+    require(result.size == longText.size() && result.capacity == longText.size() && std::string(result.pointer) == longText);
+    std::free(result.pointer);
+
+    const std::string largest(4094, 'z');
+    require(Vtable(facet).transform(&result, collate, largest.data(), largest.data() + largest.size()) == &result);
+    require(allocations == 3 && lastSize == 4095 && result.capacity == 4094 && std::string(result.pointer) == largest);
+    std::free(result.pointer);
+
+    const std::string tooLong(4095, 'z');
+    reject([&] { Vtable(facet).transform(&result, collate, tooLong.data(), tooLong.data() + tooLong.size()); });
+    const std::string embedded("a\0b", 3);
+    reject([&] { Vtable(facet).transform(&result, collate, embedded.data(), embedded.data() + embedded.size()); });
+    reject([&] { Vtable(facet).transform(nullptr, collate, text, text + 1); });
+    require(allocations == 3);
+
+    Vtable(facet).facet.retain(facet);
+    Vtable(facet).facet.retain(facet);
+    require(facet->references == 2);
+    require(Vtable(facet).facet.release(facet) == nullptr && facet->references == 1);
+    require(Vtable(facet).facet.release(facet) == facet && facet->references == 0);
+    Vtable(facet).facet.destroy(facet);
+    require(frees == 0);
+    Vtable(facet).facet.deleteObject(facet);
+    require(frees == 1 && lastFree == collate);
+    return 0;
+}

docs/dev/TechnicalDebt.md

@@ -169,6 +169,7 @@ Throughout the project, every function at every stage either **does exactly what
 - [libScePlayerInvitationDialog](../../core/libs/prx/libScePlayerInvitationDialog/libScePlayerInvitationDialog.cpp) simulates dialog completion without displaying UI or sending invitations; its parameter ABI remains unverified.
 - [64-bit LDS atomics](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_*_u64`, `ds_*_b64`, `ds_*_f64`) read and write the two dwords under a workgroup lock stored after the guest LDS, as LDS is a 32-bit array without 64-bit atomics. They are atomic with respect to each other only: 32-bit accesses to the same dwords issued concurrently are not ordered by the lock. 64-bit GDS atomics throw. `ds_min/max(_rtn)_f64` and `ds_cmpst(_rtn)_f64` follow the NaN and signed-zero rules measured for the f32 forms; the f64 forms were not measured on hardware.
 - [wcsrtombs_s](../../core/libs/prx/libc/src/Multibyte.cpp) (libc) follows the console's `libc.prx`, but a runtime-constraint violation only returns its error: the default handler there writes the message and a newline to `stderr` before returning, which is not done here. Conversion uses the C locale of `wcrtomb` (`MB_CUR_MAX` 1), so the path where a multibyte character does not fit the remaining space is not reached
+- [collate<char>::_Getcat](../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) builds the facet for the C locale only and throws for any other locale name. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size, and for results of 4095 characters or more, whose string storage uses the guest's over-aligned large allocation.
 - [Thread exit destructors](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) run when a libkernel thread finishes; on the main or a host thread only `_sceLibcInternalThreadDtors` runs them, as `exit` does, else they are dropped
 - [sceFontOpenFontSet](../../core/libs/prx/libSceFont/src/SystemFont.cpp) (libSceFont) opens only the primary file of a system font set. Characters the console takes from the set's other files (Hangul, Thai and Arabic in Japanese and Chinese sets, Arabic in European sets) are reported as unsupported glyphs. The console file names are those of the PS4 firmware; the names of the PS5-only JG2 sets (`0x1A......`) are unknown, so they use the SSTJpPro files. When a console file is missing, an openly licensed substitute (Noto Sans, Noto Sans Mono, Noto Sans Thai, Noto Sans CJK) is loaded instead. It is not the system font: its metrics differ, so text width, line height and wrapping differ from the console. System font faces are loaded by FreeType in host memory, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for.
 - [Font writing lines](../../core/libs/prx/libSceFont/src/WritingLine.cpp) (libSceFont) - `sceFontCreateWritingLine`, `sceFontWritingLineWritesOrder`, `sceFontWritingLineRefersRenderStep`, `sceFontWritingLineGetRenderMetrics` and `sceFontWritingLineClear` follow [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/main/src/libs/libFont.cpp): runs are placed left to right, each at the line's advance so far, with no spacing or adjustment. Only horizontal and left-to-right lines, writing attribute 0 and non-null run metrics are modelled; other values, the metrics of an empty line and unknown or null handles throw, since no reference gives their result or error code. `sceFontWritingLineGetOrderingSpace` still throws (KytyPS5's values are placeholders). Lines are allocated on the host, not in the font memory