fix(libc): call __dynamic_cast and __cxa_init_primary_exception with the guest ABI

Every other export of the C++ runtime is declared APS5_VABI, but these two
were not. On Windows a title calls them with the System V convention while
they read the Microsoft one, so __dynamic_cast took its fourth argument, the
src2dst hint, as the source pointer: an ordinary downcast (hint 0) returned
null, and the negative hints made it read through an invalid pointer.

guest_dynamic_cast calls both through APS5_VABI declarations, as a title
does, and checks a single-inheritance downcast, a failed cross cast, a cast
of a base object and the header __cxa_init_primary_exception fills in.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

core/libs/prx/libc/CMakeLists.txt

@@ -618,6 +618,18 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(guest_dynamic_cast_tests EXCLUDE_FROM_ALL ${LIBS_INCLUDE_DIR}/tests/GuestDynamicCast.cpp)
+target_include_directories(guest_dynamic_cast_tests PRIVATE ${LIBS_INCLUDE_DIR})
+target_link_libraries(guest_dynamic_cast_tests PRIVATE libc)
+configure_windows_unwind(guest_dynamic_cast_tests)
+if(BUILD_TESTING)
+    set_target_properties(guest_dynamic_cast_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
+    add_test(NAME guest_dynamic_cast COMMAND guest_dynamic_cast_tests)
+    if(WIN32)
+        set_tests_properties(guest_dynamic_cast PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 if(MINGW)
     if(BUILD_TESTING)
         enable_language(ASM)

core/libs/prx/libc/src/exception/Runtime.cpp

@@ -274,7 +274,7 @@ void APS5_VABI __cxa_free_exception_nid_postfix(void* object) {
     ThrowFromSite(object, type, destructor, __builtin_return_address(0));
 }
 
-LibcException::Header* __cxa_init_primary_exception_nid_postfix(void* object, std::type_info* type, void (*destructor)(void*)) {
+LibcException::Header* APS5_VABI __cxa_init_primary_exception_nid_postfix(void* object, std::type_info* type, void (*destructor)(void*)) {
     using namespace LibcException;
     auto* header = FromObject(object);
     AllocationOf(header)->references.store(0, std::memory_order_relaxed);
@@ -396,8 +396,8 @@ void APS5_VABI __cxa_rethrow_primary_exception_nid_postfix(void* object) {
     InvokeTerminate(header->terminate);
 }
 
-void* __dynamic_cast_nid_postfix(const void* source, const __cxxabiv1::__class_type_info* sourceType,
-                               const __cxxabiv1::__class_type_info* destinationType, std::ptrdiff_t) {
+void* APS5_VABI __dynamic_cast_nid_postfix(const void* source, const __cxxabiv1::__class_type_info* sourceType,
+                                         const __cxxabiv1::__class_type_info* destinationType, std::ptrdiff_t) {
     using namespace LibcException;
     if (!source) return nullptr;
     auto* vtable = *static_cast<const std::uintptr_t* const*>(source);

core/libs/tests/GuestDynamicCast.cpp

@@ -0,0 +1,45 @@
+#include "prx/libc/include/exceptions/Runtime.hpp"
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <typeinfo>
+
+struct TypeRecord { const void* vtable; const char* name; const TypeRecord* base; };
+struct Vtable { std::intptr_t offsetToTop; const TypeRecord* type; const void* first; };
+struct Object { const void* vtable; };
+
+extern "C" {
+extern const unsigned char _ZTVN10__cxxabiv117__class_type_infoE_nid_postfix[];
+extern const unsigned char _ZTVN10__cxxabiv120__si_class_type_infoE_nid_postfix[];
+void* APS5_VABI __dynamic_cast_nid_postfix(const void*, const void*, const void*, std::ptrdiff_t);
+LibcException::Header* APS5_VABI __cxa_init_primary_exception_nid_postfix(void*, std::type_info*, void (*)(void*));
+}
+
+namespace {
+
+void Require(bool value) { if (!value) std::abort(); }
+
+void Destroy(void*) {}
+
+const TypeRecord BaseType{_ZTVN10__cxxabiv117__class_type_infoE_nid_postfix + 16, "4Base", nullptr};
+const TypeRecord DerivedType{_ZTVN10__cxxabiv120__si_class_type_infoE_nid_postfix + 16, "7Derived", &BaseType};
+const TypeRecord OtherType{_ZTVN10__cxxabiv120__si_class_type_infoE_nid_postfix + 16, "5Other", &BaseType};
+const Vtable BaseVtable{0, &BaseType, nullptr};
+const Vtable DerivedVtable{0, &DerivedType, nullptr};
+
+} // namespace
+
+int main() {
+    const Object derived{&DerivedVtable.first};
+    const Object base{&BaseVtable.first};
+    Require(__dynamic_cast_nid_postfix(&derived, &BaseType, &DerivedType, 0) == &derived);
+    Require(__dynamic_cast_nid_postfix(&derived, &BaseType, &OtherType, 0) == nullptr);
+    Require(__dynamic_cast_nid_postfix(&base, &BaseType, &DerivedType, 0) == nullptr);
+    Require(__dynamic_cast_nid_postfix(nullptr, &BaseType, &DerivedType, 0) == nullptr);
+    void* object = __cxa_allocate_exception_nid_postfix(16);
+    auto* type = reinterpret_cast<std::type_info*>(const_cast<TypeRecord*>(&DerivedType));
+    LibcException::Header* header = __cxa_init_primary_exception_nid_postfix(object, type, Destroy);
+    Require(header == LibcException::FromObject(object));
+    Require(header->type == type && header->destructor == Destroy && header->adjusted == object);
+    __cxa_free_exception_nid_postfix(object);
+}