From 22d1fbbc735f72fbf61e18e4f0bc6a76c52fa697 Mon Sep 17 00:00:00 2001
From: Nahuel Genchi <65604505+NahuelGenchi@users.noreply.github.com>
Date: Fri, 9 Oct 2026 15:01:37 -0300
Subject: [PATCH] fix(libc): throw bad_alloc from operator new when the default
 heap rejects a size

---
 core/libs/prx/libc/CMakeLists.txt      |  5 ++++
 core/libs/prx/libc/src/StdioExtras.cpp |  4 +++
 core/libs/tests/GuestNewHandler.cpp    | 36 +++++++++++++++++++++++++-
 3 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt
index f631dc835..9b1103975 100644
--- a/core/libs/prx/libc/CMakeLists.txt
+++ b/core/libs/prx/libc/CMakeLists.txt
@@ -417,6 +417,11 @@ if(BUILD_TESTING)
         set_tests_properties(guest_new_handler PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME guest_new_handler_default_heap COMMAND guest_new_handler_tests default)
+    if(WIN32)
+        set_tests_properties(guest_new_handler_default_heap PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test_executable(guest_search_tests ${LIBS_INCLUDE_DIR}/tests/GuestSearch.cpp)
     target_include_directories(guest_search_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(guest_search_tests PRIVATE libc)
diff --git a/core/libs/prx/libc/src/StdioExtras.cpp b/core/libs/prx/libc/src/StdioExtras.cpp
index 0e74f5986..79e53ccac 100644
--- a/core/libs/prx/libc/src/StdioExtras.cpp
+++ b/core/libs/prx/libc/src/StdioExtras.cpp
@@ -5,6 +5,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <new>
+#include <stdexcept>
 
 #include "prx/libc/include/FileStream.hpp"
 #include "prx/libc/include/ApplicationHeap.hpp"
@@ -29,6 +30,9 @@ void* Allocate(std::size_t size) {
         } catch (const std::bad_alloc&) {
             handler = g_newHandler.load();
             if (handler == nullptr) throw;
+        } catch (const std::length_error&) {
+            handler = g_newHandler.load();
+            if (handler == nullptr) throw std::bad_alloc();
         }
         handler();
     }
diff --git a/core/libs/tests/GuestNewHandler.cpp b/core/libs/tests/GuestNewHandler.cpp
index 898358f9e..cb1a0364e 100644
--- a/core/libs/tests/GuestNewHandler.cpp
+++ b/core/libs/tests/GuestNewHandler.cpp
@@ -4,7 +4,10 @@
 #include <cstddef>
 #include <cstdio>
 #include <cstdlib>
+#include <cstring>
+#include <limits>
 #include <new>
+#include <stdexcept>
 
 extern "C" {
 using Handler = void (APS5_VABI*)();
@@ -14,6 +17,7 @@ void* APS5_VABI _Znwm_nid_postfix(std::size_t);
 void* APS5_VABI _Znam_nid_postfix(std::size_t);
 void* APS5_VABI _ZnwmRKSt9nothrow_t_nid_postfix(std::size_t, const void*) noexcept;
 void* APS5_VABI _ZnamRKSt9nothrow_t_nid_postfix(std::size_t, const void*) noexcept;
+void APS5_VABI _ZdlPv_nid_postfix(void*);
 }
 
 namespace {
@@ -85,7 +89,37 @@ bool Throws(TAction action) {
 
 }
 
-int main() {
+int RunDefaultHeap() {
+    std::array<void*, 10> none{};
+    ApplicationHeapRegister_nid_no_patch(none.data());
+    constexpr std::size_t huge = std::numeric_limits<std::size_t>::max();
+
+    REQUIRE(Throws([] { _Znwm_nid_postfix(huge); }));
+    REQUIRE(Throws([] { _Znam_nid_postfix(huge); }));
+    REQUIRE(_ZnwmRKSt9nothrow_t_nid_postfix(huge, nullptr) == nullptr);
+    REQUIRE(_ZnamRKSt9nothrow_t_nid_postfix(huge, nullptr) == nullptr);
+
+    handlerCalls = 0;
+    REQUIRE(_ZSt15set_new_handlerPFvvE_nid_postfix(&uninstallingHandler) == nullptr);
+    REQUIRE(Throws([] { _Znwm_nid_postfix(huge); }));
+    REQUIRE(handlerCalls == 3 && _ZSt15get_new_handlerv_nid_postfix() == nullptr);
+
+    handlerCalls = 0;
+    REQUIRE(_ZSt15set_new_handlerPFvvE_nid_postfix(&throwingHandler) == nullptr);
+    REQUIRE(_ZnwmRKSt9nothrow_t_nid_postfix(huge, nullptr) == nullptr && handlerCalls == 1);
+    REQUIRE(_ZSt15set_new_handlerPFvvE_nid_postfix(nullptr) == &throwingHandler);
+
+    void* small = _Znwm_nid_postfix(24);
+    REQUIRE(small != nullptr);
+    std::memset(small, 0x5a, 24);
+    _ZdlPv_nid_postfix(small);
+
+    std::puts("Guest new handler default heap checks passed");
+    return 0;
+}
+
+int main(int argc, char** argv) {
+    if (argc > 1 && std::strcmp(argv[1], "default") == 0) return RunDefaultHeap();
     std::array<void*, 10> api{};
     api[0] = reinterpret_cast<void*>(&allocate);
     api[1] = reinterpret_cast<void*>(&release);