From cadcbfd3a8533e4db7c360277f379b1f169bb5db Mon Sep 17 00:00:00 2001 From: hugodearaujo Date: Wed, 7 Oct 2026 23:53:33 -0300 Subject: [PATCH] fix(videoout): answer output mode queries with the console's codes Measured on a PS5 (firmware 13.60) through dlopen of libSceVideoOut.sprx on a handle the payload opened. sceVideoOutInitializeOutputOptions writes 0xFF to byte 2 of the 64-byte options and returns INVALID_ADDRESS for a null pointer. sceVideoOutIsOutputSupported checks the handle, then the reserved arguments, then the mode, then the options. It never answers 0: a mode the display lacks returns UNSUPPORTED_OUTPUT_MODE, and a value that is not a mode returns 0x8029001E. Options must keep the initialized words, except word 3. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceVideoOut/CMakeLists.txt | 13 +++ .../libSceVideoOut/include/VideoOutDriver.hpp | 2 + core/libs/prx/libSceVideoOut/src/Output.cpp | 36 +++++---- .../prx/libSceVideoOut/src/VideoOutDriver.cpp | 11 +++ core/libs/prx/libSceVideoOut/tests/Flip.cpp | 8 +- core/libs/tests/GuestVideoOutOutputMode.cpp | 81 +++++++++++++++++++ docs/dev/TechnicalDebt.md | 2 +- 7 files changed, 132 insertions(+), 21 deletions(-) create mode 100644 core/libs/tests/GuestVideoOutOutputMode.cpp diff --git a/core/libs/prx/libSceVideoOut/CMakeLists.txt b/core/libs/prx/libSceVideoOut/CMakeLists.txt index 5f3b3e9ce..53c76143a 100644 --- a/core/libs/prx/libSceVideoOut/CMakeLists.txt +++ b/core/libs/prx/libSceVideoOut/CMakeLists.txt @@ -83,6 +83,19 @@ if(BUILD_TESTING) endif() endif() +if(BUILD_TESTING) + add_test_executable(guest_video_out_output_mode_tests ${LIBS_INCLUDE_DIR}/tests/GuestVideoOutOutputMode.cpp) + target_include_directories(guest_video_out_output_mode_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_video_out_output_mode_tests PRIVATE libSceVideoOut libSceAgcDriver libkernel libc) + configure_windows_unwind(guest_video_out_output_mode_tests) + add_test(NAME guest_video_out_output_mode COMMAND guest_video_out_output_mode_tests) + set_tests_properties(guest_video_out_output_mode PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_video_out_output_mode PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + if(BUILD_TESTING) add_test_executable(guest_buffer_reuse_tests ${LIBS_INCLUDE_DIR}/tests/GuestBufferReuse.cpp) target_include_directories(guest_buffer_reuse_tests PRIVATE ${LIBS_INCLUDE_DIR}) diff --git a/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp b/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp index afd66844c..cc7bcfd47 100644 --- a/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp +++ b/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp @@ -37,6 +37,7 @@ static constexpr int VIDEO_OUT_ERROR_FLIP_QUEUE_FULL = -2144796654; static constexpr int VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE = -2144796650; static constexpr int VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE = -2144796647; static constexpr int VIDEO_OUT_ERROR_INVALID_EVENT = -2144796659; +static constexpr int VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE = -2144796642; static constexpr int VIDEO_OUT_BUS_TYPE_MAIN = 0; static constexpr int VIDEO_OUT_BUS_TYPE_OVERLAY = 1; @@ -201,6 +202,7 @@ public: bool Close(int handle); std::shared_ptr GetConfig(int handle); bool IsOpen(int handle); + bool HasConfig(int handle); // 0, or VIDEO_OUT_ERROR_FLIP_QUEUE_FULL when the title has VIDEO_OUT_FLIP_QUEUE_CAPACITY flips pending. int SubmitFlip(int handle, int index, int flipMode, int64_t flipArg); diff --git a/core/libs/prx/libSceVideoOut/src/Output.cpp b/core/libs/prx/libSceVideoOut/src/Output.cpp index bf4f0ff4d..9608feb97 100644 --- a/core/libs/prx/libSceVideoOut/src/Output.cpp +++ b/core/libs/prx/libSceVideoOut/src/Output.cpp @@ -11,18 +11,26 @@ #include "prx/libc/include/General.hpp" #include "prx/libSceVideoOut/include/VideoOutDriver.hpp" +static constexpr std::uint32_t VIDEO_OUT_OUTPUT_OPTIONS_FIRST_WORD = 0x00FF0000; +static constexpr std::size_t VIDEO_OUT_OUTPUT_OPTIONS_FREE_WORD = 3; + static int validateOutputConfig(int handle, uint64_t mode, const VideoOutOutputOptions* options, void* reservedPtr, uint64_t reserved) { - if (!VideoOutDriver::Get().IsOpen(handle)) { - throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); - } - if (reservedPtr != nullptr || reserved != 0) { - throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_VALUE"); + if (!VideoOutDriver::Get().HasConfig(handle) || !VideoOutDriver::Get().IsOpen(handle)) return VIDEO_OUT_ERROR_INVALID_HANDLE; + if (reservedPtr != nullptr || reserved != 0) return VIDEO_OUT_ERROR_INVALID_VALUE; + switch (mode) { + case VIDEO_OUT_OUTPUT_MODE_DEFAULT: + break; + case 0x4: case 0x7: case 0x8: case 0xC: case 0xD: case 0xE: case VIDEO_OUT_OUTPUT_MODE_119_88HZ: case 0x10: case 0x11: case 0x13: + return VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE; + case 0x12: + throw std::runtime_error(std::string(__func__) + ": output mode 0x12 answers differently on two consoles"); + default: + return VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE; } if (options != nullptr) { - for (auto v : options->internalData) { - if (v != 0) { - throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_OPTION"); - } + for (std::size_t index = 0; index < std::size(options->internalData); ++index) { + const std::uint32_t expected = index == 0 ? VIDEO_OUT_OUTPUT_OPTIONS_FIRST_WORD : 0; + if (index != VIDEO_OUT_OUTPUT_OPTIONS_FREE_WORD && options->internalData[index] != expected) return VIDEO_OUT_ERROR_INVALID_VALUE; } } return 0; @@ -205,10 +213,9 @@ int APS5_VABI sceVideoOutWaitVblank(int handle) try { } int APS5_VABI sceVideoOutInitializeOutputOptions(VideoOutOutputOptions* options) { - if (options == nullptr) { - throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_ADDRESS"); - } + if (options == nullptr) return VIDEO_OUT_ERROR_INVALID_ADDRESS; std::memset(options, 0, sizeof(VideoOutOutputOptions)); + options->internalData[0] = VIDEO_OUT_OUTPUT_OPTIONS_FIRST_WORD; return 0; } @@ -217,7 +224,7 @@ int APS5_VABI sceVideoOutIsOutputSupported(int handle, uint64_t mode, const Vide if (result != 0) { return result; } - return mode == VIDEO_OUT_OUTPUT_MODE_DEFAULT ? 1 : 0; + return 1; } catch (const ProcessShutdown&) { LibcAwaitExit_nid_postfix(); } @@ -227,9 +234,6 @@ int APS5_VABI sceVideoOutConfigureOutput(int handle, uint64_t mode, const VideoO if (supported < 0) { return supported; } - if (supported == 0) { - return VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE; - } auto cfg = VideoOutDriver::Get().GetConfig(handle); if (cfg == nullptr) { throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); diff --git a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp index cb47a1528..4d590926f 100644 --- a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp +++ b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp @@ -371,6 +371,17 @@ bool VideoOutDriver::IsOpen(int handle) { return GetConfig(handle) != nullptr; } +bool VideoOutDriver::HasConfig(int handle) { + std::shared_ptr cfg; + { + std::lock_guard lock(mutex); + if (handle <= 0 || handle >= VIDEO_OUT_NUM_MAX || contexts[handle] == nullptr) return false; + cfg = contexts[handle]; + } + std::lock_guard cfgLock(cfg->mutex); + return cfg->opened && !cfg->closing; +} + int VideoOutDriver::SubmitFlip(int handle, int index, int flipMode, int64_t flipArg) { if (LibcShutdownToken_nid_postfix().stop_requested()) throw ProcessShutdown{}; // A title that does not pace on flipPendingNum can run ahead of the presenter now that the queue diff --git a/core/libs/prx/libSceVideoOut/tests/Flip.cpp b/core/libs/prx/libSceVideoOut/tests/Flip.cpp index 7b0522c1f..dc3d59fdc 100644 --- a/core/libs/prx/libSceVideoOut/tests/Flip.cpp +++ b/core/libs/prx/libSceVideoOut/tests/Flip.cpp @@ -235,10 +235,10 @@ void testControls() { check(handle >= 0, "open with the highest priority on every CPU failed"); const auto cfg = VideoOutDriver::Get().GetConfig(handle); check(sceVideoOutIsOutputSupported(handle, VIDEO_OUT_OUTPUT_MODE_DEFAULT, nullptr, nullptr, 0) == 1, "default output mode is unsupported"); - for (const uint64_t unsupported : std::initializer_list{VIDEO_OUT_OUTPUT_MODE_119_88HZ, 0xd000000aull}) { - check(sceVideoOutIsOutputSupported(handle, unsupported, nullptr, nullptr, 0) == 0, "unavailable output mode is supported"); - check(sceVideoOutConfigureOutput(handle, unsupported, nullptr, nullptr, 0) == VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE, "unavailable output mode was configured"); - } + check(sceVideoOutIsOutputSupported(handle, VIDEO_OUT_OUTPUT_MODE_119_88HZ, nullptr, nullptr, 0) == VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE, "unavailable output mode is supported"); + check(sceVideoOutConfigureOutput(handle, VIDEO_OUT_OUTPUT_MODE_119_88HZ, nullptr, nullptr, 0) == VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE, "unavailable output mode was configured"); + check(sceVideoOutIsOutputSupported(handle, 0xd000000aull, nullptr, nullptr, 0) == VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE, "unknown output mode is supported"); + check(sceVideoOutConfigureOutput(handle, 0xd000000aull, nullptr, nullptr, 0) == VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE, "unknown output mode was configured"); check(sceVideoOutConfigureOutput(handle, VIDEO_OUT_OUTPUT_MODE_DEFAULT, nullptr, nullptr, 0) == 0, "default output mode was rejected"); for (int rate = 0; rate <= 2; ++rate) { check(sceVideoOutSetFlipRate(handle, rate) == 0 && cfg->flipRate == rate, "flip rate was not applied"); diff --git a/core/libs/tests/GuestVideoOutOutputMode.cpp b/core/libs/tests/GuestVideoOutOutputMode.cpp new file mode 100644 index 000000000..6762cd446 --- /dev/null +++ b/core/libs/tests/GuestVideoOutOutputMode.cpp @@ -0,0 +1,81 @@ +#include "SceTypes.hpp" +#include "prx/libc/include/Shutdown.hpp" +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceVideoOutOpen(int userId, int busType, int index, const void* param); +int APS5_VABI sceVideoOutClose(int handle); +int APS5_VABI sceVideoOutInitializeOutputOptions(VideoOutOutputOptions* options); +int APS5_VABI sceVideoOutIsOutputSupported(int handle, std::uint64_t mode, const VideoOutOutputOptions* options, void* reservedPtr, std::uint64_t reserved); +int APS5_VABI sceVideoOutConfigureOutput(int handle, std::uint64_t mode, const VideoOutOutputOptions* options, void* reservedPtr, std::uint64_t reserved); +} + +static constexpr int SYSTEM_USER = 255; +static constexpr int MAIN_BUS = 0; +static constexpr int INVALID_VALUE = static_cast(0x80290001); +static constexpr int INVALID_ADDRESS = static_cast(0x80290002); +static constexpr int INVALID_HANDLE = static_cast(0x8029000B); +static constexpr int UNSUPPORTED_OUTPUT_MODE = static_cast(0x80290016); +static constexpr int UNKNOWN_OUTPUT_MODE = static_cast(0x8029001E); + +static void Require(bool value) { if (!value) std::abort(); } + +int main() { + std::array raw{}; + raw.fill(0xAA); + VideoOutOutputOptions options{}; + std::memcpy(&options, raw.data(), raw.size()); + Require(sceVideoOutInitializeOutputOptions(&options) == 0); + std::memcpy(raw.data(), &options, raw.size()); + for (std::size_t index = 0; index < raw.size(); ++index) Require(raw[index] == (index == 2 ? 0xFF : 0)); + Require(sceVideoOutInitializeOutputOptions(nullptr) == INVALID_ADDRESS); + + for (int handle : {-1, 0, 1, 0x7FFFFFFF}) { + Require(sceVideoOutIsOutputSupported(handle, 1, &options, nullptr, 0) == INVALID_HANDLE); + Require(sceVideoOutIsOutputSupported(handle, 2, nullptr, &options, 1) == INVALID_HANDLE); + } + + const int handle = sceVideoOutOpen(SYSTEM_USER, MAIN_BUS, 0, nullptr); + Require(handle > 0); + Require(sceVideoOutIsOutputSupported(handle, 1, &options, nullptr, 0) == 1); + Require(sceVideoOutIsOutputSupported(handle, 1, nullptr, nullptr, 0) == 1); + for (std::uint64_t mode : {0x4ull, 0x7ull, 0x8ull, 0xCull, 0xDull, 0xEull, 0xFull, 0x10ull, 0x11ull, 0x13ull}) { + Require(sceVideoOutIsOutputSupported(handle, mode, &options, nullptr, 0) == UNSUPPORTED_OUTPUT_MODE); + Require(sceVideoOutConfigureOutput(handle, mode, &options, nullptr, 0) == UNSUPPORTED_OUTPUT_MODE); + } + for (std::uint64_t mode : {0x0ull, 0x2ull, 0x3ull, 0x5ull, 0x6ull, 0x9ull, 0xAull, 0xBull, 0x14ull, 0x20ull, 0x3Full, 0x100ull, 0xD000000Aull, 0x8000000000000001ull, ~0ull}) { + Require(sceVideoOutIsOutputSupported(handle, mode, &options, nullptr, 0) == UNKNOWN_OUTPUT_MODE); + } + bool thrown = false; + try { + sceVideoOutIsOutputSupported(handle, 0x12, &options, nullptr, 0); + } catch (const std::runtime_error&) { + thrown = true; + } + Require(thrown); + + Require(sceVideoOutIsOutputSupported(handle, 1, &options, &options, 0) == INVALID_VALUE); + Require(sceVideoOutIsOutputSupported(handle, 1, &options, nullptr, 1) == INVALID_VALUE); + Require(sceVideoOutIsOutputSupported(handle, 2, &options, nullptr, 1) == INVALID_VALUE); + VideoOutOutputOptions zeroed{}; + Require(sceVideoOutIsOutputSupported(handle, 1, &zeroed, nullptr, 0) == INVALID_VALUE); + Require(sceVideoOutIsOutputSupported(handle, 2, &zeroed, nullptr, 0) == UNKNOWN_OUTPUT_MODE); + Require(sceVideoOutIsOutputSupported(handle, 4, &zeroed, nullptr, 0) == UNSUPPORTED_OUTPUT_MODE); + for (std::size_t word = 0; word < 16; ++word) { + VideoOutOutputOptions changed = options; + changed.internalData[word] ^= 1; + Require(sceVideoOutIsOutputSupported(handle, 1, &changed, nullptr, 0) == (word == 3 ? 1 : INVALID_VALUE)); + } + VideoOutOutputOptions freeWord = options; + freeWord.internalData[3] = 0xFFFFFFFF; + Require(sceVideoOutIsOutputSupported(handle, 1, &freeWord, nullptr, 0) == 1); + Require(sceVideoOutConfigureOutput(handle, 1, &options, nullptr, 0) == 0); + + Require(sceVideoOutClose(handle) == 0); + Require(sceVideoOutIsOutputSupported(handle, 1, &options, nullptr, 0) == INVALID_HANDLE); + LibcRunShutdown_nid_postfix(); +} diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 969f607b6..03d9e3d24 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -72,7 +72,7 @@ Throughout the project, every function at every stage either **does exactly what - [sceVideoOutRegisterBuffers2](../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - buffer option 8 (STRICT_COLORIMETRY) is presented as option 0, with the stored values unchanged; what the console changes for it is unknown. Only options 0 and 8 are accepted - [sceAgcGetRegisterDefaultsInternal](../../core/libs/prx/libSceAgc/Misc/src/RegisterDefaults.cpp) (libSceAgc) - returns the version 0 internal table, as `sceAgcGetRegisterDefaults` returns the version 0 public one; [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) returns the table of the version passed to `sceAgcInit` instead, and null before it - [sceVideoOutRegisterBuffers2](../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - DCC buffers (category 1): `dcc_control` is taken as the CB_DCC_CONTROL block layout, only bits 0x10026c are accepted; `dcc_cb_register_clear_color` as the register-clear texel, only 32-bit values are presented -- [sceVideoOutIsOutputSupported](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - output mode encoding unknown: every mode except the default reports not supported (0), and `sceVideoOutConfigureOutput` returns `VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE` for it, instead of returning `VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE` for values the console does not define. PPSA12544 probes `0xd000000a` and selects 7680 pixels when it is supported, 3840 otherwise +- [sceVideoOutIsOutputSupported](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - only the default mode (1) is reported as supported. The console answers per display and per title: with a 120 Hz TV it also answers 1 for mode 15 (119.88 Hz), and [PS5_Vulkan](https://github.com/mihawk-99/PS5_Vulkan/blob/5b5e4fc2d80fdb67a4f61ba9e6a8a424026bb8a5/docs/HARDWARE_FINDINGS.md) measured that `sceVideoOutConfigureOutput` accepts 15 only from a title whose param.json declares high-frame-rate output (attribute3 0x80040). Modes 4, 7, 8, 12-17 and 19 return `VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE` here, as they did on the measured display. Mode 18 throws, because one console answered 0x80290016 and another 0x8029001E. The name `VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE` for 0x8029001E is ours. Word 3 of the output options is accepted with any value, as on the console, and its meaning is unknown. PPSA12544 probes `0xd000000a`, which the console answers with 0x8029001E, so the title selects 3840 pixels - [sceVideoOutGetResolutionStatus](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - the 48-byte layout follows the declarations in ps5-opengl and EVO-PLAYER-PS5 (#1578), which agree; not confirmed on a console. `screenSizeInInch` is 0, the value EDID uses for an undefined size, since the host display's size is not read; `flags` is 0, its bits are unknown. The pane equals the full size because `sceVideoOutSetWindowModeMargins` applies no margins - [sceVideoOutOpen](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - the open param's first word is unknown (PPSA21564 passes 16; it is not the byte size, since the affinity mask is at offset 16); only 16 is accepted. Whether the param continues past offset 24 is also unknown - [s_setkill](../../core/shader/recompiler/RdnaDecoder/src/RdnaScalarOpDecoder.cpp) (core/shader/recompiler) - decoded as `s_endpgm` for any immediate: measured on RDNA2 hardware (gfx1035), a store before it is visible and nothing after it runs ([#771](https://github.com/boykopovar/AnyPS5/pull/771)); whether the KILL status bit it sets is observable differently from `s_endpgm` is unverified