fix(videoout): read the open param as the service thread's priority and affinity

and throws for any non-zero word after it. Its layout matches shadPS4's
ServiceThreadParams instead: an unknown first word (16), a set-priority flag
and a priority (256..767), a set-affinity flag and a 64-bit CPU mask at
offset 16. PPSA21564 passes 16, 1, 0x100, 1 and 0x1FFF, so it stops at
startup on main.

sceVideoOutOpen now validates each field and reads the mask only when its
flag is set, so a 16-byte param with both flags clear still works. The
priority and affinity are not applied, since the present and vblank threads
are host threads; this and the unknown first word are listed in the
technical debt.

core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp

@@ -4,6 +4,7 @@
 #include <array>
 #include <chrono>
 #include <condition_variable>
+#include <cstddef>
 #include <cstdint>
 #include <list>
 #include <mutex>
@@ -37,10 +38,23 @@ static constexpr int VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE = -2144796647;
 static constexpr int VIDEO_OUT_ERROR_INVALID_EVENT = -2144796659;
 
 static constexpr int VIDEO_OUT_BUS_TYPE_MAIN = 0;
-static constexpr std::uint32_t VIDEO_OUT_OPEN_PARAM_SIZE = 16;
 static constexpr int VIDEO_OUT_BUS_TYPE_OVERLAY = 1;
 static constexpr int VIDEO_OUT_BUS_TYPE_SUB = 2;
 
+static constexpr std::uint32_t VIDEO_OUT_OPEN_PARAM_FIRST_WORD = 16;
+static constexpr std::int32_t VIDEO_OUT_SERVICE_THREAD_PRIORITY_HIGHEST = 256;
+static constexpr std::int32_t VIDEO_OUT_SERVICE_THREAD_PRIORITY_LOWEST = 767;
+static constexpr std::uint64_t VIDEO_OUT_SERVICE_THREAD_AFFINITY_ALL = 0x1FFF;
+
+struct VideoOutOpenParam {
+    std::uint32_t firstWord;
+    std::uint32_t setPriority;
+    std::int32_t priority;
+    std::uint32_t setAffinity;
+    std::uint64_t affinity;
+};
+static_assert(offsetof(VideoOutOpenParam, affinity) == 16);
+
 static constexpr int VIDEO_OUT_BUFFER_NUM_MAX = 16;
 static constexpr int VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX = 4;
 static constexpr int VIDEO_OUT_NUM_MAX = 4;

core/libs/prx/libSceVideoOut/src/Output.cpp

@@ -1,5 +1,6 @@
 #include <algorithm>
 #include <array>
+#include <cstddef>
 #include <cstdint>
 #include <cstring>
 #include <mutex>
@@ -30,19 +31,31 @@ static int validateOutputConfig(int handle, uint64_t mode, const VideoOutOutputO
     return 0;
 }
 
+static void validateOpenParam(const void* param) {
+    if (param == nullptr) return;
+    VideoOutOpenParam openParam{};
+    std::memcpy(&openParam, param, offsetof(VideoOutOpenParam, affinity));
+    if (openParam.firstWord != VIDEO_OUT_OPEN_PARAM_FIRST_WORD) {
+        throw std::runtime_error(std::string(__func__) + ": unsupported first word");
+    }
+    if (openParam.setPriority > 1 || openParam.setAffinity > 1) {
+        throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_VALUE");
+    }
+    if (openParam.setPriority == 1 && (openParam.priority < VIDEO_OUT_SERVICE_THREAD_PRIORITY_HIGHEST || openParam.priority > VIDEO_OUT_SERVICE_THREAD_PRIORITY_LOWEST)) {
+        throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_VALUE");
+    }
+    if (openParam.setAffinity == 1) {
+        std::memcpy(&openParam.affinity, static_cast<const std::byte*>(param) + offsetof(VideoOutOpenParam, affinity), sizeof(openParam.affinity));
+        if (openParam.affinity == 0 || (openParam.affinity & ~VIDEO_OUT_SERVICE_THREAD_AFFINITY_ALL) != 0) {
+            throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_VALUE");
+        }
+    }
+}
+
 extern "C" {
 
 int APS5_VABI sceVideoOutOpen(int userId, int busType, int index, const void* param) try {
-    if (param != nullptr) {
-        std::array<std::uint32_t, VIDEO_OUT_OPEN_PARAM_SIZE / sizeof(std::uint32_t)> words{};
-        std::memcpy(words.data(), param, sizeof(words));
-        if (words[0] != VIDEO_OUT_OPEN_PARAM_SIZE) {
-            throw std::runtime_error(std::string(__func__) + ": unsupported param size");
-        }
-        if (std::any_of(words.begin() + 1, words.end(), [](std::uint32_t word) { return word != 0; })) {
-            throw std::runtime_error(std::string(__func__) + ": param options not implemented");
-        }
-    }
+    validateOpenParam(param);
     if (userId != 255 && userId != 0) {
         throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_VALUE");
     }

core/libs/prx/libSceVideoOut/tests/Flip.cpp

@@ -159,15 +159,37 @@ void testDecode() {
     expectFailure([&] { AgcDriver::DisplayBufferSize(buffer); });
 }
 
+void testOpenParam() {
+    using Words = std::array<std::uint32_t, 6>;
+    constexpr auto first = VIDEO_OUT_OPEN_PARAM_FIRST_WORD;
+    for (const Words& rejected : {
+             Words{24, 0, 0, 0, 0, 0},
+             Words{first, 2, 0, 0, 0, 0},
+             Words{first, 0, 0, 0x101, 0, 0},
+             Words{first, 1, 255, 0, 0, 0},
+             Words{first, 1, 768, 0, 0, 0},
+             Words{first, 0, 0, 1, 0, 0},
+             Words{first, 0, 0, 1, 0x2000, 0},
+             Words{first, 0, 0, 1, 0, 1},
+         }) {
+        expectFailure([&] { sceVideoOutOpen(255, 0, 0, rejected.data()); });
+    }
+    std::array<std::uint32_t, 4> zeroed{first, 0, 0, 0};
+    const auto zeroedHandle = sceVideoOutOpen(255, 0, 0, zeroed.data());
+    check(zeroedHandle >= 0, "open with a zeroed param failed");
+    sceVideoOutClose(zeroedHandle);
+    for (const Words& accepted : {Words{first, 1, 767, 0, 0, 0}, Words{first, 0, 0, 1, 1, 0}}) {
+        const auto handle = sceVideoOutOpen(255, 0, 0, accepted.data());
+        check(handle >= 0, "open with a valid service thread setting failed");
+        sceVideoOutClose(handle);
+    }
+}
+
 void testControls() {
-    std::array<std::uint32_t, 4> openParam{VIDEO_OUT_OPEN_PARAM_SIZE, 0, 0, 0};
+    testOpenParam();
+    const std::array<std::uint32_t, 22> openParam{VIDEO_OUT_OPEN_PARAM_FIRST_WORD, 1, 0x100, 1, 0x1FFF};
     const auto handle = sceVideoOutOpen(255, 0, 0, openParam.data());
-    check(handle >= 0, "open with a zeroed param failed");
-    openParam[0] = 24;
-    expectFailure([&] { sceVideoOutOpen(255, 0, 0, openParam.data()); });
-    openParam[0] = VIDEO_OUT_OPEN_PARAM_SIZE;
-    openParam[2] = 1;
-    expectFailure([&] { sceVideoOutOpen(255, 0, 0, openParam.data()); });
+    check(handle >= 0, "open with the highest priority on every CPU failed");
     const auto cfg = VideoOutDriver::Get().GetConfig(handle);
     for (int rate = 0; rate <= 2; ++rate) {
         check(sceVideoOutSetFlipRate(handle, rate) == 0 && cfg->flipRate == rate, "flip rate was not applied");

docs/dev/TechnicalDebt.md

@@ -15,10 +15,12 @@ Throughout the project, every function at every stage either **does exactly what
 - [libSceCommonDialog](../../core/libs/prx/libSceCommonDialog/Export.cpp)
 - [libSceHmd](../../core/libs/prx/libSceHmd/Export.cpp) implements only the disconnected-headset path: initialization succeeds, device queries report `NotDetected`, and opening a device returns `DeviceDisconnected`. Headset support, tracking, rendering and additional HMD exports are not implemented; SDK-level ABI compatibility and in-game behaviour remain unverified.
 - [libSceNpCommerce](../../core/libs/prx/libSceNpCommerce/Export.cpp) - the PS Store icon show/hide calls do nothing
+- [sceVideoOutOpen](../../core/libs/prx/libSceVideoOut/src/Output.cpp) validates the priority and CPU affinity that the open param requests for the VideoOut service thread but does not apply them: the port's present and vblank threads are host threads, and guest priorities and affinities do not reach host scheduling
 - The shader recompiler [skips baryctric coordinates](../../core/shader/recompiler/Recompiler.cpp) (is not even passed to SpirvTargetOptions at row 212).
 
 ### Unknown function info
 
+- [sceVideoOutOpen](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - the open param's first word is unknown (PPSA21564 passes 16; it is not the byte size, since the affinity mask is at offset 16); only 16 is accepted. Whether the param continues past offset 24 is also unknown
 - [AMPR WaitOnAddress / WaitOnCounter](../../core/libs/prx/libkernel/Apr/src/Apr.cpp) (libSceAmpr) - compare encoding assumed to be the WAIT_REG_MEM one (0 always, 1 <, 2 <=, 3 ==, 4 !=, 5 >=, 6 >)
 - [sceAgcSetSubmitMode](../../core/libs/prx/libSceAgc/Misc/src/Suspend.cpp) (libSceAgc) - mode values unknown; only 0 is accepted
 - [zARR5aCmkoY](../../core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp) (libSceAgc) - unknown name, signature