Merge branch 'main' into fix/videoout-high-frame-rate

.github/pull_request_template.md

@@ -13,4 +13,5 @@
 - [ ] New third-party code is a submodule built from source
 - [ ] Generic behaviour, not specific to one title
 - [ ] Depends on: <!-- #PR, or none -->
+- [ ] Unblocks: <!-- title ID and Game bring-up discussion, e.g. PPSA01325 #3232, or none -->
 - [ ] AI-assisted: <!-- yes / no -->

.github/workflows/build.yml

@@ -39,6 +39,8 @@ jobs:
       pr: ${{ steps.gate.outputs.pr }}
       head: ${{ steps.gate.outputs.head }}
       relinker: ${{ steps.gate.outputs.relinker }}
+      register-context: ${{ steps.gate.outputs.register-context }}
+      write-watch: ${{ steps.gate.outputs.write-watch }}
       full: ${{ steps.gate.outputs.full }}
     steps:
       - uses: actions/github-script@v8
@@ -55,6 +57,8 @@ jobs:
               core.setOutput('pr', '');
               core.setOutput('head', '');
               core.setOutput('relinker', true);
+              core.setOutput('register-context', true);
+              core.setOutput('write-watch', true);
               core.setOutput('full', true);
               await core.summary.addRaw(`Building main at ${context.sha}: all builds and tests enabled`).write();
               return;
@@ -97,16 +101,32 @@ jobs:
               });
               relinker = commits.length > 0;
             }
+            const files = await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number, per_page: 100 });
+            const paths = files.flatMap((file) => [file.filename, file.previous_filename].filter(Boolean));
+            const touches = (...prefixes) => manual || files.length >= 3000 ||
+              paths.some((path) => prefixes.some((prefix) => prefix.endsWith('/') ? path.startsWith(prefix) : path === prefix));
+            const workflow = '.github/workflows/build.yml';
+            const libc = 'core/libs/prx/libc/';
+            relinker = relinker && touches(
+              'core/relinker/', libc, 'core/libs/prx/libkernel/AppMetadata/', 'core/libs/tests/ParamJsonParser.cpp',
+              'core/shader/recompiler/tests/CacheFileReadTests.cpp', 'core/shader/recompiler/ShaderCacheDirectory.hpp',
+              'tools/', 'cmake/', 'CMakeLists.txt', workflow
+            );
+            const registerContext = touches('core/libs/tests/register_context/', 'core/libs/tests/RegisterContext.cpp', libc, workflow);
+            const writeWatch = touches('core/libs/tests/write_watch/', 'core/libs/tests/WriteWatch.cpp', libc, workflow);
             const { data: main } = await github.rest.repos.getBranch({ owner, repo, branch: 'main' });
             core.setOutput('sha', main.commit.sha);
             core.setOutput('pr', String(pull_number));
             core.setOutput('head', pr.head.sha);
             core.setOutput('relinker', relinker);
+            core.setOutput('register-context', registerContext);
+            core.setOutput('write-watch', writeWatch);
             core.setOutput('full', manual);
             core.info(`PR #${pull_number} at ${pr.head.sha} merged into main at ${main.commit.sha}`);
             await core.summary.addRaw(
               `PR #${pull_number} at ${pr.head.sha} merged into main at ${main.commit.sha}: full builds ${manual ? 'enabled' : 'disabled'}, ` +
-              `relinker ${relinker ? 'enabled' : 'skipped because the author has no commit in main'}`
+              `relinker ${relinker ? 'enabled' : 'skipped: no relinker file changed or the author has no commit in main'}, ` +
+              `register context tests ${registerContext ? 'enabled' : 'skipped'}, write watch tests ${writeWatch ? 'enabled' : 'skipped'}`
             ).write();
 
   relinker:
@@ -162,6 +182,8 @@ jobs:
       - run: ctest --test-dir build-relinker --output-on-failure --timeout 120
 
   register-context:
+    needs: eligibility
+    if: ${{ needs.eligibility.outputs.register-context == 'true' }}
     strategy:
       fail-fast: false
       matrix:
@@ -176,6 +198,8 @@ jobs:
       - run: ctest --test-dir build/register-context --output-on-failure --repeat until-fail:3 --no-tests=error
 
   write-watch:
+    needs: eligibility
+    if: ${{ needs.eligibility.outputs.write-watch == 'true' }}
     strategy:
       fail-fast: false
       matrix:

CMakeLists.txt

@@ -66,10 +66,16 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress.py)
         add_test(NAME progress_encoding COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_encoding.py)
+        add_test(NAME progress_render_sliver COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_render_sliver.py)
         add_test(NAME import_audit COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py)
         add_test(NAME conventions_function_bodies COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_comments.py)
+        add_test(NAME conventions_wrapped_declarations COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_wrapped_declarations.py)
+        add_test(NAME conventions_ci_gate COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_ci_gate.py)
         add_test(NAME test_registration COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_test_registration.py)
         add_test(NAME check_skips COMMAND ${Python3_EXECUTABLE}
@@ -80,6 +86,8 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/hw-oracle/test_hw_oracle.py)
         add_test(NAME pr_overlap_exports COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_exports.py)
+        add_test(NAME pr_overlap_wrapped_exports COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_wrapped_exports.py)
         add_test(NAME nid_database_cache COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_nid_names.py)
         add_test(NAME release_package COMMAND ${Python3_EXECUTABLE}

CONTRIBUTING.md

@@ -37,7 +37,7 @@ For pull requests, full Linux and Windows builds and tests run only when a colla
 gh workflow run build.yml --repo boykopovar/AnyPS5 --ref main --field pr=1234
 ```
 
-This manual run also builds and tests the relinker on Linux, Windows and macOS, regardless of the author's previous contributions or the changed files. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main`; full builds never run automatically for pull requests. All build jobs build the selected pull request merged into the current `main` (GitHub's merge commit), so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds.
+This manual run also builds and tests the relinker on Linux, Windows and macOS, regardless of the author's previous contributions or the changed files. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main` and it changes a file the relinker-only build uses; full builds never run automatically for pull requests. Every job that runs automatically for pull requests runs only when the pull request changes its files: a `paths` filter on the `pull_request` trigger, or `needs: eligibility` with an `if` on the files output it sets in `build.yml`. The Conventions check fails a workflow change that adds a job without one. All build jobs build the selected pull request merged into the current `main` (GitHub's merge commit), so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds.
 
 The Conventions check runs automatically for every contributor, including first-time contributors. It uses the base branch checker and reads pull request Git objects without executing pull request code. The Conventions check runs on every pull request and fails when a rule on this page is broken. It accepts code comments only when the pull request also changes [TechnicalDebt](docs/dev/TechnicalDebt.md), and only UTF-8 text files. Run it locally before pushing:
 
@@ -54,6 +54,9 @@ The [pull request template](.github/pull_request_template.md) is the checklist f
 - Before starting, check that no open pull request already implements the same functions.
 - Keep the branch up to date with `main` and resolve conflicts yourself; rebasing and force-pushing is fine.
 - If a pull request needs another one first, say so on the checklist line (`Depends on: #N`; `none` otherwise).
+- If a pull request fixes a stop or a wrong result seen on a title, name it on the checklist line with the title ID and its [Game bring-up](https://github.com/boykopovar/AnyPS5/discussions/categories/game-bring-up) discussion (`Unblocks: PPSA01325 #3232`; `none` otherwise). Reviewers review these first.
+- Fixes a title needs go to `main` as pull requests. A Game bring-up discussion lists the open pull requests and local changes the title still needs; update the list when one is merged, closed or replaced.
+- The PR overlap comment lists open pull requests that change the same lines or exports; agree with their authors which one goes first and declare it with `Depends on`.
 - Run the tests before opening the pull request and describe what was tested (OS, title or homebrew).
 - Investigation notes, reports, screenshots and logs go in the pull request, not in the repository. Images for documentation go in the [gist](https://gist.github.com/boykopovar/0e53f2e1426f29ecd41e3b51540b8a90) comments.
 - Say whether the change was written with AI assistance. The author of the pull request is responsible for every line of it.

core/libs/CMakeLists.txt

@@ -26,6 +26,13 @@ target_compile_features(nid_patcher PRIVATE cxx_std_20)
 
 if(MINGW)
     target_link_options(nid_patcher PRIVATE -static -static-libgcc -static-libstdc++)
+    include(CheckCXXSourceCompiles)
+    set(CMAKE_REQUIRED_FLAGS -fno-asynchronous-unwind-tables)
+    check_cxx_source_compiles("#ifdef __USING_SJLJ_EXCEPTIONS__\n#error\n#endif\nint main() { return 0; }" ANYPS5_MINGW_DWARF_EXCEPTIONS)
+    unset(CMAKE_REQUIRED_FLAGS)
+    if(NOT ANYPS5_MINGW_DWARF_EXCEPTIONS)
+        message(FATAL_ERROR "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} selects SjLj exceptions with -fno-asynchronous-unwind-tables; the libraries need DWARF unwinding. Use the MinGW-w64 toolchain listed in docs/dev/BUILD.md.")
+    endif()
     add_compile_options(-fno-asynchronous-unwind-tables)
 endif()
 
@@ -290,6 +297,11 @@ if(BUILD_TESTING)
     endif()
     add_test(NAME host_thread_local COMMAND host_thread_local_tests)
     set_tests_properties(host_thread_local PROPERTIES TIMEOUT 15)
+    if(MINGW)
+        add_test(NAME prx_thread_local_destructors COMMAND ${CMAKE_COMMAND} -DNM=${CMAKE_NM} -DOBJDUMP=${CMAKE_OBJDUMP}
+            -DMODULES=$<TARGET_FILE_DIR:libc> -P ${CMAKE_CURRENT_SOURCE_DIR}/tests/ThreadLocalDestructors.cmake)
+        set_tests_properties(prx_thread_local_destructors PROPERTIES TIMEOUT 300)
+    endif()
 
     add_test_executable(savedata_core_write_replacement_tests tests/SaveDataWriteReplacement.cpp)
     target_include_directories(savedata_core_write_replacement_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})

core/libs/SceTypes.hpp

@@ -1448,7 +1448,12 @@ struct SaveDataMountInfo {
 
 struct SceSaveDataTitleId { char data[10]; char pad[2]; };
 struct SceSaveDataDirName { char data[32]; };
-struct SaveDataSearchInfo { std::uint8_t opaque[128]; };
+struct SaveDataSearchInfo {
+    std::uint64_t blocks;
+    std::uint64_t free_blocks;
+    std::uint8_t reserved[32];
+};
+static_assert(sizeof(SaveDataSearchInfo) == 48);
 struct SaveDataMemoryData { void* buf; std::size_t buf_size; std::size_t offset; };
 
 struct SaveDataMount3 {
@@ -1571,6 +1576,13 @@ struct SaveDataBackup {
     std::uint8_t reserved[32];
 };
 
+struct SaveDataConvert {
+    std::int32_t user_id;
+    std::int32_t pad;
+    const SceSaveDataDirName* dir_name;
+    std::uint8_t unknown[48];
+};
+
 struct AppContentInitParam { char reserved[32]; };
 
 struct AppContentBootParam {
@@ -1590,6 +1602,19 @@ struct ContentExportInitParam2 {
     std::int64_t reserved1;
 };
 
+struct ContentExportParam {
+    char title[257];
+    char comment[257];
+    char content_type[65];
+};
+static_assert(sizeof(ContentExportParam) == 579);
+
+struct ContentExportCallbackParam {
+    std::size_t content_length;
+    void* func;
+    void* user_data;
+};
+
 struct ContentSearchInitParam { std::size_t memory_size; };
 
 struct PngEncCreateParam {

core/libs/prx/libSceAgc/Acb/src/Control.cpp

@@ -4,6 +4,7 @@
 #include "prx/libSceAgc/Command/include/Memory.hpp"
 #include "prx/libSceAgc/Command/include/Packet.hpp"
 #include "prx/libSceAgc/DcbState/include/Marker.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp"
 #include <cstdint>
 #include <cstddef>
 #include "SceTypes.hpp"
@@ -11,10 +12,15 @@
 
 extern "C" {
 
-// unknown signature
-void* APS5_VABI sceAgcAcbAtomicGds_0900(void) {
-    NotImplemented_nid_no_patch(__func__);
-    return nullptr;
+std::uint32_t* APS5_VABI sceAgcAcbAtomicGds_0900(CommandBuffer* buf, std::uint32_t control, std::uint32_t mode, std::uint32_t gdsOp, std::uint32_t src, std::uint16_t offset, std::uint16_t index, std::uint32_t mask, std::uint32_t data, std::uint64_t compareData, std::uint64_t extraData) {
+    const auto operation = (control & 0xffu) | ((control & 0x200u) << 7u) |
+        ((control & 0x20u) << 13u) | ((control & 0x40u) << 14u) |
+        ((control & 0x20u) == 0 && mode == 1 ? 0x20000u : 0u);
+    return Agc::Command::Emit(buf, 0x1du, {
+        operation, ((gdsOp & 1u) << 8u) | (src & 0x3fu), offset, index,
+        mask & 0x00ff00ffu, data, static_cast<std::uint32_t>(compareData),
+        static_cast<std::uint32_t>(compareData >> 32u), static_cast<std::uint32_t>(extraData),
+        static_cast<std::uint32_t>(extraData >> 32u)}, __func__);
 }
 
 std::uint32_t* APS5_VABI sceAgcAcbJump(CommandBuffer* buf, std::uint8_t cachePolicy, const std::uint32_t* target, std::uint32_t sizeInDwords) {
@@ -46,11 +52,12 @@ std::uint32_t APS5_VABI sceAgcAcbRewindGetSize() {
 }
 
 std::uint32_t* APS5_VABI sceAgcAcbWaitUntilSafeForRendering(CommandBuffer* buf, std::uint32_t videoOutHandle, std::uint32_t displayBufferIndex) {
-    (void)buf;
-    (void)videoOutHandle;
-    (void)displayBufferIndex;
-    NotImplemented_nid_no_patch(__func__);
-    return nullptr;
+    auto* packet = Agc::Command::Allocate(buf, AgcDriver::RenderingWaitPacketWords, __func__);
+    packet[0] = AgcDriver::RenderingWaitPacketHeader;
+    packet[1] = videoOutHandle;
+    packet[2] = displayBufferIndex;
+    packet[3] = 0;
+    return packet;
 }
 
 std::uint32_t* APS5_VABI sceAgcAcbSetFlip(CommandBuffer* buf, std::uint32_t videoOutHandle, std::int32_t displayBufferIndex, std::uint32_t flipMode, std::int64_t flipArg) {

core/libs/prx/libSceAgc/CMakeLists.txt

@@ -68,7 +68,7 @@ endforeach()
 
 add_test_executable(agc_command_tests EXCLUDE_FROM_ALL tests/Commands.cpp)
 target_include_directories(agc_command_tests PRIVATE ${LIBS_INCLUDE_DIR})
-target_link_libraries(agc_command_tests PRIVATE libSceAgc libc)
+target_link_libraries(agc_command_tests PRIVATE libSceAgc libSceAgcDriver libc)
 configure_windows_unwind(agc_command_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_command_tests PROPERTIES EXCLUDE_FROM_ALL OFF)

core/libs/prx/libSceAgc/DcbState/src/Workload.cpp

@@ -1,28 +1,59 @@
 #include "prx/libSceAgc/DcbState/include/Workload.hpp"
 
 #include "prx/libSceAgc/Command/include/Packet.hpp"
+#include "prx/libSceAgcDriver/Resource/include/Registration.hpp"
 #include <cstdint>
 #include <cstddef>
+#include <cstring>
 #include "SceTypes.hpp"
 #include "prx/libc/include/General.hpp"
 
+namespace {
+
+constexpr std::uint32_t OpcodeNop = 0x10;
+constexpr std::uint32_t ActivePacketWords = 18;
+constexpr std::uint32_t CompletePacketWords = 12;
+constexpr std::uint32_t MaxWorkloadId = 63;
+constexpr std::uint32_t MaxActiveCount = 63;
+
+uint32_t* NewPacket(CommandBuffer* buf, std::uint32_t words, const char* function) {
+ auto* packet = Agc::Command::Allocate(buf, words, function);
+ std::memset(packet, 0, words * sizeof(std::uint32_t));
+ packet[0] = Agc::Command::Header(OpcodeNop, words, 0);
+ return packet;
+}
+
+}
+
 extern "C" {
 
 uint32_t* APS5_VABI sceAgcDcbSetWorkloadComplete(CommandBuffer* buf, uint32_t stream_id, uint32_t workload_id) {
- (void)buf;
- (void)stream_id;
- (void)workload_id;
- NotImplemented_nid_no_patch(__func__);
- return nullptr;
+ if (buf == nullptr || workload_id > MaxWorkloadId || !AgcDriverWorkloadStreamRegistered_nid_postfix(stream_id)) return nullptr;
+ const std::uint64_t clearMask = ~(std::uint64_t{1} << workload_id);
+ auto* packet = NewPacket(buf, CompletePacketWords, __func__);
+ packet[1] = stream_id;
+ packet[2] = workload_id;
+ packet[3] = static_cast<std::uint32_t>(clearMask);
+ packet[4] = static_cast<std::uint32_t>(clearMask >> 32);
+ return packet;
 }
 
 uint32_t* APS5_VABI sceAgcDcbSetWorkloadsActive(CommandBuffer* buf, uint32_t stream_id, const uint32_t* workload_ids, uint32_t workload_count) {
- (void)buf;
- (void)stream_id;
- (void)workload_ids;
- (void)workload_count;
- NotImplemented_nid_no_patch(__func__);
- return nullptr;
+ if (buf == nullptr || workload_ids == nullptr || workload_count == 0 || workload_count > MaxActiveCount) return nullptr;
+ std::uint64_t mask = 0;
+ for (std::uint32_t index = 0; index < workload_count; ++index) {
+  const auto id = workload_ids[index];
+  if (id > MaxWorkloadId) return nullptr;
+  const std::uint64_t bit = std::uint64_t{1} << id;
+  if ((mask & bit) != 0) return nullptr;
+  mask |= bit;
+ }
+ if (!AgcDriverWorkloadStreamRegistered_nid_postfix(stream_id)) return nullptr;
+ auto* packet = NewPacket(buf, ActivePacketWords, __func__);
+ packet[1] = stream_id;
+ packet[2] = static_cast<std::uint32_t>(mask);
+ packet[3] = static_cast<std::uint32_t>(mask >> 32);
+ return packet;
 }
 
 }

core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp

@@ -53,10 +53,21 @@ std::uint32_t CreateInterpolantValueSplitF16(std::uint32_t psWord, const ShaderS
         : CreateInterpolantValue(psWord, gsSemantic);
 }
 
-int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader* gs, const Shader* ps, CreateInterpolantValueFn createValue) {
+std::uint32_t UpdateInterpolantValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) {
+    const std::uint32_t f16 = (psWord >> 20u) & 0x3u;
+    if (f16 != 0 && f16 != InterpolantF16HiOnly && ((psWord >> 28u) & 0x3u) != ((psWord >> 30u) & 0x3u)) {
+        throw std::runtime_error(std::string(__func__) + ": DEFAULT_VAL of an f16 input with different low and high defaults is unknown");
+    }
+    return CreateInterpolantValueSplitF16(psWord, gsSemantic);
+}
+
+int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader* gs, const Shader* ps, CreateInterpolantValueFn createValue, bool fillIdentityTail) {
     if (regs == nullptr) {
         throw std::runtime_error(std::string(fn) + ": regs is null");
     }
+    if (ps == nullptr && !fillIdentityTail) {
+        throw std::runtime_error(std::string(fn) + ": ps is null");
+    }
     if (ps != nullptr && ps->num_input_semantics > 32) {
         throw std::runtime_error(std::string(fn) + ": input semantic count exceeds 32");
     }
@@ -66,13 +77,14 @@ int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader*
     auto* output = regs;
     regs = values.data();
     if (ps == nullptr || ps->num_input_semantics == 0) {
-        FillIdentityInterpolants(regs, 0);
+        const std::uint32_t count = fillIdentityTail ? InterpolantRegisterCount : 0;
+        if (fillIdentityTail) FillIdentityInterpolants(regs, 0);
         if (ps != nullptr) {
-            AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, 32}, {});
+            AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, count}, {});
             if (gs != nullptr) AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0);
         }
         transaction.Commit();
-        std::copy(values.begin(), values.end(), output);
+        std::copy_n(values.begin(), count, output);
         return 0;
     }
 
@@ -98,11 +110,12 @@ int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader*
         SetInterpolantRegister(regs, i, createValue(ShaderSemanticWord(psSemantic), gsSemantic));
     }
 
-    FillIdentityInterpolants(regs, ps->num_input_semantics);
-    AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, 32}, {});
+    const std::uint32_t count = fillIdentityTail ? InterpolantRegisterCount : ps->num_input_semantics;
+    if (fillIdentityTail) FillIdentityInterpolants(regs, ps->num_input_semantics);
+    AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, count}, {});
     AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0);
     transaction.Commit();
-    std::copy(values.begin(), values.end(), output);
+    std::copy_n(values.begin(), count, output);
     return 0;
 }
 
@@ -111,12 +124,17 @@ int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader*
 extern "C" {
 
 int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
-    return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValue);
+    return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValue, true);
 }
 
 APS5_EXPORT("dbOlWdppb4o", sceAgcUnknownCreateInterpolantMapping);
 int APS5_VABI sceAgcUnknownCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
-    return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16);
+    return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16, true);
+}
+
+APS5_EXPORT("vieBRwlh1Lw", sceAgcUnknown_vieBRwlh1Lw);
+int APS5_VABI sceAgcUnknown_vieBRwlh1Lw(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
+    return CreateInterpolantMapping(__func__, regs, gs, ps, UpdateInterpolantValue, false);
 }
 
 }

core/libs/prx/libSceAgc/Unimplemented.cpp

@@ -90,10 +90,4 @@ int APS5_VABI sceAgcGetShaderInstrumentation() {
  return 0;
 }
 
-APS5_EXPORT("vieBRwlh1Lw", sceAgcUnknown_vieBRwlh1Lw);
-int APS5_VABI sceAgcUnknown_vieBRwlh1Lw(void) {
- NotImplemented_nid_no_patch(__func__);
- return 0;
-}
-
 }

core/libs/prx/libSceAgc/tests/Commands.cpp

@@ -13,6 +13,7 @@
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbResetQueue(CommandBuffer* buf, std::uint32_t op, std::uint32_t state);
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbClearState(CommandBuffer* buf, std::uint32_t command);
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetFlip(CommandBuffer* buf, std::uint32_t handle, std::int32_t index, std::uint32_t mode, std::int64_t argument);
+extern "C" std::uint32_t* APS5_VABI sceAgcAcbWaitUntilSafeForRendering(CommandBuffer* buf, std::uint32_t handle, std::uint32_t index);
 extern "C" int APS5_VABI sceAgcSuspendPoint();
 extern "C" int APS5_VABI sceAgcInit(std::uint32_t version);
 extern "C" void* APS5_VABI sceAgcGetRegisterDefaults();
@@ -29,6 +30,10 @@ extern "C" std::uint32_t* APS5_VABI sceAgcCbSetShRegisterRangeDirect(CommandBuff
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbContextStateOp_0100(CommandBuffer* buf, std::uint32_t operation);
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbPushMarker(CommandBuffer* buf, const char* str, std::uint32_t color);
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbPopMarker(CommandBuffer* buf);
+extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetWorkloadComplete(CommandBuffer* buf, std::uint32_t stream_id, std::uint32_t workload_id);
+extern "C" int APS5_VABI sceAgcDriverRegisterWorkloadStream(std::uint32_t stream_id, const void* stream);
+extern "C" int APS5_VABI sceAgcDriverUnregisterWorkloadStream(std::uint32_t stream_id);
+extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetWorkloadsActive(CommandBuffer* buf, std::uint32_t stream_id, const std::uint32_t* workload_ids, std::uint32_t workload_count);
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetMarker(CommandBuffer* buf, const char* str, std::uint32_t color);
 extern "C" std::uint32_t* APS5_VABI sceAgcAcbPushMarker(CommandBuffer* buf, const char* str, std::uint32_t color);
 extern "C" std::uint32_t* APS5_VABI sceAgcAcbPopMarker(CommandBuffer* buf);
@@ -37,6 +42,7 @@ extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetIndexBuffer(CommandBuffer* buf,
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetBaseIndirectArgs(CommandBuffer* buf, std::uint32_t shaderType, const volatile void* indirectBaseAddr);
 extern "C" std::uint32_t* APS5_VABI sceAgcSetNop(CommandBuffer* buf, std::uint32_t sizeDw);
 extern "C" int APS5_VABI sceAgcDebugRaiseException(std::uint32_t exceptionId);
+extern "C" std::uint32_t* APS5_VABI sceAgcAcbAtomicGds_0900(CommandBuffer* buf, std::uint32_t control, std::uint32_t mode, std::uint32_t gdsOp, std::uint32_t src, std::uint16_t offset, std::uint16_t index, std::uint32_t mask, std::uint32_t data, std::uint64_t compareData, std::uint64_t extraData);
 
 namespace {
 
@@ -206,6 +212,38 @@ void testMarkers() {
     expectFailure([] { sceAgcAcbSetMarker(nullptr, "frame", 0); });
 }
 
+void testWorkloads() {
+    Storage dcb;
+    const std::uint32_t ids[2] = {3, 4};
+    check(sceAgcDcbSetWorkloadsActive(&dcb.buffer, 1, ids, 2) == nullptr, "workloads active packet for an unregistered stream");
+    check(sceAgcDcbSetWorkloadComplete(&dcb.buffer, 1, 3) == nullptr, "workload complete packet for an unregistered stream");
+    check(dcb.buffer.cursor_up == dcb.words.data(), "unregistered stream advanced the cursor");
+    const std::uint32_t record[8] = {};
+    check(sceAgcDriverRegisterWorkloadStream(1, record) == 0, "workload stream registration failed");
+    const auto* active = sceAgcDcbSetWorkloadsActive(&dcb.buffer, 1, ids, 2);
+    check(active == dcb.words.data() && active[0] == Agc::Command::Header(0x10, 18, 0) && active[1] == 1 && active[2] == 0x18 && active[3] == 0, "workloads active packet");
+    for (std::uint32_t word = 4; word < 18; ++word) check(active[word] == 0, "workloads active packet tail is not zero");
+    const auto* complete = sceAgcDcbSetWorkloadComplete(&dcb.buffer, 1, 33);
+    check(complete == dcb.words.data() + 18 && complete[0] == Agc::Command::Header(0x10, 12, 0) && complete[1] == 1 && complete[2] == 33 && complete[3] == 0xffffffffu && complete[4] == 0xfffffffdu, "workload complete packet");
+    for (std::uint32_t word = 5; word < 12; ++word) check(complete[word] == 0, "workload complete packet tail is not zero");
+    check(dcb.buffer.cursor_up == dcb.words.data() + 30, "incorrect workload cursor advance");
+    const std::uint32_t duplicate[2] = {5, 5};
+    const std::uint32_t outOfRange[1] = {64};
+    const std::uint32_t top[1] = {63};
+    check(sceAgcDcbSetWorkloadsActive(&dcb.buffer, 1, duplicate, 2) == nullptr, "duplicate workload ids accepted");
+    check(sceAgcDcbSetWorkloadsActive(&dcb.buffer, 1, outOfRange, 1) == nullptr, "workload id 64 accepted");
+    check(sceAgcDcbSetWorkloadsActive(&dcb.buffer, 1, top, 1) != nullptr, "workload id 63 rejected");
+    check(sceAgcDcbSetWorkloadsActive(&dcb.buffer, 1, ids, 0) == nullptr, "workload count 0 accepted");
+    check(sceAgcDcbSetWorkloadsActive(&dcb.buffer, 1, nullptr, 2) == nullptr, "null workload ids accepted");
+    check(sceAgcDcbSetWorkloadsActive(&dcb.buffer, 0, ids, 2) == nullptr, "stream id 0 accepted");
+    check(sceAgcDcbSetWorkloadsActive(&dcb.buffer, 32, ids, 2) == nullptr, "stream id 32 accepted");
+    check(sceAgcDcbSetWorkloadComplete(&dcb.buffer, 1, 64) == nullptr, "workload complete id 64 accepted");
+    check(sceAgcDcbSetWorkloadsActive(nullptr, 1, ids, 2) == nullptr, "null command buffer accepted");
+    check(sceAgcDcbSetWorkloadComplete(nullptr, 1, 3) == nullptr, "null command buffer accepted for complete");
+    check(sceAgcDriverUnregisterWorkloadStream(1) == 0, "workload stream unregistration failed");
+    check(sceAgcDcbSetWorkloadComplete(&dcb.buffer, 1, 3) == nullptr, "workload complete packet after unregistration");
+}
+
 void testIndexBuffer() {
     Storage storage;
     alignas(4) std::uint16_t indices[2]{};
@@ -316,6 +354,17 @@ void testFlip() {
     exhausted.buffer.cursor_down = exhausted.words.data() + 5;
     expectFailure([&] { sceAgcDcbSetFlip(&exhausted.buffer, 1, 0, 1, 0); });
     check(exhausted.buffer.cursor_up == exhausted.words.data(), "failed flip allocation advanced cursor");
+    Storage acb;
+    acb.words.fill(0xdeadbeefu);
+    auto* wait = sceAgcAcbWaitUntilSafeForRendering(&acb.buffer, 0xfedcba98u, 2);
+    const std::array<std::uint32_t, 4> expectedWait{0xc0021018u, 0xfedcba98u, 2, 0};
+    check(wait == acb.words.data() && std::equal(expectedWait.begin(), expectedWait.end(), wait), "compute rendering wait packet mismatch");
+    check(acb.buffer.cursor_up == wait + 4 && wait[4] == 0xdeadbeefu, "compute rendering wait overran allocation");
+    expectFailure([] { sceAgcAcbWaitUntilSafeForRendering(nullptr, 1, 0); });
+    Storage full;
+    full.buffer.cursor_down = full.words.data() + 3;
+    expectFailure([&] { sceAgcAcbWaitUntilSafeForRendering(&full.buffer, 1, 0); });
+    check(full.buffer.cursor_up == full.words.data(), "failed compute rendering wait advanced cursor");
     check(sceAgcSuspendPoint() == 0, "empty suspend failed");
 }
 
@@ -404,6 +453,101 @@ void testMemory() {
     check(storage.words == beforeMask, "invalid mask patch modified packet memory");
 }
 
+struct AtomicGdsAllocation {
+    Storage storage;
+    std::uint32_t capacity = 15;
+    std::uint32_t calls = 0;
+    bool accepted = true;
+};
+
+bool APS5_VABI growAtomicGds(CommandBuffer* buffer, std::uint32_t count, void* userData) {
+    auto& allocation = *static_cast<AtomicGdsAllocation*>(userData);
+    check(count == 15, "atomic GDS allocation omitted packet or reserved words");
+    ++allocation.calls;
+    if (!allocation.accepted) return false;
+    buffer->bottom = allocation.storage.words.data();
+    buffer->top = buffer->bottom + allocation.capacity;
+    buffer->cursor_up = buffer->bottom;
+    buffer->cursor_down = buffer->top;
+    return true;
+}
+
+void testAtomicGds() {
+    Storage storage;
+    storage.words.fill(0xdeadbeefu);
+    storage.buffer.cursor_up += 2;
+    auto* packet = sceAgcAcbAtomicGds_0900(&storage.buffer, 0xffffffffu, 1, 0xffffffffu, 0xffffffffu,
+        0xabcd, 0xfedc, 0x12345678u, 0x87654321u, 0xfedcba9876543210ull, 0x0123456789abcdefull);
+    const std::array<std::uint32_t, 11> expected{
+        0xc0091d00u, 0x001500ffu, 0x13fu, 0xabcdu, 0xfedcu, 0x00340078u,
+        0x87654321u, 0x76543210u, 0xfedcba98u, 0x89abcdefu, 0x01234567u};
+    check(packet == storage.words.data() + 2, "atomic GDS returned the wrong packet address");
+    check(std::equal(expected.begin(), expected.end(), packet), "atomic GDS packet or guest ABI mismatch");
+    check(storage.buffer.cursor_up == packet + expected.size(), "atomic GDS did not advance by eleven words");
+    check(storage.words[0] == 0xdeadbeefu && storage.words[1] == 0xdeadbeefu &&
+        std::all_of(storage.words.begin() + 13, storage.words.end(), [](auto word) { return word == 0xdeadbeefu; }),
+        "atomic GDS wrote outside its packet");
+
+    const std::array<std::array<std::uint32_t, 3>, 9> controls{{
+        {0, 0, 0}, {0, 1, 0x00020000u}, {0, 2, 0}, {0, 0xffffffffu, 0},
+        {0x20u, 1, 0x00040020u}, {0x40u, 1, 0x00120040u},
+        {0x200u, 1, 0x00030000u}, {0xfffffd00u, 0, 0}, {0xffu, 1, 0x001400ffu}}};
+    for (const auto& entry : controls) {
+        Storage controlStorage;
+        auto* controlPacket = sceAgcAcbAtomicGds_0900(&controlStorage.buffer, entry[0], entry[1],
+            0xfffffffeu, 0xffffffc0u, 0, 0, 0xff00ff00u, 0, 0, 0);
+        std::array<std::uint32_t, 11> controlExpected{};
+        controlExpected[0] = 0xc0091d00u;
+        controlExpected[1] = entry[2];
+        check(std::equal(controlExpected.begin(), controlExpected.end(), controlPacket), "atomic GDS control or field masking mismatch");
+    }
+
+    auto emit = [](CommandBuffer* buffer) {
+        return sceAgcAcbAtomicGds_0900(buffer, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+    };
+    Storage exact;
+    exact.buffer.cursor_down = exact.words.data() + 15;
+    exact.buffer.reserved_dw = 4;
+    check(emit(&exact.buffer) == exact.words.data() && exact.buffer.cursor_up == exact.words.data() + 11,
+        "atomic GDS rejected an exact fit with reserved space");
+    const auto exactBefore = exact.words;
+    expectFailure([&] { emit(&exact.buffer); });
+    check(exact.words == exactBefore && exact.buffer.cursor_up == exact.words.data() + 11,
+        "atomic GDS overwrote reserved space");
+    Storage exhausted;
+    exhausted.buffer.cursor_down = exhausted.words.data() + 10;
+    const auto before = exhausted.words;
+    expectFailure([&] { emit(&exhausted.buffer); });
+    check(exhausted.words == before && exhausted.buffer.cursor_up == exhausted.words.data(),
+        "failed atomic GDS allocation modified the buffer");
+    expectFailure([&] { emit(nullptr); });
+
+    Storage noDown;
+    noDown.buffer.top = noDown.words.data() + 11;
+    noDown.buffer.cursor_down = nullptr;
+    check(emit(&noDown.buffer) == noDown.words.data() && noDown.buffer.cursor_up == noDown.buffer.top,
+        "atomic GDS did not honor a buffer with no down cursor");
+
+    AtomicGdsAllocation allocation;
+    CommandBuffer growing{nullptr, nullptr, nullptr, nullptr, growAtomicGds, &allocation, 4};
+    check(emit(&growing) == allocation.storage.words.data() && allocation.calls == 1 &&
+        growing.cursor_up == allocation.storage.words.data() + 11, "atomic GDS did not use the guest allocation callback");
+    const std::array<std::uint32_t, 11> zeroPacket{0xc0091d00u};
+    check(std::equal(zeroPacket.begin(), zeroPacket.end(), allocation.storage.words.begin()),
+        "atomic GDS callback allocation produced the wrong packet");
+    for (const bool accepted : {false, true}) {
+        AtomicGdsAllocation failure;
+        failure.accepted = accepted;
+        failure.capacity = 14;
+        const auto failureBefore = failure.storage.words;
+        CommandBuffer failed{nullptr, nullptr, nullptr, nullptr, growAtomicGds, &failure, 4};
+        expectFailure([&] { emit(&failed); });
+        check(failure.calls == 1 && failure.storage.words == failureBefore &&
+            failed.cursor_up == (accepted ? failure.storage.words.data() : nullptr),
+            "atomic GDS wrote a packet after failed or insufficient allocation");
+    }
+}
+
 void testDebugRaiseException() {
     int result = -1;
     try {
@@ -451,6 +595,7 @@ int main(int argc, char** argv) {
         testClearState();
         testIndexedIndirectDraws();
         testMarkers();
+        testWorkloads();
         testIndexBuffer();
         testIndirectBase();
         testContextState();
@@ -459,6 +604,7 @@ int main(int argc, char** argv) {
         testRegisterRange();
         testPacketPayloadAddress();
         testMemory();
+        testAtomicGds();
         testDebugRaiseException();
         testDefaults();
         LibcRunShutdown_nid_postfix();

core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp

@@ -13,12 +13,14 @@
 
 extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps);
 extern "C" int APS5_VABI sceAgcUnknownCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps);
+extern "C" int APS5_VABI sceAgcUnknown_vieBRwlh1Lw(ShaderRegister* regs, const Shader* gs, const Shader* ps);
 
 namespace {
 
 using Registers = std::array<ShaderRegister, 32>;
 const Shader* mappedPixel = nullptr;
 Registers preparedMapping{};
+std::size_t preparedCount = 0;
 unsigned mappings = 0;
 unsigned links = 0;
 
@@ -65,6 +67,11 @@ void testIdentity() {
     regs = filled();
     check(sceAgcUnknownCreateInterpolantMapping(regs.data(), nullptr, &ps) == 0, "mapping without inputs failed");
     checkIdentity(regs, 0, "mapping without inputs is not the identity");
+    regs = filled();
+    check(sceAgcUnknown_vieBRwlh1Lw(regs.data(), nullptr, &ps) == 0, "update without inputs failed");
+    check(preparedCount == 0u, "update without inputs prepared registers");
+    const auto untouched = filled();
+    check(std::memcmp(regs.data(), untouched.data(), sizeof(regs)) == 0, "update without inputs wrote registers");
 }
 
 void testMapping() {
@@ -99,6 +106,15 @@ void testMapping() {
     check(split[4].value == 0x03180002u, "full f16 interpolant mapping changed");
     checkIdentity(split, static_cast<std::uint32_t>(inputs.size()), "unused interpolants are not the identity");
 
+    auto update = filled();
+    const auto untouched = filled();
+    const std::size_t written = inputs.size() * sizeof(ShaderRegister);
+    check(sceAgcUnknown_vieBRwlh1Lw(update.data(), &gs, &ps) == 0, "update mapping failed");
+    check(preparedCount == inputs.size(), "update prepared registers past its inputs");
+    check(std::memcmp(update.data(), preparedMapping.data(), written) == 0, "prepared update differs from published registers");
+    check(std::memcmp(update.data(), split.data(), written) == 0, "update mapping differs from the split f16 mapping");
+    check(std::memcmp(update.data() + inputs.size(), untouched.data(), sizeof(update) - written) == 0, "update wrote past its inputs");
+
     auto regular = filled();
     check(sceAgcCreateInterpolantMapping_0100(regular.data(), &gs, &ps) == 0, "mapping failed");
     check(std::memcmp(regular.data(), preparedMapping.data(), sizeof(regular)) == 0, "prepared mapping differs from published registers");
@@ -106,6 +122,51 @@ void testMapping() {
     checkIdentity(regular, static_cast<std::uint32_t>(inputs.size()), "unused interpolants are not the identity");
 }
 
+void testUpdateMapping() {
+    std::array<ShaderSemantic, 4> inputs{semantic(1, 0, 0), semantic(2, 0, 1), semantic(3, 0, 2), semantic(4, 0, 3)};
+    inputs[0].is_custom = 1;
+    inputs[0].default_value = 2;
+    inputs[1].default_value = 1;
+    inputs[1].default_value_hi = 1;
+    inputs[2].default_value_hi = 2;
+    inputs[3].is_custom = 1;
+    inputs[3].default_value = 3;
+    inputs[3].default_value_hi = 3;
+    std::array<ShaderSemantic, 3> outputs{semantic(1, 5, 0), semantic(2, 7, 1), semantic(4, 9, 3)};
+    Shader gs{};
+    gs.output_semantics = outputs.data();
+    gs.num_output_semantics = static_cast<std::uint16_t>(outputs.size());
+    Shader ps{};
+    ps.input_semantics = inputs.data();
+    ps.num_input_semantics = static_cast<std::uint32_t>(inputs.size());
+
+    auto update = filled();
+    const auto untouched = filled();
+    check(sceAgcUnknown_vieBRwlh1Lw(update.data(), &gs, &ps) == 0, "update mapping failed");
+    check(update[0].value == 0x00000625u, "custom interpolant update changed");
+    check(update[1].value == 0x01380107u, "low f16 half update changed");
+    check(update[2].value == 0x02480220u, "unmatched high f16 half update changed");
+    check(update[3].value == 0x03680709u, "full f16 interpolant update changed");
+    for (std::uint32_t i = 0; i < inputs.size(); ++i) {
+        check(update[i].offset == ShaderRegs::SPI_PS_INPUT_CNTL_0 + i, "updated register offset changed");
+    }
+    const std::size_t written = inputs.size() * sizeof(ShaderRegister);
+    check(std::memcmp(update.data() + inputs.size(), untouched.data(), sizeof(update) - written) == 0, "update wrote past its inputs");
+
+    auto create = filled();
+    check(sceAgcUnknownCreateInterpolantMapping(create.data(), &gs, &ps) == 0, "split f16 mapping failed");
+    check(std::memcmp(create.data(), update.data(), written) == 0, "update differs from the split f16 mapping");
+    checkIdentity(create, static_cast<std::uint32_t>(inputs.size()), "unused interpolants are not the identity");
+
+    for (const std::uint32_t index : {1u, 3u}) {
+        inputs[index].default_value = 0;
+        update = filled();
+        expectFailure([&] { sceAgcUnknown_vieBRwlh1Lw(update.data(), &gs, &ps); });
+        check(std::memcmp(update.data(), untouched.data(), sizeof(update)) == 0, "rejected update wrote registers");
+        inputs[index].default_value = inputs[index].default_value_hi;
+    }
+}
+
 void testRejections() {
     std::array<ShaderSemantic, 33> inputs{};
     Shader gs{};
@@ -117,18 +178,23 @@ void testRejections() {
     const auto saved = regs;
     expectFailure([&] { sceAgcCreateInterpolantMapping_0100(regs.data(), &gs, &ps); });
     expectFailure([&] { sceAgcUnknownCreateInterpolantMapping(regs.data(), &gs, &ps); });
+    expectFailure([&] { sceAgcUnknown_vieBRwlh1Lw(regs.data(), &gs, &ps); });
     check(std::memcmp(regs.data(), saved.data(), sizeof(regs)) == 0, "rejected mapping wrote registers");
     ps.num_input_semantics = 1;
     expectFailure([&] { sceAgcUnknownCreateInterpolantMapping(regs.data(), nullptr, &ps); });
     expectFailure([&] { sceAgcUnknownCreateInterpolantMapping(nullptr, &gs, &ps); });
+    expectFailure([&] { sceAgcUnknown_vieBRwlh1Lw(regs.data(), nullptr, &ps); });
+    expectFailure([&] { sceAgcUnknown_vieBRwlh1Lw(nullptr, &gs, &ps); });
+    expectFailure([&] { sceAgcUnknown_vieBRwlh1Lw(regs.data(), &gs, nullptr); });
 }
 
 }
 
 extern "C" void AgcDriverResolveShaderAbi_nid_postfix(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive) {
-    check(shader != nullptr && context.size() == preparedMapping.size() && primitive.empty(), "invalid mapping preparation call");
+    check(shader != nullptr && context.size() <= preparedMapping.size() && primitive.empty(), "invalid mapping preparation call");
     mappedPixel = shader;
     std::copy(context.begin(), context.end(), preparedMapping.begin());
+    preparedCount = context.size();
     ++mappings;
 }
 
@@ -141,8 +207,9 @@ int main() {
     try {
         testIdentity();
         testMapping();
+        testUpdateMapping();
         testRejections();
-        check(mappings == 3u && links == 2u, "mapping preparation call count changed");
+        check(mappings == 7u && links == 5u, "mapping preparation call count changed");
         LibcRunShutdown_nid_postfix();
         std::puts("AGC interpolant mapping tests passed");
         return 0;

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -595,6 +595,20 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_device_memory_fallback_tests EXCLUDE_FROM_ALL tests/DeviceMemoryFallback.cpp)
+target_include_directories(agc_device_memory_fallback_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
+target_link_libraries(agc_device_memory_fallback_tests PRIVATE libSceAgcDriver libc)
+set_target_properties(agc_device_memory_fallback_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_device_memory_fallback_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_device_memory_fallback_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_device_memory_fallback COMMAND agc_device_memory_fallback_tests)
+    if(WIN32)
+        set_tests_properties(agc_device_memory_fallback PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_display_buffer_dcc_tests EXCLUDE_FROM_ALL tests/DisplayBufferDcc.cpp)
 target_include_directories(agc_display_buffer_dcc_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
 target_link_libraries(agc_display_buffer_dcc_tests PRIVATE libSceAgcDriver libc)
@@ -657,6 +671,7 @@ add_test_executable(agc_driver_flip_tests EXCLUDE_FROM_ALL
 
 target_include_directories(agc_driver_flip_tests PRIVATE
         ${LIBS_INCLUDE_DIR}
+        ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
 )
 
 target_link_libraries(agc_driver_flip_tests PRIVATE
@@ -811,6 +826,8 @@ if(BUILD_TESTING)
     endif()
     add_test(NAME agc_host_import_unmap COMMAND agc_driver_recorder_tests --host-import-unmap-only)
     set_tests_properties(agc_host_import_unmap PROPERTIES SKIP_RETURN_CODE 77)
+    add_test(NAME agc_parked_import_unmap COMMAND agc_driver_recorder_tests --parked-import-unmap-only)
+    set_tests_properties(agc_parked_import_unmap PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30 ENVIRONMENT "APS5_PIN_WAIT_MS=3000")
     add_test(NAME agc_unit_shadow COMMAND agc_driver_recorder_tests --unit-shadow-only)
     set_tests_properties(agc_unit_shadow PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30 ENVIRONMENT "APS5_PIN_WAIT_MS=3000")
     add_test(NAME agc_descriptor_pool_recycling COMMAND agc_driver_recorder_tests --descriptor-pool-only)
@@ -819,8 +836,10 @@ if(BUILD_TESTING)
     set_tests_properties(agc_draw_snapshot_cap PROPERTIES SKIP_RETURN_CODE 77)
     add_test(NAME agc_multisample_guest_access COMMAND agc_driver_recorder_tests --multisample-only)
     set_tests_properties(agc_multisample_guest_access PROPERTIES SKIP_RETURN_CODE 77)
+    add_test(NAME agc_draw_inputs_in_place COMMAND agc_driver_recorder_tests --draw-inputs-in-place-only)
+    set_tests_properties(agc_draw_inputs_in_place PROPERTIES SKIP_RETURN_CODE 77)
     if(WIN32)
-        set_tests_properties(agc_host_import_unmap agc_unit_shadow agc_descriptor_pool_recycling agc_draw_snapshot_cap agc_multisample_guest_access PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_host_import_unmap agc_parked_import_unmap agc_unit_shadow agc_descriptor_pool_recycling agc_draw_snapshot_cap agc_multisample_guest_access agc_draw_inputs_in_place PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
@@ -988,6 +1007,8 @@ target_include_directories(agc_driver_swap_alt_blend_tests PRIVATE $<TARGET_PROP
 
 target_include_directories(agc_driver_packed_unorm_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_color1555_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 foreach(stage IN ITEMS vert frag)
     set(agcProvokingShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/ProvokingVertex.${stage})
     set(agcProvokingShaderSpv ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}.spv)
@@ -1008,6 +1029,8 @@ target_include_directories(agc_driver_provoking_vertex_tests PRIVATE ${agcTestSh
 
 target_include_directories(agc_driver_prepared_exports_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_buffer_descriptor_runtime_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 target_include_directories(agc_driver_prepared_publication_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_include_directories(agc_driver_prepared_graphics_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
@@ -1021,6 +1044,12 @@ target_include_directories(agc_driver_prepared_shaders_tests PRIVATE
 
 if(BUILD_TESTING)
     set_target_properties(agc_shader_mode_register_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_driver_readonly_resummarize COMMAND agc_driver_tests --readonly-resummarize)
+    set_tests_properties(agc_driver_readonly_resummarize PROPERTIES TIMEOUT 10)
+    if(WIN32)
+        set_tests_properties(agc_driver_readonly_resummarize PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_shader_mode_register COMMAND agc_shader_mode_register_tests)
     set_tests_properties(agc_shader_mode_register PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 20)
     if(WIN32)
@@ -1038,7 +1067,7 @@ if(BUILD_TESTING)
         set_tests_properties(agc_shader_bindless agc_shader_srt_read_patching agc_shader_frontend_pair agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_vector_window_tests agc_constant_bit_test_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests})
-        if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests")
+        if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests")
             continue()
         endif()
         string(REGEX REPLACE "_tests$" "" agcTestName "${agcTest}")
@@ -1067,6 +1096,28 @@ if(BUILD_TESTING)
         set_tests_properties(agc_shared_write_partial_collect agc_shared_write_partial_collect_two_pass PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    set_target_properties(agc_driver_pipeline_optimized_readback_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    foreach(pipelineReadbackCase IN ITEMS d32 d16 color32 color64)
+        foreach(pipelineReadbackMode IN ITEMS default fast)
+            set(pipelineReadbackTest agc_driver_pipeline_optimized_readback_${pipelineReadbackCase})
+            set(pipelineReadbackEnvironment "ANYPS5_NO_SHADER_CACHE=1")
+            if(pipelineReadbackMode STREQUAL "fast")
+                string(APPEND pipelineReadbackTest "_fast")
+                list(APPEND pipelineReadbackEnvironment "APS5_NO_PIPELINE_LTO=1")
+            endif()
+            add_test(NAME ${pipelineReadbackTest} COMMAND agc_driver_pipeline_optimized_readback_tests ${pipelineReadbackCase})
+            set_tests_properties(${pipelineReadbackTest} PROPERTIES ENVIRONMENT "${pipelineReadbackEnvironment}" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+            if(WIN32)
+                set_tests_properties(${pipelineReadbackTest} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+            endif()
+        endforeach()
+    endforeach()
+    add_test(NAME agc_vector_window_stage_inputs COMMAND agc_vector_window_tests --stage-inputs-only)
+    set_tests_properties(agc_vector_window_stage_inputs PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    set_tests_properties(agc_vector_window_stage_inputs agc_driver_relative_stage_inputs PROPERTIES TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_vector_window_stage_inputs PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_f64_denorm_preserve_spirv COMMAND agc_driver_f64_fma_subnormals_tests --verify-spv-denorm-mode)
     set_tests_properties(agc_driver_f64_denorm_preserve_spirv PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
@@ -1123,14 +1174,16 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_pm4_priority_refused PROPERTIES ENVIRONMENT
             "ANYPS5_NO_SHADER_CACHE=1;APS5_THREAD_PRIORITY=high;DBUS_SYSTEM_BUS_ADDRESS=unix:path=/nonexistent/aps5-no-rtkit" SKIP_RETURN_CODE 77)
     endif()
+    add_test(NAME agc_driver_present_mode COMMAND agc_driver_flip_tests present-mode)
     add_test(NAME agc_driver_flip_compute COMMAND agc_driver_flip_tests compute)
     set_tests_properties(agc_driver_flip_compute PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77)
     if(WIN32)
-        set_tests_properties(agc_driver_flip_compute PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_driver_flip_compute agc_driver_present_mode PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_driver_draw_cache_reuse_off COMMAND agc_driver_draw_cache_reuse_tests)
     set_tests_properties(agc_driver_draw_cache_reuse_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_GPL=1" SKIP_RETURN_CODE 77)
+    set_tests_properties(agc_driver_draw_cache_reuse agc_driver_draw_cache_reuse_off PROPERTIES FAIL_REGULAR_EXPRESSION "FATAL:")
     if(WIN32)
         set_tests_properties(agc_driver_draw_cache_reuse_off PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
@@ -1686,3 +1739,16 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
+
+add_test_executable(agc_constant_phi_tests EXCLUDE_FROM_ALL tests/ConstantPhi.cpp)
+target_link_libraries(agc_constant_phi_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_constant_phi_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_constant_phi_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_constant_phi_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_constant_phi COMMAND agc_constant_phi_tests)
+    if(WIN32)
+        set_tests_properties(agc_constant_phi PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp

@@ -37,6 +37,9 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const
 struct PreparedGraphicsStage {
     std::shared_ptr<const ShaderSnapshot> snapshot;
     PreparedShaders::Entry entry;
+    ShaderRecompiler::ShaderStage stage = ShaderRecompiler::ShaderStage::Vertex;
+    std::uint32_t pushOffset = 0;
+    std::shared_ptr<const ShaderRecompiler::RecompileResult> registration;
 };
 
 std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target);

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp

@@ -103,6 +103,7 @@ std::shared_ptr<const ShaderSnapshot> RegisteredHeader(const ShaderRegistry& reg
 std::shared_ptr<const ShaderSnapshot> RegisteredProgram(const ShaderRegistry& registry, std::uint64_t address, std::initializer_list<std::uint8_t> types);
 void PublishRegisteredShader(std::shared_ptr<ShaderRegistry>& registry, const std::shared_ptr<const ShaderSnapshot>& snapshot);
 void SubmitRegistrationPreparation(std::function<void()> task);
+void SubmitLibraryPrebuild(std::function<void()> task);
 void CloseRegistrationPreparation();
 
 void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target);

core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp

@@ -101,6 +101,7 @@ std::uint64_t CollectWritesUncached(std::uint64_t address, std::size_t bytes);
 // tracker mutex, so every collect that bumped before it has finished its walk and every later
 // collect stamps newer.
 std::uint64_t TrackerGeneration();
+std::uint64_t NewestStamp(std::uint64_t address, std::size_t bytes);
 // Whether no CPU write touched the range since `generation`: an uncached resetting collect of the
 // range's pages first (a game store not collected yet becomes a block stamp), then the compare over
 // the stamps collects make (dirty pages), not the MarkWritten stamps of the driver's own GPU label

core/libs/prx/libSceAgcDriver/Execution/include/PresentMode.hpp

@@ -0,0 +1,27 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_PRESENTMODE_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_PRESENTMODE_HPP
+
+#include <vulkan/vulkan.h>
+#include <algorithm>
+#include <span>
+#include <stdexcept>
+#include <string_view>
+
+namespace AgcDriver {
+
+inline VkPresentModeKHR SelectPresentMode(const char* requested, std::span<const VkPresentModeKHR> supported) {
+    VkPresentModeKHR mode = VK_PRESENT_MODE_FIFO_KHR;
+    if (requested != nullptr) {
+        const std::string_view name(requested);
+        if (name == "mailbox") mode = VK_PRESENT_MODE_MAILBOX_KHR;
+        else if (name == "immediate") mode = VK_PRESENT_MODE_IMMEDIATE_KHR;
+        else if (name == "fifo_latest_ready") mode = VK_PRESENT_MODE_FIFO_LATEST_READY_KHR;
+        else if (name != "fifo") throw std::runtime_error("Vulkan presentation: ANYPS5_PRESENT_MODE must be fifo, fifo_latest_ready, mailbox or immediate");
+    }
+    if (std::find(supported.begin(), supported.end(), mode) == supported.end()) throw std::runtime_error("Vulkan presentation: the surface does not support the requested present mode");
+    return mode;
+}
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -299,9 +299,11 @@ public:
     };
     IndirectDrawSupport DrawIndirectSupport() const;
     void EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span<const Graphics::CompiledShader> shaders, std::span<const Graphics::GuestMemorySnapshot> snapshots = {});
+    bool PrebuildGraphicsLibraries(const Graphics::ShaderStages& stages, std::span<const Graphics::CompiledShader> shaders);
 
 private:
     VulkanDevice(const PresentationWindow* window, bool compileOnly);
+    friend struct VulkanDeviceTestAccess;
     // The device's Graphics::Context: a copy of the one built at setup (its instance functions
     // resolved then, its function table filled then), or with APS5_NO_CONTEXT_CACHE=1 built anew.
     Graphics::Context graphicsContext() const;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
+#include "ThreadOwned.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
@@ -10,8 +11,8 @@
 namespace AgcDriver::DriverDetail {
 
 std::map<std::size_t, Driver::ResolvedDispatch>& Driver::resolvedAhead() {
-    static thread_local std::map<std::size_t, ResolvedDispatch> resolved;
-    return resolved;
+    static thread_local std::map<std::size_t, ResolvedDispatch>* resolved = nullptr;
+    return ShaderRecompiler::ThreadOwned(resolved);
 }
 
 Driver::GroupCaptureStats& Driver::groupCaptureStats() {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp

@@ -64,6 +64,7 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr<Vulk
             for (const bool flag : {pixel.wave32, pixel.hasPerspectiveCenterVgpr, pixel.perspectiveCentroid, pixel.posX, pixel.posY, pixel.posZ, pixel.posW, pixel.frontFace, pixel.ancillary, pixel.sampleShading, pixel.noPerspective, pixel.linearCentroid, pixel.pixelKillEnable, pixel.depthExportEnable, pixel.sampleMaskExportEnable, pixel.earlyZ, pixel.executeOnNoop}) mix(flag);
             mix(static_cast<std::uint64_t>(pixel.conservativeZExport));
             mix(pixel.orderedPixelShader);
+            mix(pixel.stencilRefExportEnable);
             for (const auto value : pixel.targetOutputMode) mix(value);
             for (const auto value : pixel.targetExportMapping) mix(value);
             for (const auto value : pixel.targetExportPacking) mix(static_cast<std::uint64_t>(value));

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp

@@ -23,6 +23,7 @@ std::optional<DrawVerdict> Driver::precheckDraw(const QueueState& queue, const S
         localDevice->ColorMetadataPass(*pass);
         return DrawVerdict::Drawn;
     }
+    if (Graphics::DepthResummarizeDrawsNothing(queue)) return DrawVerdict::Nothing;
     rejected = Graphics::DepthMaintenanceRejection(queue);
     if (!rejected.empty()) return DrawVerdict::Rejected;
     static const bool traceIndirectEnabled = std::getenv("APS5_TRACE_INDIRECT_DRAWS") != nullptr;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp

@@ -98,7 +98,7 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) {
     if (std::memcmp(&s.viewport, &t.viewport, sizeof(VkViewport)) != 0 || std::memcmp(&s.scissor, &t.scissor, sizeof(VkRect2D)) != 0) return false;
     const auto& p = a.pixel;
     const auto& q = b.pixel;
-    if (p.interpolatorCount != q.interpolatorCount || p.interpolatorSettings != q.interpolatorSettings || p.wave32 != q.wave32 || p.inputAddr != q.inputAddr || p.hasPerspectiveCenterVgpr != q.hasPerspectiveCenterVgpr || p.perspectiveCentroid != q.perspectiveCentroid || p.posX != q.posX || p.posY != q.posY || p.posZ != q.posZ || p.posW != q.posW || p.frontFace != q.frontFace || p.ancillary != q.ancillary || p.sampleShading != q.sampleShading || p.noPerspective != q.noPerspective || p.linearCentroid != q.linearCentroid || p.pixelKillEnable != q.pixelKillEnable || p.depthExportEnable != q.depthExportEnable || p.sampleMaskExportEnable != q.sampleMaskExportEnable || p.earlyZ != q.earlyZ || p.executeOnNoop != q.executeOnNoop || p.conservativeZExport != q.conservativeZExport || p.orderedPixelShader != q.orderedPixelShader || p.targetOutputMode != q.targetOutputMode || p.targetExportMapping != q.targetExportMapping) return false;
+    if (p.interpolatorCount != q.interpolatorCount || p.interpolatorSettings != q.interpolatorSettings || p.wave32 != q.wave32 || p.inputAddr != q.inputAddr || p.hasPerspectiveCenterVgpr != q.hasPerspectiveCenterVgpr || p.perspectiveCentroid != q.perspectiveCentroid || p.posX != q.posX || p.posY != q.posY || p.posZ != q.posZ || p.posW != q.posW || p.frontFace != q.frontFace || p.ancillary != q.ancillary || p.sampleShading != q.sampleShading || p.noPerspective != q.noPerspective || p.linearCentroid != q.linearCentroid || p.pixelKillEnable != q.pixelKillEnable || p.depthExportEnable != q.depthExportEnable || p.sampleMaskExportEnable != q.sampleMaskExportEnable || p.earlyZ != q.earlyZ || p.executeOnNoop != q.executeOnNoop || p.conservativeZExport != q.conservativeZExport || p.orderedPixelShader != q.orderedPixelShader || p.stencilRefExportEnable != q.stencilRefExportEnable || p.targetOutputMode != q.targetOutputMode || p.targetExportMapping != q.targetExportMapping) return false;
     if (a.roles != b.roles || a.programs.size() != b.programs.size()) return false;
     for (std::size_t i = 0; i < a.programs.size(); ++i) {
         const auto& x = a.programs[i];

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp

@@ -55,6 +55,7 @@ void Driver::stop() {
 }
 
 Driver::Driver() {
+    static_cast<void>(GuestMemory::TrackerGeneration());
     ShaderMemory::SetWaitedMsProvider(&Graphics::Recorder::ThreadWaitedMs);
     try {
         LibcRegisterShutdown_nid_postfix([] { Driver::Get().Shutdown(); });

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp

@@ -19,9 +19,12 @@
 #include <cstdlib>
 #include <cstring>
 #include <deque>
+#include <exception>
 #include <functional>
 #include <list>
+#include <atomic>
 #include <pthread.h>
+#include <sstream>
 #include <stdexcept>
 #include <thread>
 #include "RdnaDecoder/RdnaInstructionDecoder.hpp"
@@ -29,13 +32,25 @@
 namespace AgcDriver::DriverDetail {
 
 std::shared_ptr<const ShaderRecompiler::SourceHandle> PrepareShaderWithDiagnostics(const ShaderRecompiler::RecompileRequest& request, ShaderRecompiler::ShaderPreparationContext* preparation) {
+    std::exception_ptr failure;
+    std::string dump;
     try {
-        return ShaderRecompiler::PrepareShader(request, preparation);
+        auto handle = ShaderRecompiler::PrepareShader(request, preparation);
+        return handle;
     } catch (...) {
+        failure = std::current_exception();
         try {
-            if (std::getenv("APS5_DUMP_SHADERS") != nullptr) static_cast<void>(Driver::dumpRequest(request.shader.codeAddress, request));
+            if (std::getenv("APS5_DUMP_SHADERS") != nullptr) dump = Driver::dumpRequest(request.shader.codeAddress, request);
         } catch (...) {}
-        throw;
+    }
+    try {
+        std::rethrow_exception(failure);
+    } catch (const std::exception& error) {
+        std::ostringstream where;
+        where << "shader 0x" << std::hex << request.shader.codeAddress << " stage " << static_cast<unsigned>(request.shader.stage);
+        if (!dump.empty()) where << " (" << dump << ")";
+        where << ": ";
+        throw std::runtime_error(where.str() + error.what());
     }
 }
 
@@ -571,6 +586,7 @@ struct RegisteredPreparation {
     std::array<ShaderRecompiler::MemoryRegion, 2> memory{};
     std::optional<ShaderRecompiler::RecompileRequest> request;
     std::vector<std::uint64_t> abiKey;
+    std::shared_ptr<VulkanDevice> prebuildDevice;
 };
 
 std::unique_ptr<RegisteredPreparation> PlanRegistered(const ShaderSnapshot& snapshot, const VulkanDevice& device, const QueueState& state, bool registration, bool* deferred = nullptr) {
@@ -669,6 +685,18 @@ std::unique_ptr<RegisteredPreparation> PlanRegistered(const ShaderSnapshot& snap
     return plan;
 }
 
+void PrebuildStageLibrary(const std::shared_ptr<VulkanDevice>& device, const Graphics::ShaderStages& stages, ShaderRecompiler::ShaderStage stage, std::shared_ptr<const ShaderRecompiler::RecompileResult> result, std::uint32_t pushOffset) {
+    if (result == nullptr || (stage != ShaderRecompiler::ShaderStage::Vertex && stage != ShaderRecompiler::ShaderStage::Fragment)) return;
+    SubmitLibraryPrebuild([device, stages, stage, result = std::move(result), pushOffset] {
+        const std::array<Graphics::CompiledShader, 1> shaders{{{stage, result.get(), pushOffset}}};
+        try {
+            static_cast<void>(device->PrebuildGraphicsLibraries(stages, shaders));
+        } catch (const std::exception& error) {
+            APS5_LOG_OUT("shader library prebuild skipped: %s", error.what());
+        }
+    });
+}
+
 std::vector<PreparedShaders::Entry> PrepareRegistered(RegisteredPreparation& plan) {
     auto& request = *plan.request;
     std::vector<PreparedShaders::Entry> entries;
@@ -678,6 +706,13 @@ std::vector<PreparedShaders::Entry> PrepareRegistered(RegisteredPreparation& pla
         entries.push_back({plan.codeOffset, PrepareShaderWithDiagnostics(request, &preparation)});
     };
     append();
+    if (plan.prebuildDevice != nullptr && request.target.registrationModules && request.shader.stage == ShaderRecompiler::ShaderStage::Fragment) {
+        try {
+            PrebuildStageLibrary(plan.prebuildDevice, {Graphics::ShaderPath::Vertex, 0u, 32u, request.context.waveSize, {}, {}}, request.shader.stage, ShaderRecompiler::RegistrationResult(request, *entries.front().handle), request.layout.pushConstantOffsetBytes);
+        } catch (const std::exception& error) {
+            APS5_LOG_OUT("shader library prebuild skipped: %s", error.what());
+        }
+    }
     if (request.context.compute) {
         request.context.compute->partialThreads = {1, 1, 1};
         append();
@@ -696,10 +731,32 @@ std::vector<PreparedShaders::Entry> PrepareRegistered(const ShaderSnapshot& snap
 class RegistrationPreparePool {
 public:
     static RegistrationPreparePool& Instance() {
-        static auto* pool = new RegistrationPreparePool();
+        static auto* pool = new RegistrationPreparePool(std::max(2u, std::thread::hardware_concurrency() / 2u));
         return *pool;
     }
 
+    static RegistrationPreparePool& Prebuild() {
+        static auto* pool = [] {
+            const char* text = std::getenv("APS5_PREBUILD_THREADS");
+            auto* created = new RegistrationPreparePool(text != nullptr ? std::max(1u, static_cast<unsigned>(std::strtoul(text, nullptr, 10))) : 1u);
+            PrebuildSlot().store(created);
+            return created;
+        }();
+        return *pool;
+    }
+
+    static RegistrationPreparePool* ExistingPrebuild() {
+        return PrebuildSlot().load();
+    }
+
+private:
+    static std::atomic<RegistrationPreparePool*>& PrebuildSlot() {
+        static std::atomic<RegistrationPreparePool*> slot{nullptr};
+        return slot;
+    }
+
+public:
+
     void Submit(std::function<void()> task) {
         {
             std::unique_lock lock(mutex);
@@ -720,8 +777,7 @@ public:
     }
 
 private:
-    RegistrationPreparePool() {
-        const auto count = std::max(2u, std::thread::hardware_concurrency() / 2u);
+    explicit RegistrationPreparePool(unsigned count) {
         pthread_attr_t attributes;
         pthread_attr_init(&attributes);
         pthread_attr_setstacksize(&attributes, 64u << 20u);
@@ -802,8 +858,13 @@ void SubmitRegistrationPreparation(std::function<void()> task) {
     RegistrationPreparePool::Instance().Submit(std::move(task));
 }
 
+void SubmitLibraryPrebuild(std::function<void()> task) {
+    RegistrationPreparePool::Prebuild().Submit(std::move(task));
+}
+
 void CloseRegistrationPreparation() {
     RegistrationPreparePool::Instance().Close();
+    if (auto* prebuild = RegistrationPreparePool::ExistingPrebuild()) prebuild->Close();
 }
 
 std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target) {
@@ -860,7 +921,15 @@ std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decod
         }
         const auto bytes = handle->artifact->bindings.pushConstantSizeBytes;
         require(bytes <= room, "prepared graphics stages exceed the push constant block");
-        prepared.push_back({program.snapshot, {program.codeOffset, handle}});
+        std::shared_ptr<const ShaderRecompiler::RecompileResult> registration;
+        if (target.registrationModules) {
+            try {
+                registration = ShaderRecompiler::RegistrationResult(request, *handle);
+            } catch (const std::exception& error) {
+                APS5_LOG_OUT("shader library prebuild skipped: %s", error.what());
+            }
+        }
+        prepared.push_back({program.snapshot, {program.codeOffset, handle}, program.binary.stage, offset, std::move(registration)});
         pushOffset += bytes;
     }
     transaction.Commit();
@@ -930,6 +999,11 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
         if (stage.snapshot->type == 1) continue;
         ResolvePreparedGraphics(*stage.snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target());
     }
+    if (decoded.state.stages.path == Graphics::ShaderPath::Vertex) {
+        for (const auto& stage : prepared) {
+            if (stage.registration != nullptr) PrebuildStageLibrary(localDevice, decoded.state.stages, stage.stage, stage.registration, stage.pushOffset);
+        }
+    }
     PreparedShaderState::GraphicsAbi abi{key, registry, {}};
     for (const auto& stage : prepared) abi.stages.push_back(stage.snapshot);
     auto& graphicsAbis = transaction.Edit(*owner).graphicsAbis;
@@ -970,14 +1044,20 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegist
     struct RegisteredAbiKeyStorage {};
     auto& key = HostThreadLocal<std::vector<std::uint64_t>, RegisteredAbiKeyStorage>();
     BuildRegisteredAbiKey(state, *localDevice, key);
+    const auto prepare = [&] {
+        const auto plan = PlanRegistered(*snapshot, *localDevice, state, false);
+        if (plan == nullptr) return std::vector<PreparedShaders::Entry>{};
+        plan->prebuildDevice = localDevice;
+        return PrepareRegistered(*plan);
+    };
     const bool committedAbi = CommittedAbi(*snapshot, key);
-    auto entries = committedAbi ? std::vector<PreparedShaders::Entry>{} : PrepareRegistered(*snapshot, *localDevice, state, false);
+    auto entries = committedAbi ? std::vector<PreparedShaders::Entry>{} : prepare();
     const auto& registeredAbis = transaction.Read(*snapshot).registeredAbis;
     if (std::ranges::find(registeredAbis, key) != registeredAbis.end()) {
         transaction.Commit();
         return;
     }
-    if (committedAbi) entries = PrepareRegistered(*snapshot, *localDevice, state, false);
+    if (committedAbi) entries = prepare();
     auto& prepared = transaction.Edit(*snapshot);
     for (auto& entry : entries) {
         const auto duplicate = std::ranges::any_of(prepared.entries, [&](const auto& existing) { return existing.handle->artifact == entry.handle->artifact; });
@@ -1164,6 +1244,7 @@ void Driver::RegisterShader(const Shader* shader) {
     std::shared_ptr<RegisteredPreparation> preparation;
     if (AsyncRegistrationPrepare()) {
         preparation = PlanRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred);
+        if (preparation != nullptr) preparation->prebuildDevice = localDevice;
         if (preparation != nullptr && (snapshot.type == 0 || snapshot.type == 1)) BuildRegisteredAbiKey(registered, *localDevice, preparation->abiKey);
         snapshot.prepared->pending = preparation != nullptr;
     } else {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp

@@ -5,6 +5,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
+#include "prx/libc/include/HostThreadLocal.hpp"
 #include <cstdlib>
 #include <mutex>
 #include <set>
@@ -54,8 +55,8 @@ bool Driver::traceLateLabels() {
 }
 
 std::vector<Driver::PendingWait>& Driver::pendingWaits() {
-    static thread_local std::vector<PendingWait> waits;
-    return waits;
+    struct WaitsTag {};
+    return HostThreadLocal<std::vector<PendingWait>, WaitsTag>();
 }
 
 void Driver::notePendingWait(std::span<const std::uint32_t> packet, std::uint32_t queue, std::uint64_t received) {

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -423,6 +423,17 @@ struct PageStates {
         arena.forget(address, bytes);
         image.forget(address, bytes);
     }
+
+    void record(std::uintptr_t base, std::uintptr_t regionEnd, bool readable, bool writable, std::uint64_t generation) {
+        for (PageSpan* span : {&arena, &image}) {
+            if (!readable || span->size == 0 || regionEnd <= span->base || base >= span->base + span->size) continue;
+            const std::uint8_t value = PageReadable | (writable ? PageWritable : 0u);
+            const auto first = std::max(base, span->base);
+            const auto last = std::min(regionEnd, span->base + span->size);
+            for (auto at = first; at < last; at += PageBytes) span->store(at, value);
+            if (GuestAllocations::GuestAllocationsGeneration_nid_postfix() != generation) span->forget(first, static_cast<std::size_t>(last - first));
+        }
+    }
 };
 
 PageStates& Pages() {
@@ -533,18 +544,16 @@ bool describePages(std::uintptr_t address, std::size_t bytes, Emit&& emit) {
         const bool committed = memory.State == MEM_COMMIT && (memory.Protect & (PAGE_GUARD | PAGE_NOACCESS)) == 0;
         const bool readable = committed && readableProtection(protection);
         const bool writable = readable && writableProtection(protection);
-        for (PageSpan* span : {&pages.arena, &pages.image}) {
-            if (!readable || span->size == 0 || regionEnd <= span->base || base >= span->base + span->size) continue;
-            const std::uint8_t value = PageReadable | (writable ? PageWritable : 0u);
-            const auto first = std::max(base, span->base);
-            const auto last = std::min(regionEnd, span->base + span->size);
-            for (auto at = first; at < last; at += PageBytes) span->store(at, value);
-            if (GuestAllocations::GuestAllocationsGeneration_nid_postfix() != generation) span->forget(first, static_cast<std::size_t>(last - first));
-        }
+        pages.record(base, regionEnd, readable, writable, generation);
         const auto next = std::min(end, regionEnd);
         if (!emit(PageRun{cursor, next, readable, writable})) return true;
         cursor = next;
 #else
+        const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix();
+        const auto recordRun = [&](std::uintptr_t next, bool readable, bool writable) {
+            constexpr auto mask = static_cast<std::uintptr_t>(PageBytes - 1);
+            pages.record(cursor & ~mask, (next + mask) & ~mask, readable, writable, generation);
+        };
         if (const int fd = HostMapping::QueryFd(); fd >= 0) {
             procmap_query query{};
             query.size = sizeof(query);
@@ -559,8 +568,10 @@ bool describePages(std::uintptr_t address, std::size_t bytes, Emit&& emit) {
                     continue;
                 }
                 const bool readable = (query.vma_flags & PROCMAP_QUERY_VMA_READABLE) != 0;
+                const bool writable = readable && (query.vma_flags & PROCMAP_QUERY_VMA_WRITABLE) != 0;
                 const auto next = std::min<std::uintptr_t>(end, query.vma_end);
-                if (!emit(PageRun{cursor, next, readable, readable && (query.vma_flags & PROCMAP_QUERY_VMA_WRITABLE) != 0})) return true;
+                recordRun(next, readable, writable);
+                if (!emit(PageRun{cursor, next, readable, writable})) return true;
                 cursor = next;
                 continue;
             }
@@ -589,8 +600,11 @@ bool describePages(std::uintptr_t address, std::size_t bytes, Emit&& emit) {
                 found = true;
                 break;
             }
+            const bool readable = permissions[0] == 'r';
+            const bool writable = readable && permissions[1] == 'w';
             const auto next = std::min(end, last);
-            if (!emit(PageRun{cursor, next, permissions[0] == 'r', permissions[0] == 'r' && permissions[1] == 'w'})) return true;
+            recordRun(next, readable, writable);
+            if (!emit(PageRun{cursor, next, readable, writable})) return true;
             cursor = next;
             found = true;
             break;
@@ -780,7 +794,7 @@ struct WriteTracker {
 
     bool covers(std::uint64_t address, std::size_t bytes) const {
 #ifdef _WIN32
-        return coverage.Covers(address, bytes);
+        return coverage.Covers(address, bytes) && !GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(static_cast<std::uintptr_t>(address), bytes);
 #else
         return address + bytes <= LeafCount * LeafBlocks * WriteBlockBytes && GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(static_cast<std::uintptr_t>(address), bytes);
 #endif
@@ -859,6 +873,40 @@ struct WriteTracker {
 #endif
     }
 
+    static constexpr std::size_t UnstampedSlots = 1u << 16;
+    std::vector<std::uint32_t> unstampedStores;
+
+    void noteUnstampedStore(std::uint64_t address, std::uint64_t end) {
+        if (unstampedStores.empty()) unstampedStores.assign(UnstampedSlots, 0);
+        const auto first = address / WriteBlockBytes;
+        const auto last = (end - 1) / WriteBlockBytes;
+        if (last - first >= UnstampedSlots) {
+            std::fill(unstampedStores.begin(), unstampedStores.end(), generation);
+            return;
+        }
+        for (auto block = first; block <= last; ++block) unstampedStores[block % UnstampedSlots] = generation;
+    }
+
+    std::uint32_t newestStamp(std::uint64_t address, std::uint64_t end) const {
+        std::uint32_t newest = 0;
+        if (!unstampedStores.empty()) {
+            const auto first = address / WriteBlockBytes;
+            const auto last = (end - 1) / WriteBlockBytes;
+            if (last - first >= UnstampedSlots) newest = *std::max_element(unstampedStores.begin(), unstampedStores.end());
+            else for (auto block = first; block <= last; ++block) newest = std::max(newest, unstampedStores[block % UnstampedSlots]);
+        }
+        if (!watched) return newest;
+#ifdef _WIN32
+        address = std::max<std::uint64_t>(address, base);
+        end = std::min<std::uint64_t>(end, base + size);
+#else
+        end = std::min<std::uint64_t>(end, LeafCount * LeafBlocks * WriteBlockBytes);
+#endif
+        if (address >= end) return newest;
+        for (auto block = blockOf(address); block <= blockOf(end - 1); ++block) newest = std::max(newest, stampOf(block));
+        return newest;
+    }
+
     bool unchanged(std::uint64_t first, std::uint64_t last, std::uint64_t since) const {
         if (first == last) return stampOf(first) <= since;
         const auto newer = [since](std::uint32_t stamp) { return stamp > since; };
@@ -1059,9 +1107,6 @@ bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop,
 
 std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoized) {
     if (bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return 0;
-#ifdef _WIN32
-    if (GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(address, bytes)) return 0;
-#endif
     auto& tracker = Tracker();
     // Whole pages, so a page shared with the next range is collected with either.
     constexpr std::uint64_t page = 4096;
@@ -1190,9 +1235,6 @@ bool WriteWatched() {
 
 bool Watched(std::uint64_t address, std::size_t bytes) {
     if (bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return false;
-#ifdef _WIN32
-    if (GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(address, bytes)) return false;
-#endif
     auto& tracker = Tracker();
     const auto lock = lockTracker(tracker);
     tracker.initialize();
@@ -1201,9 +1243,6 @@ bool Watched(std::uint64_t address, std::size_t bytes) {
 
 bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t generation) {
     if (bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return false;
-#ifdef _WIN32
-    if (GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(address, bytes)) return false;
-#endif
     auto& tracker = Tracker();
     const auto lock = lockTracker(tracker);
     tracker.initialize();
@@ -1220,9 +1259,6 @@ bool UnchangedSinceAll(std::span<const UnchangedQuery> queries) {
     if (!tracker.watched) return false;
     for (const auto& [address, bytes, generation] : queries) {
         if (generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false;
-#ifdef _WIN32
-        if (GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(address, bytes)) return false;
-#endif
         const auto first = tracker.blockOf(address);
         const auto last = tracker.blockOf(address + bytes - 1);
         if (!tracker.unchanged(first, last, generation)) return false;
@@ -1235,7 +1271,10 @@ std::uint64_t storeOwn(std::uint64_t address, std::size_t bytes, const std::func
     const auto stampStored = [](WriteTracker& tracker, std::pair<std::uint64_t, std::uint64_t> stored) -> std::uint64_t {
         if (stored.second <= stored.first) return 0;
         ++tracker.generation;
-        if (!tracker.watched || !tracker.covers(stored.first, static_cast<std::size_t>(stored.second - stored.first))) return 0;
+        if (!tracker.watched || !tracker.covers(stored.first, static_cast<std::size_t>(stored.second - stored.first))) {
+            tracker.noteUnstampedStore(stored.first, stored.second);
+            return 0;
+        }
         for (auto block = tracker.blockOf(stored.first); block <= tracker.blockOf(stored.second - 1); ++block) {
             tracker.stamp(block, tracker.generation, StampKind::Driver);
             tracker.noteDriverStore(block, stored.first, stored.second, tracker.generation);
@@ -1249,9 +1288,6 @@ std::uint64_t storeOwn(std::uint64_t address, std::size_t bytes, const std::func
     constexpr std::uint64_t page = 4096;
     const auto first = address & ~(page - 1);
     const auto stop = (address + bytes + page - 1) & ~(page - 1);
-#ifdef _WIN32
-    if (GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(first, static_cast<std::size_t>(stop - first))) return stampStored(tracker, store());
-#endif
     if (!walkWrites(tracker, first, stop, StampKind::Cpu)) return stampStored(tracker, store());
     const auto stored = store();
     walkWrites(tracker, first, stop, StampKind::Driver);
@@ -1280,7 +1316,10 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
     tracker.initialize();
     if (bytes == 0) return 0;
     ++tracker.generation;
-    if (!tracker.watched || !tracker.covers(address, bytes)) return 0;
+    if (!tracker.watched || !tracker.covers(address, bytes)) {
+        tracker.noteUnstampedStore(address, address + bytes);
+        return 0;
+    }
 #ifndef _WIN32
     {
         const auto blockFirst = address & ~(std::uint64_t{WriteBlockBytes} - 1);
@@ -1288,6 +1327,7 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
         StampRuns runs{tracker, StampKind::Fresh};
         if (!GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(static_cast<std::uintptr_t>(blockFirst), static_cast<std::size_t>(blockStop - blockFirst), &stampWrittenRun, &runs)) {
             unwatchLocked(tracker, blockFirst, static_cast<std::size_t>(blockStop - blockFirst));
+            tracker.noteUnstampedStore(address, address + bytes);
             return 0;
         }
         if (runs.stamped != 0) ++tracker.generation;
@@ -1307,6 +1347,9 @@ bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generati
     const auto lock = lockTracker(tracker);
     tracker.initialize();
     if (!tracker.watched || generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return true;
+#ifdef _WIN32
+    if (GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(address, bytes)) return true;
+#endif
     const auto end = address + bytes;
     const auto first = tracker.blockOf(address);
     const auto last = tracker.blockOf(end - 1);
@@ -1344,6 +1387,13 @@ std::uint64_t TrackerGeneration() {
     return tracker.generation;
 }
 
+std::uint64_t NewestStamp(std::uint64_t address, std::size_t bytes) {
+    auto& tracker = Tracker();
+    const auto lock = lockTracker(tracker);
+    tracker.initialize();
+    return bytes == 0 ? 0 : tracker.newestStamp(address, address + bytes);
+}
+
 bool UnchangedSinceCollected(std::uint64_t address, std::size_t bytes, std::uint64_t generation) {
     if (CollectWritesUncached(address, bytes) == 0) return false;
     auto& tracker = Tracker();

core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp

@@ -249,7 +249,7 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
                 require((packet[1] & 0xffff0000u) != 0x68750000u, "typed user-data and legacy flip markers are not implemented");
                 break;
             case 0x09: size(2); break;
-            case 0x06: graphics(); size(4); require(packet[3] == 0, "unsupported rendering wait mode"); break;
+            case 0x06: size(4); require(packet[3] == 0, "unsupported rendering wait mode"); break;
             case 0x0b: {
                 const auto data = std::as_bytes(packet.subspan(1));
                 require(std::find(data.begin(), data.end(), std::byte{}) != data.end(), "unterminated marker text");

core/libs/prx/libSceAgcDriver/Execution/src/ShaderDeviceProfile.cpp

@@ -47,6 +47,7 @@ ShaderDeviceProfile::ShaderDeviceProfile(const ShaderRecompiler::SpirvTarget& ta
     checkFeature(spv::CapabilityFloat64, core.shaderFloat64 == VK_TRUE, "shaderFloat64");
     checkFeature(spv::CapabilityTessellation, core.tessellationShader == VK_TRUE, "tessellationShader");
     checkFeature(spv::CapabilityImageGatherExtended, core.shaderImageGatherExtended == VK_TRUE, "shaderImageGatherExtended");
+    checkFeature(spv::CapabilityClipDistance, core.shaderClipDistance == VK_TRUE, "shaderClipDistance");
     checkFeature(spv::CapabilityMinLod, core.shaderResourceMinLod == VK_TRUE, "shaderResourceMinLod");
     checkFeature(spv::CapabilityStorageImageMultisample, core.shaderStorageImageMultisample == VK_TRUE, "shaderStorageImageMultisample");
     checkFeature(spv::CapabilityStorageImageReadWithoutFormat, core.shaderStorageImageReadWithoutFormat == VK_TRUE, "shaderStorageImageReadWithoutFormat");

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -13,6 +13,7 @@
 #include "prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp"
 #include "prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp"
+#include "prx/libSceAgcDriver/Execution/include/PresentMode.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp"
@@ -241,6 +242,7 @@ struct VulkanDevice::State {
     VkExtent2D extent{};
     std::vector<VkImage> images;
     VkFence acquireFence = VK_NULL_HANDLE;
+    VkPresentModeKHR presentMode = VK_PRESENT_MODE_FIFO_KHR;
     struct RetiredSwapchain {
         VkSwapchainKHR swapchain = VK_NULL_HANDLE;
         std::vector<VkSemaphore> rendered;
@@ -262,6 +264,7 @@ struct VulkanDevice::State {
     bool fragmentShaderBarycentric = false;
     bool geometryShader = false;
     bool sampleRateShading = false;
+    bool shaderClipDistance = false;
     bool shaderClock = false;
     bool narrowSubgroupClock = false;
     bool float64DenormPreserve = false;
@@ -284,6 +287,7 @@ struct VulkanDevice::State {
     bool maintenance8 = false;
     std::uint32_t srgbDecodeFormats = 0;
     bool storageImageWithoutFormat = false;
+    VkSampleCountFlags sampleLocationSampleCounts = 0;
     bool depthClamp = false;
     bool depthBounds = false;
     bool depthBiasClamp = false;
@@ -590,13 +594,19 @@ struct VulkanDevice::State {
 
     void WriteFrameDump(const PresentSlot& slot) {
         require(slot.dumpBuffer != nullptr, "frame dump was not recorded");
-        std::vector<std::byte> full(slot.dumpBuffer->Bytes().begin(), slot.dumpBuffer->Bytes().end());
+        const auto full = slot.dumpBuffer->Bytes();
         const auto index = slot.dumpIndex;
-        const auto width = slot.dumpWidth;
-        const auto height = slot.dumpHeight;
+        const auto scale = slot.dumpScale;
+        require(scale != 0 && full.size() == static_cast<std::size_t>(slot.dumpWidth) * slot.dumpHeight * 4, "invalid frame dump pixels");
+        const auto width = (slot.dumpWidth + scale - 1) / scale;
+        const auto height = (slot.dumpHeight + scale - 1) / scale;
+        std::vector<std::byte> pixels(static_cast<std::size_t>(width) * height * 4);
+        for (std::uint32_t y = 0; y < height; ++y) {
+            for (std::uint32_t x = 0; x < width; ++x) std::memcpy(pixels.data() + (static_cast<std::size_t>(y) * width + x) * 4, full.data() + (static_cast<std::size_t>(y) * scale * slot.dumpWidth + static_cast<std::size_t>(x) * scale) * 4, 4);
+        }
         static const auto start = std::chrono::steady_clock::now();
-        std::fprintf(stderr, "[gpu] frame_%03d.bmp: %ux%u display read back on the GPU at %.1f s\n", index, width, height, std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count());
-        dumpWriter.Enqueue(index, width, height, slot.dumpScale, std::move(full));
+        std::fprintf(stderr, "[gpu] frame_%03d.bmp: %ux%u display read back on the GPU at %.1f s\n", index, slot.dumpWidth, slot.dumpHeight, std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count());
+        dumpWriter.Enqueue(index, width, height, 1, std::move(pixels));
     }
 
     // Blocks on the fence, or polls it every 50 us with APS5_PRESENT_POLL_FENCE=1 (the difference
@@ -1093,6 +1103,11 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         state->capabilities.push_back(spv::CapabilityFragmentShaderPixelInterlockEXT);
         state->spirvExtensions.push_back("SPV_EXT_fragment_shader_interlock");
     }
+    if (hasExtension(VK_EXT_SHADER_STENCIL_EXPORT_EXTENSION_NAME)) {
+        deviceExtensions.push_back(VK_EXT_SHADER_STENCIL_EXPORT_EXTENSION_NAME);
+        state->capabilities.push_back(spv::CapabilityStencilExportEXT);
+        state->spirvExtensions.push_back("SPV_EXT_shader_stencil_export");
+    }
     if (state->shaderClock) {
         deviceExtensions.push_back(VK_KHR_SHADER_CLOCK_EXTENSION_NAME);
         state->capabilities.push_back(spv::CapabilityShaderClockKHR);
@@ -1150,6 +1165,31 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     state->spirvExtensions.push_back("SPV_KHR_physical_storage_buffer");
     state->depthRangeUnrestricted = hasExtension(VK_EXT_DEPTH_RANGE_UNRESTRICTED_EXTENSION_NAME);
     if (state->depthRangeUnrestricted) deviceExtensions.push_back(VK_EXT_DEPTH_RANGE_UNRESTRICTED_EXTENSION_NAME);
+    if (hasExtension(VK_EXT_SAMPLE_LOCATIONS_EXTENSION_NAME)) {
+        VkPhysicalDeviceSampleLocationsPropertiesEXT locations{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLE_LOCATIONS_PROPERTIES_EXT};
+        VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &locations};
+        state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &properties);
+        if (locations.sampleLocationSubPixelBits >= 4 && locations.sampleLocationCoordinateRange[0] <= 0.0f && locations.sampleLocationCoordinateRange[1] >= 15.0f / 16.0f) {
+            const auto multisampleProperties = state->InstanceFunction<PFN_vkGetPhysicalDeviceMultisamplePropertiesEXT>("vkGetPhysicalDeviceMultisamplePropertiesEXT");
+            for (const auto samples : {VK_SAMPLE_COUNT_2_BIT, VK_SAMPLE_COUNT_4_BIT, VK_SAMPLE_COUNT_8_BIT}) {
+                if ((locations.sampleLocationSampleCounts & samples) == 0) continue;
+                VkMultisamplePropertiesEXT multisample{VK_STRUCTURE_TYPE_MULTISAMPLE_PROPERTIES_EXT};
+                multisampleProperties(selected, samples, &multisample);
+                if (multisample.maxSampleLocationGridSize.width != 0 && multisample.maxSampleLocationGridSize.height != 0) state->sampleLocationSampleCounts |= samples;
+            }
+        }
+        if (state->sampleLocationSampleCounts != 0) deviceExtensions.push_back(VK_EXT_SAMPLE_LOCATIONS_EXTENSION_NAME);
+    }
+    const char* fifoLatestReadyExtension = hasExtension(VK_KHR_PRESENT_MODE_FIFO_LATEST_READY_EXTENSION_NAME) ? VK_KHR_PRESENT_MODE_FIFO_LATEST_READY_EXTENSION_NAME : hasExtension("VK_EXT_present_mode_fifo_latest_ready") ? "VK_EXT_present_mode_fifo_latest_ready" : nullptr;
+    const char* requestedPresentMode = std::getenv("ANYPS5_PRESENT_MODE");
+    VkPhysicalDevicePresentModeFifoLatestReadyFeaturesKHR fifoLatestReadyFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PRESENT_MODE_FIFO_LATEST_READY_FEATURES_KHR};
+    bool fifoLatestReady = false;
+    if (requestedPresentMode != nullptr && std::strcmp(requestedPresentMode, "fifo_latest_ready") == 0 && fifoLatestReadyExtension != nullptr) {
+        VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &fifoLatestReadyFeatures};
+        state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &features);
+        fifoLatestReady = fifoLatestReadyFeatures.presentModeFifoLatestReady == VK_TRUE;
+        if (fifoLatestReady) deviceExtensions.push_back(fifoLatestReadyExtension);
+    }
     // Guest dispatches cover whole thread groups past the edge of small images; robust image access
     // drops those writes instead of faulting the device.
     const bool imageRobustness = hasExtension(VK_EXT_IMAGE_ROBUSTNESS_EXTENSION_NAME);
@@ -1348,6 +1388,8 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     state->sampleRateShading = enabled.sampleRateShading == VK_TRUE;
     if (enabled.geometryShader) state->capabilities.push_back(spv::CapabilityGeometry);
     enabled.shaderClipDistance = available.shaderClipDistance;
+    state->shaderClipDistance = enabled.shaderClipDistance == VK_TRUE;
+    if (enabled.shaderClipDistance) state->capabilities.push_back(spv::CapabilityClipDistance);
     if (enabled.shaderStorageImageWriteWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageWriteWithoutFormat);
     if (enabled.shaderStorageImageReadWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageReadWithoutFormat);
     state->storageImageWithoutFormat = enabled.shaderStorageImageWriteWithoutFormat == VK_TRUE && enabled.shaderStorageImageReadWithoutFormat == VK_TRUE;
@@ -1417,6 +1459,12 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         minLodFeatures.pNext = const_cast<void*>(deviceInfo.pNext);
         deviceInfo.pNext = &minLodFeatures;
     }
+    fifoLatestReadyFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PRESENT_MODE_FIFO_LATEST_READY_FEATURES_KHR};
+    fifoLatestReadyFeatures.presentModeFifoLatestReady = VK_TRUE;
+    if (fifoLatestReady) {
+        fifoLatestReadyFeatures.pNext = const_cast<void*>(deviceInfo.pNext);
+        deviceInfo.pNext = &fifoLatestReadyFeatures;
+    }
     if (state->maintenance8) {
         maintenance8Features.pNext = const_cast<void*>(deviceInfo.pNext);
         deviceInfo.pNext = &maintenance8Features;
@@ -1593,6 +1641,13 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         std::vector<VkSurfaceFormatKHR> formats(formatCount);
         check(getFormats(selected, state->surface, &formatCount, formats.data()), "vkGetPhysicalDeviceSurfaceFormatsKHR");
         require(std::any_of(formats.begin(), formats.end(), [](const auto& format) { return format.format == VK_FORMAT_B8G8R8A8_UNORM && format.colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR; }), "BGRA8 sRGB-nonlinear surface format is unavailable");
+        std::uint32_t presentModeCount = 0;
+        auto getPresentModes = state->InstanceFunction<PFN_vkGetPhysicalDeviceSurfacePresentModesKHR>("vkGetPhysicalDeviceSurfacePresentModesKHR");
+        check(getPresentModes(selected, state->surface, &presentModeCount, nullptr), "vkGetPhysicalDeviceSurfacePresentModesKHR");
+        std::vector<VkPresentModeKHR> presentModes(presentModeCount);
+        check(getPresentModes(selected, state->surface, &presentModeCount, presentModes.data()), "vkGetPhysicalDeviceSurfacePresentModesKHR");
+        presentModes.resize(presentModeCount);
+        state->presentMode = SelectPresentMode(std::getenv("ANYPS5_PRESENT_MODE"), presentModes);
         VkSwapchainCreateInfoKHR swapchain{VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR};
         swapchain.surface = state->surface;
         swapchain.minImageCount = surface.minImageCount;
@@ -1604,7 +1659,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         swapchain.imageSharingMode = VK_SHARING_MODE_EXCLUSIVE;
         swapchain.preTransform = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR;
         swapchain.compositeAlpha = VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR;
-        swapchain.presentMode = VK_PRESENT_MODE_FIFO_KHR;
+        swapchain.presentMode = state->presentMode;
         swapchain.clipped = VK_FALSE;
         check(state->DeviceFunction<PFN_vkCreateSwapchainKHR>("vkCreateSwapchainKHR")(state->device, &swapchain, nullptr, &state->swapchain), "vkCreateSwapchainKHR");
         std::uint32_t imageCount = 0;
@@ -2283,7 +2338,7 @@ void VulkanDevice::Resize(std::uint32_t width, std::uint32_t height) {
     create.imageSharingMode = VK_SHARING_MODE_EXCLUSIVE;
     create.preTransform = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR;
     create.compositeAlpha = VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR;
-    create.presentMode = VK_PRESENT_MODE_FIFO_KHR;
+    create.presentMode = state->presentMode;
     create.oldSwapchain = state->swapchain;
     state->retiredSwapchains.reserve(state->retiredSwapchains.size() + 1);
     VkSwapchainKHR replacement = VK_NULL_HANDLE;
@@ -2469,6 +2524,7 @@ FrameDumps& Dumps() {
 bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) {
     static const bool storeAtFlip = Graphics::StorageTexture::StoreAtFlipRequested(std::getenv("APS5_STORE_AT_FLIP"));
     if (storeAtFlip) Graphics::StorageTexture::FlushAllPending("flip");
+    else Graphics::StorageTexture::FlushDumpSelected("flip");
     if (buffer.tilingMode == 1) {
         const auto pixels = ReadDisplayBuffer(buffer);
         return present(buffer.width, buffer.height, true, pixels);
@@ -2907,6 +2963,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const {
     target.narrowSubgroupClock = state->narrowSubgroupClock;
     target.float64DenormPreserve = state->float64DenormPreserve;
     target.fixedPushSlots = state->graphicsPipelineLibrary;
+    target.registrationModules = state->graphicsPipelineLibrary;
     target.srgbDecodeFormats = state->srgbDecodeFormats;
     if (state->meshShader) {
         const auto& mesh = state->meshLimits;
@@ -3012,6 +3069,7 @@ Graphics::Context VulkanDevice::buildContext() const {
     context.imageInt64Atomics = state->imageInt64Atomics;
     context.geometryShader = state->geometryShader;
     context.sampleRateShading = state->sampleRateShading;
+    context.shaderClipDistance = state->shaderClipDistance;
     context.nullDescriptors = state->shaderProfile != nullptr && state->shaderProfile->NullDescriptors();
     context.bufferInt64Atomics = state->bufferInt64Atomics;
     context.placeholderImages = state->placeholderImages.get();
@@ -3019,6 +3077,7 @@ Graphics::Context VulkanDevice::buildContext() const {
     context.imageViewMinLod = state->imageViewMinLod;
     context.pipelineExecutableInfo = state->pipelineExecutableInfo;
     context.srgbDecodeFormats = state->srgbDecodeFormats;
+    context.sampleLocationSampleCounts = state->sampleLocationSampleCounts;
     context.memoryProperties2 = state->memoryBudget ? state->InstanceFunction<PFN_vkGetPhysicalDeviceMemoryProperties2>("vkGetPhysicalDeviceMemoryProperties2") : nullptr;
     context.singlePassStorage = state->storageImageWithoutFormat;
     return context;
@@ -3036,6 +3095,44 @@ void VulkanDevice::ColorMetadataPass(const Graphics::ColorMetadataPass& pass) {
     Graphics::RunColorMetadataPass(graphicsContext(), pass);
 }
 
+bool VulkanDevice::PrebuildGraphicsLibraries(const Graphics::ShaderStages& stages, std::span<const Graphics::CompiledShader> shaders) {
+    PerformanceTimer timing("Vulkan.GraphicsPipelinePrebuild");
+    const auto context = graphicsContext();
+    if (!context.graphicsPipelineLibrary || context.descriptorCache == nullptr) return false;
+    Graphics::State state{};
+    state.stages = stages;
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.color.format = VK_FORMAT_R8G8B8A8_UNORM;
+    state.color.exportIndex = 0;
+    state.colors.push_back(state.color);
+    state.hasColorTarget = true;
+    state.blend.colorWriteMask = 15;
+    state.blends.push_back(state.blend);
+    state.renderExtent = {1, 1};
+    std::vector<Graphics::CompiledShader> group(shaders.begin(), shaders.end());
+    if (group.size() == 1u) {
+        const bool vertex = group.front().stage == ShaderRecompiler::ShaderStage::Fragment;
+        static const auto stub = [](bool vertexStub) {
+            auto result = std::make_shared<ShaderRecompiler::RecompileResult>();
+            const std::uint32_t model = vertexStub ? 0u : 4u;
+            std::vector<std::uint32_t> words{0x07230203u, 0x00010000u, 0u, 6u, 0u, 0x00020011u, 1u, 0x0003000eu, 0u, 1u, 0x0005000fu, model, 4u, 0x6e69616du, 0u};
+            if (!vertexStub) words.insert(words.end(), {0x00030010u, 4u, 7u});
+            words.insert(words.end(), {0x00020013u, 2u, 0x00030021u, 3u, 2u, 0x00050036u, 2u, 4u, 0u, 3u, 0x000200f8u, 5u, 0x000100fdu, 0x00010038u});
+            result->spirv = std::move(words);
+            result->variantId = vertexStub ? ~std::uint64_t{0} - 1u : ~std::uint64_t{0} - 2u;
+            result->runtimeAbiVersion = ShaderRecompiler::RuntimeAbi::Version;
+            return result;
+        };
+        static const auto vertexStub = stub(true);
+        static const auto fragmentStub = stub(false);
+        if (vertex) group.insert(group.begin(), {ShaderRecompiler::ShaderStage::Vertex, vertexStub.get(), 0u});
+        else group.push_back({ShaderRecompiler::ShaderStage::Fragment, fragmentStub.get(), Graphics::PipelinePushSlotBytes});
+    }
+    const auto layouts = Graphics::DescribeSetLayouts(context, group);
+    return Graphics::Pipeline::Prebuild(context, state, Graphics::VertexInputLayout{}, layouts.View(), group);
+}
+
 void VulkanDevice::Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span<const Graphics::CompiledShader> shaders, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::shared_ptr<const DrawRecipe>* recipe) {
     for (const auto& shader : shaders) {
         require(shader.program != nullptr, "missing compiled shader");

core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp

@@ -2,6 +2,7 @@
 #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_COLORTARGETLAYOUT_HPP
 
 #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
 #include <cstddef>
 #include <cstdint>
 #include <span>
@@ -33,6 +34,7 @@ constexpr TextureTileMode ColorTextureTileMode(ColorTileMode mode) {
 }
 
 ColorTileMode DecodeColorTileMode(std::uint32_t attrib3);
+TileMipLayout ComputeColorTargetMipLayout(ColorTileMode mode, std::uint32_t bytesPerElement, std::uint32_t surfaceWidth, std::uint32_t surfaceHeight, std::uint32_t mipCount, std::uint32_t mipLevel);
 
 class ColorTargetLayout {
 public:

core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp

@@ -5,7 +5,9 @@
 #define VK_NO_PROTOTYPES
 #endif
 #include <vulkan/vulkan.h>
+#include <atomic>
 #include <cstdint>
+#include <cstdio>
 #include <memory>
 #include <stdexcept>
 #include <string>
@@ -158,6 +160,7 @@ struct Context {
     bool imageInt64Atomics = false;
     bool geometryShader = false;
     bool sampleRateShading = false;
+    bool shaderClipDistance = false;
     bool nullDescriptors = false;
     bool bufferInt64Atomics = false;
     const PlaceholderImages* placeholderImages = nullptr;
@@ -170,6 +173,7 @@ struct Context {
     bool provokingVertexLast = false;
     bool provokingVertexModePerPipeline = false;
     bool singlePassStorage = false;
+    VkSampleCountFlags sampleLocationSampleCounts = 0;
 
     template<typename TFunction>
     TFunction Function(const char* name) const {
@@ -193,6 +197,28 @@ struct Context {
         }
         throw std::runtime_error("AGC graphics: required Vulkan memory type is unavailable");
     }
+
+    std::uint32_t DeviceMemoryType(std::uint32_t mask, VkDeviceSize bytes, const char* what) const {
+        const auto type = MemoryType(mask, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+        if (memoryProperties2 == nullptr || physical == VK_NULL_HANDLE) return type;
+        VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
+        VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2};
+        properties.pNext = &reported;
+        memoryProperties2(physical, &properties);
+        const auto heap = memory.memoryTypes[type].heapIndex;
+        const auto budget = reported.heapBudget[heap];
+        const auto usage = reported.heapUsage[heap];
+        if (budget == 0 || (usage < budget && bytes <= budget - usage)) return type;
+        for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) {
+            if ((mask & (1u << i)) == 0 || (memory.memoryHeaps[memory.memoryTypes[i].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) continue;
+            static std::atomic<std::uint64_t> placements{0}, placedBytes{0};
+            const auto count = placements.fetch_add(1) + 1;
+            const auto total = placedBytes.fetch_add(bytes) + bytes;
+            if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local heap %u uses %llu of its %llu MiB budget, %llu bytes placed in host memory type %u; %llu such allocations, %llu MiB\n", what, heap, static_cast<unsigned long long>(usage >> 20u), static_cast<unsigned long long>(budget >> 20u), static_cast<unsigned long long>(bytes), i, static_cast<unsigned long long>(count), static_cast<unsigned long long>(total >> 20u));
+            return i;
+        }
+        return type;
+    }
 };
 
 inline void EndRenderPass(const Context& context, VkCommandBuffer commands) {

core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp

@@ -13,9 +13,11 @@ namespace AgcDriver::Graphics {
 class Texture;
 
 VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target);
+bool SampleLocationsCompatibleDepth(const Context& context, VkFormat format, std::uint32_t samples);
 std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel);
 void ClearDepthSurfaces(VkDevice device);
 bool DepthSurfaceAt(std::uint64_t address);
+bool DepthSurfaceAt(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource);
 std::uint64_t HtileDepthClearAddress(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const std::array<std::uint32_t, 3>& numThreads);
 void NoteHtileDepthClear(std::uint64_t htileAddress);
 bool DepthSurfaceHolds(const Context& context, const GuestTextureResource& resource, const Texture* texture);

core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp

@@ -32,6 +32,11 @@ DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uin
 DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes);
 void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCopy& copy, Recorder::SnapshotUse use, std::uint32_t derived);
 const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment);
+struct InPlaceIndices {
+    std::optional<std::uint32_t> highest;
+    bool reused = false;
+};
+InPlaceIndices InPlaceIndexRange(Recorder& recorder, std::uint64_t address, std::size_t bytes, std::uint32_t indexSize, bool skipRestart);
 
 std::array<std::uint32_t, 4> MeshIndexBufferDescriptor(const Pm4::DrawParameters& draw);
 

core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp

@@ -41,7 +41,9 @@ public:
     // before, during and after the pass (GENERAL for resident targets, which then need no
     // transitions).
     Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+    Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
     ~Pipeline();
+    static bool Prebuild(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders);
     Pipeline(const Pipeline&) = delete;
     Pipeline& operator=(const Pipeline&) = delete;
     VkPipelineLayout Layout() const;
@@ -87,6 +89,7 @@ private:
     VkFormat depthFormat = VK_FORMAT_UNDEFINED;
     VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_UNDEFINED;
     std::vector<std::uint32_t> colorSlots;
+    bool sampleLocationsEnabled = false;
     std::vector<CachedFramebuffer> framebuffers;
 };
 
@@ -97,6 +100,15 @@ private:
 // only evicted once no recorded draw holds them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per
 // draw as before.
 std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+struct DescribedSetLayouts {
+    PipelineSetLayouts layouts;
+    std::vector<std::uint32_t> mainKey;
+    std::vector<std::uint32_t> fragmentKey;
+    PipelineSetLayouts View() const { return {layouts.main, layouts.fragment, layouts.split, mainKey, fragmentKey}; }
+};
+DescribedSetLayouts DescribeSetLayouts(const Context& context, std::span<const CompiledShader> shaders);
+bool SplitsSets(const Context& context, std::span<const CompiledShader> shaders);
 // Destroys the cached pipelines of a device; to be called before the device goes away. Without it,
 // entries of a gone device are recognised by their buffer pool (made and reset with the device, so
 // it tells device instances apart when the loader reuses a VkDevice handle) and forgotten unused.
@@ -106,11 +118,12 @@ bool HasStencil(VkFormat format);
 void ValidateViewport(const Context& context, const VkViewport& viewport);
 void ValidateDepthBounds(const Context& context, const State& state);
 void ValidateProvokingVertex(const Context& context, const State& state, std::span<const CompiledShader> shaders = {});
+bool ValidateSampleLocations(const Context& context, const State& state);
 
 void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment);
 bool ShadersWriteMemory(std::span<const CompiledShader> shaders);
 // Returns the color attachment locations the pixel shader writes.
-std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false, bool bufferInt64Atomics = false);
+std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false, bool bufferInt64Atomics = false, bool shaderClipDistance = false);
 
 }
 

core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp

@@ -19,6 +19,8 @@ struct PipelineLibraryKeys {
     std::vector<std::byte> fragmentOutput;
     std::vector<std::byte> renderPass;
     std::vector<std::byte> layout;
+    std::vector<std::byte> preRasterizationLayout;
+    std::vector<std::byte> fragmentLayout;
 };
 
 std::span<const VkDynamicState> PipelineLibraryDynamicStates();
@@ -31,7 +33,7 @@ struct OptimizedPipeline {
     void Release() noexcept;
 };
 
-VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr<OptimizedPipeline>* optimized = nullptr, std::span<VkShaderModule> modules = {});
+VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr<OptimizedPipeline>* optimized = nullptr, bool partsOnly = false, std::span<VkShaderModule> modules = {});
 
 void WaitForOptimizedPipelines(VkDevice device);
 

core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp

@@ -99,6 +99,10 @@ public:
     static constexpr std::size_t DrawInputEntries = 16384;
     std::shared_ptr<Buffer> ReusableDrawSnapshot(std::uint64_t address, std::size_t bytes, SnapshotUse use = SnapshotUse::Storage, std::uint32_t* derived = nullptr, std::uint64_t generation = 0);
     void KeepDrawSnapshot(std::uint64_t address, std::size_t bytes, std::uint64_t generation, std::uint64_t registryGeneration, std::shared_ptr<Buffer> buffer, SnapshotUse use = SnapshotUse::Storage, std::uint32_t derived = 0);
+    static constexpr std::size_t IndexRangeEntries = 16384;
+    static constexpr std::size_t KeptIndexRangeIndices = 4096;
+    bool KnownHighestIndex(std::uint64_t address, std::size_t bytes, SnapshotUse use, std::optional<std::uint32_t>& highest);
+    void KeepHighestIndex(std::uint64_t address, std::size_t bytes, SnapshotUse use, std::uint64_t generation, std::uint64_t registryGeneration, std::optional<std::uint32_t> highest);
     void OnComplete(std::function<void()> action);
     void NotePendingWrite(std::uint64_t address, std::size_t bytes);
     void NotePendingFill(std::uint64_t address, std::size_t bytes, std::uint8_t value);
@@ -808,6 +812,14 @@ private:
     std::map<DrawSnapshotKey, DrawSnapshot> drawSnapshots;
     std::array<DrawSnapshotPool, 2> drawSnapshotPools;
     void eraseDrawSnapshot(std::map<DrawSnapshotKey, DrawSnapshot>::iterator entry);
+    struct IndexRange {
+        std::uint64_t generation;
+        std::uint64_t registryGeneration;
+        std::list<DrawSnapshotKey>::iterator recent;
+        std::optional<std::uint32_t> highest;
+    };
+    std::map<DrawSnapshotKey, IndexRange> indexRanges;
+    std::list<DrawSnapshotKey> indexRangeRecency;
     bool refreshDrawSnapshot(DrawSnapshot& entry, std::uint64_t address, std::size_t bytes);
 };
 

core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp

@@ -51,6 +51,14 @@ bool StorageImageServesKeys(const StorageTexture& image, std::uint64_t dccAddres
 // a storage image other than `except` has results pending in [address, address + bytes).
 bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const StorageTexture* except);
 
+struct PipelineSetLayouts {
+    VkDescriptorSetLayout main = VK_NULL_HANDLE;
+    VkDescriptorSetLayout fragment = VK_NULL_HANDLE;
+    bool split = false;
+    std::span<const std::uint32_t> mainKey;
+    std::span<const std::uint32_t> fragmentKey;
+};
+
 class DescriptorCache {
 public:
     explicit DescriptorCache(const Context& context);
@@ -117,6 +125,9 @@ public:
     std::vector<VkDescriptorSetLayoutBinding> LayoutBindings() const;
     bool LayoutUpdateAfterBind() const;
     std::span<const VkDescriptorBindingFlags> LayoutBindingFlags() const { return layoutBindingFlags; }
+    VkDescriptorSetLayout FragmentLayout() const { return fragment.layout; }
+    bool SplitSets() const { return splitSets; }
+    PipelineSetLayouts SetLayouts() const { return {_layout, fragment.layout, splitSets, layoutKey, fragmentLayoutKey}; }
     void Bind(VkCommandBuffer commands, VkPipelineBindPoint bindPoint, VkPipelineLayout layout) const;
     struct DrawBindings {
         DrawBindings() = default;
@@ -127,6 +138,7 @@ public:
             std::shared_ptr<Buffer> buffer;
         };
         DescriptorCache* cache = nullptr;
+        DescriptorCache::SetAllocation fragmentAllocation;
         DescriptorCache::SetAllocation allocation;
         std::vector<Snapshot> snapshots;
         ~DrawBindings();
@@ -165,6 +177,7 @@ public:
     bool ReadsImage(const StorageTexture* image) const;
     bool HasCachedStorageImages() const;
     const std::vector<std::uint32_t>& LayoutKey() const { return layoutKey; }
+    const std::vector<std::uint32_t>& FragmentLayoutKey() const { return fragmentLayoutKey; }
     // Debug aid: each bound guest resource with the fraction of sampled bytes that are nonzero.
     std::string Describe() const;
 
@@ -428,6 +441,17 @@ private:
     VkDescriptorPool pool = VK_NULL_HANDLE;
     // The cache pool the set was allocated from, freed back to it on release.
     DescriptorCache::SetAllocation cacheAllocation;
+    struct SetObjects {
+        VkDescriptorSetLayout layout = VK_NULL_HANDLE;
+        bool ownsLayout = false;
+        VkDescriptorSet set = VK_NULL_HANDLE;
+        VkDescriptorPool pool = VK_NULL_HANDLE;
+        DescriptorCache::SetAllocation cacheAllocation;
+    };
+    bool splitSets = false;
+    std::vector<std::uint32_t> fragmentLayoutKey;
+    SetObjects fragment;
+    std::uint32_t setOf(const Binding& binding) const;
     std::vector<Allocation> allocations;
     std::vector<VkDescriptorBufferInfo> descriptorBuffers;
     std::vector<VkDescriptorImageInfo> descriptorImages;
@@ -479,6 +503,7 @@ private:
     // the descriptor counts the set was sized for, and the compute stage of a deferred build.
     std::vector<Binding> bindings;
     std::vector<VkDescriptorBindingFlags> layoutBindingFlags;
+    std::vector<VkDescriptorBindingFlags> fragmentBindingFlags;
     std::vector<std::uint32_t> refreshResourceKey;
     struct DeferredImages {
         const ShaderRecompiler::DescriptorBinding* binding;

core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp

@@ -116,10 +116,13 @@ struct State {
     VkPipelineColorBlendAttachmentState blend;
     std::array<float, 4> blendConstants;
     std::uint32_t samples = 1;
+    std::array<std::uint8_t, 8> sampleLocations{};
+    bool customSampleLocations = false;
 };
 
 ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup = false);
 State DecodeState(const QueueState& queue, bool vertexWorkgroup = false);
+std::array<VkSampleLocationEXT, 8> SampleLocations(const State& state);
 std::array<std::uint8_t, 8> ExportMappings(const State& state);
 std::array<ShaderRecompiler::ColorExportPacking, 8> ExportPackings(const State& state);
 ColorTarget DecodeColorBuffer(const Registers& context, std::uint32_t slot);
@@ -134,6 +137,7 @@ struct ColorMetadataPass {
 };
 std::optional<ColorMetadataPass> DecodeColorMetadataPass(const QueueState& queue);
 std::string DepthMaintenanceRejection(const QueueState& queue);
+bool DepthResummarizeDrawsNothing(const QueueState& queue);
 // The message DecodeState (or the pixel stage decode after it) would throw for the register rules
 // this precheck covers, evaluated without exceptions before the draw is decoded; empty when they
 // pass (DecodeState still checks everything). A register a rule needs that is absent is no verdict.

core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp

@@ -178,6 +178,7 @@ public:
     static bool FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except = nullptr, const char* reason = "memory access", PublishScope scope = PublishScope::Whole, bool* published = nullptr);
     static void FlushAllPending(const char* reason);
     static bool AnyPending();
+    static void FlushDumpSelected(const char* reason);
     static bool StoreAtFlipRequested(const char* value);
     // See PendingSerial: a change of a surface's source outside the registry (a unit shadow
     // retile) moves it too.

core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp

@@ -227,6 +227,45 @@ inline VertexCopyPlan PlanVertexCopies(std::span<const VertexFetch> fetches) {
     return plan;
 }
 
+inline std::optional<std::uint32_t> ZeroFetchVertex(std::span<const ShaderRecompiler::VertexAttribute> attributes) {
+    std::uint32_t zeroFrom = 0;
+    for (const auto& attribute : attributes) {
+        const auto stride = (attribute.resource.fields[1] >> 16u) & 0x3fffu;
+        if (NullVertexDescriptor(attribute) || attribute.fetchIndex != 0 || stride == 0) continue;
+        if (VertexBufferOutOfBoundsSelect(attribute) > 1u) return std::nullopt;
+        zeroFrom = std::max(zeroFrom, attribute.resource.fields[2]);
+    }
+    return zeroFrom;
+}
+
+inline std::optional<std::uint32_t> ClampDrawIndices(std::span<std::byte> indices, std::uint32_t indexSize, bool skipRestart, std::uint32_t limit) {
+    Require(indexSize == 2 || indexSize == 4, "unsupported index size");
+    const auto restartIndex = indexSize == 2 ? 0xffffu : 0xffffffffu;
+    std::optional<std::uint32_t> highest;
+    for (std::size_t offset = 0; offset + indexSize <= indices.size(); offset += indexSize) {
+        std::uint32_t index = 0;
+        if (indexSize == 2) {
+            std::uint16_t value = 0;
+            std::memcpy(&value, indices.data() + offset, sizeof(value));
+            index = value;
+        } else {
+            std::memcpy(&index, indices.data() + offset, sizeof(index));
+        }
+        if (skipRestart && index == restartIndex) continue;
+        if (index > limit) {
+            index = limit;
+            if (indexSize == 2) {
+                const auto value = static_cast<std::uint16_t>(index);
+                std::memcpy(indices.data() + offset, &value, sizeof(value));
+            } else {
+                std::memcpy(indices.data() + offset, &index, sizeof(index));
+            }
+        }
+        highest = std::max(highest.value_or(0u), index);
+    }
+    return highest;
+}
+
 inline std::optional<std::uint32_t> HighestDrawIndex(std::span<const std::byte> indices, std::uint32_t indexSize, bool skipRestart) {
     Require(indexSize == 2 || indexSize == 4, "unsupported index size");
     const auto restartIndex = indexSize == 2 ? 0xffffu : 0xffffffffu;

core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp"
 #include <array>
 #include <bit>
@@ -52,6 +53,19 @@ ColorTileMode DecodeColorTileMode(std::uint32_t attrib3) {
     return static_cast<ColorTileMode>(mode);
 }
 
+TileMipLayout ComputeColorTargetMipLayout(ColorTileMode mode, std::uint32_t bytesPerElement, std::uint32_t surfaceWidth, std::uint32_t surfaceHeight, std::uint32_t mipCount, std::uint32_t mipLevel) {
+    auto mips = ComputeElementMipLayout(ColorTextureTileMode(mode), bytesPerElement, surfaceWidth, surfaceHeight, mipCount);
+    require(mipLevel < mips.size(), "AGC graphics: color target mip is outside its surface");
+    auto mip = mips[mipLevel];
+    const ColorTargetLayout target(mip.width, mip.height, mode, bytesPerElement);
+    require(mip.tiledSize == target.Bytes(), "AGC graphics: color target mip tiling differs from its surface layout");
+    mip.tiledOffset = 0;
+    mip.linearOffset = 0;
+    mip.pitchBytes = mip.width * bytesPerElement;
+    mip.linearSize = target.LinearBytes();
+    return mip;
+}
+
 namespace {
 
 struct SwizzleTables {

core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp

@@ -34,6 +34,7 @@ public:
         const VkImageAspectFlags aspects = VK_IMAGE_ASPECT_DEPTH_BIT | (target.stencilAddress != 0 ? VK_IMAGE_ASPECT_STENCIL_BIT : 0u);
         try {
             VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
+            if (SampleLocationsCompatibleDepth(context, target.format, target.samples)) info.flags = VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT;
             info.imageType = VK_IMAGE_TYPE_2D;
             info.format = target.format;
             info.extent = {target.extent.width, target.extent.height, 1};
@@ -187,6 +188,16 @@ std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel) {
     return width * height * bytesPerTexel;
 }
 
+bool SampleLocationsCompatibleDepth(const Context& context, VkFormat format, std::uint32_t samples) {
+    if (samples <= 1 || (context.sampleLocationSampleCounts & samples) == 0) return false;
+    VkImageFormatProperties properties{};
+    const auto result = context.imageFormatProperties(context.physical, format, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL,
+        VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT, VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT, &properties);
+    if (result == VK_ERROR_FORMAT_NOT_SUPPORTED) return false;
+    Check(result, "vkGetPhysicalDeviceImageFormatProperties sample locations depth");
+    return (properties.sampleCounts & samples) != 0;
+}
+
 VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target) {
     std::lock_guard lock(surfacesMutex());
     const bool cleared = target.htileAddress != 0 && clearedHtiles().erase(target.htileAddress) != 0;
@@ -284,6 +295,22 @@ void NoteDepthMetadataFill(std::uint64_t address, std::size_t bytes, std::uint32
     }
 }
 
+bool DepthSurfaceAt(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource) {
+    std::lock_guard lock(surfacesMutex());
+    const auto& list = surfaces();
+    const auto found = std::find_if(list.rbegin(), list.rend(), [&](const auto& surface) {
+        return surface->context.device == context.device && (surface->target.address == resource.baseAddress || (surface->target.stencilAddress != 0 && surface->target.stencilAddress == resource.baseAddress));
+    });
+    if (found == list.rend()) return false;
+    const auto& target = (*found)->target;
+    if (resource.width != target.extent.width || resource.height != target.extent.height) return false;
+    const bool stencil = target.stencilAddress != 0 && target.stencilAddress == resource.baseAddress;
+    const bool d16 = target.format == VK_FORMAT_D16_UNORM || target.format == VK_FORMAT_D16_UNORM_S8_UINT;
+    const auto expected = stencil ? VK_FORMAT_R8_UINT : d16 ? VK_FORMAT_R16_UNORM : VK_FORMAT_R32_SFLOAT;
+    const bool depthBits = !stencil && words.size() >= 4 && ShaderRecompiler::DepthBitsTextureWidth(words[1], words[3]) == (d16 ? 16u : 32u);
+    return ResolveTextureFormat(resource.format) == expected || depthBits;
+}
+
 bool DepthSurfaceAt(std::uint64_t address) {
     std::lock_guard lock(surfacesMutex());
     return std::any_of(surfaces().begin(), surfaces().end(), [&](const auto& surface) { return surface->target.address == address || surface->target.stencilAddress == address; });

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -50,6 +50,11 @@ std::uint64_t DrawRenderPassKey(const Context& context, const State& state, std:
     }
     mix(state.renderExtent.width);
     mix(state.renderExtent.height);
+    mix(state.samples);
+    mix(state.customSampleLocations);
+    if (state.customSampleLocations) {
+        for (const auto location : state.sampleLocations) mix(location);
+    }
     if (!context.provokingVertexModePerPipeline) mix(state.provokingVertexMode);
     return key;
 }
@@ -732,7 +737,7 @@ std::set<std::uint32_t> CachedFragmentOutputs(const Context& context, std::span<
     }
     std::set<std::uint32_t> outputs;
     try {
-        outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric, context.descriptorIndexing, context.imageInt64Atomics, context.geometryShader, context.sampleRateShading, context.bufferInt64Atomics);
+        outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric, context.descriptorIndexing, context.imageInt64Atomics, context.geometryShader, context.sampleRateShading, context.bufferInt64Atomics, context.shaderClipDistance);
     } catch (const std::exception& error) {
         if (keyed) {
             std::lock_guard lock(validationMutex());
@@ -971,6 +976,25 @@ const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address
     return import;
 }
 
+InPlaceIndices InPlaceIndexRange(Recorder& recorder, std::uint64_t address, std::size_t bytes, std::uint32_t indexSize, bool skipRestart) {
+    const std::span<const std::byte> range(reinterpret_cast<const std::byte*>(address), bytes);
+    InPlaceIndices indices;
+    if (bytes < Recorder::KeptIndexRangeIndices * indexSize) {
+        indices.highest = HighestDrawIndex(range, indexSize, skipRestart);
+        return indices;
+    }
+    const auto use = indexSize == 2 ? (skipRestart ? Recorder::SnapshotUse::Index16Restart : Recorder::SnapshotUse::Index16) : (skipRestart ? Recorder::SnapshotUse::Index32Restart : Recorder::SnapshotUse::Index32);
+    const auto registryGeneration = GuestAllocations::GuestAllocationsGeneration_nid_postfix();
+    const auto generation = GuestMemory::CollectWrites(address, bytes);
+    if (generation != 0 && recorder.KnownHighestIndex(address, bytes, use, indices.highest)) {
+        indices.reused = true;
+        return indices;
+    }
+    indices.highest = HighestDrawIndex(range, indexSize, skipRestart);
+    recorder.KeepHighestIndex(address, bytes, use, generation, registryGeneration, indices.highest);
+    return indices;
+}
+
 namespace {
 
 // The draw's inputs before its resources (prepareDrawInputs): the validated parameters, the index
@@ -1077,10 +1101,11 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P
         if (import == nullptr) copy = CopyDrawInput(context, context.recorder, draw.indexAddress, static_cast<std::size_t>(indexBytes), draw.indexSize, use);
         else GuestMemory::CheckRange(reinterpret_cast<const void*>(draw.indexAddress), static_cast<std::size_t>(indexBytes), draw.indexSize);
         std::optional<std::uint32_t> highest;
-        if (!copy.reused) {
-            const auto bytes = import != nullptr ? std::span<const std::byte>(reinterpret_cast<const std::byte*>(draw.indexAddress), static_cast<std::size_t>(indexBytes)) : copy.buffer->Bytes().first(static_cast<std::size_t>(indexBytes));
-            highest = HighestDrawIndex(bytes, draw.indexSize, skipRestart);
-            if (import == nullptr) KeepDrawInput(context.recorder, draw.indexAddress, copy, use, skipRestart ? (highest ? *highest + 1u : 0u) : highest.value_or(0u));
+        if (import != nullptr) {
+            highest = InPlaceIndexRange(*context.recorder, draw.indexAddress, static_cast<std::size_t>(indexBytes), draw.indexSize, skipRestart).highest;
+        } else if (!copy.reused) {
+            highest = HighestDrawIndex(copy.buffer->Bytes().first(static_cast<std::size_t>(indexBytes)), draw.indexSize, skipRestart);
+            KeepDrawInput(context.recorder, draw.indexAddress, copy, use, skipRestart ? (highest ? *highest + 1u : 0u) : highest.value_or(0u));
         } else if (!skipRestart) {
             highest = copy.derived;
         } else if (copy.derived != 0) {
@@ -1090,6 +1115,24 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P
             inputs.nothing = true;
             return inputs;
         }
+        constexpr std::uint32_t ZeroFetchSlack = 0x10000u;
+        if (const auto zeroFrom = state.stages.mesh ? std::nullopt : ZeroFetchVertex(shaders.front().program->vertexAttributes)) {
+            const auto limit = *zeroFrom > draw.firstVertex ? *zeroFrom - draw.firstVertex : 0u;
+            if (*highest >= limit && *highest - limit >= ZeroFetchSlack) {
+                GuestMemory::FlushGpuWrites(draw.indexAddress, static_cast<std::size_t>(indexBytes));
+                auto clamped = std::make_shared<Buffer>(context, static_cast<std::size_t>(indexBytes), VK_BUFFER_USAGE_INDEX_BUFFER_BIT);
+                const auto bytes = clamped->Bytes().first(static_cast<std::size_t>(indexBytes));
+                GuestMemory::Read(draw.indexAddress, bytes, draw.indexSize);
+                highest = ClampDrawIndices(bytes, draw.indexSize, skipRestart, limit);
+                import = nullptr;
+                copy = DrawInputCopy{};
+                copy.buffer = std::move(clamped);
+                if (!highest) {
+                    inputs.nothing = true;
+                    return inputs;
+                }
+            }
+        }
         Require(*highest <= context.limits.maxDrawIndexedIndexValue, "index exceeds the device's indexed draw limit");
         Require(!fanGeometry || !state.primitiveRestart || *highest != (draw.indexSize == 2 ? 0xffffu : 0xffffffffu), "primitive restart in a triangle fan geometry draw is unsupported");
         inputs.maxIndex = *highest;
@@ -1739,7 +1782,9 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
     APS5_LOG_CHARS_OUT_DEBUG("Pipeline Begin OK");
     if (drawBindings != nullptr) {
         const auto set = drawBindings->allocation.set;
-        context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, record.pipeline->Layout(), 0, 1, &set, 0, nullptr);
+        const auto fragmentSet = drawBindings->fragmentAllocation.set;
+        if (set != VK_NULL_HANDLE) context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, record.pipeline->Layout(), 0, 1, &set, 0, nullptr);
+        if (fragmentSet != VK_NULL_HANDLE) context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, record.pipeline->Layout(), ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet, 1, &fragmentSet, 0, nullptr);
     } else {
         resources.Bind(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, record.pipeline->Layout());
     }
@@ -1924,11 +1969,11 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters&
             continue;
         }
         materializeCmaskClear(context, color, nullptr);
-        Require(!color.mipTail, "rendering into a packed mip tail needs the resident image of its surface");
+        Require(!color.mipTail || binding.gpuTiling, "rendering into a packed mip tail requires the GPU tiled fallback");
         Require(color.depth == 1, "rendering into a 3D color target needs the resident image of its surface");
         Require(color.arraySlice == 0 || !ColorTileModeIsXor(color.tileMode), "rendering into a slice of a 2D array color target with an XOR tile mode needs the resident image of its array");
         if (binding.gpuTiling) {
-            binding.mip = ColorTargetMip(color, colorLayout);
+            binding.mip = color.mipTail ? ComputeColorTargetMipLayout(color.tileMode, color.elementBytes, color.surfaceExtent.width, color.surfaceExtent.height, color.mipCount, color.mip) : ColorTargetMip(color, colorLayout);
             binding.tiled = std::make_unique<Buffer>(context, colorLayout.Bytes(), copies);
             binding.tiledDevice = std::make_unique<DeviceBuffer>(context, colorLayout.Bytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
             binding.linearDevice = std::make_unique<DeviceBuffer>(context, colorLayout.LinearBytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
@@ -2091,6 +2136,12 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters&
         masked->blends.at(state.colors[index].exportIndex).colorWriteMask &= VK_COLOR_COMPONENT_R_BIT;
     }
     const State& pipelineState = masked.has_value() ? *masked : state;
+    static const bool verifySetLayouts = std::getenv("APS5_VERIFY_SET_LAYOUTS") != nullptr;
+    if (verifySetLayouts && context.descriptorCache != nullptr) {
+        const auto described = DescribeSetLayouts(context, shaders);
+        const auto built = resources->SetLayouts();
+        Require(described.layouts.main == built.main && described.layouts.fragment == built.fragment && described.layouts.split == built.split && std::ranges::equal(described.mainKey, built.mainKey) && std::ranges::equal(described.fragmentKey, built.fragmentKey), "set layouts described from the shaders differ from the draw's");
+    }
     auto pipeline = CachedPipeline(context, pipelineState, inputs.vertexInput, *resources, shaders, lean ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
     // Resident targets' views are stable while their storage image lives, so the framebuffer is
     // reused with the pipeline; a per-draw RenderTarget gets a framebuffer of its own.

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -142,6 +142,12 @@ bool addressSpaceCacheEnabled() {
     return !disabled;
 }
 
+struct ImportFailure {
+    std::uint64_t base;
+    std::uint64_t bytes;
+    GuestAllocations::Mapped ranges;
+};
+
 // Imports persist across draws, keyed by allocation base, and are dropped when their allocation leaves
 // the registered set; a dropped import is destroyed once the recorded work that may read it completed.
 struct HostImports {
@@ -152,7 +158,7 @@ struct HostImports {
     std::map<std::uint64_t, HostImport> imports;
     std::map<VkDevice, std::vector<std::shared_ptr<RetiredImport>>> parked;
     std::uint64_t liveBytes = 0;
-    std::set<std::uint64_t> failed;
+    std::map<std::uint64_t, ImportFailure> failed;
     std::set<std::uint64_t> unaliasedSpans;
     // Registry generation the imports were last reconciled with.
     std::uint64_t refreshedGeneration = 0;
@@ -219,6 +225,26 @@ void retireImport(VkDevice device, HostImports& state, std::map<std::uint64_t, H
     state.imports.erase(it);
 }
 
+bool releaseParkedImports(HostImports& state, std::uint64_t begin, std::uint64_t end) {
+    std::vector<std::shared_ptr<RetiredImport>> released;
+    {
+        std::lock_guard lock(state.mutex);
+        for (auto device = state.parked.begin(); device != state.parked.end();) {
+            auto& list = device->second;
+            for (auto it = list.begin(); it != list.end();) {
+                if ((*it)->entry.base < end && begin < (*it)->entry.base + (*it)->entry.bytes) {
+                    released.push_back(std::move(*it));
+                    it = list.erase(it);
+                } else {
+                    ++it;
+                }
+            }
+            device = list.empty() ? state.parked.erase(device) : std::next(device);
+        }
+    }
+    return !released.empty();
+}
+
 const char* bindImport(const Context& context, HostImport& entry, VkExternalMemoryHandleTypeFlagBits handleType, const void* import, std::uint32_t importTypes, bool& allocated, VkResult& failure) {
     const auto failed = [&](const char* step, VkResult result) -> const char* {
         if (entry.buffer) context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, entry.buffer, nullptr);
@@ -595,7 +621,8 @@ bool runCovers(const GuestAllocations::Lease& lease, std::uint64_t begin, std::u
     return !run.ranges.empty() && end <= run.base + run.bytes;
 }
 
-bool sameRun(const HostImport& entry, const ImportRun& run) {
+template<typename Entry>
+bool sameRun(const Entry& entry, const ImportRun& run) {
     if (run.base != entry.base || run.bytes != entry.bytes || run.ranges.size() != entry.ranges.size()) return false;
     for (std::size_t index = 0; index < run.ranges.size(); ++index) {
         if (entry.ranges[index].owner_before(run.ranges[index]) || run.ranges[index].owner_before(entry.ranges[index])) return false;
@@ -629,7 +656,11 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
     }
     if (auto it = state.imports.lower_bound(base); it != state.imports.begin() && std::prev(it)->second.base + std::prev(it)->second.bytes > base) retireImport(context.device, state, std::prev(it), lease);
     const auto alignment = context.hostImportAlignment;
-    if (alignment == 0 || base % alignment != 0 || bytes % alignment != 0 || state.failed.contains(base)) return nullptr;
+    if (alignment == 0 || base % alignment != 0 || bytes % alignment != 0) return nullptr;
+    if (const auto failed = state.failed.find(base); failed != state.failed.end()) {
+        if (sameRun(failed->second, run)) return nullptr;
+        state.failed.erase(failed);
+    }
     // Pinned imports count against the driver's system memory budget; past it ordinary host
     // allocations fail, so imports stop at APS5_HOST_IMPORT_MIB (default 6 GiB, which covers the
     // registered memory of address-based shaders; past it they copy gigabytes per dispatch).
@@ -661,7 +692,7 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
     for (std::uint64_t cursor = base; cursor < base + bytes;) {
         MEMORY_BASIC_INFORMATION info{};
         if (VirtualQuery(reinterpret_cast<const void*>(cursor), &info, sizeof(info)) == 0 || info.State != MEM_COMMIT || (info.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) {
-            state.failed.insert(base);
+            state.failed.insert_or_assign(base, ImportFailure{base, bytes, GuestAllocations::Mapped(entry.ranges.begin(), entry.ranges.end())});
             return nullptr;
         }
         const auto protection = info.Protect & 0xffu;
@@ -673,7 +704,7 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
         cursor = reinterpret_cast<std::uint64_t>(info.BaseAddress) + info.RegionSize;
     }
     if (!writable && readOnly) {
-        state.failed.insert(base);
+        state.failed.insert_or_assign(base, ImportFailure{base, bytes, GuestAllocations::Mapped(entry.ranges.begin(), entry.ranges.end())});
         return nullptr;
     }
     if (!writable) {
@@ -692,7 +723,7 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
     }
 #else
     if (!GuestMemory::Accessible(reinterpret_cast<const void*>(base), static_cast<std::size_t>(bytes), true)) {
-        state.failed.insert(base);
+        state.failed.insert_or_assign(base, ImportFailure{base, bytes, GuestAllocations::Mapped(entry.ranges.begin(), entry.ranges.end())});
         return nullptr;
     }
 #endif
@@ -712,7 +743,7 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
 #ifdef _WIN32
         GuestArena::GuestArenaUnmapAlias_nid_postfix(entry.alias);
 #endif
-        state.failed.insert(base);
+        state.failed.insert_or_assign(base, ImportFailure{base, bytes, GuestAllocations::Mapped(entry.ranges.begin(), entry.ranges.end())});
         std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx failed at %s (%d); falling back to copies\n", static_cast<unsigned long long>(base), static_cast<unsigned long long>(bytes), step, static_cast<int>(result));
 #ifdef _WIN32
         static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr;
@@ -800,7 +831,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
         retireImport(context.device, state, it, lease);
         it = next;
     }
-    for (auto it = state.failed.begin(); it != state.failed.end();) it = leasedRangeAt(lease, *it) != nullptr ? std::next(it) : state.failed.erase(it);
+    for (auto it = state.failed.begin(); it != state.failed.end();) it = sameRun(it->second, importRun(lease, it->first)) ? std::next(it) : state.failed.erase(it);
 }
 
 const HostImport* findImport(HostImports& state, std::uint64_t begin, std::uint64_t end) {
@@ -1534,6 +1565,7 @@ bool WaitForLeasesAndImports(std::uintptr_t address, std::size_t bytes) noexcept
             if (recorder != nullptr) recorder->FlushDeferredReleases();
             const auto end = static_cast<std::uint64_t>(address) + bytes;
             auto& imports = Imports();
+            if (releaseParkedImports(imports, address, end)) retiredImport = true;
             const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
             {
                 std::lock_guard importsLock(imports.mutex);
@@ -1585,6 +1617,7 @@ bool WaitForLeasesAndImports(std::uintptr_t address, std::size_t bytes) noexcept
 
             const auto end = static_cast<std::uint64_t>(address) + bytes;
             auto& imports = Imports();
+            if (releaseParkedImports(imports, address, end)) retiredImport = true;
             bool overlapsImport = false;
             {
                 std::lock_guard importsLock(imports.mutex);

core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp

@@ -2,6 +2,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp"
 #include <algorithm>
 #include <stdexcept>
 #include <string>
@@ -152,6 +153,12 @@ GuestTextureResource DecodeTextureResource(std::span<const std::uint32_t> words)
             Require(height == 1 && depth == 0 && baseArray == 0, "guest 1D texture descriptor has a nonzero height, depth or base array");
             break;
         case TextureDimension::k2D:
+            if (depth != 0 && tileMode == TextureTileMode::kLinear) {
+                const auto elementsPerPitchUnit = 256u / BytesPerElement(format);
+                const auto defaultPitch = (width + elementsPerPitchUnit - 1u) / elementsPerPitchUnit * elementsPerPitchUnit;
+                Require(!IsBlockCompressed(format) && maxMip == 0 && baseArray == 0 && depth + 1u == defaultPitch, "guest 2D linear texture descriptor sets a custom pitch other than its default pitch, which is not implemented" + describe());
+                depth = 0;
+            }
             Require(depth == 0 && baseArray == 0, "guest 2D texture descriptor has a nonzero depth or base array");
             break;
         case TextureDimension::k3D:

core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp

@@ -2,6 +2,7 @@
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp"
 #include <algorithm>
@@ -94,7 +95,13 @@ void ValidateDepthBounds(const Context& context, const State& state) {
 
 void ValidateProvokingVertex(const Context& context, const State& state, std::span<const CompiledShader> shaders) {
     Require(state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT || state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT, "invalid provoking vertex mode");
-    if (state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT) return;
+    if (state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT) {
+        for (const auto& shader : shaders) {
+            if (shader.stage != ShaderRecompiler::ShaderStage::Fragment || shader.program == nullptr) continue;
+            Require(std::none_of(shader.program->fragmentParameters.begin(), shader.program->fragmentParameters.end(), [](const ShaderRecompiler::FragmentParameter& parameter) { return parameter.flatP10; }), "v_interp_mov p10 of a flat input with the first provoking vertex is not measured");
+        }
+        return;
+    }
     Require(context.provokingVertexLast, "last provoking vertex requires VK_EXT_provoking_vertex with provokingVertexLast enabled");
     Require(!state.rectList && state.stages.path == ShaderPath::Vertex && !state.stages.mesh && !state.stages.tessellation, "last provoking vertex is unsupported for generated primitive pipelines");
     Require(state.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN, "last provoking vertex is unsupported for this primitive topology");
@@ -106,6 +113,14 @@ void ValidateProvokingVertex(const Context& context, const State& state, std::sp
     }
 }
 
+bool ValidateSampleLocations(const Context& context, const State& state) {
+    const bool extensionSupportsSamples = (context.sampleLocationSampleCounts & state.samples) != 0;
+    Require(!state.customSampleLocations || extensionSupportsSamples, "custom sample locations require VK_EXT_sample_locations support for the draw's sample count");
+    const bool enabled = extensionSupportsSamples && (!state.depth || SampleLocationsCompatibleDepth(context, state.depth->format, state.depth->samples));
+    Require(!state.customSampleLocations || enabled, "custom sample locations require a compatible depth format and sample count");
+    return enabled;
+}
+
 void ValidateViewport(const Context& context, const VkViewport& viewport) {
     Require(std::isfinite(viewport.minDepth) && std::isfinite(viewport.maxDepth), "non-finite viewport depth range");
     Require(context.depthRangeUnrestricted || (viewport.minDepth >= 0 && viewport.minDepth <= 1 && viewport.maxDepth >= 0 && viewport.maxDepth <= 1), "viewport depth outside [0, 1] requires VK_EXT_depth_range_unrestricted");
@@ -143,6 +158,9 @@ bool ShadersWriteMemory(std::span<const CompiledShader> shaders) {
 
 namespace {
 
+thread_local bool prebuilding = false;
+thread_local bool prebuilt = false;
+
 VkShaderStageFlags pushStagesOf(const Context& context) {
     return PreRasterizationPushStages(context.meshShader, context.tessellationShader, context.geometryShader);
 }
@@ -154,7 +172,7 @@ void appendKey(std::vector<std::byte>& key, const TValue& value) {
     key.insert(key.end(), bytes.begin(), bytes.end());
 }
 
-bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& state, const VertexInputLayout& input, const ShaderResources& resources, std::span<const CompiledShader> shaders) {
+bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& state, const VertexInputLayout& input, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders) {
     using Stage = ShaderRecompiler::ShaderStage;
     if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList || SplitsAlternateAlpha(state)) return false;
     keys = {};
@@ -188,6 +206,17 @@ bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State&
     if (state.stages.tessellation) appendKey(keys.preRasterization, state.stages.tessellation->inputControlPoints);
     appendKey(keys.fragmentShader, state.samples);
     appendKey(keys.fragmentOutput, state.samples);
+    const auto sampleLocationsEnabled = ValidateSampleLocations(context, state);
+    appendKey(keys.fragmentShader, sampleLocationsEnabled);
+    appendKey(keys.fragmentOutput, sampleLocationsEnabled);
+    appendKey(keys.fragmentShader, state.customSampleLocations);
+    appendKey(keys.fragmentOutput, state.customSampleLocations);
+    if (state.customSampleLocations) {
+        for (std::uint32_t sample = 0; sample < state.samples; ++sample) {
+            appendKey(keys.fragmentShader, state.sampleLocations[sample]);
+            appendKey(keys.fragmentOutput, state.sampleLocations[sample]);
+        }
+    }
     for (const auto& blend : state.blends) appendKey(keys.fragmentOutput, blend);
     for (const auto value : state.blendConstants) appendKey(keys.fragmentOutput, value);
     appendKey(keys.renderPass, state.blends.size());
@@ -197,15 +226,38 @@ bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State&
     }
     appendKey(keys.renderPass, state.depth.has_value());
     if (state.depth) appendKey(keys.renderPass, state.depth->format);
-    appendKey(keys.layout, pushStagesOf(context));
-    for (const auto word : resources.LayoutKey()) appendKey(keys.layout, word);
+    if (!resources.split) {
+        appendKey(keys.layout, pushStagesOf(context));
+        for (const auto word : resources.mainKey) appendKey(keys.layout, word);
+        keys.preRasterizationLayout = keys.fragmentLayout = keys.layout;
+        return true;
+    }
+    const auto splitKey = [&](std::vector<std::byte>& key, std::uint32_t sets) {
+        appendKey(key, ~0u);
+        appendKey(key, sets);
+        appendKey(key, pushStagesOf(context));
+        if ((sets & 1u) != 0u) {
+            appendKey(key, resources.mainKey.size());
+            for (const auto word : resources.mainKey) appendKey(key, word);
+        }
+        if ((sets & 2u) != 0u) {
+            appendKey(key, resources.fragmentKey.size());
+            for (const auto word : resources.fragmentKey) appendKey(key, word);
+        }
+    };
+    splitKey(keys.layout, 3u);
+    splitKey(keys.preRasterizationLayout, 1u);
+    splitKey(keys.fragmentLayout, 2u);
     return true;
 }
 
 }
 
-Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout) : context(context), _modules(shaders.size()), attachments(state.colors.size() + (state.depth ? 1u : 0u)), colorAttachments(state.colors.size()), depthBounds(state.depth.has_value() && state.depthBoundsTest), depthBias(state.depth.has_value() && state.depthBias), dynamicRendering(context.graphicsPipelineLibrary) {
+Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout) : Pipeline(context, state, vertexInput, resources.SetLayouts(), shaders, attachmentLayout) {}
+
+Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout) : context(context), _modules(shaders.size()), attachments(state.colors.size() + (state.depth ? 1u : 0u)), colorAttachments(state.colors.size()), depthBounds(state.depth.has_value() && state.depthBoundsTest), depthBias(state.depth.has_value() && state.depthBias), dynamicRendering(context.graphicsPipelineLibrary) {
     PerformanceTimer timing("Vulkan.GraphicsPipeline");
+    sampleLocationsEnabled = ValidateSampleLocations(context, state);
     // A cached pipeline may outlive its device's teardown (see ClearCachedPipelines); it must not keep
     // the buffer pool, which is reset with the device, alive past it.
     this->context.bufferPool.reset();
@@ -259,12 +311,13 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
         }
         // A descriptor set layout with the same bindings as this one is compatible with the pipeline
         // layout, so later draws bind their own ShaderResources' set under it.
-        const auto setLayout = resources.Layout();
+        const std::array<VkDescriptorSetLayout, 2> setLayouts{resources.main, resources.fragment};
         const VkPushConstantRange push{pushStages, 0, PipelinePushSlotBytes};
         const std::array<VkPushConstantRange, 2> slots{{{pushStagesOf(context), 0, PipelinePushSlotBytes}, {VK_SHADER_STAGE_FRAGMENT_BIT, PipelinePushSlotBytes, PipelinePushSlotBytes}}};
         VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
-        layoutInfo.setLayoutCount = 1;
-        layoutInfo.pSetLayouts = &setLayout;
+        layoutInfo.flags = resources.split ? VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT : 0;
+        layoutInfo.setLayoutCount = resources.split ? 2u : 1u;
+        layoutInfo.pSetLayouts = setLayouts.data();
         layoutInfo.pushConstantRangeCount = dynamicRendering ? static_cast<std::uint32_t>(slots.size()) : pushStages != 0 ? 1 : 0;
         layoutInfo.pPushConstantRanges = dynamicRendering ? slots.data() : pushStages != 0 ? &push : nullptr;
         Check(context.Function<PFN_vkCreatePipelineLayout>("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &layout), "vkCreatePipelineLayout graphics");
@@ -367,6 +420,11 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
         }
         VkPipelineMultisampleStateCreateInfo samples{VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO};
         samples.rasterizationSamples = static_cast<VkSampleCountFlagBits>(state.samples);
+        const auto locations = SampleLocations(state);
+        VkPipelineSampleLocationsStateCreateInfoEXT sampleLocations{VK_STRUCTURE_TYPE_PIPELINE_SAMPLE_LOCATIONS_STATE_CREATE_INFO_EXT};
+        sampleLocations.sampleLocationsEnable = VK_TRUE;
+        sampleLocations.sampleLocationsInfo = {VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT, nullptr, samples.rasterizationSamples, {1, 1}, state.samples, locations.data()};
+        if (sampleLocationsEnabled) samples.pNext = &sampleLocations;
         VkPipelineDepthStencilStateCreateInfo depthStencil{VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO};
         depthStencil.depthTestEnable = state.depthTest;
         depthStencil.depthWriteEnable = state.depthWrite;
@@ -420,7 +478,19 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
         pipelineInfo.renderPass = renderPass;
         if (dynamicRendering) pipelineInfo.pNext = &rendering;
         timing.Mark("modules_and_state");
-        if (libraries) pipeline = LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized, _modules);
+        if (prebuilding && !libraries) {
+            for (const auto module : _modules) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, module, nullptr);
+            _modules.clear();
+            return;
+        }
+        if (prebuilding) {
+            static_cast<void>(LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized, true, _modules));
+            for (const auto module : _modules) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, module, nullptr);
+            _modules.clear();
+            prebuilt = true;
+            return;
+        }
+        if (libraries) pipeline = LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized, false, _modules);
         else Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines");
         if (alternateAlpha) {
             blend.pAttachments = secondChannels.data();
@@ -440,6 +510,15 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
     }
 }
 
+bool Pipeline::Prebuild(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders) {
+    struct Scope {
+        Scope() { prebuilding = true; prebuilt = false; }
+        ~Scope() { prebuilding = false; }
+    } scope;
+    const Pipeline pipeline(context, state, vertexInput, resources, shaders);
+    return prebuilt;
+}
+
 Pipeline::~Pipeline() {
     release();
 }
@@ -563,6 +642,16 @@ void Pipeline::Begin(VkCommandBuffer commands, const Framebuffer& framebuffer, V
     begin.renderPass = renderPass;
     begin.framebuffer = framebuffer.Handle();
     begin.renderArea = {{0, 0}, extent};
+    const auto locations = SampleLocations(state);
+    const VkSampleLocationsInfoEXT info{VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT, nullptr, static_cast<VkSampleCountFlagBits>(state.samples), {1, 1}, state.samples, locations.data()};
+    const VkAttachmentSampleLocationsEXT initial{static_cast<std::uint32_t>(state.colors.size()), info};
+    const VkSubpassSampleLocationsEXT post{0, info};
+    VkRenderPassSampleLocationsBeginInfoEXT sampleLocations{VK_STRUCTURE_TYPE_RENDER_PASS_SAMPLE_LOCATIONS_BEGIN_INFO_EXT};
+    sampleLocations.attachmentInitialSampleLocationsCount = state.depth ? 1 : 0;
+    sampleLocations.pAttachmentInitialSampleLocations = state.depth ? &initial : nullptr;
+    sampleLocations.postSubpassSampleLocationsCount = 1;
+    sampleLocations.pPostSubpassSampleLocations = &post;
+    if (sampleLocationsEnabled) begin.pNext = &sampleLocations;
     context.Resolved(&DeviceFunctions::cmdBeginRenderPass, "vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE);
     Continue(commands, state);
 }
@@ -639,7 +728,7 @@ void append(std::vector<std::byte>& key, const TValue& value) {
 // (the recompiler could not identify it, so nothing else may share its pipeline). The rect-list
 // control and evaluation stages are generated from the vertex and fragment results, which the key
 // already names, so they carry no id of their own.
-void pipelineKey(std::vector<std::byte>& key, const Context& context, const State& state, const VertexInputLayout& input, const ShaderResources& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout) {
+void pipelineKey(std::vector<std::byte>& key, const Context& context, const State& state, const VertexInputLayout& input, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout) {
     using Stage = ShaderRecompiler::ShaderStage;
     key.clear();
     append(key, context.device);
@@ -671,8 +760,10 @@ void pipelineKey(std::vector<std::byte>& key, const Context& context, const Stat
         append(key, attribute.format);
         append(key, attribute.offset);
     }
-    append(key, resources.LayoutKey().size());
-    for (const auto word : resources.LayoutKey()) append(key, word);
+    append(key, resources.mainKey.size());
+    for (const auto word : resources.mainKey) append(key, word);
+    append(key, resources.fragmentKey.size());
+    for (const auto word : resources.fragmentKey) append(key, word);
     append(key, state.hasColorTarget);
     append(key, state.rectList);
     append(key, state.topology);
@@ -687,6 +778,8 @@ void pipelineKey(std::vector<std::byte>& key, const Context& context, const Stat
     for (const auto& blend : state.blends) append(key, blend);
     for (const auto value : state.blendConstants) append(key, value);
     append(key, state.samples);
+    append(key, state.customSampleLocations);
+    if (state.customSampleLocations) append(key, state.sampleLocations);
     append(key, state.colors.size());
     for (const auto& color : state.colors) append(key, color.format);
     if (state.blends.size() != state.colors.size()) {
@@ -790,6 +883,10 @@ void reportPipelines(PipelineStore& store) {
 }
 
 std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout) {
+    return CachedPipeline(context, state, vertexInput, resources.SetLayouts(), shaders, attachmentLayout);
+}
+
+std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout) {
     static const bool disabled = std::getenv("APS5_NO_PIPELINE_CACHE") != nullptr;
     if (disabled) return std::make_shared<Pipeline>(context, state, vertexInput, resources, shaders, attachmentLayout);
     auto& store = Pipelines();

core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp

@@ -8,6 +8,7 @@
 #include <thread>
 #include <map>
 #include <mutex>
+#include <optional>
 #include <string_view>
 
 namespace AgcDriver::Graphics {
@@ -38,9 +39,9 @@ struct DeviceLibraries {
     PFN_vkDestroyPipeline destroyPipeline = nullptr;
     PFN_vkDestroyPipelineLayout destroyLayout = nullptr;
     PFN_vkDestroyShaderModule destroyModule = nullptr;
+    std::vector<VkShaderModule> modules;
     std::map<Key, VkPipelineLayout> layouts;
     std::array<std::map<Key, VkPipeline>, 4> parts;
-    std::vector<VkShaderModule> modules;
     PipelineLibraryCounters counters;
 };
 
@@ -101,22 +102,32 @@ std::span<const VkDynamicState> PipelineLibraryDynamicStates() {
     return dynamicStates;
 }
 
-VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr<OptimizedPipeline>* optimized, std::span<VkShaderModule> modules) {
+VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr<OptimizedPipeline>* optimized, bool partsOnly, std::span<VkShaderModule> modules) {
     Require(info.renderPass == VK_NULL_HANDLE && info.pDynamicState != nullptr, "pipeline library parts need dynamic rendering and the library dynamic states");
     auto& store = Libraries();
-    std::lock_guard lock(store.mutex);
+    std::unique_lock lock(store.mutex);
     auto& device = store.devices[context.device];
     if (device.destroyPipeline == nullptr) {
         device.destroyPipeline = context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline");
         device.destroyLayout = context.Function<PFN_vkDestroyPipelineLayout>("vkDestroyPipelineLayout");
         device.destroyModule = context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule");
     }
-    auto pipelineLayout = device.layouts.find(keys.layout);
-    if (pipelineLayout == device.layouts.end()) {
+    const auto layoutFor = [&](const Key& key, std::optional<std::uint32_t> set) {
+        auto found = device.layouts.find(key);
+        if (found != device.layouts.end()) return found->second;
+        auto info = layout;
+        std::array<VkDescriptorSetLayout, 2> sets{VK_NULL_HANDLE, VK_NULL_HANDLE};
+        if (set) {
+            sets.at(*set) = layout.pSetLayouts[*set];
+            info.pSetLayouts = sets.data();
+        }
         VkPipelineLayout handle = VK_NULL_HANDLE;
-        Check(context.Function<PFN_vkCreatePipelineLayout>("vkCreatePipelineLayout")(context.device, &layout, nullptr, &handle), "vkCreatePipelineLayout library");
-        pipelineLayout = device.layouts.emplace(keys.layout, handle).first;
-    }
+        Check(context.Function<PFN_vkCreatePipelineLayout>("vkCreatePipelineLayout")(context.device, &info, nullptr, &handle), "vkCreatePipelineLayout library");
+        return device.layouts.emplace(key, handle).first->second;
+    };
+    const bool split = (layout.flags & VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT) != 0;
+    const auto fullLayout = layoutFor(keys.layout, std::nullopt);
+    const std::array<VkPipelineLayout, 4> partLayouts{VK_NULL_HANDLE, split ? layoutFor(keys.preRasterizationLayout, 0u) : fullLayout, split ? layoutFor(keys.fragmentLayout, 1u) : fullLayout, VK_NULL_HANDLE};
     std::vector<VkPipelineShaderStageCreateInfo> preRasterization;
     const VkPipelineShaderStageCreateInfo* fragment = nullptr;
     bool mesh = false;
@@ -135,7 +146,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
     };
     const std::array<const Key*, 4> partKeys{&keys.vertexInput, &keys.preRasterization, &keys.fragmentShader, &keys.fragmentOutput};
     const Key none;
-    const std::array<const Key*, 4> contextKeys{&none, &keys.layout, &keys.layout, &keys.renderPass};
+    const std::array<const Key*, 4> contextKeys{&none, &keys.preRasterizationLayout, &keys.fragmentLayout, &keys.renderPass};
     const VkPipelineRenderingCreateInfoKHR shaderRendering{VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO_KHR};
     VkPipelineDepthStencilStateCreateInfo dynamicDepth{VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO};
     static const bool backgroundLinking = std::getenv("APS5_NO_PIPELINE_LTO") == nullptr;
@@ -155,7 +166,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
         VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &library};
         create.flags = VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT;
         create.pDynamicState = info.pDynamicState;
-        create.layout = pipelineLayout->second;
+        create.layout = partLayouts[part];
         switch (flags[part]) {
         case VK_GRAPHICS_PIPELINE_LIBRARY_VERTEX_INPUT_INTERFACE_BIT_EXT:
             create.pVertexInputState = info.pVertexInputState;
@@ -182,7 +193,15 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
             break;
         }
         VkPipeline handle = VK_NULL_HANDLE;
-        Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &create, nullptr, &handle), "vkCreateGraphicsPipelines library");
+        if (partsOnly) lock.unlock();
+        const auto created = context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &create, nullptr, &handle);
+        if (partsOnly) lock.lock();
+        Check(created, "vkCreateGraphicsPipelines library");
+        if (const auto raced = cache.find(key); raced != cache.end()) {
+            device.destroyPipeline(context.device, handle, nullptr);
+            libraries.push_back(raced->second);
+            continue;
+        }
         cache.emplace(key, handle);
         if (optimize) {
             for (std::uint32_t stage = 0; stage < create.stageCount; ++stage) {
@@ -196,6 +215,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
         ++device.counters.built[part];
         libraries.push_back(handle);
     }
+    if (partsOnly) return VK_NULL_HANDLE;
     PerformanceTimer timing("Vulkan.GraphicsPipelineLink");
     VkPipelineLibraryCreateInfoKHR linked{VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR};
     linked.libraryCount = static_cast<std::uint32_t>(libraries.size());
@@ -212,7 +232,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
         *optimized = slot;
         const auto createPipelines = context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines");
         const auto cache = context.pipelineCache;
-        const auto libraryLayout = pipelineLayout->second;
+        const auto libraryLayout = fullLayout;
         enqueue(store, context.device, [slot, createPipelines, cache, libraryLayout, libraries, &store, devicePointer = &device] {
             if (slot->released.load()) return;
             VkPipelineLibraryCreateInfoKHR linked{VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR};
@@ -255,8 +275,8 @@ void ClearPipelineLibraries(VkDevice device) {
     for (auto& part : libraries.parts) {
         for (const auto& [key, handle] : part) libraries.destroyPipeline(device, handle, nullptr);
     }
-    for (const auto module : libraries.modules) libraries.destroyModule(device, module, nullptr);
     for (const auto& [key, handle] : libraries.layouts) libraries.destroyLayout(device, handle, nullptr);
+    for (const auto module : libraries.modules) libraries.destroyModule(device, module, nullptr);
     store.devices.erase(found);
 }
 

core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp

@@ -2506,11 +2506,19 @@ bool Recorder::refreshDrawSnapshot(DrawSnapshot& entry, std::uint64_t address, s
     const auto aligned = address / block * block;
     const auto end = address + bytes;
     const auto count = static_cast<std::size_t>((end - 1) / block - address / block + 1);
-    thread_local std::vector<std::uint64_t> generations;
-    thread_local std::vector<std::uint8_t> changed;
-    thread_local std::vector<std::pair<std::size_t, std::size_t>> differing;
-    thread_local std::vector<CompareSpan> spans;
-    thread_local std::vector<std::uint8_t> equal;
+    struct Scratch {
+        std::vector<std::uint64_t> generations;
+        std::vector<std::uint8_t> changed;
+        std::vector<std::pair<std::size_t, std::size_t>> differing;
+        std::vector<CompareSpan> spans;
+        std::vector<std::uint8_t> equal;
+    };
+    auto& scratch = HostThreadLocal<Scratch, Scratch>();
+    auto& generations = scratch.generations;
+    auto& changed = scratch.changed;
+    auto& differing = scratch.differing;
+    auto& spans = scratch.spans;
+    auto& equal = scratch.equal;
     generations.assign(count, entry.generation);
     changed.assign(count, GuestMemory::BlockWritten);
     differing.clear();
@@ -2561,6 +2569,40 @@ void Recorder::KeepDrawSnapshot(std::uint64_t address, std::size_t bytes, std::u
     pool.bytes += bytes;
 }
 
+bool Recorder::KnownHighestIndex(std::uint64_t address, std::size_t bytes, SnapshotUse use, std::optional<std::uint32_t>& highest) {
+    const auto found = indexRanges.find({address, use, bytes});
+    if (found == indexRanges.end()) return false;
+    if (found->second.registryGeneration != GuestAllocations::GuestAllocationsGeneration_nid_postfix() || !GuestMemory::UnchangedSince(address, bytes, found->second.generation)) {
+        indexRangeRecency.erase(found->second.recent);
+        indexRanges.erase(found);
+        return false;
+    }
+    indexRangeRecency.splice(indexRangeRecency.end(), indexRangeRecency, found->second.recent);
+    highest = found->second.highest;
+    return true;
+}
+
+void Recorder::KeepHighestIndex(std::uint64_t address, std::size_t bytes, SnapshotUse use, std::uint64_t generation, std::uint64_t registryGeneration, std::optional<std::uint32_t> highest) {
+    Require(use != SnapshotUse::Storage && use != SnapshotUse::Vertex, "a highest index belongs to an index buffer");
+    if (generation == 0) return;
+    const DrawSnapshotKey key{address, use, bytes};
+    if (const auto found = indexRanges.find(key); found != indexRanges.end()) {
+        indexRangeRecency.erase(found->second.recent);
+        indexRanges.erase(found);
+    }
+    while (indexRanges.size() >= IndexRangeEntries) {
+        indexRanges.erase(indexRangeRecency.front());
+        indexRangeRecency.pop_front();
+    }
+    indexRangeRecency.push_back(key);
+    try {
+        indexRanges.emplace(key, IndexRange{generation, registryGeneration, std::prev(indexRangeRecency.end()), highest});
+    } catch (...) {
+        indexRangeRecency.pop_back();
+        throw;
+    }
+}
+
 void Recorder::OnComplete(std::function<void()> action) {
     ensureOpen();
     open->completions.push_back(std::move(action));

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp

@@ -142,6 +142,7 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con
     const bool pixelKillEnable = ((shaderControl >> 6u) & 0x1u) != 0;
     const bool depthExportEnable = (shaderControl & 0x1u) != 0;
     const bool sampleMaskExportEnable = ((shaderControl >> 8u) & 0x1u) != 0;
+    const bool stencilRefExportEnable = ((shaderControl >> 1u) & 0x1u) != 0;
     const auto zOrder = (shaderControl >> 4u) & 0x3u;
     const auto loaded = [&](PixelInput input) { return (activeInputs & PixelInputBit(input)) != 0; };
     return ShaderRecompiler::ShaderPixelStageInfo{
@@ -163,12 +164,13 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con
         .pixelKillEnable = pixelKillEnable,
         .depthExportEnable = depthExportEnable,
         .sampleMaskExportEnable = sampleMaskExportEnable,
-        .earlyZ = zOrder == 1u && !pixelKillEnable && !depthExportEnable && !sampleMaskExportEnable,
+        .earlyZ = zOrder == 1u && !pixelKillEnable && !depthExportEnable && !sampleMaskExportEnable && !stencilRefExportEnable,
         .executeOnNoop = ((shaderControl >> 10u) & 0x1u) != 0,
         .conservativeZExport = static_cast<ShaderRecompiler::ConservativeZExport>(conservativeZExport),
         .orderedPixelShader = ((shaderControl >> 16u) & 0x1u) != 0,
         .targetOutputMode = targetOutputMode,
-        .targetExportMapping = nullProgram ? std::array<std::uint8_t, 8>{} : exportMappings
+        .targetExportMapping = nullProgram ? std::array<std::uint8_t, 8>{} : exportMappings,
+        .stencilRefExportEnable = stencilRefExportEnable
     };
 }
 

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp

@@ -2,10 +2,12 @@
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp"
 #include "ThreadOwned.hpp"
+#include "prx/libc/include/HostThreadLocal.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
 #include <algorithm>
 #include <atomic>
@@ -278,9 +280,10 @@ struct ImageMemoEntry {
 };
 
 constexpr std::size_t ImageMemoSlots = 1024;
-thread_local std::unique_ptr<std::array<ImageMemoEntry, ImageMemoSlots>> imageMemo;
+struct ImageMemoTag {};
 
 ImageMemoEntry& imageMemoSlot(std::span<const std::uint32_t> words) {
+    auto& imageMemo = HostThreadLocal<std::unique_ptr<std::array<ImageMemoEntry, ImageMemoSlots>>, ImageMemoTag>();
     if (imageMemo == nullptr) imageMemo = std::make_unique<std::array<ImageMemoEntry, ImageMemoSlots>>();
     const auto hash = hashWords(14695981039346656037ull, words.data(), words.size());
     return (*imageMemo)[(hash ^ (hash >> 29u)) % ImageMemoSlots];
@@ -834,9 +837,9 @@ GuestTextureResource StorageSurface(const Context& context, const GuestTextureRe
 // `guestBytes` is the surface size when the caller described the surface already (0: described here).
 std::shared_ptr<StorageTexture> cachedStorageTexture(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& viewed, std::uint32_t mip, std::uint64_t guestBytes) {
     RequireNoMultisampleTarget(viewed.baseAddress, guestBytes != 0 ? static_cast<std::size_t>(guestBytes) : 1u, "a storage image access");
-    if (DepthSurfaceAt(viewed.baseAddress)) {
-        char text[112];
-        std::snprintf(text, sizeof(text), "AGC graphics: storage image access to depth/stencil surface 0x%llx is not implemented", static_cast<unsigned long long>(viewed.baseAddress));
+    if (DepthSurfaceAt(context, words, viewed)) {
+        char text[256];
+        std::snprintf(text, sizeof(text), "AGC graphics: storage image access to depth/stencil surface 0x%llx is not implemented (storage=%ux%u guest format=%u mip=%u dimension=%d tile=%d)", static_cast<unsigned long long>(viewed.baseAddress), viewed.width, viewed.height, viewed.format, mip, static_cast<int>(viewed.dimension), static_cast<int>(viewed.tileMode));
         throw std::runtime_error(text);
     }
     static const bool disabled = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr;
@@ -1164,6 +1167,14 @@ VkDescriptorSetLayout CreateSetLayout(const Context& context, std::span<const Vk
     return layout;
 }
 
+std::vector<VkDescriptorPoolSize> PoolSizes(std::span<const VkDescriptorSetLayoutBinding> bindings) {
+    std::map<VkDescriptorType, std::uint32_t> counts;
+    for (const auto& binding : bindings) counts[binding.descriptorType] += binding.descriptorCount;
+    std::vector<VkDescriptorPoolSize> sizes;
+    for (const auto& [type, count] : counts) sizes.push_back({type, count});
+    return sizes;
+}
+
 VkDescriptorPool CreateSetPool(const Context& context, std::span<const VkDescriptorPoolSize> sizes, bool updateAfterBind) {
     VkDescriptorPoolCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
     info.flags = updateAfterBind ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0;
@@ -1337,7 +1348,7 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
                 updateAfterBind = true;
             };
             for (const auto& binding : shader.program->bindings) {
-                Require(binding.descriptorSet == 0, "unexpected descriptor set: every shader resource must use descriptor set zero");
+                Require(binding.descriptorSet == ShaderRecompiler::RuntimeAbi::DescriptorSet || (shader.stage == ShaderRecompiler::ShaderStage::Fragment && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet), "unexpected descriptor set: a shader resource must use its stage's descriptor set");
                 Require(occupied.insert(binding.binding).second, "duplicate shader binding");
                 const bool addressRole = binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable || binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer;
                 const bool bufferRole = addressRole || binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers || binding.role == ShaderRecompiler::DescriptorRole::ShaderData || binding.role == ShaderRecompiler::DescriptorRole::FlattenedSrt || binding.role == ShaderRecompiler::DescriptorRole::Gds;
@@ -1414,23 +1425,62 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
         if (drawBuild && !usesBda && std::all_of(allocations.begin(), allocations.end(), [&](const Allocation& allocation) { return !allocation.guest || allocation.pushByte >= 0 || (allocation.dataAllocation >= 0 && allocation.dataByte < allocations[static_cast<std::size_t>(allocation.dataAllocation)].size); })) guestMemory.AllowAdjustedRegions();
         guestMemory.UploadPrepare(usesBda);
         timing.uploadMs = phase(BuildPhase::Upload);
-        std::vector<VkDescriptorSetLayoutBinding> description;
+        splitSets = SplitsSets(context, shaders);
+        std::array<std::vector<VkDescriptorSetLayoutBinding>, 2> descriptions;
         for (const auto& binding : bindings) {
-            description.push_back(binding.layout);
-            layoutBindingFlags.push_back(updateAfterBind && binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT : 0);
-            layoutKey.insert(layoutKey.end(), {binding.layout.binding, static_cast<std::uint32_t>(binding.layout.descriptorType), binding.layout.descriptorCount, binding.layout.stageFlags});
+            const auto set = setOf(binding);
+            descriptions[set].push_back(binding.layout);
+            (set == 0 ? layoutBindingFlags : fragmentBindingFlags).push_back(updateAfterBind && binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT : 0);
+            auto& key = set == 0 ? layoutKey : fragmentLayoutKey;
+            key.insert(key.end(), {binding.layout.binding, static_cast<std::uint32_t>(binding.layout.descriptorType), binding.layout.descriptorCount, binding.layout.stageFlags});
+        }
+        if (updateAfterBind) {
+            layoutKey.push_back(VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT);
+            if (splitSets) fragmentLayoutKey.push_back(VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT);
         }
-        if (updateAfterBind) layoutKey.push_back(VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT);
         static const bool noLayoutCache = std::getenv("APS5_NO_LAYOUT_CACHE") != nullptr;
         static const bool noPoolCache = std::getenv("APS5_NO_POOL_CACHE") != nullptr;
-        if (context.descriptorCache != nullptr && !noLayoutCache) {
-            _layout = context.descriptorCache->Layout(layoutKey, description, updateAfterBind, layoutBindingFlags);
-        } else {
+        const auto createLayout = [&](const std::vector<std::uint32_t>& key, const std::vector<VkDescriptorSetLayoutBinding>& description, const std::vector<VkDescriptorBindingFlags>& flags, bool& owns) {
+            if (context.descriptorCache != nullptr && !noLayoutCache) return context.descriptorCache->Layout(key, description, updateAfterBind, flags);
             ValidateDescriptorLayout(context, description, updateAfterBind);
-            _layout = CreateSetLayout(context, description, updateAfterBind, layoutBindingFlags);
-            ownsLayout = true;
+            owns = true;
+            return CreateSetLayout(context, description, updateAfterBind, flags);
+        };
+        _layout = createLayout(layoutKey, descriptions[0], layoutBindingFlags, ownsLayout);
+        if (splitSets) {
+            fragment.layout = createLayout(fragmentLayoutKey, descriptions[1], fragmentBindingFlags, fragment.ownsLayout);
+            {
+                const auto sizes = PoolSizes(descriptions[1]);
+                if (context.descriptorCache != nullptr && !noPoolCache) {
+                    fragment.cacheAllocation = context.descriptorCache->Allocate(fragment.layout, sizes, updateAfterBind);
+                    fragment.set = fragment.cacheAllocation.set;
+                }
+                if (fragment.set == VK_NULL_HANDLE) {
+                    fragment.pool = CreateSetPool(context, sizes, updateAfterBind);
+                    VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};
+                    allocation.descriptorPool = fragment.pool;
+                    allocation.descriptorSetCount = 1;
+                    allocation.pSetLayouts = &fragment.layout;
+                    Check(context.Function<PFN_vkAllocateDescriptorSets>("vkAllocateDescriptorSets")(context.device, &allocation, &fragment.set), "vkAllocateDescriptorSets");
+                }
+            }
         }
-        if (!bindings.empty()) {
+        if (splitSets) {
+            const auto sizes = PoolSizes(descriptions[0]);
+            if (context.descriptorCache != nullptr && !noPoolCache) {
+                cacheAllocation = context.descriptorCache->Allocate(_layout, sizes, updateAfterBind);
+                _set = cacheAllocation.set;
+            }
+            if (_set == VK_NULL_HANDLE) {
+                pool = CreateSetPool(context, sizes, updateAfterBind);
+                VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};
+                allocation.descriptorPool = pool;
+                allocation.descriptorSetCount = 1;
+                allocation.pSetLayouts = &_layout;
+                Check(context.Function<PFN_vkAllocateDescriptorSets>("vkAllocateDescriptorSets")(context.device, &allocation, &_set), "vkAllocateDescriptorSets");
+            }
+        }
+        if (!splitSets && !bindings.empty()) {
             // The set is sized from the plan: every image element becomes exactly one descriptor
             // when stage B looks it up.
             std::vector<VkDescriptorPoolSize> sizes;
@@ -1463,6 +1513,48 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
     }
 }
 
+bool SplitsSets(const Context& context, std::span<const CompiledShader> shaders) {
+    const auto fragmentSet = [](const CompiledShader& shader) { return shader.program != nullptr && std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [](const auto& binding) { return binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet; }); };
+    const auto fragmentStage = [](const CompiledShader& shader) { return shader.stage == ShaderRecompiler::ShaderStage::Fragment; };
+    return std::any_of(shaders.begin(), shaders.end(), fragmentSet) || (context.graphicsPipelineLibrary && std::any_of(shaders.begin(), shaders.end(), fragmentStage));
+}
+
+DescribedSetLayouts DescribeSetLayouts(const Context& context, std::span<const CompiledShader> shaders) {
+    Require(context.descriptorCache != nullptr, "described set layouts need the descriptor cache");
+    DescribedSetLayouts result;
+    std::array<std::vector<VkDescriptorSetLayoutBinding>, 2> descriptions;
+    const bool split = SplitsSets(context, shaders);
+    bool updateAfterBind = false;
+    std::uint64_t storageBuffers = 0;
+    for (const auto& shader : shaders) {
+        Require(shader.program != nullptr, "missing compiled shader");
+        const VkShaderStageFlags flags = VulkanStage(shader.stage);
+        std::uint64_t stageStorageBuffers = 0;
+        std::uint64_t stageResources = 0;
+        for (const auto& binding : shader.program->bindings) {
+            const bool image = binding.role == ShaderRecompiler::DescriptorRole::GuestImages || binding.role == ShaderRecompiler::DescriptorRole::GuestSamplers;
+            const auto type = !image ? VK_DESCRIPTOR_TYPE_STORAGE_BUFFER : binding.kind == ShaderRecompiler::DescriptorKind::StorageImage ? VK_DESCRIPTOR_TYPE_STORAGE_IMAGE : binding.kind == ShaderRecompiler::DescriptorKind::SampledImage ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_SAMPLER;
+            if (type != VK_DESCRIPTOR_TYPE_SAMPLER) stageResources += binding.count;
+            if (type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER) {
+                stageStorageBuffers += binding.count;
+                storageBuffers += binding.count;
+            }
+            if (stageStorageBuffers > context.limits.maxPerStageDescriptorStorageBuffers || stageResources > context.limits.maxPerStageResources) updateAfterBind = true;
+            descriptions[binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet ? 1 : 0].push_back({binding.binding, type, binding.count, flags, nullptr});
+        }
+    }
+    if (storageBuffers > context.limits.maxDescriptorSetStorageBuffers) updateAfterBind = true;
+    for (std::size_t set = 0; set < descriptions.size(); ++set) {
+        auto& key = set == 0 ? result.mainKey : result.fragmentKey;
+        for (const auto& binding : descriptions[set]) key.insert(key.end(), {binding.binding, static_cast<std::uint32_t>(binding.descriptorType), binding.descriptorCount, binding.stageFlags});
+        if (updateAfterBind && (set == 0 || split)) key.push_back(VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT);
+    }
+    result.layouts.split = split;
+    result.layouts.main = context.descriptorCache->Layout(result.mainKey, descriptions[0], updateAfterBind);
+    if (split) result.layouts.fragment = context.descriptorCache->Layout(result.fragmentKey, descriptions[1], updateAfterBind);
+    return result;
+}
+
 void ShaderResources::buildComplete() {
     PerformanceTimer frameTiming("Resources.Complete");
     Require(!completed, "cannot update a published descriptor heap");
@@ -1490,7 +1582,7 @@ void ShaderResources::buildComplete() {
         if (usesBda) bda = std::make_unique<BdaResources>(context, guestMemory);
         else if (usesFaultBuffer) bda = std::make_unique<BdaResources>(context);
         phase(BuildPhase::Bda);
-        if (_set != VK_NULL_HANDLE) {
+        if (_set != VK_NULL_HANDLE || fragment.set != VK_NULL_HANDLE) {
             // One update call for the whole set: the info arrays are sized up front so every write's
             // pointer into them stays valid until the call.
             std::size_t bufferCount = 0;
@@ -1512,7 +1604,7 @@ void ShaderResources::buildComplete() {
             };
             for (const auto& binding : bindings) {
                 VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
-                write.dstSet = _set;
+                write.dstSet = setOf(binding) == 0 ? _set : fragment.set;
                 write.dstBinding = binding.layout.binding;
                 write.descriptorCount = binding.layout.descriptorCount;
                 write.descriptorType = binding.layout.descriptorType;
@@ -3332,10 +3424,19 @@ void ShaderResources::release() noexcept {
     if (cacheAllocation.pool && context.descriptorCache != nullptr) context.descriptorCache->Free(cacheAllocation);
     if (pool) context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")(context.device, pool, nullptr);
     if (_layout && ownsLayout) context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(context.device, _layout, nullptr);
+    if (fragment.cacheAllocation.pool && context.descriptorCache != nullptr) context.descriptorCache->Free(fragment.cacheAllocation);
+    if (fragment.pool) context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")(context.device, fragment.pool, nullptr);
+    if (fragment.layout && fragment.ownsLayout) context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(context.device, fragment.layout, nullptr);
     cacheAllocation = {};
     pool = VK_NULL_HANDLE;
     _set = VK_NULL_HANDLE;
     _layout = VK_NULL_HANDLE;
+    fragment = {};
+}
+
+std::uint32_t ShaderResources::setOf(const Binding& binding) const {
+    constexpr auto count = static_cast<std::uint32_t>(ShaderRecompiler::RuntimeAbi::Binding::Count);
+    return splitSets && binding.layout.binding / count == static_cast<std::uint32_t>(ShaderRecompiler::RuntimeAbi::Stage::Fragment) ? 1u : 0u;
 }
 
 VkDescriptorSetLayout ShaderResources::Layout() const {
@@ -3354,11 +3455,12 @@ bool ShaderResources::LayoutUpdateAfterBind() const {
 
 ShaderResources::DrawBindings::~DrawBindings() {
     if (cache != nullptr && allocation.set != VK_NULL_HANDLE) cache->Free(allocation);
+    if (cache != nullptr && fragmentAllocation.set != VK_NULL_HANDLE) cache->Free(fragmentAllocation);
 }
 
 std::optional<std::vector<ShaderResources::MovedBuffer>> ShaderResources::MovedReadOnlyBuffers(std::span<const CompiledShader> shaders, Recorder& recorder) const {
     std::vector<MovedBuffer> moved;
-    if (_set == VK_NULL_HANDLE || usesBda) return moved;
+    if ((_set == VK_NULL_HANDLE && fragment.set == VK_NULL_HANDLE) || usesBda) return moved;
     for (const auto& shader : shaders) {
         if (shader.program == nullptr) return std::nullopt;
         for (const auto& binding : shader.program->bindings) {
@@ -3411,7 +3513,7 @@ std::size_t ShaderResources::MaxDrawSnapshotBytes() {
 }
 
 std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindings(Recorder& recorder, std::span<const MovedBuffer> moved) const {
-    if (_set == VK_NULL_HANDLE || usesBda) return {};
+    if ((_set == VK_NULL_HANDLE && fragment.set == VK_NULL_HANDLE) || usesBda) return {};
     const auto reads = guestMemory.InPlaceReads();
     const auto maxSnapshotBytes = MaxDrawSnapshotBytes();
     std::size_t totalSnapshotBytes = 0;
@@ -3458,13 +3560,17 @@ std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindi
     }
     if (selected.empty()) return {};
     Require(context.descriptorCache != nullptr, "draw snapshots require a descriptor cache");
-    std::map<VkDescriptorType, std::uint32_t> counts;
-    for (const auto& binding : bindings) counts[binding.layout.descriptorType] += binding.layout.descriptorCount;
-    std::vector<VkDescriptorPoolSize> sizes;
-    for (const auto& [type, count] : counts) sizes.push_back({type, count});
+    std::array<std::vector<VkDescriptorSetLayoutBinding>, 2> descriptions;
+    for (const auto& binding : bindings) descriptions[setOf(binding)].push_back(binding.layout);
     result->cache = context.descriptorCache;
-    result->allocation = result->cache->Allocate(_layout, sizes);
-    Require(result->allocation.set != VK_NULL_HANDLE, "draw snapshot descriptor allocation failed");
+    if (!descriptions[0].empty() || splitSets) {
+        result->allocation = result->cache->Allocate(_layout, PoolSizes(descriptions[0]));
+        Require(result->allocation.set != VK_NULL_HANDLE, "draw snapshot descriptor allocation failed");
+    }
+    if (!descriptions[1].empty() || splitSets) {
+        result->fragmentAllocation = result->cache->Allocate(fragment.layout, PoolSizes(descriptions[1]));
+        Require(result->fragmentAllocation.set != VK_NULL_HANDLE, "draw snapshot descriptor allocation failed");
+    }
     auto buffers = descriptorBuffers;
     auto writes = descriptorWrites;
     std::vector<std::size_t> snapshotIndices(allocations.size(), selected.size());
@@ -3472,7 +3578,7 @@ std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindi
     for (std::size_t index = 0; index < bindings.size(); ++index) {
         const auto& binding = bindings[index];
         auto& write = writes[index];
-        write.dstSet = result->allocation.set;
+        write.dstSet = setOf(binding) == 0 ? result->allocation.set : result->fragmentAllocation.set;
         if (binding.allocations.empty()) continue;
         const auto offset = static_cast<std::size_t>(write.pBufferInfo - descriptorBuffers.data());
         write.pBufferInfo = buffers.data() + offset;
@@ -3490,8 +3596,8 @@ std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindi
 }
 
 void ShaderResources::Bind(VkCommandBuffer commands, VkPipelineBindPoint bindPoint, VkPipelineLayout layout) const {
-    if (_set == VK_NULL_HANDLE) return;
-    context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, bindPoint, layout, 0, 1, &_set, 0, nullptr);
+    if (_set != VK_NULL_HANDLE) context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, bindPoint, layout, 0, 1, &_set, 0, nullptr);
+    if (fragment.set != VK_NULL_HANDLE) context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, bindPoint, layout, ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet, 1, &fragment.set, 0, nullptr);
 }
 
 namespace {

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp

@@ -43,6 +43,7 @@ struct Module {
     std::map<std::uint32_t, std::string> outputs;
     std::optional<std::string> secondSource;
     bool position = false;
+    bool clipDistance = false;
     bool primitiveIndices = false;
     bool fragmentBarycentric = false;
     std::map<std::uint32_t, std::vector<std::uint32_t>> modes;
@@ -109,6 +110,11 @@ struct Module {
             if (!input) position = true;
             return;
         }
+        if (value == spv::BuiltInClipDistance && !input && (vertex || stage == Stage::TessellationEvaluation)) {
+            Require(!clipDistance && Signature(type) == "f32[1]", "invalid or duplicate clip distance output");
+            clipDistance = true;
+            return;
+        }
         if (stage == Stage::TessellationControl || stage == Stage::TessellationEvaluation || stage == Stage::Mesh) {
             const auto signature = Signature(type);
             if (!input && value == spv::BuiltInPosition) {
@@ -131,12 +137,12 @@ struct Module {
             Require(value == spv::BuiltInPosition && signature == "f32x4" && !position, "unsupported or duplicate vertex built-in output");
             position = true;
         } else {
-            Require((storage == spv::StorageClassInput && ((value == spv::BuiltInFragCoord && signature == "f32x4") || ((value == spv::BuiltInFrontFacing || value == spv::BuiltInHelperInvocation) && signature == "bool") || ((value == spv::BuiltInSampleId || value == spv::BuiltInLayer) && signature == "i32"))) || (storage == spv::StorageClassOutput && ((value == spv::BuiltInFragDepth && signature == "f32") || (value == spv::BuiltInSampleMask && (signature == "i32[1]" || signature == "u32[1]")))), "unsupported fragment built-in");
+            Require((storage == spv::StorageClassInput && ((value == spv::BuiltInFragCoord && signature == "f32x4") || ((value == spv::BuiltInFrontFacing || value == spv::BuiltInHelperInvocation) && signature == "bool") || ((value == spv::BuiltInSampleId || value == spv::BuiltInLayer) && signature == "i32"))) || (storage == spv::StorageClassOutput && ((value == spv::BuiltInFragDepth && signature == "f32") || (value == spv::BuiltInFragStencilRefEXT && signature == "i32") || (value == spv::BuiltInSampleMask && (signature == "i32[1]" || signature == "u32[1]")))), "unsupported fragment built-in");
         }
     }
 };
 
-Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading, bool bufferInt64Atomics) {
+Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading, bool bufferInt64Atomics, bool shaderClipDistance) {
     using Stage = ShaderRecompiler::ShaderStage;
     Require(compiled.program != nullptr, "missing compiled shader");
     const auto& shader = *compiled.program;
@@ -226,12 +232,17 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
                     fragment &&
                     capability == spv::CapabilityFragmentShaderPixelInterlockEXT;
 
+                const bool isStencilExportCapability =
+                    fragment &&
+                    capability == spv::CapabilityStencilExportEXT;
+
                 // Enabled unconditionally or by the device setup in VulkanDevice.
                 const bool isFeatureCapability =
                     capability == spv::CapabilitySampled1D ||
                     capability == spv::CapabilityImage1D ||
                     capability == spv::CapabilityImageGatherExtended ||
                     capability == spv::CapabilityImageQuery ||
+                    ((vertex || evaluation) && shaderClipDistance && capability == spv::CapabilityClipDistance) ||
                     (fragment && geometryShader && capability == spv::CapabilityGeometry) ||
                     (fragment && sampleRateShading && capability == spv::CapabilitySampleRateShading) ||
                     capability == spv::CapabilityImageMSArray ||
@@ -266,6 +277,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
                     isMeshCapability ||
                     isGeometryCapability ||
                     isInterlockCapability ||
+                    isStencilExportCapability ||
                     isFeatureCapability ||
                     isDescriptorIndexingCapability ||
                     ((imageInt64Atomics || bufferInt64Atomics) && capability == spv::CapabilityInt64Atomics) ||
@@ -286,6 +298,10 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
                     Require(fragment, "SPV_EXT_fragment_shader_interlock requires a fragment shader");
                     break;
                 }
+                if (extension == "SPV_EXT_shader_stencil_export") {
+                    Require(fragment, "SPV_EXT_shader_stencil_export requires a fragment shader");
+                    break;
+                }
                 if (extension == "SPV_KHR_fragment_shader_barycentric") {
                     Require(fragment && fragmentShaderBarycentric, "SPV_KHR_fragment_shader_barycentric requires enabled fragmentShaderBarycentric in a fragment shader");
                     break;
@@ -413,7 +429,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
         mode(spv::ExecutionModeOutputVertices, {3u});
         Require(module.modes.size() == 4, "unsupported geometry execution mode");
     } else if (fragment) {
-        for (const auto& [name, operands] : module.modes) Require(operands.empty() && (name == spv::ExecutionModeOriginUpperLeft || name == spv::ExecutionModeEarlyFragmentTests || name == spv::ExecutionModeDepthReplacing || name == spv::ExecutionModeDepthLess || name == spv::ExecutionModeDepthGreater || name == spv::ExecutionModePixelInterlockOrderedEXT), "unsupported fragment execution mode");
+        for (const auto& [name, operands] : module.modes) Require(operands.empty() && (name == spv::ExecutionModeOriginUpperLeft || name == spv::ExecutionModeEarlyFragmentTests || name == spv::ExecutionModeDepthReplacing || name == spv::ExecutionModeDepthLess || name == spv::ExecutionModeDepthGreater || name == spv::ExecutionModePixelInterlockOrderedEXT || name == spv::ExecutionModeStencilRefReplacingEXT), "unsupported fragment execution mode");
     } else Require(module.modes.empty(), "unsupported vertex execution mode");
     std::set<std::pair<std::uint32_t, std::uint32_t>> descriptors;
     bool push = false;
@@ -547,7 +563,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
 
 }
 
-std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading, bool bufferInt64Atomics) {
+std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading, bool bufferInt64Atomics, bool shaderClipDistance) {
     using Stage = ShaderRecompiler::ShaderStage;
     const bool tessellation = state.stages.path == ShaderPath::Tessellation;
     const bool mesh = state.stages.path == ShaderPath::Geometry;
@@ -567,8 +583,8 @@ std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders,
         const auto expected = state.rectList ? (i == 0 ? Stage::Vertex : tessStages[i]) : tessellation ? tessStages[i] : geometry && i == 1 ? Stage::Geometry : i == shaders.size() - 1 ? Stage::Fragment : state.stages.path == ShaderPath::Geometry ? Stage::Mesh : Stage::Vertex;
         Require(shaders[i].program != nullptr, "missing compiled shader");
         Require(shaders[i].stage == expected, "graphics stage order disagrees");
-        for (const auto& binding : shaders[i].program->bindings) Require(binding.descriptorSet == 0, "graphics resource uses a descriptor set other than zero");
-        const auto current = Inspect(shaders[i], state, subgroup, fragmentShaderBarycentric, descriptorIndexing, imageInt64Atomics, geometryShader, sampleRateShading, bufferInt64Atomics);
+        for (const auto& binding : shaders[i].program->bindings) Require(binding.descriptorSet == ShaderRecompiler::RuntimeAbi::DescriptorSet || (shaders[i].stage == ShaderRecompiler::ShaderStage::Fragment && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet), "graphics resource uses a descriptor set other than its stage's");
+        const auto current = Inspect(shaders[i], state, subgroup, fragmentShaderBarycentric, descriptorIndexing, imageInt64Atomics, geometryShader, sampleRateShading, bufferInt64Atomics, shaderClipDistance);
         if (i != 0) {
             for (const auto& [location, signature] : current.inputs) {
                 const auto output = previous.outputs.find(location);

core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp

@@ -119,9 +119,11 @@ bool zFormatSupported(std::uint32_t format) {
 
 std::uint32_t shaderControlMask(std::uint32_t zFormat) {
     constexpr std::uint32_t zExportEnable = 0x1u;
+    constexpr std::uint32_t stencilTestValExportEnable = 0x2u;
     constexpr std::uint32_t maskExportEnable = 0x100u;
     auto mask = ShaderControlMask;
     if (zFormat != 0u && zFormatSupported(zFormat)) mask &= ~zExportEnable;
+    if (zFormat == 2u || zFormat == 9u) mask &= ~stencilTestValExportEnable;
     if (zFormat == 9u) mask &= ~maskExportEnable;
     return mask;
 }
@@ -154,7 +156,7 @@ bool depthPlanesAbsent(const Registers& cx) {
     return z != cx.end() && stencil != cx.end() && (z->second & 3u) == 0 && (stencil->second & 1u) == 0;
 }
 
-VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint32_t refMask, bool readOnly) {
+VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint32_t refMask, bool readOnly, bool testValExport) {
     VkStencilOpState face{};
     face.compareOp = static_cast<VkCompareOp>(compare);
     face.compareMask = (refMask >> 8u) & 0xffu;
@@ -172,6 +174,10 @@ VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint
             case 2:
             case 3:
             case 4: {
+                if (testValExport) {
+                    if (op != 3) throw std::runtime_error("AGC graphics: stencil replacement by a value other than the test value is unsupported with a pixel shader that exports the stencil test value");
+                    return VK_STENCIL_OP_REPLACE;
+                }
                 const auto value = op == 2 ? 0xffu : op == 3 ? test : opValue;
                 if (((reference ^ value) & fixed & face.writeMask) != 0) {
                     std::snprintf(detail, sizeof(detail), "AGC graphics: stencil replacement 0x%02x against test value 0x%02x (masks 0x%02x/0x%02x) is unsupported", value, test, face.compareMask, face.writeMask);
@@ -203,7 +209,7 @@ VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint
     return face;
 }
 
-void decodeDepth(const Registers& cx, std::uint32_t depthControl, State& result) {
+void decodeDepth(const Registers& cx, std::uint32_t depthControl, bool testValExport, State& result) {
     zero(cx, 0x000, 0x00001f9cu, "depth copy, resummarize or decompress draws (DB_RENDER_CONTROL)");
     const bool depthClear = (read(cx, 0x000) & 1u) != 0;
     const bool stencilClear = (read(cx, 0x000) & 2u) != 0;
@@ -223,6 +229,7 @@ void decodeDepth(const Registers& cx, std::uint32_t depthControl, State& result)
     const bool depthReadOnly = (view & 0x01000000u) != 0;
     const bool stencilReadOnly = (view & 0x02000000u) != 0;
     Require(!stencilClear || (stencil && !stencilReadOnly), "stencil clear requires a writable stencil plane");
+    Require(!stencilClear || !testValExport, "stencil clear with a pixel shader that exports the stencil test value is unsupported");
     Require(!depthClear || (zFormat != 0 && !depthReadOnly), "depth clear requires a writable depth plane");
     DepthTarget depth{};
     depth.samples = 1u << ((read(cx, 0x010) >> 2u) & 3u);
@@ -274,8 +281,8 @@ void decodeDepth(const Registers& cx, std::uint32_t depthControl, State& result)
         result.stencilBack = clear;
     } else if (result.stencilTest) {
         const auto ops = read(cx, 0x10b);
-        result.stencilFront = stencilFace((depthControl >> 8u) & 7u, ops, read(cx, 0x10c), stencilReadOnly);
-        result.stencilBack = (depthControl & 0x80u) != 0 ? stencilFace((depthControl >> 20u) & 7u, ops >> 12u, read(cx, 0x10d), stencilReadOnly) : result.stencilFront;
+        result.stencilFront = stencilFace((depthControl >> 8u) & 7u, ops, read(cx, 0x10c), stencilReadOnly, testValExport);
+        result.stencilBack = (depthControl & 0x80u) != 0 ? stencilFace((depthControl >> 20u) & 7u, ops >> 12u, read(cx, 0x10d), stencilReadOnly, testValExport) : result.stencilFront;
     }
 }
 
@@ -317,6 +324,36 @@ constexpr std::array<std::array<std::int8_t, 2>, 2> StandardLocations2{{{4, 4},
 constexpr std::array<std::array<std::int8_t, 2>, 4> StandardLocations4{{{-2, -6}, {6, -2}, {-6, 2}, {2, 6}}};
 constexpr std::array<std::array<std::int8_t, 2>, 8> StandardLocations8{{{1, -3}, {-1, 3}, {5, 1}, {-3, -5}, {-5, 5}, {-7, -1}, {3, 7}, {7, -7}}};
 
+std::string singleSampleMsaaRejection(const QueueState& queue) {
+    const auto& cx = queue.context;
+    const auto word = [](const Registers& registers, std::uint32_t offset, RegisterBank bank = RegisterBank::Context) -> std::optional<std::uint32_t> {
+        const auto it = find(registers, offset, bank);
+        if (it == registers.end()) return std::nullopt;
+        return it->second;
+    };
+    for (std::uint32_t pixel = 0; pixel < 4; ++pixel) {
+        const auto offset = 0x2feu + pixel * 4u;
+        if (const auto locations = word(cx, offset); locations && (*locations & 0xffu) != 0) return zeroMessage(offset, *locations, "MSAA_ENABLE on a single-sample target whose sample is off the pixel center");
+    }
+    if (const auto alphaToMask = word(cx, 0x2dc); alphaToMask && (*alphaToMask & 1u) != 0) return zeroMessage(0x2dc, *alphaToMask, "MSAA_ENABLE on a single-sample target with alpha-to-coverage");
+    if (const auto conservative = word(cx, 0x313); conservative && (*conservative & ~0x001f6000u) != 0) return zeroMessage(0x313, *conservative, "MSAA_ENABLE on a single-sample target with conservative rasterization");
+    constexpr const char* primitives = "MSAA_ENABLE on a single-sample target for primitives other than triangles";
+    const auto stages = word(cx, 0x2d5);
+    if (!stages) return {};
+    if ((*stages & 0x20u) != 0) {
+        if (const auto output = word(cx, 0x29b); output && ((*output & 0x80000000u) != 0 || (*output & 0x3fu) != 2u)) return zeroMessage(0x29b, *output, primitives);
+    } else if ((*stages & 4u) != 0) {
+        if (const auto parameters = word(cx, 0x2db)) {
+            const auto domain = *parameters & 3u;
+            const auto topology = (*parameters >> 5u) & 7u;
+            if ((domain != 1u && domain != 2u) || (topology != 2u && topology != 3u)) return zeroMessage(0x2db, *parameters, primitives);
+        }
+    } else if (const auto primitive = word(queue.userConfig, 0x242, RegisterBank::UserConfig); primitive && *primitive != 4u && *primitive != 5u && *primitive != 6u) {
+        return zeroMessage(0x242, *primitive, primitives);
+    }
+    return {};
+}
+
 std::uint32_t decodeSamples(const Registers& cx) {
     const auto config = read(cx, 0x2f8);
     zero(cx, 0x2f8, ~(0x7u | 0x1e000u | 0x700000u), "coverage conversion or nonstandard multisampling");
@@ -327,7 +364,12 @@ std::uint32_t decodeSamples(const Registers& cx) {
     }
     Require(log2 <= 3u, "more than eight samples are unsupported");
     Require(((config >> 20u) & 7u) == log2, "exposing fewer samples than are rasterized is unsupported");
-    const auto samples = 1u << log2;
+    return 1u << log2;
+}
+
+void decodeSampleLocations(const Registers& cx, State& state) {
+    const auto samples = state.samples;
+    if (samples == 1) return;
     const std::span<const std::array<std::int8_t, 2>> standard = samples == 2 ? std::span<const std::array<std::int8_t, 2>>(StandardLocations2) : samples == 4 ? std::span<const std::array<std::int8_t, 2>>(StandardLocations4) : std::span<const std::array<std::int8_t, 2>>(StandardLocations8);
     for (std::uint32_t pixel = 0; pixel < 4; ++pixel) {
         for (std::uint32_t sample = 0; sample < samples; ++sample) {
@@ -335,10 +377,14 @@ std::uint32_t decodeSamples(const Registers& cx) {
             const auto bits = (word >> (8u * (sample % 4u))) & 0xffu;
             const auto x = static_cast<std::int8_t>(static_cast<std::uint8_t>(bits << 4u)) >> 4;
             const auto y = static_cast<std::int8_t>(static_cast<std::uint8_t>(bits & 0xf0u)) >> 4;
-            Require(x == standard[sample][0] && y == standard[sample][1], "nonstandard sample locations are unsupported");
+            if (pixel == 0) {
+                state.sampleLocations[sample] = static_cast<std::uint8_t>(bits);
+                state.customSampleLocations |= x != standard[sample][0] || y != standard[sample][1];
+            } else {
+                Require(bits == state.sampleLocations[sample], "nonstandard sample locations varying between pixels are unsupported");
+            }
         }
     }
-    return samples;
 }
 
 bool colorControlSupported(std::uint32_t colorControl, bool hasColorTarget) {
@@ -495,6 +541,9 @@ DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number,
             if (number == floating) return {VK_FORMAT_R32G32B32A32_SFLOAT, 16};
             if (number == uint) return {VK_FORMAT_R32G32B32A32_UINT, 16};
             return fail();
+        case 17:
+            if (swap != 0 || number != unorm) return fail();
+            return {VK_FORMAT_A1R5G5B5_UNORM_PACK16, 2, 0xc6u};
         default:
             return fail();
     }
@@ -519,6 +568,23 @@ void intersect(VkRect2D& result, const Registers& registers, std::uint32_t offse
 
 }
 
+std::array<VkSampleLocationEXT, 8> SampleLocations(const State& state) {
+    Require(state.samples == 1 || state.samples == 2 || state.samples == 4 || state.samples == 8, "invalid sample count");
+    std::array<VkSampleLocationEXT, 8> locations{};
+    for (std::uint32_t sample = 0; sample < state.samples; ++sample) {
+        std::array<int, 2> position{};
+        if (state.customSampleLocations) {
+            const auto bits = state.sampleLocations[sample];
+            position = {static_cast<int>(bits & 7u) - static_cast<int>(bits & 8u), static_cast<int>((bits >> 4u) & 7u) - static_cast<int>((bits >> 4u) & 8u)};
+        } else if (state.samples != 1) {
+            const auto& standard = state.samples == 2 ? StandardLocations2[sample] : state.samples == 4 ? StandardLocations4[sample] : StandardLocations8[sample];
+            position = {standard[0], standard[1]};
+        }
+        locations[sample] = {0.5f + position[0] / 16.0f, 0.5f + position[1] / 16.0f};
+    }
+    return locations;
+}
+
 ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup) {
     const auto value = read(queue.context, 0x2d5);
     const auto validate = [&](bool condition, const char* reason) {
@@ -576,7 +642,22 @@ ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup) {
     return result;
 }
 
+bool DepthResummarizeDrawsNothing(const QueueState& queue) {
+    const auto& cx = queue.context;
+    const auto control = find(cx, 0x000);
+    const auto depthControl = find(cx, 0x200);
+    const auto zInfo = find(cx, 0x010);
+    const auto stencilInfo = find(cx, 0x011);
+    const auto targetMask = find(cx, 0x8e);
+    const auto shaderMask = find(cx, 0x8f);
+    if (control == cx.end() || control->second != 0x10u || depthControl == cx.end() || zInfo == cx.end() || stencilInfo == cx.end() || targetMask == cx.end() || shaderMask == cx.end()) return false;
+    const auto depth = depthControl->second;
+    const auto format = zInfo->second & 3u;
+    return (depth & 2u) != 0 && (depth & 0xc000000cu) == 0 && depthPassThrough(depth) && (format == 1u || format == 3u) && (stencilInfo->second & 1u) == 0 && ColorWriteMask(cx) == 0 && PixelProgramSkipped(queue);
+}
+
 std::string DepthMaintenanceRejection(const QueueState& queue) {
+    if (DepthResummarizeDrawsNothing(queue)) return {};
     const auto control = find(queue.context, 0x000);
     if (control == queue.context.end() || (control->second & ~0x2063u) == 0) return {};
     const auto depthControl = find(queue.context, 0x200);
@@ -619,13 +700,14 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
     }
     zero(cx, 0x207, ~LayerExports, "clip distances, layer, viewport or auxiliary vertex exports");
     result.samples = decodeSamples(cx);
+    decodeSampleLocations(cx, result);
     {
         const auto depthControl = read(cx, 0x200);
         const auto renderControl = find(cx, 0x000);
         const bool clear = renderControl != cx.end() && (renderControl->second & 3u) != 0;
         const auto tests = planeTests(cx, depthControl);
         if (clear || (tests != 0 && depthSurfaceBound(cx))) {
-            decodeDepth(cx, depthControl, result);
+            decodeDepth(cx, depthControl, (read(cx, 0x203) & 2u) != 0 && !PixelProgramSkipped(queue), result);
         } else if (depthPassThrough(depthControl) || ((depthControl & 0xbu) != 0 && (tests == 0 || depthPlanesAbsent(cx)))) {
             static bool reported = false;
             if (!reported) {
@@ -648,8 +730,12 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
     result.alphaToCoverage = read(cx, 0x2dc) == AlphaToCoverageUndithered;
     if (!result.alphaToCoverage) zero(cx, 0x2dc, AlphaToCoverageMask, "dithered or rounded alpha-to-coverage");
     Require(!result.depth || result.depth->samples == result.samples, "the depth target's sample count differs from the rasterizer's");
-    zero(cx, 0x292, result.samples > 1 ? ScanModeMask & ~1u : ScanModeMask, "scan conversion mode");
+    zero(cx, 0x292, ScanModeMask & ~1u, "scan conversion mode");
     Require(result.samples == 1 || (read(cx, 0x292) & 1u) != 0, "multisampled targets rasterized without MSAA_ENABLE are unsupported");
+    if (result.samples == 1 && (read(cx, 0x292) & 1u) != 0) {
+        for (std::uint32_t pixel = 0; pixel < 4; ++pixel) static_cast<void>(read(cx, 0x2feu + pixel * 4u));
+        if (auto reason = singleSampleMsaaRejection(queue); !reason.empty()) throw std::runtime_error(reason);
+    }
     zero(cx, 0x293, ScanControlMask, "sample iteration, primitive discard or out-of-order rasterization");
     zero(cx, 0x80, ~0u, "window offset");
     zero(cx, 0x8d, ScreenOffsetMask, "reserved PA_SU_HARDWARE_SCREEN_OFFSET bits");
@@ -1004,7 +1090,7 @@ std::optional<ColorMetadataPass> DecodeColorMetadataPass(const QueueState& queue
     if (mode != 2u && mode != 6u && !resolve) return std::nullopt;
     ColorMetadataPass pass{resolve ? ColorMetadataPass::Mode::Resolve : mode == 2u ? ColorMetadataPass::Mode::EliminateFastClear : ColorMetadataPass::Mode::DccDecompress, {}, std::nullopt};
     Require((control->second & ~0x70u) == 0xcc0000u, "CB metadata pass with a nonstandard ROP, dual quads disabled or degamma");
-    Require((read(cx, 0x200) & 0xfu) == 0 && (read(cx, 0x0) & 0xfu) == 0, "CB metadata pass with depth or stencil work");
+    Require(depthPlanesAbsent(cx) || ((read(cx, 0x200) & 0xfu) == 0 && (read(cx, 0x0) & 0xfu) == 0), "CB metadata pass with depth or stencil work");
     const auto samples = resolve ? decodeSamples(cx) : 1u;
     if (!resolve) zero(cx, 0x2f8, ~0u, "multisampling or coverage conversion");
     zero(cx, 0x80, ~0u, "window offset");
@@ -1100,7 +1186,10 @@ std::string DrawRejection(const QueueState& queue, bool indexed) {
     }
     if (auto reason = nonzero(cx, 0x2f8, ~(0x7u | 0x1e000u | 0x700000u), "coverage conversion or nonstandard multisampling"); !reason.empty()) return reason;
     const bool multisampled = value(cx, 0x2f8, word) && (word & 7u) != 0;
-    if (auto reason = nonzero(cx, 0x292, multisampled ? ScanModeMask & ~1u : ScanModeMask, "scan conversion mode"); !reason.empty()) return reason;
+    if (auto reason = nonzero(cx, 0x292, ScanModeMask & ~1u, "scan conversion mode"); !reason.empty()) return reason;
+    if (!multisampled && value(cx, 0x292, word) && (word & 1u) != 0) {
+        if (auto reason = singleSampleMsaaRejection(queue); !reason.empty()) return reason;
+    }
     if (auto reason = nonzero(cx, 0x293, ScanControlMask, "sample iteration, primitive discard or out-of-order rasterization"); !reason.empty()) return reason;
     if (auto reason = nonzero(cx, 0x80, ~0u, "window offset"); !reason.empty()) return reason;
     if (auto reason = nonzero(cx, 0x8d, ScreenOffsetMask, "reserved PA_SU_HARDWARE_SCREEN_OFFSET bits"); !reason.empty()) return reason;

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -286,7 +286,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex
         VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
         allocation.allocationSize = requirements.size;
         allocationBytes = requirements.size;
-        allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+        allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "texture");
         Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &owned->memory), "vkAllocateMemory texture");
         Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory");
 
@@ -940,7 +940,7 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler,
         VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
         allocation.allocationSize = requirements.size;
         memoryBytes = requirements.size;
-        allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+        allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "storage texture");
         Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory storage texture");
         Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory storage");
     } catch (...) {
@@ -1909,7 +1909,7 @@ bool StorageTexture::compareUntracked(std::uint64_t address, std::size_t bytes,
     const auto first = address / blockBytes;
     Require(changed.size() == (end - 1) / blockBytes - first + 1, "texture comparison block count differs");
     const auto* recorder = Recorder::Active();
-    const std::array<std::uint64_t, 4> stamp{GuestMemory::CollectEpoch(), GuestMemory::TrackerGeneration(), GuestMemory::ForgetSerial(), recorder != nullptr ? recorder->NewestWriteNote(address, bytes) : 0};
+    const std::array<std::uint64_t, 4> stamp{GuestMemory::CollectEpoch(), GuestMemory::NewestStamp(address, bytes), GuestMemory::ForgetSerial(), recorder != nullptr ? recorder->NewestWriteNote(address, bytes) : 0};
     const bool cacheable = memoize && stamp[0] != 0 && (stamp[2] & 1u) == 0 && address == descriptor.baseAddress && bytes == guestBytes;
     if (cacheable && comparedGuestBytes == stamp) {
         std::fill(changed.begin(), changed.end(), GuestMemory::BlockUnchanged);
@@ -2976,6 +2976,51 @@ bool StorageTexture::PublishShadowsOnly(std::uint64_t address, std::size_t bytes
     return PublishShadow(address, bytes, scope, PublishReasonFor(reason)) != 0;
 }
 
+namespace {
+
+struct StorageDumpSelector {
+    std::uint64_t address = 0;
+    std::uint32_t width = 0, height = 0, layers = 0, format = 0;
+    bool anyFormat = true;
+};
+
+const StorageDumpSelector& StorageDumpSelection() {
+    static const StorageDumpSelector selector = [] {
+        StorageDumpSelector parsed;
+        const char* text = std::getenv("APS5_DUMP_STORAGE");
+        if (text == nullptr) return parsed;
+        if (const int fields = std::sscanf(text, "%ux%ux%u:%u", &parsed.width, &parsed.height, &parsed.layers, &parsed.format); fields >= 3) {
+            parsed.anyFormat = fields < 4;
+            return parsed;
+        }
+        parsed.address = std::strtoull(text, nullptr, 16);
+        return parsed;
+    }();
+    return selector;
+}
+
+bool StorageDumpSelects(const GuestTextureResource& descriptor) {
+    const auto& selector = StorageDumpSelection();
+    if (selector.address != 0) return descriptor.baseAddress == selector.address;
+    return selector.width != 0 && descriptor.width == selector.width && descriptor.height == selector.height && descriptor.depthOrLastArray + 1 == selector.layers && (selector.anyFormat || descriptor.format == selector.format);
+}
+
+}
+
+void StorageTexture::FlushDumpSelected(const char* reason) {
+    const auto& selector = StorageDumpSelection();
+    if (selector.address == 0 && selector.width == 0) return;
+    std::vector<std::pair<std::uint64_t, std::size_t>> ranges;
+    {
+        auto& pending = Pending();
+        std::lock_guard lock(pending.mutex);
+        for (const auto* texture : pending.textures) {
+            if (StorageDumpSelects(texture->descriptor)) ranges.emplace_back(texture->descriptor.baseAddress, static_cast<std::size_t>(texture->guestBytes));
+        }
+    }
+    for (const auto& [address, bytes] : ranges) static_cast<void>(FlushPending(address, bytes, nullptr, reason));
+}
+
 void StorageTexture::FlushAllPending(const char* reason) {
     static_cast<void>(FlushPending(0, std::numeric_limits<std::size_t>::max(), nullptr, reason));
 }
@@ -4257,15 +4302,15 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
             detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear.Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, tiled.Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, mip, true, layer, geometry.thick, {.pipeBankXor = descriptor.pipeBankXor});
         }
     }
-    // Debug aid: APS5_DUMP_STORAGE=<hex address> saves that storage image's first mip after each of
-    // its first 8 write-backs as storage_<address>_<n>.raw (u32 width, height, VkFormat, then rows).
-    static const std::uint64_t dumpAddress = [] { const char* text = std::getenv("APS5_DUMP_STORAGE"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }();
+    const bool dumpSelected = StorageDumpSelects(descriptor);
     static int dumps = 0;
     std::unique_ptr<Buffer> dump;
-    if (dumpAddress != 0 && descriptor.baseAddress == dumpAddress && dumps < 8) {
-        dump = std::make_unique<Buffer>(context, static_cast<std::size_t>(mips[0].linearSize), VK_BUFFER_USAGE_TRANSFER_DST_BIT);
+    if (dumpSelected && dumps < 8) {
+        dump = std::make_unique<Buffer>(context, static_cast<std::size_t>(mips[0].linearSize * arrayLayers), VK_BUFFER_USAGE_TRANSFER_DST_BIT);
         RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
-        CopyBuffer(context, commands, linear.Handle(), mips[0].linearOffset, dump->Handle(), 0, mips[0].linearSize);
+        for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) {
+            if (geometry.HasLayer(0, layer)) CopyBuffer(context, commands, linear.Handle(), geometry.LinearLayerOffset(layer) + mips[0].linearOffset, dump->Handle(), layer * mips[0].linearSize, mips[0].linearSize);
+        }
     }
     const auto toCopy = WholeBufferBarrier(tiled.Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
     VkImageMemoryBarrier backToGeneral = toSource;

core/libs/prx/libSceAgcDriver/PostCompile.cmake

@@ -28,7 +28,7 @@ foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_gra
     endif()
 endforeach()
 
-target_sources(agc_driver_bda_device_tests PRIVATE tests/ColorTransferTests.cpp Graphics/src/ColorTargetLayout.cpp Execution/src/GuestMemory.cpp)
+target_sources(agc_driver_bda_device_tests PRIVATE tests/ColorTransferTests.cpp Graphics/src/ColorTargetLayout.cpp Graphics/src/TextureTiling.cpp Graphics/src/TextureFormat.cpp Execution/src/GuestMemory.cpp)
 target_include_directories(agc_driver_bda_device_tests PRIVATE ${agcTextureDetileGeneratedRoot})
 
 if(TARGET agc_driver_visual_test)

core/libs/prx/libSceAgcDriver/Resource/include/Registration.hpp

@@ -1,4 +1,8 @@
 #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_RESOURCE_INCLUDE_REGISTRATION_HPP
 #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_RESOURCE_INCLUDE_REGISTRATION_HPP
 
+#include <cstdint>
+
+extern "C" bool AgcDriverWorkloadStreamRegistered_nid_postfix(std::uint32_t streamId);
+
 #endif

core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp

@@ -2,15 +2,27 @@
 
 #include <cstdint>
 #include <cstddef>
+#include <mutex>
 #include "SceTypes.hpp"
 #include "prx/libc/include/General.hpp"
 
 // Resource registration feeds GPU debugging tools, none of which are attached; the game treats this code as benign.
 static constexpr int SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE = static_cast<int>(0x8A6C9018);
 static constexpr uint32_t RESOURCE_REGISTRATION_MAX_NAME_LENGTH = 0xfc;
+static constexpr uint32_t WORKLOAD_STREAM_MIN_ID = 1;
+static constexpr uint32_t WORKLOAD_STREAM_MAX_ID = 31;
+
+static std::mutex g_workloadStreamLock;
+static uint32_t g_workloadStreams = 0;
 
 extern "C" {
 
+bool AgcDriverWorkloadStreamRegistered_nid_postfix(uint32_t streamId) {
+    if (streamId < WORKLOAD_STREAM_MIN_ID || streamId > WORKLOAD_STREAM_MAX_ID) return false;
+    std::lock_guard lock(g_workloadStreamLock);
+    return (g_workloadStreams & (1u << streamId)) != 0;
+}
+
 int APS5_VABI sceAgcDriverRegisterOwner(uint32_t* owner_handle, const char* name) {
     (void)owner_handle;
     (void)name;
@@ -37,8 +49,10 @@ int APS5_VABI sceAgcDriverRegisterMultipleResources(uint32_t* resourceHandle, ui
 }
 
 int APS5_VABI sceAgcDriverRegisterWorkloadStream(uint32_t stream_id, const void* stream) {
- (void)stream_id;
- (void)stream;
+ if (stream != nullptr && stream_id >= WORKLOAD_STREAM_MIN_ID && stream_id <= WORKLOAD_STREAM_MAX_ID) {
+  std::lock_guard lock(g_workloadStreamLock);
+  g_workloadStreams |= 1u << stream_id;
+ }
  return 0;
 }
 
@@ -69,7 +83,10 @@ int APS5_VABI sceAgcDriverUnregisterAllResourcesForOwner(uint32_t owner_handle)
 }
 
 int APS5_VABI sceAgcDriverUnregisterWorkloadStream(uint32_t stream_id) {
-    (void)stream_id;
+    if (stream_id >= WORKLOAD_STREAM_MIN_ID && stream_id <= WORKLOAD_STREAM_MAX_ID) {
+        std::lock_guard lock(g_workloadStreamLock);
+        g_workloadStreams &= ~(1u << stream_id);
+    }
     return 0;
 }
 

core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp

@@ -119,6 +119,62 @@ void importSpanTests(const Context& context) {
     ::operator delete(block, std::align_val_t{piece});
 }
 
+void failedImportRemapTests(const Context& context, const BdaTestAccess& access) {
+    constexpr std::size_t piece = 65536;
+    auto* block = static_cast<std::byte*>(::operator new(4 * piece, std::align_val_t{piece}));
+    auto importing = context;
+    importing.hostImportAlignment = piece;
+    const auto address = reinterpret_cast<std::uintptr_t>(block);
+    const auto add = [&](std::size_t index) {
+        GuestAllocations::Mutation mutation;
+        mutation.Add(block + index * piece, piece, true, true, true);
+    };
+    const auto remove = [&](std::size_t index) {
+        GuestAllocations::Mutation mutation;
+        mutation.Remove(block + index * piece);
+    };
+    SetHostImportWindow(0);
+    for (std::size_t replaced = 0; replaced < 2; ++replaced) {
+        add(0);
+        add(1);
+        std::weak_ptr<const GuestAllocations::Range> owner;
+        {
+            const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
+            const auto found = std::find_if(lease.begin(), lease.end(), [&](const auto& range) { return range->address == address + replaced * piece; });
+            Require(found != lease.end(), "failed import fixture has no registered owner");
+            owner = *found;
+        }
+        access.limitMemory(0);
+        const auto attempts = access.allocationAttempts();
+        Require(HostImportFor(importing, address, 2 * piece) == nullptr && access.allocationAttempts() == attempts + 1, "host import failure was not exercised");
+        Require(owner.use_count() == 1, "a failed import pins its owner");
+        Require(HostImportFor(importing, address, 2 * piece) == nullptr && access.allocationAttempts() == attempts + 1, "an unchanged failed import was retried");
+        add(3);
+        Require(HostImportFor(importing, address, 2 * piece) == nullptr && access.allocationAttempts() == attempts + 1, "an unrelated mapping retried a failed import");
+        remove(replaced);
+        Require(owner.expired(), "a failed import kept the replaced owner alive");
+        add(replaced);
+        access.limitMemory(std::nullopt);
+        const auto* imported = HostImportFor(importing, address, 2 * piece);
+        Require(imported != nullptr && imported->base == address && imported->bytes == 2 * piece && access.allocationAttempts() == attempts + 2, "same-address remap retained a failed import decision");
+        remove(0);
+        remove(1);
+        remove(3);
+        ClearHostImports(importing.device);
+    }
+    add(0);
+    add(1);
+    access.limitMemory(0);
+    Require(HostImportFor(importing, address, 2 * piece) == nullptr, "span failure was not exercised");
+    remove(1);
+    access.limitMemory(std::nullopt);
+    const auto* shortened = HostImportFor(importing, address, piece);
+    Require(shortened != nullptr && shortened->bytes == piece, "a failed span suppressed an import of its remaining range");
+    remove(0);
+    ClearHostImports(importing.device);
+    ::operator delete(block, std::align_val_t{piece});
+}
+
 void importWindowTests(const Context& context) {
     constexpr std::size_t piece = 65536;
     constexpr std::size_t window = 2 * piece;
@@ -1105,6 +1161,7 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
     }
     importCrossingTests(context, access);
     importSpanTests(context);
+    failedImportRemapTests(context, access);
     importWindowTests(context);
     heapMirrorTests(context, access);
     importedHeapMirrorTests(context, access);

core/libs/prx/libSceAgcDriver/tests/ConstantPhi.cpp

@@ -0,0 +1,91 @@
+#include "IntermediateRepresentation/IrBuilder.hpp"
+#include "Optimization/ConstantFolder.hpp"
+#include "Optimization/DeadCodeEliminator.hpp"
+#include <cstdio>
+#include <stdexcept>
+
+using namespace ShaderRecompiler;
+
+namespace {
+
+void Require(bool condition, const char* message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+void CheckDiamond(unsigned mode) {
+    IrProgram program;
+    for (std::uint32_t id = 0; id < 4u; ++id) {
+        auto& block = program.CreateBlock();
+        program.BlockOrder().push_back(&block);
+        BlockInfo info;
+        info.id = id;
+        info.terminator.kind = TerminatorKind::Return;
+        program.Metadata().blockInfo.push_back(info);
+    }
+    auto& entry = *program.BlockOrder()[0];
+    auto& left = *program.BlockOrder()[1];
+    auto& right = *program.BlockOrder()[2];
+    auto& merge = *program.BlockOrder()[3];
+    program.SetEntryBlock(entry);
+    entry.AddBranch(&left);
+    entry.AddBranch(&right);
+    left.AddBranch(&merge);
+    right.AddBranch(&merge);
+    auto& entryInfo = program.Metadata().blockInfo[0];
+    entryInfo.terminator.kind = TerminatorKind::ConditionalBranch;
+    entryInfo.terminator.trueBlock = 1u;
+    entryInfo.terminator.falseBlock = 2u;
+    for (std::uint32_t id = 1; id < 3u; ++id) {
+        program.Metadata().blockInfo[id].terminator.kind = TerminatorKind::Branch;
+        program.Metadata().blockInfo[id].terminator.trueBlock = 3u;
+    }
+    IrBuilder builder(program);
+    builder.SetInsertionPoint(entry);
+    auto& lane = builder.Emit(IrOpcode::LaneId, IrType::U32, {});
+    auto& limit = builder.Constant(16u);
+    auto& condition = builder.Emit(IrOpcode::ULessThan32, IrType::Bool, {&lane, &limit});
+    entryInfo.condition = &condition;
+    auto& one = builder.Constant(1u);
+    auto& two = builder.Constant(2u);
+    IrValue* first = &lane;
+    IrValue* second = &lane;
+    if (mode == 0u || mode == 1u) {
+        builder.SetInsertionPoint(left);
+        first = mode == 0u ? &builder.Emit(IrOpcode::SelectU32, IrType::U32, {&condition, &one, &two}) : &builder.Emit(IrOpcode::LaneId, IrType::U32, {});
+        builder.SetInsertionPoint(right);
+        second = mode == 0u ? &builder.Emit(IrOpcode::SelectU32, IrType::U32, {&condition, &one, &two}) : &builder.Emit(IrOpcode::LaneId, IrType::U32, {});
+    } else if (mode == 3u || mode == 4u) {
+        first = &one;
+        second = &program.CreateValue(IrOpcode::Void, IrType::U32);
+        second->SetImmediateU32(mode == 3u ? 1u : 2u);
+    }
+    builder.SetInsertionPoint(merge);
+    auto& phi = builder.Emit(IrOpcode::Phi, IrType::U32, {});
+    phi.AddPhiOperand(&left, first);
+    phi.AddPhiOperand(&right, second);
+    auto& reference = builder.Emit(IrOpcode::ReferenceU32, IrType::Void, {&phi});
+    ValidateProgram(program, true);
+    ConstantFolder{}.Fold(program);
+    DeadCodeEliminator{}.RemoveIdentities(program);
+    DeadCodeEliminator{}.Eliminate(program);
+    ValidateProgram(program, true);
+    const auto* result = reference.Argument(0)->Resolve();
+    if (mode == 0u || mode == 1u || mode == 4u) {
+        Require(result == &phi && phi.Argument(0) == first && phi.Argument(1) == second, "phi lost distinct branch definitions or constants");
+    } else {
+        Require(result == first, "phi did not fold identical definitions or equal constants");
+    }
+}
+
+}
+
+int main() {
+    try {
+        for (unsigned mode = 0; mode < 5u; ++mode) CheckDiamond(mode);
+        std::puts("constant phi dominance tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "%s\n", error.what());
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/ControlFlow.cpp

@@ -566,6 +566,16 @@ done:
   buffer_store_dword v1, off, s[0:3], 0
   s_endpgm)",
          Store({0x7d880090u, 0xbf860004u, 0x7d880088u, 0xbf870003u, 0x4a020081u, 0xbf820005u, 0x4a020082u, 0x7d880084u, 0xbf860001u, 0x4a020285u, 0x4a020283u}), Split::Clone},
+        {"memory access in a single-block loop latch", R"(
+  s_mov_b32 s8, 0
+  v_mov_b32 v1, 0
+loop:
+  buffer_store_dword v1, off, s[0:3], 0
+  s_add_u32 s8, s8, 1
+  s_cmp_lt_u32 s8, 4
+  s_cbranch_scc1 loop
+  s_endpgm)",
+         {0xbe880380u, 0x7e020280u, 0xe0700000u, 0x80000100u, 0x80088108u, 0xbf0a8408u, 0xbf85fffbu, 0xbf810000u}, Split::None},
         {"shared tail in a loop", R"(
   s_mov_b32 s8, 0
   v_mov_b32 v1, 0

core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp

@@ -186,5 +186,26 @@ void RunDepthSurfaceReuseTests() {
     Require(lookup(context, {stencil, FormatR8G8B8A8Unorm, 320, 180, TileRenderTarget64KB}) == nullptr, "an RGBA8 view of a stencil plane's own extent must be read as reused memory");
     Require(lookup(context, {stencil, FormatR32Float, 320, 180}) == nullptr, "an R32F view of a stencil plane's own extent must be read as reused memory");
 
+    const auto storageAliasesDepth = [&](const Context& owner, const View& view) {
+        const auto words = tsharp(view);
+        return DepthSurfaceAt(owner, words, DecodeTextureResource(words));
+    };
+    Require(storageAliasesDepth(context, {depth32, FormatR32Float, 384, 384}), "storage access to the actual D32 plane must remain rejected");
+    Require(storageAliasesDepth(context, {depth16, FormatR16Uint, 256, 256}), "raw D16 storage access must remain rejected");
+    Require(!storageAliasesDepth(context, {depth32, FormatR32Float, 128, 128}), "temporary memory reused at a different extent is not a depth storage alias");
+    Require(!storageAliasesDepth(context, {depth32, FormatR11G11B10Float, 384, 384}), "temporary memory reused for a color format is not a depth storage alias");
+    Require(!storageAliasesDepth(context, {depth32, FormatR32Uint, 384, 384, TileRenderTarget64KB}), "raw integer color memory without a depth layout is not a depth storage alias");
+    auto otherDevice = context;
+    static int otherDeviceTag = 0;
+    otherDevice.device = reinterpret_cast<VkDevice>(&otherDeviceTag);
+    Require(!storageAliasesDepth(otherDevice, {depth32, FormatR32Float, 384, 384}), "a depth surface on another device must not block a storage lookup");
+
+    constexpr std::uint64_t stencil16 = 0x60000000;
+    DepthSurfaceView(context, {depth16, stencil16, {128, 128}, VK_FORMAT_D16_UNORM_S8_UINT, 1.0f, 0});
+    Require(!storageAliasesDepth(context, {depth16, FormatR16Unorm, 256, 256}), "an older depth extent at the same address must not override the newest surface");
+    Require(storageAliasesDepth(context, {depth16, FormatR16Unorm, 128, 128}), "the newest depth extent must still reject actual storage access");
+    Require(storageAliasesDepth(context, {stencil16, FormatR8Uint, 128, 128}), "storage access to the actual stencil plane must remain rejected");
+    Require(!storageAliasesDepth(context, {stencil16, FormatR11G11B10Float, 128, 128}), "a stencil address reused for color is not a stencil storage alias");
+
     ClearDepthSurfaces(context.device);
 }

core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp

@@ -0,0 +1,71 @@
+#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
+#include <cstdint>
+#include <cstdio>
+#include <exception>
+#include <string>
+
+using namespace AgcDriver::Graphics;
+
+namespace {
+
+int failures = 0;
+
+void Expect(bool condition, const std::string& what) {
+    if (condition) return;
+    std::fprintf(stderr, "FAIL: %s\n", what.c_str());
+    ++failures;
+}
+
+constexpr std::uint32_t HostType = 0;
+constexpr std::uint32_t DeviceType = 1;
+constexpr std::uint64_t MiB = 1ull << 20u;
+
+std::uint64_t reportedBudget = 0;
+std::uint64_t reportedUsage = 0;
+
+VKAPI_ATTR void VKAPI_CALL mockMemoryProperties2(VkPhysicalDevice, VkPhysicalDeviceMemoryProperties2* properties) {
+    auto* reported = static_cast<VkPhysicalDeviceMemoryBudgetPropertiesEXT*>(properties->pNext);
+    reported->heapBudget[0] = reportedBudget;
+    reported->heapUsage[0] = reportedUsage;
+}
+
+Context mockContext(bool budgetExtension) {
+    Context context{};
+    context.physical = reinterpret_cast<VkPhysicalDevice>(static_cast<std::uintptr_t>(1));
+    context.memoryProperties2 = budgetExtension ? mockMemoryProperties2 : nullptr;
+    context.memory.memoryHeapCount = 2;
+    context.memory.memoryHeaps[0] = {4096 * MiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT};
+    context.memory.memoryHeaps[1] = {16384 * MiB, 0};
+    context.memory.memoryTypeCount = 2;
+    context.memory.memoryTypes[HostType] = {0, 1};
+    context.memory.memoryTypes[DeviceType] = {VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 0};
+    return context;
+}
+
+void Choose(const char* what, bool budgetExtension, std::uint64_t budget, std::uint64_t usage, std::uint32_t mask, std::uint32_t expected) {
+    reportedBudget = budget;
+    reportedUsage = usage;
+    const auto type = mockContext(budgetExtension).DeviceMemoryType(mask, 64 * MiB, "test");
+    Expect(type == expected, std::string(what) + ": chose memory type " + std::to_string(type) + ", expected " + std::to_string(expected));
+}
+
+}
+
+int main() {
+    try {
+        const std::uint32_t both = (1u << HostType) | (1u << DeviceType);
+        Choose("no VK_EXT_memory_budget", false, 0, 0, both, DeviceType);
+        Choose("budget not reported", true, 0, 4000 * MiB, both, DeviceType);
+        Choose("room in the device-local budget", true, 3000 * MiB, 2000 * MiB, both, DeviceType);
+        Choose("allocation fills the budget exactly", true, 3000 * MiB, 2936 * MiB, both, DeviceType);
+        Choose("allocation exceeds the budget", true, 3000 * MiB, 2937 * MiB, both, HostType);
+        Choose("usage already over the budget", true, 2952 * MiB, 3633 * MiB, both, HostType);
+        Choose("no host type allowed", true, 3000 * MiB, 3000 * MiB, 1u << DeviceType, DeviceType);
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "FAIL: %s\n", error.what());
+        return 1;
+    }
+    if (failures != 0) return 1;
+    std::puts("device memory placement tests passed");
+    return 0;
+}

core/libs/prx/libSceAgcDriver/tests/Flip.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/PresentMode.hpp"
 #include "prx/libc/include/Shutdown.hpp"
 #include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp"
 #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
@@ -265,6 +266,46 @@ void testFlipHold() {
     AgcDriverUnregisterVideoOutput_nid_postfix(7, output);
 }
 
+void testComputeRenderingWait() {
+    auto output = std::make_shared<Output>();
+    AgcDriverRegisterVideoOutput_nid_postfix(7, output);
+    std::array<std::uint32_t, 4> wait{0xc0021018, 8, 0, 0};
+    Packet packet{wait.data(), 4, 0, {}};
+    check(expectFailure([&] { sceAgcDriverSubmitAcb(0x20, &packet); }).find("unregistered video output") != std::string::npos, "a compute rendering wait on an unregistered output was accepted");
+    wait = {0xc0021018, 7, 0, 1};
+    check(expectFailure([&] { sceAgcDriverSubmitAcb(0x20, &packet); }).find("rendering wait mode") != std::string::npos, "a compute rendering wait with a mode was accepted");
+    {
+        std::lock_guard lock(output->state->mutex);
+        output->state->gate = true;
+    }
+    struct GateRelease {
+        std::shared_ptr<State> state;
+        ~GateRelease() {
+            {
+                std::lock_guard lock(state->mutex);
+                state->gate = false;
+            }
+            state->changed.notify_all();
+        }
+    } gateRelease{output->state};
+    wait = {0xc0021018, 7, 0, 0};
+    check(sceAgcDriverSubmitAcb(0x20, &packet) == 0, "compute rendering wait submission failed");
+    {
+        std::unique_lock lock(output->state->mutex);
+        check(output->state->changed.wait_for(lock, std::chrono::seconds(5), [&] { return output->state->waiting; }), "compute worker did not reach the rendering wait");
+    }
+    auto idle = std::async(std::launch::async, [] { AgcDriverWaitIdle_nid_postfix(); });
+    check(idle.wait_for(std::chrono::milliseconds(200)) == std::future_status::timeout, "the driver went idle past a closed compute rendering wait");
+    {
+        std::lock_guard lock(output->state->mutex);
+        output->state->gate = false;
+    }
+    output->state->changed.notify_all();
+    check(idle.wait_for(std::chrono::seconds(5)) == std::future_status::ready, "a compute rendering wait stayed blocked after release");
+    idle.get();
+    AgcDriverUnregisterVideoOutput_nid_postfix(7, output);
+}
+
 void testFailure() {
     auto output = std::make_shared<Output>();
     output->state->fail = true;
@@ -298,10 +339,34 @@ void testReset(bool compute) {
 
 }
 
+void testPresentMode() {
+    using AgcDriver::SelectPresentMode;
+    const std::array wayland{VK_PRESENT_MODE_FIFO_KHR, VK_PRESENT_MODE_IMMEDIATE_KHR};
+    const std::array all{VK_PRESENT_MODE_IMMEDIATE_KHR, VK_PRESENT_MODE_MAILBOX_KHR, VK_PRESENT_MODE_FIFO_KHR};
+    check(SelectPresentMode(nullptr, wayland) == VK_PRESENT_MODE_FIFO_KHR, "the default present mode is not FIFO");
+    check(SelectPresentMode("fifo", wayland) == VK_PRESENT_MODE_FIFO_KHR, "fifo was not selected");
+    check(SelectPresentMode("immediate", wayland) == VK_PRESENT_MODE_IMMEDIATE_KHR, "immediate was not selected");
+    check(SelectPresentMode("mailbox", all) == VK_PRESENT_MODE_MAILBOX_KHR, "mailbox was not selected");
+    const std::array latest{VK_PRESENT_MODE_FIFO_KHR, VK_PRESENT_MODE_FIFO_LATEST_READY_KHR};
+    check(SelectPresentMode("fifo_latest_ready", latest) == VK_PRESENT_MODE_FIFO_LATEST_READY_KHR, "fifo_latest_ready was not selected");
+    check(SelectPresentMode(nullptr, latest) == VK_PRESENT_MODE_FIFO_KHR, "the default changed on a surface with fifo_latest_ready");
+    check(expectFailure([&] { SelectPresentMode("fifo_latest_ready", wayland); }).find("does not support") != std::string::npos, "fifo_latest_ready was accepted on a surface without it");
+    check(expectFailure([&] { SelectPresentMode("mailbox", wayland); }).find("does not support") != std::string::npos, "an unsupported present mode was accepted");
+    check(expectFailure([&] { SelectPresentMode("Immediate", all); }).find("must be fifo, fifo_latest_ready, mailbox or immediate") != std::string::npos, "an unknown present mode name was accepted");
+    check(expectFailure([&] { SelectPresentMode("", all); }).find("must be fifo, fifo_latest_ready, mailbox or immediate") != std::string::npos, "an empty present mode name was accepted");
+    const std::array<VkPresentModeKHR, 0> none{};
+    check(expectFailure([&] { SelectPresentMode(nullptr, none); }).find("does not support") != std::string::npos, "FIFO was selected on a surface without it");
+}
+
 int main(int argc, char** argv) {
     try {
+        if (argc == 2 && std::string(argv[1]) == "present-mode") {
+            testPresentMode();
+            std::puts("present mode selection tests passed");
+            return 0;
+        }
         if (argc == 2) testReset(std::string(argv[1]) == "compute");
-        else { testFlipAndBoundary(); testFlipHold(); testFailure(); }
+        else { testFlipAndBoundary(); testFlipHold(); testComputeRenderingWait(); testFailure(); }
         const auto shutdown = expectFailure([] { LibcRunShutdown_nid_postfix(); });
         check(shutdown.find(argc == 2 ? (std::string(argv[1]) == "compute" ? "not readable" : "required shader register") : "intentional flip failure") != std::string::npos, "shutdown lost worker failure");
         std::puts("AGC flip and suspend tests passed");

core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp

@@ -19,8 +19,8 @@ void Check(bool condition, const std::string& message) {
 }
 
 SpirvTarget Target() {
-    static constexpr std::array<std::uint32_t, 3> capabilities{
-        spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess,
+    static constexpr std::array<std::uint32_t, 4> capabilities{
+        spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilityClipDistance,
     };
     static constexpr std::array<std::string_view, 4> extensions{
         "SPV_EXT_descriptor_indexing", "SPV_KHR_8bit_storage", "SPV_KHR_float_controls", "SPV_KHR_physical_storage_buffer",
@@ -35,6 +35,7 @@ struct Counts {
     std::size_t calls = 0;
     std::size_t functions = 0;
     std::size_t notInlined = 0;
+    std::size_t compareExchanges = 0;
 };
 
 Counts Count(std::span<const std::uint32_t> words) {
@@ -46,6 +47,7 @@ Counts Count(std::span<const std::uint32_t> words) {
         const auto opcode = words[offset] & spv::OpCodeMask;
         if (opcode == spv::OpSwitch) ++counts.switches;
         if (opcode == spv::OpFunctionCall) ++counts.calls;
+        if (opcode == spv::OpAtomicCompareExchange) ++counts.compareExchanges;
         if (opcode == spv::OpFunction) {
             ++counts.functions;
             if (count > 3 && (words[offset + 3] & spv::FunctionControlDontInlineMask) != 0) ++counts.notInlined;
@@ -55,24 +57,27 @@ Counts Count(std::span<const std::uint32_t> words) {
     return counts;
 }
 
-Counts Compile(std::size_t loads, bool formattedStores = false, bool gpu = false) {
+Counts Compile(std::size_t loads, bool formattedStores = false, bool gpu = false, bool vertex = false, bool registration = false) {
     std::vector<std::uint32_t> code;
     if (gpu) code.push_back(0x7e160500u);
     for (std::uint32_t load = 0; load < loads; ++load) code.insert(code.end(), {0xe0000000u | (load * 4u), 0x80020000u | ((10u + load) << 8u)});
     code.push_back(0xbf8c3f70u);
     for (std::uint32_t load = 0; load < loads; ++load) code.insert(code.end(), {(formattedStores ? 0xe0100000u : 0xe0700000u) | (load * 4u), 0x80010000u | ((10u + load) << 8u)});
+    if (vertex) code.insert(code.end(), {0xf80008cfu, 0x03020100u});
     code.push_back(0xbf810000u);
     const std::span<const std::uint32_t> words(code);
     const std::array<MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(words)}}};
     const std::array<std::uint32_t, 12> userData{0u, 0u, 0u, 0u, 0x10000000u, 0u, 0x1000u, 0x31016facu, 0x20000000u, 0u, 0x1000u, 0x31038facu};
     RecompileRequest request{};
-    request.shader = {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), words, 0, {}};
+    request.shader = {vertex ? ShaderStage::Vertex : ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), words, 0, {}};
     request.target = Target();
+    request.target.registrationModules = registration;
     request.layout = {0, 0, 0, 128};
     request.context.waveSize = 32;
     request.context.userDataBaseRegister = 0;
     request.context.userData = userData;
-    request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
+    if (vertex) request.context.vertex = ShaderVertexStageInfo{};
+    else request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
     request.context.memory = memory;
     request.useCache = false;
     return Count(Recompile(request).spirv.Words());
@@ -85,6 +90,12 @@ int main() {
         const auto none = Compile(0);
         const auto one = Compile(1);
         const auto three = Compile(3);
+        const auto gpuOne = Compile(1, false, true);
+        const auto vertexSpecialized = Compile(1, false, false, true, false);
+        const auto vertexRegistration = Compile(1, false, false, true, true);
+        Check(vertexRegistration.compareExchanges == vertexSpecialized.compareExchanges, "formatted buffer once: in registration form a store through an aligned V# kept the unaligned compare-exchange path (" + std::to_string(vertexRegistration.compareExchanges) + " compare-exchanges, " + std::to_string(vertexSpecialized.compareExchanges) + " specialized)");
+        Check(vertexRegistration.words < vertexSpecialized.words + vertexSpecialized.words / 2u, "formatted buffer once: in registration form a formatted load from a known descriptor kept every format's decoder (" + std::to_string(vertexRegistration.words) + " words, " + std::to_string(vertexSpecialized.words) + " specialized)");
+        Check(one.words < gpuOne.words / 2u, "formatted buffer once: a formatted load from a known descriptor kept every format's decoder (" + std::to_string(one.words) + " words, a GPU-selected descriptor needs " + std::to_string(gpuOne.words) + ")");
         Check(one.calls == none.calls + 1 && one.functions == none.functions + 1, "formatted buffer once: one formatted load emitted " + std::to_string(one.calls - none.calls) + " calls and " + std::to_string(one.functions - none.functions) + " functions more than none, expected one call to one shared function");
         Check(three.calls == one.calls + 2 && three.functions == one.functions, "formatted buffer once: two more formatted loads from the same descriptor emitted " + std::to_string(three.calls - one.calls) + " calls and " + std::to_string(three.functions - one.functions) + " functions more, expected two calls and no function");
         Check(three.switches == one.switches, "formatted buffer once: two more formatted loads from the same descriptor emitted " + std::to_string(three.switches - one.switches) + " more format switches");
@@ -94,12 +105,11 @@ int main() {
         Check(storeOne.functions == none.functions + 2 && storeOne.calls == none.calls + 2, "formatted buffer once: a formatted load and a formatted store to another resource emitted " + std::to_string(storeOne.functions - none.functions) + " functions and " + std::to_string(storeOne.calls - none.calls) + " calls more than none, expected one function and one call each");
         Check(storeThree.functions == storeOne.functions && storeThree.calls == storeOne.calls + 4 && storeThree.switches == storeOne.switches, "formatted buffer once: two more formatted loads and stores emitted " + std::to_string(storeThree.functions - storeOne.functions) + " functions, " + std::to_string(storeThree.calls - storeOne.calls) + " calls and " + std::to_string(storeThree.switches - storeOne.switches) + " switches more, expected four calls only");
         Check(three.words - one.words < one.words / 10u, "formatted buffer once: 2 more formatted loads added " + std::to_string(three.words - one.words) + " SPIR-V words to a " + std::to_string(one.words) + "-word module");
-        const auto gpuOne = Compile(1, false, true);
         const auto gpuThree = Compile(3, false, true);
         Check(gpuThree.functions == gpuOne.functions && gpuThree.calls > gpuOne.calls, "formatted buffer once: two more formatted loads from a GPU-selected descriptor emitted " + std::to_string(gpuThree.functions - gpuOne.functions) + " more functions, expected calls to the shared one");
         Check(gpuThree.switches == gpuOne.switches, "formatted buffer once: two more formatted loads from a GPU-selected descriptor emitted " + std::to_string(gpuThree.switches - gpuOne.switches) + " more format switches");
         Check(gpuThree.words - gpuOne.words < gpuOne.words / 10u, "formatted buffer once: 2 more formatted loads from a GPU-selected descriptor added " + std::to_string(gpuThree.words - gpuOne.words) + " SPIR-V words to a " + std::to_string(gpuOne.words) + "-word module");
-        std::printf("formatted buffer once tests passed (%zu words for 1 load, %zu for 3; GPU-selected %zu and %zu)\n", one.words, three.words, gpuOne.words, gpuThree.words);
+        std::printf("formatted buffer once tests passed (%zu words for 1 load, %zu for 3; GPU-selected %zu and %zu; vertex %zu specialized, %zu registration)\n", one.words, three.words, gpuOne.words, gpuThree.words, vertexSpecialized.words, vertexRegistration.words);
         return 0;
     } catch (const std::exception& error) {
         std::fprintf(stderr, "%s\n", error.what());

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -450,6 +450,73 @@ void hardwareScreenOffsetTests() {
     expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "missing register");
 }
 
+void MsaaEnableSingleSampleTests() {
+    using AgcDriver::Graphics::DecodeState;
+    using AgcDriver::Graphics::DrawRejection;
+    const auto enabled = [] {
+        auto queue = makeState();
+        queue.context[0x1b3] = 2;
+        queue.context[0x1b4] = 2;
+        queue.context[0x292] = 3;
+        for (std::uint32_t pixel = 0; pixel < 4; ++pixel) queue.context[0x2fe + pixel * 4u] = 0;
+        return queue;
+    };
+    const auto refused = [](const AgcDriver::QueueState& queue, std::string_view reason) {
+        expectFailure([&] { DecodeState(queue); }, reason);
+        const auto rejection = DrawRejection(queue, false);
+        Require(rejection.find(reason) != std::string::npos, "the precheck did not refuse MSAA_ENABLE on a single-sample target (" + std::string(reason) + "): " + rejection);
+    };
+    auto queue = enabled();
+    auto clear = queue;
+    clear.context[0x292] = 2;
+    const auto baseline = DecodeState(clear);
+    for (const auto locations : {0x00000000u, 0xcc44cc00u}) {
+        for (std::uint32_t pixel = 0; pixel < 4; ++pixel) queue.context[0x2fe + pixel * 4u] = locations;
+        const auto state = DecodeState(queue);
+        Require(state.samples == 1 && state.colors.size() == 1 && state.colors[0].samples == 1 && !state.alphaToCoverage, "MSAA_ENABLE on a single-sample target did not decode as single-sampled");
+        Require(state.topology == baseline.topology && state.conservativeRasterization == baseline.conservativeRasterization && state.cullMode == baseline.cullMode && state.frontFace == baseline.frontFace, "MSAA_ENABLE on a single-sample target changed the rasterizer state");
+        Require(state.scissor.offset.x == baseline.scissor.offset.x && state.scissor.offset.y == baseline.scissor.offset.y && state.scissor.extent.width == baseline.scissor.extent.width && state.scissor.extent.height == baseline.scissor.extent.height, "MSAA_ENABLE on a single-sample target changed the scissor");
+        Require(state.colors[0].address == baseline.colors[0].address && state.colors[0].format == baseline.colors[0].format && state.blend.colorWriteMask == baseline.blend.colorWriteMask, "MSAA_ENABLE on a single-sample target changed the color target");
+        const auto rejection = DrawRejection(queue, false);
+        Require(rejection.empty(), "the precheck refused MSAA_ENABLE on a single-sample target: " + rejection);
+    }
+    for (std::uint32_t pixel = 0; pixel < 4; ++pixel) {
+        for (const auto location : {0x04u, 0x40u, 0xc4u}) {
+            queue = enabled();
+            queue.context[0x2fe + pixel * 4u] = location;
+            refused(queue, "off the pixel center");
+        }
+    }
+    queue = enabled();
+    queue.context.erase(0x306);
+    expectFailure([&] { DecodeState(queue); }, "missing register");
+    queue = enabled();
+    queue.context[0x2dc] = 0xaa01;
+    refused(queue, "MSAA_ENABLE on a single-sample target with alpha-to-coverage");
+    queue = enabled();
+    queue.context[0x313] = 0x6001;
+    refused(queue, "MSAA_ENABLE on a single-sample target with conservative rasterization");
+    for (const auto primitive : {2u, 7u, 17u}) {
+        queue = enabled();
+        queue.userConfig[0x242] = primitive;
+        refused(queue, "primitives other than triangles");
+    }
+    queue = enabled();
+    queue.userConfig[0x242] = 1;
+    Require(DrawRejection(queue, false).find("primitives other than triangles") != std::string::npos, "the precheck accepted MSAA_ENABLE on a single-sample target for points");
+    for (const auto primitive : {5u, 6u}) {
+        queue = enabled();
+        queue.userConfig[0x242] = primitive;
+        Require(DecodeState(queue).samples == 1 && DrawRejection(queue, false).empty(), "MSAA_ENABLE on a single-sample target was refused for a triangle fan or strip");
+    }
+    queue = enabled();
+    queue.context[0x292] = 7;
+    refused(queue, "scan conversion mode");
+    clear.context[0x2fe] = 0x44;
+    clear.context[0x2dc] = 0xaa01;
+    Require(DecodeState(clear).alphaToCoverage && DrawRejection(clear, false).empty(), "a single-sample target without MSAA_ENABLE was refused over its sample locations or alpha-to-coverage");
+}
+
 void ShaderStageTests() {
     auto queue = makeState();
     for (const auto routing : {0x2000u, 0x2010u, 0x02002000u, 0x02002010u}) {
@@ -1021,6 +1088,26 @@ void ReversedComponentOrderTests() {
     }
 }
 
+void Color1555TargetTests() {
+    auto queue = makeState();
+    queue.context[0x31c] = (queue.context[0x31c] & ~((0x1fu << 2u) | (7u << 8u) | (3u << 11u))) | (17u << 2u);
+    const auto state = AgcDriver::Graphics::DecodeState(queue);
+    Require(state.colors.size() == 1 && state.colors[0].format == VK_FORMAT_A1R5G5B5_UNORM_PACK16 && state.colors[0].elementBytes == 2 && state.colors[0].componentMapping == 0xc6u, "a 1_5_5_5 target did not render through A1R5G5B5 with X and Z exchanged");
+    Require(AgcDriver::Graphics::ExportMappings(state)[0] == 0xc6u && state.blends[0].colorWriteMask == 0xfu, "a 1_5_5_5 target did not write all four channels through its export mapping");
+    queue.context[0x8e] = 0x1u;
+    Require(AgcDriver::Graphics::DecodeState(queue).blends[0].colorWriteMask == VK_COLOR_COMPONENT_B_BIT, "a 1_5_5_5 target writing only X did not write the A1R5G5B5 channel in its low bits");
+    queue.context[0x8e] = 0xfu;
+    for (std::uint32_t i = 0; i < 4; ++i) queue.context[0x105 + i] = 0;
+    queue.context[0x1e0] = 0x40000504u;
+    const auto blended = AgcDriver::Graphics::DecodeState(queue);
+    Require(blended.blends[0].blendEnable && blended.blends[0].srcColorBlendFactor == VK_BLEND_FACTOR_SRC_ALPHA && blended.blends[0].dstColorBlendFactor == VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA, "alpha blending into a 1_5_5_5 target was not decoded");
+    queue.context[0x1e0] = 0;
+    for (const auto& [number, swap] : {std::pair{1u, 0u}, std::pair{4u, 0u}, std::pair{0u, 1u}}) {
+        queue.context[0x31c] = (queue.context[0x31c] & ~((7u << 8u) | (3u << 11u))) | (number << 8u) | (swap << 11u);
+        expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "unsupported color format 17 number type " + std::to_string(number) + " component swap " + std::to_string(swap));
+    }
+}
+
 void CompactedExportTests() {
     alignas(256) static std::array<std::byte, 1024> slotFourMemory{};
     const auto slotFour = reinterpret_cast<std::uintptr_t>(slotFourMemory.data());
@@ -1299,6 +1386,32 @@ void depthMaintenanceTests() {
     auto absent = makeState();
     absent.context.erase(0x000);
     Require(AgcDriver::Graphics::DepthMaintenanceRejection(absent).empty(), "an absent depth control produced a maintenance verdict");
+    auto readOnly = makeState();
+    readOnly.context[0x000] = 0x10;
+    readOnly.context[0x200] = 0x007007f3;
+    readOnly.context[0x010] = 0xa0000183;
+    readOnly.context[0x011] = 0x20000180;
+    readOnly.context[0x8e] = 0xffffffff;
+    readOnly.context[0x8f] = 0;
+    readOnly.shader.erase(0x8);
+    readOnly.shader.erase(0x9);
+    Require(AgcDriver::Graphics::DepthMaintenanceRejection(readOnly).empty(), "a resummarize draw with pass-through read-only depth and no stencil plane was rejected");
+    for (const auto [offset, value] : {std::pair{0x200u, 0x007007f7u}, {0x200u, 0x007007fbu}, {0x200u, 0x007007e3u}, {0x011u, 0x20000181u}, {0x8fu, 0xfu}, {0x000u, 0x11u}, {0x010u, 0xa0000182u}}) {
+        auto changed = readOnly;
+        changed.context[offset] = value;
+        Require(!AgcDriver::Graphics::DepthMaintenanceRejection(changed).empty(), "a resummarize draw with surface work or unsupported depth passed");
+    }
+    auto pixel = readOnly;
+    pixel.shader[0x8] = 1;
+    pixel.shader[0x9] = 0;
+    pixel.context[0x1c4] = 0;
+    pixel.context[0x203] = 0x40;
+    Require(!AgcDriver::Graphics::DepthMaintenanceRejection(pixel).empty(), "a resummarize draw with a running pixel stage passed");
+    for (const auto offset : {0x200u, 0x010u, 0x011u, 0x8eu, 0x8fu}) {
+        auto missing = readOnly;
+        missing.context.erase(offset);
+        Require(!AgcDriver::Graphics::DepthMaintenanceRejection(missing).empty(), "a resummarize draw without a required register passed");
+    }
 }
 
 // SPI_SHADER_Z_FORMAT (0x1c4) and the export enables of DB_SHADER_CONTROL (0x203): Z export needs a
@@ -1434,6 +1547,23 @@ void metadataPassTests() {
     queue.context[0x202] = 0xcc0060;
     queue.context[0x200] = 2;
     expectFailure([&] { DecodeColorMetadataPass(queue); }, "depth or stencil work");
+    queue.context[0x10] = 0x80000180;
+    queue.context[0x11] = 0x20000180;
+    queue.context[0x200] = 0x700763;
+    queue.context[0x0] = 0x20;
+    Require(DecodeColorMetadataPass(queue).has_value(), "depth and stencil state without depth or stencil planes refused the pass");
+    queue.context[0x10] = 0x80000183;
+    expectFailure([&] { DecodeColorMetadataPass(queue); }, "depth or stencil work");
+    queue.context[0x10] = 0x80000180;
+    queue.context[0x11] = 0x20000181;
+    expectFailure([&] { DecodeColorMetadataPass(queue); }, "depth or stencil work");
+    queue.context[0x11] = 0x20000180;
+    queue.context.erase(0x200);
+    queue.context.erase(0x0);
+    Require(DecodeColorMetadataPass(queue).has_value(), "unwritten depth and render control refused a pass without depth or stencil planes");
+    queue.context.erase(0x10);
+    queue.context.erase(0x11);
+    queue.context[0x0] = 0;
     queue.context[0x200] = 0x70;
     Require(DecodeColorMetadataPass(queue).has_value(), "an always-pass depth function without a test refused the pass");
     queue.context[0x91] = 0x40020;
@@ -1721,6 +1851,18 @@ void uint16ExportTests() {
     expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "color export format 8");
 }
 
+VkSampleCountFlags depthSampleCounts = VK_SAMPLE_COUNT_4_BIT;
+VkResult depthSampleResult = VK_SUCCESS;
+std::uint32_t depthSampleQueries = 0;
+
+VKAPI_ATTR VkResult VKAPI_CALL sampleLocationDepthProperties(VkPhysicalDevice, VkFormat format, VkImageType type, VkImageTiling tiling, VkImageUsageFlags usage, VkImageCreateFlags flags, VkImageFormatProperties* properties) {
+    Require(format == VK_FORMAT_D32_SFLOAT && type == VK_IMAGE_TYPE_2D && tiling == VK_IMAGE_TILING_OPTIMAL, "sample location query changed the depth image format or type");
+    Require(usage == (VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT) && flags == VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT, "sample location query differs from depth image creation");
+    ++depthSampleQueries;
+    properties->sampleCounts = depthSampleCounts;
+    return depthSampleResult;
+}
+
 void multisampleTests() {
     using AgcDriver::Graphics::ColorMetadataPass;
     using AgcDriver::Graphics::DecodeColorMetadataPass;
@@ -1729,13 +1871,19 @@ void multisampleTests() {
         auto queue = makeState();
         queue.context[0x2f8] = 0x00100001;
         queue.context[0x292] = 3;
+        queue.context[0x3b8] = 0x0dc00000u;
         for (std::uint32_t pixel = 0; pixel < 4; ++pixel) queue.context[0x2fe + pixel * 4u] = 0xcc44;
         queue.context[0x31d] = 0x9000;
-        queue.context[0x3b8] |= (0x18u << 19u) | 0x4000000u;
         queue.context[0x1b3] = 2;
         queue.context[0x1b4] = 2;
         return queue;
     };
+    const auto cmaskAddress = reinterpret_cast<std::uintptr_t>(cmaskMemory.data());
+    const auto setCmaskAddress = [&](auto& target, std::uint32_t slot = 0u) {
+        const auto stride = slot * 0xfu;
+        target.context[0x31f + stride] = static_cast<std::uint32_t>(cmaskAddress >> 8u);
+        target.context[0x398 + slot] = static_cast<std::uint32_t>(cmaskAddress >> 40u);
+    };
     auto queue = multisampled();
     auto state = DecodeState(queue);
     Require(state.samples == 2 && state.colors.size() == 1 && state.colors[0].samples == 2 && state.colors[0].cmaskAddress == 0, "2x multisampling did not decode");
@@ -1752,6 +1900,75 @@ void multisampleTests() {
     }
     state = DecodeState(queue);
     Require(state.samples == 8 && state.colors.size() == 1 && state.colors[0].samples == 8, "the standard 8x pattern (sample 7 at (7, -7), PPSA99902's registers) did not decode as standard");
+    queue.context[0x2f8] = 0x00200002;
+    queue.context[0x31d] = 0x12000;
+    for (std::uint32_t pixel = 0; pixel < 4; ++pixel) queue.context[0x2fe + pixel * 4u] = 0x22eea66a;
+    state = DecodeState(queue);
+    Require(state.samples == 4 && state.customSampleLocations, "uniform custom sample locations were not decoded");
+    const auto locations = AgcDriver::Graphics::SampleLocations(state);
+    const std::array<VkSampleLocationEXT, 4> expected{{{0.125f, 0.875f}, {0.875f, 0.125f}, {0.375f, 0.375f}, {0.625f, 0.625f}}};
+    for (std::size_t sample = 0; sample < expected.size(); ++sample) Require(locations[sample].x == expected[sample].x && locations[sample].y == expected[sample].y, "custom sample coordinates or indices changed");
+    AgcDriver::Graphics::Context sampleContext{};
+    expectFailure([&] { AgcDriver::Graphics::ValidateSampleLocations(sampleContext, state); }, "VK_EXT_sample_locations");
+    sampleContext.sampleLocationSampleCounts = VK_SAMPLE_COUNT_2_BIT;
+    expectFailure([&] { AgcDriver::Graphics::ValidateSampleLocations(sampleContext, state); }, "sample count");
+    sampleContext.sampleLocationSampleCounts |= VK_SAMPLE_COUNT_4_BIT;
+    AgcDriver::Graphics::ValidateSampleLocations(sampleContext, state);
+    sampleContext.imageFormatProperties = sampleLocationDepthProperties;
+    using AgcDriver::Graphics::SampleLocationsCompatibleDepth;
+    Require(SampleLocationsCompatibleDepth(sampleContext, VK_FORMAT_D32_SFLOAT, 4), "supported custom sample depth format was refused");
+    state.depth = AgcDriver::Graphics::DepthTarget{};
+    state.depth->format = VK_FORMAT_D32_SFLOAT;
+    state.depth->samples = 4;
+    Require(AgcDriver::Graphics::ValidateSampleLocations(sampleContext, state), "supported custom depth sample locations were disabled");
+    depthSampleCounts = VK_SAMPLE_COUNT_2_BIT;
+    Require(!SampleLocationsCompatibleDepth(sampleContext, VK_FORMAT_D32_SFLOAT, 4), "unsupported custom sample depth count was accepted");
+    expectFailure([&] { AgcDriver::Graphics::ValidateSampleLocations(sampleContext, state); }, "compatible depth format and sample count");
+    depthSampleCounts = VK_SAMPLE_COUNT_4_BIT;
+    depthSampleResult = VK_ERROR_FORMAT_NOT_SUPPORTED;
+    Require(!SampleLocationsCompatibleDepth(sampleContext, VK_FORMAT_D32_SFLOAT, 4), "unsupported custom sample depth format was accepted");
+    expectFailure([&] { AgcDriver::Graphics::ValidateSampleLocations(sampleContext, state); }, "compatible depth format and sample count");
+    state.customSampleLocations = false;
+    Require(!AgcDriver::Graphics::ValidateSampleLocations(sampleContext, state), "standard locations did not fall back for an incompatible depth format");
+    depthSampleResult = VK_ERROR_OUT_OF_HOST_MEMORY;
+    expectFailure([&] { SampleLocationsCompatibleDepth(sampleContext, VK_FORMAT_D32_SFLOAT, 4); }, "Vulkan result");
+    depthSampleResult = VK_SUCCESS;
+    const auto queries = depthSampleQueries;
+    Require(!SampleLocationsCompatibleDepth({}, VK_FORMAT_D32_SFLOAT, 4) && !SampleLocationsCompatibleDepth(sampleContext, VK_FORMAT_D32_SFLOAT, 1), "sample location depth compatibility requires the extension and multisampling");
+    Require(depthSampleQueries == queries, "ineligible sample location depth formats were queried");
+    for (std::uint32_t pixel = 0; pixel < 4; ++pixel) queue.context[0x2fe + pixel * 4u] = 0x622ae6ae;
+    state = DecodeState(queue);
+    Require(!state.customSampleLocations, "standard sample locations require the extension");
+    const auto standard = AgcDriver::Graphics::SampleLocations(state);
+    const std::array<VkSampleLocationEXT, 4> expectedStandard{{{0.375f, 0.125f}, {0.875f, 0.375f}, {0.125f, 0.625f}, {0.625f, 0.875f}}};
+    for (std::size_t sample = 0; sample < expectedStandard.size(); ++sample) Require(standard[sample].x == expectedStandard[sample].x && standard[sample].y == expectedStandard[sample].y, "standard sample coordinate scale or Y direction changed");
+    AgcDriver::Graphics::ValidateSampleLocations({}, state);
+    for (const auto samples : {2u, 8u}) {
+        queue = multisampled();
+        const auto log2 = samples == 2 ? 1u : 3u;
+        queue.context[0x2f8] = (log2 << 20u) | log2;
+        queue.context[0x31d] = (log2 << 12u) | (log2 << 15u);
+        for (std::uint32_t pixel = 0; pixel < 4; ++pixel) {
+            queue.context[0x2fe + pixel * 4u] = 0x77887788;
+            queue.context[0x2ff + pixel * 4u] = 0x77887788;
+        }
+        state = DecodeState(queue);
+        Require(state.customSampleLocations && state.samples == samples, "a custom 2x or 8x pattern did not decode");
+        const auto endpoints = AgcDriver::Graphics::SampleLocations(state);
+        for (std::uint32_t sample = 0; sample < samples; ++sample) {
+            const auto expected = sample % 2u == 0 ? 0.0f : 15.0f / 16.0f;
+            Require(endpoints[sample].x == expected && endpoints[sample].y == expected, "signed sample coordinate endpoints changed");
+        }
+    }
+    queue.context[0x2f8] = 0x00300003;
+    queue.context[0x31d] = 0x1b000;
+    for (std::uint32_t pixel = 0; pixel < 4; ++pixel) {
+        queue.context[0x2fe + pixel * 4u] = 0xbd153fd1u;
+        queue.context[0x2ff + pixel * 4u] = 0x9773f95bu;
+    }
+    state = DecodeState(queue);
+    Require(state.samples == 8 && !state.customSampleLocations && state.colors.size() == 1 && state.colors[0].samples == 8, "the standard 8x pattern (sample 7 at (7, -7), PPSA99902's registers) did not decode as standard");
+    AgcDriver::Graphics::ValidateSampleLocations({}, state);
     queue = multisampled();
     queue.context[0x31d] = 0x1000;
     expectFailure([&] { DecodeState(queue); }, "fewer fragments than samples");
@@ -1765,28 +1982,28 @@ void multisampleTests() {
     expectFailure([&] { DecodeState(queue); }, "coverage conversion");
     queue = multisampled();
     queue.context[0x31c] |= 0x6000;
-    queue.context[0x31f] = 0x1234;
-    Require(DecodeState(queue).colors[0].cmaskAddress == 0x123400, "the CMASK address of a multisampled target did not decode");
+    setCmaskAddress(queue);
+    Require(DecodeState(queue).colors[0].cmaskAddress == cmaskAddress, "the CMASK address of a multisampled target did not decode");
     queue = makeState();
     queue.context[0x31c] |= 0x4000;
     expectFailure([&] { DecodeState(queue); }, "color compression");
     queue = multisampled();
     queue.context[0x31c] |= 0x2000;
-    queue.context[0x31f] = 0x1234;
+    setCmaskAddress(queue);
     const auto fastCleared = DecodeState(queue).colors[0];
-    Require(fastCleared.cmaskAddress == 0x123400 && fastCleared.cmaskFastClear && fastCleared.cmaskBytes == 0x1000, "a multisampled FAST_CLEAR target did not decode its CMASK at the metablock size");
+    Require(fastCleared.cmaskAddress == cmaskAddress && fastCleared.cmaskFastClear && fastCleared.cmaskBytes == 0x1000, "a multisampled FAST_CLEAR target did not decode its CMASK at the metablock size");
     queue = multisampled();
     queue.context[0x31c] |= 0x4000;
-    queue.context[0x31f] = 0x1234;
+    setCmaskAddress(queue);
     const auto compressed = DecodeState(queue).colors[0];
-    Require(compressed.cmaskAddress == 0x123400 && !compressed.cmaskFastClear && compressed.cmaskBytes == 0x1000, "a multisampled COMPRESSION target did not decode its CMASK at the metablock size");
+    Require(compressed.cmaskAddress == cmaskAddress && !compressed.cmaskFastClear && compressed.cmaskBytes == 0x1000, "a multisampled COMPRESSION target did not decode its CMASK at the metablock size");
     queue = multisampled();
     queue.context[0x31c] |= 0x2000;
     queue.context[0x31f] = 0;
     expectFailure([&] { DecodeState(queue); }, "without a CMASK address");
     queue = multisampled();
     queue.context[0x31c] |= 0x2000;
-    queue.context[0x31f] = 0x1234;
+    setCmaskAddress(queue);
     std::vector<AgcDriver::Graphics::RegisterRead> reads;
     AgcDriver::Graphics::RegisterReadLog() = &reads;
     static_cast<void>(DecodeState(queue));
@@ -1800,6 +2017,8 @@ void multisampleTests() {
     queue.context[0x32c] = 0;
     queue.context[0x3b1] = queue.context[0x3b0];
     queue.context[0x3b9] = queue.context[0x3b8];
+    setCmaskAddress(queue);
+    setCmaskAddress(queue, 1);
     auto pass = DecodeColorMetadataPass(queue);
     Require(pass.has_value() && pass->mode == ColorMetadataPass::Mode::Resolve && pass->source.has_value() && pass->source->samples == 2 && pass->targets.size() == 1 && pass->targets[0].samples == 1, "a CB resolve of a multisampled target did not decode");
     queue.context[0x32b] = queue.context[0x31c] ^ (1u << 8u);
@@ -1812,8 +2031,7 @@ void multisampleTests() {
     queue.context[0x3b1] = queue.context[0x3b0];
     queue.context[0x3b9] = queue.context[0x3b8];
     queue.context[0x32b] = queue.context[0x31c] | 0x2000u;
-    queue.context[0x32e] = static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(cmaskMemory.data()) >> 8u);
-    queue.context[0x399] = static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(cmaskMemory.data()) >> 40u);
+    setCmaskAddress(queue, 1);
     expectFailure([&] { DecodeColorMetadataPass(queue); }, "fast-clear color target");
     queue = multisampled();
     queue.context[0x2f8] = 0;
@@ -3574,6 +3792,131 @@ void orderedPixelShaderTests() {
     expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(orderedPixelRequest(0x30600u, plain))); }, "fragmentShaderPixelInterlock");
 }
 
+constexpr std::array<std::uint32_t, 4> StencilRefExportCode{0x7e0e02f2u, 0xf8001882u, 0x700u, 0xbf810000u};
+constexpr std::array<std::uint32_t, 4> DepthStencilRefExportCode{0x7e0e02f2u, 0xf8001883u, 0x707u, 0xbf810000u};
+
+AgcDriver::QueueState stencilRefExportQueue(std::uint32_t shaderControl, std::uint32_t zFormat) {
+    auto queue = makeState();
+    queue.context[0x1b3] = 0x300u;
+    queue.context[0x1b4] = 0x300u;
+    queue.context[0x1c4] = zFormat;
+    queue.context[0x203] = shaderControl;
+    return queue;
+}
+
+ShaderRecompiler::RecompileRequest stencilRefExportRequest(std::uint32_t shaderControl, std::span<const std::uint32_t> capabilities, std::span<const std::uint32_t> code = StencilRefExportCode) {
+    const auto queue = stencilRefExportQueue(shaderControl, 2u);
+    ShaderRecompiler::RecompileRequest request{};
+    request.shader = {ShaderRecompiler::ShaderStage::Fragment, 0x30000u, code, 0, {}};
+    request.context.waveSize = 64;
+    request.context.pixel = AgcDriver::Graphics::DecodePixelStageInfo(queue.context, IdentityExports);
+    request.target.vulkanVersion = 0x00401000u;
+    request.target.spirvVersion = 0x00010300u;
+    request.target.subgroupSize = 64;
+    request.target.supportedCapabilities = capabilities;
+    request.layout.pushConstantSizeBytes = 128;
+    request.useCache = false;
+    return request;
+}
+
+std::string stencilRefShape(std::span<const std::uint32_t> words) {
+    std::string shape;
+    std::uint32_t variable = 0;
+    for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
+        const auto op = static_cast<spv::Op>(words[at] & 0xffffu);
+        if (op == spv::OpCapability && words[at + 1] == spv::CapabilityStencilExportEXT) shape += "capability ";
+        if (op == spv::OpExtension && std::string_view(reinterpret_cast<const char*>(&words[at + 1])) == "SPV_EXT_shader_stencil_export") shape += "extension ";
+        if (op == spv::OpExecutionMode && words[at + 2] == spv::ExecutionModeEarlyFragmentTests) shape += "early ";
+        if (op == spv::OpExecutionMode && words[at + 2] == spv::ExecutionModeDepthReplacing) shape += "depth ";
+        if (op == spv::OpExecutionMode && words[at + 2] == spv::ExecutionModeStencilRefReplacingEXT) shape += "replacing ";
+        if (op == spv::OpDecorate && words[at + 2] == spv::DecorationBuiltIn && words[at + 3] == spv::BuiltInFragStencilRefEXT) {
+            variable = words[at + 1];
+            shape += "builtin ";
+        }
+        if (op == spv::OpStore && variable != 0 && words[at + 1] == variable) shape += "store ";
+    }
+    return shape;
+}
+
+void stencilRefExportTests() {
+    for (const auto shaderControl : {0x52u, 0x12u, 0x02u, 0x03u}) {
+        for (const auto zFormat : {2u, 9u}) {
+            const auto queue = stencilRefExportQueue(shaderControl, zFormat);
+            static_cast<void>(AgcDriver::Graphics::DecodeState(queue));
+            Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "the precheck rejected DB_SHADER_CONTROL " + std::to_string(shaderControl) + " with Z format " + std::to_string(zFormat));
+            const auto pixel = AgcDriver::Graphics::DecodePixelStageInfo(queue.context, IdentityExports);
+            Require(pixel.stencilRefExportEnable, "STENCIL_TEST_VAL_EXPORT_ENABLE did not reach the pixel stage");
+            Require(!pixel.earlyZ, "a pixel shader that exports the stencil reference was given early tests");
+        }
+        for (const auto zFormat : {0u, 1u, 3u, 7u}) {
+            const auto queue = stencilRefExportQueue(shaderControl, zFormat);
+            expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(queue)); }, "depth export, shader coverage or ordered fragment execution");
+            Require(!AgcDriver::Graphics::DrawRejection(queue, false).empty(), "the precheck accepted a stencil reference export with Z format " + std::to_string(zFormat));
+        }
+    }
+    for (const auto shaderControl : {0x54u, 0x56u}) {
+        const auto queue = stencilRefExportQueue(shaderControl, 2u);
+        expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(queue)); }, "depth export, shader coverage or ordered fragment execution");
+        Require(!AgcDriver::Graphics::DrawRejection(queue, false).empty(), "the precheck accepted STENCIL_OP_VAL_EXPORT_ENABLE in DB_SHADER_CONTROL " + std::to_string(shaderControl));
+    }
+    Require(!AgcDriver::Graphics::DecodePixelStageInfo(stencilRefExportQueue(0x50u, 2u).context, IdentityExports).stencilRefExportEnable, "a pixel stage without STENCIL_TEST_VAL_EXPORT_ENABLE exports the stencil reference");
+    const std::array<std::uint32_t, 2> stencilExport{spv::CapabilityShader, spv::CapabilityStencilExportEXT};
+    const std::array<std::uint32_t, 1> plain{spv::CapabilityShader};
+    const ShaderRecompiler::RequestSerializer serializer;
+    std::set<std::vector<std::uint64_t>> keys;
+    const std::array<std::pair<std::uint32_t, std::string_view>, 4> cases{{{0x52u, "capability extension replacing builtin store "}, {0x12u, "capability extension replacing builtin store "}, {0x50u, ""}, {0x10u, "early "}}};
+    for (const auto& [shaderControl, expected] : cases) {
+        const auto request = stencilRefExportRequest(shaderControl, stencilExport);
+        const auto replayed = serializer.Deserialize(serializer.Serialize(request));
+        Require(replayed.request.context.pixel->stencilRefExportEnable == ((shaderControl & 2u) != 0), "the serialized request lost STENCIL_TEST_VAL_EXPORT_ENABLE");
+        const auto result = ShaderRecompiler::Recompile(request);
+        Require(stencilRefShape(result.spirv.Words()) == expected && stencilRefShape(ShaderRecompiler::Recompile(replayed.request).spirv.Words()) == expected, "DB_SHADER_CONTROL " + std::to_string(shaderControl) + " declared the wrong stencil reference export: " + stencilRefShape(result.spirv.Words()));
+        std::vector<std::uint64_t> key;
+        ShaderRecompiler::RecompileCacheKey::Build(request, key);
+        keys.insert(key);
+        AgcDriver::Graphics::State state{};
+        state.stages.path = AgcDriver::Graphics::ShaderPath::Vertex;
+        ShaderRecompiler::RecompileResult vertex;
+        vertex.spirv = makeModule({});
+        const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &result, 0}}};
+        AgcDriver::Graphics::ValidateShaders(shaders, state, VkPhysicalDeviceSubgroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}, false);
+    }
+    Require(keys.size() == cases.size(), "the recompile cache key ignores STENCIL_TEST_VAL_EXPORT_ENABLE");
+    const auto both = stencilRefShape(ShaderRecompiler::Recompile(stencilRefExportRequest(0x53u, stencilExport, DepthStencilRefExportCode)).spirv.Words());
+    Require(both == "capability extension depth replacing builtin store ", "a depth and stencil reference export declared the wrong outputs: " + both);
+    const auto depthOnly = stencilRefShape(ShaderRecompiler::Recompile(stencilRefExportRequest(0x51u, stencilExport, DepthStencilRefExportCode)).spirv.Words());
+    Require(depthOnly == "depth ", "an MRTZ stencil channel without STENCIL_TEST_VAL_EXPORT_ENABLE declared the wrong outputs: " + depthOnly);
+    expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(stencilRefExportRequest(0x52u, plain))); }, "VK_EXT_shader_stencil_export");
+    auto queue = stencilRefExportQueue(0x52u, 2u);
+    queue.context[0x000] = 0;
+    queue.context[0x002] = 0;
+    queue.context[0x007] = (1u << 16u) | 3u;
+    queue.context[0x00a] = 7;
+    queue.context[0x00b] = 0;
+    queue.context[0x010] = 0x22900983;
+    queue.context[0x011] = 0x20000181;
+    for (const auto offset : {0x012u, 0x014u}) queue.context[offset] = 0x100;
+    for (const auto offset : {0x013u, 0x015u}) queue.context[offset] = 0x200;
+    queue.context[0x10b] = 0x00030030;
+    queue.context[0x10c] = 0x05ffff02;
+    queue.context[0x10d] = 0x05ffff02;
+    queue.context[0x200] = 0x00200211;
+    auto state = AgcDriver::Graphics::DecodeState(queue);
+    Require(state.stencilTest && state.stencilFront.compareOp == VK_COMPARE_OP_EQUAL && state.stencilFront.passOp == VK_STENCIL_OP_REPLACE, "a stencil reference export changed the REPLACE_TEST decode");
+    for (const auto ops : {0x00020020u, 0x00040040u, 0x00000200u, 0x00000004u}) {
+        queue.context[0x10b] = ops;
+        expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(queue)); }, "exports the stencil test value");
+    }
+    queue.context[0x10b] = 0x00050050;
+    queue.context[0x10c] = 0x01ffff02;
+    state = AgcDriver::Graphics::DecodeState(queue);
+    Require(state.stencilFront.passOp == VK_STENCIL_OP_INCREMENT_AND_CLAMP, "a stencil reference export changed the increment decode");
+    queue.context[0x000] = 2;
+    expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(queue)); }, "stencil clear with a pixel shader that exports the stencil test value");
+    queue.context[0x203] = 0x50u;
+    static_cast<void>(AgcDriver::Graphics::DecodeState(queue));
+}
+
 void ProvokingVertexTests() {
     using namespace AgcDriver::Graphics;
     constexpr std::uint32_t lastVertex = 1u << 19u;
@@ -3622,6 +3965,14 @@ void ProvokingVertexTests() {
     Require(flatPixel.fragmentParameters.size() == 1 && flatPixel.fragmentParameters[0].flat && !flatPixel.fragmentParameters[0].perVertex, "flat v_interp_mov P0 has explicit per-vertex metadata");
     const std::array<CompiledShader, 1> flatShaders{{{ShaderRecompiler::ShaderStage::Fragment, &flatPixel, 0}}};
     ValidateProvokingVertex(supported, last, flatShaders);
+    const std::array<std::uint32_t, 7> flatP10Color{0xc8120000u, 0xc8160100u, 0xc81a0200u, 0xc81e0300u, 0xf800180fu, 0x07060504u, 0xbf810000u};
+    const auto flatP10Pixel = recompilePixel({0x400u}, flatP10Color);
+    const auto p10Inputs = locatedInputs(flatP10Pixel.spirv.Words());
+    Require(p10Inputs.size() == 1 && p10Inputs[0].location == 0 && p10Inputs[0].flat && !p10Inputs[0].perVertex, "flat v_interp_mov P10 did not compile to a flat input");
+    Require(flatP10Pixel.fragmentParameters.size() == 1 && flatP10Pixel.fragmentParameters[0].flatP10 && !flatP10Pixel.fragmentParameters[0].perVertex && !flatPixel.fragmentParameters[0].flatP10, "flat v_interp_mov P10 was not recorded apart from P0");
+    const std::array<CompiledShader, 1> flatP10Shaders{{{ShaderRecompiler::ShaderStage::Fragment, &flatP10Pixel, 0}}};
+    ValidateProvokingVertex(supported, last, flatP10Shaders);
+    expectFailure([&] { ValidateProvokingVertex(supported, first, flatP10Shaders); }, "v_interp_mov p10 of a flat input with the first provoking vertex is not measured");
     ShaderRecompiler::RecompileResult fragment{};
     const std::array<CompiledShader, 1> shaders{{{ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}}};
     for (const auto topology : {VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN}) {
@@ -4101,6 +4452,26 @@ void validationTests() {
         expectFailure([&] { AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false); }, "metadata disagrees");
         Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 2, 1) == 80, "incorrect strided vertex range");
         Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 3, 1) == 112, "an index past the records still spans its fetch");
+        {
+            auto wider = attribute;
+            wider.resource.fields[2] = 9;
+            auto instanced = attribute;
+            instanced.fetchIndex = 1;
+            instanced.resource.fields[2] = 1000;
+            auto constant = attribute;
+            constant.resource.fields[1] = 0;
+            const std::array attributes{attribute, wider, instanced, constant};
+            Require(AgcDriver::Graphics::ZeroFetchVertex(attributes) == 9u, "the first vertex reading zeros is not past the largest per-vertex record count");
+            auto clamped = attribute;
+            clamped.resource.fields[3] |= 2u << 28u;
+            const std::array selective{attribute, clamped};
+            Require(!AgcDriver::Graphics::ZeroFetchVertex(selective).has_value(), "an attribute with out-of-bounds select 2 was taken to read zeros past its records");
+            std::array<std::uint32_t, 4> indices{0xffffffffu, 2u, 0x12345u, 0xffffffffu};
+            Require(AgcDriver::Graphics::ClampDrawIndices(std::as_writable_bytes(std::span(indices)), 4, true, 9u) == 9u && indices == std::array<std::uint32_t, 4>{0xffffffffu, 2u, 9u, 0xffffffffu}, "clamping kept an index past the limit or moved a restart index");
+            Require(AgcDriver::Graphics::ClampDrawIndices(std::as_writable_bytes(std::span(indices)), 4, false, 9u) == 9u && indices == std::array<std::uint32_t, 4>{9u, 2u, 9u, 9u}, "clamping without restart kept a 0xffffffff index");
+            std::array<std::uint16_t, 2> narrow{0xffffu, 1u};
+            Require(AgcDriver::Graphics::ClampDrawIndices(std::as_writable_bytes(std::span(narrow)), 2, false, 4u) == 4u && narrow[0] == 4u && narrow[1] == 1u, "clamping 16-bit indices failed");
+        }
         attribute.fetchIndex = 1;
         Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 100, 2) == 48, "instance attributes used the vertex index");
         Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 100, 2, 1) == 80, "first instance was ignored");
@@ -4274,7 +4645,7 @@ void validationTests() {
     {
         auto vertex = pushed({.push = true, .bufferArray = 2});
         vertex.bindings.front().descriptorSet = 1;
-        expectFailure([&] { validate(vertex, 8); }, "descriptor set other than zero");
+        expectFailure([&] { validate(vertex, 8); }, "descriptor set other than its stage's");
     }
     {
         auto vertex = pushed({.push = true, .bufferArray = 2});
@@ -4591,11 +4962,13 @@ int main(int argc, char** argv) {
         DepthBoundsBiasTests();
         conservativeZExportTests();
         orderedPixelShaderTests();
+        stencilRefExportTests();
         ProvokingVertexTests();
         ProvokingVertexRenderPassTests();
         ConservativeRasterizationTests();
         DisabledColorTests();
         OpaqueDestinationAlphaTests();
+        Color1555TargetTests();
         CompactedExportTests();
         ReversedComponentOrderTests();
         ShadersWriteMemoryTests();
@@ -4605,6 +4978,7 @@ int main(int argc, char** argv) {
         uint16ExportTests();
         uint8x4TargetTests();
         multisampleTests();
+        MsaaEnableSingleSampleTests();
         ShaderStageTests();
         NggVertexWorkgroupTests();
         TuningFieldTests();

core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp

@@ -1,5 +1,6 @@
 #include "BdaTests.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
 #include "prx/libc/include/GuestHeap.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
@@ -187,5 +188,36 @@ void RunUnmappedGapTests() {
     Require(!GuestMemory::Accessible(reinterpret_cast<const void*>(std::uintptr_t{0x18}), 4), "a near-null address counts as accessible");
     munmap(block, page);
     munmap(block + 2 * page, page);
+    if (!GuestArena::GuestArenaAvailable_nid_postfix()) return;
+    auto* guest = static_cast<std::byte*>(GuestArena::GuestArenaAllocate_nid_postfix(2 * page, page));
+    Require(mmap(guest, 2 * page, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == guest, "cannot map the arena page test pages");
+    {
+        GuestAllocations::Mutation mutation;
+        mutation.Add(guest, 2 * page, true, true, true);
+    }
+    Require(GuestMemory::Accessible(guest, page, true), "a mapped arena page is not writable");
+    Require(mprotect(guest, page, PROT_NONE) == 0, "cannot revoke the arena test page behind the registry");
+    Require(GuestMemory::Accessible(guest, page, true), "a checked arena page was queried again instead of read from the page table");
+    Require(mprotect(guest, page, PROT_READ | PROT_WRITE) == 0, "cannot restore the arena test page behind the registry");
+    Require(mprotect(guest + page, page, PROT_READ) == 0, "cannot protect the arena test's second page");
+    Require(!GuestMemory::Accessible(guest + page, page, true), "a check stored arena pages outside its range");
+    {
+        GuestAllocations::Mutation mutation;
+        mutation.Protect(guest + page, page, true, true, true, [&] { Require(mprotect(guest + page, page, PROT_READ | PROT_WRITE) == 0, "cannot restore the arena test's second page"); });
+    }
+    Require(GuestMemory::Accessible(guest, 2 * page, true), "mapped arena pages are not writable");
+    {
+        GuestAllocations::Mutation mutation;
+        mutation.Protect(guest, page, true, false, true, [&] { Require(mprotect(guest, page, PROT_READ) == 0, "cannot protect the arena test page"); });
+    }
+    Require(!GuestMemory::Accessible(guest, page, true) && GuestMemory::Accessible(guest, page) && GuestMemory::Accessible(guest + page, page, true), "an arena page protected through the registry kept its old access");
+    {
+        GuestAllocations::Mutation mutation;
+        const auto release = [](const void* pointer, std::size_t bytes, const void*, bool) { Require(munmap(const_cast<void*>(pointer), bytes) == 0, "cannot unmap an arena test page"); };
+        mutation.Unmap(guest, page, release);
+        mutation.Unmap(guest + page, page, release);
+    }
+    Require(!GuestMemory::Accessible(guest, page) && !GuestMemory::Accessible(guest + page, page), "unmapped arena pages are still accessible");
+    GuestArena::GuestArenaRelease_nid_postfix(guest, 2 * page);
 #endif
 }

core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp

@@ -160,7 +160,26 @@ void RunGuestTextureResourceTests() {
 
     Fields twoD = base;
     twoD.depth = 1;
+    rejectFields(twoD, "custom pitch other than its default pitch");
+    twoD.tileModeRaw = 0x09;
     rejectFields(twoD, "nonzero depth or base array");
+    Fields pitched = base;
+    pitched.width = 72;
+    pitched.height = 72;
+    pitched.depth = 127;
+    result = DecodeTextureResource(pack(pitched));
+    Require(result.dimension == TextureDimension::k2D && result.width == 72 && result.depthOrLastArray == 0, "linear 2D descriptor with its default pitch in DEPTH decoded incorrectly");
+    pitched.width = 8;
+    pitched.height = 8;
+    pitched.depth = 63;
+    result = DecodeTextureResource(pack(pitched));
+    Require(result.width == 8 && result.depthOrLastArray == 0, "linear 2D descriptor with a one-unit default pitch decoded incorrectly");
+    pitched.depth = 127;
+    rejectFields(pitched, "custom pitch other than its default pitch");
+    pitched.depth = 63;
+    pitched.maxMip = 1;
+    pitched.lastLevel = 1;
+    rejectFields(pitched, "custom pitch other than its default pitch");
     twoD = base;
     twoD.baseArray = 1;
     rejectFields(twoD, "nonzero depth or base array");

core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp

@@ -8,6 +8,7 @@
 #include <functional>
 #include <initializer_list>
 #include <utility>
+#include <vector>
 
 using namespace ShaderRecompiler;
 
@@ -209,7 +210,7 @@ int main() {
 
     passed &= run("cross-lane reads must keep the select", [] {
         bool ok = true;
-        for (const IrOpcode opcode : {IrOpcode::ReadFirstLane, IrOpcode::DppMoveU32, IrOpcode::BpermuteU32, IrOpcode::Permlane16U32, IrOpcode::SwizzleU32}) {
+        for (const IrOpcode opcode : {IrOpcode::DppMoveU32, IrOpcode::BpermuteU32, IrOpcode::Permlane16U32, IrOpcode::SwizzleU32}) {
             Builder b;
             auto& exec = b.mask(16u);
             auto& old = b.lane();
@@ -227,6 +228,132 @@ int main() {
         return ok && b.eliminate() == 0u && !removed(written);
     });
 
+    passed &= run("a first-lane read guarded at both ends loses the source select", [] {
+        Builder b;
+        auto& exec = b.mask(16u);
+        auto& narrow = b.logicalAnd(exec, b.mask(8u));
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& read = b.emit(IrOpcode::ReadFirstLane, IrType::U32, {&written, &narrow});
+        b.keep(b.select(narrow, read, old));
+        return b.eliminate() == 1u && removed(written);
+    });
+
+    passed &= run("an unguarded first-lane result keeps the source select for empty exec", [] {
+        Builder b;
+        auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.lane(), &b.constant(0u)});
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& empty = b.logicalAnd(exec, b.mask(0u));
+        b.keep(b.emit(IrOpcode::ReadFirstLane, IrType::U32, {&written, &empty}));
+        return b.eliminate() == 0u && !removed(written);
+    });
+
+    passed &= run("a first-lane result read under the write mask keeps the select when read exec is narrower", [] {
+        Builder b;
+        auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.lane(), &b.constant(0u)});
+        auto& empty = b.logicalAnd(exec, b.mask(0u));
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& read = b.emit(IrOpcode::ReadFirstLane, IrType::U32, {&written, &empty});
+        b.keep(b.select(exec, read, old));
+        return b.eliminate() == 0u && !removed(written);
+    });
+
+    passed &= run("a first-lane read under a wider exec keeps the select", [] {
+        Builder b;
+        auto& wide = b.mask(32u);
+        auto& exec = b.logicalAnd(wide, b.mask(16u));
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& read = b.emit(IrOpcode::ReadFirstLane, IrType::U32, {&written, &wide});
+        b.keep(b.select(wide, read, old));
+        return b.eliminate() == 0u && !removed(written);
+    });
+
+    const auto selectedExec = [](bool negate, bool widerArm) {
+        Builder b;
+        auto& wide = b.mask(32u);
+        auto& exec = b.logicalAnd(wide, b.mask(16u));
+        auto& empty = b.logicalAnd(exec, b.mask(0u));
+        auto& selector = negate ? b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}) : exec;
+        auto& later = b.emit(IrOpcode::SelectU1, IrType::Bool, {&selector, negate ? (widerArm ? &wide : &empty) : &wide, negate ? &wide : (widerArm ? &wide : &empty)});
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        b.keep(b.select(later, b.add(written, 2u), old));
+        return std::pair{b.eliminate(), removed(written)};
+    };
+
+    passed &= run("selected boolean exec implies the write mask through its selector", [&] {
+        for (bool negate : {false, true}) {
+            const auto [count, gone] = selectedExec(negate, false);
+            if (count != 1u || !gone) return false;
+        }
+        return true;
+    });
+
+    passed &= run("selected boolean exec with a wider unguarded arm keeps the select", [&] {
+        for (bool negate : {false, true}) {
+            const auto [count, gone] = selectedExec(negate, true);
+            if (count != 0u || gone) return false;
+        }
+        return true;
+    });
+
+    const auto rebuiltExec = [](bool mismatchedHigh) {
+        Builder b;
+        const auto [low, high] = b.ballotWords(b.mask(16u));
+        const auto [wideLow, wideHigh] = b.ballotWords(b.mask(48u));
+        auto& exec = b.threadBit(*low, *high);
+        auto& later = b.threadBit(*low, mismatchedHigh ? *wideHigh : *high);
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        b.keep(b.select(later, b.add(written, 2u), old));
+        return std::pair{b.eliminate(), removed(written)};
+    };
+
+    passed &= run("an exec rebuilt from the same words implies the write mask", [&] {
+        const auto [count, gone] = rebuiltExec(false);
+        return count >= 1u && gone;
+    });
+
+    passed &= run("matching low mask words alone do not imply a wave64 write mask", [&] {
+        const auto [count, gone] = rebuiltExec(true);
+        return count == 0u && !gone;
+    });
+
+    const auto firstLaneInLoop = [](bool carriedResult) {
+        Builder b;
+        auto& exec = b.mask(16u);
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        IrBlock& entry = *b.block;
+        IrBlock& loop = b.newBlock();
+        entry.AddBranch(&loop);
+        loop.AddBranch(&loop);
+        b.block = &loop;
+        auto& loopExec = b.logicalAnd(exec, b.mask(8u));
+        auto& read = b.emit(IrOpcode::ReadFirstLane, IrType::U32, {&written, &loopExec});
+        if (carriedResult) {
+            auto& carried = b.phi(IrType::U32, entry, old);
+            carried.AddPhiOperand(&loop, &read);
+            b.keep(b.select(loopExec, carried, old));
+        } else {
+            b.keep(b.select(loopExec, read, old));
+        }
+        return std::pair{b.eliminate(), removed(written)};
+    };
+
+    passed &= run("a loop-local first-lane result guarded in its own iteration loses the source select", [&] {
+        const auto [count, gone] = firstLaneInLoop(false);
+        return count == 1u && gone;
+    });
+
+    passed &= run("a first-lane result carried to another loop iteration keeps the source select", [&] {
+        const auto [count, gone] = firstLaneInLoop(true);
+        return count == 0u && !gone;
+    });
+
     passed &= run("a branch condition derived from the write keeps the select", [] {
         Builder b;
         auto& exec = b.mask(16u);
@@ -413,25 +540,35 @@ int main() {
         return count == 0u && !gone;
     });
 
-    const auto chained = [](bool wideRoot) {
+    const auto chained = [](bool wideRoot, std::uint32_t kills) {
         Builder b;
         auto& wide = b.mask(48u);
         auto& exec = b.logicalAnd(wide, b.mask(32u));
         auto& old = b.lane();
         auto& written = b.select(exec, b.add(old, 1u), old);
         IrValue* later = wideRoot ? &wide : &exec;
-        for (std::uint32_t kill = 0; kill < 24u; ++kill) later = &b.logicalAnd(*later, b.mask(31u - kill));
+        for (std::uint32_t kill = 0; kill < kills; ++kill) later = &b.logicalAnd(*later, b.mask(31u - kill % 24u));
         b.keep(b.select(*later, b.add(written, 2u), old));
         return std::pair{b.eliminate(), removed(written)};
     };
 
     passed &= run("an exec narrowed by a long chain of kills still implies the write's exec", [&] {
-        const auto [count, gone] = chained(false);
+        const auto [count, gone] = chained(false, 24u);
         return count == 1u && gone;
     });
 
     passed &= run("a long chain of kills rooted at a wider exec keeps the select", [&] {
-        const auto [count, gone] = chained(true);
+        const auto [count, gone] = chained(true, 24u);
+        return count == 0u && !gone;
+    });
+
+    passed &= run("an exec narrowed by hundreds of kills still implies the write's exec", [&] {
+        const auto [count, gone] = chained(false, 300u);
+        return count == 1u && gone;
+    });
+
+    passed &= run("hundreds of kills rooted at a wider exec keep the select", [&] {
+        const auto [count, gone] = chained(true, 300u);
         return count == 0u && !gone;
     });
 
@@ -765,5 +902,42 @@ int main() {
         return b.eliminate() == 0u && !removed(written);
     });
 
+    const auto shaded = [](std::uint32_t units) {
+        Builder b;
+        b.program.Resources().stage = IrShaderStage::Pixel;
+        MemoryInfo memory;
+        memory.kind = ResourceKind::Image;
+        memory.imageSampleFlags = RdnaImageSampleFlagLevelZero;
+        b.program.Resources().memoryInfo.push_back(memory);
+        MemoryFlags flags{0u, 0u};
+        std::uint64_t bits = 0;
+        std::memcpy(&bits, &flags, sizeof(flags));
+        auto& exec = b.mask(48u);
+        auto& image = b.emit(IrOpcode::GetImageResource, IrType::ImageResource, {});
+        auto& sampler = b.emit(IrOpcode::GetSamplerResource, IrType::SamplerResource, {});
+        IrValue* color = &b.lane();
+        IrValue* address = nullptr;
+        std::uint32_t kept = 0;
+        std::vector<IrValue*> compared;
+        for (std::uint32_t unit = 0; unit < units; ++unit) {
+            auto& sum = b.add(*color, unit);
+            auto& low = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&sum, &b.constant(100u + unit)});
+            compared.push_back(&b.select(low, sum, b.add(*color, 1u)));
+            address = &b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {compared.back(), color});
+            auto& gathered = b.emit(IrOpcode::ImageGatherRaw, IrType::U32x4, {&image, &sampler, address}, bits);
+            color = &b.select(exec, b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&gathered, &b.constant(0u)}), *color);
+        }
+        b.emit(IrOpcode::ImageWrite, IrType::Void, {&image, address, &b.emit(IrOpcode::CompositeConstructU32x4, IrType::U32x4, {color, color, color, color}), &exec});
+        const auto stats = MaskedSelectEliminator{}.Eliminate(b.program);
+        for (const IrValue* select : compared) kept += removed(*select) ? 0u : 1u;
+        return std::pair{stats, kept};
+    };
+
+    passed &= run("selects that each reach the rest of the program are settled in steps that grow with the program, not with its square", [&] {
+        const auto [small, smallKept] = shaded(500u);
+        const auto [large, largeKept] = shaded(4000u);
+        return small.removedSelects == 500u && smallKept == 500u && large.removedSelects == 4000u && largeKept == 4000u && small.steps != 0u && large.steps <= 9u * small.steps;
+    });
+
     return passed ? 0 : 1;
 }

core/libs/prx/libSceAgcDriver/tests/Mesh.cpp

@@ -2,6 +2,7 @@
 #include "Recompiler.hpp"
 #include "execution/VulkanTestDevice.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
+#include <algorithm>
 #include <array>
 #include <cmath>
 #include <cstdio>
@@ -133,6 +134,26 @@ struct MeshDraw {
     VkCullModeFlags cull = VK_CULL_MODE_NONE;
 };
 
+bool ReadsNumWorkgroups(const std::vector<std::uint32_t>& words) {
+    constexpr std::uint32_t OpDecorate = 71;
+    constexpr std::uint32_t OpAccessChain = 65;
+    constexpr std::uint32_t OpLoad = 61;
+    constexpr std::uint32_t DecorationBuiltIn = 11;
+    constexpr std::uint32_t BuiltInNumWorkgroups = 24;
+    std::uint32_t variable = 0;
+    std::vector<std::uint32_t> pointers;
+    for (std::size_t at = 5; at < words.size();) {
+        const auto length = words[at] >> 16u;
+        const auto opcode = words[at] & 0xffffu;
+        if (length == 0 || at + length > words.size()) return false;
+        if (opcode == OpDecorate && length == 4 && words[at + 2] == DecorationBuiltIn && words[at + 3] == BuiltInNumWorkgroups) variable = words[at + 1];
+        if (opcode == OpAccessChain && length >= 4 && variable != 0 && words[at + 3] == variable) pointers.push_back(words[at + 2]);
+        if (opcode == OpLoad && length >= 4 && variable != 0 && (words[at + 3] == variable || std::find(pointers.begin(), pointers.end(), words[at + 3]) != pointers.end())) return true;
+        at += length;
+    }
+    return false;
+}
+
 void DrawMesh(AgcDriver::VulkanDevice& device, const MeshDraw& setup) {
     const auto target = device.Target();
     std::vector<std::uint32_t> userData(12, 0u);
@@ -149,6 +170,7 @@ void DrawMesh(AgcDriver::VulkanDevice& device, const MeshDraw& setup) {
         ShaderRecompiler::GraphicsCompileContext{0, {}, setup.mesh, std::nullopt, {setup.draw.indexAddress, setup.draw.indexCount, setup.draw.indexSize, setup.draw.instanceCount}}
     };
     const auto meshResult = ShaderRecompiler::Recompile(geometry);
+    Require(ReadsNumWorkgroups(meshResult.spirv), "the mesh program does not read NumWorkgroups, so RADV can keep a previous vertex pipeline's user SGPR count for its dispatch");
     const auto meshPush = static_cast<std::uint32_t>(meshResult.pushConstants.size());
     Require((meshPush == 0) == (setup.geometryPushBytes == 0), "the geometry program's push data does not follow its layout");
 
@@ -162,13 +184,13 @@ void DrawMesh(AgcDriver::VulkanDevice& device, const MeshDraw& setup) {
         {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(PixelCode.data()), PixelCode, 0, {}},
         {64, 0, {}, std::nullopt, pixel, std::nullopt, pixelMemory},
         target,
-        {0, 0, meshPush, ShaderRecompiler::MeshDrawPushOffsetBytes - meshPush},
+        target.fixedPushSlots ? PixelPushLayout(meshPush, target) : ShaderRecompiler::BindingLayout{0, 0, meshPush, ShaderRecompiler::MeshDrawPushOffsetBytes - meshPush},
         std::nullopt
     };
     const auto pixelResult = ShaderRecompiler::Recompile(fragment);
     const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
         {ShaderStage::Mesh, &meshResult, 0},
-        {ShaderStage::Fragment, &pixelResult, meshPush}
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(meshPush, target)}
     }};
 
     AgcDriver::Graphics::State state{};

core/libs/prx/libSceAgcDriver/tests/PositionInvariance.cpp

@@ -36,6 +36,8 @@ void requireInvariantPositions(const ShaderRecompiler::RecompileResult& result,
 
 constexpr std::array<std::uint32_t, 3> PositionExport{0xf80008cfu, 0u, 0xbf810000u};
 
+constexpr std::array<std::uint32_t, 1> VertexCapabilities{spv::CapabilityClipDistance};
+
 ShaderRecompiler::RecompileRequest baseRequest(ShaderRecompiler::ShaderStage stage) {
     using namespace ShaderRecompiler;
     RecompileRequest request{};
@@ -45,6 +47,7 @@ ShaderRecompiler::RecompileRequest baseRequest(ShaderRecompiler::ShaderStage sta
     request.target.vulkanVersion = 0x00401000u;
     request.target.spirvVersion = 0x00010300u;
     request.target.subgroupSize = 64;
+    request.target.supportedCapabilities = VertexCapabilities;
     request.layout.pushConstantSizeBytes = 128;
     request.useCache = false;
     return request;

core/libs/prx/libSceAgcDriver/tests/Recorder.cpp

@@ -2081,6 +2081,45 @@ void drawInputInPlaceTests(const Device& device, Recorder& recorder) {
     Require(InPlaceDrawInput(unrecorded, vertices, 256, 4) == nullptr, "a draw without a recorder bound its input in place");
     recorder.Sync();
     Require(!recorder.PendingReadOverlaps(vertices, 256), "an in-place draw input's read outlived its batch");
+    if (!AgcDriver::GuestMemory::Watched(address, bytes)) {
+        std::cout << "host imports are compared, not watched: kept highest indices not tested\n";
+        return;
+    }
+    const auto indices = address + 20480;
+    constexpr std::size_t kept = Recorder::KeptIndexRangeIndices * 2;
+    std::vector<std::uint16_t> first(Recorder::KeptIndexRangeIndices * 2, 1);
+    first[0] = 3;
+    first[1] = 9;
+    first[2] = 0xffff;
+    std::memcpy(reinterpret_cast<void*>(indices), first.data(), kept * 2);
+    auto range = InPlaceIndexRange(recorder, indices, kept, 2, false);
+    Require(!range.reused && range.highest == 0xffffu, "an in-place index range was not scanned at its first use");
+    range = InPlaceIndexRange(recorder, indices, kept, 2, false);
+    Require(range.reused && range.highest == 0xffffu, "an unchanged in-place index range was scanned again");
+    range = InPlaceIndexRange(recorder, indices, kept, 2, true);
+    Require(!range.reused && range.highest == 9u, "an in-place index range served a scan made under the other restart rule");
+    Require(!InPlaceIndexRange(recorder, indices, kept + 2, 2, true).reused, "an in-place index range served a scan of another size");
+    const std::uint16_t raised = 12;
+    std::memcpy(reinterpret_cast<void*>(indices + 2), &raised, sizeof(raised));
+    range = InPlaceIndexRange(recorder, indices, kept, 2, true);
+    Require(!range.reused && range.highest == 12u, "an in-place index range kept its highest index over a guest write");
+    range = InPlaceIndexRange(recorder, indices, kept, 2, true);
+    Require(range.reused && range.highest == 12u, "a rescanned in-place index range was not kept");
+    range = InPlaceIndexRange(recorder, indices, kept - 2, 2, true);
+    Require(!range.reused && range.highest == 12u, "an in-place index range below the kept size was not scanned");
+    range = InPlaceIndexRange(recorder, indices, kept - 2, 2, true);
+    Require(!range.reused && range.highest == 12u, "an in-place index range below the kept size was kept");
+    range = InPlaceIndexRange(recorder, indices, kept * 2 - 4, 4, false);
+    Require(!range.reused && range.highest == 0x000c0003u, "a 32-bit in-place index range below the kept size was not scanned");
+    Require(!InPlaceIndexRange(recorder, indices, kept * 2 - 4, 4, false).reused, "a 32-bit in-place index range below the kept size was kept");
+    Require(!InPlaceIndexRange(recorder, indices, kept * 2, 4, false).reused, "a 32-bit in-place index range was not scanned at its first use");
+    range = InPlaceIndexRange(recorder, indices, kept * 2, 4, false);
+    Require(range.reused && range.highest == 0x000c0003u, "an unchanged 32-bit in-place index range was scanned again");
+    const std::vector<std::uint16_t> restarts(Recorder::KeptIndexRangeIndices, 0xffff);
+    std::memcpy(reinterpret_cast<void*>(indices + 20480), restarts.data(), kept);
+    Require(!InPlaceIndexRange(recorder, indices + 20480, kept, 2, true).highest.has_value(), "an index range of restart indices has a highest index");
+    range = InPlaceIndexRange(recorder, indices + 20480, kept, 2, true);
+    Require(range.reused && !range.highest.has_value(), "an empty in-place index range was scanned again");
 }
 
 // Unit shadows (UnitShadow.hpp) over a host import of write-watched arena memory: a retile piece's
@@ -2528,6 +2567,71 @@ void hostImportUnmapTests(const Device& device, Recorder& recorder) {
     std::cout << "host import unmap lifetime: ok\n";
 }
 
+void parkedImportUnmapTests(const Device& device) {
+    const auto& context = device.GetContext();
+    if (context.hostImportAlignment == 0) {
+        std::cout << "host imports unavailable: parked import unmap not tested\n";
+        return;
+    }
+    const auto alignment = static_cast<std::size_t>(std::max<VkDeviceSize>(65536u, context.hostImportAlignment));
+    const auto bytes = alignment * 4u;
+#ifdef _WIN32
+    void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+#else
+    void* block = std::aligned_alloc(alignment, bytes);
+#endif
+    Require(block != nullptr, "cannot allocate the parked import unmap block");
+    const auto release = [&] {
+#ifdef _WIN32
+        VirtualFree(block, 0, MEM_RELEASE);
+#else
+        std::free(block);
+#endif
+    };
+    const auto address = reinterpret_cast<std::uint64_t>(block);
+    {
+        GuestAllocations::Mutation mutation;
+        mutation.Add(block, bytes, true, true, true);
+    }
+    std::unique_ptr<Device> replacement;
+    bool imported = false;
+    {
+        std::lock_guard gpu(GpuMutex());
+        imported = HostImportFor(context, address, bytes) != nullptr;
+    }
+    if (imported) {
+        replacement = std::make_unique<Device>();
+        std::lock_guard gpu(GpuMutex());
+        imported = HostImportFor(replacement->GetContext(), address, bytes) != nullptr;
+    }
+    if (!imported) {
+        GuestAllocations::Mutation mutation;
+        mutation.Remove(block);
+        if (replacement) ClearHostImports(replacement->GetContext().device);
+        release();
+        std::cout << "host import refused: parked import unmap not tested\n";
+        return;
+    }
+    bool unmapped = false;
+    try {
+        GuestAllocations::Mutation mutation;
+        mutation.Unmap(block, bytes, [&](const void*, std::size_t, const void*, bool) { unmapped = true; });
+    } catch (const std::exception& error) {
+        ClearHostImports(replacement->GetContext().device);
+        ClearHostImports(context.device);
+        GuestAllocations::Mutation mutation;
+        mutation.Remove(block);
+        release();
+        throw std::runtime_error(std::string("guest unmap after a device replacement: ") + error.what());
+    }
+    ClearHostImports(replacement->GetContext().device);
+    ClearImageMirrors(replacement->GetContext().device);
+    replacement.reset();
+    release();
+    Require(unmapped, "guest unmap of a range a replaced device imported did not run");
+    std::cout << "parked import unmap: ok\n";
+}
+
 void importWatchTests(const Device& device) {
     using namespace AgcDriver::GuestMemory;
     const auto& context = device.GetContext();
@@ -5292,6 +5396,37 @@ void pipelineLibraryTests(const Device& device) {
     state.samples = 1;
     state.colors.front().samples = 1;
     Require(lookup() != multisampled && counters().built == std::array<std::uint64_t, 4>{2, 2, 4, 4}, "the 1x pipeline after a 4x one built its libraries again");
+    if ((context.sampleLocationSampleCounts & VK_SAMPLE_COUNT_4_BIT) != 0) {
+        state.samples = 4;
+        state.colors.front().samples = 4;
+        state.customSampleLocations = true;
+        state.sampleLocations = {0x10, 0x21, 0x32, 0x43};
+        const auto pattern = lookup();
+        const auto beforePatternChange = counters().built;
+        state.sampleLocations[0] = 0x20;
+        const auto otherPattern = lookup();
+        Require(otherPattern != pattern && counters().built[2] == beforePatternChange[2] + 1u && counters().built[3] == beforePatternChange[3] + 1u && counters().built[0] == beforePatternChange[0] && counters().built[1] == beforePatternChange[1], "a custom sample pattern did not rebuild exactly the fragment shader and fragment output libraries");
+        state.sampleLocations[0] = 0x10;
+        Require(lookup() == pattern && counters().built[2] == beforePatternChange[2] + 1u && counters().built[3] == beforePatternChange[3] + 1u, "a cached custom sample pattern was not reused");
+
+        std::optional<VkFormat> compatibleDepth, incompatibleDepth;
+        state.customSampleLocations = false;
+        for (const auto format : {VK_FORMAT_D32_SFLOAT, VK_FORMAT_D16_UNORM, VK_FORMAT_D24_UNORM_S8_UINT, VK_FORMAT_D32_SFLOAT_S8_UINT}) {
+            state.depth = DepthTarget{};
+            state.depth->format = format;
+            state.depth->samples = state.samples;
+            (ValidateSampleLocations(context, state) ? compatibleDepth : incompatibleDepth) = format;
+        }
+        if (compatibleDepth && incompatibleDepth) {
+            state.depth->format = *compatibleDepth;
+            const auto compatible = lookup();
+            const auto beforeDepthChange = counters().built;
+            state.depth->format = *incompatibleDepth;
+            const auto incompatible = lookup();
+            Require(incompatible != compatible && counters().built[2] == beforeDepthChange[2] + 1u && counters().built[3] == beforeDepthChange[3] + 1u, "depth-format sample-location enablement did not rebuild both fragment libraries");
+        }
+        state.customSampleLocations = true;
+    }
     WaitForOptimizedPipelines(context.device);
     const auto lto = std::getenv("APS5_NO_PIPELINE_LTO") == nullptr && context.pipelineLinkTimeOptimization;
     Require(counters().optimized == (lto ? counters().linked : 0u), "a linked pipeline was not optimized in the background");
@@ -5361,6 +5496,11 @@ int main(int argc, char** argv) {
             hostImportUnmapTests(device, recorder);
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--parked-import-unmap-only") {
+            gpu.unlock();
+            parkedImportUnmapTests(device);
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--unit-shadow-only") {
             if (!AgcDriver::Graphics::UnitShadowEnabled() || device.GetContext().hostImportAlignment == 0) {
                 std::cout << "skipped, unit shadows or host imports are unavailable\n";
@@ -5374,6 +5514,11 @@ int main(int argc, char** argv) {
             std::cout << "CMASK copied clear, write-back and failure tests passed\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--draw-inputs-in-place-only") {
+            drawInputInPlaceTests(device, recorder);
+            std::cout << "In-place draw input tests passed\n";
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--multisample-only") {
             multisampleGuestAccessTests(device, recorder);
             return 0;

core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp

@@ -71,6 +71,7 @@ void requireSameArtifact(const CompiledShaderArtifact& left, const CompiledShade
     require(left.hostSubgroupSize == right.hostSubgroupSize, prefix + "host subgroup size differs");
     require(left.vertexInputs == right.vertexInputs, prefix + "vertex inputs differ");
     require(left.vertexInputPatches == right.vertexInputPatches, prefix + "vertex type patches differ");
+    require(left.registrationModule == right.registrationModule, prefix + "registration module flags differ");
     require(left.vertexOffsetSgpr == right.vertexOffsetSgpr && left.instanceOffsetSgpr == right.instanceOffsetSgpr, prefix + "offset SGPRs differ");
     require(left.vertexOffsetShared == right.vertexOffsetShared && left.instanceOffsetShared == right.instanceOffsetShared && left.vertexOffsetConflict == right.vertexOffsetConflict && left.instanceOffsetConflict == right.instanceOffsetConflict, prefix + "offset flags differ");
     require(left.parameterExports == right.parameterExports, prefix + "parameter exports differ");
@@ -153,6 +154,7 @@ RecompileResult sampleResult() {
     result.workgroupMemoryDwords = 32769u;
     result.vertexInputs = {{1, 4, 2}, {5, 2, 0}};
     result.vertexInputPatches = {{1, 20, {7, 8, 9}}, {5, 40, {10, 11, 12}}};
+    result.registrationModule = true;
     result.specialization = {{512, 4}, {516, 0x3f800000u}, {517, 0}};
     result.vertexAttributes = {{1, 4, {{0x1000u, 0x20000u, 0x30u, 0x4u}}, 2}, {5, 2, {{9u, 8u, 7u, 6u}}, 0}};
     result.vertexInputs[0].outputMask = 5u;

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -50,7 +50,7 @@
 namespace {
 
 ShaderRecompiler::SpirvTarget BufferTarget() {
-    static constexpr std::array<std::uint32_t, 5> capabilities{spv::CapabilityShader, spv::CapabilityImageGatherExtended, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess};
+    static constexpr std::array<std::uint32_t, 6> capabilities{spv::CapabilityShader, spv::CapabilityImageGatherExtended, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilityClipDistance};
     static constexpr std::array<std::string_view, 2> extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"};
     ShaderRecompiler::SpirvTarget target{};
     target.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version;
@@ -1202,7 +1202,7 @@ void verifyLanesOutsideHostSubgroup() {
 
 void verifyHalfWaveReduction() {
     using namespace ShaderRecompiler;
-    static constexpr std::array<std::uint32_t, 4> capabilities{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic};
+    static constexpr std::array<std::uint32_t, 5> capabilities{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic, spv::CapabilityClipDistance};
     const auto pixel = [](std::span<const std::uint32_t> code, std::uint32_t subgroupSize, bool arithmetic) {
         ShaderPixelStageInfo info{};
         info.inputAddr = PixelInputBit(PixelInput::PositionX);
@@ -1341,6 +1341,13 @@ void verifyHalfWaveReduction() {
     divergent.insert(divergent.begin() + 3, 0x88fe1a7eu);
     divergent.insert(divergent.begin() + 2, {0xbe860480u, 0x7d821688u, 0xbe9a246au, 0xbf880001u});
     require(reductions(pixel(divergent, 32u, true), spv::OpGroupNonUniformUMin) == 1u, "half-wave reduction: keys masked inside a divergent branch did not read lane 31 as a subgroup UMin");
+    auto windowed = umin;
+    windowed.insert(windowed.begin() + 23, 0x7e008514u);
+    require(reductions(pixel(windowed, 32u, true), spv::OpGroupNonUniformUMin) == 1u, "half-wave reduction: a UMin scan of registers in a v_movreld_b32 window did not read lane 31 as a subgroup UMin on 32 lanes");
+    require(halves(pixel(windowed, 64u, true), spv::OpGroupNonUniformUMin, 0xffffffffu), "half-wave reduction: a UMin scan of registers in a v_movreld_b32 window did not read lanes 31 and 63 as reductions of host invocations 0-31 and 32-63 on 64 lanes");
+    auto overwritten = umin;
+    overwritten.insert(overwritten.begin() + 17, {0x7ef80500u, 0x7e008514u});
+    expectFailure([&] { static_cast<void>(pixel(overwritten, 32u, true)); }, lane63, "half-wave reduction: a scan of window registers read lane 63 after a v_movreld_b32 at a runtime M0 that may overwrite them");
     const char* arithmetic = "v_readlane_b32 of lane 31 of a wave64 half-wave reduction scan needs subgroup arithmetic";
     expectFailure([&] { static_cast<void>(pixel(umin, 32u, false)); }, arithmetic, "half-wave reduction: a device without subgroup arithmetic read lane 31 on 32 lanes");
     expectFailure([&] { static_cast<void>(pixel(umin, 64u, false)); }, arithmetic, "half-wave reduction: a device without subgroup arithmetic read lane 31 on 64 lanes");
@@ -1364,7 +1371,7 @@ void verifyHalfWaveReduction() {
 
 void verifyNarrowVertexWave() {
     using namespace ShaderRecompiler;
-    static constexpr std::array<std::uint32_t, 4> capabilities{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic};
+    static constexpr std::array<std::uint32_t, 5> capabilities{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic, spv::CapabilityClipDistance};
     static constexpr std::array<std::uint32_t, 4> userData{0u, 0u, 0u, 0x2020u};
     const auto vertex = [](std::span<const std::uint32_t> code, std::uint32_t waveSize, std::uint32_t subgroupSize, std::uint32_t userDataBase) {
         RecompileRequest request{};
@@ -1411,7 +1418,7 @@ void verifyNarrowVertexWave() {
     const auto wide64 = prepared(culling, 64u);
     require(!wide64.empty() && reductions(wide64) == 0u, "narrow vertex wave: the NGG culling scan, keyed on lanes below 64 on 64 lanes, did not keep its DPP scan");
     require(prepares(patched({{6, 0x80060015u}}), 32u), "narrow vertex wave: the scan keyed on lanes below the vertex count in s3 did not prepare on 32 lanes");
-    static constexpr std::array<std::uint32_t, 3> noArithmetic{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle};
+    static constexpr std::array<std::uint32_t, 4> noArithmetic{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityClipDistance};
     const auto otherKeys = patched({{7, 0x360e0a82u}});
     auto withoutArithmetic = vertex(otherKeys, 64u, 32u, 8u);
     withoutArithmetic.target.supportedCapabilities = noArithmetic;
@@ -1542,7 +1549,7 @@ void verifyPixelRequestSerialization() {
                         value.posZ, value.posW, value.frontFace, value.ancillary, value.sampleShading,
                         value.noPerspective, value.linearCentroid, value.pixelKillEnable, value.depthExportEnable,
                         value.sampleMaskExportEnable, value.earlyZ, value.executeOnNoop, value.conservativeZExport, value.orderedPixelShader,
-                        value.targetOutputMode, value.targetExportMapping, value.alphaToCoverage);
+                        value.targetOutputMode, value.targetExportMapping, value.alphaToCoverage, value.stencilRefExportEnable);
     };
     const std::array<std::uint32_t, 1> code{0xbf810000u};
     const std::array<std::uint32_t, 3> userData{0x12345678u, 0xabcdef01u, 0x87654321u};
@@ -1586,6 +1593,7 @@ void verifyPixelRequestSerialization() {
     };
     for (std::uint32_t i = 0; i < pixel.interpolatorSettings.size(); ++i) pixel.interpolatorSettings[i] = 0x10101010u + i;
     pixel.alphaToCoverage = true;
+    pixel.stencilRefExportEnable = true;
     const std::array<std::array<std::uint8_t, 8>, 3> mappings{{
         {0x00u, 0xe4u, 0xc6u, 0x1bu, 0xffu, 0x80u, 0x55u, 0xaau},
         {0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u},
@@ -1614,6 +1622,10 @@ void verifyPixelRequestSerialization() {
         withoutCoverage.context.pixel->alphaToCoverage = false;
         RecompileCacheKey::Build(withoutCoverage, replayKey);
         require(key != replayKey && RecompileCacheKey::ContextHash(request) != RecompileCacheKey::ContextHash(withoutCoverage), "alpha-to-coverage did not change the static shader identity");
+        auto withoutStencilRef = request;
+        withoutStencilRef.context.pixel->stencilRefExportEnable = false;
+        RecompileCacheKey::Build(withoutStencilRef, replayKey);
+        require(key != replayKey && RecompileCacheKey::ContextHash(request) != RecompileCacheKey::ContextHash(withoutStencilRef), "the stencil reference export did not change the static shader identity");
     }
     request.context.pixel.reset();
     request.shader.stage = ShaderStage::Vertex;
@@ -1625,12 +1637,12 @@ void verifyPixelRequestSerialization() {
     minimal.context.waveSize = 64;
     minimal.context.pixel = ShaderPixelStageInfo{};
     const auto encoded = serializer.Serialize(minimal);
-    require(requestPrefix(encoded, 8u) == "NVNQQRAAAAA=", "new requests did not use serialization version 16");
+    require(requestPrefix(encoded, 8u) == "NVNQQREAAAA=", "new requests did not use serialization version 17");
     constexpr std::size_t mappingOffset = 8u + 37u + 18u + 163u;
-    for (std::size_t bytes = 0; bytes < 18u; ++bytes) {
-        expectFailure([&] { static_cast<void>(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source or alpha-to-coverage flag was accepted");
+    for (std::size_t bytes = 0; bytes < 19u; ++bytes) {
+        expectFailure([&] { static_cast<void>(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source, alpha-to-coverage or stencil reference export flag was accepted");
     }
-    for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQREAAAA="}) {
+    for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRIAAAA="}) {
         expectFailure([&] { static_cast<void>(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted");
     }
 }
@@ -1720,12 +1732,13 @@ void verifyLegacyPixelRequests() {
         require(pixel.targetExportPacking == std::array<ColorExportPacking, 8>{}, "a legacy request packed a color export");
         require(!pixel.dualSourceBlend, "a legacy request enabled dual-source blending");
         require(!pixel.alphaToCoverage, "a legacy request enabled alpha-to-coverage");
+        require(!pixel.stencilRefExportEnable, "a legacy request exported a stencil reference");
         require(pixel.inputAddr == 2u && pixel.hasPerspectiveCenterVgpr && pixel.targetOutputMode[0] == 9u, "legacy pixel layout was misread");
         require(pixel.conservativeZExport == (version >= 7u ? ConservativeZExport::GreaterThanZ : ConservativeZExport::AnyZ), "legacy conservative Z layout was misread");
         require(!pixel.orderedPixelShader, "a legacy request became a primitive-ordered pixel shader");
         require(request.shader.stage == ShaderStage::Fragment && request.shader.code.size() == 1u && request.shader.code[0] == 0xbf810000u && !request.context.vertex.has_value() && request.context.memory.empty(), "legacy guest context was misaligned");
         require(request.target.vulkanVersion == 0x00401000u && request.target.spirvVersion == 0x00010300u && request.target.subgroupSize == 64u && request.layout.firstBinding == 11u && request.layout.pushConstantSizeBytes == 128u, "legacy target or binding layout was misaligned");
-        require(request.useCache == (version == 1u) && request.target.nonConstantImageOffsets == (version >= 6u) && request.target.srgbDecodeFormats == 0u && !request.target.narrowSubgroupClock && !request.target.float64DenormPreserve, "legacy request trailer was misread");
+        require(request.useCache == (version == 1u) && request.target.nonConstantImageOffsets == (version >= 6u) && request.target.srgbDecodeFormats == 0u && !request.target.narrowSubgroupClock && !request.target.float64DenormPreserve && !request.target.registrationModules, "legacy request trailer was misread");
         const auto upgraded = serializer.Deserialize(serializer.Serialize(request));
         require(upgraded.request.context.pixel->targetExportMapping == pixel.targetExportMapping && RecompileCacheKey::ContextHash(upgraded.request) == RecompileCacheKey::ContextHash(request), "upgrading a legacy capture changed its pixel mapping");
     }
@@ -3218,6 +3231,16 @@ int main(int argc, char** argv) {
         auto float64Denormals = uncached;
         float64Denormals.target.float64DenormPreserve = true;
         require(RequestSerializer{}.Deserialize(RequestSerializer{}.Serialize(float64Denormals)).request.target.float64DenormPreserve, "f64 denormal preservation was lost in serialization");
+        auto registrationModules = uncached;
+        registrationModules.target.registrationModules = true;
+        registrationModules.target.registrationWordLimit = 1234u;
+        const auto roundTrip = RequestSerializer{}.Deserialize(RequestSerializer{}.Serialize(registrationModules)).request.target;
+        require(roundTrip.registrationModules && roundTrip.registrationWordLimit == 1234u, "registration modules were lost in serialization");
+        std::vector<std::uint64_t> plainKey;
+        std::vector<std::uint64_t> registrationKey;
+        RecompileCacheKey::Build(uncached, plainKey);
+        RecompileCacheKey::Build(registrationModules, registrationKey);
+        require(plainKey != registrationKey, "registration modules did not change the cache key");
         auto changedLayout = request;
         changedLayout.layout.pushConstantSizeBytes = 64;
         require(!Recompile(changedLayout).cacheHit, "binding layout change reused an incompatible variant");

core/libs/prx/libSceAgcDriver/tests/SpirvNarrowConstantStore.cpp

@@ -50,8 +50,8 @@ void CheckStorageOnlyModule(const RecompileResult& result, std::string_view name
 }
 
 SpirvTarget StorageOnlyTarget() {
-    static constexpr std::array<std::uint32_t, 3> capabilities{
-        spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess,
+    static constexpr std::array<std::uint32_t, 4> capabilities{
+        spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilityClipDistance,
     };
     static constexpr std::array<std::string_view, 4> extensions{
         "SPV_EXT_descriptor_indexing", "SPV_KHR_8bit_storage", "SPV_KHR_float_controls", "SPV_KHR_physical_storage_buffer",

core/libs/prx/libSceAgcDriver/tests/Submit.cpp

@@ -22,6 +22,7 @@
 #include <memory>
 #include <stdexcept>
 #include <string>
+#include <string_view>
 #include <thread>
 #include <vector>
 #ifdef _WIN32
@@ -111,6 +112,17 @@ void testValidation() {
     AgcDriverWaitIdle_nid_postfix();
 }
 
+void testReadOnlyResummarize() {
+    std::vector<std::uint32_t> commands;
+    for (const auto [offset, value] : {std::pair{0x000u, 0x10u}, {0x200u, 0x007007f3u}, {0x010u, 0xa0000183u}, {0x011u, 0x20000180u}, {0x8eu, 0xffffffffu}, {0x8fu, 0u}, {0x1c4u, 0u}, {0x203u, 0u}}) {
+        commands.insert(commands.end(), {0xc0016900u, offset, value});
+    }
+    commands.insert(commands.end(), {0xc0012d00u, 3u, 2u});
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    check(sceAgcDriverSubmitDcb(&packet) == 0, "read-only resummarize submission failed");
+    AgcDriverWaitIdle_nid_postfix();
+}
+
 void testClearState() {
     AgcDriver::QueueState graphics{{{0x20c, 1}}, {{0x10, 17}, {0x11, 23}}, {{0x242, 5}}};
     const auto shader = graphics.shader;
@@ -671,8 +683,14 @@ void testWorkerFailure() {
 
 }
 
-int main() {
+int main(int argc, char** argv) {
     try {
+        if (argc == 2 && std::string_view(argv[1]) == "--readonly-resummarize") {
+            testReadOnlyResummarize();
+            LibcRunShutdown_nid_postfix();
+            std::puts("Read-only resummarize submission passed");
+            return 0;
+        }
         const auto device = OpenVulkanTestDevice();
         if (!device) return VulkanTestSkipped;
         alignas(256) std::array<std::uint32_t, 64> rawCode{};
@@ -766,6 +784,7 @@ int main() {
         testHeaderWithoutProgramAddress();
         testCopyIntoHostMemory();
         testRegisteredFloatMode();
+        testReadOnlyResummarize();
         testWorkerFailure();
         check(expectFailure([] { LibcRunShutdown_nid_postfix(); }).find("required shader register") != std::string::npos, "shutdown lost worker failure");
         std::puts("AGC driver submit tests passed");

core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp

@@ -1,4 +1,5 @@
 #include "GraphicsTests.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp"
 #include <cstring>
 #include <string>
@@ -171,6 +172,18 @@ void RunTextureDetilerTests(const Context& context, const TextureDetilerTestAcce
     Require(decodePush(access.lastDispatch().pushConstants).pipeBankXor == 0xa00u, "the window's pipe/bank XOR did not reach the detiling shader");
     Require(equationSpecialization[0] == 4 && equationSpecialization[1] == 4096 && equationSpecialization[2] == 2, "SW_4KB_D_X detiling must select the equation family over 4 KiB blocks");
 
+    {
+        const auto mip = ComputeColorTargetMipLayout(ColorTileMode::RenderTarget, 4, 1024, 1024, 11, 4);
+        Require(mip.width == 64 && mip.height == 64 && mip.tail, "the first packed render-target mip has the wrong extent");
+        Require(mip.tiledOffset == 0 && mip.tiledSize == 65536 && mip.tailX == 64 && mip.tailY == 0, "the first packed render-target mip lost its shared-block origin");
+        Require(mip.pitchBytes == 256 && mip.linearSize == 16384, "packed render-target mips must use tightly packed transfer rows");
+        const auto next = ComputeColorTargetMipLayout(ColorTileMode::RenderTarget, 4, 1024, 1024, 11, 5);
+        Require(next.width == 32 && next.height == 32 && next.tail && next.tiledOffset == 0 && next.tiledSize == 65536 && next.tailX == 0 && next.tailY == 64, "the next packed render-target mip lost its shared-block origin");
+        const auto regular = ComputeColorTargetMipLayout(ColorTileMode::RenderTarget, 4, 1024, 1024, 11, 3);
+        Require(!regular.tail && regular.width == 128 && regular.height == 128 && regular.tiledOffset == 0 && regular.tiledSize == 65536, "a regular render-target mip must be relative to its own target buffer");
+        reject([] { ComputeColorTargetMipLayout(ColorTileMode::RenderTarget, 4, 1024, 1024, 11, 11); }, "outside its surface");
+    }
+
     reject([&] { detiler.Dispatch(VK_NULL_HANDLE, TextureTileMode::kStandard4KB, 4, source, 0, destination, 0, layout, 0); }, "active command buffer");
     reject([&] { detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, VK_NULL_HANDLE, 0, destination, 0, layout, 0); }, "source and destination buffers");
     reject([&] { detiler.Dispatch(commands, TextureTileMode::kStandard4KB, 4, source, 0, VK_NULL_HANDLE, 0, layout, 0); }, "source and destination buffers");

core/libs/prx/libSceAgcDriver/tests/VectorWindow.cpp

@@ -1,9 +1,13 @@
 #include "Recompiler.hpp"
+#include "ControlFlow/GraphBuilder.hpp"
+#include "RdnaDecoder/RdnaInstructionDecoder.hpp"
+#include "Translation/InstructionTranslator.hpp"
 #include <spirv/unified1/spirv.hpp>
 #include <array>
 #include <cstdint>
 #include <cstdio>
 #include <exception>
+#include <initializer_list>
 #include <span>
 #include <stdexcept>
 #include <string>
@@ -15,11 +19,12 @@ namespace {
 using namespace ShaderRecompiler;
 
 constexpr std::uint32_t MoveM0Two = 0xbefc0382u;
+constexpr std::uint32_t MoveM0GroupX = 0xbefc0304u;
 constexpr std::uint32_t MoveV4One = 0x7e0802f2u;
 constexpr std::uint32_t MoveV5Two = 0x7e0a02f4u;
 constexpr std::uint32_t MoveV6Four = 0x7e0c02f6u;
 constexpr std::uint32_t MovrelsV1V4 = 0x7e028704u;
-constexpr std::uint32_t MovreldV4V1 = 0x7e088501u;
+constexpr std::uint32_t MovreldV4V0 = 0x7e088500u;
 constexpr std::uint32_t MoveV1V5 = 0x7e020305u;
 constexpr std::uint32_t EndProgram = 0xbf810000u;
 
@@ -36,14 +41,14 @@ void Require(bool condition, const std::string& message) {
 
 std::vector<std::uint32_t> RecompileProgram(std::span<const std::uint32_t> code) {
     const std::array<std::uint32_t, 7> capabilities{spv::CapabilityShader, spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformVote, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses};
-    const std::array<std::uint32_t, 4> userData{};
+    const std::array<std::uint32_t, 4> userData{0x10000000u, 0u, 0x40u, 0x31016facu};
     const std::array<std::string_view, 2> extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"};
     RecompileRequest request{};
     request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}};
     request.context.waveSize = 32;
     request.context.userDataBaseRegister = 0;
     request.context.userData = userData;
-    request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 4u, {false, false, false}, false, 1u};
+    request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 4u, {true, false, false}, false, 1u};
     request.target.vulkanVersion = 0x00403000u;
     request.target.spirvVersion = 0x00010600u;
     request.target.subgroupSize = 32;
@@ -65,6 +70,7 @@ Shape Scan(std::span<const std::uint32_t> words) {
     std::vector<std::pair<std::uint32_t, std::uint32_t>> constants;
     std::vector<std::pair<std::uint32_t, std::uint32_t>> arrays;
     std::vector<std::pair<std::uint32_t, std::uint32_t>> functionPointers;
+    std::vector<std::uint32_t> windowVariables;
     for (std::size_t cursor = 5; cursor < words.size();) {
         const auto count = words[cursor] >> spv::WordCountShift;
         const auto op = static_cast<spv::Op>(words[cursor] & spv::OpCodeMask);
@@ -78,19 +84,162 @@ Shape Scan(std::span<const std::uint32_t> words) {
                 for (const auto& [array, length] : arrays) {
                     if (array != pointee) continue;
                     ++shape.functionArrays;
+                    windowVariables.push_back(words[cursor + 2]);
                     for (const auto& [constant, value] : constants) {
                         if (constant == length) shape.arrayLength = value;
                     }
                 }
             }
         }
-        if (op == spv::OpAccessChain) ++shape.accessChains;
+        if (op == spv::OpAccessChain) {
+            for (const auto variable : windowVariables) {
+                if (words[cursor + 3] == variable) ++shape.accessChains;
+            }
+        }
         if (op == spv::OpSelect) ++shape.selects;
         cursor += count;
     }
     return shape;
 }
 
+IrProgram TranslateStage(ShaderStageKind stage, std::uint32_t waveSize, std::uint32_t base = 0u, std::uint32_t fetchStart = 0u, std::uint32_t fetchCount = 0u) {
+    ShaderComputeInputInfo compute{};
+    compute.threadIdsNum = 3;
+    compute.threadsNum[0] = waveSize;
+    ShaderVertexInputInfo vertex{};
+    vertex.mesh.threadsNum[0] = waveSize;
+    vertex.mesh.inputPrimitive = 1u;
+    vertex.mesh.primitivesPerGroup = 1u;
+    vertex.mesh.verticesPerGroup = 1u;
+    vertex.mesh.esgsItemSize = 1u;
+    ShaderPixelInputInfo pixel{};
+    std::uint32_t next = 0u;
+    for (const auto input : {PixelInput::PerspectiveCenter, PixelInput::PerspectiveCentroid, PixelInput::LinearCenter, PixelInput::LinearCentroid,
+                            PixelInput::PositionX, PixelInput::PositionY, PixelInput::PositionZ, PixelInput::PositionW,
+                            PixelInput::FrontFace, PixelInput::Ancillary, PixelInput::LineStipple, PixelInput::PositionFixedPoint}) {
+        pixel.psInputVgpr[static_cast<std::uint32_t>(input)] = next;
+        next += PixelInputVgprCount(input);
+    }
+    std::vector<std::uint32_t> code{MoveM0Two};
+    if (fetchCount != 0u) {
+        code.push_back(0xbe802102u);
+        code.push_back(0x7e008700u | base);
+        code.push_back(0x7e1002f2u);
+        vertex.resourcesNum = 1;
+        vertex.resourcesDst[0] = {static_cast<int>(fetchStart), static_cast<int>(fetchCount), 0, 0u};
+    }
+    code.push_back(0x7e008700u | base);
+    code.push_back(EndProgram);
+    const auto decoded = RdnaInstructionDecoder{}.Decode(code);
+    if (fetchCount != 0u) {
+        const auto& call = decoded.instructions.at(1);
+        Require(call.op == RdnaOpcode::SSwappcB64 && call.family == RdnaInstructionFamily::SOP1 && call.opcodeId == 0x21u &&
+                call.destination.kind == RdnaOperandKind::ScalarRegister && call.destination.reg == 0u &&
+                call.source0.kind == RdnaOperandKind::ScalarRegister && call.source0.reg == 2u,
+                "external fetch fixture did not decode s_swappc_b64 s[0:1], s[2:3]");
+    }
+    const SwappcInfo swappc{fetchCount != 0u, 0u, 8u};
+    const auto graph = GraphBuilder{}.Build(decoded, &swappc);
+    Require(graph.hasFetchCall == (fetchCount != 0u), "stage fixture did not identify its external fetch call");
+    TranslateOptions options{};
+    options.stage = stage;
+    options.waveSize = waveSize;
+    options.userDataCount = 8u;
+    options.inputInfo = {&vertex, &pixel, &compute};
+    return InstructionTranslator{}.Translate(decoded, graph, options);
+}
+
+void RequireWindow(const IrProgram& program, std::uint32_t base, std::uint32_t limit, std::span<const std::uint32_t> initialized) {
+    Require(program.Metadata().vectorWindowBase == base && program.Metadata().vectorWindowSize == limit - base, "relative window does not include exactly the initialized stage registers");
+    std::array<bool, NumVectorRegs> stored{};
+    for (const auto* instruction : program.BlockOrder().front()->Instructions()) {
+        if (instruction->Opcode() == IrOpcode::StoreVectorWindow) {
+            Require(instruction->Argument(0)->HasImmediate(), "stage input uses a dynamic window index");
+            const auto index = base + instruction->Argument(0)->ImmediateU32();
+            Require(index < limit, "stage input lies past the window");
+            stored[index] = true;
+        } else if (instruction->Opcode() == IrOpcode::SetVectorRegister) {
+            const auto index = instruction->Argument(0)->Register().index;
+            Require(index < base, "stage input escaped the relative window into SSA");
+            stored[index] = true;
+        }
+    }
+    for (const auto index : initialized) Require(stored[index], "stage input v" + std::to_string(index) + " was not initialized");
+    bool bounded = false;
+    for (const auto& block : program.Blocks()) {
+        for (const auto* instruction : block->Instructions()) {
+            if (instruction->Opcode() == IrOpcode::ULessThan32 && instruction->Argument(1)->HasImmediate() && instruction->Argument(1)->ImmediateU32() == limit - base) bounded = true;
+        }
+    }
+    Require(bounded, "relative access did not use the final stage window bound");
+}
+
+template<typename TAction>
+bool RunStage(const char* name, TAction action) {
+    try {
+        action();
+        std::printf("vector window stage inputs: %s passed\n", name);
+        return true;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "vector window stage inputs: %s: %s\n", name, error.what());
+        return false;
+    }
+}
+
+bool StageInputs() {
+    bool passed = true;
+    const auto stage = [&](const char* name, ShaderStageKind kind, std::uint32_t limit, std::initializer_list<std::uint32_t> initialized) {
+        passed &= RunStage(name, [&] {
+            for (const auto wave : {32u, 64u}) RequireWindow(TranslateStage(kind, wave), 0u, limit, {initialized.begin(), initialized.size()});
+        });
+    };
+    stage("compute", ShaderStageKind::Compute, 3u, {0u, 1u, 2u});
+    stage("vertex", ShaderStageKind::Vertex, 9u, {5u, 8u});
+    stage("local", ShaderStageKind::Local, 6u, {2u, 3u, 5u});
+    stage("tessellation control", ShaderStageKind::TessellationControl, 2u, {0u, 1u});
+    stage("tessellation evaluation", ShaderStageKind::TessellationEvaluation, 9u, {0u, 5u, 6u, 7u, 8u});
+    stage("mesh", ShaderStageKind::Mesh, 9u, {0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u});
+    stage("pixel", ShaderStageKind::Pixel, 16u, {0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, 15u});
+    passed &= RunStage("SSA prefix", [] {
+        const std::array<std::uint32_t, 3> local{2u, 3u, 5u};
+        const std::array<std::uint32_t, 2> vertex{5u, 8u};
+        for (const auto wave : {32u, 64u}) {
+            RequireWindow(TranslateStage(ShaderStageKind::Local, wave, 4u), 4u, 6u, local);
+            RequireWindow(TranslateStage(ShaderStageKind::Vertex, wave, 16u), 16u, 17u, vertex);
+        }
+    });
+    passed &= RunStage("external fetch", [] {
+        const std::array<std::uint32_t, 2> vertex{5u, 8u};
+        for (const auto wave : {32u, 64u}) {
+            for (const auto start : {20u, 252u}) {
+                for (const auto count : {1u, 2u, 3u, 4u}) {
+                    const auto program = TranslateStage(ShaderStageKind::Vertex, wave, 0u, start, count);
+                    RequireWindow(program, 0u, start + count, vertex);
+                    std::uint32_t fetched = 0u;
+                    std::uint32_t boundedReads = 0u;
+                    for (const auto& block : program.Blocks()) {
+                        for (const auto* instruction : block->Instructions()) {
+                            if (instruction->Opcode() == IrOpcode::StoreVectorWindow) {
+                                const auto* value = instruction->Argument(1);
+                                if (value->Opcode() == IrOpcode::SelectU32) value = value->Argument(1);
+                                if (value->Opcode() == IrOpcode::GetAttribute) {
+                                    Require(instruction->Argument(0)->ImmediateU32() == start + fetched && value->Argument(1)->ImmediateU32() == fetched,
+                                            "external fetch did not initialize its entire window span");
+                                    ++fetched;
+                                }
+                            }
+                            if (instruction->Opcode() == IrOpcode::ULessThan32 && instruction->Argument(1)->HasImmediate() && instruction->Argument(1)->ImmediateU32() == start + count) ++boundedReads;
+                        }
+                    }
+                    Require(fetched == count && boundedReads == 2u, "relative accesses to the external fetch disagree about the window");
+                }
+            }
+        }
+    });
+    if (passed) std::puts("Vector window stage input and fetch tests passed");
+    return passed;
+}
+
 bool run(const char* name, bool (*test)()) {
     try {
         if (test()) return true;
@@ -103,21 +252,25 @@ bool run(const char* name, bool (*test)()) {
 
 }
 
-int main() {
+int main(int argc, char** argv) {
+    if (argc == 2 && std::string_view(argv[1]) == "--stage-inputs-only") return StageInputs() ? 0 : 1;
+    if (argc != 1) return 2;
     bool passed = true;
 
-    passed &= run("a relative read and write index a function-scope window instead of selecting over the registers", [] {
-        const std::array<std::uint32_t, 8> code{MoveM0Two, MoveV4One, MoveV5Two, MoveV6Four, MovrelsV1V4, MovreldV4V1, MoveV1V5, EndProgram};
+    passed &= run("observable relative reads and writes index a function-scope window", [] {
+        const std::array<std::uint32_t, 12> code{MoveM0GroupX, MoveV4One, MoveV5Two, MoveV6Four, MovrelsV1V4,
+            0xe0700000u, 0x80000100u, MovreldV4V0, MoveV1V5, 0xe0700004u, 0x80000100u, EndProgram};
         const auto shape = Scan(RecompileProgram(code));
-        if (shape.functionArrays != 1 || shape.arrayLength != 3 || shape.accessChains < 2 || shape.selects > 4) std::fprintf(stderr, "  window shape: arrays %zu length %u access chains %zu selects %zu\n", shape.functionArrays, shape.arrayLength, shape.accessChains, shape.selects);
-        return shape.functionArrays == 1 && shape.arrayLength == 3 && shape.accessChains >= 2 && shape.selects <= 4;
+        if (shape.functionArrays != 1 || shape.arrayLength != 3 || shape.accessChains < 2) std::fprintf(stderr, "  window shape: arrays %zu length %u access chains %zu selects %zu\n", shape.functionArrays, shape.arrayLength, shape.accessChains, shape.selects);
+        return shape.functionArrays == 1 && shape.arrayLength == 3 && shape.accessChains >= 2;
     });
 
     passed &= run("a program without relative addressing keeps its registers in SSA values", [] {
-        const std::array<std::uint32_t, 5> code{MoveV4One, MoveV5Two, MoveV6Four, MoveV1V5, EndProgram};
+        const std::array<std::uint32_t, 7> code{MoveV4One, MoveV5Two, MoveV6Four, MoveV1V5, 0xe0700000u, 0x80000100u, EndProgram};
         const auto shape = Scan(RecompileProgram(code));
         return shape.functionArrays == 0 && shape.accessChains == 0;
     });
 
+    if (passed) std::puts("Vector window shape and SSA control tests passed");
     return passed ? 0 : 1;
 }

core/libs/prx/libSceAgcDriver/tests/VertexBarrier.cpp

@@ -12,6 +12,8 @@ namespace {
 constexpr std::array<std::uint32_t, 4> BarrierExport{0xbf8a0000u, 0xf80008cfu, 0u, 0xbf810000u};
 constexpr std::array<std::uint32_t, 6> BarrierLdsExport{0xd8340000u, 0u, 0xbf8a0000u, 0xf80008cfu, 0u, 0xbf810000u};
 
+constexpr std::array<std::uint32_t, 1> VertexCapabilities{spv::CapabilityClipDistance};
+
 ShaderRecompiler::RecompileRequest vertexRequest(std::span<const std::uint32_t> code) {
     using namespace ShaderRecompiler;
     RecompileRequest request{};
@@ -21,6 +23,7 @@ ShaderRecompiler::RecompileRequest vertexRequest(std::span<const std::uint32_t>
     request.target.vulkanVersion = 0x00401000u;
     request.target.spirvVersion = 0x00010300u;
     request.target.subgroupSize = 32;
+    request.target.supportedCapabilities = VertexCapabilities;
     request.layout.pushConstantSizeBytes = 128;
     request.useCache = false;
     return request;

core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp

@@ -152,6 +152,32 @@ void CheckUnwatch() {
     Require(!UnchangedSince(base, Block, adjacent), "unwatch hid a CPU edit in unrelated memory");
 }
 
+void CheckNewestStamp() {
+    void* memory = AllocateWatched(4 * Block);
+    const auto base = reinterpret_cast<std::uint64_t>(memory);
+    const auto first = base + Block;
+    const auto second = base + 2 * Block;
+    std::memset(memory, 0x11, 4 * Block);
+    BumpCollectEpoch();
+    CollectWrites(base, 4 * Block);
+    Unwatch(first, 2 * Block);
+    const auto firstBefore = NewestStamp(first, Block);
+    const auto secondBefore = NewestStamp(second, Block);
+    MarkWritten(second + 16, 4);
+    Require(NewestStamp(second, Block) > secondBefore, "a driver store into unwatched memory did not advance its stamp");
+    Require(NewestStamp(first, Block) == firstBefore, "a driver store advanced the stamp of other unwatched memory");
+    const auto generation = TrackerGeneration();
+    static_cast<volatile std::uint8_t*>(memory)[0] = 0x22;
+    CollectWritesUncached(base, Block);
+    Require(TrackerGeneration() > generation && NewestStamp(first, Block) == firstBefore, "a CPU store into watched memory advanced the stamp of unwatched memory");
+    StoreOwnBytes(first, 1, [&] { *reinterpret_cast<std::uint8_t*>(first) = 0x42; });
+    Require(NewestStamp(first, Block) > firstBefore, "an own store into unwatched memory did not advance its stamp");
+    const auto partial = NewestStamp(base + 3 * Block - 8, Block + 8);
+    static_cast<volatile std::uint8_t*>(memory)[3 * Block] = 0x33;
+    CollectWritesUncached(base + 3 * Block, Block);
+    Require(NewestStamp(base + 3 * Block - 8, Block + 8) > partial, "a CPU store into the watched part of a partly watched range did not advance its stamp");
+}
+
 #ifdef _WIN32
 void CheckPrivateMappingReuse() {
     void* memory = AllocateWatched(3 * Block);
@@ -184,6 +210,67 @@ void CheckPrivateMappingReuse() {
     GuestArena::GuestArenaReset_nid_postfix(replacement, 3 * Block);
     GuestArena::GuestArenaRelease_nid_postfix(replacement, 3 * Block);
 }
+
+std::uint64_t ArenaHostHole() {
+    std::uintptr_t arena = 0;
+    std::size_t arenaBytes = 0;
+    GuestArena::GuestArenaRange_nid_postfix(&arena, &arenaBytes);
+    for (auto cursor = arena; cursor < arena + arenaBytes;) {
+        MEMORY_BASIC_INFORMATION info{};
+        Require(VirtualQuery(reinterpret_cast<void*>(cursor), &info, sizeof(info)) == sizeof(info), "cannot enumerate arena host holes");
+        if (GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(cursor, 1)) return cursor;
+        const auto next = reinterpret_cast<std::uintptr_t>(info.BaseAddress) + info.RegionSize;
+        Require(next > cursor, "arena enumeration did not advance");
+        cursor = next;
+    }
+    return 0;
+}
+
+void CheckHostHole() {
+    constexpr std::size_t bytes = 64;
+    const auto hole = ArenaHostHole();
+    if (hole == 0) {
+        std::cout << "host-hole query checks skipped: the arena contains no host hole\n";
+        return;
+    }
+    const auto before = TrackerGeneration();
+    Require(!Watched(hole, bytes), "an arena host hole reads as watched");
+    Require(CollectWrites(hole, bytes) == 0 && CollectWritesUncached(hole, bytes) == 0, "an arena host hole was collected");
+    auto* watched = static_cast<std::uint8_t*>(AllocateWatched(Block, true));
+    struct Cleanup {
+        void* memory;
+        ~Cleanup() { GuestArena::GuestArenaRelease_nid_postfix(memory, Block); }
+    } cleanup{watched};
+    const auto base = reinterpret_cast<std::uint64_t>(watched);
+    for (unsigned epoch = 0; epoch < 2; ++epoch) {
+        BumpCollectEpoch();
+        watched[7] ^= 1;
+        Require(CollectWrites(base + 7, 1) != 0, "a warm host-hole check could not collect its watched range");
+        for (unsigned repeat = 0; repeat < 3; ++repeat) {
+            Require(CollectWrites(base + 7, 1) != 0, "a warm host-hole check lost its watched range");
+            Require(CollectWrites(hole, bytes) == 0, "a repeated host-hole query hit a collect memo");
+            Require(CollectWrites(hole + 7, 1) == 0, "a page-rounded host-hole query hit a collect memo");
+        }
+    }
+    Require(StoredOver(hole, bytes, before), "an arena host hole reads as not stored over");
+    Require(!UnchangedSince(hole, bytes, before), "an arena host hole reads as unchanged");
+    const std::array queries{UnchangedQuery{hole, bytes, before}};
+    Require(!UnchangedSinceAll(queries), "an arena host hole reads as unchanged in a batch");
+    Require(!UnchangedSinceCollected(hole, bytes, before), "an arena host hole reads as unchanged since a collect");
+    Require(MarkWritten(hole, bytes) == 0, "a driver store into an arena host hole is stamped");
+    Require(StoreOwnBytes(hole, bytes, [] {}) == 0, "an own store into an arena host hole is stamped");
+    Require(!WrittenSince(hole, bytes, before), "an arena host hole reads its stamps as written");
+    std::array<std::uint64_t, 1> generations{before};
+    std::array<std::uint8_t, 1> changed{};
+    std::array<std::uint8_t, 1> cpu{};
+    Require(!ChangedBlocks(hole, bytes, generations, changed, cpu) && changed[0] == BlockWritten && cpu[0] == 1, "an arena host hole uses block stamps");
+    bool imported = false;
+    Require(ImportWatched(hole, bytes, [&] {
+        imported = true;
+        return true;
+    }) && imported, "an import into an arena host hole did not run");
+    Require(!Watched(hole, bytes), "an import made an arena host hole watched");
+}
 #endif
 
 void CheckVersionRanges() {
@@ -310,8 +397,10 @@ int main() {
         CheckSharedBlock();
         CheckOwnStore();
         CheckUnwatch();
+        CheckNewestStamp();
 #ifdef _WIN32
         CheckPrivateMappingReuse();
+        CheckHostHole();
 #endif
         CheckVersionRanges();
         CheckCollectBoundaries();

core/libs/prx/libSceAgcDriver/tests/execution/BufferDescriptorRuntime.cpp

@@ -0,0 +1,111 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "Optimization/ResourceProgram.hpp"
+#include "VulkanTestDevice.hpp"
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Lanes = 32;
+constexpr std::uint32_t InputBytes = 256;
+constexpr std::uint32_t Fill = 0xdeadbeefu;
+alignas(256) std::array<std::uint8_t, InputBytes> Input{};
+alignas(256) std::array<std::uint32_t, Lanes> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 7> Code{
+    0x34020082, 0xe0301000, 0x80000201, 0xbf8c3f70, 0xe0701000, 0x80010201, 0xbf810000,
+};
+
+struct Case {
+    const char* name;
+    std::uint32_t baseOffset;
+    std::uint32_t records;
+    bool present;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+ShaderRecompiler::RecompileRequest Request(std::span<const std::uint32_t> userData, const ShaderRecompiler::SpirvTarget& target, std::span<const ShaderRecompiler::MemoryRegion> memory) {
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Lanes, 1, 1}, 0u, {false, false, false}, false, 1};
+    return {
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(Code.data()), Code, 0, {}},
+        {Lanes, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        target,
+        {0, 0, 0, 128}
+    };
+}
+
+void Run(AgcDriver::VulkanDevice& device, bool registrationModules) {
+    const std::string mode = registrationModules ? "registration modules" : "specialized modules";
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(Code.data()), std::as_bytes(std::span(Code))}}};
+    std::array<std::uint32_t, 8> userData{};
+    auto target = device.ComputeTarget(Lanes);
+    target.registrationModules = registrationModules;
+    const auto request = Request(userData, target, memory);
+    const auto handle = ShaderRecompiler::PrepareShader(request);
+    const std::array<Case, 7> cases{{
+        {"aligned", 0, 128, true},
+        {"base +1", 1, 128, true},
+        {"base +2", 2, 128, true},
+        {"base +3", 3, 128, true},
+        {"records 37", 0, 37, true},
+        {"records 64 base +2", 2, 64, true},
+        {"absent", 0, 128, false},
+    }};
+    for (const auto& test : cases) {
+        Output.fill(Fill);
+        const auto input = test.present ? BufferDescriptor(Input.data() + test.baseOffset, test.records) : std::array<std::uint32_t, 4>{0u, 0u, 0u, 0x31016facu};
+        const auto output = BufferDescriptor(Output.data(), sizeof(Output));
+        std::copy(input.begin(), input.end(), userData.begin());
+        std::copy(output.begin(), output.end(), userData.begin() + 4);
+        ShaderRecompiler::SrtRuntime runtime{};
+        runtime.userData = userData;
+        const auto capture = ShaderRecompiler::CaptureResources(request, runtime, *handle);
+        const auto result = ShaderRecompiler::MaterializeShader(request, *capture, *handle);
+        Require(!result->registrationModule, "buffer descriptor runtime, " + mode + ", " + test.name + ": a compute shader was emitted in registration form");
+        device.Dispatch(*result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(Code.data()));
+        device.WaitIdle();
+        for (std::uint32_t lane = 0; lane < Lanes; ++lane) {
+            std::uint32_t expected = 0;
+            if (test.present && lane * 4u + 4u <= test.records) std::memcpy(&expected, Input.data() + test.baseOffset + lane * 4u, sizeof(expected));
+            Require(Output[lane] == expected, "buffer descriptor runtime, " + mode + ", " + test.name + ": lane " + std::to_string(lane) + " is " + Hex(Output[lane]) + ", expected " + Hex(expected));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        for (std::uint32_t index = 0; index < InputBytes; ++index) Input[index] = static_cast<std::uint8_t>(index * 41u + 3u + (index >> 3u));
+        Run(*device, false);
+        Run(*device, true);
+        std::puts("buffer descriptor runtime tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/Color1555Target.cpp

@@ -0,0 +1,299 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "Optimization/ResourceProgram.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::ColorTargetLayout;
+using AgcDriver::Graphics::ColorTileMode;
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 256;
+constexpr std::uint32_t Height = 128;
+constexpr std::uint32_t ScissorWidth = 128;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::uint32_t Color1555Info = 0x8044u;
+constexpr std::uint32_t TiledAttrib3 = 0x4dc6c000u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32Export = 9;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::array<std::uint32_t, 64>, 16> Programs{};
+std::size_t programCount = 0;
+
+using Color = std::array<float, 4>;
+
+std::uint16_t Texel(const Color& color) {
+    const auto unorm = [](float value, std::uint32_t maximum) { return static_cast<std::uint32_t>(std::lround(std::fmin(std::fmax(static_cast<double>(value), 0.0), 1.0) * maximum)); };
+    return static_cast<std::uint16_t>(unorm(color[0], 31) | (unorm(color[1], 31) << 5u) | (unorm(color[2], 31) << 10u) | (unorm(color[3], 1) << 15u));
+}
+
+std::span<const std::uint32_t> Program(const Color& color) {
+    Require(programCount < Programs.size(), "too many test programs");
+    auto& code = Programs[programCount++];
+    std::size_t at = 0;
+    for (std::uint32_t channel = 0; channel < 4; ++channel) {
+        code[at++] = 0x7e0002ffu | ((4u + channel) << 17u);
+        code[at++] = std::bit_cast<std::uint32_t>(color[channel]);
+    }
+    code[at++] = 0xf800180fu;
+    code[at++] = 0x07060504u;
+    code[at++] = 0xbf810000u;
+    return std::span<const std::uint32_t>(code.data(), at);
+}
+
+struct Block {
+    explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (watched) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (watched) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color target");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate the color target");
+        Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched");
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (watched) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (watched) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool watched;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block, std::uint32_t attrib3) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(block.Address() >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Color1555Info;
+    cx[0x31d] = 0;
+    cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u);
+    cx[0x3b8] = attrib3;
+    cx[0x390] = static_cast<std::uint32_t>(block.Address() >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    Require(color.format == VK_FORMAT_A1R5G5B5_UNORM_PACK16 && color.elementBytes == 2 && color.bytes == block.bytes, "the 1_5_5_5 unorm target did not decode as an A1R5G5B5 surface");
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::uint32_t exportFormat, std::span<const std::uint32_t> pixelCode) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto address = reinterpret_cast<std::uintptr_t>(Triangle.data());
+    const std::array<std::uint32_t, 4> vertexBuffer{static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (16u << 16u), static_cast<std::uint32_t>(Triangle.size()), 0x01016facu};
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {ScissorWidth, Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.colorWriteMask = 0xf;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+
+    const auto codeAddress = reinterpret_cast<std::uintptr_t>(pixelCode.data());
+    const AgcDriver::Registers context{{0x1b6, 0x8000}, {0x1b3, 0}, {0x1b4, 0}, {0x203, 0}, {0x1c5, exportFormat}};
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{codeAddress, std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, codeAddress, pixelCode, 0, {}},
+        {waveSize, 0, {}, std::nullopt, AgcDriver::Graphics::DecodePixelStageInfo(context, {}), std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    const auto handle = ShaderRecompiler::PrepareShader(fragment);
+    AgcDriver::DriverDetail::ShaderSnapshot snapshot{codeAddress, 0, 1, {pixelCode.begin(), pixelCode.end()}, {}};
+    snapshot.prepared->entries.push_back({0, handle});
+    fragment.shader.code = snapshot.code;
+    fragment.context.pixel = AgcDriver::Graphics::DecodePixelStageInfo(context, AgcDriver::Graphics::ExportMappings(state));
+    const ShaderRecompiler::SrtRuntime runtime{};
+    const auto materialize = [&](const ShaderRecompiler::RecompileRequest& request) {
+        const auto invocation = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request);
+        return invocation.Materialize(*invocation.Capture(runtime));
+    };
+    const auto pixelResult = materialize(fragment);
+    Require(pixelResult->variantId == ShaderRecompiler::GetPreparedArtifact(*handle).variantId, "the draw did not use the registered pixel shader");
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, pixelResult.get(), PixelPushOffset(vertexPush, target)}
+    }};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+std::vector<std::uint16_t> ReadBack(AgcDriver::VulkanDevice& device, const Block& block) {
+    AgcDriver::Graphics::StorageTexture::FlushPending(block.Address(), block.bytes, nullptr, "test read-back");
+    device.WaitIdle();
+    std::vector<std::uint16_t> stored(block.bytes / 2u);
+    AgcDriver::GuestMemory::Read(block.Address(), std::as_writable_bytes(std::span(stored)), 1);
+    return stored;
+}
+
+std::vector<std::uint16_t> Fill(Block& block) {
+    std::vector<std::uint16_t> halves(block.bytes / 2u);
+    for (std::size_t index = 0; index < halves.size(); ++index) halves[index] = static_cast<std::uint16_t>((index * 0x9e3779b9u) >> 16u);
+    std::memcpy(block.data, halves.data(), block.bytes);
+    return halves;
+}
+
+void ExpectSurface(AgcDriver::VulkanDevice& device, const Block& block, const ColorTargetLayout& layout, const std::vector<std::uint16_t>& before, std::uint16_t texel, const std::string& what) {
+    const auto stored = ReadBack(device, block);
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const auto index = layout.Offset(x, y) / 2u;
+            const bool inside = x < ScissorWidth;
+            const auto expected = inside ? texel : before[index];
+            if (stored[index] == expected) continue;
+            char detail[160];
+            std::snprintf(detail, sizeof(detail), ": pixel (%u, %u) %s the scissor holds 0x%04x, expected 0x%04x", x, y, inside ? "inside" : "outside", stored[index], expected);
+            throw std::runtime_error(what + detail);
+        }
+    }
+}
+
+void TargetTests(AgcDriver::VulkanDevice& device, Block& block, std::uint32_t attrib3, ColorTileMode mode, const char* kind) {
+    const ColorTargetLayout layout(Width, Height, mode, 2);
+    Require(layout.Bytes() == block.bytes, "the test surface size disagrees with its layout");
+    const auto color = DecodeTarget(block, attrib3);
+    auto before = Fill(block);
+    struct Case {
+        Color color;
+        std::uint16_t texel;
+    };
+    const std::array<Case, 11> cases{{
+        {{1.0f, 0.0f, 0.0f, 1.0f}, 0x801f},
+        {{0.0f, 1.0f, 0.0f, 0.0f}, 0x03e0},
+        {{0.0f, 0.0f, 1.0f, 0.0f}, 0x7c00},
+        {{0.0f, 0.0f, 0.0f, 1.0f}, 0x8000},
+        {{1.0f, 0.0f, 1.0f, 0.0f}, 0x7c1f},
+        {{1.0f, 1.0f, 1.0f, 1.0f}, 0xffff},
+        {{0.0f, 0.0f, 0.0f, 0.0f}, 0x0000},
+        {{3.0f / 31.0f, 10.0f / 31.0f, 21.0f / 31.0f, 1.0f}, (3u) | (10u << 5u) | (21u << 10u) | 0x8000u},
+        {{21.0f / 31.0f, 3.0f / 31.0f, 10.0f / 31.0f, 0.0f}, (21u) | (3u << 5u) | (10u << 10u)},
+        {{1.0f / 31.0f, 30.0f / 31.0f, 2.0f / 31.0f, 0.75f}, (1u) | (30u << 5u) | (2u << 10u) | 0x8000u},
+        {{2.0f, -1.0f, 10.0f / 31.0f, 0.25f}, (31u) | (0u << 5u) | (10u << 10u)},
+    }};
+    for (const auto& test : cases) {
+        Require(Texel(test.color) == test.texel, "the test's expected texel disagrees with the 1_5_5_5 field layout");
+        Draw(device, color, Float32Export, Program(test.color));
+        char what[160];
+        std::snprintf(what, sizeof(what), "a 32_ABGR export of (%.9g, %.9g, %.9g, %.9g) into a %s 1_5_5_5 unorm target", test.color[0], test.color[1], test.color[2], test.color[3], kind);
+        ExpectSurface(device, block, layout, before, test.texel, what);
+        before = ReadBack(device, block);
+        programCount = 0;
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Block tiled(ColorTargetLayout(Width, Height, ColorTileMode::RenderTarget, 2).Bytes());
+        Block linear(ColorTargetLayout(Width, Height, ColorTileMode::Linear, 2).Bytes());
+        TargetTests(*device, tiled, TiledAttrib3, ColorTileMode::RenderTarget, "tiled");
+        TargetTests(*device, linear, LinearAttrib3, ColorTileMode::Linear, "linear");
+        std::puts("1_5_5_5 unorm color target tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DepthFastClearHarness.hpp

@@ -4,6 +4,7 @@
 #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
 #include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
 #include <algorithm>
 #include <array>
 #include <cstddef>
@@ -96,7 +97,7 @@ inline Shaders Compile(AgcDriver::VulkanDevice& device) {
         {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(PixelCode.data()), PixelCode, 0, {}},
         {32, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
         target,
-        {0, 0, shaders.vertexPush, 128 - shaders.vertexPush}
+        PixelPushLayout(shaders.vertexPush, target)
     };
     fragment.useCache = false;
     shaders.pixel = ShaderRecompiler::Recompile(fragment);
@@ -107,7 +108,7 @@ inline std::uint32_t Draw(AgcDriver::VulkanDevice& device, const Shaders& compil
     Pixels.fill(std::byte{0});
     const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
         {ShaderStage::Vertex, &compiled.vertex, 0},
-        {ShaderStage::Fragment, &compiled.pixel, compiled.vertexPush}
+        {ShaderStage::Fragment, &compiled.pixel, PixelPushOffset(compiled.vertexPush, device.Target())}
     }};
     AgcDriver::Graphics::State state{};
     state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt};

core/libs/prx/libSceAgcDriver/tests/execution/DepthStorageReuse.cpp

@@ -0,0 +1,78 @@
+#include "DepthFastClearHarness.hpp"
+#include "VulkanTestDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdio>
+#include <cmath>
+#include <iostream>
+#include <string>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using DepthFastClearHarness::Height;
+using DepthFastClearHarness::Width;
+
+alignas(4096) std::array<float, 65536> Depth{};
+alignas(256) std::array<std::uint32_t, 64> Output{};
+alignas(256) constexpr std::array<std::uint32_t, 13> LoadCode{
+    0x7e3c0280u, 0x7e3e0280u, 0x7e140280u, 0xf0001108u, 0x00010a1eu,
+    0xbf8c3f70u, 0xf0201108u, 0x00010a1eu, 0x34060082u, 0xe0701000u, 0x80000a03u, 0xbf810000u, 0xbf810000u,
+};
+alignas(256) constexpr std::array<std::uint32_t, 10> SampleCode{
+    0x7e3c02ffu, 0x3f000000u, 0x7e3e02ffu, 0x3f000000u, 0xf09c0108u,
+    0x00610a1eu, 0x34060082u, 0xe0701000u, 0x80000a03u, 0xbf810000u,
+};
+
+void Read(AgcDriver::VulkanDevice& device, std::uint32_t format, std::uint32_t width, bool sampled, float expected) {
+    Output.fill(0xdeadbeefu);
+    const auto output = reinterpret_cast<std::uintptr_t>(Output.data());
+    const auto depth = reinterpret_cast<std::uintptr_t>(Depth.data());
+    const std::array<std::uint32_t, 16> data{
+        static_cast<std::uint32_t>(output), static_cast<std::uint32_t>(output >> 32u), sizeof(Output), 0x31016facu,
+        static_cast<std::uint32_t>(depth >> 8u), static_cast<std::uint32_t>(depth >> 40u) | (format << 20u) | (((width - 1u) & 3u) << 30u),
+        ((width - 1u) >> 2u) | ((Height - 1u) << 14u), 0x90000facu, 0u, 0u, 0u, 0u,
+        0x92u, 0u, 0u, 0u,
+    };
+    const auto code = sampled ? std::span<const std::uint32_t>(SampleCode) : std::span<const std::uint32_t>(LoadCode);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{32, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderRecompiler::ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, data, compute, std::nullopt, std::nullopt, memory}, device.Target(), {0, 0, 0, 128},
+    };
+    request.useCache = false;
+    const auto shader = ShaderRecompiler::Recompile(request);
+    device.Dispatch(shader, 1, 1, 1);
+    device.WaitIdle();
+    for (std::size_t lane = 0; lane < 32; ++lane) {
+        Require((sampled && format == 7u ? std::abs(std::bit_cast<float>(Output[lane]) - expected) <= 1.0f / 65535.0f : Output[lane] == std::bit_cast<std::uint32_t>(expected)), "depth reuse returned the wrong GPU value at lane " + std::to_string(lane) + ": got " + std::to_string(Output[lane]) + ", expected " + std::to_string(std::bit_cast<std::uint32_t>(expected)));
+    }
+    Require(std::all_of(Depth.begin(), Depth.end(), [](float value) { return value == 0.25f; }), "depth reuse changed the guest bytes");
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        const auto shaders = DepthFastClearHarness::Compile(*device);
+        const auto address = reinterpret_cast<std::uintptr_t>(Depth.data());
+        for (const auto format : {VK_FORMAT_D32_SFLOAT, VK_FORMAT_D16_UNORM}) {
+            AgcDriver::Graphics::ClearDepthSurfaces(device->Device());
+            Depth.fill(0.25f);
+            const DepthFastClearHarness::Surface surface{address, 0, 0, false, format};
+            Require(DepthFastClearHarness::Draw(*device, shaders, surface, {.depthWrite = true}) == DepthFastClearHarness::Covered, "depth producer did not cover the surface");
+            Read(*device, format == VK_FORMAT_D16_UNORM ? 7u : 22u, Width, true, 0.5f);
+            Read(*device, 22u, format == VK_FORMAT_D16_UNORM ? Width : Width / 2u, false, 0.25f);
+        }
+        std::puts("depth storage reuse GPU tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/F16DenormalFlush.cpp

@@ -0,0 +1,353 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 1;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+struct alignas(256) Shader {
+    std::array<std::uint32_t, 11> words;
+};
+
+struct Vector {
+    std::uint32_t half;
+    std::uint32_t expected[4];
+};
+
+struct Case {
+    const char* name;
+    std::uint32_t word;
+    std::span<const Vector> rows;
+};
+
+constexpr Vector CeilRows[] = {
+    {0x0001u, {0x5a5a0000u, 0x5a5a3c00u, 0x5a5a0000u, 0x5a5a3c00u}},
+    {0x0200u, {0x5a5a0000u, 0x5a5a3c00u, 0x5a5a0000u, 0x5a5a3c00u}},
+    {0x03ffu, {0x5a5a0000u, 0x5a5a3c00u, 0x5a5a0000u, 0x5a5a3c00u}},
+    {0x8001u, {0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u}},
+    {0x8200u, {0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u}},
+    {0x83ffu, {0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u}},
+    {0x0400u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0x8400u, {0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u}},
+    {0x0000u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x3c00u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0xbc00u, {0x5a5abc00u, 0x5a5abc00u, 0x5a5abc00u, 0x5a5abc00u}},
+    {0x3800u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+};
+
+constexpr Vector CvtF32Rows[] = {
+    {0x0001u, {0x00000000u, 0x33800000u, 0x00000000u, 0x33800000u}},
+    {0x0200u, {0x00000000u, 0x38000000u, 0x00000000u, 0x38000000u}},
+    {0x03ffu, {0x00000000u, 0x387fc000u, 0x00000000u, 0x387fc000u}},
+    {0x8001u, {0x80000000u, 0xb3800000u, 0x80000000u, 0xb3800000u}},
+    {0x8200u, {0x80000000u, 0xb8000000u, 0x80000000u, 0xb8000000u}},
+    {0x83ffu, {0x80000000u, 0xb87fc000u, 0x80000000u, 0xb87fc000u}},
+    {0x0400u, {0x38800000u, 0x38800000u, 0x38800000u, 0x38800000u}},
+    {0x8400u, {0xb8800000u, 0xb8800000u, 0xb8800000u, 0xb8800000u}},
+    {0x0000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}},
+    {0x3c00u, {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u}},
+    {0xbc00u, {0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u}},
+    {0x3800u, {0x3f000000u, 0x3f000000u, 0x3f000000u, 0x3f000000u}},
+};
+
+constexpr Vector CvtNormI16Rows[] = {
+    {0x0001u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x0200u, {0x5a5a0000u, 0x5a5a0001u, 0x5a5a0000u, 0x5a5a0001u}},
+    {0x03ffu, {0x5a5a0000u, 0x5a5a0002u, 0x5a5a0000u, 0x5a5a0002u}},
+    {0x8001u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x8200u, {0x5a5a0000u, 0x5a5affffu, 0x5a5a0000u, 0x5a5affffu}},
+    {0x83ffu, {0x5a5a0000u, 0x5a5afffeu, 0x5a5a0000u, 0x5a5afffeu}},
+    {0x0400u, {0x5a5a0002u, 0x5a5a0002u, 0x5a5a0002u, 0x5a5a0002u}},
+    {0x8400u, {0x5a5afffeu, 0x5a5afffeu, 0x5a5afffeu, 0x5a5afffeu}},
+    {0x0000u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x3c00u, {0x5a5a7fffu, 0x5a5a7fffu, 0x5a5a7fffu, 0x5a5a7fffu}},
+    {0xbc00u, {0x5a5a8001u, 0x5a5a8001u, 0x5a5a8001u, 0x5a5a8001u}},
+    {0x3800u, {0x5a5a4000u, 0x5a5a4000u, 0x5a5a4000u, 0x5a5a4000u}},
+    {0xb800u, {0x5a5ac000u, 0x5a5ac000u, 0x5a5ac000u, 0x5a5ac000u}},
+};
+
+constexpr Vector CvtNormU16Rows[] = {
+    {0x0001u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x0200u, {0x5a5a0000u, 0x5a5a0002u, 0x5a5a0000u, 0x5a5a0002u}},
+    {0x03ffu, {0x5a5a0000u, 0x5a5a0004u, 0x5a5a0000u, 0x5a5a0004u}},
+    {0x8001u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x8200u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x83ffu, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x0400u, {0x5a5a0004u, 0x5a5a0004u, 0x5a5a0004u, 0x5a5a0004u}},
+    {0x8400u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x0000u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x3c00u, {0x5a5affffu, 0x5a5affffu, 0x5a5affffu, 0x5a5affffu}},
+    {0xbc00u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x3800u, {0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u}},
+    {0xb800u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+};
+
+constexpr Vector ExpRows[] = {
+    {0x0001u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0x0200u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0x03ffu, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0x8001u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0x8200u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0x83ffu, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0x0400u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0x8400u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0x0000u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0x3c00u, {0x5a5a4000u, 0x5a5a4000u, 0x5a5a4000u, 0x5a5a4000u}},
+    {0xbc00u, {0x5a5a3800u, 0x5a5a3800u, 0x5a5a3800u, 0x5a5a3800u}},
+    {0xcb01u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a03fau, 0x5a5a03fau}},
+    {0xcca0u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a002du, 0x5a5a002du}},
+    {0xce3cu, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0001u, 0x5a5a0001u}},
+    {0xc800u, {0x5a5a1c00u, 0x5a5a1c00u, 0x5a5a1c00u, 0x5a5a1c00u}},
+    {0xc400u, {0x5a5a2c00u, 0x5a5a2c00u, 0x5a5a2c00u, 0x5a5a2c00u}},
+};
+
+constexpr Vector FloorRows[] = {
+    {0x0001u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x0200u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x03ffu, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x8001u, {0x5a5a8000u, 0x5a5abc00u, 0x5a5a8000u, 0x5a5abc00u}},
+    {0x8200u, {0x5a5a8000u, 0x5a5abc00u, 0x5a5a8000u, 0x5a5abc00u}},
+    {0x83ffu, {0x5a5a8000u, 0x5a5abc00u, 0x5a5a8000u, 0x5a5abc00u}},
+    {0x0400u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x8400u, {0x5a5abc00u, 0x5a5abc00u, 0x5a5abc00u, 0x5a5abc00u}},
+    {0x0000u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x3c00u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0xbc00u, {0x5a5abc00u, 0x5a5abc00u, 0x5a5abc00u, 0x5a5abc00u}},
+    {0xb800u, {0x5a5abc00u, 0x5a5abc00u, 0x5a5abc00u, 0x5a5abc00u}},
+};
+
+constexpr Vector FractRows[] = {
+    {0x0001u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0001u}},
+    {0x0200u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0200u}},
+    {0x03ffu, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a03ffu}},
+    {0x8001u, {0x5a5a0000u, 0x5a5a3bffu, 0x5a5a0000u, 0x5a5a3bffu}},
+    {0x8200u, {0x5a5a0000u, 0x5a5a3bffu, 0x5a5a0000u, 0x5a5a3bffu}},
+    {0x83ffu, {0x5a5a0000u, 0x5a5a3bffu, 0x5a5a0000u, 0x5a5a3bffu}},
+    {0x0400u, {0x5a5a0400u, 0x5a5a0400u, 0x5a5a0400u, 0x5a5a0400u}},
+    {0x8400u, {0x5a5a3bffu, 0x5a5a3bffu, 0x5a5a3bffu, 0x5a5a3bffu}},
+    {0x0000u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x3c00u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0xbc00u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0xb800u, {0x5a5a3800u, 0x5a5a3800u, 0x5a5a3800u, 0x5a5a3800u}},
+    {0x4900u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+};
+
+constexpr Vector FrexpExpRows[] = {
+    {0x0001u, {0x5a5a0000u, 0x5a5affe9u, 0x5a5a0000u, 0x5a5affe9u}},
+    {0x0200u, {0x5a5a0000u, 0x5a5afff2u, 0x5a5a0000u, 0x5a5afff2u}},
+    {0x03ffu, {0x5a5a0000u, 0x5a5afff2u, 0x5a5a0000u, 0x5a5afff2u}},
+    {0x8001u, {0x5a5a0000u, 0x5a5affe9u, 0x5a5a0000u, 0x5a5affe9u}},
+    {0x8200u, {0x5a5a0000u, 0x5a5afff2u, 0x5a5a0000u, 0x5a5afff2u}},
+    {0x83ffu, {0x5a5a0000u, 0x5a5afff2u, 0x5a5a0000u, 0x5a5afff2u}},
+    {0x0400u, {0x5a5afff3u, 0x5a5afff3u, 0x5a5afff3u, 0x5a5afff3u}},
+    {0x8400u, {0x5a5afff3u, 0x5a5afff3u, 0x5a5afff3u, 0x5a5afff3u}},
+    {0x0000u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x3c00u, {0x5a5a0001u, 0x5a5a0001u, 0x5a5a0001u, 0x5a5a0001u}},
+    {0xbc00u, {0x5a5a0001u, 0x5a5a0001u, 0x5a5a0001u, 0x5a5a0001u}},
+    {0x4900u, {0x5a5a0004u, 0x5a5a0004u, 0x5a5a0004u, 0x5a5a0004u}},
+};
+
+constexpr Vector FrexpMantRows[] = {
+    {0x0001u, {0x5a5a0000u, 0x5a5a3800u, 0x5a5a0000u, 0x5a5a3800u}},
+    {0x0200u, {0x5a5a0000u, 0x5a5a3800u, 0x5a5a0000u, 0x5a5a3800u}},
+    {0x03ffu, {0x5a5a0000u, 0x5a5a3bfeu, 0x5a5a0000u, 0x5a5a3bfeu}},
+    {0x8001u, {0x5a5a8000u, 0x5a5ab800u, 0x5a5a8000u, 0x5a5ab800u}},
+    {0x8200u, {0x5a5a8000u, 0x5a5ab800u, 0x5a5a8000u, 0x5a5ab800u}},
+    {0x83ffu, {0x5a5a8000u, 0x5a5abbfeu, 0x5a5a8000u, 0x5a5abbfeu}},
+    {0x0400u, {0x5a5a3800u, 0x5a5a3800u, 0x5a5a3800u, 0x5a5a3800u}},
+    {0x8400u, {0x5a5ab800u, 0x5a5ab800u, 0x5a5ab800u, 0x5a5ab800u}},
+    {0x0000u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x3c00u, {0x5a5a3800u, 0x5a5a3800u, 0x5a5a3800u, 0x5a5a3800u}},
+    {0xbc00u, {0x5a5ab800u, 0x5a5ab800u, 0x5a5ab800u, 0x5a5ab800u}},
+    {0x4900u, {0x5a5a3900u, 0x5a5a3900u, 0x5a5a3900u, 0x5a5a3900u}},
+};
+
+constexpr Vector LogRows[] = {
+    {0x0001u, {0x5a5afc00u, 0x5a5ace00u, 0x5a5afc00u, 0x5a5ace00u}},
+    {0x0200u, {0x5a5afc00u, 0x5a5acb80u, 0x5a5afc00u, 0x5a5acb80u}},
+    {0x03ffu, {0x5a5afc00u, 0x5a5acb00u, 0x5a5afc00u, 0x5a5acb00u}},
+    {0x8001u, {0x5a5afc00u, 0x5a5afe00u, 0x5a5afc00u, 0x5a5afe00u}},
+    {0x8200u, {0x5a5afc00u, 0x5a5afe00u, 0x5a5afc00u, 0x5a5afe00u}},
+    {0x83ffu, {0x5a5afc00u, 0x5a5afe00u, 0x5a5afc00u, 0x5a5afe00u}},
+    {0x0400u, {0x5a5acb00u, 0x5a5acb00u, 0x5a5acb00u, 0x5a5acb00u}},
+    {0x8400u, {0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u}},
+    {0x0000u, {0x5a5afc00u, 0x5a5afc00u, 0x5a5afc00u, 0x5a5afc00u}},
+    {0x3c00u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0xbc00u, {0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u}},
+    {0x4900u, {0x5a5a42a5u, 0x5a5a42a5u, 0x5a5a42a5u, 0x5a5a42a5u}},
+    {0x0801u, {0x5a5aca80u, 0x5a5aca80u, 0x5a5aca80u, 0x5a5aca80u}},
+};
+
+constexpr Vector RcpRows[] = {
+    {0x0001u, {0x5a5a7c00u, 0x5a5a7c00u, 0x5a5a7c00u, 0x5a5a7c00u}},
+    {0x0200u, {0x5a5a7c00u, 0x5a5a7800u, 0x5a5a7c00u, 0x5a5a7800u}},
+    {0x03ffu, {0x5a5a7c00u, 0x5a5a7401u, 0x5a5a7c00u, 0x5a5a7401u}},
+    {0x8001u, {0x5a5afc00u, 0x5a5afc00u, 0x5a5afc00u, 0x5a5afc00u}},
+    {0x8200u, {0x5a5afc00u, 0x5a5af800u, 0x5a5afc00u, 0x5a5af800u}},
+    {0x83ffu, {0x5a5afc00u, 0x5a5af401u, 0x5a5afc00u, 0x5a5af401u}},
+    {0x0400u, {0x5a5a7400u, 0x5a5a7400u, 0x5a5a7400u, 0x5a5a7400u}},
+    {0x8400u, {0x5a5af400u, 0x5a5af400u, 0x5a5af400u, 0x5a5af400u}},
+    {0x0000u, {0x5a5a7c00u, 0x5a5a7c00u, 0x5a5a7c00u, 0x5a5a7c00u}},
+    {0x3c00u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0xbc00u, {0x5a5abc00u, 0x5a5abc00u, 0x5a5abc00u, 0x5a5abc00u}},
+    {0x7bffu, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0100u, 0x5a5a0100u}},
+    {0x7401u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a03ffu, 0x5a5a03ffu}},
+    {0xfbffu, {0x5a5a8000u, 0x5a5a8000u, 0x5a5a8100u, 0x5a5a8100u}},
+    {0xf404u, {0x5a5a8000u, 0x5a5a8000u, 0x5a5a83fcu, 0x5a5a83fcu}},
+    {0x7800u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0200u, 0x5a5a0200u}},
+};
+
+constexpr Vector RsqRows[] = {
+    {0x0001u, {0x5a5a7c00u, 0x5a5a6c00u, 0x5a5a7c00u, 0x5a5a6c00u}},
+    {0x0200u, {0x5a5a7c00u, 0x5a5a59a8u, 0x5a5a7c00u, 0x5a5a59a8u}},
+    {0x03ffu, {0x5a5a7c00u, 0x5a5a5801u, 0x5a5a7c00u, 0x5a5a5801u}},
+    {0x8001u, {0x5a5afc00u, 0x5a5afe00u, 0x5a5afc00u, 0x5a5afe00u}},
+    {0x8200u, {0x5a5afc00u, 0x5a5afe00u, 0x5a5afc00u, 0x5a5afe00u}},
+    {0x83ffu, {0x5a5afc00u, 0x5a5afe00u, 0x5a5afc00u, 0x5a5afe00u}},
+    {0x0400u, {0x5a5a5800u, 0x5a5a5800u, 0x5a5a5800u, 0x5a5a5800u}},
+    {0x8400u, {0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u}},
+    {0x0000u, {0x5a5a7c00u, 0x5a5a7c00u, 0x5a5a7c00u, 0x5a5a7c00u}},
+    {0x3c00u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0xbc00u, {0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u}},
+    {0x4400u, {0x5a5a3800u, 0x5a5a3800u, 0x5a5a3800u, 0x5a5a3800u}},
+};
+
+constexpr Vector SinRows[] = {
+    {0x0001u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0006u}},
+    {0x0200u, {0x5a5a0000u, 0x5a5a0a48u, 0x5a5a0000u, 0x5a5a0a48u}},
+    {0x03ffu, {0x5a5a0000u, 0x5a5a0e47u, 0x5a5a0000u, 0x5a5a0e47u}},
+    {0x8001u, {0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u, 0x5a5a8006u}},
+    {0x8200u, {0x5a5a8000u, 0x5a5a8a48u, 0x5a5a8000u, 0x5a5a8a48u}},
+    {0x83ffu, {0x5a5a8000u, 0x5a5a8e47u, 0x5a5a8000u, 0x5a5a8e47u}},
+    {0x0400u, {0x5a5a0e48u, 0x5a5a0e48u, 0x5a5a0e48u, 0x5a5a0e48u}},
+    {0x8400u, {0x5a5a8e48u, 0x5a5a8e48u, 0x5a5a8e48u, 0x5a5a8e48u}},
+    {0x0000u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x3c00u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0xbc00u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x00a0u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a03edu}},
+    {0x80a0u, {0x5a5a8000u, 0x5a5a8000u, 0x5a5a8000u, 0x5a5a83edu}},
+    {0x3800u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+};
+
+constexpr Vector SqrtRows[] = {
+    {0x0001u, {0x5a5a0000u, 0x5a5a0c00u, 0x5a5a0000u, 0x5a5a0c00u}},
+    {0x0200u, {0x5a5a0000u, 0x5a5a1da8u, 0x5a5a0000u, 0x5a5a1da8u}},
+    {0x03ffu, {0x5a5a0000u, 0x5a5a1fffu, 0x5a5a0000u, 0x5a5a1fffu}},
+    {0x8001u, {0x5a5a8000u, 0x5a5afe00u, 0x5a5a8000u, 0x5a5afe00u}},
+    {0x8200u, {0x5a5a8000u, 0x5a5afe00u, 0x5a5a8000u, 0x5a5afe00u}},
+    {0x83ffu, {0x5a5a8000u, 0x5a5afe00u, 0x5a5a8000u, 0x5a5afe00u}},
+    {0x0400u, {0x5a5a2000u, 0x5a5a2000u, 0x5a5a2000u, 0x5a5a2000u}},
+    {0x8400u, {0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u}},
+    {0x0000u, {0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u, 0x5a5a0000u}},
+    {0x3c00u, {0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u, 0x5a5a3c00u}},
+    {0xbc00u, {0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u, 0x5a5afe00u}},
+    {0x4400u, {0x5a5a4000u, 0x5a5a4000u, 0x5a5a4000u, 0x5a5a4000u}},
+};
+
+constexpr Case Cases[] = {
+    {"v_ceil_f16", 0x7e14b904u, CeilRows},
+    {"v_cvt_f32_f16", 0x7e141704u, CvtF32Rows},
+    {"v_cvt_norm_i16_f16", 0x7e14c704u, CvtNormI16Rows},
+    {"v_cvt_norm_u16_f16", 0x7e14c904u, CvtNormU16Rows},
+    {"v_exp_f16", 0x7e14b104u, ExpRows},
+    {"v_floor_f16", 0x7e14b704u, FloorRows},
+    {"v_fract_f16", 0x7e14bf04u, FractRows},
+    {"v_frexp_exp_i16_f16", 0x7e14b504u, FrexpExpRows},
+    {"v_frexp_mant_f16", 0x7e14b304u, FrexpMantRows},
+    {"v_log_f16", 0x7e14af04u, LogRows},
+    {"v_rcp_f16", 0x7e14a904u, RcpRows},
+    {"v_rsq_f16", 0x7e14ad04u, RsqRows},
+    {"v_sin_f16", 0x7e14c104u, SinRows},
+    {"v_sqrt_f16", 0x7e14ab04u, SqrtRows},
+};
+
+constexpr std::uint32_t CaseCount = std::size(Cases);
+std::array<Shader, CaseCount> Shaders{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Run(AgcDriver::VulkanDevice& device, const Case& item, const Shader& shader, std::uint32_t denorm16, bool ieee) {
+    Input.fill(0u);
+    for (std::uint32_t row = 0; row < item.rows.size(); ++row) {
+        Input[row * Inputs] = item.rows[row].half;
+    }
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(shader.words);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    request.context.floatMode = ShaderRecompiler::ShaderFloatMode{(3u << 4u) | (denorm16 << 6u), true, ieee, false};
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check(const Case& item, std::uint32_t denorm16, bool ieee) {
+    for (std::uint32_t row = 0; row < item.rows.size(); ++row) {
+        const auto actual = Output[row];
+        const auto expected = item.rows[row].expected[denorm16];
+        Require(actual == expected, std::string(item.name) + " denorm16 " + std::to_string(denorm16) + (ieee ? " ieee" : " no ieee") + " input " +
+                                        Hex(item.rows[row].half) + " is " + Hex(actual) + ", expected " + Hex(expected));
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        for (std::uint32_t index = 0; index < CaseCount; ++index) {
+            Shaders[index].words = {0x34020084u, 0x34040082u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e1402ffu, 0x5a5a5a5au, Cases[index].word,
+                                    0xe0701000u, 0x80010a02u, 0xbf810000u};
+            for (std::uint32_t denorm16 = 0u; denorm16 < 4u; ++denorm16) {
+                for (const bool ieee : {false, true}) {
+                    Run(*device, Cases[index], Shaders[index], denorm16, ieee);
+                    Check(Cases[index], denorm16, ieee);
+                }
+            }
+        }
+        std::puts("f16 denormal flush tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/MsaaEnableSingleSample.cpp

@@ -0,0 +1,132 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <exception>
+#include <initializer_list>
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+
+constexpr std::byte Kept{0x40};
+
+alignas(256) constexpr std::array<std::uint32_t, 16> VertexCode{
+    0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u,
+    0x7e060280u, 0x7e0802f2u, 0xf80008cfu, 0x04030201u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u,
+};
+alignas(256) constexpr std::array<std::uint32_t, 8> PixelCode{0x7e000200u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u};
+alignas(256) std::array<std::byte, 1024> ClearMemory{};
+alignas(256) std::array<std::byte, 1024> EnabledMemory{};
+
+struct Header {
+    Shader shader{};
+    ShaderUserData userData{};
+};
+
+alignas(8) Header VertexHeader{};
+alignas(8) Header PixelHeader{};
+
+void Registered(Header& header, const std::uint32_t* code, std::size_t bytes, std::uint8_t type) {
+    header.shader.file_header = 0x34333231u;
+    header.shader.version = 0x18u;
+    header.shader.header_size = sizeof(Header);
+    header.shader.shader_size = static_cast<std::uint32_t>(bytes);
+    header.shader.code = code;
+    header.shader.type = type;
+    header.shader.user_data = &header.userData;
+    AgcDriverRegisterShader_nid_postfix(&header.shader);
+}
+
+void Append(std::vector<std::uint32_t>& commands, std::uint32_t opcode, std::initializer_list<std::uint32_t> payload) {
+    commands.push_back(0xc0000000u | (static_cast<std::uint32_t>(payload.size() - 1) << 16u) | (opcode << 8u));
+    commands.insert(commands.end(), payload);
+}
+
+std::vector<std::uint32_t> DrawBoth() {
+    const auto vertex = reinterpret_cast<std::uintptr_t>(VertexCode.data());
+    const auto pixel = reinterpret_cast<std::uintptr_t>(PixelCode.data());
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> context{
+        {0x2d5, 0x2000},
+        {0x1b6, 0}, {0x207, 0}, {0x200, 0}, {0x203, 0x800},
+        {0x2dc, 0xaa00}, {0x2f8, 0}, {0x293, 0},
+        {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa},
+        {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff},
+        {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240},
+        {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010},
+        {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, 0x28028},
+        {0x31b, 0}, {0x31d, 0}, {0x3b0, (63u << 14u) | 3u},
+        {0x3b8, 0x9000000}, {0x1e0, 0},
+        {0xc, 0}, {0xd, 0x40040},
+        {0x81, 0x80000000}, {0x82, 0x40040},
+        {0x90, 0x80000000}, {0x91, 0x40040},
+        {0x94, 0x80000000}, {0x95, 0x40040},
+        {0x10f, std::bit_cast<std::uint32_t>(32.0f)}, {0x110, std::bit_cast<std::uint32_t>(32.0f)},
+        {0x111, std::bit_cast<std::uint32_t>(-2.0f)}, {0x112, std::bit_cast<std::uint32_t>(2.0f)},
+        {0x113, std::bit_cast<std::uint32_t>(1.0f)}, {0x114, 0},
+        {0xb4, 0}, {0xb5, std::bit_cast<std::uint32_t>(1.0f)},
+        {0x2fe, 0}, {0x302, 0}, {0x306, 0}, {0x30a, 0},
+    };
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> shader{
+        {0xc8, static_cast<std::uint32_t>(vertex >> 8u)}, {0xc9, static_cast<std::uint32_t>(vertex >> 40u)}, {0x8b, 1u << 1u}, {0x8c, 0},
+        {0x008, static_cast<std::uint32_t>(pixel >> 8u)}, {0x009, static_cast<std::uint32_t>(pixel >> 40u)}, {0x00b, 1u << 1u}, {0x00c, std::bit_cast<std::uint32_t>(1.0f)},
+    };
+    std::vector<std::uint32_t> commands;
+    Append(commands, 0x79, {0x242, 4});
+    for (const auto& [offset, value] : context) Append(commands, 0x69, {offset, value});
+    for (const auto& [offset, value] : shader) Append(commands, 0x76, {offset, value});
+    const auto draw = [&](std::array<std::byte, 1024>& memory, std::uint32_t scanMode) {
+        const auto color = reinterpret_cast<std::uintptr_t>(memory.data());
+        Append(commands, 0x69, {0x318, static_cast<std::uint32_t>(color >> 8u)});
+        Append(commands, 0x69, {0x390, static_cast<std::uint32_t>(color >> 40u)});
+        Append(commands, 0x69, {0x292, scanMode});
+        Append(commands, 0x2d, {3, 2});
+    };
+    draw(ClearMemory, 2);
+    draw(EnabledMemory, 3);
+    return commands;
+}
+
+void MsaaEnableDrawsLikeDisabled() {
+    ClearMemory.fill(Kept);
+    EnabledMemory.fill(Kept);
+    Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2);
+    Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1);
+    auto commands = DrawBoth();
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw submission failed");
+    AgcDriverWaitIdle_nid_postfix();
+    const auto kept = static_cast<std::size_t>(std::count(ClearMemory.begin(), ClearMemory.end(), Kept));
+    Require(kept != 0 && kept != ClearMemory.size(), "the triangle without MSAA_ENABLE covered " + std::to_string(ClearMemory.size() - kept) + " of " + std::to_string(ClearMemory.size()) + " bytes, expected an edge through the target");
+    for (std::size_t index = 0; index < ClearMemory.size(); ++index) {
+        Require(EnabledMemory[index] == ClearMemory[index], "byte " + std::to_string(index) + " is " + std::to_string(static_cast<unsigned>(EnabledMemory[index])) + " with MSAA_ENABLE on a single-sample target and " + std::to_string(static_cast<unsigned>(ClearMemory[index])) + " without it");
+    }
+}
+
+}
+
+int main() {
+    try {
+        {
+            const auto device = OpenVulkanTestDevice();
+            if (!device) return VulkanTestSkipped;
+        }
+        MsaaEnableDrawsLikeDisabled();
+        AgcDriverShutdown_nid_postfix();
+        std::puts("MSAA_ENABLE on a single-sample target drew the same pixels as the state without it");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        AgcDriverShutdown_nid_postfix();
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/OversizedDispatch.cpp

@@ -181,7 +181,7 @@ int main() {
         } else if (device->WorkGroupCountLimit()[0] < Groups) {
             CheckSplitCoverage(*device);
         } else {
-            std::printf("oversized dispatch split not tested: the device's %u workgroup limit never splits a %ux1x1 dispatch\n", device->WorkGroupCountLimit()[0], Groups);
+            std::printf("oversized dispatch split: skipped, the device reports a %u workgroup limit, which never splits a %ux1x1 dispatch\n", device->WorkGroupCountLimit()[0], Groups);
             return VulkanTestSkipped;
         }
         AgcDriverShutdown_nid_postfix();

core/libs/prx/libSceAgcDriver/tests/execution/PackedFloat16Native.cpp

@@ -148,7 +148,7 @@ int main() {
             return std::ranges::find(target.supportedCapabilities, static_cast<std::uint32_t>(capability)) != target.supportedCapabilities.end();
         };
         if (!supports(spv::CapabilityFloat16) || !supports(spv::CapabilityRoundingModeRTE) || !supports(spv::CapabilityDenormPreserve)) {
-            std::puts("packed f16 native: the device has no native f16 with RTE rounding and kept denormals, skipped");
+            std::puts("packed f16 native: skipped, the device has no native f16 with RTE rounding and kept denormals");
             return VulkanTestSkipped;
         }
         std::vector<std::uint32_t> withoutFloat16;

core/libs/prx/libSceAgcDriver/tests/execution/PackedMipTailTarget.cpp

@@ -0,0 +1,257 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::ColorTargetLayout;
+using AgcDriver::Graphics::ColorTileMode;
+using AgcDriver::Graphics::Require;
+using AgcDriver::Graphics::StorageTexture;
+using AgcDriver::Graphics::TextureTileMode;
+using AgcDriver::Graphics::TileMipLayout;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Side = 256;
+constexpr std::uint32_t Levels = 9;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::uint32_t BlockSide = 128;
+constexpr std::size_t SurfaceBytes = BlockBytes + Side * Side * 4u + BlockBytes;
+constexpr std::uint32_t Color32FloatInfo = (4u << 2u) | (7u << 8u);
+constexpr std::uint32_t TiledAttrib3 = 0x4dc6c000u;
+constexpr std::uint32_t Float32Export = 9;
+constexpr std::uint32_t Initial = 0x5a5a5a5au;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::array<std::uint32_t, 16>, Levels> PixelCodes{};
+
+std::uint64_t AddressOf(const void* data) {
+    return static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(data));
+}
+
+std::string Hex(std::uint64_t value) {
+    char text[24];
+    std::snprintf(text, sizeof(text), "0x%llx", static_cast<unsigned long long>(value));
+    return text;
+}
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(SurfaceBytes, BlockBytes));
+        if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, SurfaceBytes, PAGE_READWRITE, SurfaceBytes);
+#else
+        void* mapped = mmap(nullptr, SurfaceBytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        if (mapped != MAP_FAILED) {
+            const auto begin = reinterpret_cast<std::uintptr_t>(mapped);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(mapped, aligned - begin);
+            if (aligned + SurfaceBytes != begin + SurfaceBytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + SurfaceBytes), begin + BlockBytes - aligned);
+            block = reinterpret_cast<std::uint8_t*>(aligned);
+            if (AgcDriver::GuestMemory::WriteWatched()) GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, SurfaceBytes);
+        }
+#endif
+        Require(block != nullptr, "packed mip tail target: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, SurfaceBytes, true, true, true);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        GuestArena::GuestArenaReset_nid_postfix(block, SurfaceBytes);
+        GuestArena::GuestArenaRelease_nid_postfix(block, SurfaceBytes);
+#else
+        munmap(block, SurfaceBytes);
+        if (AgcDriver::GuestMemory::WriteWatched()) GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, SurfaceBytes);
+#endif
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(std::uint64_t address, std::uint32_t level, const TileMipLayout& mip) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(address >> 8u);
+    cx[0x31b] = level << 26u;
+    cx[0x31c] = Color32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x3b0] = ((Levels - 1u) << 28u) | ((Side - 1u) << 14u) | (Side - 1u);
+    cx[0x3b8] = TiledAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(address >> 40u);
+    const auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    Require(color.format == VK_FORMAT_R32_SFLOAT && color.tileMode == ColorTileMode::RenderTarget, "the COLOR_32 float view did not decode as an SW_64KB_R_X R32 target");
+    Require(color.mip == level && color.mipCount == Levels && color.mipTail == mip.tail, "the color view did not keep its mip of the surface");
+    Require(color.extent.width == mip.width && color.extent.height == mip.height && color.address == address + mip.tiledOffset, "the color view did not decode the extent and address of mip " + std::to_string(level));
+    return color;
+}
+
+void DrawLevel(AgcDriver::VulkanDevice& device, std::uint64_t surface, std::uint32_t level, const TileMipLayout& mip, std::uint32_t value) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    auto& pixelWords = PixelCodes[level];
+    std::size_t at = 0;
+    for (std::uint32_t channel = 0; channel < 4; ++channel) {
+        pixelWords[at++] = 0x7e0002ffu | ((4u + channel) << 17u);
+        pixelWords[at++] = value;
+    }
+    pixelWords[at++] = 0xf800180fu;
+    pixelWords[at++] = 0x07060504u;
+    pixelWords[at++] = 0xbf810000u;
+    const std::span<const std::uint32_t> pixelCode(pixelWords.data(), at);
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto address = reinterpret_cast<std::uintptr_t>(Triangle.data());
+    const std::array<std::uint32_t, 4> vertexBuffer{static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (16u << 16u), static_cast<std::uint32_t>(Triangle.size()), 0x01016facu};
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    const auto color = DecodeTarget(surface, level, mip);
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.colorWriteMask = 0xf;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = color.extent;
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(mip.height), static_cast<float>(mip.width), -static_cast<float>(mip.height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, color.extent};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+
+    const auto codeAddress = reinterpret_cast<std::uintptr_t>(pixelCode.data());
+    const AgcDriver::Registers context{{0x1b6, 0x8000}, {0x1b3, 0}, {0x1b4, 0}, {0x203, 0}, {0x1c5, Float32Export}};
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{codeAddress, std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, codeAddress, pixelCode, 0, {}},
+        {waveSize, 0, {}, std::nullopt, AgcDriver::Graphics::DecodePixelStageInfo(context, AgcDriver::Graphics::ExportMappings(state)), std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+    device.WaitIdle();
+}
+
+void Expect(std::vector<std::uint32_t>& expected, const TileMipLayout& mip, std::uint32_t value) {
+    const ColorTargetLayout block(BlockSide, BlockSide, ColorTileMode::RenderTarget, 4);
+    const ColorTargetLayout own(mip.width, mip.height, ColorTileMode::RenderTarget, 4);
+    for (std::uint32_t y = 0; y < mip.height; ++y) {
+        for (std::uint32_t x = 0; x < mip.width; ++x) {
+            const auto offset = mip.tail ? block.Offset(x + mip.tailX, y + mip.tailY) : mip.tiledOffset + own.Offset(x, y);
+            Require(expected.at(offset / 4u) == Initial, "the expected texels of two mips overlap at byte " + Hex(offset));
+            expected[offset / 4u] = value;
+        }
+    }
+}
+
+void Check(const std::uint8_t* texels, const std::vector<std::uint32_t>& expected, const std::string& what) {
+    const auto* words = reinterpret_cast<const std::uint32_t*>(texels);
+    for (std::size_t index = 0; index < expected.size(); ++index) {
+        Require(words[index] == expected[index], what + ": surface byte " + Hex(index * 4u) + " holds " + Hex(words[index]) + ", expected " + Hex(expected[index]));
+    }
+}
+
+void Run(AgcDriver::VulkanDevice& device, std::uint8_t* texels) {
+    const auto mips = AgcDriver::Graphics::ComputeElementMipLayout(TextureTileMode::kR64KBX, 4, Side, Side, Levels);
+    Require(AgcDriver::Graphics::ComputeSurfaceSize(mips, 1) == SurfaceBytes, "the 256x256 R32 surface with 9 mips is not one tail block, mip 0 and mip 1");
+    Require(!mips[1].tail && mips[2].tail && mips[3].tail && mips[2].width == 64 && mips[3].width == 32, "mips 2 and 3 of the 256x256 R32 surface are not its first packed tail mips");
+    const auto surface = AddressOf(texels);
+    std::fill_n(reinterpret_cast<std::uint32_t*>(texels), SurfaceBytes / 4u, Initial);
+    AgcDriver::GuestMemory::CollectWrites(surface, SurfaceBytes);
+    AgcDriver::GuestMemory::BumpCollectEpoch();
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    std::vector<std::uint32_t> expected(SurfaceBytes / 4u, Initial);
+    const std::array<std::pair<std::uint32_t, float>, 3> draws{{{2u, 0.25f}, {3u, 0.75f}, {1u, 0.5f}}};
+    for (const auto& [level, drawn] : draws) {
+        const auto value = std::bit_cast<std::uint32_t>(drawn);
+        DrawLevel(device, surface, level, mips[level], value);
+        Require(StorageTexture::FindPending(surface, SurfaceBytes) == nullptr, "the draw into mip " + std::to_string(level) + " rendered into a resident image instead of the guest-memory fallback");
+        Expect(expected, mips[level], value);
+        Check(texels, expected, "after the draw into mip " + std::to_string(level));
+    }
+    AgcDriver::Graphics::ClearCachedTextures(device.Device());
+}
+
+}
+
+int main() {
+    try {
+#ifdef _WIN32
+        _putenv_s("APS5_NO_RESIDENT_TARGETS", "1");
+#else
+        setenv("APS5_NO_RESIDENT_TARGETS", "1", 1);
+#endif
+        GuestBlock block;
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device, block.Data());
+        std::puts("packed mip tail target tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/PipelineOptimizedReadback.cpp

@@ -0,0 +1,276 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <condition_variable>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <mutex>
+#include <span>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using namespace AgcDriver::Graphics;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::byte Kept{0x40};
+alignas(256) std::array<std::byte, Width * Height * 4> Pixels{};
+alignas(256) std::array<float, Width * Height> DepthMemory{};
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+alignas(256) constexpr std::array<std::uint32_t, 7> MaskedPixelCode{
+    0x7e040f00, 0x36060481, 0x7da40680, 0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000,
+};
+alignas(256) constexpr std::array<std::uint32_t, 4> FullPixelCode{
+    0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000,
+};
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+constexpr std::array<std::array<float, 4>, 12> DepthTriangles{{
+    {-1.0f, -1.0f, 0.25f, 1.0f}, {0.0f, -1.0f, 0.25f, 1.0f}, {-1.0f, 1.0f, 0.25f, 1.0f},
+    {-1.0f, 1.0f, 0.25f, 1.0f}, {0.0f, -1.0f, 0.25f, 1.0f}, {0.0f, 1.0f, 0.25f, 1.0f},
+    {0.0f, -1.0f, 0.75f, 1.0f}, {1.0f, -1.0f, 0.75f, 1.0f}, {0.0f, 1.0f, 0.75f, 1.0f},
+    {0.0f, 1.0f, 0.75f, 1.0f}, {1.0f, -1.0f, 0.75f, 1.0f}, {1.0f, 1.0f, 0.75f, 1.0f}
+}};
+
+struct PipelineTrace {
+    PFN_vkGetDeviceProcAddr resolve = nullptr;
+    PFN_vkCreateGraphicsPipelines create = nullptr;
+    PFN_vkCmdBindPipeline bind = nullptr;
+    PFN_vkCmdDraw draw = nullptr;
+    DeviceFunctions functions{};
+    std::mutex mutex;
+    std::condition_variable released;
+    bool allowOptimization = false;
+    std::vector<VkPipeline> fast;
+    std::vector<VkPipeline> optimized;
+    VkCommandBuffer boundCommands = VK_NULL_HANDLE;
+    VkPipeline bound = VK_NULL_HANDLE;
+    std::vector<VkPipeline> drawn;
+
+    void ReleaseOptimization() {
+        {
+            std::lock_guard lock(mutex);
+            allowOptimization = true;
+        }
+        released.notify_all();
+    }
+};
+
+PipelineTrace* Trace = nullptr;
+
+VKAPI_ATTR VkResult VKAPI_CALL CreatePipelines(VkDevice device, VkPipelineCache cache, std::uint32_t count, const VkGraphicsPipelineCreateInfo* infos, const VkAllocationCallbacks* allocator, VkPipeline* pipelines) {
+    const bool optimizing = std::any_of(infos, infos + count, [](const auto& info) { return (info.flags & VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT) != 0; });
+    if (optimizing) {
+        std::unique_lock lock(Trace->mutex);
+        Trace->released.wait(lock, [] { return Trace->allowOptimization; });
+    }
+    const auto result = Trace->create(device, cache, count, infos, allocator, pipelines);
+    std::lock_guard lock(Trace->mutex);
+    for (std::uint32_t index = 0; index < count; ++index) {
+        if (pipelines[index] == VK_NULL_HANDLE || (infos[index].flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) != 0) continue;
+        auto& handles = (infos[index].flags & VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT) != 0 ? Trace->optimized : Trace->fast;
+        handles.push_back(pipelines[index]);
+    }
+    return result;
+}
+
+VKAPI_ATTR void VKAPI_CALL BindPipeline(VkCommandBuffer commands, VkPipelineBindPoint point, VkPipeline pipeline) {
+    if (point == VK_PIPELINE_BIND_POINT_GRAPHICS) {
+        Trace->boundCommands = commands;
+        Trace->bound = pipeline;
+    }
+    Trace->bind(commands, point, pipeline);
+}
+
+VKAPI_ATTR void VKAPI_CALL DrawCommands(VkCommandBuffer commands, std::uint32_t vertices, std::uint32_t instances, std::uint32_t firstVertex, std::uint32_t firstInstance) {
+    Trace->drawn.push_back(Trace->boundCommands == commands ? Trace->bound : VK_NULL_HANDLE);
+    Trace->draw(commands, vertices, instances, firstVertex, firstInstance);
+}
+
+VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL Resolve(VkDevice device, const char* name) {
+    if (std::strcmp(name, "vkCreateGraphicsPipelines") == 0) return reinterpret_cast<PFN_vkVoidFunction>(CreatePipelines);
+    if (std::strcmp(name, "vkCmdBindPipeline") == 0) return reinterpret_cast<PFN_vkVoidFunction>(BindPipeline);
+    if (std::strcmp(name, "vkCmdDraw") == 0) return reinterpret_cast<PFN_vkVoidFunction>(DrawCommands);
+    return Trace->resolve(device, name);
+}
+
+struct OptimizationDrain {
+    VkDevice device;
+    ~OptimizationDrain() {
+        Trace->ReleaseOptimization();
+        WaitForOptimizedPipelines(device);
+    }
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+void CheckPixels(bool masked, std::string_view phase) {
+    for (std::uint32_t pixel = 0; pixel < Width * Height; ++pixel) {
+        const bool drawn = masked ? (pixel % Width) % 2u == 0u : pixel % Width < Width / 2u;
+        const auto expected = drawn ? std::byte{255} : Kept;
+        for (std::uint32_t channel = 0; channel < 4; ++channel) {
+            const auto actual = Pixels[pixel * 4u + channel];
+            Require(actual == expected, std::string(phase) + ": pixel " + std::to_string(pixel) + " channel " + std::to_string(channel) + " is " + std::to_string(static_cast<unsigned>(actual)) + ", expected " + std::to_string(static_cast<unsigned>(expected)));
+        }
+    }
+}
+
+void CheckDrawn(PipelineTrace& trace, bool optimized) {
+    std::lock_guard lock(trace.mutex);
+    const auto& expected = optimized ? trace.optimized : trace.fast;
+    Require(trace.drawn.size() == 1, "pipeline readback did not observe exactly one real draw");
+    Require(std::find(expected.begin(), expected.end(), trace.drawn.front()) != expected.end(), optimized ? "readback draw did not bind an optimized pipeline" : "readback draw did not bind a fast-linked pipeline");
+}
+
+void Run(AgcDriver::VulkanDevice& device, Context context, PipelineTrace& trace, std::uint32_t waveSize, VkFormat depthFormat, bool optimize) {
+    const bool masked = depthFormat == VK_FORMAT_UNDEFINED;
+    const auto target = device.Target();
+    const auto vertices = masked ? std::span<const std::array<float, 4>>(Triangle) : std::span<const std::array<float, 4>>(DepthTriangles);
+    const auto vertexBuffer = BufferDescriptor(vertices.data(), 16u, static_cast<std::uint32_t>(vertices.size()));
+    const std::vector<std::uint32_t> vertexUserData(vertexBuffer.begin(), vertexBuffer.end());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+    const auto pixelCode = masked ? std::span<const std::uint32_t>(MaskedPixelCode) : std::span<const std::uint32_t>(FullPixelCode);
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.wave32 = waveSize == 32u;
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY);
+    pixel.posX = true;
+    pixel.posY = true;
+    pixel.targetOutputMode[0] = 9;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+    State state{};
+    state.stages = {ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.color = {reinterpret_cast<std::uintptr_t>(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u};
+    state.colors = {state.color};
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {Width, Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blend.colorWriteMask = 15;
+    state.blends = {state.blend};
+    if (!masked) {
+        state.depth = DepthTarget{reinterpret_cast<std::uintptr_t>(DepthMemory.data()), 0, {Width, Height}, depthFormat, 0.5f, 0};
+        state.depthTest = true;
+        state.depthCompare = VK_COMPARE_OP_LESS;
+    }
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(vertices.size()), 0, 1, 0, false};
+    const auto render = [&](bool optimized, std::string_view phase) {
+        Pixels.fill(Kept);
+        trace.drawn.clear();
+        trace.bound = VK_NULL_HANDLE;
+        trace.boundCommands = VK_NULL_HANDLE;
+        AgcDriver::Graphics::Draw(context, state, draw, shaders);
+        device.WaitIdle();
+        CheckDrawn(trace, optimized);
+        CheckPixels(masked, phase);
+    };
+    render(false, "fast link");
+    trace.ReleaseOptimization();
+    WaitForOptimizedPipelines(context.device);
+    Require(PipelineLibraryCountersOf(context.device).linked == 1, "the readback pipeline was not linked from graphics libraries");
+    Require(trace.optimized.size() == (optimize ? 1u : 0u), "the optimized pipeline creation count differs from the requested mode");
+    render(optimize, "completed link");
+    std::printf("pipeline readback passed: fast draws %u, optimized draws %u, optimized creations %zu\n", optimize ? 1u : 2u, optimize ? 1u : 0u, trace.optimized.size());
+}
+
+}
+
+int main(int argc, char** argv) {
+    try {
+        std::string_view mode = "d32";
+        bool force = false;
+        for (int index = 1; index < argc; ++index) {
+            const std::string_view argument = argv[index];
+            if (argument == "--force-lto") force = true;
+            else {
+                Require(argument == "color32" || argument == "color64" || argument == "d32" || argument == "d16", "unknown pipeline readback case");
+                mode = argument;
+            }
+        }
+        PipelineTrace trace;
+        Trace = &trace;
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(*device);
+        if (!context.graphicsPipelineLibrary) {
+            std::puts("skipped, graphics pipeline libraries are unavailable or disabled");
+            return VulkanTestSkipped;
+        }
+        const bool masked = mode == "color32" || mode == "color64";
+        if (masked && device->Target().subgroupSize < 32u) {
+            std::puts("skipped, the masked pixel shader requires at least 32 subgroup lanes");
+            return VulkanTestSkipped;
+        }
+        if (force) context.pipelineLinkTimeOptimization = true;
+        const bool optimize = context.pipelineLinkTimeOptimization && std::getenv("APS5_NO_PIPELINE_LTO") == nullptr;
+        std::printf("pipeline readback: %s, %.*s, LTO %s%s\n", device->DeviceName().c_str(), static_cast<int>(mode.size()), mode.data(), optimize ? "enabled" : "disabled", force ? " (explicit test override)" : " (production policy)");
+        trace.resolve = context.deviceProc;
+        trace.create = context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines");
+        trace.bind = context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline");
+        trace.draw = context.Function<PFN_vkCmdDraw>("vkCmdDraw");
+        trace.functions = context.functions != nullptr ? *context.functions : DeviceFunctions{};
+        trace.functions.cmdBindPipeline = BindPipeline;
+        trace.functions.cmdDraw = DrawCommands;
+        context.functions = &trace.functions;
+        context.deviceProc = Resolve;
+        OptimizationDrain drain{context.device};
+        std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex());
+        Run(*device, context, trace, mode == "color32" ? 32u : 64u, masked ? VK_FORMAT_UNDEFINED : (mode == "d16" ? VK_FORMAT_D16_UNORM : VK_FORMAT_D32_SFLOAT), optimize);
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/PixelWaveReduction.cpp

@@ -194,13 +194,16 @@ int main() {
         }};
         for (const auto& layout : layouts) {
             for (const auto& reduction : reductions) {
-                const auto code = PixelCode(reduction);
-                std::uint32_t partial = 0;
-                for (std::uint32_t pass = 0; pass < 4; ++pass) {
-                    Draw(*device, code, layout);
-                    partial += Check(reduction, target.subgroupSize);
+                for (const bool windowed : {false, true}) {
+                    auto code = PixelCode(reduction);
+                    if (windowed) code.insert(std::find(code.begin(), code.end(), 0x7e0e02f2u), 0x7e008514u);
+                    std::uint32_t partial = 0;
+                    for (std::uint32_t pass = 0; pass < 4; ++pass) {
+                        Draw(*device, code, layout);
+                        partial += Check(reduction, target.subgroupSize);
+                    }
+                    std::printf("%s%s, laid out for %u lanes on %u-lane subgroups: %u partial waves over 4 draws\n", reduction.name, windowed ? " in a v_movreld_b32 window" : "", layout.subgroupSize, target.subgroupSize, partial);
                 }
-                std::printf("%s, laid out for %u lanes on %u-lane subgroups: %u partial waves over 4 draws\n", reduction.name, layout.subgroupSize, target.subgroupSize, partial);
             }
         }
         std::puts("pixel wave reduction tests passed");

core/libs/prx/libSceAgcDriver/tests/execution/PreparedExports.cpp

@@ -29,7 +29,7 @@ void Run(AgcDriver::VulkanDevice& device, bool integer) {
     const std::array<std::uint32_t, 4> values = integer ? std::array<std::uint32_t, 4>{11, 22, 33, 44} : std::array<std::uint32_t, 4>{0, 0x3e800000u, 0x3f000000u, 0x3f800000u};
     const std::array<std::uint32_t, 13> code{0x7e0002ffu, values[0], 0x7e0202ffu, values[1], 0x7e0402ffu, values[2], 0x7e0602ffu, values[3], 0xf800100fu, 0x03020100u, 0xf800183fu, 0x03020100u, 0xbf810000u};
     AgcDriver::Registers context{{0x1b6, 0x8000}, {0x1b3, 0}, {0x1b4, 0}, {0x203, 0}, {0x1c5, integer ? 0x7007u : 0x9009u}};
-    RecompileRequest request{{ShaderStage::Fragment, 0x20000, code, 0, {}}, {32, 0, {}, {}, AgcDriver::Graphics::DecodePixelStageInfo(context, {}), {}, {}}, device.Target(), {0, 0, 0, 128}};
+    RecompileRequest request{{ShaderStage::Fragment, 0x20000, code, 0, {}}, {32, 0, {}, {}, AgcDriver::Graphics::DecodePixelStageInfo(context, {}), {}, {}}, device.Target(), PixelPushLayout(0u, device.Target())};
     const auto handle = PrepareShader(request);
     const auto& artifact = GetPreparedArtifact(*handle);
     std::uint32_t scalarType = 0;
@@ -85,6 +85,10 @@ void Run(AgcDriver::VulkanDevice& device, bool integer) {
             request.useCache = cached;
             const auto materialized = MaterializeShader(request, *capture, *handle);
             Require(materialized->spirv.data() == pixel->spirv.data() && materialized->PipelineVariantId() == pixel->PipelineVariantId(), "export mapping did not reuse its specialized module");
+            if (device.Target().registrationModules) {
+                Require(materialized->PipelineVariantId() == first->PipelineVariantId(), "registration modules: the export mapping specialized the fragment module");
+                continue;
+            }
             for (std::size_t cursor = 5; cursor < materialized->spirv.size();) {
                 const auto count = materialized->spirv[cursor] >> 16u;
                 Require(count != 0u && count <= materialized->spirv.size() - cursor, "invalid specialized fragment instruction");
@@ -94,7 +98,7 @@ void Run(AgcDriver::VulkanDevice& device, bool integer) {
         }
         request.useCache = true;
         if (integer) continue;
-        const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::Fragment, pixel.get(), 0}}};
+        const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::Fragment, pixel.get(), PixelPushOffset(0u, device.Target())}}};
         pixels.fill(0xdeadbeef);
         otherPixels.fill(0xdeadbeef);
         device.Draw(state, draw, shaders);

core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp

@@ -1,3 +1,4 @@
+#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
 #include "VulkanTestDevice.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp"
 #include "SceShaders.hpp"
@@ -130,6 +131,30 @@ void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path
     }
     const auto stages = PrepareGraphicsStages(prepared, target);
     Require(stages.size() == (tessellation ? 4u : 2u), "graphics preparation compiled the wrong stages");
+    if (!mesh && !tessellation && target.registrationModules) {
+        const auto prebuild = [&](const std::vector<PreparedGraphicsStage>& group, std::initializer_list<std::size_t> picked) {
+            std::vector<AgcDriver::Graphics::CompiledShader> shaders;
+            for (const auto index : picked) {
+                Require(group.at(index).registration != nullptr, "registration modules: a prepared stage has no registration result");
+                shaders.push_back({group[index].stage, group[index].registration.get(), group[index].pushOffset});
+            }
+            const auto before = AgcDriver::Graphics::PipelineLibraryCountersOf(device.Device());
+            Require(device.PrebuildGraphicsLibraries(prepared.state.stages, shaders), "registration modules: the shader libraries were not prebuilt");
+            const auto after = AgcDriver::Graphics::PipelineLibraryCountersOf(device.Device());
+            Require(after.linked == before.linked, "registration modules: a prebuild linked a pipeline");
+            return std::pair{after.built[1] - before.built[1], after.built[2] - before.built[2]};
+        };
+        const auto vertexOnly = prebuild(stages, {0u});
+        Require(vertexOnly.first == 1u, "registration modules: the vertex stage alone did not build its pre-rasterization library");
+        const auto pixelOnly = prebuild(stages, {1u});
+        Require(pixelOnly.second == 1u, "registration modules: the pixel stage alone did not build its fragment library");
+        const auto pair = prebuild(stages, {0u, 1u});
+        Require(pair.first == 0u && pair.second == 0u, "registration modules: the pair of stages did not find the libraries prebuilt one stage at a time");
+        const auto again = PrepareGraphicsStages(prepared, target);
+        Require(again.front().registration != stages.front().registration, "registration modules: the second preparation reused the first result object");
+        const auto repeated = prebuild(again, {0u, 1u});
+        Require(repeated.first == 0u && repeated.second == 0u, "registration modules: another materialization of the same stages did not find the prebuilt libraries");
+    }
     for (const auto& stage : stages) stage.snapshot->prepared->entries.push_back(stage.entry);
     const auto& pixelArtifact = GetPreparedArtifact(*stages.back().entry.handle);
     Require(!pixelArtifact.fragmentParameters.empty() && pixelArtifact.fragmentParameters.front().sourceLocation == 3u, "prepared fragment lost interpolant mapping");

core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp

@@ -200,7 +200,11 @@ void FailureCapture(AgcDriver::VulkanDevice& device) {
 #else
     Require(unsetenv("APS5_DUMP_SHADERS") == 0, "cannot disable shader capture");
 #endif
-    Require(failure([&] { static_cast<void>(AgcDriver::DriverDetail::PrepareShaderWithDiagnostics(request)); }) == expected, "disabled shader capture changed the preparation failure");
+    std::ostringstream prefix;
+    prefix << "shader 0x" << std::hex << request.shader.codeAddress << " stage " << static_cast<unsigned>(request.shader.stage) << ": ";
+    const auto diagnosed = failure([&] { static_cast<void>(AgcDriver::DriverDetail::PrepareShaderWithDiagnostics(request)); });
+    Require(diagnosed.rfind(prefix.str(), 0) == 0, "diagnosed failure names no shader address and stage");
+    Require(diagnosed.substr(prefix.str().size()) == expected, "diagnosed failure changed the reason");
     Require(!std::ifstream(path, std::ios::binary).is_open(), "disabled shader capture created a request");
 #ifdef _WIN32
     Require(_putenv_s("APS5_DUMP_SHADERS", "1") == 0, "cannot enable shader capture");
@@ -208,12 +212,14 @@ void FailureCapture(AgcDriver::VulkanDevice& device) {
     Require(setenv("APS5_DUMP_SHADERS", "1", 1) == 0, "cannot enable shader capture");
 #endif
     const auto actual = failure([&] { static_cast<void>(AgcDriver::DriverDetail::PrepareShaderWithDiagnostics(request)); });
+    std::ostringstream dumpedPrefix;
+    dumpedPrefix << "shader 0x" << std::hex << request.shader.codeAddress << " stage " << static_cast<unsigned>(request.shader.stage) << " (" << path << "): ";
+    Require(actual == dumpedPrefix.str() + expected, "enabled shader capture changed the diagnosed failure");
 #ifdef _WIN32
     Require(_putenv_s("APS5_DUMP_SHADERS", saved.c_str()) == 0, "cannot restore shader capture");
 #else
     Require((previous != nullptr ? setenv("APS5_DUMP_SHADERS", saved.c_str(), 1) : unsetenv("APS5_DUMP_SHADERS")) == 0, "cannot restore shader capture");
 #endif
-    Require(actual == expected, "shader capture changed the preparation failure");
     std::ifstream file(path, std::ios::binary);
     std::ostringstream text;
     text << file.rdbuf();
@@ -531,7 +537,7 @@ void RegisteredVertexWorkgroup(const ShaderRecompiler::SpirvTarget& target, bool
         return;
     }
     alignas(256) static constexpr std::array<std::uint32_t, 8> code{
-        0xd8340000u, 0u, 0xd7600000u, 0x00017f00u, 0x7e020200u, 0xf80008cfu, 0x04030201u, 0xbf810000u
+        0xd8340000u, 0u, 0xd7600000u, 0x00017f05u, 0x7e020200u, 0xf80008cfu, 0x04030201u, 0xbf810000u
     };
     struct Header {
         Shader shader{};

core/libs/prx/libSceAgcDriver/tests/execution/ProvokingVertex.cpp

@@ -50,7 +50,7 @@ ShaderRecompiler::RecompileResult GuestPixel(const ShaderRecompiler::SpirvTarget
     request.context.waveSize = 64;
     request.context.pixel = pixel;
     request.target = target;
-    request.layout.pushConstantSizeBytes = 128;
+    request.layout = PixelPushLayout(0u, target);
     request.useCache = false;
     auto result = ShaderRecompiler::Recompile(request);
     result.variantId = 3;
@@ -130,7 +130,7 @@ int main() {
         const auto guestPixel = GuestPixel(device->Target());
         const std::array<AgcDriver::Graphics::CompiledShader, 2> guestShaders{{
             {ShaderRecompiler::ShaderStage::Vertex, &vertex, 0},
-            {ShaderRecompiler::ShaderStage::Fragment, &guestPixel, 0}
+            {ShaderRecompiler::ShaderStage::Fragment, &guestPixel, PixelPushOffset(0u, device->Target())}
         }};
         for (const auto topology : {VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP}) Run(*device, topology, guestShaders, true, "guest P0");
         std::puts("provoking vertex rendering tests passed: GLSL and guest P0, triangle list and strip, first/last/first");

core/libs/prx/libSceAgcDriver/tests/execution/RecordedKeptBytes.cpp

@@ -200,13 +200,13 @@ void Draw(AgcDriver::VulkanDevice& device, GuestBuffer& pixels, GuestBuffer& ver
         {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(PixelCode.data()), PixelCode, 0, {}},
         {64, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
         target,
-        {0, 0, vertexPush, 128 - vertexPush}
+        PixelPushLayout(vertexPush, target)
     };
     fragment.useCache = false;
     const auto pixelResult = ShaderRecompiler::Recompile(fragment);
     const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
         {ShaderStage::Vertex, &vertexResult, 0},
-        {ShaderStage::Fragment, &pixelResult, vertexPush}
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
     }};
 
     AgcDriver::Graphics::State state{};

core/libs/prx/libSceAgcDriver/tests/execution/RelativeAncillary.cpp

@@ -0,0 +1,148 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <array>
+#include <cstdint>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+constexpr std::uint32_t Extent = 64u;
+alignas(256) std::array<std::byte, Extent * Extent * 4u> pixels{};
+alignas(256) constexpr std::array<std::array<float, 4>, 6> positions{{
+    {0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f, 1.0f},
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+constexpr std::array<std::uint32_t, 6> vertexCode{
+    0xe0382000u, 0x80000005u, 0xbf8c3f70u, 0xf80008cfu, 0x03020100u, 0xbf810000u
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+void Draw(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> fragmentCode, std::uint32_t offset, std::uint32_t waveSize, std::byte expected, bool reject = false, bool implicitInputs = false) {
+    pixels.fill(std::byte{0x40});
+    const auto target = device.Target();
+    const auto userData = BufferDescriptor(positions.data(), 16u, positions.size());
+    const auto code = std::span<const std::uint32_t>(vertexCode);
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32u, 0u, userData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, {}},
+        target, {0u, 0u, 0u, 64u}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto pushBytes = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.wave32 = waveSize == 32u;
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::Ancillary);
+    pixel.ancillary = true;
+    if (implicitInputs) {
+        pixel.inputAddr |= ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionZ) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionW);
+        pixel.posX = pixel.posY = pixel.posZ = pixel.posW = true;
+    }
+    pixel.targetOutputMode[0] = 9u;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::array<std::uint32_t, 1> pixelData{offset};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(fragmentCode.data()), fragmentCode, 0, {}},
+        {waveSize, 0u, pixelData, std::nullopt, pixel, std::nullopt, {}}, target, PixelPushLayout(pushBytes, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = [&] {
+        try {
+            auto result = ShaderRecompiler::Recompile(fragment);
+            Require(!reject, "an unsupported ancillary word was accepted");
+            return result;
+        } catch (const std::exception& error) {
+            if (!reject || std::string(error.what()).find("packed pixel ancillary input has an unsupported live use") == std::string::npos) throw;
+            return ShaderRecompiler::RecompileResult{};
+        }
+    }();
+    if (reject) return;
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0u}, {ShaderStage::Fragment, &pixelResult, PixelPushOffset(pushBytes, target)}
+    }};
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt};
+    state.color = {reinterpret_cast<std::uintptr_t>(pixels.data()), {Extent, Extent}, VK_FORMAT_R8G8B8A8_UNORM, pixels.size(), 0xe4u};
+    state.colors = {state.color};
+    state.hasColorTarget = true;
+    state.renderExtent = {Extent, Extent};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0.0f, static_cast<float>(Extent), static_cast<float>(Extent), -static_cast<float>(Extent), 0.0f, 1.0f};
+    state.scissor = {{0, 0}, {Extent, Extent}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blend.colorWriteMask = 15u;
+    state.blends = {state.blend};
+    AgcDriver::Pm4::DrawParameters draw{0u, 3u, 0u, 1u, 0u, false};
+    draw.firstVertex = 3u;
+    draw.firstInstance = 2u;
+    device.Draw(state, draw, shaders);
+    device.WaitIdle();
+    for (const auto value : pixels) Require(value == expected, "relative ancillary draw produced an incorrect pixel: " + std::to_string(std::to_integer<unsigned>(value)));
+}
+
+} // namespace
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        for (const auto waveSize : {32u, 64u}) {
+            const std::array<std::uint32_t, 8> read{
+                0xbefc0300u, 0x7e028700u, 0xd5480001u, 0x02111101u,
+                0x7e020d01u, 0x060202f2u, 0xf800180fu, 0x01010101u
+            };
+            auto code = std::vector<std::uint32_t>(read.begin(), read.end());
+            code.push_back(0xbf810000u);
+            std::cerr << "wave " << waveSize << " packed read\n";
+            Draw(*device, code, 0u, waveSize, std::byte{255});
+            Draw(*device, code, 4u, waveSize, std::byte{255}, false, true);
+            const std::array<std::uint32_t, 8> prefix{
+                0xd5480000u, 0x02111100u, 0x7e000d00u, 0x7e020280u,
+                0x7e0402f2u, 0xbefc0300u, 0x7e008502u, 0x7e008702u
+            };
+            code.assign(prefix.begin(), prefix.end());
+            code.insert(code.end(), {0xf800180fu, 0x01010101u, 0xbf810000u});
+            std::cerr << "in-range\n";
+            Draw(*device, code, 1u, waveSize, std::byte{255});
+            std::cerr << "out-of-range\n";
+            Draw(*device, code, 8u, waveSize, std::byte{0});
+            code[6] = 0x7e02d100u;
+            code.erase(code.begin() + 7);
+            std::cerr << "in-range\n";
+            Draw(*device, code, 0x00010001u, waveSize, std::byte{255});
+            std::cerr << "out-of-range\n";
+            Draw(*device, code, 0x00010008u, waveSize, std::byte{0});
+            code.assign(prefix.begin(), prefix.begin() + 6);
+            code.insert(code.end(), {0xbefe0480u, 0x7e008502u, 0xbefe04c1u, 0xf800180fu, 0x01010101u, 0xbf810000u});
+            std::cerr << "inactive write\n";
+            Draw(*device, code, 1u, waveSize, std::byte{0});
+            for (const auto modifier : {0x0000e402u, 0xff00e402u}) {
+                code.assign(prefix.begin(), prefix.begin() + 6);
+                code.insert(code.end(), {0x7e0084fau, modifier, 0xf800180fu, 0x01010101u, 0xbf810000u});
+                std::cerr << "DPP write " << modifier << '\n';
+                Draw(*device, code, 1u, waveSize, modifier == 0xff00e402u ? std::byte{255} : std::byte{0});
+                Draw(*device, code, 8u, waveSize, std::byte{0});
+            }
+            const std::array<std::uint32_t, 5> unsupported{0xbefc0300u, 0x7e028700u, 0xf800180fu, 0x01010101u, 0xbf810000u};
+            Draw(*device, unsupported, 0u, waveSize, std::byte{0}, true);
+            Draw(*device, unsupported, 4u, waveSize, std::byte{0}, true, true);
+        }
+        std::cout << "relative ancillary GPU readback tests passed\n";
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/RelativeStageInputs.cpp

@@ -0,0 +1,148 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <array>
+#include <cstdint>
+#include <iostream>
+#include <span>
+#include <string>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Extent = 64u;
+alignas(256) std::array<std::byte, Extent * Extent * 4u> pixels{};
+alignas(256) std::array<std::uint32_t, 16> output{};
+alignas(256) constexpr std::array<std::array<float, 4>, 6> positions{{
+    {0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f, 1.0f},
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+constexpr std::array<std::uint32_t, 11> relativeVertexCode{
+    0xbefc0385u, 0x7e028700u, 0xbefc0388u, 0x7e008700u, 0x4a000300u,
+    0xe0382000u, 0x80000000u, 0xbf8c3f70u, 0xf80008cfu, 0x03020100u, 0xbf810000u
+};
+constexpr std::array<std::uint32_t, 6> vertexCode{
+    0xe0382000u, 0x80000005u, 0xbf8c3f70u, 0xf80008cfu, 0x03020100u, 0xbf810000u
+};
+constexpr std::array<std::uint32_t, 5> relativePixelCode{
+    0xbefc0383u, 0x7e008700u, 0xf800180fu, 0x00000000u, 0xbf810000u
+};
+constexpr std::array<std::uint32_t, 4> pixelCode{
+    0x7e0002f2u, 0xf800180fu, 0x00000000u, 0xbf810000u
+};
+constexpr std::array<std::uint32_t, 9> computeCode{
+    0xbefc0382u, 0x7e028700u, 0x34020283u, 0x4a000300u, 0x34000082u,
+    0x7e028700u, 0xe0701000u, 0x80000100u, 0xbf810000u
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, stride == 0u ? 0x31016facu : 0x01016facu};
+}
+
+void Draw(AgcDriver::VulkanDevice& device, bool relativeVertex) {
+    pixels.fill(std::byte{0x40});
+    const auto target = device.Target();
+    const auto userData = BufferDescriptor(positions.data(), 16u, positions.size());
+    const auto code = relativeVertex ? std::span<const std::uint32_t>(relativeVertexCode) : std::span<const std::uint32_t>(vertexCode);
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32u, 0u, userData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, {}},
+        target, {0u, 0u, 0u, 64u}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto pushBytes = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.wave32 = true;
+    if (!relativeVertex) {
+        pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX)
+            | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY)
+            | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionZ)
+            | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionW);
+        pixel.posX = pixel.posY = pixel.posZ = pixel.posW = true;
+    }
+    pixel.targetOutputMode[0] = 9u;
+    pixel.targetExportMapping.fill(0xe4u);
+    const auto fragmentCode = relativeVertex ? std::span<const std::uint32_t>(pixelCode) : std::span<const std::uint32_t>(relativePixelCode);
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(fragmentCode.data()), fragmentCode, 0, {}},
+        {32u, 0u, {}, std::nullopt, pixel, std::nullopt, {}}, target, PixelPushLayout(pushBytes, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0u}, {ShaderStage::Fragment, &pixelResult, PixelPushOffset(pushBytes, target)}
+    }};
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt};
+    state.color = {reinterpret_cast<std::uintptr_t>(pixels.data()), {Extent, Extent}, VK_FORMAT_R8G8B8A8_UNORM, pixels.size(), 0xe4u};
+    state.colors = {state.color};
+    state.hasColorTarget = true;
+    state.renderExtent = {Extent, Extent};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0.0f, static_cast<float>(Extent), static_cast<float>(Extent), -static_cast<float>(Extent), 0.0f, 1.0f};
+    state.scissor = {{0, 0}, {Extent, Extent}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blend.colorWriteMask = 15u;
+    state.blends = {state.blend};
+    AgcDriver::Pm4::DrawParameters draw{0u, 3u, 0u, 1u, 0u, false};
+    draw.firstVertex = relativeVertex ? 1u : 3u;
+    draw.firstInstance = 2u;
+    device.Draw(state, draw, shaders);
+    device.WaitIdle();
+    for (const auto value : pixels) Require(value == std::byte{255}, "relative stage input draw did not cover every pixel in white");
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device) {
+    output.fill(0xdeadbeefu);
+    const auto userData = BufferDescriptor(output.data(), 0u, sizeof(output));
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{8u, 1u, 2u}, 0u, {false, false, false}, false, 3u};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(computeCode.data()), computeCode, 0, {}},
+        {32u, 0u, userData, compute, std::nullopt, std::nullopt, {}}, device.Target(), {0u, 0u, 0u, 128u}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1u, 1u, 1u);
+    device.WaitIdle();
+    for (std::size_t index = 0; index < output.size(); ++index) {
+        Require(output[index] == index / 8u, "relative stage input dispatch did not preserve the implicit Z input at output " + std::to_string(index)
+            + ": got " + std::to_string(output[index]) + ", expected " + std::to_string(index / 8u));
+    }
+}
+
+template<typename TAction>
+bool Run(const char* name, TAction action) {
+    try {
+        action();
+        std::cout << "relative stage inputs: " << name << " GPU readback passed\n";
+        return true;
+    } catch (const std::exception& error) {
+        std::cerr << "relative stage inputs: " << name << ": " << error.what() << '\n';
+        return false;
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        bool passed = Run("vertex", [&] { Draw(*device, true); });
+        passed &= Run("pixel", [&] { Draw(*device, false); });
+        passed &= Run("compute", [&] { Dispatch(*device); });
+        if (!passed) return 1;
+        std::cout << "relative stage input tests passed\n";
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp

@@ -42,6 +42,8 @@ constexpr std::uint32_t Type2D = 9;
 constexpr std::size_t SurfaceBytes = std::size_t{Side} * Side * 4u;
 constexpr std::size_t Surfaces = 12;
 constexpr std::size_t BlockBytes = SurfaceBytes * Surfaces;
+constexpr std::size_t RegistrationGapBytes = 65536;
+constexpr std::size_t AllocationBytes = BlockBytes + RegistrationGapBytes;
 constexpr std::uint64_t Budget = 1ull << 20u;
 constexpr std::uint64_t Ceiling = Budget + Budget / 4u;
 
@@ -164,9 +166,9 @@ class Block {
 public:
     explicit Block(const Context& context) : context(context) {
 #ifdef _WIN32
-        memory = VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+        memory = VirtualAlloc(nullptr, AllocationBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
 #else
-        memory = std::aligned_alloc(65536, BlockBytes);
+        memory = std::aligned_alloc(65536, AllocationBytes);
 #endif
         Require(memory != nullptr, "cannot allocate the texture block");
         std::memset(memory, 0x5a, BlockBytes);
@@ -449,7 +451,9 @@ void viewEvictionTests(const Context& context) {
     const std::weak_ptr<Texture> view = Sampled(context, imported.Surface(0));
     Require(!view.expired() && TextureCacheUsage().sampledBytes == 0, "the view of a cached storage image did not enter the sampled texture cache at 0 bytes");
     Sampled(context, snapshots.Surface(0));
-    const auto capacity = static_cast<std::size_t>(Budget / TextureCacheUsage().sampledBytes);
+    const auto snapshotBytes = TextureCacheUsage().sampledBytes;
+    Require(snapshotBytes != 0, "the snapshot fixture unexpectedly became a view of a cached storage image");
+    const auto capacity = static_cast<std::size_t>(Budget / snapshotBytes);
     for (std::size_t surface = 1; surface <= capacity; ++surface) Sampled(context, snapshots.Surface(surface));
     Require(!view.expired(), "the sampled texture cache evicted a view of a cached storage image, which frees no memory");
 }

core/libs/prx/libSceAgcDriver/tests/execution/StencilRefExport.cpp

@@ -0,0 +1,159 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <array>
+#include <cstdio>
+#include <functional>
+#include <iostream>
+#include <span>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::uint32_t WaveSize = 64;
+constexpr std::byte Kept{0x40};
+alignas(256) std::array<std::byte, Width * Height * 4> Pixels{};
+alignas(256) std::array<std::byte, 128 * 128 * 4> DepthMemory{};
+alignas(256) std::array<std::byte, 256 * 256> StencilMemory{};
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 4> PlainPixelCode{
+    0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 7> ColumnPixelCode{
+    0x7e040f00, 0x7e0e02f2, 0xf8000082, 0x00000200, 0xf800180f, 0x07070707, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 7> RowPixelCode{
+    0x7e040f01, 0x7e0e02f2, 0xf8000082, 0x00000200, 0xf800180f, 0x07070707, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::array<float, 4>, 3> Vertices{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f},
+}};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+ShaderRecompiler::ShaderPixelStageInfo PixelInfo(std::uint32_t shaderControl) {
+    const AgcDriver::Registers context{
+        {0x1b3, 0x300u}, {0x1b4, 0x300u}, {0x1b6, 0u}, {0x1c5, 9u}, {0x203, shaderControl},
+    };
+    return AgcDriver::Graphics::DecodePixelStageInfo(context, {0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u});
+}
+
+void Draw(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> pixelCode, std::uint32_t shaderControl, const VkStencilOpState& stencil) {
+    Pixels.fill(Kept);
+    const auto target = device.Target();
+
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Vertices.data(), 16u, static_cast<std::uint32_t>(Vertices.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {WaveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    const std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {WaveSize, 0, pixelUserData, std::nullopt, PixelInfo(shaderControl), std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, WaveSize, WaveSize, std::nullopt, std::nullopt};
+    state.color = {reinterpret_cast<std::uintptr_t>(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u};
+    state.colors = {state.color};
+    state.hasColorTarget = true;
+    state.depth = AgcDriver::Graphics::DepthTarget{reinterpret_cast<std::uintptr_t>(DepthMemory.data()), reinterpret_cast<std::uintptr_t>(StencilMemory.data()), {Width, Height}, VK_FORMAT_D32_SFLOAT_S8_UINT, 1.0f, 0};
+    state.stencilTest = true;
+    state.stencilFront = stencil;
+    state.stencilBack = stencil;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {Width, Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blend.colorWriteMask = 15;
+    state.blends = {state.blend};
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Vertices.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+    device.WaitIdle();
+}
+
+void Check(const std::function<bool(std::uint32_t, std::uint32_t)>& drawn, std::string_view what) {
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const auto expected = drawn(x, y) ? std::byte{255} : Kept;
+            const auto actual = Pixels[(y * Width + x) * 4u];
+            Require(actual == expected, std::string(what) + ": pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") is " + std::to_string(static_cast<unsigned>(actual)) + ", expected " + (drawn(x, y) ? "drawn" : "stencil-rejected"));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!TargetHasCapability(device->Target(), spv::CapabilityStencilExportEXT)) {
+            std::puts("skipped, the device lacks VK_EXT_shader_stencil_export");
+            return VulkanTestSkipped;
+        }
+        constexpr std::uint32_t exported = 0x52u;
+        constexpr std::uint32_t plain = 0x50u;
+        const VkStencilOpState replace{VK_STENCIL_OP_KEEP, VK_STENCIL_OP_REPLACE, VK_STENCIL_OP_KEEP, VK_COMPARE_OP_ALWAYS, 0xffu, 0xffu, 0u};
+        const auto equal = [](std::uint32_t reference) { return VkStencilOpState{VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_COMPARE_OP_EQUAL, 0xffu, 0u, reference}; };
+        Draw(*device, ColumnPixelCode, exported, replace);
+        Check([](std::uint32_t, std::uint32_t) { return true; }, "the draw that replaces the stencil with the exported column");
+        for (const auto column : {0u, 5u, 37u, 63u}) {
+            Draw(*device, PlainPixelCode, plain, equal(column));
+            Check([&](std::uint32_t x, std::uint32_t) { return x == column; }, "stencil equal to " + std::to_string(column) + " after the exported column was stored");
+        }
+        Draw(*device, ColumnPixelCode, exported, equal(200u));
+        Check([](std::uint32_t, std::uint32_t) { return true; }, "the exported column tested against the stored column");
+        Draw(*device, RowPixelCode, exported, equal(200u));
+        Check([](std::uint32_t x, std::uint32_t y) { return x == y; }, "the exported row tested against the stored column");
+        Draw(*device, ColumnPixelCode, plain, equal(9u));
+        Check([](std::uint32_t x, std::uint32_t) { return x == 9u; }, "a column export without STENCIL_TEST_VAL_EXPORT_ENABLE");
+        std::puts("stencil reference export tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/SubvectorLoop.cpp

@@ -0,0 +1,209 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <optional>
+#include <span>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 64;
+constexpr std::uint32_t Columns = 9;
+constexpr std::uint32_t Ones = 0xffffffffu;
+constexpr std::uint32_t Marker = 0x5a5a5a5au;
+alignas(256) std::array<std::uint32_t, Threads> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Columns> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 49> ExecKeptCode{
+    0x34020082, 0xe0301000, 0x80000401, 0xbf8c3f70, 0x7e140280, 0x7e160280, 0x7e180280, 0x7e1a0280,
+    0x7e1c0280, 0xbe940380, 0xbea603ff, 0x5a5a5a5a, 0x7daa0880, 0xbda6000a, 0x80148114, 0x4a141481,
+    0x7e160214, 0x7e180226, 0xbe96047e, 0x7e1a0216, 0x7e1c0217, 0xbf098414, 0xbf850001, 0xbe26fff6,
+    0xbe98047e, 0xbefe04c1, 0x7e1e0226, 0x7e200218, 0x7e220219, 0x7e240214, 0xe0701000, 0x80010a01,
+    0xe0701100, 0x80010b01, 0xe0701200, 0x80010c01, 0xe0701300, 0x80010d01, 0xe0701400, 0x80010e01,
+    0xe0701500, 0x80010f01, 0xe0701600, 0x80011001, 0xe0701700, 0x80011101, 0xe0701800, 0x80011201,
+    0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 50> ExecResetCode{
+    0x34020082, 0xe0301000, 0x80000401, 0xbf8c3f70, 0x7e140280, 0x7e160280, 0x7e180280, 0x7e1a0280,
+    0x7e1c0280, 0xbe940380, 0xbea603ff, 0x5a5a5a5a, 0x7daa0880, 0xbda6000b, 0x80148114, 0x4a141481,
+    0x7e160214, 0x7e180226, 0xbe96047e, 0x7e1a0216, 0x7e1c0217, 0xbf098414, 0xbf850002, 0xbefe04c1,
+    0xbe26fff5, 0xbe98047e, 0xbefe04c1, 0x7e1e0226, 0x7e200218, 0x7e220219, 0x7e240214, 0xe0701000,
+    0x80010a01, 0xe0701100, 0x80010b01, 0xe0701200, 0x80010c01, 0xe0701300, 0x80010d01, 0xe0701400,
+    0x80010e01, 0xe0701500, 0x80010f01, 0xe0701600, 0x80011001, 0xe0701700, 0x80011101, 0xe0701800,
+    0x80011201, 0xbf810000,
+};
+
+constexpr std::array<const char*, Columns> Names{"passes", "last pass", "s38 in the pass", "exec lo in the pass", "exec hi in the pass", "s38 after the loop", "exec lo after the loop", "exec hi after the loop", "total passes"};
+
+struct Group {
+    std::uint32_t first;
+    std::uint32_t last;
+    std::array<std::uint32_t, Columns> values;
+};
+
+struct Case {
+    const char* name;
+    std::uint64_t mask;
+    std::vector<Group> groups;
+};
+
+const std::vector<Case> ExecKeptCases{
+    {"all lanes", ~0ull, {
+        {0, 31, {1, 1, Ones, Ones, 0, Ones, Ones, Ones, 2}},
+        {32, 63, {1, 2, Ones, 0, Ones, Ones, Ones, Ones, 2}},
+    }},
+    {"low half", 0xffffffffull, {
+        {0, 31, {1, 1, 0, Ones, 0, 0, Ones, 0, 1}},
+        {32, 63, {0, 0, 0, 0, 0, 0, Ones, 0, 1}},
+    }},
+    {"high half", 0xffffffffull << 32, {
+        {0, 31, {0, 0, 0, 0, 0, 0, 0, Ones, 1}},
+        {32, 63, {1, 1, 0, 0, Ones, 0, 0, Ones, 1}},
+    }},
+    {"no lane", 0ull, {
+        {0, 63, {0, 0, 0, 0, 0, Marker, 0, 0, 0}},
+    }},
+    {"lanes 3 and 40", (1ull << 3) | (1ull << 40), {
+        {0, 63, {0, 0, 0, 0, 0, 8, 8, 0x100u, 2}},
+        {3, 3, {1, 1, 0x100u, 8, 0, 8, 8, 0x100u, 2}},
+        {40, 40, {1, 2, 8, 0, 0x100u, 8, 8, 0x100u, 2}},
+    }},
+    {"lane 3", 1ull << 3, {
+        {0, 63, {0, 0, 0, 0, 0, 0, 8, 0, 1}},
+        {3, 3, {1, 1, 0, 8, 0, 0, 8, 0, 1}},
+    }},
+    {"lane 40", 1ull << 40, {
+        {0, 63, {0, 0, 0, 0, 0, 0, 0, 0x100u, 1}},
+        {40, 40, {1, 1, 0, 0, 0x100u, 0, 0, 0x100u, 1}},
+    }},
+};
+
+const std::vector<Case> ExecResetCases{
+    {"all lanes", ~0ull, {
+        {0, 63, {0, 0, 0, 0, 0, Ones, Ones, Ones, 1}},
+        {0, 31, {1, 1, Ones, Ones, 0, Ones, Ones, Ones, 1}},
+    }},
+    {"low half", 0xffffffffull, {
+        {0, 63, {0, 0, 0, 0, 0, 0, 0, Ones, 1}},
+        {0, 31, {1, 1, 0, Ones, 0, 0, 0, Ones, 1}},
+    }},
+    {"high half", 0xffffffffull << 32, {
+        {0, 63, {0, 0, 0, 0, 0, 0, 0, Ones, 1}},
+        {32, 63, {1, 1, 0, 0, Ones, 0, 0, Ones, 1}},
+    }},
+    {"no lane", 0ull, {
+        {0, 63, {0, 0, 0, 0, 0, Marker, 0, 0, 0}},
+    }},
+    {"lanes 3 and 40", (1ull << 3) | (1ull << 40), {
+        {0, 63, {0, 0, 0, 0, 0, 0x100u, 0x100u, Ones, 1}},
+        {3, 3, {1, 1, 0x100u, 8, 0, 0x100u, 0x100u, Ones, 1}},
+    }},
+    {"lane 3", 1ull << 3, {
+        {0, 63, {0, 0, 0, 0, 0, 0, 0, Ones, 1}},
+        {3, 3, {1, 1, 0, 8, 0, 0, 0, Ones, 1}},
+    }},
+    {"lane 40", 1ull << 40, {
+        {0, 63, {0, 0, 0, 0, 0, 0, 0, Ones, 1}},
+        {40, 40, {1, 1, 0, 0, 0x100u, 0, 0, Ones, 1}},
+    }},
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::uint64_t mask) {
+    for (std::uint32_t lane = 0; lane < Threads; ++lane) Input[lane] = ((mask >> lane) & 1u) != 0u ? 1u : 0u;
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {64, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check(const char* variant, const Case& testCase) {
+    std::array<std::array<std::uint32_t, Columns>, Threads> expected{};
+    for (const auto& group : testCase.groups) {
+        for (std::uint32_t lane = group.first; lane <= group.last; ++lane) expected[lane] = group.values;
+    }
+    for (std::uint32_t lane = 0; lane < Threads; ++lane) {
+        for (std::uint32_t column = 0; column < Columns; ++column) {
+            const auto actual = Output[column * Threads + lane];
+            Require(actual == expected[lane][column], std::string("subvector loop ") + variant + ", initial exec " + testCase.name + ": lane " + std::to_string(lane) + " " + Names[column] + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[lane][column]));
+        }
+    }
+}
+
+void CheckWave32Throws(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code) {
+    std::vector<std::uint32_t> userData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{32, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.ComputeTarget(32),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    try {
+        static_cast<void>(ShaderRecompiler::Recompile(request));
+    } catch (const std::runtime_error& error) {
+        Require(std::string(error.what()).find("s_subvector_loop_begin in a wave32 shader is not measured") != std::string::npos, std::string("wave32 subvector loop threw an unexpected error: ") + error.what());
+        return;
+    }
+    Require(false, "a wave32 subvector loop was recompiled");
+}
+
+void RunCases(AgcDriver::VulkanDevice& device, const char* variant, std::span<const std::uint32_t> code, const std::vector<Case>& cases) {
+    for (const auto& testCase : cases) {
+        Run(device, code, testCase.mask);
+        Check(variant, testCase);
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        CheckWave32Throws(*device, ExecKeptCode);
+        if (device->Target().subgroupSize < 32u) {
+            std::printf("skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize);
+            return VulkanTestSkipped;
+        }
+        RunCases(*device, "exec kept by the body", ExecKeptCode, ExecKeptCases);
+        RunCases(*device, "body resets exec", ExecResetCode, ExecResetCases);
+        std::puts("subvector loop tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}