Merge remote-tracking branch 'upstream/main' into feat/ampr-nop-04-writes

# Conflicts:
#	docs/dev/TechnicalDebt.md

core/Decoder/Png/CMakeLists.txt

@@ -16,6 +16,7 @@ function(add_png_decoder target runtimeTarget)
 
     if(BUILD_TESTING)
         add_test_executable(${target}_tests ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/tests/Png.cpp)
+        target_include_directories(${target}_tests SYSTEM PRIVATE ${PROJECT_SOURCE_DIR}/3rdparty/stb)
         target_link_libraries(${target}_tests PRIVATE ${target})
         configure_windows_unwind(${target}_tests)
         add_test(NAME ${target} COMMAND ${target}_tests)

core/Decoder/Png/include/Decoder/Png.hpp

@@ -35,9 +35,16 @@ std::optional<Header> ParseHeader(std::span<const std::uint8_t> png);
 
 std::optional<Image> Decode(std::span<const std::uint8_t> png);
 
+inline constexpr std::uint8_t FILTER_NONE = 1 << 0;
+inline constexpr std::uint8_t FILTER_SUB = 1 << 1;
+inline constexpr std::uint8_t FILTER_UP = 1 << 2;
+inline constexpr std::uint8_t FILTER_AVERAGE = 1 << 3;
+inline constexpr std::uint8_t FILTER_PAETH = 1 << 4;
+inline constexpr std::uint8_t FILTER_ALL = FILTER_NONE | FILTER_SUB | FILTER_UP | FILTER_AVERAGE | FILTER_PAETH;
+
 struct EncodeOptions {
     int compressionLevel = 8;
-    int filter = -1;
+    std::uint8_t filters = FILTER_ALL;
 };
 
 std::vector<std::uint8_t> Encode(std::span<const std::uint8_t> pixels, std::uint32_t width, std::uint32_t height,

core/Decoder/Png/src/Png.cpp

@@ -4,7 +4,8 @@
 #include <array>
 #include <climits>
 #include <cstddef>
-#include <mutex>
+#include <cstdlib>
+#include <memory>
 #include <stdexcept>
 
 #define STB_IMAGE_WRITE_STATIC
@@ -26,6 +27,17 @@ constexpr std::array<std::uint8_t, 8> SIGNATURE = {0x89, 0x50, 0x4E, 0x47, 0x0D,
 constexpr std::size_t CHUNK_OVERHEAD = 12;
 constexpr std::size_t IHDR_SIZE = 13;
 constexpr std::uint32_t MAX_DIMENSION = 0x7FFFFFFF;
+constexpr std::uint8_t FILTER_TYPE_COUNT = 5;
+constexpr std::array<std::uint8_t, 5> COLOR_TYPE_BY_CHANNELS = {0, 0, 4, 2, 6};
+constexpr std::array<std::uint32_t, 256> CRC_TABLE = [] {
+    std::array<std::uint32_t, 256> table{};
+    for (std::uint32_t index = 0; index < table.size(); ++index) {
+        std::uint32_t value = index;
+        for (int bit = 0; bit < 8; ++bit) value = (value >> 1) ^ ((value & 1) != 0 ? 0xEDB88320u : 0u);
+        table[index] = value;
+    }
+    return table;
+}();
 
 std::uint32_t readBigEndian32(const std::uint8_t* bytes) {
     return static_cast<std::uint32_t>(bytes[0]) << 24 | static_cast<std::uint32_t>(bytes[1]) << 16
@@ -51,10 +63,57 @@ bool isValidFormat(std::uint8_t bitDepth, std::uint8_t colorType) {
     }
 }
 
-void appendBytes(void* context, void* data, int size) {
-    auto* output = static_cast<std::vector<std::uint8_t>*>(context);
-    const auto* bytes = static_cast<const std::uint8_t*>(data);
-    output->insert(output->end(), bytes, bytes + size);
+void appendBigEndian32(std::vector<std::uint8_t>& output, std::uint32_t value) {
+    for (int shift = 24; shift >= 0; shift -= 8) output.push_back(static_cast<std::uint8_t>(value >> shift));
+}
+
+void appendChunk(std::vector<std::uint8_t>& output, const char* type, std::span<const std::uint8_t> data) {
+    appendBigEndian32(output, static_cast<std::uint32_t>(data.size()));
+    const std::size_t checkedFrom = output.size();
+    output.insert(output.end(), type, type + 4);
+    output.insert(output.end(), data.begin(), data.end());
+    std::uint32_t crc = 0xFFFFFFFFu;
+    for (std::size_t i = checkedFrom; i < output.size(); ++i) crc = (crc >> 8) ^ CRC_TABLE[(crc ^ output[i]) & 0xFF];
+    appendBigEndian32(output, ~crc);
+}
+
+int paethPredictor(int left, int up, int upLeft) {
+    const int estimate = left + up - upLeft;
+    const int leftDistance = std::abs(estimate - left);
+    const int upDistance = std::abs(estimate - up);
+    const int upLeftDistance = std::abs(estimate - upLeft);
+    if (leftDistance <= upDistance && leftDistance <= upLeftDistance) return left;
+    return upDistance <= upLeftDistance ? up : upLeft;
+}
+
+std::uint64_t filterRow(const std::uint8_t* row, const std::uint8_t* previousRow, std::size_t rowSize, std::size_t pixelSize,
+                        std::uint8_t filterType, std::uint8_t* filtered) {
+    std::uint64_t cost = 0;
+    for (std::size_t i = 0; i < rowSize; ++i) {
+        const int left = i >= pixelSize ? row[i - pixelSize] : 0;
+        const int up = previousRow ? previousRow[i] : 0;
+        const int upLeft = previousRow && i >= pixelSize ? previousRow[i - pixelSize] : 0;
+        int prediction = 0;
+        switch (filterType) {
+        case 1:
+            prediction = left;
+            break;
+        case 2:
+            prediction = up;
+            break;
+        case 3:
+            prediction = (left + up) >> 1;
+            break;
+        case 4:
+            prediction = paethPredictor(left, up, upLeft);
+            break;
+        default:
+            break;
+        }
+        filtered[i] = static_cast<std::uint8_t>(row[i] - prediction);
+        cost += filtered[i] < 0x80 ? filtered[i] : 0x100u - filtered[i];
+    }
+    return cost;
 }
 
 }  // namespace
@@ -105,25 +164,50 @@ std::optional<Image> Decode(std::span<const std::uint8_t> png) {
 std::vector<std::uint8_t> Encode(std::span<const std::uint8_t> pixels, std::uint32_t width, std::uint32_t height,
                                  std::uint32_t channels, EncodeOptions options) {
     if (channels < 1 || channels > 4) throw std::invalid_argument("Png::Encode: channels must be 1-4");
-    if (options.compressionLevel < 0 || options.compressionLevel > 9 || options.filter < -1 || options.filter > 4) {
+    if (options.compressionLevel < 0 || options.compressionLevel > 9 || options.filters == 0 || (options.filters & ~FILTER_ALL) != 0) {
         throw std::invalid_argument("Png::Encode: unsupported options");
     }
-    if (width == 0 || height == 0 || static_cast<std::uint64_t>(width) * channels > INT_MAX || height > INT_MAX) {
+    const std::uint64_t rowSize = static_cast<std::uint64_t>(width) * channels;
+    if (width == 0 || height == 0 || rowSize > INT_MAX || (rowSize + 1) * height > INT_MAX) {
         throw std::invalid_argument("Png::Encode: unsupported image size");
     }
-    if (pixels.size() < static_cast<std::size_t>(width) * height * channels) {
+    if (pixels.size() < rowSize * height) {
         throw std::invalid_argument("Png::Encode: pixel buffer is too small");
     }
 
-    std::vector<std::uint8_t> output;
-    const int stride = static_cast<int>(width * channels);
-    static std::mutex optionsMutex;
-    std::lock_guard lock(optionsMutex);
-    stbi_write_png_compression_level = options.compressionLevel;
-    stbi_write_force_png_filter = options.filter;
-    const int written = stbi_write_png_to_func(appendBytes, &output, static_cast<int>(width), static_cast<int>(height),
-                                               static_cast<int>(channels), pixels.data(), stride);
-    if (written == 0) throw std::runtime_error("Png::Encode: encoding failed");
+    std::vector<std::uint8_t> filtered((rowSize + 1) * height);
+    std::vector<std::uint8_t> candidate(rowSize);
+    for (std::uint32_t y = 0; y < height; ++y) {
+        const std::uint8_t* row = pixels.data() + rowSize * y;
+        const std::uint8_t* previousRow = y == 0 ? nullptr : row - rowSize;
+        std::uint8_t* filteredRow = filtered.data() + (rowSize + 1) * y;
+        std::uint64_t bestCost = UINT64_MAX;
+        for (std::uint8_t filterType = 0; filterType < FILTER_TYPE_COUNT; ++filterType) {
+            if ((options.filters & (1u << filterType)) == 0) continue;
+            const std::uint64_t cost =filterRow(row, previousRow, rowSize, channels, filterType, candidate.data());
+            if (cost >= bestCost) continue;
+            bestCost = cost;
+            filteredRow[0] = filterType;
+            std::copy(candidate.begin(), candidate.end(), filteredRow + 1);
+        }
+    }
+
+    int compressedSize = 0;
+    const std::unique_ptr<unsigned char, void (*)(unsigned char*)> compressed(
+        stbi_zlib_compress(filtered.data(), static_cast<int>(filtered.size()), &compressedSize, options.compressionLevel),
+        [](unsigned char* memory) { STBIW_FREE(memory); });
+    if (!compressed) throw std::runtime_error("Png::Encode: encoding failed");
+
+    std::vector<std::uint8_t> header;
+    appendBigEndian32(header, width);
+    appendBigEndian32(header, height);
+    header.insert(header.end(), {8, COLOR_TYPE_BY_CHANNELS[channels], 0, 0, 0});
+
+    std::vector<std::uint8_t> output(SIGNATURE.begin(), SIGNATURE.end());
+    output.reserve(SIGNATURE.size() + 3 * CHUNK_OVERHEAD + IHDR_SIZE + static_cast<std::size_t>(compressedSize));
+    appendChunk(output, "IHDR", header);
+    appendChunk(output, "IDAT", std::span<const std::uint8_t>(compressed.get(), static_cast<std::size_t>(compressedSize)));
+    appendChunk(output, "IEND", {});
     return output;
 }
 

core/Decoder/Png/tests/Png.cpp

@@ -1,10 +1,17 @@
 #include "Decoder/Png.hpp"
 
+#include <algorithm>
 #include <cstdint>
 #include <cstdlib>
 #include <stdexcept>
 #include <vector>
 
+#define STB_IMAGE_STATIC
+#define STB_IMAGE_IMPLEMENTATION
+#define STBI_ONLY_PNG
+#define STBI_NO_STDIO
+#include "stb_image.h"
+
 static void Require(bool value) { if (!value) std::abort(); }
 
 template<typename TFunction>
@@ -84,11 +91,63 @@ static void RequireRoundTrip(std::uint32_t channels, Decoder::Png::ColorType col
     }
 }
 
+static std::vector<std::uint8_t> RowFilters(const std::vector<std::uint8_t>& png, std::uint32_t height, std::uint32_t rowSize) {
+    std::vector<std::uint8_t> stream;
+    for (std::size_t offset = 8; offset + 12 <= png.size();) {
+        const std::size_t length = static_cast<std::size_t>(png[offset]) << 24 | static_cast<std::size_t>(png[offset + 1]) << 16
+            | static_cast<std::size_t>(png[offset + 2]) << 8 | png[offset + 3];
+        Require(length <= png.size() - offset - 12);
+        const std::uint8_t* data = png.data() + offset + 8;
+        if (std::equal(data - 4, data, "IDAT")) stream.insert(stream.end(), data, data + length);
+        offset += 12 + length;
+    }
+
+    int size = 0;
+    char* rows = stbi_zlib_decode_malloc(reinterpret_cast<const char*>(stream.data()), static_cast<int>(stream.size()), &size);
+    Require(rows && static_cast<std::size_t>(size) == static_cast<std::size_t>(height) * (rowSize + 1));
+    std::vector<std::uint8_t> filters;
+    for (std::uint32_t y = 0; y < height; ++y) filters.push_back(static_cast<std::uint8_t>(rows[y * (rowSize + 1)]));
+    stbi_image_free(rows);
+    return filters;
+}
+
+static void RequireFilterSets(std::uint32_t channels) {
+    constexpr std::uint32_t width = 16;
+    constexpr std::uint32_t height = 6;
+    const std::uint32_t rowSize = width * channels;
+    std::vector<std::uint8_t> pixels(rowSize * height);
+    std::uint32_t noise = 12345;
+    for (std::uint32_t y = 0; y < height; ++y) {
+        for (std::uint32_t i = 0; i < rowSize; ++i) {
+            noise = noise * 1103515245u + 12345u;
+            pixels[y * rowSize + i] = y == 2 ? static_cast<std::uint8_t>(noise >> 16) : static_cast<std::uint8_t>(i / channels * 8);
+        }
+    }
+    std::copy_n(pixels.begin() + 2 * rowSize, rowSize, pixels.begin() + 3 * rowSize);
+
+    for (std::uint8_t filters = 1; filters <= Decoder::Png::FILTER_ALL; ++filters) {
+        const std::vector<std::uint8_t> png = Decoder::Png::Encode(pixels, width, height, channels, {6, filters});
+        for (const std::uint8_t filter : RowFilters(png, height, rowSize)) Require(filter < 5 && (filters >> filter & 1) != 0);
+
+        const auto image = Decoder::Png::Decode(png);
+        Require(image.has_value() && image->width == width && image->height == height);
+        for (std::size_t pixel = 0; pixel < width * height; ++pixel) {
+            Require(image->pixels[pixel * 4] == pixels[pixel * channels]);
+            Require(image->pixels[pixel * 4 + 3] == (channels % 2 == 0 ? pixels[pixel * channels + channels - 1] : 255));
+        }
+    }
+
+    const std::vector<std::uint8_t> subOrUp = RowFilters(
+        Decoder::Png::Encode(pixels, width, height, channels, {6, Decoder::Png::FILTER_SUB | Decoder::Png::FILTER_UP}), height, rowSize);
+    Require(subOrUp[0] == 1 && subOrUp[1] == 2 && subOrUp[3] == 2 && subOrUp[4] == 1 && subOrUp[5] == 2);
+}
+
 int main() {
     RequireRoundTrip(1, Decoder::Png::ColorType::Grayscale);
     RequireRoundTrip(2, Decoder::Png::ColorType::GrayscaleAlpha);
     RequireRoundTrip(3, Decoder::Png::ColorType::Rgb);
     RequireRoundTrip(4, Decoder::Png::ColorType::Rgba);
+    for (const std::uint32_t channels : {1, 2, 3, 4}) RequireFilterSets(channels);
 
     const auto paletteHeader = Decoder::Png::ParseHeader(PALETTE_TRNS);
     Require(paletteHeader.has_value());
@@ -140,13 +199,14 @@ int main() {
     Require(ThrowsInvalidArgument([&] { Decoder::Png::Encode(pixels, 4, 4, 5); }));
     Require(ThrowsInvalidArgument([&] { Decoder::Png::Encode(pixels, 0, 4, 4); }));
     Require(ThrowsInvalidArgument([&] { Decoder::Png::Encode(pixels, 4, 5, 4); }));
-    Require(ThrowsInvalidArgument([&] { Decoder::Png::Encode(pixels, 4, 4, 4, {10, -1}); }));
-    Require(ThrowsInvalidArgument([&] { Decoder::Png::Encode(pixels, 4, 4, 4, {8, 5}); }));
+    Require(ThrowsInvalidArgument([&] { Decoder::Png::Encode(pixels, 4, 4, 4, {10, Decoder::Png::FILTER_ALL}); }));
+    Require(ThrowsInvalidArgument([&] { Decoder::Png::Encode(pixels, 4, 4, 4, {8, 0}); }));
+    Require(ThrowsInvalidArgument([&] { Decoder::Png::Encode(pixels, 4, 4, 4, {8, Decoder::Png::FILTER_ALL + 1}); }));
 
     std::vector<std::uint8_t> gradient(64 * 64 * 3);
     for (std::size_t i = 0; i < gradient.size(); ++i) gradient[i] = static_cast<std::uint8_t>(i / 3 % 64 * 4);
-    const auto stored = Decoder::Png::Encode(gradient, 64, 64, 3, {0, 0});
-    const auto compressed = Decoder::Png::Encode(gradient, 64, 64, 3, {9, 1});
+    const auto stored = Decoder::Png::Encode(gradient, 64, 64, 3, {0, Decoder::Png::FILTER_NONE});
+    const auto compressed = Decoder::Png::Encode(gradient, 64, 64, 3, {9, Decoder::Png::FILTER_SUB});
     Require(compressed.size() < stored.size());
     for (const auto& png : {stored, compressed}) {
         const auto image = Decoder::Png::Decode(png);

core/libs/CMakeLists.txt

@@ -170,6 +170,16 @@ if(BUILD_TESTING)
     add_test(NAME host_thread_local COMMAND host_thread_local_tests)
     set_tests_properties(host_thread_local PROPERTIES TIMEOUT 15)
 
+    add_test_executable(guest_np_cpp_web_api_intrusive_ptr_tests tests/GuestNpCppWebApiIntrusivePtr.cpp)
+    target_include_directories(guest_np_cpp_web_api_intrusive_ptr_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_np_cpp_web_api_intrusive_ptr_tests PRIVATE libSceNpCppWebApi libc)
+    configure_windows_unwind(guest_np_cpp_web_api_intrusive_ptr_tests)
+    add_test(NAME guest_np_cpp_web_api_intrusive_ptr COMMAND guest_np_cpp_web_api_intrusive_ptr_tests)
+    if(WIN32)
+        set_tests_properties(guest_np_cpp_web_api_intrusive_ptr PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpCppWebApi>")
+    endif()
+
     add_test_executable(savedata_core_write_replacement_tests tests/SaveDataWriteReplacement.cpp)
     target_include_directories(savedata_core_write_replacement_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(savedata_core_write_replacement_tests PRIVATE libSceSaveData)

core/libs/SceTypes.hpp

@@ -389,6 +389,18 @@ struct AjmDecAt9ConfigDataInfo {
     std::uint32_t superframe_size;
 };
 
+struct AjmDecMp3ParseFrame {
+    std::uint64_t frame_size;
+    std::uint32_t num_channels;
+    std::uint32_t samples_per_channel;
+    std::uint32_t bitrate;
+    std::uint32_t sample_rate;
+    std::uint32_t encoder_delay;
+    std::uint32_t num_frames;
+    std::uint32_t total_samples;
+    std::uint32_t ofl_type;
+};
+
 using AudioOut2ContextHandle = std::uint64_t;
 using AudioOut2PortHandle = std::uint64_t;
 using AudioOut2UserHandle = std::uintptr_t;

core/libs/prx/libSceAgc/Acb/src/Memory.cpp

@@ -47,14 +47,7 @@ std::uint32_t APS5_VABI sceAgcAcbAtomicGdsGetSize() {
 }
 
 std::uint32_t* APS5_VABI sceAgcAcbPrimeUtcl2(CommandBuffer* buf, const volatile void* address, std::uint32_t sizeInBytes) {
-    const auto guestAddress = reinterpret_cast<std::uintptr_t>(address);
-    auto* packet = Agc::Command::Allocate(buf, 5, __func__);
-    packet[0] = Agc::Command::Header(0x10u, 5);
-    packet[1] = 0;
-    packet[2] = static_cast<std::uint32_t>(guestAddress);
-    packet[3] = static_cast<std::uint32_t>(guestAddress >> 32u);
-    packet[4] = sizeInBytes;
-    return packet;
+    return Agc::Command::WritePrimeUtcl2(buf, address, sizeInBytes, __func__);
 }
 
 std::uint32_t APS5_VABI sceAgcAcbPrimeUtcl2GetSize() {

core/libs/prx/libSceAgc/CMakeLists.txt

@@ -30,6 +30,7 @@ add_library(libSceAgc SHARED EXCLUDE_FROM_ALL
         ${CMAKE_CURRENT_SOURCE_DIR}/DcbState/src/Registers.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/DcbState/src/Workload.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Misc/src/PacketInfo.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Misc/src/Platform.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Misc/src/RegisterDefaults.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Misc/src/ShaderFusion.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Misc/src/Suspend.cpp
@@ -69,9 +70,23 @@ target_link_libraries(agc_command_tests PRIVATE libSceAgc libc)
 configure_windows_unwind(agc_command_tests)
 configure_windows_unwind(libSceAgc)
 
+add_test_executable(agc_platform_tests EXCLUDE_FROM_ALL tests/Platform.cpp)
+target_include_directories(agc_platform_tests PRIVATE ${LIBS_INCLUDE_DIR})
+target_link_libraries(agc_platform_tests PRIVATE libSceAgc libc)
+configure_windows_unwind(agc_platform_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_platform_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
+    add_test(NAME agc_platform_tests COMMAND agc_platform_tests)
+    if(WIN32)
+        set_tests_properties(agc_platform_tests PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_register_command_tests EXCLUDE_FROM_ALL
         tests/RegisterCommands.cpp
         Command/src/Packet.cpp
+        Cb/src/Registers.cpp
         DcbState/src/Registers.cpp
         Patch/src/IndirectRegisters.cpp
 )
@@ -120,6 +135,21 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_interpolant_mapping_tests EXCLUDE_FROM_ALL tests/InterpolantMapping.cpp)
+target_include_directories(agc_interpolant_mapping_tests PRIVATE ${LIBS_INCLUDE_DIR})
+target_link_libraries(agc_interpolant_mapping_tests PRIVATE libSceAgc libc)
+set_target_properties(agc_interpolant_mapping_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_interpolant_mapping_tests)
+
+if(BUILD_TESTING)
+    set_target_properties(agc_interpolant_mapping_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_interpolant_mapping COMMAND agc_interpolant_mapping_tests)
+    if(WIN32)
+        set_tests_properties(agc_interpolant_mapping PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 if(BUILD_TESTING)
     add_test_executable(guest_shader_link_tests ${LIBS_INCLUDE_DIR}/tests/GuestShaderLink.cpp)
     target_include_directories(guest_shader_link_tests PRIVATE ${LIBS_INCLUDE_DIR})
@@ -150,3 +180,23 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
+
+add_test_executable(agc_prime_utcl2_tests EXCLUDE_FROM_ALL
+        tests/PrimeUtcl2.cpp
+        Command/src/Packet.cpp
+        Command/src/Memory.cpp
+        Acb/src/Memory.cpp
+        DcbFlow/src/Memory.cpp
+        DcbState/src/Display.cpp
+)
+target_include_directories(agc_prime_utcl2_tests PRIVATE ${LIBS_INCLUDE_DIR})
+target_link_libraries(agc_prime_utcl2_tests PRIVATE libc)
+configure_windows_unwind(agc_prime_utcl2_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_prime_utcl2_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
+    add_test(NAME agc_prime_utcl2_tests COMMAND agc_prime_utcl2_tests)
+    if(WIN32)
+        set_tests_properties(agc_prime_utcl2_tests PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()

core/libs/prx/libSceAgc/Cb/src/Registers.cpp

@@ -6,6 +6,16 @@
 #include "SceTypes.hpp"
 #include "prx/libc/include/General.hpp"
 
+namespace {
+
+std::uint32_t RegisterListSize(std::uint32_t numRegs, const char* function) {
+    constexpr std::uint32_t bytesPerRegister = 3u * sizeof(std::uint32_t);
+    Agc::Command::Require(numRegs <= 0xffffffffu / bytesPerRegister, function, "register list size overflow");
+    return numRegs * bytesPerRegister;
+}
+
+}
+
 extern "C" {
 
 std::uint32_t* APS5_VABI sceAgcCbSetUcRegisterRangeDirect(CommandBuffer* buf, std::uint32_t offset, const std::uint32_t* values, std::uint32_t numValues) {
@@ -29,9 +39,7 @@ std::uint32_t* APS5_VABI sceAgcCbSetShRegistersDirect(CommandBuffer* buf, const
 }
 
 std::uint32_t APS5_VABI sceAgcCbSetShRegistersDirectGetSize(std::uint32_t numRegs) {
-    (void)numRegs;
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+    return RegisterListSize(numRegs, __func__);
 }
 
 std::uint32_t* APS5_VABI sceAgcCbSetUcRegistersDirect(CommandBuffer* buf, const volatile ShaderRegister* regs, std::uint32_t numRegs) {
@@ -39,9 +47,7 @@ std::uint32_t* APS5_VABI sceAgcCbSetUcRegistersDirect(CommandBuffer* buf, const
 }
 
 std::uint32_t APS5_VABI sceAgcCbSetUcRegistersDirectGetSize(std::uint32_t numRegs) {
-    (void)numRegs;
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+    return RegisterListSize(numRegs, __func__);
 }
 
 }

core/libs/prx/libSceAgc/Command/include/Memory.hpp

@@ -8,6 +8,7 @@ namespace Agc::Command {
 std::uint32_t* WriteDma(CommandBuffer* buffer, bool compute, std::uint8_t engine, std::uint8_t dst, std::uint8_t dstCachePolicy, std::uint64_t dstAddress, std::uint8_t src, std::uint8_t srcCachePolicy, std::uint64_t srcAddress, std::uint32_t numBytes, std::uint8_t waitForPrevious, std::uint8_t writeConfirm, std::uint8_t blockEngine, const char* function);
 std::uint32_t* WriteCopyData(CommandBuffer* buffer, bool compute, std::uint8_t dst, std::uint8_t dstCachePolicy, std::uint64_t dstAddress, std::uint8_t src, std::uint8_t srcCachePolicy, std::uint64_t srcAddressOrImmediate, std::uint8_t itemSize, std::uint8_t writeConfirm, const char* function);
 std::uint32_t* WriteAtomicMem(CommandBuffer* buffer, std::uint8_t atomicOp, std::uint8_t command, std::uint8_t cachePolicy, const volatile void* address, std::uint64_t srcData, std::uint64_t compareData, std::uint16_t loopInterval, const char* function);
+std::uint32_t* WritePrimeUtcl2(CommandBuffer* buffer, const volatile void* address, std::uint32_t sizeInBytes, const char* function);
 std::uint32_t* WriteData(CommandBuffer* buffer, bool compute, std::uint8_t dst, std::uint8_t cachePolicy, std::uint64_t address, const void* data, std::uint32_t count, std::uint8_t increment, std::uint8_t writeConfirm, const char* function);
 std::uint32_t* WriteWait(CommandBuffer* buffer, std::uint8_t size, std::uint8_t compareFunction, std::uint8_t operation, std::uint8_t cachePolicy, const volatile void* address, std::uint64_t reference, std::uint64_t mask, std::uint32_t pollCycles, const char* function);
 std::uint32_t* ValidateWait(std::uint32_t* packet, const char* function);

core/libs/prx/libSceAgc/Command/src/Memory.cpp

@@ -49,6 +49,11 @@ std::uint32_t* WriteAtomicMem(CommandBuffer* buffer, std::uint8_t atomicOp, std:
     return Emit(buffer, 0x1eu, {atomicOp | (static_cast<std::uint32_t>(command) << 8u) | (static_cast<std::uint32_t>(cachePolicy) << 25u), static_cast<std::uint32_t>(guestAddress), static_cast<std::uint32_t>(guestAddress >> 32u), static_cast<std::uint32_t>(srcData), static_cast<std::uint32_t>(srcData >> 32u), static_cast<std::uint32_t>(compareData), static_cast<std::uint32_t>(compareData >> 32u), loopInterval}, function);
 }
 
+std::uint32_t* WritePrimeUtcl2(CommandBuffer* buffer, const volatile void* address, std::uint32_t sizeInBytes, const char* function) {
+    const auto guestAddress = reinterpret_cast<std::uintptr_t>(address);
+    return Emit(buffer, 0x10u, {0, static_cast<std::uint32_t>(guestAddress), static_cast<std::uint32_t>(guestAddress >> 32u), sizeInBytes}, function);
+}
+
 std::uint32_t* WriteData(CommandBuffer* buffer, bool compute, std::uint8_t dst, std::uint8_t cachePolicy, std::uint64_t address, const void* data, std::uint32_t count, std::uint8_t increment, std::uint8_t writeConfirm, const char* function) {
     Require(data != nullptr && count != 0 && count <= 0x3ffdu, function, "invalid write data payload");
     CheckBits(dst, compute ? 0xfu : 0x1fu, function);

core/libs/prx/libSceAgc/DcbState/src/Display.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgc/DcbState/include/Display.hpp"
 
+#include "prx/libSceAgc/Command/include/Memory.hpp"
 #include "prx/libSceAgc/Command/include/Packet.hpp"
 #include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp"
 #include <cstdint>
@@ -21,16 +22,11 @@ uint32_t* APS5_VABI sceAgcDcbSetFlip(CommandBuffer* buf, uint32_t video_out_hand
 }
 
 uint32_t* APS5_VABI sceAgcDcbPrimeUtcl2(CommandBuffer* buf, const volatile void* address, uint32_t size_in_bytes) {
-    (void)buf;
-    (void)address;
-    (void)size_in_bytes;
-    NotImplemented_nid_no_patch(__func__);
-    return nullptr;
+    return Agc::Command::WritePrimeUtcl2(buf, address, size_in_bytes, __func__);
 }
 
 std::uint32_t APS5_VABI sceAgcDcbPrimeUtcl2GetSize() {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+    return 20;
 }
 
 }

core/libs/prx/libSceAgc/Misc/include/Platform.hpp

@@ -0,0 +1,4 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGC_MISC_INCLUDE_PLATFORM_HPP
+#define CORE_LIBS_PRX_LIBSCEAGC_MISC_INCLUDE_PLATFORM_HPP
+
+#endif

core/libs/prx/libSceAgc/Misc/src/Platform.cpp

@@ -0,0 +1,17 @@
+#include "prx/libSceAgc/Misc/include/Platform.hpp"
+#include "prx/libSceAgc/Command/include/Packet.hpp"
+
+#include <cstdint>
+#include <cstddef>
+#include "SceTypes.hpp"
+#include "prx/libc/include/General.hpp"
+
+extern "C" {
+
+int APS5_VABI sceAgcGetIsTrinityMode(bool* isTrinityMode) {
+    Agc::Command::CheckAddress(reinterpret_cast<std::uintptr_t>(isTrinityMode), alignof(bool), __func__);
+    *isTrinityMode = false;
+    return 0;
+}
+
+}

core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp

@@ -8,11 +8,48 @@
 #include "prx/libSceAgc/Shader/include/ShaderUtils.hpp"
 #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp"
 
-extern "C" {
+namespace {
 
-APS5_EXPORT("HV4j+E0MBHE", sceAgcCreateInterpolantMapping);
-int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
-    constexpr auto fn = __func__;
+constexpr std::uint32_t InterpolantRegisterCount = 32;
+constexpr std::uint32_t InterpolantF16HiOnly = 2;
+
+using CreateInterpolantValueFn = std::uint32_t (*)(std::uint32_t psWord, const ShaderSemantic* gsSemantic);
+
+std::uint32_t CreateInterpolantValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) {
+    const std::uint32_t value = ((psWord & 0x00300000u) != 0)
+        ? CreateInterpolantF16Value(psWord, gsSemantic)
+        : CreateInterpolantNonF16Value(psWord, gsSemantic);
+
+    return (gsSemantic == nullptr)
+        ? CreateInterpolantDefaultValue(value, psWord)
+        : CreateInterpolantMappingValue(value, psWord, ShaderSemanticWord(*gsSemantic));
+}
+
+std::uint32_t CreateInterpolantF16HiValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) {
+    std::uint32_t value = ((psWord << 4u) & 0x03000000u) | 0x00080000u;
+    if (gsSemantic == nullptr || (psWord & ShaderSemanticWord(*gsSemantic) & 0x00200000u) == 0) {
+        value |= 0x00000020u;
+    }
+    value = ApplyInterpolantDefaultValueHi(value, psWord);
+    value |= ((psWord >> 30u) & 0x3u) << 8u;
+    if (gsSemantic == nullptr) {
+        return value;
+    }
+
+    value |= (ShaderSemanticWord(*gsSemantic) >> 8u) & 0x1Fu;
+    if ((psWord & 0x01400000u) != 0) {
+        value |= 0x00000400u;
+    }
+    return value;
+}
+
+std::uint32_t CreateInterpolantValueSplitF16(std::uint32_t psWord, const ShaderSemantic* gsSemantic) {
+    return (((psWord >> 20u) & 0x3u) == InterpolantF16HiOnly)
+        ? CreateInterpolantF16HiValue(psWord, gsSemantic)
+        : CreateInterpolantValue(psWord, gsSemantic);
+}
+
+int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader* gs, const Shader* ps, CreateInterpolantValueFn createValue) {
     if (regs == nullptr) {
         throw std::runtime_error(std::string(fn) + ": regs is null");
     }
@@ -26,6 +63,10 @@ int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister* regs, const Shader*
         throw std::runtime_error(std::string(fn) + ": ps->input_semantics is null but num_input_semantics != 0");
     }
 
+    if (ps->num_input_semantics > InterpolantRegisterCount) {
+        throw std::runtime_error(std::string(fn) + ": ps->num_input_semantics exceeds the interpolant register count");
+    }
+
     if (gs == nullptr) {
         throw std::runtime_error(std::string(fn) + ": gs is null");
     }
@@ -37,17 +78,7 @@ int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister* regs, const Shader*
     for (std::uint32_t i = 0; i < ps->num_input_semantics; ++i) {
         const ShaderSemantic& psSemantic = ps->input_semantics[i];
         const ShaderSemantic* gsSemantic = FindOutputSemantic(gs, psSemantic.semantic);
-        const std::uint32_t psWord = ShaderSemanticWord(psSemantic);
-
-        std::uint32_t value = ((psWord & 0x00300000u) != 0)
-            ? CreateInterpolantF16Value(psWord, gsSemantic)
-            : CreateInterpolantNonF16Value(psWord, gsSemantic);
-
-        value = (gsSemantic == nullptr)
-            ? CreateInterpolantDefaultValue(value, psWord)
-            : CreateInterpolantMappingValue(value, psWord, ShaderSemanticWord(*gsSemantic));
-
-        SetInterpolantRegister(regs, i, value);
+        SetInterpolantRegister(regs, i, createValue(ShaderSemanticWord(psSemantic), gsSemantic));
     }
 
     FillIdentityInterpolants(regs, ps->num_input_semantics);
@@ -55,3 +86,17 @@ int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister* regs, const Shader*
 }
 
 }
+
+extern "C" {
+
+APS5_EXPORT("HV4j+E0MBHE", sceAgcCreateInterpolantMapping);
+int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
+    return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValue);
+}
+
+APS5_EXPORT("dbOlWdppb4o", sceAgcUnknownCreateInterpolantMapping);
+int APS5_VABI sceAgcUnknownCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
+    return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16);
+}
+
+}

core/libs/prx/libSceAgc/Unimplemented.cpp

@@ -70,10 +70,6 @@ int APS5_VABI sceAgcGetGsOversubscription() {
  return 0;
 }
 
-int APS5_VABI sceAgcGetIsTrinityMode() {
-    return 0;
-}
-
 int APS5_VABI sceAgcGetSemaphoreLabel() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
@@ -84,8 +80,7 @@ int APS5_VABI sceAgcSetAmmSemaphoreMemory() {
  return 0;
 }
 
-APS5_EXPORT("6nths4DHNrs", sceAgcUnknown_6nths4DHNrs);
-int APS5_VABI sceAgcUnknown_6nths4DHNrs() {
+int APS5_VABI sceAgcCbMemsetExclusive() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
@@ -96,38 +91,32 @@ int APS5_VABI sceAgcUnknown_7Wa3aeJgeVU() {
  return 0;
 }
 
-APS5_EXPORT("EJBA4dbmvfg", sceAgcUnknown_EJBA4dbmvfg);
-int APS5_VABI sceAgcUnknown_EJBA4dbmvfg() {
+int APS5_VABI sceAgcAsyncWriteDataPatchSetDst() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-APS5_EXPORT("ICkECTBxrMw", sceAgcUnknown_ICkECTBxrMw);
-int APS5_VABI sceAgcUnknown_ICkECTBxrMw() {
+int APS5_VABI sceAgcGetGsPrimPayload() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-APS5_EXPORT("RTpj-tIlvZc", sceAgcUnknown_RTpj_MtIlvZc);
-int APS5_VABI sceAgcUnknown_RTpj_MtIlvZc() {
+int APS5_VABI sceAgcSetShaderInstrumentation() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-APS5_EXPORT("SwI6QxqwAC0", sceAgcUnknown_SwI6QxqwAC0);
-int APS5_VABI sceAgcUnknown_SwI6QxqwAC0() {
+int APS5_VABI sceAgcGetShaderInstrumentation() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-APS5_EXPORT("d4NZIlguzv0", sceAgcUnknown_d4NZIlguzv0);
-int APS5_VABI sceAgcUnknown_d4NZIlguzv0() {
+int APS5_VABI sceAgcAsyncWriteDataPatchSetAddressOrOffset() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-APS5_EXPORT("eAy8eGNsCuU", sceAgcUnknown_eAy8eGNsCuU);
-int APS5_VABI sceAgcUnknown_eAy8eGNsCuU() {
+int APS5_VABI sceAgcWriteDataPatchSetCachePolicy() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
@@ -138,14 +127,12 @@ int APS5_VABI sceAgcUnknown_rP5xLdOf26k() {
  return 0;
 }
 
-APS5_EXPORT("tmy-+rBpspY", sceAgcUnknown_tmy_M_PrBpspY);
-int APS5_VABI sceAgcUnknown_tmy_M_PrBpspY() {
+int APS5_VABI sceAgcWriteDataPatchSetDst() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-APS5_EXPORT("y5K5tPktiL8", sceAgcUnknown_y5K5tPktiL8);
-int APS5_VABI sceAgcUnknown_y5K5tPktiL8() {
+int APS5_VABI sceAgcAsyncWriteDataPatchSetCachePolicy() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }

core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp

@@ -0,0 +1,134 @@
+#include "SceShaders.hpp"
+#include "prx/libc/include/Shutdown.hpp"
+#include "prx/libc/include/general/VabiMacros.hpp"
+#include "prx/libSceAgc/Shader/include/ShaderConstants.hpp"
+
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <stdexcept>
+
+extern "C" int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps);
+extern "C" int APS5_VABI sceAgcUnknownCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps);
+
+namespace {
+
+using Registers = std::array<ShaderRegister, 32>;
+
+void check(bool condition, const char* message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+template <typename TAction>
+void expectFailure(TAction action) {
+    try {
+        action();
+    } catch (const std::exception& error) {
+        check(error.what()[0] != '\0', "empty exception message");
+        return;
+    }
+    throw std::runtime_error("expected an exception");
+}
+
+Registers filled() {
+    Registers regs{};
+    std::memset(regs.data(), 0xcc, sizeof(regs));
+    return regs;
+}
+
+ShaderSemantic semantic(std::uint32_t id, std::uint32_t hardwareMapping, std::uint32_t f16) {
+    ShaderSemantic value{};
+    value.semantic = id;
+    value.hardware_mapping = hardwareMapping;
+    value.is_f16 = f16;
+    return value;
+}
+
+void checkIdentity(const Registers& regs, std::uint32_t first, const char* message) {
+    for (std::uint32_t i = first; i < regs.size(); ++i) {
+        check(regs[i].offset == ShaderRegs::SPI_PS_INPUT_CNTL_0 + i && regs[i].value == i, message);
+    }
+}
+
+void testIdentity() {
+    auto regs = filled();
+    check(sceAgcUnknownCreateInterpolantMapping(regs.data(), nullptr, nullptr) == 0, "mapping without shaders failed");
+    checkIdentity(regs, 0, "mapping without a pixel shader is not the identity");
+    Shader ps{};
+    regs = filled();
+    check(sceAgcUnknownCreateInterpolantMapping(regs.data(), nullptr, &ps) == 0, "mapping without inputs failed");
+    checkIdentity(regs, 0, "mapping without inputs is not the identity");
+}
+
+void testMapping() {
+    std::array<ShaderSemantic, 5> inputs{
+        semantic(1, 0, 2), semantic(2, 0, 2), semantic(5, 0, 2), semantic(3, 0, 0), semantic(4, 0, 3)};
+    inputs[0].is_flat_shaded = 1;
+    inputs[0].default_value = 1;
+    inputs[0].default_value_hi = 3;
+    inputs[1].default_value_hi = 1;
+    inputs[2].is_flat_shaded = 1;
+    inputs[2].default_value_hi = 2;
+    inputs[3].is_flat_shaded = 1;
+    inputs[3].default_value = 2;
+    std::array<ShaderSemantic, 4> outputs{semantic(1, 7, 3), semantic(2, 4, 1), semantic(3, 9, 0), semantic(4, 2, 1)};
+    Shader gs{};
+    gs.output_semantics = outputs.data();
+    gs.num_output_semantics = static_cast<std::uint16_t>(outputs.size());
+    Shader ps{};
+    ps.input_semantics = inputs.data();
+    ps.num_input_semantics = static_cast<std::uint32_t>(inputs.size());
+
+    auto split = filled();
+    check(sceAgcUnknownCreateInterpolantMapping(split.data(), &gs, &ps) == 0, "split f16 mapping failed");
+    for (std::uint32_t i = 0; i < inputs.size(); ++i) {
+        check(split[i].offset == ShaderRegs::SPI_PS_INPUT_CNTL_0 + i, "interpolant register offset changed");
+    }
+    check(split[0].value == 0x02680707u, "high f16 half provided by the gs is not mapped");
+    check(split[1].value == 0x02280124u, "high f16 half missing from the gs does not use its default");
+    check(split[2].value == 0x02480220u, "unmatched high f16 half does not use its default");
+    check(split[3].value == 0x00000609u, "32-bit interpolant mapping changed");
+    check(split[4].value == 0x03180002u, "full f16 interpolant mapping changed");
+    checkIdentity(split, static_cast<std::uint32_t>(inputs.size()), "unused interpolants are not the identity");
+
+    auto regular = filled();
+    check(sceAgcCreateInterpolantMapping(regular.data(), &gs, &ps) == 0, "mapping failed");
+    check(regular[3].value == split[3].value && regular[4].value == split[4].value, "mappings differ outside the high f16 half mode");
+    checkIdentity(regular, static_cast<std::uint32_t>(inputs.size()), "unused interpolants are not the identity");
+}
+
+void testRejections() {
+    std::array<ShaderSemantic, 33> inputs{};
+    Shader gs{};
+    Shader ps{};
+    ps.input_semantics = inputs.data();
+    ps.num_input_semantics = static_cast<std::uint32_t>(inputs.size());
+    std::array<ShaderRegister, 64> regs{};
+    std::memset(regs.data(), 0xcc, sizeof(regs));
+    const auto saved = regs;
+    expectFailure([&] { sceAgcCreateInterpolantMapping(regs.data(), &gs, &ps); });
+    expectFailure([&] { sceAgcUnknownCreateInterpolantMapping(regs.data(), &gs, &ps); });
+    check(std::memcmp(regs.data(), saved.data(), sizeof(regs)) == 0, "rejected mapping wrote registers");
+    ps.num_input_semantics = 1;
+    expectFailure([&] { sceAgcUnknownCreateInterpolantMapping(regs.data(), nullptr, &ps); });
+    expectFailure([&] { sceAgcUnknownCreateInterpolantMapping(nullptr, &gs, &ps); });
+}
+
+}
+
+int main() {
+    try {
+        testIdentity();
+        testMapping();
+        testRejections();
+        LibcRunShutdown_nid_postfix();
+        std::puts("AGC interpolant mapping tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "%s\n", error.what());
+        try { LibcRunShutdown_nid_postfix(); }
+        catch (const std::exception& shutdown) { std::fprintf(stderr, "shutdown: %s\n", shutdown.what()); }
+        return 1;
+    }
+}

core/libs/prx/libSceAgc/tests/Platform.cpp

@@ -0,0 +1,53 @@
+#include "prx/libc/include/general/VabiMacros.hpp"
+#include "prx/libc/include/Shutdown.hpp"
+
+#include <array>
+#include <cstdio>
+#include <stdexcept>
+
+extern "C" int APS5_VABI sceAgcGetIsTrinityMode(bool* isTrinityMode);
+
+namespace {
+
+void check(bool condition, const char* message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+template <typename TAction>
+void expectFailure(TAction action) {
+    try {
+        action();
+    } catch (const std::runtime_error& error) {
+        check(error.what()[0] != '\0', "empty exception message");
+        return;
+    }
+    throw std::runtime_error("expected an exception");
+}
+
+void testTrinityMode() {
+    std::array<bool, 3> flags{true, true, true};
+    check(sceAgcGetIsTrinityMode(&flags[1]) == 0, "Trinity mode query did not return 0");
+    check(!flags[1], "base PS5 GPU reported as Trinity");
+    check(flags[0] && flags[2], "Trinity mode query wrote past its one-byte flag");
+}
+
+void testRejections() {
+    expectFailure([] { sceAgcGetIsTrinityMode(nullptr); });
+}
+
+}
+
+int main() {
+    try {
+        testTrinityMode();
+        testRejections();
+        LibcRunShutdown_nid_postfix();
+        std::puts("AGC platform tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "%s\n", error.what());
+        try { LibcRunShutdown_nid_postfix(); }
+        catch (const std::exception& shutdown) { std::fprintf(stderr, "shutdown: %s\n", shutdown.what()); }
+        return 1;
+    }
+}

core/libs/prx/libSceAgc/tests/PrimeUtcl2.cpp

@@ -0,0 +1,99 @@
+#include "prx/libSceAgc/Command/include/Packet.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <stdexcept>
+
+extern "C" {
+std::uint32_t* APS5_VABI sceAgcAcbPrimeUtcl2(CommandBuffer*, const volatile void*, std::uint32_t);
+std::uint32_t APS5_VABI sceAgcAcbPrimeUtcl2GetSize();
+std::uint32_t* APS5_VABI sceAgcDcbPrimeUtcl2(CommandBuffer*, const volatile void*, std::uint32_t);
+std::uint32_t APS5_VABI sceAgcDcbPrimeUtcl2GetSize();
+}
+
+namespace {
+
+constexpr std::uint32_t Sentinel = 0xabcdef01u;
+constexpr std::uint64_t Address = 0x0000112233440000ull;
+constexpr std::uint32_t SizeInBytes = 0x00345000u;
+
+void check(bool condition, const char* message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+template <typename TAction>
+void expectFailure(TAction action) {
+    try {
+        action();
+    } catch (const std::runtime_error&) {
+        return;
+    }
+    throw std::runtime_error("expected invalid input to fail");
+}
+
+struct Storage {
+    std::array<std::uint32_t, 16> words{};
+    CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
+
+    Storage() {
+        words.fill(Sentinel);
+    }
+
+    void limitTo(std::uint32_t count) {
+        buffer.cursor_down = buffer.cursor_up + count;
+    }
+};
+
+std::uint32_t* writeCompute(CommandBuffer* buffer) {
+    return sceAgcAcbPrimeUtcl2(buffer, reinterpret_cast<const volatile void*>(Address), SizeInBytes);
+}
+
+std::uint32_t* writeDraw(CommandBuffer* buffer) {
+    return sceAgcDcbPrimeUtcl2(buffer, reinterpret_cast<const volatile void*>(Address), SizeInBytes);
+}
+
+template <typename TWriter, typename TSize>
+void testCommand(TWriter writer, TSize size) {
+    const std::array expected{0xc0031000u, 0u, 0x33440000u, 0x1122u, SizeInBytes};
+    const auto count = static_cast<std::uint32_t>(expected.size());
+    check(size() == count * sizeof(std::uint32_t), "size query does not match the prime packet");
+
+    Storage storage;
+    auto* packet = writer(&storage.buffer);
+    check(packet == storage.words.data() && std::equal(expected.begin(), expected.end(), packet), "incorrect prime packet");
+    check(storage.buffer.cursor_up == packet + size() / sizeof(std::uint32_t), "cursor advance does not match the size query");
+    check(std::all_of(storage.words.begin() + count, storage.words.end(), [](std::uint32_t word) { return word == Sentinel; }), "prime command overwrote following words");
+
+    auto* second = writer(&storage.buffer);
+    check(second == packet + count && std::equal(expected.begin(), expected.end(), second), "second prime packet does not follow the first");
+
+    Storage exact;
+    exact.limitTo(size() / sizeof(std::uint32_t));
+    check(writer(&exact.buffer) == exact.words.data() && exact.buffer.cursor_up == exact.buffer.cursor_down, "prime packet does not fill the queried size");
+
+    Storage shortBuffer;
+    shortBuffer.limitTo(size() / sizeof(std::uint32_t) - 1u);
+    const auto before = shortBuffer.words;
+    expectFailure([&] { writer(&shortBuffer.buffer); });
+    check(shortBuffer.words == before && shortBuffer.buffer.cursor_up == shortBuffer.words.data(), "failed prime write modified the buffer");
+
+    expectFailure([&] { writer(nullptr); });
+}
+
+}
+
+int main() {
+    try {
+        testCommand(writeCompute, sceAgcAcbPrimeUtcl2GetSize);
+        testCommand(writeDraw, sceAgcDcbPrimeUtcl2GetSize);
+        check(sceAgcAcbPrimeUtcl2GetSize() == sceAgcDcbPrimeUtcl2GetSize(), "compute and draw prime sizes differ");
+        std::puts("AGC prime UTCL2 tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "%s\n", error.what());
+        return 1;
+    }
+}

core/libs/prx/libSceAgc/tests/RegisterCommands.cpp

@@ -21,6 +21,10 @@ std::uint32_t APS5_VABI sceAgcDcbSetUcRegistersIndirectGetSize(std::uint32_t);
 int APS5_VABI sceAgcSetCxRegIndirectPatchSetNumRegisters(std::uint32_t*, std::uint32_t);
 int APS5_VABI sceAgcSetShRegIndirectPatchSetNumRegisters(std::uint32_t*, std::uint32_t);
 int APS5_VABI sceAgcSetUcRegIndirectPatchSetNumRegisters(std::uint32_t*, std::uint32_t);
+std::uint32_t* APS5_VABI sceAgcCbSetShRegistersDirect(CommandBuffer*, const volatile ShaderRegister*, std::uint32_t);
+std::uint32_t* APS5_VABI sceAgcCbSetUcRegistersDirect(CommandBuffer*, const volatile ShaderRegister*, std::uint32_t);
+std::uint32_t APS5_VABI sceAgcCbSetShRegistersDirectGetSize(std::uint32_t);
+std::uint32_t APS5_VABI sceAgcCbSetUcRegistersDirectGetSize(std::uint32_t);
 }
 
 namespace {
@@ -101,12 +105,60 @@ void testIndirect() {
     }
 }
 
+void testDirectList() {
+    constexpr std::uint32_t sentinel = 0xabcdef01u;
+    constexpr std::uint32_t wordSize = sizeof(std::uint32_t);
+    const std::array writers{sceAgcCbSetShRegistersDirect, sceAgcCbSetUcRegistersDirect};
+    const std::array sizes{sceAgcCbSetShRegistersDirectGetSize, sceAgcCbSetUcRegistersDirectGetSize};
+    const std::array opcodes{0x7600u, 0x7900u};
+    const std::array<ShaderRegister, 4> scattered{{{0x12u, 1}, {0x11u, 2}, {0x20u, 3}, {0xffffu, 4}}};
+    const std::array<ShaderRegister, 4> adjacent{{{0x10u, 5}, {0x11u, 6}, {0x12u, 7}, {0x13u, 8}}};
+    const std::array<ShaderRegister, 4> paired{{{0x10u, 5}, {0x11u, 6}, {0x20u, 7}, {0x21u, 8}}};
+    for (std::size_t i = 0; i < writers.size(); ++i) {
+        const auto single = 0xc0010000u | opcodes[i];
+        for (std::uint32_t count = 1; count <= scattered.size(); ++count) {
+            const auto words = sizes[i](count) / wordSize;
+            Storage storage;
+            check(sizes[i](count) % wordSize == 0 && words != 0 && words <= storage.words.size(), "register list size is not a usable word count");
+            storage.words.fill(sentinel);
+            storage.buffer.cursor_down = storage.buffer.cursor_up + words;
+            auto* packet = writers[i](&storage.buffer, scattered.data(), count);
+            check(packet == storage.words.data() && storage.buffer.cursor_up == storage.buffer.cursor_down, "separate registers do not fill the queried size");
+            for (std::uint32_t reg = 0; reg < count; ++reg) {
+                const std::array expected{single, scattered[reg].offset, scattered[reg].value};
+                check(std::equal(expected.begin(), expected.end(), packet + reg * expected.size()), "incorrect register list packet");
+            }
+            check(std::all_of(storage.words.begin() + words, storage.words.end(), [](std::uint32_t word) { return word == sentinel; }), "register list overwrote following words");
+            Storage shortStorage;
+            shortStorage.buffer.cursor_down = shortStorage.buffer.cursor_up + words - 1u;
+            expectFailure([&] { writers[i](&shortStorage.buffer, scattered.data(), count); });
+        }
+        const auto limit = sizes[i](4) / wordSize;
+        Storage merged;
+        merged.buffer.cursor_down = merged.buffer.cursor_up + limit;
+        const std::array<std::uint32_t, 6> expectedMerged{0xc0040000u | opcodes[i], 0x10u, 5, 6, 7, 8};
+        auto* packet = writers[i](&merged.buffer, adjacent.data(), 4);
+        check(std::equal(expectedMerged.begin(), expectedMerged.end(), packet) && merged.buffer.cursor_up == packet + expectedMerged.size(), "adjacent registers were not merged into one range");
+        Storage split;
+        split.buffer.cursor_down = split.buffer.cursor_up + limit;
+        const std::array<std::uint32_t, 8> expectedSplit{0xc0020000u | opcodes[i], 0x10u, 5, 6, 0xc0020000u | opcodes[i], 0x20u, 7, 8};
+        packet = writers[i](&split.buffer, paired.data(), 4);
+        check(std::equal(expectedSplit.begin(), expectedSplit.end(), packet) && split.buffer.cursor_up == packet + expectedSplit.size(), "register pairs were not written as two ranges");
+        check(sizes[i](0) == 0, "an empty register list has a size");
+        check(sizes[i](0x15555555u) == 0xfffffffcu, "largest register list size mismatch");
+        for (const auto count : {0x15555556u, 0x40000000u, 0xffffffffu}) {
+            expectFailure([&] { sizes[i](count); });
+        }
+    }
+}
+
 }
 
 int main() {
     try {
         testDirect();
         testIndirect();
+        testDirectList();
         std::puts("AGC register command tests passed");
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -279,6 +279,11 @@ target_link_libraries(agc_dpp_write_tests PRIVATE shader_recompiler_agc_driver l
 set_target_properties(agc_dpp_write_tests PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(agc_dpp_write_tests)
 
+add_test_executable(agc_masked_select_tests EXCLUDE_FROM_ALL tests/MaskedSelect.cpp)
+target_link_libraries(agc_masked_select_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_masked_select_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_masked_select_tests)
+
 add_test_executable(agc_vop1_control_decode_tests EXCLUDE_FROM_ALL tests/Vop1ControlDecode.cpp)
 target_link_libraries(agc_vop1_control_decode_tests PRIVATE shader_recompiler_agc_driver libc)
 set_target_properties(agc_vop1_control_decode_tests PROPERTIES CXX_EXTENSIONS OFF)
@@ -498,7 +503,7 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
 endforeach()
 
 if(BUILD_TESTING)
-    foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_dpp_write_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests ${agcExecutionTests})
+    foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests ${agcExecutionTests})
         string(REGEX REPLACE "_tests$" "" agcTestName "${agcTest}")
         set_target_properties(${agcTest} PROPERTIES EXCLUDE_FROM_ALL OFF)
         add_test(NAME ${agcTestName} COMMAND ${agcTest})
@@ -681,6 +686,14 @@ if(BUILD_TESTING)
         set_tests_properties(ray_tracing_miss PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test_executable(wave32_wide_subgroup_tests ${LIBS_INCLUDE_DIR}/tests/Wave32WideSubgroup.cpp)
+    target_link_libraries(wave32_wide_subgroup_tests PRIVATE shader_recompiler_agc_driver libc)
+    configure_windows_unwind(wave32_wide_subgroup_tests)
+    add_test(NAME wave32_wide_subgroup COMMAND wave32_wide_subgroup_tests)
+    if(WIN32)
+        set_tests_properties(wave32_wide_subgroup PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test_executable(vector_compare_i16_u16_tests ${LIBS_INCLUDE_DIR}/tests/VectorCompareI16U16.cpp)
     target_link_libraries(vector_compare_i16_u16_tests PRIVATE shader_recompiler_agc_driver libc)
     configure_windows_unwind(vector_compare_i16_u16_tests)

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -128,7 +128,7 @@ private:
     static std::array<WriteRecord, 16384>& writeHistory();
     static std::size_t& writeCursor();
     static void dumpPackets(std::span<const std::uint32_t> commands, const std::uint32_t* guest = nullptr);
-    static void validate(std::span<const std::uint32_t> commands, std::uint32_t queue, const std::uint32_t* guest = nullptr);
+    static void validate(const Submission& submission, const std::uint32_t* guest = nullptr);
     static void reportSkip(const char* kind, const std::string& what);
     static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request);
     static bool matchesFillKernel(std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute);

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp

@@ -25,6 +25,7 @@ struct Submission {
     std::shared_ptr<const ShaderRegistry> shaders;
     std::map<std::size_t, std::shared_ptr<IFlipRequest>> flips;
     std::map<std::size_t, std::shared_ptr<IRenderingWait>> renderingWaits;
+    std::map<std::size_t, std::size_t> conditionalEnds;
     bool suspend = false;
     bool waitFree = false;
 

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp

@@ -39,6 +39,8 @@ using ShaderRegistry = std::map<std::uint64_t, std::shared_ptr<const ShaderSnaps
 
 bool FailureMemo();
 
+std::uint64_t NullPixelProgramAddress();
+
 std::shared_ptr<const ShaderRecompiler::SourceHandle> SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, std::uint64_t deviceSerial, const ShaderRecompiler::RecompileRequest& request, bool bypass, const std::string** poisoned = nullptr);
 
 }

core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp

@@ -96,6 +96,8 @@ bool WaitSatisfiedUnchecked(std::span<const std::uint32_t> packet);
 // holds will store), without reading memory.
 bool WaitComparesValue(std::span<const std::uint32_t> packet, std::uint64_t value);
 std::size_t WaitAwaitedBytes(std::span<const std::uint32_t> packet);
+std::size_t ConditionalWords(std::span<const std::uint32_t> packet);
+std::uint32_t ReadCondition(std::span<const std::uint32_t> packet);
 // A label write (RELEASE_MEM with a data select, WRITE_DATA to memory): the destination and the
 // bytes it stores, so the write can be recorded on the GPU behind the work it signals. Packets
 // without a memory destination decode to nothing. No allocation per label (tens of thousands per

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 
@@ -15,6 +16,8 @@ std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Su
         return (static_cast<std::uint64_t>(readRegister(queue.shader, base)) << 8u) | (static_cast<std::uint64_t>(high) << 40u);
     };
     const auto prepare = [&](std::uint64_t address, std::uint8_t type, Stage stage, std::uint32_t rsrc2, std::uint32_t userDataBase) {
+        const bool nullPixel = address == 0 && stage == Stage::Fragment;
+        if (nullPixel) address = NullPixelProgramAddress();
         auto it = submission.shaders->upper_bound(address);
         require(it != submission.shaders->begin(), "graphics program does not belong to a registered shader");
         --it;
@@ -22,7 +25,7 @@ std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Su
         require(address - snapshot.codeAddress < snapshot.code.size() * sizeof(std::uint32_t), "graphics program is outside registered shader code");
         require(snapshot.type == type, "graphics program refers to an incompatible shader binary type");
         Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, rsrc2);
-        const auto resources = readRegister(queue.shader, rsrc2);
+        const auto resources = nullPixel && !queue.shader.contains(rsrc2) ? 0u : readRegister(queue.shader, rsrc2);
         const auto userCount = ((resources >> 1u) & 0x1fu) | (((resources >> 27u) & 1u) << 5u);
         require(userCount <= 32, "graphics user SGPR count exceeds the register bank");
         const auto codeOffset = static_cast<std::size_t>((address - snapshot.codeAddress) / sizeof(std::uint32_t));
@@ -86,9 +89,14 @@ std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Su
         } else {
             append(0xc8, 2, Stage::Vertex, 0x8b, 0x8c, Role::Main);
         }
+        const bool nullPixel = Graphics::PixelProgramUnset(queue);
+        if (nullPixel) {
+            const auto rejection = Graphics::NullPixelProgramRejection(queue);
+            require(rejection.empty(), rejection.c_str());
+        }
         append(0x008, 1, Stage::Fragment, 0x00b, 0x00c, Role::Fragment);
         programs.back().firstUserSgpr = 0;
-        product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(graphics));
+        product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(graphics), nullPixel);
         return product;
     }
 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp

@@ -54,7 +54,7 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) {
     const auto& s = a.state;
     const auto& t = b.state;
     const auto sameColor = [](const Graphics::ColorTarget& x, const Graphics::ColorTarget& y) {
-        return x.address == y.address && x.extent.width == y.extent.width && x.extent.height == y.extent.height && x.format == y.format && x.bytes == y.bytes && x.componentMapping == y.componentMapping && x.tileMode == y.tileMode && x.elementBytes == y.elementBytes && x.dccAddress == y.dccAddress && x.dccAlphaOnMsb == y.dccAlphaOnMsb && x.slot == y.slot;
+        return x.address == y.address && x.extent.width == y.extent.width && x.extent.height == y.extent.height && x.format == y.format && x.bytes == y.bytes && x.componentMapping == y.componentMapping && x.tileMode == y.tileMode && x.elementBytes == y.elementBytes && x.dccAddress == y.dccAddress && x.dccAlphaOnMsb == y.dccAlphaOnMsb && x.slot == y.slot && x.exportIndex == y.exportIndex;
     };
     const auto sameBlend = [](const VkPipelineColorBlendAttachmentState& x, const VkPipelineColorBlendAttachmentState& y) {
         return x.blendEnable == y.blendEnable && x.srcColorBlendFactor == y.srcColorBlendFactor && x.dstColorBlendFactor == y.dstColorBlendFactor && x.colorBlendOp == y.colorBlendOp && x.srcAlphaBlendFactor == y.srcAlphaBlendFactor && x.dstAlphaBlendFactor == y.dstAlphaBlendFactor && x.alphaBlendOp == y.alphaBlendOp && x.colorWriteMask == y.colorWriteMask;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "ThreadOwned.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include <bit>
@@ -192,7 +193,8 @@ bool Driver::fillBuffer(QueueState& queue, std::uint32_t queueId, std::span<cons
             ++cpuFills;
 
             localDevice->WaitIdle();
-            static thread_local std::vector<std::byte> block;
+            static thread_local std::vector<std::byte>* blockSlot = nullptr;
+            auto& block = ShaderRecompiler::ThreadOwned(blockSlot);
             const auto chunk = std::min<std::size_t>(bytes, 1u << 20u);
             block.resize(chunk);
             for (std::size_t at = 0; at < chunk; at += 16) std::memcpy(block.data() + at, pattern.data(), 16);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/CaptureValidation.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "ThreadOwned.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include <cstdlib>
 #include <cstring>
@@ -14,7 +15,8 @@ bool Driver::captureStable(std::span<const ShaderRecompiler::MemoryRegion> captu
     double waited = 0;
     PendingView pending;
     pending.Load();
-    thread_local std::vector<std::byte> known;
+    thread_local std::vector<std::byte>* knownSlot = nullptr;
+    auto& known = ShaderRecompiler::ThreadOwned(knownSlot);
     for (const auto& region : captured) {
         if (!stable) break;
         known.resize(region.bytes.size());
@@ -58,7 +60,8 @@ bool Driver::captureStable(std::span<const ShaderRecompiler::MemoryRegion> captu
 bool Driver::validateCaptured(std::uint64_t program, std::uint32_t queue, std::span<const ShaderRecompiler::MemoryRegion> captured, const ShaderRecompiler::RecompileResult& compiled, bool inPlace, const PendingView& view, bool* unmapped, std::optional<SampledReadScope>* sampling, const DataMask* data) {
     using Policy = ShaderMemory::PendingWrite;
 
-    thread_local std::vector<std::size_t> pending;
+    thread_local std::vector<std::size_t>* pendingSlot = nullptr;
+    auto& pending = ShaderRecompiler::ThreadOwned(pendingSlot);
     pending.clear();
     for (std::size_t i = 0; i < captured.size(); ++i) {
         const auto& region = captured[i];
@@ -66,7 +69,8 @@ bool Driver::validateCaptured(std::uint64_t program, std::uint32_t queue, std::s
     }
 
     const auto compareMasked = [&](const ShaderRecompiler::MemoryRegion& region, std::size_t first) {
-        thread_local std::vector<std::byte> live;
+        thread_local std::vector<std::byte>* liveSlot = nullptr;
+        auto& live = ShaderRecompiler::ThreadOwned(liveSlot);
         live.resize(region.bytes.size());
         const auto copied = GuestMemory::CopyMapped(region.guestAddress, live);
         if (copied != GuestMemory::Compare::Equal) return copied;
@@ -84,9 +88,12 @@ bool Driver::validateCaptured(std::uint64_t program, std::uint32_t queue, std::s
     };
 
     static constexpr std::size_t NoKnownValue = std::numeric_limits<std::size_t>::max();
-    thread_local std::vector<Policy> policies;
-    thread_local std::vector<std::size_t> knownOffsets;
-    thread_local std::vector<std::byte> knownBytes;
+    thread_local std::vector<Policy>* policiesSlot = nullptr;
+    thread_local std::vector<std::size_t>* knownOffsetsSlot = nullptr;
+    thread_local std::vector<std::byte>* knownBytesSlot = nullptr;
+    auto& policies = ShaderRecompiler::ThreadOwned(policiesSlot);
+    auto& knownOffsets = ShaderRecompiler::ThreadOwned(knownOffsetsSlot);
+    auto& knownBytes = ShaderRecompiler::ThreadOwned(knownBytesSlot);
     policies.assign(pending.size(), Policy::None);
     knownOffsets.assign(pending.size(), NoKnownValue);
     knownBytes.clear();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp

@@ -165,6 +165,10 @@ ShaderMemory::PendingWrite Driver::classifyPendingWrite(std::uint64_t address, s
     }
     const auto first = address & ~3ull;
     const auto limit = address + bytes;
+    if (Graphics::Recorder::WideLabelIn(first, static_cast<std::size_t>(limit - first))) {
+        reason = &ValidateCounters::syncedLabel;
+        return Policy::Sync;
+    }
     for (std::uint64_t dword = first; dword < limit; dword += 4) {
         if (Graphics::Recorder::LookupLabelValue(dword, 4, 0).has_value()) {
             reason = &ValidateCounters::syncedLabel;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/CommandValidation.cpp

@@ -47,8 +47,12 @@ void Driver::dumpPackets(std::span<const std::uint32_t> commands, const std::uin
     }
 }
 
-void Driver::validate(std::span<const std::uint32_t> commands, std::uint32_t queue, const std::uint32_t* guest) {
+void Driver::validate(const Submission& submission, const std::uint32_t* guest) {
+    const std::span<const std::uint32_t> commands = submission.commands;
+    const auto queue = submission.queue;
+    std::vector<std::size_t> guarded;
     for (std::size_t cursor = 0; cursor < commands.size();) {
+        std::erase_if(guarded, [cursor](std::size_t end) { return end <= cursor; });
         const auto header = commands[cursor];
         if (Pm4::FillerPacket(header)) { ++cursor; continue; }
         if ((header & 0xc0000000u) != 0xc0000000u) {
@@ -65,6 +69,12 @@ void Driver::validate(std::span<const std::uint32_t> commands, std::uint32_t que
         require(count <= commands.size() - cursor, "truncated PM4 packet");
         try {
             Pm4::Validate(commands.subspan(cursor, count), queue);
+            if (header == FlipPacketHeader && !guarded.empty()) throw std::runtime_error("a flip inside a conditional execution range (conditional flip reservation) is not implemented");
+            if (((header >> 8u) & 0xffu) == 0x22u) {
+                const auto end = submission.conditionalEnds.find(cursor);
+                if (end == submission.conditionalEnds.end()) throw std::runtime_error("conditional execution range is not made of whole packets of its command buffer");
+                guarded.push_back(end->second);
+            }
         } catch (const std::exception& error) {
             dumpPackets(commands, guest);
             throw std::runtime_error("AGC driver: " + Pm4::Name(header) + " at DWORD " + std::to_string(cursor) + ": " + error.what());

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "ThreadOwned.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
@@ -97,7 +98,8 @@ void Driver::execute(const Submission& submission) {
 
     static const bool profilePackets = std::getenv("APS5_PROFILE_DRAW") != nullptr;
 
-    thread_local PacketProfile packetProfile;
+    thread_local PacketProfile* packetProfileSlot = nullptr;
+    auto& packetProfile = ShaderRecompiler::ThreadOwned(packetProfileSlot);
     ++packetProfile.submissions;
 
     bumpEpoch(&EpochBumps::submissions);
@@ -109,6 +111,7 @@ void Driver::execute(const Submission& submission) {
         const auto count = Pm4::PacketWords(header);
         const auto packet = std::span(submission.commands).subspan(cursor, count);
         const auto opcode = (header >> 8u) & 0xffu;
+        auto nextCursor = cursor + count;
         std::shared_lock deviceUse(deviceReplacement, std::defer_lock);
         if (opcode != 0x3c && opcode != 0x93 && header != RenderingWaitPacketHeader && header != FlipPacketHeader) deviceUse.lock();
 
@@ -270,6 +273,11 @@ void Driver::execute(const Submission& submission) {
                 }
             }); });
             finishDrawPacket(drawn);
+        } else if (opcode == 0x22) {
+            recordQueuedLabelsBeforeRead(submission.queue);
+            const auto condition = Pm4::ReadCondition(packet);
+            if (condition == 0) nextCursor = submission.conditionalEnds.at(cursor);
+            if (traceGpu) std::fprintf(stderr, "[gpu] %.1f queue 0x%x COND_EXEC at DWORD %zu reads 0x%x at 0x%llx: %s %zu dwords\n", TraceMs(), submission.queue, cursor, condition, static_cast<unsigned long long>(packet[1] | (static_cast<std::uint64_t>(packet[2]) << 32u)), condition == 0 ? "skips" : "executes", submission.conditionalEnds.at(cursor) - cursor - count);
         } else if (sampleDump && !wroteOnGpu) {
             dumpSampleCounters(packet[2] | (static_cast<std::uint64_t>(packet[3]) << 32u));
         } else if (opcode != 0x42 && opcode != 0x46 && opcode != 0x58) {
@@ -282,7 +290,7 @@ void Driver::execute(const Submission& submission) {
             if (endOfPipeInterrupt && !interruptDeferred) AgcDriverDeliverEopInterrupt(submission.queue);
         }
         if (drawPacket || (sampleDump && wroteOnGpu)) Graphics::Recorder::CountRecordedWork();
-        cursor += count;
+        cursor = nextCursor;
     }
 
     static const bool submitAtEnd = std::getenv("APS5_SUBMIT_AT_END") != nullptr;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp

@@ -11,6 +11,7 @@ namespace AgcDriver::DriverDetail {
 
 void Driver::copyCommands(Submission& submission, const std::uint32_t* guest, std::size_t words) {
     submission.commands.clear();
+    submission.conditionalEnds.clear();
     std::size_t budget = std::size_t{1} << 26u;
     copySegment(submission, guest, words, budget);
 }
@@ -18,34 +19,57 @@ void Driver::copyCommands(Submission& submission, const std::uint32_t* guest, st
 bool Driver::copySegment(Submission& submission, const std::uint32_t* guest, std::size_t words, std::size_t& budget) {
     require(words <= budget, "command buffer jumps exceed the copy limit (a jump loop?)");
     budget -= words;
+    std::vector<std::pair<std::size_t, std::size_t>> guarded;
+    const auto reach = [&](std::size_t cursor, std::size_t next) {
+        std::erase_if(guarded, [&](const auto& range) {
+            if (range.first != cursor) return false;
+            submission.conditionalEnds.emplace(range.second, submission.commands.size());
+            return true;
+        });
+        for (const auto& range : guarded) require(range.first >= next, "conditional execution range ends inside a packet");
+    };
     for (std::size_t cursor = 0; cursor < words;) {
         const auto header = guest[cursor];
-        if (Pm4::FillerPacket(header)) { submission.commands.push_back(header); ++cursor; continue; }
+        if (Pm4::FillerPacket(header)) {
+            reach(cursor, cursor + 1);
+            submission.commands.push_back(header);
+            ++cursor;
+            continue;
+        }
         const auto count = (header & 0xc0000000u) == 0xc0000000u ? Pm4::PacketWords(header) : words - cursor;
         if ((header & 0xc0000000u) != 0xc0000000u || count > words - cursor) {
             submission.commands.insert(submission.commands.end(), guest + cursor, guest + words);
             return false;
         }
+        reach(cursor, cursor + count);
         const auto opcode = (header >> 8u) & 0xffu;
         if (opcode == 0x3fu) {
             require(count == 4, "invalid INDIRECT_BUFFER size");
             const auto* target = reinterpret_cast<const std::uint32_t*>(static_cast<std::uintptr_t>(guest[cursor + 1] & ~3u) | (static_cast<std::uintptr_t>(guest[cursor + 2] & 0xffffu) << 32u));
             const std::size_t targetWords = guest[cursor + 3] & 0xfffffu;
             const bool chain = (guest[cursor + 3] & (1u << 20u)) != 0;
+            require(!chain || guarded.empty(), "a chained INDIRECT_BUFFER inside a conditional execution range is not implemented");
             GuestMemory::CheckRange(target, targetWords * sizeof(std::uint32_t), alignof(std::uint32_t));
-            if (copySegment(submission, target, targetWords, budget)) return true;
+            if (copySegment(submission, target, targetWords, budget)) {
+                require(guarded.empty(), "a REWIND inside a conditional execution range is not implemented");
+                return true;
+            }
             if (chain) return false;
             cursor += count;
             continue;
         }
+        if (opcode == 0x22u && count == 5) guarded.emplace_back(cursor + count + Pm4::ConditionalWords(std::span(guest + cursor, count)), submission.commands.size());
         submission.commands.insert(submission.commands.end(), guest + cursor, guest + cursor + count);
         cursor += count;
         if (opcode == 0x59u) {
+            require(guarded.empty(), "a REWIND inside a conditional execution range is not implemented");
             submission.rewindTail = guest + cursor;
             submission.rewindWords = words - cursor;
             return true;
         }
     }
+    reach(words, words);
+    require(guarded.empty(), "conditional execution range exceeds its command buffer");
     return false;
 }
 
@@ -104,7 +128,7 @@ void Driver::executeRewindTail(const Submission& stalled) {
     Submission tail{};
     tail.queue = stalled.queue;
     copyCommands(tail, stalled.rewindTail, stalled.rewindWords);
-    validate(tail.commands, tail.queue, stalled.rewindTail);
+    validate(tail, stalled.rewindTail);
     waitForFlipRoom(tail);
     {
         std::lock_guard lock(mutex);
@@ -135,7 +159,7 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) {
         copyCommands(submission, descriptor.addr, descriptor.dw_num);
     }
     const auto copied = profile ? std::chrono::steady_clock::now() : start;
-    validate(submission.commands, queue, descriptor.addr);
+    validate(submission, descriptor.addr);
     waitForFlipRoom(submission);
     static const bool trace = std::getenv("APS5_TRACE_GPU") != nullptr;
     if (trace) std::fprintf(stderr, "[gpu] %.1f submit queue=0x%x dwords=%zu at %p\n", TraceMs(), queue, submission.commands.size(), static_cast<const void*>(descriptor.addr));

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp

@@ -77,6 +77,22 @@ std::shared_ptr<const ShaderRecompiler::SourceHandle> SourceHandleFor(const Shad
     return handle;
 }
 
+alignas(256) static const std::uint32_t NullPixelCode[64] = {0xbf810000u};
+static const Shader NullPixelShader = [] {
+    Shader shader{};
+    shader.file_header = 0x34333231u;
+    shader.version = 0x18u;
+    shader.code = NullPixelCode;
+    shader.header_size = sizeof(Shader);
+    shader.shader_size = sizeof(NullPixelCode);
+    shader.type = 1;
+    return shader;
+}();
+
+std::uint64_t NullPixelProgramAddress() {
+    return reinterpret_cast<std::uintptr_t>(NullPixelCode);
+}
+
 void Driver::RegisterShader(const Shader* shader) {
     CheckFailure();
     GuestMemory::CheckRange(shader, sizeof(Shader), alignof(Shader));
@@ -107,6 +123,13 @@ void Driver::RegisterShader(const Shader* shader) {
     if (shaders == nullptr) shaders = std::make_shared<ShaderRegistry>();
     else if (shaders.use_count() != 1) shaders = std::make_shared<ShaderRegistry>(*shaders);
     shaders->insert_or_assign(address, std::make_shared<const ShaderSnapshot>(std::move(snapshot)));
+    if (shaders->find(NullPixelProgramAddress()) == shaders->end()) {
+        ShaderSnapshot null{NullPixelProgramAddress(), reinterpret_cast<std::uintptr_t>(&NullPixelShader), NullPixelShader.type, {}, {}};
+        null.code.assign(std::begin(NullPixelCode), std::end(NullPixelCode));
+        null.header.resize(sizeof(Shader));
+        std::memcpy(null.header.data(), &NullPixelShader, sizeof(Shader));
+        shaders->insert_or_assign(NullPixelProgramAddress(), std::make_shared<const ShaderSnapshot>(std::move(null)));
+    }
 }
 
 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp

@@ -2,13 +2,14 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "ThreadOwned.hpp"
 #include <cstdlib>
 
 namespace AgcDriver::DriverDetail {
 
 DeferredLabels& deferredLabels() {
-    static thread_local DeferredLabels deferred;
-    return deferred;
+    static thread_local DeferredLabels* deferred = nullptr;
+    return ShaderRecompiler::ThreadOwned(deferred);
 }
 
 bool DeferLabels() {
@@ -91,7 +92,8 @@ void Driver::recordDeferredLabels(VulkanDevice* localDevice, std::uint32_t queue
     auto& deferred = deferredLabels();
     if (deferred.labels.empty()) return;
 
-    static thread_local std::vector<DeferredLabel> labels;
+    static thread_local std::vector<DeferredLabel>* recording = nullptr;
+    auto& labels = ShaderRecompiler::ThreadOwned(recording);
     labels.clear();
     labels.swap(deferred.labels);
     struct Clear {

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "ThreadOwned.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
 #include "prx/libc/include/GuestArena.hpp"
 #include "prx/libc/include/GuestWriteWatch.hpp"
@@ -169,8 +170,8 @@ struct ThreadMemoryCounters {
 
 MemoryCounter& CounterFor(MemoryCounterKind kind) {
     if (SharedMemoryCounters()) return Profile().shared.counters[kind];
-    thread_local ThreadMemoryCounters thread;
-    return thread.counters.counters[kind];
+    thread_local ThreadMemoryCounters* thread = nullptr;
+    return ShaderRecompiler::ThreadOwned(thread).counters.counters[kind];
 }
 
 // Every thread's counters, the retired and the shared ones summed: calls, bytes, nanoseconds per kind.

core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp

@@ -166,7 +166,7 @@ std::string_view UnsupportedReason(std::uint32_t header) {
         }
     }
     switch (opcode) {
-        case 0x11: case 0x12: case 0x13: case 0x15: case 0x16: case 0x26: case 0x27:
+        case 0x11: case 0x12: case 0x13: case 0x15: case 0x16: case 0x22: case 0x26: case 0x27:
         case 0x2a: case 0x2d: case 0x2f: case 0x35: case 0x37: case 0x40: case 0x42: case 0x46: case 0x50:
         case 0x58: case 0x63: case 0x64: case 0x69: case 0x76: case 0x79: case 0x7a:
         case 0x81: case 0x83: case 0x9f: return {};
@@ -176,7 +176,6 @@ std::string_view UnsupportedReason(std::uint32_t header) {
         case 0x3a: case 0x8d:
             return "graphics draw, shader stages and guest render-target materialization are not implemented";
         case 0x20: return "GPU query predication is not implemented";
-        case 0x22: return "conditional command execution and conditional flip reservation are not implemented";
         case 0x33: case 0x3f: return "nested command buffers, branching and nested flip reservation are not implemented";
         case 0x3c: case 0x93: return {};
         case 0x39:
@@ -194,8 +193,11 @@ std::string_view UnsupportedReason(std::uint32_t header) {
     }
 }
 
+constexpr std::uint32_t ConditionCachePolicy = 3u << 25u;
+constexpr std::uint32_t ConditionalWordsMask = 0x3fffu;
 constexpr std::uint32_t DmaSourceCachePolicy = 3u << 13u;
 constexpr std::uint32_t DmaDestinationCachePolicy = 3u << 25u;
+constexpr std::uint32_t WriteDataCachePolicy = 3u << 25u;
 
 void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
     require(!packet.empty(), "truncated PM4 header");
@@ -296,6 +298,13 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
             require(packet.size() == 3 || packet.size() == 4, "invalid indirect dispatch size");
             require((packet.back() & ~0xa020u) == 0x41u, "indirect dispatch modifiers are not implemented");
             break;
+        case 0x22:
+            size(5);
+            require((packet[1] & 3u) == 0, "COND_EXEC reserved address bits are not implemented");
+            require(packet[2] <= 0xffffu, "COND_EXEC address bits above 48 are not implemented");
+            require((packet[3] & ~(queue == 0 ? 0u : ConditionCachePolicy)) == 0, "COND_EXEC reserved control fields are not implemented");
+            require((packet[4] & ~ConditionalWordsMask) == 0, "COND_EXEC reserved count bits are not implemented");
+            break;
         case 0x42: size(2); require(packet[1] == 0, "unsupported PFP_SYNC_ME payload"); break;
         case 0x46: {
             require((packet[1] & ~0x73fu) == 0, "unsupported EVENT_WRITE flags or reserved bits");
@@ -380,7 +389,7 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
             break;
         case 0x37: {
             require(packet.size() >= 5, "WRITE_DATA has no data");
-            require((packet[1] & ~0x40110f00u) == 0, "WRITE_DATA engine, cache or reserved fields are not implemented");
+            require((packet[1] & ~(0x40110f00u | WriteDataCachePolicy)) == 0, "WRITE_DATA engine or reserved fields are not implemented");
             const auto destination = (packet[1] >> 8u) & 0xfu;
             require(destination == 1 || destination == 2 || (queue != 0 && destination == 5), "WRITE_DATA register or GDS destination is not implemented");
             require((packet[2] & 3u) == 0, "misaligned WRITE_DATA destination");
@@ -462,6 +471,19 @@ std::size_t WaitAwaitedBytes(std::span<const std::uint32_t> packet) {
     return wide && (packet.size() < 8 || packet[7] != 0) ? 8 : 4;
 }
 
+std::size_t ConditionalWords(std::span<const std::uint32_t> packet) {
+    require(packet.size() == 5 && ((packet[0] >> 8u) & 0xffu) == 0x22u, "expected COND_EXEC packet");
+    return packet[4] & ConditionalWordsMask;
+}
+
+std::uint32_t ReadCondition(std::span<const std::uint32_t> packet) {
+    require(packet.size() == 5 && ((packet[0] >> 8u) & 0xffu) == 0x22u, "expected COND_EXEC packet");
+    std::uint32_t value = 0;
+    const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Wait);
+    GuestMemory::Read(address(packet[1], packet[2]), std::as_writable_bytes(std::span(&value, 1)), 4);
+    return value;
+}
+
 std::optional<LabelWrite> DecodeLabelWrite(std::span<const std::uint32_t> packet) {
     const auto opcode = (packet[0] >> 8u) & 0xffu;
     if (opcode == 0x49 && packet.size() >= 7) {
@@ -537,7 +559,7 @@ bool UsesGpuCacheBarrier(std::span<const std::uint32_t> packet) {
 
 bool AccessesMemory(std::uint32_t header) {
     switch ((header >> 8u) & 0xffu) {
-        case 0x27: case 0x3c: case 0x93: case 0x49: case 0x16: case 0x2d: case 0x35: case 0x24: case 0x25: case 0x2c: case 0x38: case 0x37: case 0x40: case 0x50: case 0x63: case 0x64: case 0x83: case 0x9f: return true;
+        case 0x22: case 0x27: case 0x3c: case 0x93: case 0x49: case 0x16: case 0x2d: case 0x35: case 0x24: case 0x25: case 0x2c: case 0x38: case 0x37: case 0x40: case 0x50: case 0x63: case 0x64: case 0x83: case 0x9f: return true;
         default: return false;
     }
 }

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -191,6 +191,7 @@ struct VulkanDevice::State {
     bool primitiveListRestart = false;
     bool depthClipControl = false;
     bool imageViewMinLod = false;
+    bool pipelineExecutableInfo = false;
     bool maintenance8 = false;
     bool depthClamp = false;
     bool depthBounds = false;
@@ -802,7 +803,6 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
         state->capabilities.push_back(5283);
         state->spirvExtensions.push_back("SPV_EXT_mesh_shader");
     }
-    APS5_LOG_OUT("Vulkan features tessellationAvailable=%u mesh=%u depthClip=%u depthRangeUnrestricted=%u", static_cast<unsigned>(state->tessellationShader), static_cast<unsigned>(state->meshShader), static_cast<unsigned>(state->depthClipControl), static_cast<unsigned>(state->depthRangeUnrestricted));
     const float priority = 1.0f;
     VkDeviceQueueCreateInfo queueInfo{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};
     queueInfo.queueFamilyIndex = family;
@@ -823,6 +823,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
     enabled.tessellationShader = available.tessellationShader;
     state->tessellationShader = enabled.tessellationShader == VK_TRUE;
     if (state->tessellationShader) state->capabilities.push_back(3);
+    APS5_LOG_OUT("Vulkan features tessellationAvailable=%u mesh=%u depthClip=%u depthRangeUnrestricted=%u", static_cast<unsigned>(state->tessellationShader), static_cast<unsigned>(state->meshShader), static_cast<unsigned>(state->depthClipControl), static_cast<unsigned>(state->depthRangeUnrestricted));
     require(available.samplerAnisotropy && available.textureCompressionBC, "device lacks sampler anisotropy or BC texture compression support required for texture sampling");
     enabled.samplerAnisotropy = VK_TRUE;
     enabled.textureCompressionBC = VK_TRUE;
@@ -957,6 +958,20 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
     } else {
         std::fprintf(stderr, "[gpu] timeline semaphores unavailable or disabled; drains wait under the GPU mutex\n");
     }
+    VkPhysicalDevicePipelineExecutablePropertiesFeaturesKHR pipelineFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_EXECUTABLE_PROPERTIES_FEATURES_KHR};
+    const auto pipelineStats = std::getenv("APS5_PIPELINE_STATS");
+    if (pipelineStats && std::string_view(pipelineStats) == "1") {
+        Graphics::Require(hasExtension(VK_KHR_PIPELINE_EXECUTABLE_PROPERTIES_EXTENSION_NAME), "APS5_PIPELINE_STATS requires VK_KHR_pipeline_executable_properties");
+        VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &pipelineFeatures};
+        state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &features);
+        Graphics::Require(pipelineFeatures.pipelineExecutableInfo == VK_TRUE, "APS5_PIPELINE_STATS requires pipelineExecutableInfo");
+        state->pipelineExecutableInfo = true;
+        deviceExtensions.push_back(VK_KHR_PIPELINE_EXECUTABLE_PROPERTIES_EXTENSION_NAME);
+        deviceInfo.enabledExtensionCount = static_cast<std::uint32_t>(deviceExtensions.size());
+        deviceInfo.ppEnabledExtensionNames = deviceExtensions.data();
+        pipelineFeatures.pNext = byteFeatures.pNext;
+        byteFeatures.pNext = &pipelineFeatures;
+    }
     bdaFeatures.pNext = &byteFeatures;
     deviceInfo.pNext = &bdaFeatures;
     check(state->InstanceFunction<PFN_vkCreateDevice>("vkCreateDevice")(selected, &deviceInfo, nullptr, &state->device), "vkCreateDevice");
@@ -2305,6 +2320,7 @@ Graphics::Context VulkanDevice::buildContext() const {
     context.descriptorIndexing = state->descriptorIndexing;
     context.primitiveListRestart = state->primitiveListRestart;
     context.imageViewMinLod = state->imageViewMinLod;
+    context.pipelineExecutableInfo = state->pipelineExecutableInfo;
     return context;
 }
 
@@ -3309,6 +3325,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
         layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr;
         check(state->DeviceFunction<PFN_vkCreatePipelineLayout>("vkCreatePipelineLayout")(state->device, &layoutInfo, nullptr, &objects->layout), "vkCreatePipelineLayout");
         VkComputePipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
+        pipelineInfo.flags = context.pipelineExecutableInfo ? VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR : 0;
         pipelineInfo.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
         pipelineInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
         pipelineInfo.stage.module = objects->module;
@@ -3319,6 +3336,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
         pipelineInfo.layout = objects->layout;
         if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] creating a pipeline for %zu SPIR-V words (program 0x%llx)\n", shader.spirv.size(), static_cast<unsigned long long>(programAddress));
         check(state->DeviceFunction<PFN_vkCreateComputePipelines>("vkCreateComputePipelines")(state->device, context.pipelineCache, 1, &pipelineInfo, nullptr, &objects->pipeline), "vkCreateComputePipelines");
+        Graphics::LogPipelineStatistics_nid_no_patch(context, objects->pipeline);
         timing.Mark("pipeline_create");
         if (pipelineKey != 0) {
             // Find-or-insert: objects another dispatch of the variant mapped meanwhile serve this

core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp

@@ -135,6 +135,7 @@ struct Context {
     bool descriptorIndexing = false;
     bool primitiveListRestart = false;
     bool imageViewMinLod = false;
+    bool pipelineExecutableInfo = false;
 
     template<typename TFunction>
     TFunction Function(const char* name) const {

core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp

@@ -61,6 +61,7 @@ struct GuestTextureResource {
     std::uint64_t dccAddress = 0;
     bool dccAlphaOnMsb = false;
     std::uint32_t minLod = 0;
+    std::uint32_t allocatedMipCount = 0;
 };
 
 float EffectiveMinLod(const GuestTextureResource& resource);

core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp

@@ -8,6 +8,8 @@ namespace AgcDriver::Graphics {
 
 struct VertexInputLayout;
 
+void LogPipelineStatistics_nid_no_patch(const Context& context, VkPipeline pipeline);
+
 // The attachments of one render pass instance. Work recorded with it references the handle until
 // the batch completes, so a recorded draw keeps the object (Recorder::Keep) like its pipeline.
 class Framebuffer {

core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp

@@ -235,6 +235,7 @@ public:
     // Entries leave the table with their batch (a memory bound only). The note also enters the
     // label's range as a pending write of the open batch (the flush hook syncs CPU reads), marked
     // as the label's own so it does not count as an overwrite of the entry.
+    static constexpr std::size_t LabelTableBytes = 64;
     void NoteLabel(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue);
     // Late rule: an entry whose stamp is not newer than the wait's submission is still the value
     // memory will hold, unless a CPU store touched the dword since it was recorded (the table
@@ -285,6 +286,7 @@ public:
     static std::optional<LabelHit> LookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, LabelRefusal* refusal = nullptr);
     // The value alone, for callers asking whether any label is pending in a dword (afterStamp 0).
     static std::optional<std::uint64_t> LookupLabelValue(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp);
+    static bool WideLabelIn(std::uint64_t address, std::size_t bytes);
     // A label a queue worker decoded but has not recorded yet (Driver.cpp DeferredLabels): it
     // enters the table with no batch, and only a lookup made on the noting thread (a WAIT_REG_MEM
     // of the same queue) takes its value, since that queue's later work follows the label in queue
@@ -520,6 +522,7 @@ private:
         std::uint64_t readGeneration = 0;
         // Dword addresses this batch noted in the label table (removed when it finishes).
         std::vector<std::uint64_t> labelDwords;
+        std::vector<std::multimap<std::uint64_t, std::uint64_t>::iterator> wideLabels;
         // Labels stored by this batch's completion actions (see AfterCompletions); subtracted from
         // the lock-free pending count when the batch finishes, whether or not its completions ran.
         std::uint32_t completionLabelCount = 0;
@@ -641,6 +644,13 @@ private:
     PFN_vkEndCommandBuffer endCommandBuffer = nullptr;
     PFN_vkQueueSubmit queueSubmit = nullptr;
     PFN_vkCmdPipelineBarrier cmdPipelineBarrier = nullptr;
+    PFN_vkCmdBeginQuery cmdBeginQuery = nullptr;
+    PFN_vkCmdEndQuery cmdEndQuery = nullptr;
+    PFN_vkCmdResetQueryPool cmdResetQueryPool = nullptr;
+    PFN_vkCmdCopyQueryPoolResults cmdCopyQueryPoolResults = nullptr;
+    PFN_vkCmdBindPipeline cmdBindPipeline = nullptr;
+    PFN_vkCmdPushConstants cmdPushConstants = nullptr;
+    PFN_vkCmdDispatch cmdDispatch = nullptr;
     template<typename TFunction>
     TFunction function(TFunction resolved, const char* name) const {
         return resolved != nullptr ? resolved : context.Function<TFunction>(name);
@@ -709,6 +719,9 @@ private:
     // The queued entries (batch always null), by dword; a lookup on the noting queue's thread
     // prefers them (its newest store in program order), every other lookup sees `labels` alone.
     std::map<std::uint64_t, LabelEntry> queuedLabels;
+    std::multimap<std::uint64_t, std::uint64_t> wideLabels;
+    std::uint64_t wideLabelBytes = 0;
+    bool wideLabelInLocked(std::uint64_t address, std::size_t bytes) const;
     // labels.size(), readable without the table mutex: a noted write skips the mutex while the
     // table is empty (most of the time between label groups).
     std::atomic<std::size_t> recordedLabels{0};

core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp

@@ -10,7 +10,7 @@
 
 namespace AgcDriver::Graphics {
 
-ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& context, const std::array<std::uint8_t, 8>& exportMappings);
+ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& context, const std::array<std::uint8_t, 8>& exportMappings, bool nullProgram = false);
 ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& shader);
 // A guest range DecodeVertexStageInfo read (an attribute word, a vertex V#) with the bytes as read:
 // the draw cache validates them by value with the stage's capture (design_cpu_final rule RD).

core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp

@@ -411,6 +411,7 @@ private:
     std::vector<std::uint64_t> storageKeys;
     std::vector<bool> storageFirstLayer;
     std::vector<bool> storageWritten;
+    std::vector<bool> storageAtomic;
     std::vector<std::shared_ptr<Sampler>> samplers;
     bool reusable = false;
     std::vector<DirectRegion> directRegions;

core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp

@@ -50,6 +50,7 @@ struct ColorTarget {
     std::uint32_t slot = 0;
     std::uint32_t depth = 1;
     std::uint32_t depthSlice = 0;
+    std::uint32_t exportIndex = 0;
 };
 
 struct DepthTarget {
@@ -110,6 +111,8 @@ std::optional<ColorMetadataPass> DecodeColorMetadataPass(const QueueState& queue
 // this precheck covers, evaluated without exceptions before the draw is decoded; empty when they
 // pass (DecodeState still checks everything). A register a rule needs that is absent is no verdict.
 std::string DrawRejection(const QueueState& queue, bool indexed);
+bool PixelProgramUnset(const QueueState& queue);
+std::string NullPixelProgramRejection(const QueueState& queue);
 
 // The recording facade of the draw decoders (design_cpu_final M8, step 8a): every register read
 // of DecodeShaderStages, DecodeState, DrawRejection, DecodePixelStageInfo and Driver::draw's

core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp

@@ -114,6 +114,7 @@ public:
     // so successive mip writes of a chain share one image and one write-back.
     VkImageView View(std::uint32_t mip);
     VkImageView FirstLayerView(std::uint32_t mip);
+    VkImageView AtomicView(std::uint32_t mip, bool firstLayer);
     // Render targets live in the same images: draws attach mip 0 through a view of the color
     // buffer's format and mark the image dirty like a storage write.
     bool Attachable() const { return attachable; }
@@ -308,6 +309,27 @@ private:
     // The windows moving the surface-relative runs: whole tile blocks of thin tiled mips, a tail
     // mip whole (its block holds every tail level), in slice order.
     std::vector<SliceWindow> sliceWindows(std::span<const std::pair<std::uint64_t, std::uint64_t>> runs) const;
+    // The bytes of the surface-relative runs no element of the surface holds (the padding of partly
+    // covered tile blocks and of linear rows, tail blocks, the bytes between mips; every byte of a
+    // thick surface), ascending: a write-back keeps the guest's bytes there.
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> uncoveredBytes(std::span<const std::pair<std::uint64_t, std::uint64_t>> runs) const;
+    // The seed of a write-back's tiled scratch where its copies would carry bytes no element holds:
+    // those bytes' current contents (a fresh unit shadow's, else the import's), copied into the
+    // scratch before the retile, so the copies out of the scratch store them unchanged. `copied`
+    // lists the surface-relative [begin, end) ranges the write-back copies out of the scratch, each
+    // with the scratch offset of its begin.
+    struct CopiedBytes {
+        std::uint64_t begin;
+        std::uint64_t end;
+        std::uint64_t scratch;
+    };
+    struct PaddingSeeds {
+        std::vector<std::pair<VkBuffer, std::vector<VkBufferCopy>>> copies;
+        std::vector<std::shared_ptr<ShadowSlab>> slabs;
+        // Guest ranges read from the import.
+        std::vector<std::pair<std::uint64_t, std::uint64_t>> importReads;
+    };
+    PaddingSeeds paddingSeeds(const HostImport& import, std::vector<CopiedBytes> copied) const;
     // The direct upload and GPU-direct write-back of the runs through their windows (block units);
     // both return the bytes moved. The upload reads each unit from its unit shadow while fresh,
     // else from the import (`discard`: the image has no layout yet, a whole-surface first upload).
@@ -373,7 +395,7 @@ private:
     bool clearByKeysFill(DccKeys keys, std::uint8_t key);
     bool overlaps(std::uint64_t address, std::size_t bytes) const;
     bool pendingUnitInside(std::uint64_t address, std::size_t bytes) const;
-    VkImageView createView(std::uint32_t mip, bool firstLayer = false) const;
+    VkImageView createView(std::uint32_t mip, bool firstLayer, VkFormat format) const;
     void release() noexcept;
 
     Context context;
@@ -421,6 +443,7 @@ private:
     std::uint32_t defaultMip = 0;
     std::map<std::uint32_t, VkImageView> extraViews;
     std::map<std::uint32_t, VkImageView> firstLayerViews;
+    std::map<std::pair<std::uint32_t, bool>, VkImageView> atomicViews;
     bool attachable = false;
     std::map<std::tuple<VkFormat, std::uint32_t, std::uint32_t>, VkImageView> attachmentViews;
     VkFormat storageFormat = VK_FORMAT_UNDEFINED;

core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp

@@ -5,6 +5,7 @@
 #include <algorithm>
 #include <array>
 #include <cstdint>
+#include <utility>
 #include <vector>
 
 namespace AgcDriver::Graphics {
@@ -34,10 +35,15 @@ std::array<std::uint32_t, 3> ThickBlockExtent(TextureTileMode tileMode, std::uin
 // (the detiler's blockExtent). A mip's tiled bytes are its blocks in row-major order, blocksPerRow per
 // row (TileMipLayout), except the tail mips, which share one block. Throws for linear tiling.
 std::array<std::uint32_t, 3> ThinBlockLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement);
+// The bytes of a thin mip (mip-relative [begin, end), ascending and merged) that hold one of its
+// elements each: its whole tile blocks (every element of the block inside the mip), or its rows
+// without the pitch padding when linear. A tail mip has none (its block also holds the other tail
+// levels and their padding). A retile of the mip writes every byte of these and no other.
+std::vector<std::pair<std::uint64_t, std::uint64_t>> CoveredMipBytes(TextureTileMode tileMode, std::uint32_t bytesPerElement, const TileMipLayout& mip);
 
-// 3D surface. Each slab holds the whole mip chain, smallest level first. Depth slice z of a level is detiled
-// from slab z / blockDepth (slabBytes apart) with the swizzle's slice input set to z, into linear
-// slices sliceLinearBytes apart.
+// 3D surface. Each slab holds the whole mip chain: the mip tail block, if any, then the other levels from the
+// smallest to level 0. Depth slice z of a level is detiled from slab z / blockDepth (slabBytes apart) with the
+// swizzle's slice input set to z, into linear slices sliceLinearBytes apart.
 struct ThickLayout {
     std::vector<TileMipLayout> mips;
     std::uint32_t depth;
@@ -68,6 +74,7 @@ struct SurfaceGeometry {
     bool HasLayer(std::uint32_t level, std::uint32_t layer) const { return imageDepth <= 1 || layer < std::max(imageDepth >> level, 1u); }
 };
 SurfaceGeometry DescribeSurface(const GuestTextureResource& descriptor);
+bool LevelsFitAllocation(const GuestTextureResource& surface, std::uint32_t levels);
 
 }
 

core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp

@@ -100,7 +100,8 @@ bool AnyShadowedOverlaps(std::span<const std::pair<std::uint64_t, std::uint64_t>
 // from one buffer (a slab while the unit is fresh, else the import) at `offset` (byte x of the run
 // is at offset + (x - begin)), split at unit freshness and slab boundaries with one tracker query
 // over the span. A tail block (surface-relative, every unit of it) must have one source: a mixed
-// one is published (TailMip) and read from the import.
+// one is published (TailMip) and read from the import. `countReads` false: not counted as detiled
+// bytes (a write-back reading the current bytes of its padding).
 struct ShadowRun {
     std::uint64_t begin;
     std::uint64_t end;
@@ -109,7 +110,7 @@ struct ShadowRun {
     bool shadow;
     std::shared_ptr<ShadowSlab> slab;
 };
-std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, std::span<const std::pair<std::uint64_t, std::uint64_t>> tailBlocks);
+std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, std::span<const std::pair<std::uint64_t, std::uint64_t>> tailBlocks, bool countReads = true);
 
 // The slab destination of a retile copy [begin, end) (guest addresses inside one slab: the caller
 // splits at SlabBoundary), making the slab under the budget; nullopt when refused (the caller writes

core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp

@@ -262,7 +262,7 @@ void BdaResources::CheckFault() const {
             ShaderRecompiler::BdaAbi::Range range{};
             std::memcpy(&range, bytes.data() + sizeof(header) + index * sizeof(range), sizeof(range));
             if (report.address < range.begin || report.address >= range.end) continue;
-            message << std::hex << "; the store hit 0x" << range.begin << "+0x" << range.end - range.begin << ", read-only in the BDA table: stores through GPU-selected descriptors reach only writable ranges imported in place, not ones served by a mirror or a copy (past APS5_HOST_IMPORT_MIB, or refused by the driver)";
+            message << std::hex << "; the store hit 0x" << range.begin << "+0x" << range.end - range.begin << ", read-only in the BDA table: stores through GPU-selected descriptors and FLAT/GLOBAL addresses reach only writable ranges imported in place, not ones served by a mirror or a copy (past APS5_HOST_IMPORT_MIB, or refused by the driver)";
             break;
         }
     }
@@ -276,7 +276,7 @@ namespace AgcDriver::Graphics {
 void BdaResources::markWrittenPages() const {
     namespace Abi = ShaderRecompiler::BdaAbi;
     auto* words = reinterpret_cast<std::uint32_t*>(fault->Bytes().data());
-    Require(words[Abi::WrittenOverflowWord] == 0, "more than " + std::to_string(Abi::WrittenPageSlots) + " pages stored to through GPU-selected buffer descriptors in one use are not implemented");
+    Require(words[Abi::WrittenOverflowWord] == 0, "more than " + std::to_string(Abi::WrittenPageSlots) + " pages stored to through the BDA table in one use are not implemented");
     bool any = false;
     for (std::uint32_t slot = 0; slot < Abi::WrittenPageSlots; ++slot) {
         const auto page = words[Abi::WrittenSlotsWord + slot];

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -1312,7 +1312,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
     for (const auto view : record.targetViews) mix(reinterpret_cast<std::uint64_t>(view));
     if (state.blends.size() != state.colors.size()) {
         mix(state.blends.size());
-        for (const auto& color : state.colors) mix(color.slot);
+        for (const auto& color : state.colors) mix(color.exportIndex);
     }
     mix(state.renderExtent.width);
     mix(state.renderExtent.height);
@@ -1558,7 +1558,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters&
             }
         }
         timer.phase(PhaseReadTarget);
-        binding.target = std::make_unique<RenderTarget>(context, color, state.blends.at(color.slot).blendEnable != 0);
+        binding.target = std::make_unique<RenderTarget>(context, color, state.blends.at(color.exportIndex).blendEnable != 0);
         targetViews.push_back(binding.target->View());
     }
     if (state.depth) targetViews.push_back(DepthSurfaceView(context, *state.depth));

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "ThreadOwned.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
@@ -671,7 +672,8 @@ void refreshMirror(ImageMirror& mirror, std::uint64_t address, std::uint64_t byt
 void refreshHeapMirrors(std::vector<ImageMirror*>& mirrors, std::vector<RefreshBlock>& blocks) {
     constexpr std::uint64_t block = 65536;
     std::sort(mirrors.begin(), mirrors.end(), [](const ImageMirror* left, const ImageMirror* right) { return left->base < right->base; });
-    thread_local std::vector<std::uint8_t> changed;
+    thread_local std::vector<std::uint8_t>* changedSlot = nullptr;
+    auto& changed = ShaderRecompiler::ThreadOwned(changedSlot);
     for (std::size_t first = 0; first < mirrors.size();) {
         auto last = first + 1;
         while (last < mirrors.size() && mirrors[last]->base == mirrors[last - 1]->base + mirrors[last - 1]->bytes) ++last;

core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp

@@ -1,6 +1,7 @@
 #include <cstdio>
 #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
 #include <algorithm>
 #include <stdexcept>
 #include <string>
@@ -129,10 +130,9 @@ GuestTextureResource DecodeTextureResource(std::span<const std::uint32_t> words)
     } catch (const std::exception& error) {
         throw std::runtime_error(std::string(error.what()) + describe());
     }
-    Require(baseLevel <= maxMip, "guest texture descriptor starts past the surface's last mip level" + describe());
     // Views may name levels past MAX_MIP (a 512x512 view through 1x1 over a 9-level surface); the
     // hardware never addresses them, so the view ends at the surface's last level.
-    lastLevel = std::min(lastLevel, maxMip);
+    if (baseLevel <= maxMip) lastLevel = std::min(lastLevel, maxMip);
     // XOR swizzles fold a pipe/bank XOR into the low address bits; only unmodified 64 KiB bases are modeled.
     Require(XorSwizzleMode(tileMode) == 0 || (baseAddress & 0xffffu) == 0, "guest texture descriptor combines an XOR swizzle with a pipe/bank XOR base which is not implemented");
 
@@ -164,6 +164,7 @@ GuestTextureResource DecodeTextureResource(std::span<const std::uint32_t> words)
     result.depthOrLastArray = depth;
     result.baseArray = baseArray;
     result.mipCount = maxMip + 1u;
+    result.allocatedMipCount = maxMip + 1u;
     result.baseLevel = baseLevel;
     result.lastLevel = lastLevel;
     result.tileMode = tileMode;
@@ -176,6 +177,10 @@ GuestTextureResource DecodeTextureResource(std::span<const std::uint32_t> words)
     result.dccAddress = metaCompress ? metaAddr << 8u : 0u;
     result.dccAlphaOnMsb = dccAlphaPos;
     result.minLod = minLod;
+    if (baseLevel > maxMip) {
+        Require(LevelsFitAllocation(result, lastLevel + 1u), "guest texture descriptor starts past the surface's last mip level at levels that would move the surface's own" + describe());
+        result.mipCount = lastLevel + 1u;
+    }
     return result;
 }
 

core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp

@@ -14,6 +14,51 @@
 
 namespace AgcDriver::Graphics {
 
+void LogPipelineStatistics_nid_no_patch(const Context& context, VkPipeline pipeline) {
+    if (!context.pipelineExecutableInfo) return;
+    const auto getProperties = context.Function<PFN_vkGetPipelineExecutablePropertiesKHR>("vkGetPipelineExecutablePropertiesKHR");
+    const auto getStatistics = context.Function<PFN_vkGetPipelineExecutableStatisticsKHR>("vkGetPipelineExecutableStatisticsKHR");
+    VkPipelineInfoKHR pipelineInfo{VK_STRUCTURE_TYPE_PIPELINE_INFO_KHR};
+    pipelineInfo.pipeline = pipeline;
+    std::uint32_t count = 0;
+    Check(getProperties(context.device, &pipelineInfo, &count, nullptr), "vkGetPipelineExecutablePropertiesKHR count");
+    std::vector<VkPipelineExecutablePropertiesKHR> properties(count, {VK_STRUCTURE_TYPE_PIPELINE_EXECUTABLE_PROPERTIES_KHR});
+    Check(getProperties(context.device, &pipelineInfo, &count, properties.data()), "vkGetPipelineExecutablePropertiesKHR");
+    properties.resize(count);
+    for (std::uint32_t index = 0; index < properties.size(); ++index) {
+        VkPipelineExecutableInfoKHR executable{VK_STRUCTURE_TYPE_PIPELINE_EXECUTABLE_INFO_KHR};
+        executable.pipeline = pipeline;
+        executable.executableIndex = index;
+        count = 0;
+        Check(getStatistics(context.device, &executable, &count, nullptr), "vkGetPipelineExecutableStatisticsKHR count");
+        std::vector<VkPipelineExecutableStatisticKHR> statistics(count, {VK_STRUCTURE_TYPE_PIPELINE_EXECUTABLE_STATISTIC_KHR});
+        Check(getStatistics(context.device, &executable, &count, statistics.data()), "vkGetPipelineExecutableStatisticsKHR");
+        statistics.resize(count);
+        const auto& property = properties[index];
+        std::fprintf(stderr, "[pipeline-stats] executable=%u stages=0x%x subgroup=%u name=%s\n", index, property.stages, property.subgroupSize, property.name);
+        for (const auto& statistic : statistics) {
+            std::fprintf(stderr, "[pipeline-stats] executable=%u %s=", index, statistic.name);
+            switch (statistic.format) {
+            case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_BOOL32_KHR:
+                std::fprintf(stderr, "%s", statistic.value.b32 ? "true" : "false");
+                break;
+            case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_INT64_KHR:
+                std::fprintf(stderr, "%lld", static_cast<long long>(statistic.value.i64));
+                break;
+            case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR:
+                std::fprintf(stderr, "%llu", static_cast<unsigned long long>(statistic.value.u64));
+                break;
+            case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_FLOAT64_KHR:
+                std::fprintf(stderr, "%.17g", statistic.value.f64);
+                break;
+            default:
+                throw std::runtime_error("unsupported pipeline statistic format");
+            }
+            std::fprintf(stderr, " (%s)\n", statistic.description);
+        }
+    }
+}
+
 Framebuffer::Framebuffer(const Context& context, VkRenderPass renderPass, std::span<const VkImageView> targets, VkExtent2D extent) : context(context) {
     // Cached objects outlive their device's teardown; they must not keep its buffer pool alive past it.
     this->context.bufferPool.reset();
@@ -49,7 +94,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
     // A cached pipeline may outlive its device's teardown (see ClearCachedPipelines); it must not keep
     // the buffer pool, which is reset with the device, alive past it.
     this->context.bufferPool.reset();
-    Require(state.blends.size() == (state.colors.empty() ? 0u : state.colors.back().slot + 1u) && state.colors.size() <= state.blends.size(), "blend states do not match decoded color state");
+    Require(state.blends.size() == (state.colors.empty() ? 0u : state.colors.back().exportIndex + 1u) && state.colors.size() <= state.blends.size(), "blend states do not match decoded color state");
     Require(state.blends.size() <= context.limits.maxColorAttachments, "color targets exceed device attachment limits");
     Require(state.hasColorTarget || (context.limits.framebufferNoAttachmentsSampleCounts & VK_SAMPLE_COUNT_1_BIT) != 0, "device does not support single-sample rendering without attachments");
     Require(!depthBounds || context.depthBounds, "device does not support the depth bounds test");
@@ -106,7 +151,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
             color.initialLayout = attachmentLayout;
             color.finalLayout = attachmentLayout;
             colors.push_back(color);
-            references.at(state.colors[index].slot) = {index, attachmentLayout};
+            references.at(state.colors[index].exportIndex) = {index, attachmentLayout};
         }
         VkSubpassDescription subpass{};
         subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS;
@@ -177,6 +222,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
         blend.pAttachments = state.blends.empty() ? nullptr : state.blends.data();
         std::copy(state.blendConstants.begin(), state.blendConstants.end(), blend.blendConstants);
         VkGraphicsPipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO};
+        pipelineInfo.flags = context.pipelineExecutableInfo ? VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR : 0;
         pipelineInfo.stageCount = static_cast<std::uint32_t>(stages.size());
         pipelineInfo.pStages = stages.data();
         VkPipelineTessellationStateCreateInfo tessellation{VK_STRUCTURE_TYPE_PIPELINE_TESSELLATION_STATE_CREATE_INFO};
@@ -195,6 +241,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
         pipelineInfo.layout = layout;
         pipelineInfo.renderPass = renderPass;
         Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines");
+        LogPipelineStatistics_nid_no_patch(context, pipeline);
     } catch (...) {
         release();
         throw;
@@ -355,7 +402,7 @@ std::vector<std::byte> pipelineKey(const Context& context, const State& state, c
     append(key, state.colors.size());
     for (const auto& color : state.colors) append(key, color.format);
     if (state.blends.size() != state.colors.size()) {
-        for (const auto& color : state.colors) append(key, color.slot);
+        for (const auto& color : state.colors) append(key, color.exportIndex);
     }
     append(key, state.depth.has_value());
     if (state.depth) {

core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp

@@ -1074,6 +1074,13 @@ Recorder::Recorder(const Context& context, bool timelineSemaphores) : context(co
         endCommandBuffer = context.Function<PFN_vkEndCommandBuffer>("vkEndCommandBuffer");
         queueSubmit = context.Function<PFN_vkQueueSubmit>("vkQueueSubmit");
         cmdPipelineBarrier = context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier");
+        cmdBeginQuery = context.Function<PFN_vkCmdBeginQuery>("vkCmdBeginQuery");
+        cmdEndQuery = context.Function<PFN_vkCmdEndQuery>("vkCmdEndQuery");
+        cmdResetQueryPool = context.Function<PFN_vkCmdResetQueryPool>("vkCmdResetQueryPool");
+        cmdCopyQueryPoolResults = context.Function<PFN_vkCmdCopyQueryPoolResults>("vkCmdCopyQueryPoolResults");
+        cmdBindPipeline = context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline");
+        cmdPushConstants = context.Function<PFN_vkCmdPushConstants>("vkCmdPushConstants");
+        cmdDispatch = context.Function<PFN_vkCmdDispatch>("vkCmdDispatch");
     }
     if (!timelineSemaphores) return;
     // The timeline starts at 0 and every Submit signals its serial (1, 2, ...): a value that only
@@ -1208,6 +1215,20 @@ std::optional<Recorder::LabelHit> Recorder::LookupLabel(std::uint64_t address, s
     return labelTableOwner->lookupLabel(address, bytes, afterStamp, refusal);
 }
 
+bool Recorder::WideLabelIn(std::uint64_t address, std::size_t bytes) {
+    std::lock_guard tableLock(labelTableMutex);
+    return labelTableOwner != nullptr && labelTableOwner->wideLabelInLocked(address, bytes);
+}
+
+bool Recorder::wideLabelInLocked(std::uint64_t address, std::size_t bytes) const {
+    if (wideLabels.empty() || bytes == 0) return false;
+    const auto end = address + bytes;
+    for (auto it = wideLabels.lower_bound(address >= wideLabelBytes ? address - wideLabelBytes + 1 : 0); it != wideLabels.end() && it->first < end; ++it) {
+        if (it->second > address) return true;
+    }
+    return false;
+}
+
 std::optional<std::uint64_t> Recorder::LookupLabelValue(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp) {
     const auto hit = LookupLabel(address, bytes, afterStamp);
     if (!hit.has_value()) return std::nullopt;
@@ -1837,7 +1858,7 @@ std::uint32_t Recorder::beginTiming(std::uint64_t key) {
                 return NoTiming;
             }
         }
-        context.Function<PFN_vkCmdResetQueryPool>("vkCmdResetQueryPool")(commands, open->queries, 0, queryCount);
+        function(cmdResetQueryPool, "vkCmdResetQueryPool")(commands, open->queries, 0, queryCount);
     }
     if (open->timedKeys.size() >= queryCount / 2) {
         timingDropped.fetch_add(1, std::memory_order_relaxed);
@@ -1926,7 +1947,7 @@ bool Recorder::gpuSampleCounter() {
 
 void Recorder::endSamples(Batch& batch) {
     if (!batch.sampleActive) return;
-    context.Function<PFN_vkCmdEndQuery>("vkCmdEndQuery")(batch.commands, batch.samples, 0);
+    function(cmdEndQuery, "vkCmdEndQuery")(batch.commands, batch.samples, 0);
     batch.sampleActive = false;
     if (sampleCounterState > 0 && batch.samplesDrawn) pendingSamples.push_back({batch.samplePool, batch.samples});
     batch.samplesDrawn = false;
@@ -1934,10 +1955,10 @@ void Recorder::endSamples(Batch& batch) {
 
 void Recorder::foldSamples(Batch& batch, VkDeviceAddress target) {
     const auto commands = batch.commands;
-    const auto copy = context.Function<PFN_vkCmdCopyQueryPoolResults>("vkCmdCopyQueryPoolResults");
-    const auto bind = context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline");
-    const auto push = context.Function<PFN_vkCmdPushConstants>("vkCmdPushConstants");
-    const auto dispatch = context.Function<PFN_vkCmdDispatch>("vkCmdDispatch");
+    const auto copy = function(cmdCopyQueryPoolResults, "vkCmdCopyQueryPoolResults");
+    const auto bind = function(cmdBindPipeline, "vkCmdBindPipeline");
+    const auto push = function(cmdPushConstants, "vkCmdPushConstants");
+    const auto dispatch = function(cmdDispatch, "vkCmdDispatch");
     std::size_t done = 0;
     do {
         const auto count = static_cast<std::uint32_t>(std::min<std::size_t>(SampleFoldCapacity, pendingSamples.size() - done));
@@ -2011,15 +2032,15 @@ bool Recorder::RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress rec
     }
     if (meshArgumentState < 0) return false;
     recordBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
-    context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, meshArgumentPipeline);
+    function(cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, meshArgumentPipeline);
     struct {
         VkDeviceAddress record;
         VkDeviceAddress arguments;
         std::array<std::uint32_t, 8> rules;
     } parameters{record, arguments, {rules[0], rules[1], rules[2], rules[3], rules[4], rules[5], rules[6], 0u}};
     static_assert(sizeof(parameters) == PushBytes);
-    context.Function<PFN_vkCmdPushConstants>("vkCmdPushConstants")(commands, meshArgumentLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, PushBytes, &parameters);
-    context.Function<PFN_vkCmdDispatch>("vkCmdDispatch")(commands, 1, 1, 1);
+    function(cmdPushConstants, "vkCmdPushConstants")(commands, meshArgumentLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, PushBytes, &parameters);
+    function(cmdDispatch, "vkCmdDispatch")(commands, 1, 1, 1);
     recordBarrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_MESH_SHADER_BIT_EXT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_SHADER_READ_BIT);
     CountBarriers(CommandClass::Draw, 2);
     return true;
@@ -2049,8 +2070,8 @@ void Recorder::beginSamples(Batch& batch) {
             throw;
         }
     }
-    context.Function<PFN_vkCmdResetQueryPool>("vkCmdResetQueryPool")(batch.commands, batch.samples, 0, 1);
-    context.Function<PFN_vkCmdBeginQuery>("vkCmdBeginQuery")(batch.commands, batch.samples, 0, context.occlusionQueryPrecise ? VK_QUERY_CONTROL_PRECISE_BIT : 0u);
+    function(cmdResetQueryPool, "vkCmdResetQueryPool")(batch.commands, batch.samples, 0, 1);
+    function(cmdBeginQuery, "vkCmdBeginQuery")(batch.commands, batch.samples, 0, context.occlusionQueryPrecise ? VK_QUERY_CONTROL_PRECISE_BIT : 0u);
     batch.sampleActive = true;
     batch.samplesDrawn = false;
 }
@@ -2501,7 +2522,16 @@ void Recorder::NoteLabel(std::uint64_t address, std::span<const std::byte> bytes
     ensureOpen();
     // The table entry before the write note: the note bumps the write generation a poller
     // watches, and a poller that sees the bump then finds the label without the GPU mutex.
-    noteLabelOn(*open, address, bytes, stamp, queue);
+    const bool tabled = bytes.size() <= LabelTableBytes;
+    if (tabled) {
+        noteLabelOn(*open, address, bytes, stamp, queue);
+    } else {
+        std::lock_guard tableLock(labelTableMutex);
+        labels.erase(labels.lower_bound(address), labels.lower_bound(address + bytes.size()));
+        recordedLabels.store(labels.size(), std::memory_order_relaxed);
+        open->wideLabels.push_back(wideLabels.emplace(address, address + bytes.size()));
+        wideLabelBytes = std::max<std::uint64_t>(wideLabelBytes, bytes.size());
+    }
     if (activeRecorder == this && pendingLabelSince.load(std::memory_order_relaxed) == NoPendingLabel) {
         pendingLabelSince.store(std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_release);
     }
@@ -2524,6 +2554,7 @@ bool Recorder::PendingLabelIn(std::uint64_t address, std::size_t bytes) const {
     if (bytes == 0) return false;
     const auto end = address + bytes;
     if (const auto first = labels.lower_bound(address); first != labels.end() && first->first < end) return true;
+    if (wideLabelInLocked(address, bytes)) return true;
     // A queued label of another queue is unordered against the caller on hardware (nothing that
     // queue recorded could have satisfied a wait yet); it is still reported, since the table mutex
     // is held anyway and a caller deciding a CPU store wants the conservative answer.
@@ -3256,15 +3287,17 @@ void Recorder::finish(std::unique_ptr<Batch> batch, bool wait, int source) {
     }
     // The batch's label entries leave the table (a later label to the same dword already replaced
     // its entry and belongs to another batch). Correctness never depended on this removal.
-    if (!batch->labelDwords.empty()) {
+    if (!batch->labelDwords.empty() || !batch->wideLabels.empty()) {
         std::lock_guard tableLock(labelTableMutex);
         for (const auto dword : batch->labelDwords) {
             const auto found = labels.find(dword);
             if (found != labels.end() && found->second.batch == batch.get()) labels.erase(found);
         }
+        for (const auto entry : batch->wideLabels) wideLabels.erase(entry);
         recordedLabels.store(labels.size(), std::memory_order_relaxed);
     }
     batch->labelDwords.clear();
+    batch->wideLabels.clear();
     release(*batch);
 }
 

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp

@@ -25,6 +25,7 @@ constexpr std::uint32_t spiPsInputAddr = 0x1B4;
 constexpr std::uint32_t spiPsInControl = 0x1B6;
 constexpr std::uint32_t dbShaderControl = 0x203;
 constexpr std::uint32_t spiShaderColFormat = 0x1C5;
+constexpr std::uint32_t defaultPixelInputs = 0x2u;
 
 std::uint32_t read(const Registers& registers, std::uint32_t offset, RegisterBank bank) {
     NoteRegisterRead(bank, offset);
@@ -37,6 +38,12 @@ std::uint32_t read(const Registers& registers, std::uint32_t offset, RegisterBan
     return it->second;
 }
 
+std::uint32_t readOr(const Registers& registers, std::uint32_t offset, RegisterBank bank, std::uint32_t fallback) {
+    NoteRegisterRead(bank, offset);
+    const auto it = registers.find(offset);
+    return it == registers.end() ? fallback : it->second;
+}
+
 template <typename T> T _readHeaderPod(std::span<const std::byte> header, std::uint64_t headerAddress, const void* pointer) {
     if (pointer == nullptr) throw std::runtime_error("AGC graphics: null AGC header pointer");
     const auto address = reinterpret_cast<std::uint64_t>(pointer);
@@ -84,14 +91,14 @@ ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers&
     };
 }
 
-ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& context, const std::array<std::uint8_t, 8>& exportMappings) {
+ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& context, const std::array<std::uint8_t, 8>& exportMappings, bool nullProgram) {
     const auto inControl = read(context, spiPsInControl, RegisterBank::Context);
     const auto inputNum = inControl & 0x3Fu;
     if (inputNum > 32u) {
         throw std::runtime_error("AGC graphics: SPI_PS_IN_CONTROL input count exceeds 32");
     }
-    const auto ena = read(context, spiPsInputEna, RegisterBank::Context);
-    const auto addr = read(context, spiPsInputAddr, RegisterBank::Context);
+    const auto ena = nullProgram ? readOr(context, spiPsInputEna, RegisterBank::Context, defaultPixelInputs) : read(context, spiPsInputEna, RegisterBank::Context);
+    const auto addr = nullProgram ? readOr(context, spiPsInputAddr, RegisterBank::Context, defaultPixelInputs) : read(context, spiPsInputAddr, RegisterBank::Context);
     const auto activeInputs = ena & addr;
     using ShaderRecompiler::PixelInput;
     using ShaderRecompiler::PixelInputBit;
@@ -114,7 +121,7 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con
     if (((shaderControl >> 13u) & 0x3u) != 0) {
         throw std::runtime_error("AGC graphics: DB_SHADER_CONTROL.CONSERVATIVE_Z_EXPORT is unsupported");
     }
-    const auto colFormat = read(context, spiShaderColFormat, RegisterBank::Context);
+    const auto colFormat = nullProgram ? readOr(context, spiShaderColFormat, RegisterBank::Context, 0u) : read(context, spiShaderColFormat, RegisterBank::Context);
     std::array<std::uint8_t, 8> targetOutputMode{};
     for (std::uint32_t i = 0; i < 8u; ++i) {
         targetOutputMode[i] = static_cast<std::uint8_t>((colFormat >> (4u * i)) & 0xFu);

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp"
+#include "ThreadOwned.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
@@ -186,14 +187,19 @@ struct LookupRecord {
     const StorageTexture* source;
 };
 
-thread_local std::vector<LookupRecord> lookupLog;
+thread_local std::vector<LookupRecord>* lookupLogSlot = nullptr;
+
+std::vector<LookupRecord>& lookupLog() {
+    return ShaderRecompiler::ThreadOwned(lookupLogSlot);
+}
 
 void logLookup(const LookupRecord& record) {
     // The bound drops the oldest half, never everything: one build's lookups (a few dozen at most)
     // are the newest records, and a capture must find them even when the bound trips mid-build.
     constexpr std::size_t bound = 1024;
-    if (lookupLog.size() >= bound) lookupLog.erase(lookupLog.begin(), lookupLog.begin() + bound / 2);
-    lookupLog.push_back(record);
+    auto& log = lookupLog();
+    if (log.size() >= bound) log.erase(log.begin(), log.begin() + bound / 2);
+    log.push_back(record);
 }
 
 std::shared_ptr<StorageTexture> cachedStorageTexture(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource, std::uint32_t mip, std::uint64_t guestBytes = 0);
@@ -511,22 +517,64 @@ std::array<std::uint32_t, 8> SurfaceKey(const Context& context, const GuestTextu
     return {static_cast<std::uint32_t>(resource.baseAddress), static_cast<std::uint32_t>(resource.baseAddress >> 32u), resource.width, resource.height, (resource.depthOrLastArray << 16u) | (resource.mipCount & 0xffffu), (static_cast<std::uint32_t>(resource.tileMode) << 12u) | (static_cast<std::uint32_t>(resource.dimension) << 20u), resource.baseArray, static_cast<std::uint32_t>(StorageFormatForGuest(context, resource.format))};
 }
 
+struct ExtendedSurfaces {
+    std::mutex mutex;
+    std::map<std::array<std::uint32_t, 8>, std::uint32_t> levels;
+    std::atomic<bool> any{false};
+};
+
+ExtendedSurfaces& ExtendedChains() {
+    static ExtendedSurfaces surfaces;
+    return surfaces;
+}
+
+std::uint32_t AllocatedLevels(const GuestTextureResource& resource) {
+    return resource.allocatedMipCount != 0 ? resource.allocatedMipCount : resource.mipCount;
+}
+
+GuestTextureResource StorageSurface(const Context& context, const GuestTextureResource& viewed) {
+    auto& surfaces = ExtendedChains();
+    const bool extended = viewed.mipCount > AllocatedLevels(viewed);
+    if (!extended && !surfaces.any.load(std::memory_order_acquire)) return viewed;
+    auto surface = viewed;
+    surface.mipCount = AllocatedLevels(viewed);
+    const auto identity = SurfaceKey(context, surface);
+    std::lock_guard lock(surfaces.mutex);
+    if (extended) {
+        auto& levels = surfaces.levels[identity];
+        levels = std::max(levels, viewed.mipCount);
+        surfaces.any.store(true, std::memory_order_release);
+        surface.mipCount = levels;
+    } else if (const auto it = surfaces.levels.find(identity); it != surfaces.levels.end()) {
+        surface.mipCount = it->second;
+    } else {
+        surface.mipCount = viewed.mipCount;
+    }
+    return surface;
+}
+
 // `guestBytes` is the surface size when the caller described the surface already (0: described here).
-std::shared_ptr<StorageTexture> cachedStorageTexture(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource, std::uint32_t mip, std::uint64_t guestBytes) {
-    if (DepthSurfaceAt(resource.baseAddress)) {
+std::shared_ptr<StorageTexture> cachedStorageTexture(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& viewed, std::uint32_t mip, std::uint64_t guestBytes) {
+    if (DepthSurfaceAt(viewed.baseAddress)) {
         char text[112];
-        std::snprintf(text, sizeof(text), "AGC graphics: storage image access to depth/stencil surface 0x%llx is not implemented", static_cast<unsigned long long>(resource.baseAddress));
+        std::snprintf(text, sizeof(text), "AGC graphics: storage image access to depth/stencil surface 0x%llx is not implemented", static_cast<unsigned long long>(viewed.baseAddress));
         throw std::runtime_error(text);
     }
     static const bool disabled = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr;
-    if (disabled) return std::make_shared<StorageTexture>(context, *context.detiler, resource, mip);
+    if (disabled) return std::make_shared<StorageTexture>(context, *context.detiler, viewed, mip);
     static_cast<void>(words);
     const bool profile = LookupOutcomes::Profiled();
     auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
     auto& counters = TextureCounts();
+    const auto resource = StorageSurface(context, viewed);
     const StorageKey key{context.device, SurfaceKey(context, resource)};
     auto& cache = StorageTextures();
     std::lock_guard lock(cache.mutex);
+    if (resource.mipCount > AllocatedLevels(resource)) {
+        auto allocated = resource;
+        allocated.mipCount = AllocatedLevels(resource);
+        if (const auto found = findStorage(cache, {context.device, SurfaceKey(context, allocated)}); found != cache.entries.end()) evictStorage(cache, found);
+    }
     auto it = findStorage(cache, key);
     if (it != cache.entries.end() && MetadataMoved(*it->texture, resource)) {
         evictStorage(cache, it);
@@ -793,7 +841,8 @@ struct ThreadBuildProfile {
 };
 
 ThreadBuildProfile& ThreadBuilds() {
-    thread_local ThreadBuildProfile profile;
+    thread_local ThreadBuildProfile* profileSlot = nullptr;
+    auto& profile = ShaderRecompiler::ThreadOwned(profileSlot);
     return profile;
 }
 
@@ -1007,7 +1056,7 @@ void ShaderResources::buildComplete() {
                         break;
                     case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
                         write.pImageInfo = images.data() + images.size();
-                        for (const auto index : binding.imageAllocations) images.push_back({VK_NULL_HANDLE, storageFirstLayer[index] ? storageTextures[index]->FirstLayerView(storageMips[index]) : storageTextures[index]->View(storageMips[index]), VK_IMAGE_LAYOUT_GENERAL});
+                        for (const auto index : binding.imageAllocations) images.push_back({VK_NULL_HANDLE, storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageFirstLayer[index] ? storageTextures[index]->FirstLayerView(storageMips[index]) : storageTextures[index]->View(storageMips[index]), VK_IMAGE_LAYOUT_GENERAL});
                         break;
                     case VK_DESCRIPTOR_TYPE_SAMPLER:
                         write.pImageInfo = images.data() + images.size();
@@ -1152,6 +1201,7 @@ std::vector<std::uint32_t> ShaderResources::ContentKey(const CompiledShader& sha
         packBits(binding.imageWritten);
         packBits(binding.samplerDepthCompare);
         packBits(binding.imageDepthCompare);
+        packBits(binding.imageAtomic);
         // Read-only elements are bound without a write set: an object built for one written set
         // must not serve a build with another (the variant implies it, this makes it explicit).
         packBits(binding.bufferWritten);
@@ -1310,7 +1360,8 @@ bool EpochRevalidate() {
 // ValidatedSurface); the last record of an object is the freshest.
 void ShaderResources::captureValidation() {
     const auto record = [](const void* object) -> const LookupRecord* {
-        for (auto it = lookupLog.rbegin(); it != lookupLog.rend(); ++it) {
+        const auto& log = lookupLog();
+        for (auto it = log.rbegin(); it != log.rend(); ++it) {
             if (it->object == object) return &*it;
         }
         return nullptr;
@@ -1319,7 +1370,7 @@ void ShaderResources::captureValidation() {
     for (std::size_t i = 0; i < textures.size(); ++i) {
         if (const auto* found = record(textures[i].get())) validatedTextures[i] = {found->resource, found->bytes, found->keys, found->generation, 0, found->source, true};
     }
-    lookupLog.clear();
+    lookupLog().clear();
 }
 
 // Proves every texture and storage image still current without repeating its lookup: exactly the
@@ -1350,12 +1401,17 @@ bool ShaderResources::fastRevalidate(std::uint64_t serialBefore, std::span<const
     if (validatedTextures.size() != textures.size()) return fail(FastFail::NoRecord);
     const bool unchanged = pendingSerialSeen != 0 && pendingSerialSeen == serialBefore;
     const bool keyProofs = KeyFastPath();
-    thread_local std::vector<GuestMemory::UnchangedQuery> queries;
-    thread_local std::vector<StorageTexture::PendingQuery> pending;
+    thread_local std::vector<GuestMemory::UnchangedQuery>* queriesSlot = nullptr;
+    auto& queries = ShaderRecompiler::ThreadOwned(queriesSlot);
+    thread_local std::vector<StorageTexture::PendingQuery>* pendingSlot = nullptr;
+    auto& pending = ShaderRecompiler::ThreadOwned(pendingSlot);
     // The element each pending query stands for, in query order.
-    thread_local std::vector<PendingOverlap> owners;
-    thread_local std::vector<const StorageTexture*> images;
-    thread_local std::vector<DccKeys> scannedKeys;
+    thread_local std::vector<PendingOverlap>* ownersSlot = nullptr;
+    auto& owners = ShaderRecompiler::ThreadOwned(ownersSlot);
+    thread_local std::vector<const StorageTexture*>* imagesSlot = nullptr;
+    auto& images = ShaderRecompiler::ThreadOwned(imagesSlot);
+    thread_local std::vector<DccKeys>* scannedKeysSlot = nullptr;
+    auto& scannedKeys = ShaderRecompiler::ThreadOwned(scannedKeysSlot);
     queries.clear();
     pending.clear();
     owners.clear();
@@ -1628,7 +1684,7 @@ bool ShaderResources::Revalidate(std::span<const CompiledShader> shaders, ProofR
     // they must hand back the very objects the set's views belong to. The build appended them stage
     // by stage, binding by binding, so the walk repeats that order.
     const auto fullWalk = [&] {
-        lookupLog.clear();
+        lookupLog().clear();
         std::size_t textureIndex = 0;
         std::size_t storageIndex = 0;
         for (const auto& shader : shaders) {
@@ -1664,8 +1720,10 @@ bool ShaderResources::Revalidate(std::span<const CompiledShader> shaders, ProofR
         captureValidation();
         return true;
     };
-    thread_local std::vector<PendingOverlap> overlapping;
-    thread_local std::vector<PendingOverlap> refreshed;
+    thread_local std::vector<PendingOverlap>* overlappingSlot = nullptr;
+    auto& overlapping = ShaderRecompiler::ThreadOwned(overlappingSlot);
+    thread_local std::vector<PendingOverlap>* refreshedSlot = nullptr;
+    auto& refreshed = ShaderRecompiler::ThreadOwned(refreshedSlot);
     refreshed.clear();
     FastFail reason = FastFail::Count;
     bool accepted = false;
@@ -1711,7 +1769,8 @@ bool ShaderResources::Revalidate(std::span<const CompiledShader> shaders, ProofR
             // The walk after the refresh, or beside a proof that accepted a foreign overlap, must
             // find every object in place and current: another object, or an upload (a moved
             // content version), is a decision the proof got wrong.
-            thread_local std::vector<std::pair<const StorageTexture*, std::uint64_t>> versions;
+            thread_local std::vector<std::pair<const StorageTexture*, std::uint64_t>>* versionsSlot = nullptr;
+            auto& versions = ShaderRecompiler::ThreadOwned(versionsSlot);
             versions.clear();
             for (const auto& surface : validatedTextures) {
                 if (surface.source != nullptr) versions.emplace_back(surface.source, surface.source->Version());
@@ -1744,7 +1803,8 @@ bool ShaderResources::Revalidate(std::span<const CompiledShader> shaders, ProofR
     };
     if (EpochRevalidate()) {
         if (!directRegions.empty() && !(pendingSerialSeen != 0 && pendingSerialSeen == StorageTexture::PendingSerial())) {
-            thread_local std::vector<StorageTexture::PendingQuery> regions;
+            thread_local std::vector<StorageTexture::PendingQuery>* regionsSlot = nullptr;
+            auto& regions = ShaderRecompiler::ThreadOwned(regionsSlot);
             regions.clear();
             for (const auto& region : directRegions) regions.push_back({region.begin, region.end, nullptr, nullptr, false});
             StorageTexture::ScanPending(regions);
@@ -2505,6 +2565,7 @@ void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBind
         storageFirstLayer.push_back(firstLayer);
         // Images the shader only reads have nothing to store back.
         storageWritten.push_back(element >= binding.imageWritten.size() || binding.imageWritten[element]);
+        storageAtomic.push_back(element < binding.imageAtomic.size() && binding.imageAtomic[element]);
         describedRanges.push_back({"storage", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast<int>(resource.tileMode), resource.dccAddress});
         item.imageAllocations.push_back(storageTextures.size() - 1);
     }

core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp

@@ -70,7 +70,7 @@ constexpr std::uint32_t ScanModeMask = ~2u;
 constexpr std::uint32_t ScanControlMask = ~0x06003fffu;
 constexpr std::uint32_t ScreenOffsetMask = ~0x01ff01ffu;
 // Bits 26/27 (ZCLIP_NEAR/FAR_DISABLE) become depth clamping; bit 19 selects the [0, 1] clip space.
-constexpr std::uint32_t ClipControlMask = ~(0x80000u | 0x0c000000u);
+constexpr std::uint32_t ClipControlMask = ~(0x80000u | 0x01000000u | 0x0c000000u);
 
 // Debug aid: APS5_IGNORE_DEPTH_TEST=1 renders depth- and stencil-tested draws without a depth
 // target as if their tests always passed (wrong occlusion, but the draws run), so stages that
@@ -501,28 +501,34 @@ State DecodeState(const QueueState& queue) {
     // matters where the shader exports.
     const auto targetMask = read(cx, 0x8e) & shaderMask;
     APS5_LOG_OUT_DEBUG("CB_TARGET_MASK=0x%x CB_SHADER_MASK=0x%x", targetMask, shaderMask);
-    std::uint32_t slotCount = 0;
+    std::vector<std::uint32_t> exportSlots;
     for (std::uint32_t slot = 0; slot < 8; ++slot) {
-        if (((targetMask >> (4u * slot)) & 0xfu) != 0) slotCount = slot + 1;
+        if (((shaderMask >> (4u * slot)) & 0xfu) != 0) exportSlots.push_back(slot);
     }
     const auto written = [&](std::uint32_t slot) { return ((targetMask >> (4u * slot)) & 0xfu) != 0; };
-    result.hasColorTarget = slotCount != 0;
-    APS5_LOG_OUT_DEBUG("hasColorTarget=%u slots=%u", result.hasColorTarget ? 1u : 0u, slotCount);
+    std::uint32_t exportCount = 0;
+    for (std::uint32_t index = 0; index < exportSlots.size(); ++index) {
+        if (written(exportSlots[index])) exportCount = index + 1;
+    }
+    result.hasColorTarget = exportCount != 0;
+    APS5_LOG_OUT_DEBUG("hasColorTarget=%u exports=%u", result.hasColorTarget ? 1u : 0u, exportCount);
 
     // CB_COLOR_CONTROL mode 0 disables color writes, which only matters when a target is written.
     if (const auto colorControl = read(cx, 0x202); !colorControlSupported(colorControl, result.hasColorTarget)) throw std::runtime_error(colorControlMessage(colorControl));
     zero(cx, 0x1c4, ~0u, "depth or sample-mask export");
-    const auto exportFormat = read(cx, 0x1c5);
+    const auto exportFormat = result.hasColorTarget ? read(cx, 0x1c5) : 0u;
     APS5_LOG_OUT_DEBUG("Export format=%u", exportFormat);
     // SPI_SHADER_POS_FORMAT: POS0 must be a 4-component position; later vectors carry the misc/clip
     // exports that PA_CL_VS_OUT_CNTL validation above already limits to ignored layer/viewport data.
     Require((read(cx, 0x1c3) & 0xfu) == 4, "additional position exports are unsupported");
-    for (std::uint32_t slot = 0; slot < slotCount; ++slot) {
+    for (std::uint32_t index = 0; index < exportCount; ++index) {
+        const auto slot = exportSlots[index];
         if (!written(slot)) continue;
         // Export formats only matter for the targets the draw writes.
-        const auto slotExport = (exportFormat >> (4u * slot)) & 0xfu;
+        const auto slotExport = (exportFormat >> (4u * index)) & 0xfu;
         if (slotExport == 0 || slotExport == 7 || slotExport == 8 || slotExport > 9) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " is unsupported");
-        const auto color = DecodeColorBuffer(cx, slot);
+        auto color = DecodeColorBuffer(cx, slot);
+        color.exportIndex = index;
         APS5_LOG_OUT_DEBUG("Color %u address=0x%llx extent=%ux%u bytes=%llu VkFormat=%u", slot, static_cast<unsigned long long>(color.address), color.extent.width, color.extent.height, static_cast<unsigned long long>(color.bytes), static_cast<unsigned>(color.format));
         if (result.colors.empty()) {
             result.renderExtent = color.extent;
@@ -566,7 +572,7 @@ State DecodeState(const QueueState& queue) {
     intersect(result.scissor, cx, 0x90, false);
     if ((read(cx, 0x292) & 2u) != 0) intersect(result.scissor, cx, 0x94, false);
     APS5_LOG_OUT_DEBUG("Scissor offset=(%d,%d) extent=%ux%u", result.scissor.offset.x, result.scissor.offset.y, result.scissor.extent.width, result.scissor.extent.height);
-    result.blends.assign(slotCount, VkPipelineColorBlendAttachmentState{});
+    result.blends.assign(exportCount, VkPipelineColorBlendAttachmentState{});
     for (const auto& color : result.colors) {
         const auto slot = color.slot;
         const auto blend = read(cx, 0x1e0 + slot);
@@ -595,9 +601,9 @@ State DecodeState(const QueueState& queue) {
             }
             for (std::uint32_t i = 0; i < 4; ++i) result.blendConstants[i] = readFloat(cx, 0x105 + i);
         }
-        result.blends[slot] = state;
+        result.blends[color.exportIndex] = state;
     }
-    if (!result.colors.empty()) result.blend = result.blends[result.colors.front().slot];
+    if (!result.colors.empty()) result.blend = result.blends[result.colors.front().exportIndex];
     APS5_LOG_OUT_DEBUG("DecodeState done colorTarget=%u render=%ux%u topology=%u", result.hasColorTarget ? 1u : 0u, result.renderExtent.width, result.renderExtent.height, static_cast<unsigned>(result.topology));
     return result;
 }
@@ -606,7 +612,7 @@ std::array<std::uint8_t, 8> ExportMappings(const State& state) {
     std::array<std::uint8_t, 8> mappings{};
     mappings.fill(0xe4u);
     for (const auto& color : state.colors) {
-        if (color.slot < mappings.size()) mappings[color.slot] = color.componentMapping;
+        if (color.exportIndex < mappings.size()) mappings[color.exportIndex] = color.componentMapping;
     }
     return mappings;
 }
@@ -770,8 +776,7 @@ std::string DrawRejection(const QueueState& queue, bool indexed) {
     std::uint32_t targetMask = 0, shaderMask = 0;
     if (value(cx, 0x8e, targetMask) && value(cx, 0x8f, shaderMask) && value(cx, 0x202, word) && !colorControlSupported(word, (targetMask & shaderMask) != 0)) return colorControlMessage(word);
     if (auto reason = nonzero(cx, 0x1c4, ~0u, "depth or sample-mask export"); !reason.empty()) return reason;
-    // The pixel stage decode (ShaderInputState.cpp) reads these after DecodeState and the program
-    // prepare; a bank without them fails there with this message.
+    if (PixelProgramUnset(queue)) return NullPixelProgramRejection(queue);
     for (const auto offset : {0x1b3u, 0x1b4u, 0x1c5u}) {
         if (find(cx, offset) != cx.end()) continue;
         char text[64];
@@ -781,6 +786,20 @@ std::string DrawRejection(const QueueState& queue, bool indexed) {
     return {};
 }
 
+bool PixelProgramUnset(const QueueState& queue) {
+    const auto low = find(queue.shader, 0x008, RegisterBank::Shader);
+    const auto high = find(queue.shader, 0x009, RegisterBank::Shader);
+    return low != queue.shader.end() && high != queue.shader.end() && low->second == 0 && high->second == 0;
+}
+
+std::string NullPixelProgramRejection(const QueueState& queue) {
+    const auto targetMask = find(queue.context, 0x8e);
+    const auto shaderMask = find(queue.context, 0x8f);
+    if (targetMask == queue.context.end() || shaderMask == queue.context.end()) return "AGC graphics: a draw without a pixel program needs CB_TARGET_MASK and CB_SHADER_MASK";
+    if ((targetMask->second & shaderMask->second) != 0) return "AGC graphics: a draw without a pixel program writes color";
+    return {};
+}
+
 std::vector<RegisterRead>*& RegisterReadLog() {
     static thread_local std::vector<RegisterRead>* log = nullptr;
     return log;

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -691,7 +691,7 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler,
         uploadReason = "first";
         upload();
         defaultMip = mipLevel;
-        view = createView(mipLevel);
+        view = createView(mipLevel, false, storageFormat);
         {
             auto& live = Live();
             std::lock_guard lock(live.mutex);
@@ -942,15 +942,18 @@ bool AdjacentGenerationEnabled() {
 
 }
 
-VkImageView StorageTexture::createView(std::uint32_t mip, bool firstLayer) const {
+VkImageView StorageTexture::createView(std::uint32_t mip, bool firstLayer, VkFormat format) const {
     Require(mip < descriptor.mipCount, "storage texture mip level is outside the texture");
     Require(!firstLayer || descriptor.dimension == TextureDimension::k2DArray, "a first-layer storage view needs a 2D array surface");
     const auto viewLayerCount = firstLayer ? 1u : geometry.imageLayers - descriptor.baseArray;
+    VkImageViewUsageCreateInfo usage{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO};
+    usage.usage = VK_IMAGE_USAGE_STORAGE_BIT;
     VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
+    viewInfo.pNext = format == storageFormat ? nullptr : &usage;
     viewInfo.image = image;
     // Storage views address one mip; cube faces are written as array layers.
     viewInfo.viewType = firstLayer ? VK_IMAGE_VIEW_TYPE_2D : descriptor.dimension == TextureDimension::k1D ? VK_IMAGE_VIEW_TYPE_1D : descriptor.dimension == TextureDimension::k2D ? VK_IMAGE_VIEW_TYPE_2D : descriptor.dimension == TextureDimension::k3D ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D_ARRAY;
-    viewInfo.format = storageFormat;
+    viewInfo.format = format;
     viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY};
     viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, mip, 1u, descriptor.baseArray, viewLayerCount};
     VkImageView created = VK_NULL_HANDLE;
@@ -984,7 +987,7 @@ VkImageView StorageTexture::View(std::uint32_t mip) {
     if (mip == defaultMip) return view;
     const auto found = extraViews.find(mip);
     if (found != extraViews.end()) return found->second;
-    const auto created = createView(mip);
+    const auto created = createView(mip, false, storageFormat);
     extraViews.emplace(mip, created);
     return created;
 }
@@ -992,11 +995,21 @@ VkImageView StorageTexture::View(std::uint32_t mip) {
 VkImageView StorageTexture::FirstLayerView(std::uint32_t mip) {
     const auto found = firstLayerViews.find(mip);
     if (found != firstLayerViews.end()) return found->second;
-    const auto created = createView(mip, true);
+    const auto created = createView(mip, true, storageFormat);
     firstLayerViews.emplace(mip, created);
     return created;
 }
 
+VkImageView StorageTexture::AtomicView(std::uint32_t mip, bool firstLayer) {
+    if (storageFormat == VK_FORMAT_R32_UINT) return firstLayer ? FirstLayerView(mip) : View(mip);
+    Require(storageFormat == VK_FORMAT_R32_SINT || storageFormat == VK_FORMAT_R32_SFLOAT, "storage image atomics need a surface of one 32-bit component");
+    const auto found = atomicViews.find({mip, firstLayer});
+    if (found != atomicViews.end()) return found->second;
+    const auto created = createView(mip, firstLayer, VK_FORMAT_R32_UINT);
+    atomicViews.emplace(std::pair{mip, firstLayer}, created);
+    return created;
+}
+
 const char* LookupOutcomes::Name(Kind kind) {
     static constexpr const char* names[Count] = {"sampled fast hit", "sampled fast miss", "sampled hit view", "sampled hit cleared view", "sampled hit snapshot", "sampled made view", "sampled made snapshot", "storage hit", "storage made", "refresh unchanged", "refresh compared", "upload direct", "upload cpu", "upload clear", "dcc scan", "pending flush"};
     return kind < Count ? names[kind] : "?";
@@ -1617,6 +1630,83 @@ std::vector<StorageTexture::SliceWindow> StorageTexture::sliceWindows(std::span<
     return windows;
 }
 
+std::vector<std::pair<std::uint64_t, std::uint64_t>> StorageTexture::uncoveredBytes(std::span<const std::pair<std::uint64_t, std::uint64_t>> runs) const {
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> covered;
+    if (!geometry.thick) {
+        const auto elementBytes = BytesPerElement(descriptor.format);
+        for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) {
+            for (std::uint32_t level = 0; level < descriptor.mipCount; ++level) {
+                if (!geometry.HasLayer(level, layer)) continue;
+                const auto sliceBegin = geometry.GuestLayerOffset(layer) + mips[level].tiledOffset;
+                for (const auto& [begin, end] : CoveredMipBytes(descriptor.tileMode, elementBytes, mips[level])) covered.emplace_back(sliceBegin + begin, sliceBegin + end);
+            }
+        }
+        std::sort(covered.begin(), covered.end());
+    }
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> sorted(runs.begin(), runs.end());
+    std::sort(sorted.begin(), sorted.end());
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> uncovered;
+    const auto add = [&](std::uint64_t begin, std::uint64_t end) {
+        if (begin >= end) return;
+        if (!uncovered.empty() && uncovered.back().second >= begin) uncovered.back().second = std::max(uncovered.back().second, end);
+        else uncovered.emplace_back(begin, end);
+    };
+    std::size_t next = 0;
+    for (const auto& [runBegin, runEnd] : sorted) {
+        while (next < covered.size() && covered[next].second <= runBegin) ++next;
+        auto at = runBegin;
+        for (auto i = next; i < covered.size() && covered[i].first < runEnd; ++i) {
+            add(at, std::min(covered[i].first, runEnd));
+            at = std::max(at, covered[i].second);
+        }
+        add(at, runEnd);
+    }
+    return uncovered;
+}
+
+StorageTexture::PaddingSeeds StorageTexture::paddingSeeds(const HostImport& import, std::vector<CopiedBytes> copied) const {
+    PaddingSeeds seeds;
+    std::sort(copied.begin(), copied.end(), [](const CopiedBytes& a, const CopiedBytes& b) { return a.begin < b.begin; });
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> runs;
+    for (const auto& range : copied) runs.emplace_back(range.begin, range.end);
+    const auto uncovered = uncoveredBytes(runs);
+    if (uncovered.empty()) return seeds;
+    std::vector<CopiedBytes> pieces;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> pieceRuns;
+    std::size_t next = 0;
+    for (const auto& [begin, end] : uncovered) {
+        while (next < copied.size() && copied[next].end <= begin) ++next;
+        for (auto i = next; i < copied.size() && copied[i].begin < end; ++i) {
+            const auto from = std::max(begin, copied[i].begin);
+            const auto to = std::min(end, copied[i].end);
+            if (from >= to) continue;
+            pieces.push_back({from, to, copied[i].scratch + (from - copied[i].begin)});
+            pieceRuns.emplace_back(from, to);
+        }
+    }
+    if (pieces.empty()) return seeds;
+    const auto sources = ShadowSources(context, import, descriptor.baseAddress, pieceRuns, {}, false);
+    std::size_t first = 0;
+    for (const auto& source : sources) {
+        while (first < pieces.size() && pieces[first].end <= source.begin) ++first;
+        for (auto i = first; i < pieces.size() && pieces[i].begin < source.end; ++i) {
+            const auto from = std::max(source.begin, pieces[i].begin);
+            const auto to = std::min(source.end, pieces[i].end);
+            if (from >= to) continue;
+            auto target = std::find_if(seeds.copies.begin(), seeds.copies.end(), [&](const auto& entry) { return entry.first == source.buffer; });
+            if (target == seeds.copies.end()) target = seeds.copies.insert(seeds.copies.end(), {source.buffer, {}});
+            target->second.push_back({source.offset + (from - source.begin), pieces[i].scratch + (from - pieces[i].begin), to - from});
+            if (!source.shadow) {
+                const auto guestBegin = descriptor.baseAddress + from;
+                if (!seeds.importReads.empty() && seeds.importReads.back().second == guestBegin) seeds.importReads.back().second = descriptor.baseAddress + to;
+                else seeds.importReads.emplace_back(guestBegin, descriptor.baseAddress + to);
+            }
+        }
+        if (source.shadow && std::find(seeds.slabs.begin(), seeds.slabs.end(), source.slab) == seeds.slabs.end()) seeds.slabs.push_back(source.slab);
+    }
+    return seeds;
+}
+
 std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, bool discard) {
     const auto elementBytes = BytesPerElement(descriptor.format);
     // Each run's pieces by source (a unit shadow's slab while fresh, else the import), the tail
@@ -1808,6 +1898,17 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
         }
     }
     std::sort(seeds.begin(), seeds.end(), [](const ShadowedRange& a, const ShadowedRange& b) { return a.begin < b.begin; });
+    std::vector<CopiedBytes> copied;
+    for (std::size_t i = 0; i < windows.size(); ++i) {
+        const auto& window = windows[i];
+        if (i != 0 && scratchPositions[i] == scratchPositions[i - 1]) continue;
+        for (const auto& [from, to] : runs) {
+            const auto begin = std::max(from, window.tiledBegin);
+            const auto end = std::min(to, window.tiledEnd);
+            if (begin < end) copied.push_back({begin, end, scratchPositions[i] + (begin - window.tiledBegin)});
+        }
+    }
+    const auto padding = paddingSeeds(import, std::move(copied));
     // A queued label or key store inside a seeded unit lands before the seed copies the import.
     auto flushBegin = firstStored;
     auto flushEnd = lastStored;
@@ -1816,7 +1917,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
         flushEnd = std::max(flushEnd, seed.end);
     }
     auto linear = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
-    auto tiledScratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
+    auto tiledScratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
     detiler.BeginBatch();
     auto* recorder = Recorder::Active();
     std::unique_ptr<CommandBatch> batch;
@@ -1831,12 +1932,14 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
         recorder->Keep(linear);
         recorder->Keep(tiledScratch);
         for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab);
+        for (const auto& slab : padding.slabs) recorder->Keep(slab);
         if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self));
         Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4);
         if (Recorder::BarrierValidate()) {
             const std::pair<VkImage, bool> read{image, false};
             std::vector<std::pair<std::uint64_t, std::uint64_t>> seedReads;
             for (const auto& seed : seeds) seedReads.emplace_back(seed.begin, seed.end);
+            seedReads.insert(seedReads.end(), padding.importReads.begin(), padding.importReads.end());
             recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{seedReads, imported, std::span(&read, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT});
         }
     } else {
@@ -1853,7 +1956,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
     toSource.image = image;
     toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers};
     context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource);
-    if (!seeds.empty()) {
+    if (!seeds.empty() || !padding.copies.empty()) {
         // The seeded units' import bytes (every earlier writer of them, host stores included)
         // precede the seed copies; the previous command's trailing barrier may have covered that.
         constexpr VkAccessFlags transferAccess = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
@@ -1869,6 +1972,9 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
             context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, import.buffer, slab->buffer, 1, &seed);
             NoteShadowSeed(begin, end);
         }
+        // The padding's current bytes, under the retile that follows (the import-ready barrier
+        // orders the scratch writes).
+        for (const auto& [source, copies] : padding.copies) context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast<std::uint32_t>(copies.size()), copies.data());
     }
     std::vector<VkBufferImageCopy> regions;
     for (std::size_t i = 0; i < windows.size(); ++i) {
@@ -3224,7 +3330,10 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
         // untouched blocks are copied into the imported bytes in place, recorded behind the work that
         // produced the image; nothing crosses to the CPU.
         auto linear = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(sliceLinearBytes * arrayLayers), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
-        auto tiledScratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
+        auto tiledScratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
+        std::vector<CopiedBytes> copied;
+        for (const auto& [from, to] : keep) copied.push_back({from - descriptor.baseAddress, to - descriptor.baseAddress, from - descriptor.baseAddress});
+        const auto padding = paddingSeeds(*import, std::move(copied));
         detiler.BeginBatch();
         auto* recorder = Recorder::Active();
         std::unique_ptr<CommandBatch> batch;
@@ -3239,12 +3348,13 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
             timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack);
             recorder->Keep(linear);
             recorder->Keep(tiledScratch);
+            for (const auto& slab : padding.slabs) recorder->Keep(slab);
             // The image itself must outlive the recorded retile: the cache may evict it right after.
             if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self));
-            Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4);
+            Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, padding.copies.empty() ? 4 : 5);
             if (Recorder::BarrierValidate()) {
                 const std::pair<VkImage, bool> read{image, false};
-                recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{{}, keep, std::span(&read, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT});
+                recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{padding.importReads, keep, std::span(&read, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT});
             }
         } else {
             batch = std::make_unique<CommandBatch>(context);
@@ -3264,6 +3374,13 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
         toSource.image = image;
         toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers};
         context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource);
+        if (!padding.copies.empty()) {
+            // The current bytes of the copied ranges no element holds go under the retile (every
+            // earlier writer of the import, host stores included, first; the import-ready barrier
+            // below orders the scratch writes).
+            RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
+            for (const auto& [source, copies] : padding.copies) context.Function<PFN_vkCmdCopyBuffer>("vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast<std::uint32_t>(copies.size()), copies.data());
+        }
         const auto regions = CopyRegions(storedLayers);
         context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast<std::uint32_t>(regions.size()), regions.data());
         const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
@@ -3455,6 +3572,8 @@ void StorageTexture::release() noexcept {
     extraViews.clear();
     for (const auto& [mip, extra] : firstLayerViews) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, extra, nullptr);
     firstLayerViews.clear();
+    for (const auto& [key, atomic] : atomicViews) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, atomic, nullptr);
+    atomicViews.clear();
     for (const auto& [format, attachment] : attachmentViews) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, attachment, nullptr);
     attachmentViews.clear();
     if (view) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, view, nullptr);

core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp

@@ -255,6 +255,30 @@ std::array<std::uint32_t, 3> ThinBlockLayout(TextureTileMode tileMode, std::uint
     return {block.blockSize, block.blockWidth, block.blockHeight};
 }
 
+std::vector<std::pair<std::uint64_t, std::uint64_t>> CoveredMipBytes(TextureTileMode tileMode, std::uint32_t bytesPerElement, const TileMipLayout& mip) {
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> covered;
+    if (mip.tail || mip.width == 0 || mip.height == 0) return covered;
+    const auto add = [&](std::uint64_t begin, std::uint64_t end) {
+        end = std::min(end, mip.tiledSize);
+        if (begin >= end) return;
+        if (!covered.empty() && covered.back().second == begin) covered.back().second = end;
+        else covered.emplace_back(begin, end);
+    };
+    if (tileMode == TextureTileMode::kLinear) {
+        const auto rowBytes = static_cast<std::uint64_t>(mip.width) * bytesPerElement;
+        for (std::uint32_t row = 0; row < mip.height; ++row) add(static_cast<std::uint64_t>(row) * mip.pitchBytes, static_cast<std::uint64_t>(row) * mip.pitchBytes + rowBytes);
+        return covered;
+    }
+    const auto block = ThinBlockLayout(tileMode, bytesPerElement);
+    const auto wholeColumns = std::min(mip.width / block[1], mip.blocksPerRow);
+    const auto wholeRows = mip.height / block[2];
+    for (std::uint32_t row = 0; row < wholeRows && wholeColumns != 0; ++row) {
+        const auto first = static_cast<std::uint64_t>(row) * mip.blocksPerRow;
+        add(first * block[0], (first + wholeColumns) * block[0]);
+    }
+    return covered;
+}
+
 ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t depth, std::uint32_t mipCount) {
     Require(width != 0 && height != 0 && depth != 0, "cannot compute layout for a zero-sized 3D texture");
     Require(mipCount != 0 && mipCount <= 16u, "3D texture mip count is out of range");
@@ -265,6 +289,8 @@ ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, s
     result.mips.resize(mipCount);
     std::array<std::uint32_t, 3> block{CalcLinearBlockWidth(bytesPerElement), 1u, 1u};
     std::uint32_t blockBytes = 0;
+    std::uint32_t maxTailLevels = 0;
+    auto firstTailLevel = mipCount;
     if (tileMode != TextureTileMode::kLinear) {
         block = ThickBlockExtent(tileMode, bytesPerElement);
         blockBytes = tileMode == TextureTileMode::kStandard4KB ? 4096u : 65536u;
@@ -272,18 +298,22 @@ ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, s
         const auto tailWidth = log2BlockBytes % 3u == 1u ? block[0] >> 1u : block[0];
         const auto tailHeight = log2BlockBytes % 3u == 0u ? block[1] >> 1u : block[1];
         const auto effectiveLog2 = log2BlockBytes - (log2BlockBytes - 8u) / 3u;
-        const auto maxTailLevels = effectiveLog2 <= 11u ? 1u + (1u << (effectiveLog2 - 9u)) : effectiveLog2 - 4u;
-        for (std::uint32_t level = 0; mipCount > 1 && level < mipCount; ++level)
-            Require(std::max(width >> level, 1u) > tailWidth || std::max(height >> level, 1u) > tailHeight || mipCount - level > maxTailLevels, "3D texture mip tails are not implemented");
+        maxTailLevels = effectiveLog2 <= 11u ? 1u + (1u << (effectiveLog2 - 9u)) : effectiveLog2 - 4u;
+        for (std::uint32_t level = 0; mipCount > 1 && level < mipCount; ++level) {
+            if (ShiftCeil(width, level) <= tailWidth && ShiftCeil(height, level) <= tailHeight && mipCount - level <= maxTailLevels) {
+                firstTailLevel = level;
+                break;
+            }
+        }
     }
     result.blockDepth = block[2];
-    std::uint64_t tiledOffset = 0;
+    std::uint64_t tiledOffset = firstTailLevel < mipCount ? blockBytes : 0u;
     std::uint64_t linearOffset = 0;
     for (auto level = mipCount; level-- > 0;) {
         auto& mip = result.mips[level];
         mip.width = std::max(width >> level, 1u);
         mip.height = std::max(height >> level, 1u);
-        mip.tail = false;
+        mip.tail = level >= firstTailLevel;
         mip.tailX = 0;
         mip.tailY = 0;
         const bool tiled = tileMode != TextureTileMode::kLinear;
@@ -291,17 +321,34 @@ ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, s
         const auto levelHeight = tiled ? std::max((height + (1u << level) - 1u) >> level, 1u) : mip.height;
         const auto paddedWidth = AlignUp(levelWidth, block[0]);
         mip.pitchBytes = paddedWidth * bytesPerElement;
-        if (!tiled) {
+        if (mip.tail) {
+            constexpr std::uint32_t microBlock[5][2] = {{8, 4}, {4, 4}, {4, 4}, {4, 2}, {2, 2}};
+            const auto element = static_cast<std::size_t>(std::countr_zero(bytesPerElement));
+            const auto slot = maxTailLevels - 1u - (level - firstTailLevel);
+            const auto slotOffset = slot > 6u ? 16u << slot : slot << 8u;
+            for (std::uint32_t bit = 0; bit < 6u; ++bit) {
+                mip.tailX |= ((slotOffset >> (9u + 2u * bit)) & 1u) << bit;
+                mip.tailY |= ((slotOffset >> (8u + 2u * bit)) & 1u) << bit;
+            }
+            mip.tailX *= microBlock[element][0];
+            mip.tailY *= microBlock[element][1];
+            mip.blocksPerRow = 1u;
+            mip.pitchBytes = block[0] * bytesPerElement;
+            mip.tiledOffset = 0;
+            mip.tiledSize = blockBytes;
+        } else if (!tiled) {
             mip.blocksPerRow = paddedWidth;
             mip.tiledSize = static_cast<std::uint64_t>(mip.pitchBytes) * mip.height;
         } else {
             mip.blocksPerRow = paddedWidth / block[0];
             mip.tiledSize = static_cast<std::uint64_t>(mip.blocksPerRow) * (AlignUp(levelHeight, block[1]) / block[1]) * blockBytes;
         }
+        if (!mip.tail) {
+            mip.tiledOffset = tiledOffset;
+            tiledOffset += mip.tiledSize;
+        }
         mip.linearSize = static_cast<std::uint64_t>(mip.pitchBytes) * mip.height;
-        mip.tiledOffset = tiledOffset;
         mip.linearOffset = linearOffset;
-        tiledOffset += mip.tiledSize;
         linearOffset += mip.linearSize;
     }
     result.slabBytes = tiledOffset;
@@ -345,6 +392,25 @@ SurfaceGeometry DescribeSurface(const GuestTextureResource& descriptor) {
     return geometry;
 }
 
+bool LevelsFitAllocation(const GuestTextureResource& surface, std::uint32_t levels) {
+    Require(levels >= surface.mipCount && levels <= 16u, "a view's mip chain must cover the surface's levels and at most 16");
+    auto view = surface;
+    view.mipCount = levels;
+    const auto allocated = DescribeSurface(surface);
+    const auto extended = DescribeSurface(view);
+    if (allocated.guestBytes != extended.guestBytes || allocated.layerBytes != extended.layerBytes || allocated.layers != extended.layers) return false;
+    for (std::uint32_t level = 0; level < surface.mipCount; ++level) {
+        const auto& before = allocated.mips[level];
+        const auto& after = extended.mips[level];
+        if (before.tiledOffset != after.tiledOffset || before.tiledSize != after.tiledSize || before.blocksPerRow != after.blocksPerRow || before.tail != after.tail || before.tailX != after.tailX || before.tailY != after.tailY) return false;
+    }
+    for (auto level = surface.mipCount; level < levels; ++level) {
+        const auto& mip = extended.mips[level];
+        if (mip.tiledOffset + mip.tiledSize > extended.layerBytes) return false;
+    }
+    return true;
+}
+
 std::vector<TileMipLayout> ComputeElementMipLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount) {
     Require(width != 0 && height != 0 && mipCount != 0 && mipCount <= 16u, "invalid surface mip chain");
     if (tileMode == TextureTileMode::kLinear) return ComputeLinearMipLayout(bytesPerElement, 1u, 1u, width, height, mipCount);

core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp

@@ -592,12 +592,12 @@ std::optional<ShadowDestination> ShadowDestinationFor(const Context& context, co
     return destination;
 }
 
-std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, std::span<const std::pair<std::uint64_t, std::uint64_t>> tailBlocks) {
+std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, std::span<const std::pair<std::uint64_t, std::uint64_t>> tailBlocks, bool countReads) {
     std::vector<ShadowRun> result;
     if (runs.empty()) return result;
     const auto importPiece = [&](std::uint64_t begin, std::uint64_t end) {
         result.push_back({begin, end, import.buffer, surfaceBase + begin - import.base, false, nullptr});
-        Stats().detiledImportBytes.fetch_add(end - begin, std::memory_order_relaxed);
+        if (countReads) Stats().detiledImportBytes.fetch_add(end - begin, std::memory_order_relaxed);
     };
     std::shared_ptr<UnitShadow> shadow;
     if (UnitShadowEnabled()) {
@@ -608,7 +608,7 @@ std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& i
     }
     if (shadow == nullptr) {
         for (const auto& [begin, end] : runs) importPiece(begin, end);
-        Stats().detiledImport.fetch_add(1, std::memory_order_relaxed);
+        if (countReads) Stats().detiledImport.fetch_add(1, std::memory_order_relaxed);
         return result;
     }
     const auto first = shadow->UnitOf(surfaceBase + runs.front().first);
@@ -663,6 +663,7 @@ std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& i
             (isFresh ? shadowBytes : importBytes) += pieceEnd - pieceBegin;
         }
     }
+    if (!countReads) shadowBytes = importBytes = 0;
     if (shadowBytes != 0) {
         Stats().detiledShadow.fetch_add(1, std::memory_order_relaxed);
         Stats().detiledShadowBytes.fetch_add(shadowBytes, std::memory_order_relaxed);

core/libs/prx/libSceAgcDriver/Unimplemented.cpp

@@ -85,26 +85,22 @@ int APS5_VABI sceAgcDriverUnregisterWorkloadStream() {
  return 0;
 }
 
-APS5_EXPORT("+iAOE3jCnkk", sceAgcDriverUnknown__PiAOE3jCnkk);
-int APS5_VABI sceAgcDriverUnknown__PiAOE3jCnkk() {
+int APS5_VABI sceAgcDriverSetSubmitValidationMode() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-APS5_EXPORT("ICaGtkEIXTk", sceAgcDriverUnknown_ICaGtkEIXTk);
-int APS5_VABI sceAgcDriverUnknown_ICaGtkEIXTk() {
+int APS5_VABI sceAgcDriverGetSubmitValidationConfig() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-APS5_EXPORT("WHIOMbb+iIU", sceAgcDriverUnknown_WHIOMbb_PiIU);
-int APS5_VABI sceAgcDriverUnknown_WHIOMbb_PiIU() {
+int APS5_VABI sceAgcDriverGetSubmitValidationMode() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-APS5_EXPORT("chJWZcNSzjk", sceAgcDriverUnknown_chJWZcNSzjk);
-int APS5_VABI sceAgcDriverUnknown_chJWZcNSzjk() {
+int APS5_VABI sceAgcDriverSetSubmitValidationConfig() {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }

core/libs/prx/libSceAgcDriver/tests/Flip.cpp

@@ -114,6 +114,12 @@ void testFlipAndBoundary() {
     Packet rejected{rollback.data(), 12, 0, {}};
     expectFailure([&] { sceAgcDriverSubmitDcb(&rejected); });
     check(output->state->alive == 0 && output->state->ready == 0, "rejected submission retained or executed a reservation");
+    alignas(4) static std::uint32_t condition = 1;
+    const auto conditionAddress = reinterpret_cast<std::uintptr_t>(&condition);
+    std::array<std::uint32_t, 11> guarded{0xc0032200, static_cast<std::uint32_t>(conditionAddress), static_cast<std::uint32_t>(conditionAddress >> 32u), 0, 6, 0xc004105c, 7, 0, 1, 0, 0};
+    Packet conditionalFlip{guarded.data(), 11, 0, {}};
+    check(expectFailure([&] { sceAgcDriverSubmitDcb(&conditionalFlip); }).find("a flip inside a conditional execution range") != std::string::npos, "a flip inside a COND_EXEC range was not rejected");
+    check(output->state->alive == 0 && output->state->ready == 0, "a rejected conditional flip retained or executed a reservation");
     {
         std::lock_guard lock(output->state->mutex);
         output->state->block = true;

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -191,6 +191,25 @@ void stateTests() {
     log.clear();
     static_cast<void>(AgcDriver::Graphics::DecodeState(queue));
     Require(log.empty(), "the register facade recorded without a log");
+    queue = makeState();
+    queue.context.erase(0x1b3);
+    queue.context.erase(0x1b4);
+    queue.context.erase(0x1c5);
+    queue.shader[0x008] = 0x100;
+    queue.shader[0x009] = 0;
+    Require(!AgcDriver::Graphics::PixelProgramUnset(queue), "a pixel program address was read as unset");
+    Require(AgcDriver::Graphics::DrawRejection(queue, true).find("missing register at DWORD 0x1b3") != std::string::npos, "a real pixel program without SPI_PS_INPUT_ENA was accepted");
+    expectFailure([&] { AgcDriver::Graphics::DecodePixelStageInfo(queue.context, std::array<std::uint8_t, 8>{0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u}); }, "missing register");
+    queue.shader[0x008] = 0;
+    Require(AgcDriver::Graphics::PixelProgramUnset(queue), "a zero pixel program address was not read as unset");
+    Require(AgcDriver::Graphics::DrawRejection(queue, true).find("writes color") != std::string::npos, "a draw without a pixel program that writes color was accepted");
+    queue.context[0x8e] = 0;
+    Require(AgcDriver::Graphics::DrawRejection(queue, true).empty(), "a depth-only draw without a pixel program was rejected");
+    state = AgcDriver::Graphics::DecodeState(queue);
+    Require(!state.hasColorTarget, "a depth-only draw without a pixel program decoded a color target");
+    const auto unset = AgcDriver::Graphics::DecodePixelStageInfo(queue.context, AgcDriver::Graphics::ExportMappings(state), true);
+    Require(unset.inputAddr == ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PerspectiveCenter), "unset pixel inputs of the null program did not read as PERSP_CENTER_ENA");
+    for (const auto mode : unset.targetOutputMode) Require(mode == 0, "an unset SPI_SHADER_COL_FORMAT exported a color");
 }
 
 void hardwareScreenOffsetTests() {
@@ -449,6 +468,36 @@ void DisabledColorTests() {
     expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "missing register");
 }
 
+void CompactedExportTests() {
+    alignas(256) static std::array<std::byte, 1024> slotFourMemory{};
+    const auto slotFour = reinterpret_cast<std::uintptr_t>(slotFourMemory.data());
+    auto queue = makeState();
+    for (const auto offset : {0x31bu, 0x31cu, 0x31du}) queue.context[offset + 4u * 0xfu] = queue.context.at(offset);
+    for (const auto offset : {0x3b0u, 0x3b8u}) queue.context[offset + 4u] = queue.context.at(offset);
+    queue.context[0x318 + 4u * 0xfu] = static_cast<std::uint32_t>(slotFour >> 8u);
+    queue.context[0x390 + 4u] = static_cast<std::uint32_t>(slotFour >> 40u);
+    queue.context[0x1e4] = 0x40010001u;
+    for (std::uint32_t i = 0; i < 4; ++i) queue.context[0x105 + i] = 0;
+    queue.context[0x8e] = 0x3000fu;
+    queue.context[0x8f] = 0xf000fu;
+    queue.context[0x1c5] = 0x44u;
+    auto state = AgcDriver::Graphics::DecodeState(queue);
+    Require(state.colors.size() == 2 && state.blends.size() == 2, "two compacted exports did not give two attachments");
+    Require(state.colors[0].slot == 0 && state.colors[0].exportIndex == 0 && state.colors[0].address == reinterpret_cast<std::uintptr_t>(colorMemory.data()), "export 0 did not reach MRT slot 0");
+    Require(state.colors[1].slot == 4 && state.colors[1].exportIndex == 1 && state.colors[1].address == slotFour, "export 1 did not reach MRT slot 4, the second slot CB_SHADER_MASK enables");
+    Require(!state.blends[0].blendEnable && state.blends[1].blendEnable && state.blends[1].colorWriteMask == (VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT), "export 1 did not take MRT slot 4's blend control and target mask");
+    Require(AgcDriver::Graphics::ExportMappings(state)[1] == state.colors[1].componentMapping, "export 1 did not take MRT slot 4's component mapping");
+    queue.context[0x1c5] = 0x90009u;
+    expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "color export format 0");
+    queue.context[0x8e] = 0xf000fu;
+    queue.context[0x8f] = 0xf0f0fu;
+    queue.context[0x1c5] = 0x999u;
+    state = AgcDriver::Graphics::DecodeState(queue);
+    Require(state.colors.size() == 2 && state.blends.size() == 3, "a write-masked export between written ones changed the attachment count");
+    Require(state.colors[0].exportIndex == 0 && state.colors[1].slot == 4 && state.colors[1].exportIndex == 2, "export 2 did not reach MRT slot 4, the third slot CB_SHADER_MASK enables");
+    Require(state.blends[1].colorWriteMask == 0 && !state.blends[1].blendEnable && state.blends[2].blendEnable, "the write-masked export 1 was not left unused");
+}
+
 void DepthStencilTests() {
     auto queue = makeState();
     queue.context[0x000] = 0;
@@ -679,9 +728,23 @@ void DepthClipTests() {
     expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "inverted viewport depth clamp");
     queue.context[0xb4] = 0;
     for (std::uint32_t bit = 0; bit < 32; ++bit) {
-        if (bit == 19 || bit == 26 || bit == 27) continue;
-        queue.context[0x204] = 1u << bit;
-        expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "PA_CL_CLIP_CNTL");
+        if (bit == 19 || bit == 24 || bit == 26 || bit == 27) continue;
+        for (const auto linearBit : {0u, 0x01000000u}) {
+            queue.context[0x204] = (1u << bit) | linearBit;
+            expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "PA_CL_CLIP_CNTL");
+            Require(AgcDriver::Graphics::DrawRejection(queue, false).find("PA_CL_CLIP_CNTL") != std::string::npos, "the precheck accepted an unsupported PA_CL_CLIP_CNTL bit");
+        }
+    }
+    queue = makeState();
+    queue.context[0x1b3] = 2;
+    queue.context[0x1b4] = 2;
+    for (const auto clip : {0x00080000u, 0x00000000u, 0x0c080000u, 0x04000000u}) {
+        queue.context[0x204] = clip;
+        const auto plain = AgcDriver::Graphics::DecodeState(queue);
+        queue.context[0x204] = clip | 0x01000000u;
+        const auto linearClip = AgcDriver::Graphics::DecodeState(queue);
+        Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "the precheck rejected DX_LINEAR_ATTR_CLIP_ENA");
+        Require(linearClip.negativeOneToOne == plain.negativeOneToOne && linearClip.depthClamp == plain.depthClamp && linearClip.viewport.minDepth == plain.viewport.minDepth && linearClip.viewport.maxDepth == plain.viewport.maxDepth, "DX_LINEAR_ATTR_CLIP_ENA changed the clip space, depth clamping or depth range");
     }
 }
 
@@ -1860,6 +1923,7 @@ int main() {
         DepthStencilTests();
         DepthBoundsBiasTests();
         DisabledColorTests();
+        CompactedExportTests();
         metadataPassTests();
         ShaderStageTests();
         PixelInputLayoutTests();

core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp

@@ -1,5 +1,6 @@
 #include "GraphicsTests.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
 #include <array>
 #include <string>
 #include <string_view>
@@ -269,6 +270,57 @@ void RunGuestTextureResourceTests() {
     const auto partial = DecodeTextureResource(pack(partialMips));
     Require(partial.lastLevel == 1 && partial.mipCount == 3, "a view over part of the mip chain decoded wrongly");
 
+    Fields pastLast = base;
+    pastLast.base40 = 0x55ea000ull;
+    pastLast.format = 71;
+    pastLast.width = 1920;
+    pastLast.height = 1080;
+    pastLast.tileModeRaw = 0x1b;
+    pastLast.maxMip = 5;
+    pastLast.baseLevel = 6;
+    pastLast.lastLevel = 6;
+    const auto tailView = DecodeTextureResource(pack(pastLast));
+    Require(tailView.baseLevel == 6 && tailView.lastLevel == 6 && tailView.mipCount == 7, "a view one level past the last mip must address that level of the chain");
+    auto allocated = tailView;
+    allocated.mipCount = 6;
+    const auto allocatedSurface = DescribeSurface(allocated);
+    const auto viewSurface = DescribeSurface(tailView);
+    Require(allocatedSurface.guestBytes == 0x1640000u && viewSurface.guestBytes == 0x1640000u, "a 1920x1080 64 bpp SW_64KB_R_X chain must take addrlib's 0x1640000 bytes with 6 and with 7 levels");
+    constexpr std::array<std::uint64_t, 7> addrlibOffsets{0x650000u, 0x1d0000u, 0x90000u, 0x30000u, 0x10000u, 0u, 0u};
+    for (std::uint32_t level = 0; level < 7; ++level) {
+        Require(viewSurface.mips[level].tiledOffset == addrlibOffsets[level] && viewSurface.mips[level].tail == (level >= 5), "the 7-level chain must place each level at its addrlib offset");
+        if (level < 6) Require(allocatedSurface.mips[level].tiledOffset == addrlibOffsets[level] && allocatedSurface.mips[level].tail == viewSurface.mips[level].tail && allocatedSurface.mips[level].tailX == viewSurface.mips[level].tailX && allocatedSurface.mips[level].tailY == viewSurface.mips[level].tailY, "the level past the last mip must not move the allocated levels");
+    }
+    Require(viewSurface.mips[5].tailX == 64 && viewSurface.mips[5].tailY == 0, "the last allocated level must sit in its addrlib tail slot");
+    Require(viewSurface.mips[6].tailX == 0 && viewSurface.mips[6].tailY == 32, "the level past the last mip must sit in its addrlib tail slot");
+    pastLast.lastLevel = 8;
+    const auto deeperView = DecodeTextureResource(pack(pastLast));
+    Require(deeperView.mipCount == 9, "a view past the last mip must cover every level it names");
+    const auto deeperSurface = DescribeSurface(deeperView);
+    Require(deeperSurface.guestBytes == 0x1640000u && deeperSurface.mips[7].tail && deeperSurface.mips[7].tailX == 32 && deeperSurface.mips[7].tailY == 0 && deeperSurface.mips[8].tail && deeperSurface.mips[8].tailX == 0 && deeperSurface.mips[8].tailY == 16, "levels 7 and 8 must sit in their addrlib tail slots");
+
+    Fields pastLinear = base;
+    pastLinear.maxMip = 1;
+    pastLinear.baseLevel = 2;
+    pastLinear.lastLevel = 2;
+    rejectFields(pastLinear, "would move the surface's own");
+    auto linearChain = partial;
+    linearChain.mipCount = 2;
+    const auto linearAllocated = DescribeSurface(linearChain).guestBytes;
+    linearChain.mipCount = 3;
+    Require(linearAllocated == 0x1800u && DescribeSurface(linearChain).guestBytes == 0x1c00u, "a third level must grow a 16x16 32 bpp linear chain from addrlib's 0x1800 to 0x1c00 bytes");
+
+    Fields pastUntailed = pastLast;
+    pastUntailed.maxMip = 0;
+    pastUntailed.baseLevel = 1;
+    pastUntailed.lastLevel = 1;
+    rejectFields(pastUntailed, "would move the surface's own");
+    auto untailedChain = tailView;
+    untailedChain.mipCount = 1;
+    const auto untailedAllocated = DescribeSurface(untailedChain).guestBytes;
+    untailedChain.mipCount = 2;
+    Require(untailedAllocated == 0xff0000u && DescribeSurface(untailedChain).guestBytes == 0x1470000u, "a second level must grow a 1920x1080 64 bpp SW_64KB_R_X surface from addrlib's 0xff0000 to 0x1470000 bytes");
+
     std::array<std::uint32_t, 4> shortWords{};
     reject([&] { DecodeTextureResource(shortWords); }, "8 dwords");
 

core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp

@@ -0,0 +1,378 @@
+#include "IntermediateRepresentation/IrProgram.hpp"
+#include "Optimization/MaskedSelectEliminator.hpp"
+#include "RdnaDecoder/RdnaInstruction.hpp"
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <exception>
+#include <functional>
+#include <initializer_list>
+#include <utility>
+
+using namespace ShaderRecompiler;
+
+namespace {
+
+struct Builder {
+    IrProgram program;
+    IrBlock* block = nullptr;
+
+    Builder() {
+        block = &newBlock();
+        program.SetEntryBlock(*block);
+    }
+
+    IrBlock& newBlock() {
+        auto& created = program.CreateBlock();
+        program.BlockOrder().push_back(&created);
+        return created;
+    }
+
+    IrValue& constant(std::uint32_t value) {
+        auto& created = program.CreateValue(IrOpcode::Void, IrType::U32);
+        created.SetImmediateU32(value);
+        return created;
+    }
+
+    IrValue& emit(IrOpcode opcode, IrType type, std::initializer_list<IrValue*> arguments, std::uint64_t flags = 0) {
+        auto& created = program.CreateValue(opcode, type, flags);
+        for (auto* argument : arguments) created.AddArgument(argument);
+        block->AppendInstruction(&created);
+        return created;
+    }
+
+    IrValue& lane() {
+        return emit(IrOpcode::LaneId, IrType::U32, {});
+    }
+
+    IrValue& mask(std::uint32_t lanes) {
+        return emit(IrOpcode::ULessThan32, IrType::Bool, {&lane(), &constant(lanes)});
+    }
+
+    IrValue& select(IrValue& condition, IrValue& active, IrValue& inactive) {
+        return emit(IrOpcode::SelectU32, IrType::U32, {&condition, &active, &inactive});
+    }
+
+    IrValue& add(IrValue& value, std::uint32_t immediate) {
+        return emit(IrOpcode::IAdd32, IrType::U32, {&value, &constant(immediate)});
+    }
+
+    IrValue& logicalAnd(IrValue& left, IrValue& right) {
+        return emit(IrOpcode::LogicalAnd, IrType::Bool, {&left, &right});
+    }
+
+    IrValue& threadBit(IrValue& low, IrValue& high) {
+        auto& laneId = lane();
+        auto& word = select(emit(IrOpcode::ULessThan32, IrType::Bool, {&laneId, &constant(32u)}), low, high);
+        auto& shift = emit(IrOpcode::BitwiseAnd32, IrType::U32, {&laneId, &constant(31u)});
+        auto& shifted = emit(IrOpcode::ShiftRightLogical32, IrType::U32, {&word, &shift});
+        auto& bit = emit(IrOpcode::BitwiseAnd32, IrType::U32, {&shifted, &constant(1u)});
+        return emit(IrOpcode::INotEqual32, IrType::Bool, {&bit, &constant(0u)});
+    }
+
+    IrValue& ballotBit(IrValue& predicate) {
+        auto& ballot = emit(IrOpcode::Ballot, IrType::U32x4, {&predicate});
+        auto& low = emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &constant(0u)});
+        auto& high = emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &constant(1u)});
+        return threadBit(low, high);
+    }
+
+    void keep(IrValue& value) {
+        (void)emit(IrOpcode::ReferenceU32, IrType::Void, {&value});
+    }
+
+    std::uint32_t eliminate() {
+        return MaskedSelectEliminator{}.Eliminate(program).removedSelects;
+    }
+};
+
+bool removed(const IrValue& value) {
+    return value.Parent() == nullptr;
+}
+
+bool expect(const char* name, bool condition) {
+    if (!condition) std::fprintf(stderr, "%s\n", name);
+    return condition;
+}
+
+bool run(const char* name, const std::function<bool()>& test) {
+    try {
+        return expect(name, test());
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "%s: %s\n", name, error.what());
+        return false;
+    }
+}
+
+}
+
+int main() {
+    bool passed = true;
+
+    passed &= run("a write read only under its own exec must lose its select", [] {
+        Builder b;
+        auto& exec = b.mask(16u);
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& next = b.select(exec, b.add(written, 2u), old);
+        b.keep(next);
+        return b.eliminate() == 1u && removed(written) && !removed(next);
+    });
+
+    passed &= run("a write read under a wider exec must keep its select", [] {
+        Builder b;
+        auto& wide = b.mask(32u);
+        auto& exec = b.logicalAnd(wide, b.mask(16u));
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& next = b.select(wide, b.add(written, 2u), old);
+        b.keep(next);
+        return b.eliminate() == 0u && !removed(written);
+    });
+
+    passed &= run("a write read under a narrower exec must lose its select", [] {
+        Builder b;
+        auto& exec = b.mask(32u);
+        auto& narrow = b.logicalAnd(exec, b.mask(16u));
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& next = b.select(narrow, b.add(written, 2u), old);
+        b.keep(next);
+        return b.eliminate() == 1u && removed(written);
+    });
+
+    passed &= run("a write whose old lanes flow into the next write must keep its select", [] {
+        Builder b;
+        auto& exec = b.mask(16u);
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& next = b.select(exec, b.add(old, 2u), written);
+        b.keep(next);
+        return b.eliminate() == 0u && !removed(written);
+    });
+
+    passed &= run("a chain of writes under one exec must collapse once the last one goes", [] {
+        Builder b;
+        auto& exec = b.mask(16u);
+        auto& old = b.lane();
+        auto& first = b.select(exec, b.add(old, 1u), old);
+        auto& second = b.select(exec, b.add(first, 2u), first);
+        auto& sink = b.select(exec, b.add(second, 3u), old);
+        b.keep(sink);
+        return b.eliminate() == 2u && removed(first) && removed(second) && !removed(sink);
+    });
+
+    passed &= run("cross-lane reads must keep the select", [] {
+        bool ok = true;
+        for (const IrOpcode opcode : {IrOpcode::ReadFirstLane, IrOpcode::DppMoveU32, IrOpcode::BpermuteU32, IrOpcode::Permlane16U32, IrOpcode::SwizzleU32}) {
+            Builder b;
+            auto& exec = b.mask(16u);
+            auto& old = b.lane();
+            auto& written = b.select(exec, b.add(old, 1u), old);
+            auto& read = b.emit(opcode, IrType::U32, {&written, &exec});
+            b.keep(b.select(exec, read, old));
+            ok &= b.eliminate() == 0u && !removed(written);
+        }
+        Builder b;
+        auto& exec = b.mask(16u);
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.emit(IrOpcode::INotEqual32, IrType::Bool, {&written, &b.constant(0u)})});
+        b.keep(b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)}));
+        return ok && b.eliminate() == 0u && !removed(written);
+    });
+
+    passed &= run("a branch condition derived from the write keeps the select", [] {
+        Builder b;
+        auto& exec = b.mask(16u);
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& taken = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&written, &b.constant(0u)});
+        (void)b.emit(IrOpcode::Reference, IrType::Void, {&taken});
+        return b.eliminate() == 0u && !removed(written);
+    });
+
+    passed &= run("a compare masked by the same exec must not keep the select", [] {
+        Builder b;
+        auto& exec = b.mask(16u);
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& compare = b.logicalAnd(exec, b.emit(IrOpcode::INotEqual32, IrType::Bool, {&written, &b.constant(0u)}));
+        auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&compare});
+        b.keep(b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)}));
+        return b.eliminate() == 1u && removed(written);
+    });
+
+    passed &= run("an exec rebuilt from a ballot of a narrower mask implies the write's exec", [] {
+        Builder b;
+        auto& exec = b.mask(48u);
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& later = b.ballotBit(b.logicalAnd(exec, b.mask(40u)));
+        auto& next = b.select(later, b.add(written, 2u), old);
+        b.keep(next);
+        return b.eliminate() == 1u && removed(written);
+    });
+
+    passed &= run("an exec rebuilt from a ballot of a wider mask keeps the select", [] {
+        Builder b;
+        auto& wide = b.mask(48u);
+        auto& exec = b.logicalAnd(wide, b.mask(40u));
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& later = b.ballotBit(wide);
+        b.keep(b.select(later, b.add(written, 2u), old));
+        return b.eliminate() == 0u && !removed(written);
+    });
+
+    passed &= run("a select under the full wave's exec must go whatever reads it", [] {
+        Builder b;
+        auto& full = b.threadBit(b.constant(0xffffffffu), b.constant(0xffffffffu));
+        auto& old = b.lane();
+        auto& written = b.select(full, b.add(old, 1u), old);
+        b.keep(written);
+        return b.eliminate() != 0u && removed(written);
+    });
+
+    passed &= run("a partial wave's exec is not the full wave", [] {
+        Builder b;
+        auto& partial = b.threadBit(b.constant(0xffffffffu), b.constant(0x0000ffffu));
+        auto& old = b.lane();
+        auto& written = b.select(partial, b.add(old, 1u), old);
+        b.keep(written);
+        return b.eliminate() == 0u && !removed(written);
+    });
+
+    const auto diamond = [](bool wideRead) {
+        Builder b;
+        auto& wide = b.mask(48u);
+        auto& exec = b.logicalAnd(wide, b.mask(16u));
+        auto& old = b.lane();
+        IrBlock& head = *b.block;
+        IrBlock& body = b.newBlock();
+        IrBlock& merge = b.newBlock();
+        head.AddBranch(&body);
+        head.AddBranch(&merge);
+        body.AddBranch(&merge);
+        b.block = &body;
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        b.block = &merge;
+        auto& phi = b.program.CreateValue(IrOpcode::Phi, IrType::U32);
+        merge.AppendInstruction(&phi);
+        phi.AddPhiOperand(&head, &old);
+        phi.AddPhiOperand(&body, &written);
+        b.keep(b.select(wideRead ? wide : exec, b.add(phi, 2u), old));
+        return std::pair{b.eliminate(), removed(written)};
+    };
+
+    passed &= run("a write merged by a phi and read under its own exec must lose its select", [&] {
+        const auto [count, gone] = diamond(false);
+        return count == 1u && gone;
+    });
+
+    passed &= run("a write merged by a phi and read under the wider exec must keep its select", [&] {
+        const auto [count, gone] = diamond(true);
+        return count == 0u && !gone;
+    });
+
+    passed &= run("a write in a loop must keep its select", [] {
+        Builder b;
+        auto& exec = b.mask(16u);
+        auto& old = b.lane();
+        IrBlock& head = *b.block;
+        IrBlock& loop = b.newBlock();
+        IrBlock& exit = b.newBlock();
+        head.AddBranch(&loop);
+        loop.AddBranch(&loop);
+        loop.AddBranch(&exit);
+        b.block = &loop;
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& next = b.select(exec, b.add(written, 2u), old);
+        b.keep(next);
+        return b.eliminate() == 0u && !removed(written);
+    });
+
+    passed &= run("a write read inside a later loop must keep its select", [] {
+        Builder b;
+        auto& exec = b.mask(16u);
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        IrBlock& head = *b.block;
+        IrBlock& loop = b.newBlock();
+        head.AddBranch(&loop);
+        loop.AddBranch(&loop);
+        b.block = &loop;
+        b.keep(b.select(exec, b.add(written, 2u), old));
+        return b.eliminate() == 0u && !removed(written);
+    });
+
+    const auto sample = [](IrShaderStage stage, std::uint32_t sampleFlags) {
+        Builder b;
+        b.program.Resources().stage = stage;
+        MemoryInfo memory;
+        memory.kind = ResourceKind::Image;
+        memory.imageSampleFlags = sampleFlags;
+        b.program.Resources().memoryInfo.push_back(memory);
+        auto& exec = b.mask(16u);
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        auto& address = b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {&written, &old});
+        auto& image = b.emit(IrOpcode::GetImageResource, IrType::ImageResource, {});
+        auto& sampler = b.emit(IrOpcode::GetSamplerResource, IrType::SamplerResource, {});
+        MemoryFlags flags{0u, 0u};
+        std::uint64_t bits = 0;
+        std::memcpy(&bits, &flags, sizeof(flags));
+        auto& sampled = b.emit(IrOpcode::ImageSampleRaw, IrType::U32x4, {&image, &sampler, &address}, bits);
+        b.keep(b.select(exec, b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&sampled, &b.constant(0u)}), old));
+        b.eliminate();
+        return removed(written);
+    };
+
+    passed &= run("an implicit-LOD pixel sample reads its quad's coordinates and keeps the select", [&] {
+        return !sample(IrShaderStage::Pixel, 0u);
+    });
+
+    passed &= run("an explicit-LOD sample reads only its lane and drops the select", [&] {
+        return sample(IrShaderStage::Pixel, RdnaImageSampleFlagLevelZero) && sample(IrShaderStage::Mesh, 0u);
+    });
+
+    const auto guarded = [](IrOpcode opcode, bool sameExec, bool asActive) {
+        Builder b;
+        auto& wide = b.mask(48u);
+        auto& exec = b.logicalAnd(wide, b.mask(16u));
+        auto& old = b.lane();
+        auto& written = b.select(exec, b.add(old, 1u), old);
+        IrValue& active = asActive ? b.emit(IrOpcode::INotEqual32, IrType::Bool, {&written, &b.constant(0u)}) : sameExec ? exec : wide;
+        switch (opcode) {
+            case IrOpcode::WriteSharedU32:
+                b.emit(opcode, IrType::Void, {&old, &written, &active});
+                break;
+            case IrOpcode::LoadBufferU32:
+                b.keep(b.select(exec, b.emit(opcode, IrType::U32, {&b.emit(IrOpcode::GetBufferResource, IrType::BufferResource, {}), &written, &b.constant(0u), &b.constant(0u), &active}), old));
+                break;
+            default:
+                b.emit(opcode, IrType::Void, {&b.emit(IrOpcode::CompositeConstructU32x4, IrType::U32x4, {&written, &old, &old, &old}), &active});
+                break;
+        }
+        b.eliminate();
+        return removed(written);
+    };
+
+    passed &= run("a store, load or export whose exec implies the write's exec drops the select", [&] {
+        return guarded(IrOpcode::WriteSharedU32, true, false) && guarded(IrOpcode::LoadBufferU32, true, false) && guarded(IrOpcode::SetAttribute, true, false);
+    });
+
+    passed &= run("a store, load or export under a wider exec keeps the select", [&] {
+        return !guarded(IrOpcode::WriteSharedU32, false, false) && !guarded(IrOpcode::LoadBufferU32, false, false) && !guarded(IrOpcode::SetAttribute, false, false);
+    });
+
+    passed &= run("a value used as a store's exec keeps the select", [&] {
+        return !guarded(IrOpcode::WriteSharedU32, true, true);
+    });
+
+    passed &= run("an atomic keeps the select", [&] {
+        return !guarded(IrOpcode::SharedAtomicIAdd32, true, false);
+    });
+
+    return passed ? 0 : 1;
+}

core/libs/prx/libSceAgcDriver/tests/Pm4.cpp

@@ -2,12 +2,16 @@
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
 #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
+#include "prx/libSceAgcDriver/Submit/include/Acb.hpp"
 #include "prx/libc/include/Shutdown.hpp"
+#include <atomic>
+#include <chrono>
 #include <cstdio>
 #include <cstring>
 #include <limits>
 #include <set>
 #include <stdexcept>
+#include <thread>
 #include <vector>
 #ifdef _WIN32
 #ifndef NOMINMAX
@@ -290,6 +294,9 @@ void testMemory() {
     check(data[0] == 22 && data[1] == 12, "WRITE_DATA fixed destination failed");
     execute(state, makePacket(0x37, {0x40000100, low(data.data()), high(data.data()), 41, 42}));
     check(data[0] == 41 && data[1] == 42, "WRITE_DATA from the PFP failed");
+    execute(state, makePacket(0x37, {0x04100200, low(data.data()), high(data.data()), 61, 62}));
+    check(data[0] == 61 && data[1] == 62, "WRITE_DATA with a cache policy failed");
+    expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x37, {0x08000100, low(data.data()), high(data.data()), 71}), 0); }, "reserved");
     expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x37, {0x80000100, low(data.data()), high(data.data()), 51}), 0); }, "engine");
     execute(state, makePacket(0x81, {4, 31, 32}));
     execute(state, makePacket(0x83, {4, 2, low(data.data()), high(data.data())}));
@@ -405,6 +412,60 @@ void testMemorySynchronization() {
     check(memory.destination == 42, "failed synchronization changed the destination");
 }
 
+void testConditionalValidation() {
+    alignas(4) static std::uint32_t condition = 0;
+    const auto valid = makePacket(0x22, {low(&condition), high(&condition), 0, 0x3fff});
+    check(AgcDriver::Pm4::UnsupportedReason(valid[0]).empty() && AgcDriver::Pm4::AccessesMemory(valid[0]), "COND_EXEC is rejected or does not synchronize guest memory");
+    AgcDriver::Pm4::Validate(valid, 0);
+    AgcDriver::Pm4::Validate(valid, 0x20);
+    check(AgcDriver::Pm4::ConditionalWords(valid) == 0x3fff, "COND_EXEC count decoded wrong");
+    check(AgcDriver::Pm4::ConditionalWords(makePacket(0x22, {low(&condition), high(&condition), 0, 0})) == 0, "empty COND_EXEC range decoded wrong");
+    AgcDriver::Pm4::Validate(makePacket(0x22, {low(&condition), high(&condition), 3u << 25u, 5}), 0x20);
+    const auto invalidWord = [&](std::size_t word, std::uint32_t value, std::uint32_t queue, const char* text) {
+        auto packet = valid;
+        packet[word] = value;
+        expectFailure([&] { AgcDriver::Pm4::Validate(packet, queue); }, text);
+    };
+    invalidWord(1, low(&condition) | 1u, 0, "reserved address bits");
+    invalidWord(1, low(&condition) | 2u, 0x20, "reserved address bits");
+    invalidWord(2, 0x10000u, 0, "above 48");
+    invalidWord(3, 3u << 25u, 0, "reserved control fields");
+    invalidWord(3, 1u, 0x20, "reserved control fields");
+    invalidWord(3, 1u << 27u, 0x20, "reserved control fields");
+    invalidWord(4, 0x4000u, 0, "reserved count bits");
+    invalidWord(4, 0x80000005u, 0x20, "reserved count bits");
+    expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x22, {0, 0, 0}), 0); }, "packet size");
+    expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x22, {0, 0, 0, 0, 0}), 0); }, "packet size");
+    expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x22, {0, 0, 0, 0}, 1), 0); }, "header flags");
+    expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x22, {0, 0, 0, 0}, 2), 0x20); }, "header flags");
+    expectFailure([] { AgcDriver::Pm4::ConditionalWords(makePacket(0x37, {0x100, 0, 0, 0})); }, "expected COND_EXEC");
+    AgcDriver::QueueState state;
+    expectFailure([&] { AgcDriver::Pm4::Execute(valid, state); }, "driver execution");
+}
+
+void testConditionReadSynchronization() {
+    alignas(4) static std::uint32_t condition = 0;
+    static std::size_t flushed = 0;
+    condition = 0;
+    flushed = 0;
+    struct FlushHookReset {
+        ~FlushHookReset() { AgcDriver::GuestMemory::SetFlushHook(nullptr); }
+    } reset;
+    AgcDriver::GuestMemory::SetFlushHook([](std::uint64_t address, std::size_t bytes) {
+        check(address == reinterpret_cast<std::uintptr_t>(&condition) && bytes == sizeof(condition), "COND_EXEC synchronized an unrelated range");
+        condition = 0x80;
+        ++flushed;
+    });
+    const auto packet = makePacket(0x22, {low(&condition), high(&condition), 0, 5});
+    check(AgcDriver::Pm4::ReadCondition(packet) == 0x80 && flushed == 1, "COND_EXEC read its condition before the GPU work that writes it");
+    AgcDriver::GuestMemory::SetFlushHook([](std::uint64_t, std::size_t) {
+        throw std::runtime_error("condition synchronization failed");
+    });
+    expectFailure([&] { AgcDriver::Pm4::ReadCondition(packet); }, "condition synchronization failed");
+    AgcDriver::GuestMemory::SetFlushHook(nullptr);
+    expectFailure([] { AgcDriver::Pm4::ReadCondition(makePacket(0x22, {0x1000, 0, 0, 5})); }, "guest");
+}
+
 void testWriteChangedKeepsUntouchedBytes() {
     alignas(256) static std::uint8_t guest[256];
     std::memset(guest, 0, sizeof(guest));
@@ -541,6 +602,97 @@ void testDriverSubmission() {
     check(destination[0] == 0, "rejected submission executed a prefix");
 }
 
+std::vector<std::uint32_t> joinPackets(std::initializer_list<std::vector<std::uint32_t>> packets) {
+    std::vector<std::uint32_t> words;
+    for (const auto& packet : packets) words.insert(words.end(), packet.begin(), packet.end());
+    return words;
+}
+
+std::vector<std::uint32_t> writeWord(std::uint32_t& target, std::uint32_t value) {
+    return makePacket(0x37, {0x00100200, low(&target), high(&target), value});
+}
+
+std::vector<std::uint32_t> conditional(const std::uint32_t& condition, std::uint32_t words, std::uint32_t control = 0) {
+    return makePacket(0x22, {low(&condition), high(&condition), control, words});
+}
+
+std::vector<std::uint32_t> indirectBuffer(const std::vector<std::uint32_t>& target, bool chain = false) {
+    return makePacket(0x3f, {low(target.data()), high(target.data()), static_cast<std::uint32_t>(target.size()) | (chain ? 1u << 20u : 0u)});
+}
+
+void submitWords(std::vector<std::uint32_t>& words, std::uint32_t queue = 0) {
+    Packet packet{words.data(), static_cast<std::uint32_t>(words.size()), 0, {}};
+    check((queue == 0 ? sceAgcDriverSubmitDcb(&packet) : sceAgcDriverSubmitAcb(queue, &packet)) == 0, "conditional submission failed");
+}
+
+void testConditionalSubmission() {
+    alignas(8) static std::uint32_t zero = 0, one = 1, condition = 0;
+    static std::array<std::uint32_t, 16> results{};
+    results.fill(0);
+    const auto run = [](std::vector<std::uint32_t> words, std::uint32_t queue = 0) {
+        submitWords(words, queue);
+        AgcDriverWaitIdle_nid_postfix();
+    };
+    run(joinPackets({conditional(one, 5), writeWord(results[0], 11), writeWord(results[1], 12)}));
+    check(results[0] == 11 && results[1] == 12, "COND_EXEC skipped a range whose condition is set");
+    run(joinPackets({conditional(zero, 6), {0x80000000u}, writeWord(results[0], 21), writeWord(results[1], 22)}));
+    check(results[0] == 11 && results[1] == 22, "COND_EXEC did not skip exactly its range when the condition is zero");
+    run(joinPackets({conditional(zero, 0), writeWord(results[2], 23)}));
+    check(results[2] == 23, "an empty COND_EXEC range skipped the next packet");
+    alignas(4) static std::uint32_t never = 0;
+    never = 0;
+    auto skippedWait = joinPackets({conditional(zero, 9), makePacket(0x3c, {0x13, low(&never), high(&never), 1, 0xffffffffu, 0x190}), makePacket(0x10, {0}, 0x30), writeWord(results[14], 24)});
+    submitWords(skippedWait);
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10);
+    while (std::atomic_ref<std::uint32_t>(results[14]).load() != 24 && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    const bool skipped = std::atomic_ref<std::uint32_t>(results[14]).load() == 24;
+    std::atomic_ref<std::uint32_t>(never).store(1);
+    AgcDriverWaitIdle_nid_postfix();
+    check(skipped, "a WAIT_REG_MEM inside a skipped COND_EXEC range was executed");
+
+    condition = 1;
+    run(joinPackets({writeWord(condition, 0), conditional(condition, 5), writeWord(results[0], 31), writeWord(results[1], 32)}));
+    check(condition == 0 && results[0] == 11 && results[1] == 32, "COND_EXEC read its condition before an earlier packet of its queue stored it");
+    run(joinPackets({writeWord(condition, 9), conditional(condition, 5), writeWord(results[0], 41)}));
+    check(results[0] == 41, "COND_EXEC skipped a range whose condition an earlier packet of its queue set");
+    condition = 0;
+    run(joinPackets({conditional(condition, 5, 3u << 25u), writeWord(results[3], 42), writeWord(condition, 1), conditional(condition, 5, 1u << 25u), writeWord(results[4], 43)}), 0x20);
+    check(results[3] == 0 && results[4] == 43, "compute queue COND_EXEC with a cache policy evaluated wrong");
+
+    run(joinPackets({conditional(one, 10), conditional(zero, 10), writeWord(results[5], 51), writeWord(results[6], 52), writeWord(results[7], 53)}));
+    check(results[5] == 0 && results[6] == 0 && results[7] == 53, "a nested COND_EXEC range reaching past the outer range was not skipped exactly");
+    run(joinPackets({conditional(zero, 10), conditional(one, 10), writeWord(results[5], 54), writeWord(results[6], 55), writeWord(results[7], 56)}));
+    check(results[5] == 0 && results[6] == 55 && results[7] == 56, "a skipped COND_EXEC range did not skip the COND_EXEC inside it");
+
+    static std::vector<std::uint32_t> target;
+    target = joinPackets({writeWord(results[8], 61), conditional(zero, 5), writeWord(results[9], 62), writeWord(results[10], 63)});
+    run(joinPackets({conditional(zero, 4), indirectBuffer(target), writeWord(results[11], 64)}));
+    check(results[8] == 0 && results[10] == 0 && results[11] == 64, "COND_EXEC over an INDIRECT_BUFFER did not skip the whole buffer");
+    run(joinPackets({conditional(one, 4), indirectBuffer(target), writeWord(results[11], 65)}));
+    check(results[8] == 61 && results[9] == 0 && results[10] == 63 && results[11] == 65, "COND_EXEC over an INDIRECT_BUFFER or inside one evaluated wrong");
+
+    const auto rejected = [](std::vector<std::uint32_t> words, const char* text) {
+        results[12] = 0;
+        expectFailure([&] { submitWords(words); }, text);
+        AgcDriverWaitIdle_nid_postfix();
+        check(results[12] == 0, "a rejected conditional submission executed a prefix");
+    };
+    const auto sentinel = writeWord(results[12], 1);
+    rejected(joinPackets({sentinel, conditional(one, 3), writeWord(results[13], 1)}), "ends inside a packet");
+    rejected(joinPackets({sentinel, conditional(one, 6), writeWord(results[13], 1)}), "exceeds its command buffer");
+    rejected(joinPackets({sentinel, conditional(one, 6), {0xc004105cu, 7, 0, 1, 0, 0}}), "a flip inside a conditional execution range");
+    rejected(joinPackets({sentinel, conditional(one, 2), makePacket(0x59, {0x80000000u})}), "REWIND inside a conditional execution range");
+    rejected(joinPackets({sentinel, conditional(one, 4), indirectBuffer(target, true)}), "chained INDIRECT_BUFFER");
+    rejected(joinPackets({sentinel, conditional(one, 2), {0x40000000u, 0}}), "whole packets");
+    rejected(joinPackets({sentinel, makePacket(0x22, {low(&one), high(&one), 0, 0x4005}), writeWord(results[13], 1)}), "reserved count bits");
+    static std::vector<std::uint32_t> unaligned, overlong;
+    unaligned = joinPackets({conditional(one, 3), writeWord(results[13], 1)});
+    overlong = joinPackets({conditional(one, 10), writeWord(results[13], 1)});
+    rejected(joinPackets({sentinel, indirectBuffer(unaligned)}), "ends inside a packet");
+    rejected(joinPackets({sentinel, indirectBuffer(overlong), writeWord(results[13], 1)}), "exceeds its command buffer");
+    check(results[13] == 0, "a rejected conditional submission executed a guarded packet");
+}
+
 void testAsyncMemoryFailure() {
     auto commands = makePacket(0x37, {0x100, 0x1000, 0, 1});
     Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
@@ -572,11 +724,14 @@ int main(int argc, char** argv) {
         testCopies();
         testMemoryCopyDecode();
         testMemorySynchronization();
+        testConditionalValidation();
+        testConditionReadSynchronization();
         testEventWrite();
         testAcquireMem();
         testDriverSubmission();
+        testConditionalSubmission();
         LibcRunShutdown_nid_postfix();
-        std::puts("PM4 catalog, registers, state, memory and submission tests passed");
+        std::puts("PM4 catalog, registers, state, memory, conditional execution and submission tests passed");
         return 0;
     } catch (const std::exception& error) {
         std::fprintf(stderr, "%s\n", error.what());

core/libs/prx/libSceAgcDriver/tests/Recorder.cpp

@@ -429,6 +429,38 @@ void lateLabelTests(Recorder& recorder) {
     Require(!Recorder::LookupLabel(a, 4, 5).has_value() && !Recorder::LookupLabel(b, 8, 5).has_value() && !recorder.PendingLabelIn(0x70000, 0x400), "(7) entries outlived their batch");
 }
 
+void largeLabelTests(Recorder& recorder) {
+    constexpr std::uint64_t slot = 0x71000, small = slot + 0x10, outside = slot - 4, wide = 0x72000;
+    const std::array<std::byte, 4> one{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}};
+    std::array<std::byte, 256> clear{};
+    std::array<std::byte, Recorder::LabelTableBytes> limit{};
+    limit.fill(std::byte{3});
+    recorder.Sync();
+    Require(recorder.PendingLabels() == 0 && !Recorder::PendingLabelSince().has_value() && !Recorder::WideLabelIn(0, ~0ull), "the table is not empty before the large label tests");
+    recorder.NoteLabel(small, one, 100, 0);
+    recorder.NoteLabel(outside, one, 100, 0);
+    Recorder::CloseLabelGroup(AgcDriver::GuestMemory::TrackerGeneration());
+    Require(recorder.PendingLabels() == 2 && Recorder::LookupLabel(small, 4, 50).has_value(), "4-byte labels did not enter the table");
+    recorder.NoteLabel(slot, clear, 101, 0);
+    Recorder::CloseLabelGroup(AgcDriver::GuestMemory::TrackerGeneration());
+    Require(recorder.PendingLabels() == 1 && Recorder::LookupLabel(outside, 4, 50).has_value(), "a label larger than the table bound entered dwords or removed one outside its range");
+    Require(!Recorder::LookupLabel(small, 4, 50).has_value() && !Recorder::LookupLabel(slot, 8, 0).has_value(), "an older label under a larger one still composes the value");
+    Require(Recorder::WideLabelIn(slot, 4) && Recorder::WideLabelIn(slot + clear.size() - 4, 4) && Recorder::WideLabelIn(slot - 0x100, 0x104) && !Recorder::WideLabelIn(slot - 0x100, 0x100) && !Recorder::WideLabelIn(slot + clear.size(), 4), "a large label's range is wrong");
+    Require(recorder.PendingLabelIn(slot + 0x20, 0x20) && recorder.PendingWriteOverlaps(slot, clear.size()), "a large label is not pending");
+    Require(Recorder::PendingLabelSince().has_value(), "a large label did not start the label flush deadline");
+    recorder.NoteLabel(small, one, 102, 0);
+    Recorder::CloseLabelGroup(AgcDriver::GuestMemory::TrackerGeneration());
+    const auto later = Recorder::LookupLabel(small, 4, 101);
+    Require(later.has_value() && later->value == 1 && later->stamp == 102, "a label after a large one is not served");
+    recorder.NoteLabel(wide, limit, 103, 0);
+    Recorder::CloseLabelGroup(AgcDriver::GuestMemory::TrackerGeneration());
+    Require(recorder.PendingLabels() == 2 + Recorder::LabelTableBytes / 4 && !Recorder::WideLabelIn(wide, limit.size()), "a label at the table bound did not enter every dword");
+    const auto bound = Recorder::LookupLabel(wide + Recorder::LabelTableBytes - 8, 8, 0);
+    Require(bound.has_value() && bound->value == 0x0303030303030303ull && bound->stamp == 103, "a label at the table bound is not served");
+    recorder.Sync();
+    Require(recorder.PendingLabels() == 0 && !Recorder::WideLabelIn(0, ~0ull) && !recorder.PendingLabelIn(slot, clear.size()) && !recorder.PendingWriteOverlaps(slot, clear.size()), "large label tests left entries, ranges or writes behind");
+}
+
 // (6) GuestMemory::UnchangedSinceCollected: false outside the watched arena, and around a
 // MarkWritten or a CPU write of the block inside it.
 void unchangedSinceTests() {
@@ -768,6 +800,91 @@ void movedMetadataTests(const Device& device, Recorder& recorder) {
     Require(holds(*back, {0x55, 0x55, 0x55, 0x55}), "the image remade under the first keys does not hold the stored texels");
 }
 
+void viewPastLastMipTests(const Device& device, Recorder& recorder) {
+    const auto& base = device.GetContext();
+    if (base.hostImportAlignment == 0) {
+        std::cout << "host imports unavailable: views past the last mip not tested\n";
+        return;
+    }
+    constexpr std::uint32_t side = 256;
+    constexpr std::size_t bytes = 0x60000;
+#ifdef _WIN32
+    void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+#else
+    void* block = std::aligned_alloc(65536, bytes);
+#endif
+    Require(block != nullptr, "cannot allocate the past-last-mip block");
+    auto* texels = static_cast<std::uint8_t*>(block);
+    const auto address = reinterpret_cast<std::uint64_t>(block);
+    {
+        GuestAllocations::Mutation mutation;
+        mutation.Add(block, bytes, true, true);
+    }
+    struct Unregister {
+        const Context& context;
+        void* block;
+        std::uint64_t address;
+        ~Unregister() {
+            ClearCachedTextures(context.device);
+            {
+                GuestAllocations::Mutation mutation;
+                mutation.Remove(block);
+            }
+            HostImportFor(context, address, bytes);
+        }
+    } unregister{base, block, address};
+    if (HostImportFor(base, address, bytes) == nullptr) {
+        std::cout << "host import of the past-last-mip block refused: views past the last mip not tested\n";
+        return;
+    }
+    TextureDetiler detiler(base);
+    auto context = base;
+    context.detiler = &detiler;
+    const auto view = [&](std::uint32_t level) {
+        const std::array<std::uint32_t, 8> words{
+            static_cast<std::uint32_t>(address >> 8u),
+            static_cast<std::uint32_t>((address >> 40u) & 0xffu) | (56u << 20u) | (((side - 1u) & 3u) << 30u),
+            ((side - 1u) >> 2u) | ((side - 1u) << 14u),
+            0xfacu | (level << 12u) | (level << 16u) | (0x1bu << 20u) | (9u << 28u),
+            0u,
+            2u << 4u,
+            0u,
+            0u,
+        };
+        return DecodeTextureResource(words);
+    };
+    const auto texel = [&](const StorageTexture& image, std::uint32_t level) {
+        Buffer readback(context, 4, VK_BUFFER_USAGE_TRANSFER_DST_BIT);
+        const auto commands = recorder.Commands();
+        RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
+        VkBufferImageCopy copy{};
+        copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, 1};
+        copy.imageExtent = {1, 1, 1};
+        context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, image.Image(), VK_IMAGE_LAYOUT_GENERAL, readback.Handle(), 1, &copy);
+        RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT);
+        recorder.Submit();
+        device.WaitQueue();
+        recorder.Sync();
+        std::uint32_t value = 0;
+        std::memcpy(&value, readback.Bytes().data(), sizeof(value));
+        return value;
+    };
+    std::memset(texels, 0x55, bytes);
+    std::memset(texels + 0x8400, 0x66, 4);
+    std::memset(texels + 0x4800, 0x77, 4);
+    const auto allocatedView = view(2);
+    const auto pastView = view(3);
+    Require(allocatedView.mipCount == 3 && pastView.mipCount == 4 && DescribeSurface(pastView).guestBytes == bytes, "a 256x256 32 bpp SW_64KB_R_X surface viewed past MAX_MIP 2 decoded wrongly");
+    const auto allocated = CachedStorageSurface(context, allocatedView);
+    Require(allocated->Descriptor().mipCount == 3, "the allocated levels' storage image has an unexpected level count");
+    const auto past = CachedStorageSurface(context, pastView);
+    Require(past->Descriptor().mipCount == 4, "the storage image of a view past the last mip lacks the level it names");
+    Require(!StorageImageCached(context, allocated.get()), "the allocated levels' image is still the surface's after a view past its last mip");
+    Require(CachedStorageSurface(context, allocatedView) == past, "a view of the allocated levels took another image than the view past the last mip");
+    Require(texel(*past, 2) == 0x66666666u, "level 2 was not read from its addrlib tail slot");
+    Require(texel(*past, 3) == 0x77777777u, "the level past the last mip was not read from its addrlib tail slot");
+}
+
 void resourceReadTests(const Device& device, Recorder& recorder) {
     const auto& context = device.GetContext();
     if (context.hostImportAlignment == 0) {
@@ -1700,6 +1817,143 @@ void staleGenerationTests(const Device& device, Recorder& recorder) {
 #endif
 }
 
+// A write-back stores the texels of an image and leaves every other byte of the surface's memory
+// as it was: the padding of the tile blocks the surface covers only partly (its last block column
+// and row), the tail block of a mip chain, the bytes between the mips and the pitch padding of a
+// linear surface. Another surface over the memory, or the CPU, reads those bytes. Each case clears
+// every mip of an image over memory holding a pattern no texel equals, stores it and reads the
+// memory back (watched memory through the flush hook, after a store into the import's unit shadow):
+// each 4-byte word is the clear texel or the pattern, and the texels are exactly the surface's.
+// The 64 KiB-aligned tiled cases store through block windows, the others whole layers.
+void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watched) {
+    const auto& base = device.GetContext();
+    if (base.hostImportAlignment == 0) {
+        std::cout << "host imports unavailable: write-back padding not tested\n";
+        return;
+    }
+    struct Case {
+        const char* name;
+        TextureTileMode tileMode;
+        std::uint32_t width;
+        std::uint32_t height;
+        std::uint32_t mipCount;
+        std::uint64_t offset;
+    };
+    constexpr Case cases[] = {
+        {"64 KiB R_X 200x150", TextureTileMode::kR64KBX, 200, 150, 1, 0},
+        {"64 KiB R_X 200x150, 3 mips", TextureTileMode::kR64KBX, 200, 150, 3, 0},
+        {"64 KiB R_X 520x260, 6 mips", TextureTileMode::kR64KBX, 520, 260, 6, 0},
+        {"64 KiB R_X 200x150 off a 64 KiB boundary, 3 mips", TextureTileMode::kR64KBX, 200, 150, 3, 4096},
+        {"4 KiB S 100x70, 2 mips", TextureTileMode::kStandard4KB, 100, 70, 2, 0},
+        {"linear 200x150", TextureTileMode::kLinear, 200, 150, 1, 0},
+    };
+    const auto pattern = [](std::size_t i) { return static_cast<std::uint8_t>(i * 131u + 7u); };
+    constexpr std::array<std::uint8_t, 4> red{255, 0, 0, 255};
+    std::string failures;
+    for (const auto& test : cases) {
+        const std::string what = std::string(watched ? "(watched) " : "") + test.name;
+        GuestTextureResource resource{};
+        resource.width = test.width;
+        resource.height = test.height;
+        resource.mipCount = test.mipCount;
+        resource.tileMode = test.tileMode;
+        resource.dimension = TextureDimension::k2D;
+        resource.format = 56;
+        resource.dstSelX = 4;
+        resource.dstSelY = 5;
+        resource.dstSelZ = 6;
+        resource.dstSelW = 7;
+        const auto geometry = DescribeSurface(resource);
+        const auto surfaceBytes = static_cast<std::size_t>(geometry.guestBytes);
+        std::size_t texelCount = 0;
+        for (const auto& mip : geometry.mips) texelCount += static_cast<std::size_t>(mip.width) * mip.height;
+        // The surface, then a 64 KiB unit past it that no store may reach.
+        const auto bytes = (static_cast<std::size_t>(test.offset) + surfaceBytes + 65535) / 65536 * 65536 + 65536;
+        void* block = nullptr;
+        if (watched) {
+            block = AllocateWatched(bytes, 65536);
+            if (block == nullptr) {
+                std::cout << "no write watching: write-back padding in watched memory not tested\n";
+                return;
+            }
+        } else {
+#ifdef _WIN32
+            block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+#else
+            block = std::aligned_alloc(65536, bytes);
+#endif
+        }
+        Require(block != nullptr, "cannot allocate the write-back padding block");
+        auto* memory = static_cast<std::uint8_t*>(block);
+        for (std::size_t i = 0; i < bytes; ++i) memory[i] = pattern(i);
+        const auto blockAddress = reinterpret_cast<std::uint64_t>(block);
+        const auto address = blockAddress + test.offset;
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Add(block, bytes, true, true);
+        }
+        struct Unregister {
+            const Context& context;
+            void* block;
+            std::uint64_t address;
+            std::size_t bytes;
+            bool watched;
+            ~Unregister() {
+                {
+                    GuestAllocations::Mutation mutation;
+                    mutation.Remove(block);
+                }
+                HostImportFor(context, address, bytes);
+                if (watched) ReleaseWatched(block, bytes);
+            }
+        } unregister{base, block, blockAddress, bytes, watched};
+        if (HostImportFor(base, blockAddress, bytes) == nullptr) {
+            std::cout << "host import of the write-back padding block refused: write-back padding not tested\n";
+            return;
+        }
+        TextureDetiler detiler(base);
+        auto context = base;
+        context.detiler = &detiler;
+        resource.baseAddress = address;
+        {
+            auto image = std::make_shared<StorageTexture>(context, detiler, resource, 0);
+            const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, test.mipCount, 0, 1};
+            const auto commands = recorder.Commands();
+            recorder.Keep(image);
+            const VkClearColorValue value{{1.0f, 0.0f, 0.0f, 1.0f}};
+            RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
+            context.Function<PFN_vkCmdClearColorImage>("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &value, 1, &range);
+            RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
+            image->MarkDirty();
+            image->WriteBack();
+            recorder.Submit();
+            device.WaitQueue();
+            recorder.Sync();
+            std::vector<std::byte> read(bytes);
+            if (watched) AgcDriver::GuestMemory::Read(blockAddress, read);
+            else std::memcpy(read.data(), memory, bytes);
+            std::size_t texels = 0, garbage = 0, outside = 0;
+            for (std::size_t i = 0; i < bytes; i += 4) {
+                const bool inside = i >= test.offset && i < test.offset + surfaceBytes;
+                bool isTexel = inside, isPattern = true;
+                for (std::size_t c = 0; c < 4; ++c) {
+                    const auto byte = std::to_integer<std::uint8_t>(read[i + c]);
+                    isTexel = isTexel && byte == red[c];
+                    isPattern = isPattern && byte == pattern(i + c);
+                }
+                if (isTexel) ++texels;
+                else if (!isPattern) ++(inside ? garbage : outside);
+            }
+            if (garbage != 0 || outside != 0 || texels != texelCount) {
+                failures += what + ": the write-back stored " + std::to_string(texels) + " texels of " + std::to_string(texelCount) + " and changed " + std::to_string(garbage) + " words no texel holds (" + std::to_string(outside) + " past the surface)\n";
+            }
+        }
+        recorder.Sync();
+    }
+    if (!failures.empty()) throw std::runtime_error("write-back padding:\n" + failures);
+    std::cout << "write-back padding" << (watched ? " (watched)" : "") << ": ok\n";
+}
+
 void importWindowTests(const Device& device, Recorder& recorder) {
     using namespace AgcDriver::GuestMemory;
     const auto& base = device.GetContext();
@@ -2319,6 +2573,44 @@ void firstLayerViewTests(const Device& device, Recorder& recorder) {
     expectRed(program.Red(flatTexture.View(), flatTexture.Layout(), 0.0f), 0x20 / 255.0f, "a 2D texture over the surface does not read its first layer");
 }
 
+void atomicViewTests(const Device& device, Recorder& recorder) {
+    const auto& context = device.GetContext();
+    TextureDetiler detiler(context);
+    auto withDetiler = context;
+    withDetiler.detiler = &detiler;
+    for (const std::uint32_t format : {20u, 21u, 22u, 56u}) {
+        GuestTextureResource resource{};
+        resource.width = 64;
+        resource.height = 4;
+        resource.mipCount = 1;
+        resource.tileMode = TextureTileMode::kLinear;
+        resource.dimension = TextureDimension::k2D;
+        resource.format = format;
+        resource.dstSelX = 4;
+        resource.dstSelY = 5;
+        resource.dstSelZ = 6;
+        resource.dstSelW = 7;
+        const auto geometry = DescribeSurface(resource);
+        std::vector<std::uint8_t> memory(static_cast<std::size_t>(geometry.guestBytes) + 256);
+        resource.baseAddress = (reinterpret_cast<std::uintptr_t>(memory.data()) + 255) & ~std::uintptr_t{255};
+        auto image = std::make_shared<StorageTexture>(withDetiler, detiler, resource, 0);
+        recorder.Keep(image);
+        if (format == 56u) {
+            bool refused = false;
+            try {
+                static_cast<void>(image->AtomicView(0, false));
+            } catch (const std::exception&) {
+                refused = true;
+            }
+            Require(refused, "an 8_8_8_8 storage image has an atomic view");
+            continue;
+        }
+        const auto atomic = image->AtomicView(0, false);
+        Require(atomic != VK_NULL_HANDLE && atomic == image->AtomicView(0, false), "atomic storage views are not reused");
+        Require((atomic == image->View(0)) == (format == 20u), "only a 32_UINT storage image serves atomics through its own view");
+    }
+}
+
 void keysFillTests(const Device& device, Recorder& recorder) {
     const auto& base = device.GetContext();
     if (base.hostImportAlignment == 0) {
@@ -2444,6 +2736,7 @@ int main() {
         batchStampTests(recorder);
         labelTests(recorder);
         lateLabelTests(recorder);
+        largeLabelTests(recorder);
         unchangedSinceTests();
         closeRaceTests(device, recorder);
         keyProofTests(device, recorder);
@@ -2455,10 +2748,13 @@ int main() {
         RunResidentPresentTests(device.GetContext());
         storeRunTests(device, recorder);
         movedMetadataTests(device, recorder);
+        viewPastLastMipTests(device, recorder);
         keysFillTests(device, recorder);
         unitShadowTests(device, recorder);
         storageRefreshTests(device, recorder, false);
         storageRefreshTests(device, recorder, true);
+        writeBackPaddingTests(device, recorder, false);
+        writeBackPaddingTests(device, recorder, true);
         importWatchTests(device);
         staleGenerationTests(device, recorder);
         importWindowTests(device, recorder);
@@ -2466,6 +2762,7 @@ int main() {
         dataRefreshTests(device, recorder);
         minLodTests(device, recorder);
         firstLayerViewTests(device, recorder);
+        atomicViewTests(device, recorder);
         metadataPassTests(device, recorder);
         pendingKeyStoreTests(device, recorder);
         sampleDumpTests(device, recorder);

core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp

@@ -31,7 +31,7 @@ void setEnvironment(const char* name, const std::string& value) {
 }
 
 bool sameBinding(const DescriptorBinding& left, const DescriptorBinding& right) {
-    return left.kind == right.kind && left.role == right.role && left.descriptorSet == right.descriptorSet && left.binding == right.binding && left.count == right.count && left.guestDescriptor == right.guestDescriptor && left.readOnly == right.readOnly && left.imageShape == right.imageShape && left.samplerDepthCompare == right.samplerDepthCompare && left.imageWritten == right.imageWritten && left.imageDepthCompare == right.imageDepthCompare && left.bufferAtomic == right.bufferAtomic && left.bufferWritten == right.bufferWritten;
+    return left.kind == right.kind && left.role == right.role && left.descriptorSet == right.descriptorSet && left.binding == right.binding && left.count == right.count && left.guestDescriptor == right.guestDescriptor && left.readOnly == right.readOnly && left.imageShape == right.imageShape && left.samplerDepthCompare == right.samplerDepthCompare && left.imageWritten == right.imageWritten && left.imageDepthCompare == right.imageDepthCompare && left.imageAtomic == right.imageAtomic && left.bufferAtomic == right.bufferAtomic && left.bufferWritten == right.bufferWritten;
 }
 
 bool sameBindings(const std::vector<DescriptorBinding>& left, const std::vector<DescriptorBinding>& right) {
@@ -92,6 +92,7 @@ DescriptorBinding sampleBinding(std::uint32_t seed) {
     binding.samplerDepthCompare = {true, false, true};
     binding.imageDepthCompare = {false, true, false};
     binding.imageWritten = {false, true, true};
+    binding.imageAtomic = {false, true, false};
     binding.bufferAtomic = {true};
     binding.bufferWritten = {false, false, true, true, false};
     return binding;

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -893,6 +893,146 @@ void verifyTwoLaneUniformValues() {
     require(multiplies(32u) == oneLane, "two-lane uniform values: a two-lane invocation computes a scalar value once per lane");
 }
 
+void verifyFunctionLdsBound() {
+    using namespace ShaderRecompiler;
+    const auto build = [](const auto& body) {
+        IrProgram program;
+        program.Resources().stage = IrShaderStage::Pixel;
+        for (const std::uint32_t offset : {0u, 256u, 512u, 0xfffffff0u}) {
+            MemoryInfo lds;
+            lds.kind = ResourceKind::Lds;
+            lds.offset = offset;
+            program.Resources().memoryInfo.push_back(lds);
+        }
+        auto& block = program.CreateBlock();
+        program.SetEntryBlock(block);
+        program.BlockOrder().push_back(&block);
+        const auto emit = [&](IrOpcode opcode, IrType type, std::initializer_list<IrValue*> arguments, std::uint32_t memory = ~0u) -> IrValue& {
+            std::uint64_t bits = 0;
+            if (memory != ~0u) {
+                MemoryFlags flags{memory, 0u};
+                std::memcpy(&bits, &flags, sizeof(flags));
+            }
+            auto& value = program.CreateValue(opcode, type, bits);
+            for (auto* argument : arguments) value.AddArgument(argument);
+            block.AppendInstruction(&value);
+            return value;
+        };
+        const auto constant = [&](std::uint32_t immediate) -> IrValue& {
+            auto& value = program.CreateValue(IrOpcode::Void, IrType::U32);
+            value.SetImmediateU32(immediate);
+            return value;
+        };
+        auto& active = program.CreateValue(IrOpcode::Void, IrType::U1);
+        active.SetImmediateBool(true);
+        body(program, block, emit, constant, active);
+        return std::pair{FunctionLdsDwords(program), AnalyzeProgramRequirements(program)};
+    };
+
+    const auto [laneSlots, laneRequirements] = build([](IrProgram&, IrBlock&, auto& emit, auto& constant, IrValue& active) {
+        auto& lane = emit(IrOpcode::LaneId, IrType::U32, {});
+        auto& address = emit(IrOpcode::ShiftLeftLogical32, IrType::U32, {&lane, &constant(2u)});
+        for (const std::uint32_t memory : {0u, 1u, 2u}) emit(IrOpcode::WriteSharedU32, IrType::Void, {&address, &lane, &active}, memory);
+        emit(IrOpcode::LoadSharedU32, IrType::U32, {&address, &active}, 2u);
+    });
+    require(laneSlots == 192u, "function LDS: lane-strided slots up to byte 767 must take 192 dwords");
+    require(laneRequirements.functionLds && laneRequirements.functionLdsDwords == 192u, "function LDS: the requirements do not carry the bounded size");
+    {
+        std::vector<std::uint32_t> rebased;
+        for (const auto& [inst, address] : laneRequirements.functionLdsAddresses) rebased.push_back(address);
+        std::sort(rebased.begin(), rebased.end());
+        require(rebased == std::vector<std::uint32_t>{0u, 0u, 0u, 0u}, "function LDS: lane-strided slots must be addressed without the lane term");
+    }
+
+    const auto [mixed, mixedRequirements] = build([](IrProgram&, IrBlock&, auto& emit, auto& constant, IrValue& active) {
+        auto& lane = emit(IrOpcode::LaneId, IrType::U32, {});
+        auto& address = emit(IrOpcode::ShiftLeftLogical32, IrType::U32, {&lane, &constant(2u)});
+        emit(IrOpcode::WriteSharedU32, IrType::Void, {&address, &lane, &active}, 0u);
+        emit(IrOpcode::LoadSharedU32, IrType::U32, {&constant(8u), &active}, 0u);
+    });
+    require(mixed == 64u && mixedRequirements.functionLdsAddresses.empty(), "function LDS: a constant address beside lane-strided ones must keep the lane term");
+
+    const auto [strides, stridesRequirements] = build([](IrProgram&, IrBlock&, auto& emit, auto& constant, IrValue& active) {
+        auto& lane = emit(IrOpcode::LaneId, IrType::U32, {});
+        auto& four = emit(IrOpcode::ShiftLeftLogical32, IrType::U32, {&lane, &constant(2u)});
+        auto& eight = emit(IrOpcode::IMul32, IrType::U32, {&lane, &constant(8u)});
+        emit(IrOpcode::WriteSharedU32, IrType::Void, {&four, &lane, &active}, 0u);
+        emit(IrOpcode::LoadSharedU32, IrType::U32, {&eight, &active}, 0u);
+    });
+    require(strides == 128u && stridesRequirements.functionLdsAddresses.empty(), "function LDS: two lane strides must keep the lane term");
+
+    const auto [halfWord, halfWordRequirements] = build([](IrProgram&, IrBlock&, auto& emit, auto& constant, IrValue& active) {
+        auto& lane = emit(IrOpcode::LaneId, IrType::U32, {});
+        auto& address = emit(IrOpcode::ShiftLeftLogical32, IrType::U32, {&lane, &constant(1u)});
+        emit(IrOpcode::WriteSharedU32, IrType::Void, {&address, &lane, &active}, 0u);
+    });
+    require(halfWord == 64u && halfWordRequirements.functionLdsAddresses.empty(), "function LDS: a lane stride that is not whole dwords must keep the lane term");
+
+    const auto [summed, summedRequirements] = build([](IrProgram&, IrBlock&, auto& emit, auto& constant, IrValue& active) {
+        auto& lane = emit(IrOpcode::LaneId, IrType::U32, {});
+        auto& scaled = emit(IrOpcode::IMul32, IrType::U32, {&constant(16u), &lane});
+        auto& low = emit(IrOpcode::IAdd32, IrType::U32, {&scaled, &constant(4u)});
+        auto& other = emit(IrOpcode::LaneId, IrType::U32, {});
+        auto& shifted = emit(IrOpcode::ShiftLeftLogical32, IrType::U32, {&other, &constant(4u)});
+        auto& high = emit(IrOpcode::IAdd32, IrType::U32, {&constant(1024u), &shifted});
+        emit(IrOpcode::WriteSharedU32x2, IrType::Void, {&low, &lane, &lane, &active}, 0u);
+        emit(IrOpcode::LoadSharedU32x4, IrType::U32x4, {&high, &active}, 1u);
+    });
+    {
+        std::vector<std::uint32_t> rebased;
+        for (const auto& [inst, address] : summedRequirements.functionLdsAddresses) rebased.push_back(address);
+        std::sort(rebased.begin(), rebased.end());
+        require(rebased == std::vector<std::uint32_t>{4u, 1024u}, "function LDS: lane ids of separate instructions with one stride must share the rebase");
+        require(summedRequirements.functionLdsDwords == 384u && summed == 576u, "function LDS: the rebased array must hold the rebased addresses");
+    }
+
+    const auto [wide, wideRequirements] = build([](IrProgram&, IrBlock&, auto& emit, auto& constant, IrValue& active) {
+        auto& lane = emit(IrOpcode::LaneId, IrType::U32, {});
+        auto& masked = emit(IrOpcode::BitwiseAnd32, IrType::U32, {&lane, &constant(7u)});
+        auto& scaled = emit(IrOpcode::IMul32, IrType::U32, {&masked, &constant(16u)});
+        auto& chosen = emit(IrOpcode::SelectU32, IrType::U32, {&active, &scaled, &constant(0x100u)});
+        emit(IrOpcode::LoadSharedU32x4, IrType::U32x4, {&chosen, &active}, 0u);
+    });
+    require(wide == 128u, "function LDS: a 4-dword access at byte 0x100 must take 68 dwords, rounded to 128");
+    require(wideRequirements.functionLdsDwords == 128u, "function LDS: the requirements do not carry the wide access's size");
+    require(wideRequirements.functionLdsAddresses.empty(), "function LDS: a selected address must keep the lane term");
+
+    const auto [unbounded, unboundedRequirements] = build([](IrProgram&, IrBlock&, auto& emit, auto& constant, IrValue& active) {
+        auto& user = emit(IrOpcode::GetUserData, IrType::U32, {&constant(0u)});
+        emit(IrOpcode::WriteSharedU32, IrType::Void, {&user, &user, &active}, 0u);
+    });
+    require(unbounded == FunctionLdsDwordLimit && unboundedRequirements.functionLdsDwords == FunctionLdsDwordLimit, "function LDS: an address without a bound must keep the full array");
+
+    const auto wrapping = build([](IrProgram&, IrBlock&, auto& emit, auto& constant, IrValue& active) {
+        emit(IrOpcode::LoadSharedU32, IrType::U32, {&constant(0x20u), &active}, 3u);
+    }).first;
+    require(wrapping == FunctionLdsDwordLimit, "function LDS: an offset that can wrap the address must keep the full array");
+
+    const auto loop = build([](IrProgram& program, IrBlock& block, auto& emit, auto& constant, IrValue& active) {
+        auto& phi = program.CreateValue(IrOpcode::Phi, IrType::U32);
+        block.AppendInstruction(&phi);
+        auto& next = emit(IrOpcode::IAdd32, IrType::U32, {&phi, &constant(4u)});
+        phi.AddPhiOperand(&block, &constant(0u));
+        phi.AddPhiOperand(&block, &next);
+        emit(IrOpcode::WriteSharedU32, IrType::Void, {&phi, &next, &active}, 0u);
+    }).first;
+    require(loop == FunctionLdsDwordLimit, "function LDS: a loop-carried address must keep the full array");
+
+    const auto joined = build([](IrProgram& program, IrBlock& block, auto& emit, auto& constant, IrValue& active) {
+        auto& phi = program.CreateValue(IrOpcode::Phi, IrType::U32);
+        block.AppendInstruction(&phi);
+        phi.AddPhiOperand(&block, &constant(0x40u));
+        phi.AddPhiOperand(&block, &constant(0x3fcu));
+        emit(IrOpcode::WriteSharedU32, IrType::Void, {&phi, &constant(1u), &active}, 0u);
+    }).first;
+    require(joined == 256u, "function LDS: a phi of bounded addresses must take its largest");
+
+    const auto unsized = build([](IrProgram&, IrBlock&, auto& emit, auto& constant, IrValue& active) {
+        emit(IrOpcode::LoadShared, IrType::U32, {&constant(0u), &active}, 0u);
+    }).first;
+    require(unsized == FunctionLdsDwordLimit, "function LDS: an access without a known width must keep the full array");
+}
+
 int main() {
     try {
         using namespace ShaderRecompiler;
@@ -908,6 +1048,7 @@ int main() {
         verifyComputedTexelOffsets();
         verifyWaveUniformValues();
         verifyTwoLaneUniformValues();
+        verifyFunctionLdsBound();
 #if ANYPS5_ENABLE_SPIRV_TOOLS
         const std::vector<std::uint32_t> minimalSpirv{
             0x07230203u, 0x00010000u, 0u, 5u, 0u,

core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp

@@ -1,7 +1,10 @@
 #include "GraphicsTests.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
 #include <algorithm>
 #include <array>
+#include <bit>
+#include <span>
 #include <string>
 #include <string_view>
 #include <utility>
@@ -22,6 +25,57 @@ void reject(TAction action, std::string_view reason) {
     throw std::runtime_error(std::string("expected texture tiling rejection: ") + std::string(reason));
 }
 
+struct ElementAddress {
+    std::uint32_t mip;
+    std::uint32_t x;
+    std::uint32_t y;
+    std::uint32_t z;
+    std::uint64_t address;
+};
+
+std::uint64_t equationOffset(const TextureSwizzleEquation& equation, std::uint32_t x, std::uint32_t y, std::uint32_t z) {
+    std::uint64_t offset = 0;
+    for (std::uint32_t bit = 0; bit < 16; ++bit) {
+        const auto mask = equation.bits[bit];
+        const auto selected = (x & (mask & 0xfffu)) ^ ((y << 12) & (mask & 0xfff000u)) ^ ((z << 24) & (mask & 0xff000000u));
+        offset |= static_cast<std::uint64_t>(std::popcount(selected) & 1u) << bit;
+    }
+    return offset;
+}
+
+GuestTextureResource thickVolume(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t depth, std::uint32_t mipCount) {
+    GuestTextureResource resource{};
+    resource.width = width;
+    resource.height = height;
+    resource.depthOrLastArray = depth - 1u;
+    resource.mipCount = mipCount;
+    resource.tileMode = tileMode;
+    resource.dimension = TextureDimension::k3D;
+    resource.format = format;
+    return resource;
+}
+
+void requireThickAddresses(const GuestTextureResource& resource, std::uint32_t bytesPerElement, std::uint32_t firstTailLevel, std::uint64_t guestBytes, std::span<const ElementAddress> expected, const std::string& what) {
+    const auto geometry = DescribeSurface(resource);
+    Require(geometry.thick && geometry.mips.size() == resource.mipCount && geometry.guestBytes == guestBytes, what + ": guest size differs from addrlib");
+    for (std::uint32_t level = 0; level < resource.mipCount; ++level) Require(geometry.mips[level].tail == (level >= firstTailLevel), what + ": mip " + std::to_string(level) + " is on the wrong side of the mip tail");
+    const auto blockBytes = resource.tileMode == TextureTileMode::kStandard4KB ? 4096u : 65536u;
+    const auto* equation = FindTextureSwizzleEquation(resource.tileMode == TextureTileMode::kStandard4KB ? 0x105u : 0x109u, bytesPerElement);
+    Require(equation != nullptr, what + ": missing thick swizzle equation");
+    const auto block = ThickBlockExtent(resource.tileMode, bytesPerElement);
+    for (const auto& element : expected) {
+        Require(geometry.HasLayer(element.mip, element.z), what + ": sample slice lies outside its level");
+        const auto& mip = geometry.mips.at(element.mip);
+        auto address = geometry.GuestLayerOffset(element.z) + mip.tiledOffset;
+        if (mip.tail) {
+            address += equationOffset(*equation, element.x + mip.tailX, element.y + mip.tailY, element.z);
+        } else {
+            address += (static_cast<std::uint64_t>(element.y / block[1]) * mip.blocksPerRow + element.x / block[0]) * blockBytes + equationOffset(*equation, element.x, element.y, element.z);
+        }
+        Require(address == element.address, what + ": mip " + std::to_string(element.mip) + " element (" + std::to_string(element.x) + ", " + std::to_string(element.y) + ", " + std::to_string(element.z) + ") detiles from " + std::to_string(address) + " instead of addrlib's " + std::to_string(element.address));
+    }
+}
+
 }
 
 void RunTextureTilingTests() {
@@ -142,11 +196,72 @@ void RunTextureTilingTests() {
         const auto odd = DescribeSurface(volume);
         Require(odd.mips[1].width == 16 && odd.mips[1].blocksPerRow == 3 && odd.mips[1].tiledSize == 12288, "a 3D level must be padded from its size rounded up, as addrlib does");
         Require(odd.mips[0].blocksPerRow == 5 && odd.mips[0].tiledOffset == 12288 && odd.mips[0].tiledSize == 40960, "3D mip 0 of a non-power-of-two volume changed");
-        volume.width = 8;
-        volume.height = 8;
-        volume.depthOrLastArray = 7;
-        volume.mipCount = 2;
-        reject([&] { DescribeSurface(volume); }, "3D texture mip tails are not implemented");
+    }
+
+    {
+        constexpr ElementAddress chainInTail[] = {{0, 0, 0, 0, 0x800}, {0, 7, 7, 7, 0xffc}, {0, 4, 2, 4, 0xe20}, {0, 2, 7, 0, 0x968}, {0, 7, 0, 7, 0xed4}, {1, 0, 0, 0, 0x300}, {1, 3, 3, 3, 0x3fc}, {1, 2, 1, 2, 0x3c8}, {1, 1, 3, 0, 0x32c}, {1, 3, 0, 3, 0x3d4}};
+        requireThickAddresses(thickVolume(TextureTileMode::kStandard4KB, 56, 8, 8, 8, 2), 4, 0, 4096, chainInTail, "SW_4KB_S 32 bpp 8x8x8, 2 levels");
+        constexpr ElementAddress unevenTail[] = {{0, 0, 0, 0, 0x6000}, {0, 32, 19, 11, 0x1f0b8}, {0, 16, 6, 6, 0x85a0}, {0, 11, 19, 0, 0xc06c}, {0, 32, 0, 11, 0x1a090}, {1, 0, 0, 0, 0x3000}, {1, 15, 9, 5, 0x4e5c}, {1, 8, 3, 3, 0x40b8}, {1, 5, 9, 0, 0x3a0c}, {1, 15, 0, 5, 0x4654}, {2, 0, 0, 0, 0x1000}, {2, 7, 4, 2, 0x13c4}, {2, 4, 1, 1, 0x1218}, {2, 2, 4, 0, 0x1140}, {2, 7, 0, 2, 0x12c4}, {3, 0, 0, 0, 0x800}, {3, 3, 1, 0, 0x84c}, {3, 2, 0, 0, 0x840}, {3, 1, 1, 0, 0x80c}, {3, 3, 0, 0, 0x844}, {4, 0, 0, 0, 0x300}, {4, 1, 0, 0, 0x304}, {4, 0, 0, 0, 0x300}, {5, 0, 0, 0, 0x200}};
+        requireThickAddresses(thickVolume(TextureTileMode::kStandard4KB, 56, 33, 20, 12, 6), 4, 3, 131072, unevenTail, "SW_4KB_S 32 bpp 33x20x12, 6 levels");
+        constexpr ElementAddress wideTail[] = {{0, 0, 0, 0, 0x20000}, {0, 39, 23, 19, 0xb0bf8}, {0, 20, 8, 10, 0x2e280}, {0, 13, 23, 0, 0x41b28}, {0, 39, 0, 19, 0x902d8}, {1, 0, 0, 0, 0x10000}, {1, 19, 11, 9, 0x1e178}, {1, 10, 4, 5, 0x11c50}, {1, 6, 11, 0, 0x14360}, {1, 19, 0, 9, 0x1a058}, {2, 0, 0, 0, 0x8000}, {2, 9, 5, 4, 0x9c28}, {2, 5, 2, 2, 0x8388}, {2, 3, 5, 0, 0x8868}, {2, 9, 0, 4, 0x9408}, {3, 0, 0, 0, 0x4000}, {3, 4, 2, 1, 0x4310}, {3, 2, 1, 1, 0x4070}, {3, 1, 2, 0, 0x4108}, {3, 4, 0, 1, 0x4210}, {4, 0, 0, 0, 0x1000}, {4, 1, 0, 0, 0x1008}, {5, 0, 0, 0, 0xa00}};
+        requireThickAddresses(thickVolume(TextureTileMode::kStandard64KB, 71, 40, 24, 20, 6), 8, 2, 786432, wideTail, "SW_64KB_S 64 bpp 40x24x20, 6 levels");
+        constexpr ElementAddress byteTail[] = {{0, 0, 0, 0, 0x20000}, {0, 99, 59, 69, 0x11c8af}, {0, 50, 20, 35, 0x8d116}, {0, 33, 59, 0, 0x4c829}, {0, 99, 0, 69, 0xf8087}, {1, 0, 0, 0, 0x10000}, {1, 49, 29, 34, 0x7d919}, {1, 25, 10, 17, 0x13a25}, {1, 16, 29, 0, 0x15908}, {1, 49, 0, 34, 0x79011}, {2, 0, 0, 0, 0x8000}, {2, 24, 14, 16, 0xbb20}, {2, 12, 5, 8, 0x8748}, {2, 8, 14, 0, 0x8b20}, {2, 24, 0, 16, 0xb200}, {3, 0, 0, 0, 0x4000}, {3, 11, 6, 7, 0x43b7}, {3, 6, 2, 4, 0x40e2}, {3, 4, 6, 0, 0x4160}, {3, 11, 0, 7, 0x4297}, {4, 0, 0, 0, 0x1000}, {4, 5, 2, 3, 0x1075}, {4, 3, 1, 2, 0x101b}, {4, 2, 2, 0, 0x1022}, {4, 5, 0, 3, 0x1055}, {5, 0, 0, 0, 0xa00}, {5, 2, 0, 1, 0xa06}, {5, 1, 0, 0, 0xa01}, {6, 0, 0, 0, 0x900}};
+        requireThickAddresses(thickVolume(TextureTileMode::kStandard64KB, 1, 100, 60, 70, 7), 1, 2, 1179648, byteTail, "SW_64KB_S 8 bpp 100x60x70, 7 levels");
+        constexpr ElementAddress wideElementTail[] = {{0, 0, 0, 0, 0x3000}, {0, 8, 4, 2, 0x5880}, {0, 4, 1, 1, 0x4030}, {0, 3, 4, 0, 0x3a40}, {0, 8, 0, 2, 0x5080}, {1, 0, 0, 0, 0x1000}, {1, 3, 1, 0, 0x1260}, {1, 2, 0, 0, 0x1200}, {1, 1, 1, 0, 0x1060}, {1, 3, 0, 0, 0x1240}, {2, 0, 0, 0, 0x800}, {2, 1, 0, 0, 0x840}, {3, 0, 0, 0, 0x300}};
+        requireThickAddresses(thickVolume(TextureTileMode::kStandard4KB, 77, 9, 5, 3, 4), 16, 2, 24576, wideElementTail, "SW_4KB_S 128 bpp 9x5x3, 4 levels");
+    }
+
+    {
+        // CoveredMipBytes against the XOR address equations: every element slot of a covered range
+        // holds exactly one element of the mip, and every tile block left out holds a slot no
+        // element does (the bytes a write-back must keep).
+        constexpr std::array<TextureTileMode, 4> modes{TextureTileMode::kZ64KBX, TextureTileMode::kS64KBX, TextureTileMode::kD64KBX, TextureTileMode::kR64KBX};
+        constexpr std::array<std::pair<std::uint32_t, std::uint32_t>, 6> sizes{{{200, 150}, {1920, 1080}, {2432, 1368}, {960, 540}, {256, 128}, {300, 1}}};
+        for (const auto mode : modes) {
+            for (std::uint32_t bytesPerElement = 1; bytesPerElement <= 16; bytesPerElement *= 2) {
+                const auto* equation = FindTextureSwizzleEquation(XorSwizzleMode(mode), bytesPerElement);
+                Require(equation != nullptr, "missing XOR swizzle equation");
+                const auto block = ThinBlockLayout(mode, bytesPerElement);
+                for (const auto& [width, height] : sizes) {
+                    if (static_cast<std::uint64_t>(width) * height > (1u << 20) && mode != TextureTileMode::kR64KBX) continue;
+                    const auto what = "XOR mode " + std::to_string(XorSwizzleMode(mode)) + ", " + std::to_string(bytesPerElement) + " bytes, " + std::to_string(width) + "x" + std::to_string(height);
+                    for (const auto& mip : ComputeElementMipLayout(mode, bytesPerElement, width, height, 3)) {
+                        const auto covered = CoveredMipBytes(mode, bytesPerElement, mip);
+                        if (mip.tail) {
+                            Require(covered.empty(), what + ": a tail mip has covered bytes");
+                            continue;
+                        }
+                        std::vector<std::uint8_t> held(static_cast<std::size_t>(mip.tiledSize / bytesPerElement), 0);
+                        for (std::uint32_t y = 0; y < mip.height; ++y) {
+                            for (std::uint32_t x = 0; x < mip.width; ++x) {
+                                const auto offset = (static_cast<std::uint64_t>(y / block[2]) * mip.blocksPerRow + x / block[1]) * block[0] + equationOffset(*equation, x, y, 0);
+                                Require(offset % bytesPerElement == 0 && offset < mip.tiledSize, what + ": an element lies outside the mip");
+                                held[static_cast<std::size_t>(offset / bytesPerElement)] += 1;
+                            }
+                        }
+                        std::vector<bool> inCovered(static_cast<std::size_t>(mip.tiledSize / block[0]), false);
+                        std::uint64_t previous = 0;
+                        for (const auto& [begin, end] : covered) {
+                            Require(begin >= previous && begin < end && end <= mip.tiledSize && begin % block[0] == 0 && end % block[0] == 0, what + ": covered ranges are not ascending whole blocks");
+                            previous = end;
+                            for (auto slot = begin / bytesPerElement; slot < end / bytesPerElement; ++slot) Require(held[static_cast<std::size_t>(slot)] == 1, what + ": a covered byte holds no element or several");
+                            for (auto at = begin; at < end; at += block[0]) inCovered[static_cast<std::size_t>(at / block[0])] = true;
+                        }
+                        for (std::size_t index = 0; index < inCovered.size(); ++index) {
+                            if (inCovered[index]) continue;
+                            const auto first = index * block[0] / bytesPerElement;
+                            const auto last = (index + 1) * block[0] / bytesPerElement;
+                            Require(std::any_of(held.begin() + static_cast<std::ptrdiff_t>(first), held.begin() + static_cast<std::ptrdiff_t>(last), [](std::uint8_t count) { return count == 0; }), what + ": a block every byte of which holds an element is left out");
+                        }
+                    }
+                }
+            }
+        }
+        const auto linear = ComputeElementMipLayout(TextureTileMode::kLinear, 4, 200, 3, 1).front();
+        const auto rows = CoveredMipBytes(TextureTileMode::kLinear, 4, linear);
+        Require(linear.pitchBytes > 800 && rows.size() == 3 && rows[1].first == linear.pitchBytes && rows[1].second == linear.pitchBytes + 800, "linear covered bytes are not the rows without their pitch padding");
+        const auto dense = ComputeElementMipLayout(TextureTileMode::kLinear, 4, 256, 3, 1).front();
+        Require(dense.pitchBytes == 1024 && CoveredMipBytes(TextureTileMode::kLinear, 4, dense) == std::vector<std::pair<std::uint64_t, std::uint64_t>>{{0, 3072}}, "linear rows without padding are not one range");
     }
 
     reject([] { ComputeSurfaceSize({}, 1); }, "empty mip chain");

core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp

@@ -0,0 +1,200 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t MaxThreads = 64;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 8;
+constexpr std::uint32_t Written = 5;
+constexpr std::uint32_t SharedWords = 2;
+constexpr std::uint32_t Untouched = 0xdeadbeefu;
+constexpr std::uint32_t MaskedExecLo = 0x55555555u;
+constexpr std::uint32_t SharedPartial = 0x800186a0u;
+constexpr std::uint32_t SharedExhausted = 1000u;
+alignas(256) std::array<std::uint32_t, MaxThreads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, MaxThreads * Results + SharedWords> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 35> Code{
+    0x34020084, 0x34040085, 0xe0301000, 0x80000401, 0xe0301004, 0x80000501, 0xe0301008, 0x80000601,
+    0xbf8c3f70, 0x7e140280, 0x461414c1, 0x481414c1, 0x7e0e0304, 0xe0d05000, 0x80010702, 0x7e100304,
+    0xbefe03ff, 0x55555555, 0xe0d05004, 0x80010802, 0xbefe03c1, 0x7e120305, 0xe0d04800, 0x80010900,
+    0x7e160306, 0xe0d04804, 0x80010b00, 0xbf8c3f70, 0xe0701008, 0x80010702, 0xe070100c, 0x80010802,
+    0xe0701010, 0x80010a02, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 3> NoReturnCode{
+    0xe0d01000, 0x80010702, 0xbf810000,
+};
+
+struct Workgroup {
+    std::uint32_t threads;
+    std::uint32_t waveSize;
+
+    [[nodiscard]] std::string Name() const {
+        return "wave" + std::to_string(waveSize) + " " + std::to_string(threads) + " threads";
+    }
+};
+
+constexpr std::array<Workgroup, 3> Workgroups{{{32, 32}, {64, 32}, {64, 64}}};
+
+struct Row {
+    std::uint32_t memory;
+    std::uint32_t subtrahend;
+};
+
+constexpr std::array<Row, 16> Edges{{
+    {0u, 0u}, {0u, 1u}, {1u, 0u}, {1u, 1u},
+    {5u, 7u}, {7u, 5u}, {0xffffffffu, 1u}, {1u, 0xffffffffu},
+    {0x80000000u, 0x7fffffffu}, {0x7fffffffu, 0x80000000u}, {0xffffffffu, 0xffffffffu}, {0x80000000u, 0x80000000u},
+    {0x80000001u, 0x80000000u}, {0u, 0xffffffffu}, {0xffffffffu, 0u}, {0x12345678u, 0x12345679u},
+}};
+
+Row RowOf(std::uint32_t tid) {
+    if (tid < Edges.size()) return Edges[tid];
+    const std::uint32_t memory = (tid + 1u) * 0x9e3779b9u;
+    const std::uint32_t subtrahend = (tid + 1u) * 0x85ebca6bu;
+    return {memory, (tid & 3u) == 0u ? subtrahend >> 8u : subtrahend};
+}
+
+std::uint32_t SharedPartialStep(std::uint32_t tid) { return tid + 1u; }
+std::uint32_t SharedExhaustedStep(std::uint32_t tid) { return tid + 100u; }
+
+std::uint32_t SubtractClamped(std::uint32_t memory, std::uint32_t subtrahend) {
+    return memory >= subtrahend ? memory - subtrahend : 0u;
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+std::vector<std::uint32_t> UserData() {
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    return userData;
+}
+
+ShaderRecompiler::RecompileResult Recompile(const AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const Workgroup& workgroup, std::span<const std::uint32_t> userData) {
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{workgroup.threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {workgroup.waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    return ShaderRecompiler::Recompile(request);
+}
+
+void Run(AgcDriver::VulkanDevice& device, const Workgroup& workgroup) {
+    Input.fill(0u);
+    Output.fill(Untouched);
+    for (std::uint32_t tid = 0; tid < MaxThreads; ++tid) {
+        const Row row = RowOf(tid);
+        Input[tid * Inputs] = row.subtrahend;
+        Input[tid * Inputs + 1u] = SharedPartialStep(tid);
+        Input[tid * Inputs + 2u] = SharedExhaustedStep(tid);
+        Output[tid * Results] = row.memory;
+        Output[tid * Results + 1u] = row.memory;
+    }
+    Output[MaxThreads * Results] = SharedPartial;
+    Output[MaxThreads * Results + 1u] = SharedExhausted;
+    const auto userData = UserData();
+    const std::span<const std::uint32_t> code(Code);
+    const auto result = Recompile(device, code, workgroup, userData);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check(const Workgroup& workgroup) {
+    constexpr std::array<const char*, 4> names{"memory", "memory under a partial exec mask", "returned value", "data register under a partial exec mask"};
+    std::uint32_t partialTotal = 0;
+    std::uint32_t exhaustedTotal = 0;
+    for (std::uint32_t tid = 0; tid < MaxThreads; ++tid) {
+        const Row row = RowOf(tid);
+        const auto* out = &Output[tid * Results];
+        const auto where = "buffer atomic csub: " + workgroup.Name() + " thread " + std::to_string(tid);
+        if (tid >= workgroup.threads) {
+            Require(out[0] == row.memory && out[1] == row.memory, where + " is outside the workgroup but its memory changed");
+            for (std::uint32_t j = 2; j < Results; ++j) {
+                Require(out[j] == Untouched, where + " is outside the workgroup but result " + std::to_string(j) + " was written");
+            }
+            continue;
+        }
+        partialTotal += SharedPartialStep(tid);
+        exhaustedTotal += SharedExhaustedStep(tid);
+        const auto lane = out[4];
+        Require(lane < workgroup.waveSize, where + " reports lane " + std::to_string(lane));
+        const bool active = lane >= 32u || ((MaskedExecLo >> lane) & 1u) != 0u;
+        const std::uint32_t clamped = SubtractClamped(row.memory, row.subtrahend);
+        const std::array<std::uint32_t, 4> expected{clamped, active ? clamped : row.memory, row.memory, active ? row.memory : row.subtrahend};
+        for (std::uint32_t j = 0; j < expected.size(); ++j) {
+            Require(out[j] == expected[j], where + " lane " + std::to_string(lane) + " " + names[j] + " is " + Hex(out[j]) + ", expected " + Hex(expected[j]));
+        }
+        for (std::uint32_t j = Written; j < Results; ++j) {
+            Require(out[j] == Untouched, where + " result " + std::to_string(j) + " was written");
+        }
+    }
+    const std::array<std::uint32_t, SharedWords> shared{SubtractClamped(SharedPartial, partialTotal), SubtractClamped(SharedExhausted, exhaustedTotal)};
+    constexpr std::array<const char*, SharedWords> sharedNames{"dword shared by all lanes", "dword exhausted by all lanes"};
+    Require(shared[0] != 0u && shared[1] == 0u, "buffer atomic csub: " + workgroup.Name() + " does not cover both shared outcomes");
+    for (std::uint32_t j = 0; j < SharedWords; ++j) {
+        const auto actual = Output[MaxThreads * Results + j];
+        Require(actual == shared[j], "buffer atomic csub: " + workgroup.Name() + " " + sharedNames[j] + " is " + Hex(actual) + ", expected " + Hex(shared[j]));
+    }
+}
+
+void CheckNoReturnRejected(const AgcDriver::VulkanDevice& device) {
+    const auto userData = UserData();
+    const std::string reason = "buffer_atomic_csub without glc is not supported";
+    std::string failure;
+    try {
+        static_cast<void>(Recompile(device, NoReturnCode, Workgroups[0], userData));
+    } catch (const std::exception& error) {
+        failure = error.what();
+    }
+    Require(failure.find(reason) != std::string::npos, "buffer atomic csub: expected '" + reason + "', got '" + failure + "'");
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        for (const auto& workgroup : Workgroups) {
+            Run(*device, workgroup);
+            Check(workgroup);
+        }
+        CheckNoReturnRejected(*device);
+        std::puts("buffer atomic csub tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsOrSwap64.cpp

@@ -0,0 +1,296 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t MaxThreads = 64;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 128;
+constexpr std::uint32_t Checked = 28;
+alignas(256) std::array<std::uint32_t, MaxThreads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, MaxThreads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 95> Wave32Code{
+    0x34020084, 0x34060089, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xbe880304, 0xbe890305, 0xbe8a03ff,
+    0x00002134, 0xbe8b03ff, 0x31016fac, 0xe0741100, 0x80010403, 0xe0741108, 0x80010403, 0xe0741110,
+    0x80010403, 0xe0741118, 0x80010403, 0xe0741120, 0x80010403, 0xe0741128, 0x80010403, 0xe0741130,
+    0x80010403, 0xe0741138, 0x80010403, 0xbf8c3f70, 0x7e140306, 0x7e160307, 0x7e180306, 0x7e1a0307,
+    0x7e1c0306, 0x7e1e0307, 0x7e200306, 0x7e220307, 0x7e240306, 0x7e260307, 0x7e280306, 0x7e2a0307,
+    0x7e500306, 0x7e520307, 0xe1685100, 0x80010a03, 0xe1681108, 0x80012803, 0xe1405110, 0x80010c03,
+    0xe1401118, 0x80012803, 0x36a00e83, 0x7d8aa080, 0xbea83c6a, 0xe1685120, 0x80010e03, 0xe1405128,
+    0x80011003, 0xbefe03c1, 0xe1685130, 0x80021203, 0xe1405138, 0x80021403, 0xbf8c3f70, 0xe034d100,
+    0x80011603, 0xe034d108, 0x80011803, 0xe034d110, 0x80011a03, 0xe034d118, 0x80011c03, 0xe034d120,
+    0x80011e03, 0xe034d128, 0x80012003, 0xe034d130, 0x80012203, 0xe034d138, 0x80012403, 0xbf8c3f70,
+    0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xe0781020, 0x80011203, 0xe0781030, 0x80011603,
+    0xe0781040, 0x80011a03, 0xe0781050, 0x80011e03, 0xe0781060, 0x80012203, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 95> Wave64Code{
+    0x34020084, 0x34060089, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xbe880304, 0xbe890305, 0xbe8a03ff,
+    0x00004134, 0xbe8b03ff, 0x31016fac, 0xe0741100, 0x80010403, 0xe0741108, 0x80010403, 0xe0741110,
+    0x80010403, 0xe0741118, 0x80010403, 0xe0741120, 0x80010403, 0xe0741128, 0x80010403, 0xe0741130,
+    0x80010403, 0xe0741138, 0x80010403, 0xbf8c3f70, 0x7e140306, 0x7e160307, 0x7e180306, 0x7e1a0307,
+    0x7e1c0306, 0x7e1e0307, 0x7e200306, 0x7e220307, 0x7e240306, 0x7e260307, 0x7e280306, 0x7e2a0307,
+    0x7e500306, 0x7e520307, 0xe1685100, 0x80010a03, 0xe1681108, 0x80012803, 0xe1405110, 0x80010c03,
+    0xe1401118, 0x80012803, 0x36a00e83, 0x7d8aa080, 0xbea8246a, 0xe1685120, 0x80010e03, 0xe1405128,
+    0x80011003, 0xbefe04c1, 0xe1685130, 0x80021203, 0xe1405138, 0x80021403, 0xbf8c3f70, 0xe034d100,
+    0x80011603, 0xe034d108, 0x80011803, 0xe034d110, 0x80011a03, 0xe034d118, 0x80011c03, 0xe034d120,
+    0x80011e03, 0xe034d128, 0x80012003, 0xe034d130, 0x80012203, 0xe034d138, 0x80012403, 0xbf8c3f70,
+    0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xe0781020, 0x80011203, 0xe0781030, 0x80011603,
+    0xe0781040, 0x80011a03, 0xe0781050, 0x80011e03, 0xe0781060, 0x80012203, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[64][4] = {
+    {0x6a6ae768u, 0x89abcdefu, 0x466de486u, 0x00ff00ffu},
+    {0x00ff00ffu, 0x00ff00ffu, 0x80000000u, 0x2e09e4b8u},
+    {0x2be38eccu, 0xdd1775a9u, 0xab195b47u, 0xe88f40a2u},
+    {0xe9a462ceu, 0x00000000u, 0x2b1141d6u, 0xac0e36d5u},
+    {0x64ffc604u, 0xbcfdc56au, 0x2af2003cu, 0x8f4c6fc3u},
+    {0x70ce7993u, 0x00ff00ffu, 0x80000000u, 0x7b05bc1au},
+    {0x00000001u, 0xba30a818u, 0x6e325d76u, 0x7767929du},
+    {0x7e974317u, 0x00ff00ffu, 0xddd59451u, 0xdf30a0eau},
+    {0xffffffffu, 0x159840eau, 0xe41c70dbu, 0x88d1f52cu},
+    {0x89abcdefu, 0x00000000u, 0x6dd9de7bu, 0x93cd7e6fu},
+    {0x7fffffffu, 0x6d924e06u, 0x379cf45fu, 0x5a176daau},
+    {0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xdd343334u},
+    {0xf4c65508u, 0x774afa8fu, 0x4bf01f77u, 0x80000000u},
+    {0x0680d87fu, 0xe2aeba83u, 0x0cae8376u, 0x21027ce8u},
+    {0x12345678u, 0x24483dacu, 0x343089c1u, 0xfe3a9921u},
+    {0x9dd441ebu, 0x7fffffffu, 0x3f250fabu, 0x5a25508fu},
+    {0x80000000u, 0x940c48d1u, 0x00000001u, 0x12345678u},
+    {0x06a06e39u, 0xdca4271cu, 0x89abcdefu, 0x80000000u},
+    {0x00ff00ffu, 0xdadef6e0u, 0xdfd0b7a2u, 0xffffffffu},
+    {0x436d4bd7u, 0xb4373875u, 0xc71d5b33u, 0x63b433c8u},
+    {0xe1a9611du, 0xdd70b68bu, 0xc3e41dfbu, 0x80000000u},
+    {0x00ff00ffu, 0x00ff00ffu, 0x2a2b5f67u, 0xf2d9cc0du},
+    {0x89abcdefu, 0x38573260u, 0x5b120ed1u, 0x49e07752u},
+    {0xf60106b7u, 0xd61d1405u, 0xc58ccccdu, 0xbce22306u},
+    {0x7fffffffu, 0x80000000u, 0x15452b7bu, 0xb8054218u},
+    {0x12345678u, 0x00000001u, 0xa7387a4cu, 0xffffffffu},
+    {0x00000000u, 0x90322d5bu, 0x7ee058dau, 0x89abcdefu},
+    {0x7fffffffu, 0x93b8f2e0u, 0x8245e09du, 0x168b95f5u},
+    {0x80000000u, 0x4ec04618u, 0x44b6a55du, 0x89abcdefu},
+    {0x80a3acc4u, 0x7fffffffu, 0x1f016769u, 0xac7c83b9u},
+    {0x725f6ff6u, 0x00000000u, 0x45040843u, 0x00ff00ffu},
+    {0x50d14398u, 0x7f767333u, 0x2369eadcu, 0x00000001u},
+    {0xac8c599bu, 0x7fffffffu, 0xd8f36844u, 0x4761f31au},
+    {0xd078019fu, 0x60ac3d38u, 0x7fffffffu, 0x20b5d1d4u},
+    {0x2925feb3u, 0x00000001u, 0x7fffffffu, 0xb16614f4u},
+    {0x86d5360fu, 0xfaf40417u, 0xd4424852u, 0x89631325u},
+    {0xffffffffu, 0xde274210u, 0xe96e12c9u, 0x65305d1cu},
+    {0x1043c4f3u, 0x8a860eeau, 0xe9579523u, 0x7563acb0u},
+    {0x310b55c7u, 0x12345678u, 0x12345678u, 0x96188707u},
+    {0x80000000u, 0xbcabb1bau, 0x12345678u, 0xffffffffu},
+    {0x72fdad03u, 0x00000001u, 0x028f3b02u, 0x6c60da42u},
+    {0xe473cdc9u, 0x75eb16c7u, 0x12345678u, 0xe0d59f3cu},
+    {0x89abcdefu, 0x74102553u, 0x1cb3e931u, 0xe536aae9u},
+    {0x4d126f38u, 0x7d42a12au, 0x50692ce5u, 0xa68132ddu},
+    {0x744e3f0eu, 0x7fffffffu, 0xde298785u, 0x80000000u},
+    {0xad554ba1u, 0x7c5153d0u, 0xeaf8edcfu, 0xc8aab92fu},
+    {0xf8f008c5u, 0x0f119987u, 0x1bd39c7cu, 0x5dc52692u},
+    {0xd03b304du, 0x00ff00ffu, 0x89abcdefu, 0x991132c0u},
+    {0xa92c8f72u, 0x07b9077du, 0x66546794u, 0xf6a5f7c1u},
+    {0xb5691c18u, 0x5d6c710au, 0x7ae6b997u, 0xbfe4abb0u},
+    {0xc9570a1cu, 0xefa96b6eu, 0x00000000u, 0xcd12bfb3u},
+    {0x00000000u, 0x37eadcd2u, 0x3a6d60aau, 0x89abcdefu},
+    {0x114a2d32u, 0x00000000u, 0x89abcdefu, 0x12345678u},
+    {0x14090b01u, 0xffffffffu, 0x0eb083edu, 0x1a0caeefu},
+    {0x12345678u, 0x4521d7efu, 0x00000001u, 0x80000000u},
+    {0xefa94a8eu, 0x561bfb20u, 0x19b0c2f0u, 0x69b1111du},
+    {0xbcf7629fu, 0xaf1de605u, 0x41622456u, 0x00000000u},
+    {0xd47a46a9u, 0x12da9645u, 0x7ded0e8du, 0x00000000u},
+    {0x697d50dbu, 0x89abcdefu, 0xdeacacaau, 0xcf93dd30u},
+    {0x00ff00ffu, 0xb6a48819u, 0x1f752ee6u, 0x80000000u},
+    {0x12345678u, 0xbb36bb87u, 0x0888385bu, 0x5cb6c6d7u},
+    {0xd24df64du, 0x9595f0c7u, 0x00000000u, 0x7fffffffu},
+    {0xffffffffu, 0x00ff00ffu, 0x00000001u, 0xfe7ffe21u},
+    {0xdd6537d2u, 0x6c2ab542u, 0xffffffffu, 0xde5ffb8au}
+};
+
+constexpr std::uint32_t Expected32[32][28] = {
+    {0x6a6ae768u, 0x89abcdefu, 0x6a6ae768u, 0x89abcdefu, 0x6a6ae768u, 0x89abcdefu, 0x6a6ae768u, 0x89abcdefu, 0x6a6ae768u, 0x89abcdefu, 0x6a6ae768u, 0x89abcdefu, 0x6e6fe7eeu, 0x89ffcdffu, 0x6e6fe7eeu, 0x89ffcdffu, 0x466de486u, 0x00ff00ffu, 0x466de486u, 0x00ff00ffu, 0x6e6fe7eeu, 0x89ffcdffu, 0x466de486u, 0x00ff00ffu, 0x6e6fe7eeu, 0x89ffcdffu, 0x466de486u, 0x00ff00ffu},
+    {0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x80000000u, 0x2e09e4b8u, 0x80000000u, 0x2e09e4b8u, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x80ff00ffu, 0x2effe4ffu, 0x80ff00ffu, 0x2effe4ffu, 0x80000000u, 0x2e09e4b8u, 0x80000000u, 0x2e09e4b8u, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x80ff00ffu, 0x2effe4ffu, 0x80000000u, 0x2e09e4b8u},
+    {0x2be38eccu, 0xdd1775a9u, 0x2be38eccu, 0xdd1775a9u, 0x2be38eccu, 0xdd1775a9u, 0x2be38eccu, 0xdd1775a9u, 0x2be38eccu, 0xdd1775a9u, 0x2be38eccu, 0xdd1775a9u, 0xabfbdfcfu, 0xfd9f75abu, 0xabfbdfcfu, 0xfd9f75abu, 0xab195b47u, 0xe88f40a2u, 0xab195b47u, 0xe88f40a2u, 0xabfbdfcfu, 0xfd9f75abu, 0xab195b47u, 0xe88f40a2u, 0xabfbdfcfu, 0xfd9f75abu, 0xab195b47u, 0xe88f40a2u},
+    {0xe9a462ceu, 0x00000000u, 0xe9a462ceu, 0x00000000u, 0xe9a462ceu, 0x00000000u, 0xe9a462ceu, 0x00000000u, 0xe9a462ceu, 0x00000000u, 0xe9a462ceu, 0x00000000u, 0xebb563deu, 0xac0e36d5u, 0xebb563deu, 0xac0e36d5u, 0x2b1141d6u, 0xac0e36d5u, 0x2b1141d6u, 0xac0e36d5u, 0xebb563deu, 0xac0e36d5u, 0x2b1141d6u, 0xac0e36d5u, 0xebb563deu, 0xac0e36d5u, 0x2b1141d6u, 0xac0e36d5u},
+    {0x64ffc604u, 0xbcfdc56au, 0x64ffc604u, 0xbcfdc56au, 0x64ffc604u, 0xbcfdc56au, 0x64ffc604u, 0xbcfdc56au, 0x64ffc604u, 0xbcfdc56au, 0x64ffc604u, 0xbcfdc56au, 0x6effc63cu, 0xbffdefebu, 0x6effc63cu, 0xbffdefebu, 0x2af2003cu, 0x8f4c6fc3u, 0x2af2003cu, 0x8f4c6fc3u, 0x6effc63cu, 0xbffdefebu, 0x2af2003cu, 0x8f4c6fc3u, 0x6effc63cu, 0xbffdefebu, 0x2af2003cu, 0x8f4c6fc3u},
+    {0x70ce7993u, 0x00ff00ffu, 0x70ce7993u, 0x00ff00ffu, 0x70ce7993u, 0x00ff00ffu, 0x70ce7993u, 0x00ff00ffu, 0x70ce7993u, 0x00ff00ffu, 0x70ce7993u, 0x00ff00ffu, 0xf0ce7993u, 0x7bffbcffu, 0xf0ce7993u, 0x7bffbcffu, 0x80000000u, 0x7b05bc1au, 0x80000000u, 0x7b05bc1au, 0xf0ce7993u, 0x7bffbcffu, 0x80000000u, 0x7b05bc1au, 0xf0ce7993u, 0x7bffbcffu, 0x80000000u, 0x7b05bc1au},
+    {0x00000001u, 0xba30a818u, 0x00000001u, 0xba30a818u, 0x00000001u, 0xba30a818u, 0x00000001u, 0xba30a818u, 0x00000001u, 0xba30a818u, 0x00000001u, 0xba30a818u, 0x6e325d77u, 0xff77ba9du, 0x6e325d77u, 0xff77ba9du, 0x6e325d76u, 0x7767929du, 0x6e325d76u, 0x7767929du, 0x6e325d77u, 0xff77ba9du, 0x6e325d76u, 0x7767929du, 0x6e325d77u, 0xff77ba9du, 0x6e325d76u, 0x7767929du},
+    {0x7e974317u, 0x00ff00ffu, 0x7e974317u, 0x00ff00ffu, 0x7e974317u, 0x00ff00ffu, 0x7e974317u, 0x00ff00ffu, 0x7e974317u, 0x00ff00ffu, 0x7e974317u, 0x00ff00ffu, 0xffd7d757u, 0xdfffa0ffu, 0xffd7d757u, 0xdfffa0ffu, 0xddd59451u, 0xdf30a0eau, 0xddd59451u, 0xdf30a0eau, 0xffd7d757u, 0xdfffa0ffu, 0xddd59451u, 0xdf30a0eau, 0xffd7d757u, 0xdfffa0ffu, 0xddd59451u, 0xdf30a0eau},
+    {0xffffffffu, 0x159840eau, 0xffffffffu, 0x159840eau, 0xe41c70dbu, 0x88d1f52cu, 0xe41c70dbu, 0x88d1f52cu, 0xffffffffu, 0x159840eau, 0xffffffffu, 0x159840eau, 0xffffffffu, 0x9dd9f5eeu, 0xffffffffu, 0x9dd9f5eeu, 0xe41c70dbu, 0x88d1f52cu, 0xe41c70dbu, 0x88d1f52cu, 0xffffffffu, 0x159840eau, 0xffffffffu, 0x159840eau, 0xffffffffu, 0x9dd9f5eeu, 0xe41c70dbu, 0x88d1f52cu},
+    {0x89abcdefu, 0x00000000u, 0x89abcdefu, 0x00000000u, 0x89abcdefu, 0x00000000u, 0x89abcdefu, 0x00000000u, 0x89abcdefu, 0x00000000u, 0x89abcdefu, 0x00000000u, 0xedfbdfffu, 0x93cd7e6fu, 0xedfbdfffu, 0x93cd7e6fu, 0x6dd9de7bu, 0x93cd7e6fu, 0x6dd9de7bu, 0x93cd7e6fu, 0xedfbdfffu, 0x93cd7e6fu, 0x6dd9de7bu, 0x93cd7e6fu, 0xedfbdfffu, 0x93cd7e6fu, 0x6dd9de7bu, 0x93cd7e6fu},
+    {0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x7f976faeu, 0x7fffffffu, 0x7f976faeu, 0x379cf45fu, 0x5a176daau, 0x379cf45fu, 0x5a176daau, 0x7fffffffu, 0x7f976faeu, 0x379cf45fu, 0x5a176daau, 0x7fffffffu, 0x7f976faeu, 0x379cf45fu, 0x5a176daau},
+    {0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xdd343334u, 0x89abcdefu, 0xdd343334u, 0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xfd3eb3bdu, 0x89abcdefu, 0xfd3eb3bdu, 0x89abcdefu, 0xdd343334u, 0x89abcdefu, 0xdd343334u, 0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xfd3eb3bdu, 0x89abcdefu, 0xdd343334u},
+    {0xf4c65508u, 0x774afa8fu, 0xf4c65508u, 0x774afa8fu, 0x4bf01f77u, 0x80000000u, 0x4bf01f77u, 0x80000000u, 0xf4c65508u, 0x774afa8fu, 0xf4c65508u, 0x774afa8fu, 0xfff65f7fu, 0xf74afa8fu, 0xfff65f7fu, 0xf74afa8fu, 0x4bf01f77u, 0x80000000u, 0x4bf01f77u, 0x80000000u, 0xf4c65508u, 0x774afa8fu, 0xf4c65508u, 0x774afa8fu, 0xfff65f7fu, 0xf74afa8fu, 0x4bf01f77u, 0x80000000u},
+    {0x0680d87fu, 0xe2aeba83u, 0x0680d87fu, 0xe2aeba83u, 0x0cae8376u, 0x21027ce8u, 0x0cae8376u, 0x21027ce8u, 0x0680d87fu, 0xe2aeba83u, 0x0680d87fu, 0xe2aeba83u, 0x0eaedb7fu, 0xe3aefeebu, 0x0eaedb7fu, 0xe3aefeebu, 0x0cae8376u, 0x21027ce8u, 0x0cae8376u, 0x21027ce8u, 0x0680d87fu, 0xe2aeba83u, 0x0680d87fu, 0xe2aeba83u, 0x0eaedb7fu, 0xe3aefeebu, 0x0cae8376u, 0x21027ce8u},
+    {0x12345678u, 0x24483dacu, 0x12345678u, 0x24483dacu, 0x12345678u, 0x24483dacu, 0x12345678u, 0x24483dacu, 0x12345678u, 0x24483dacu, 0x12345678u, 0x24483dacu, 0x3634dff9u, 0xfe7abdadu, 0x3634dff9u, 0xfe7abdadu, 0x343089c1u, 0xfe3a9921u, 0x343089c1u, 0xfe3a9921u, 0x3634dff9u, 0xfe7abdadu, 0x343089c1u, 0xfe3a9921u, 0x3634dff9u, 0xfe7abdadu, 0x343089c1u, 0xfe3a9921u},
+    {0x9dd441ebu, 0x7fffffffu, 0x9dd441ebu, 0x7fffffffu, 0x9dd441ebu, 0x7fffffffu, 0x9dd441ebu, 0x7fffffffu, 0x9dd441ebu, 0x7fffffffu, 0x9dd441ebu, 0x7fffffffu, 0xbff54febu, 0x7fffffffu, 0xbff54febu, 0x7fffffffu, 0x3f250fabu, 0x5a25508fu, 0x3f250fabu, 0x5a25508fu, 0xbff54febu, 0x7fffffffu, 0x3f250fabu, 0x5a25508fu, 0xbff54febu, 0x7fffffffu, 0x3f250fabu, 0x5a25508fu},
+    {0x80000000u, 0x940c48d1u, 0x80000000u, 0x940c48d1u, 0x00000001u, 0x12345678u, 0x00000001u, 0x12345678u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000001u, 0x963c5ef9u, 0x80000001u, 0x963c5ef9u, 0x00000001u, 0x12345678u, 0x00000001u, 0x12345678u, 0x80000000u, 0x940c48d1u, 0x80000000u, 0x940c48d1u, 0x80000000u, 0x940c48d1u, 0x80000000u, 0x940c48d1u},
+    {0x06a06e39u, 0xdca4271cu, 0x06a06e39u, 0xdca4271cu, 0x89abcdefu, 0x80000000u, 0x89abcdefu, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x8fabefffu, 0xdca4271cu, 0x8fabefffu, 0xdca4271cu, 0x89abcdefu, 0x80000000u, 0x89abcdefu, 0x80000000u, 0x06a06e39u, 0xdca4271cu, 0x06a06e39u, 0xdca4271cu, 0x06a06e39u, 0xdca4271cu, 0x06a06e39u, 0xdca4271cu},
+    {0x00ff00ffu, 0xdadef6e0u, 0x00ff00ffu, 0xdadef6e0u, 0x00ff00ffu, 0xdadef6e0u, 0x00ff00ffu, 0xdadef6e0u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xdfffb7ffu, 0xffffffffu, 0xdfffb7ffu, 0xffffffffu, 0xdfd0b7a2u, 0xffffffffu, 0xdfd0b7a2u, 0xffffffffu, 0xdfffb7ffu, 0xffffffffu, 0xdfd0b7a2u, 0xffffffffu, 0x00ff00ffu, 0xdadef6e0u, 0x00ff00ffu, 0xdadef6e0u},
+    {0x436d4bd7u, 0xb4373875u, 0x436d4bd7u, 0xb4373875u, 0xc71d5b33u, 0x63b433c8u, 0xc71d5b33u, 0x63b433c8u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xc77d5bf7u, 0xf7b73bfdu, 0xc77d5bf7u, 0xf7b73bfdu, 0xc71d5b33u, 0x63b433c8u, 0xc71d5b33u, 0x63b433c8u, 0x436d4bd7u, 0xb4373875u, 0x436d4bd7u, 0xb4373875u, 0x436d4bd7u, 0xb4373875u, 0x436d4bd7u, 0xb4373875u},
+    {0xe1a9611du, 0xdd70b68bu, 0xe1a9611du, 0xdd70b68bu, 0xc3e41dfbu, 0x80000000u, 0xc3e41dfbu, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xe3ed7dffu, 0xdd70b68bu, 0xe3ed7dffu, 0xdd70b68bu, 0xc3e41dfbu, 0x80000000u, 0xc3e41dfbu, 0x80000000u, 0xe1a9611du, 0xdd70b68bu, 0xe1a9611du, 0xdd70b68bu, 0xe1a9611du, 0xdd70b68bu, 0xe1a9611du, 0xdd70b68bu},
+    {0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x2aff5fffu, 0xf2ffccffu, 0x2aff5fffu, 0xf2ffccffu, 0x2a2b5f67u, 0xf2d9cc0du, 0x2a2b5f67u, 0xf2d9cc0du, 0x2aff5fffu, 0xf2ffccffu, 0x2a2b5f67u, 0xf2d9cc0du, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu},
+    {0x89abcdefu, 0x38573260u, 0x89abcdefu, 0x38573260u, 0x89abcdefu, 0x38573260u, 0x89abcdefu, 0x38573260u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xdbbbcfffu, 0x79f77772u, 0xdbbbcfffu, 0x79f77772u, 0x5b120ed1u, 0x49e07752u, 0x5b120ed1u, 0x49e07752u, 0xdbbbcfffu, 0x79f77772u, 0x5b120ed1u, 0x49e07752u, 0x89abcdefu, 0x38573260u, 0x89abcdefu, 0x38573260u},
+    {0xf60106b7u, 0xd61d1405u, 0xf60106b7u, 0xd61d1405u, 0xf60106b7u, 0xd61d1405u, 0xf60106b7u, 0xd61d1405u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xf78dceffu, 0xfeff3707u, 0xf78dceffu, 0xfeff3707u, 0xc58ccccdu, 0xbce22306u, 0xc58ccccdu, 0xbce22306u, 0xf78dceffu, 0xfeff3707u, 0xc58ccccdu, 0xbce22306u, 0xf60106b7u, 0xd61d1405u, 0xf60106b7u, 0xd61d1405u},
+    {0x7fffffffu, 0x80000000u, 0x7fffffffu, 0x80000000u, 0x15452b7bu, 0xb8054218u, 0x15452b7bu, 0xb8054218u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fffffffu, 0xb8054218u, 0x7fffffffu, 0xb8054218u, 0x15452b7bu, 0xb8054218u, 0x15452b7bu, 0xb8054218u, 0x7fffffffu, 0x80000000u, 0x7fffffffu, 0x80000000u, 0x7fffffffu, 0x80000000u, 0x7fffffffu, 0x80000000u},
+    {0x12345678u, 0x00000001u, 0x12345678u, 0x00000001u, 0x12345678u, 0x00000001u, 0x12345678u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xb73c7e7cu, 0xffffffffu, 0xb73c7e7cu, 0xffffffffu, 0xa7387a4cu, 0xffffffffu, 0xa7387a4cu, 0xffffffffu, 0xb73c7e7cu, 0xffffffffu, 0xa7387a4cu, 0xffffffffu, 0x12345678u, 0x00000001u, 0x12345678u, 0x00000001u},
+    {0x00000000u, 0x90322d5bu, 0x00000000u, 0x90322d5bu, 0x00000000u, 0x90322d5bu, 0x00000000u, 0x90322d5bu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ee058dau, 0x99bbedffu, 0x7ee058dau, 0x99bbedffu, 0x7ee058dau, 0x89abcdefu, 0x7ee058dau, 0x89abcdefu, 0x7ee058dau, 0x99bbedffu, 0x7ee058dau, 0x89abcdefu, 0x00000000u, 0x90322d5bu, 0x00000000u, 0x90322d5bu},
+    {0x7fffffffu, 0x93b8f2e0u, 0x7fffffffu, 0x93b8f2e0u, 0x7fffffffu, 0x93b8f2e0u, 0x7fffffffu, 0x93b8f2e0u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0x97bbf7f5u, 0xffffffffu, 0x97bbf7f5u, 0x8245e09du, 0x168b95f5u, 0x8245e09du, 0x168b95f5u, 0xffffffffu, 0x97bbf7f5u, 0x8245e09du, 0x168b95f5u, 0x7fffffffu, 0x93b8f2e0u, 0x7fffffffu, 0x93b8f2e0u},
+    {0x80000000u, 0x4ec04618u, 0x80000000u, 0x4ec04618u, 0x80000000u, 0x4ec04618u, 0x80000000u, 0x4ec04618u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xc4b6a55du, 0xcfebcfffu, 0xc4b6a55du, 0xcfebcfffu, 0x44b6a55du, 0x89abcdefu, 0x44b6a55du, 0x89abcdefu, 0xc4b6a55du, 0xcfebcfffu, 0x44b6a55du, 0x89abcdefu, 0x80000000u, 0x4ec04618u, 0x80000000u, 0x4ec04618u},
+    {0x80a3acc4u, 0x7fffffffu, 0x80a3acc4u, 0x7fffffffu, 0x80a3acc4u, 0x7fffffffu, 0x80a3acc4u, 0x7fffffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x9fa3efedu, 0xffffffffu, 0x9fa3efedu, 0xffffffffu, 0x1f016769u, 0xac7c83b9u, 0x1f016769u, 0xac7c83b9u, 0x9fa3efedu, 0xffffffffu, 0x1f016769u, 0xac7c83b9u, 0x80a3acc4u, 0x7fffffffu, 0x80a3acc4u, 0x7fffffffu},
+    {0x725f6ff6u, 0x00000000u, 0x725f6ff6u, 0x00000000u, 0x725f6ff6u, 0x00000000u, 0x725f6ff6u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x775f6ff7u, 0x00ff00ffu, 0x775f6ff7u, 0x00ff00ffu, 0x45040843u, 0x00ff00ffu, 0x45040843u, 0x00ff00ffu, 0x775f6ff7u, 0x00ff00ffu, 0x45040843u, 0x00ff00ffu, 0x725f6ff6u, 0x00000000u, 0x725f6ff6u, 0x00000000u},
+    {0x50d14398u, 0x7f767333u, 0x50d14398u, 0x7f767333u, 0x50d14398u, 0x7f767333u, 0x50d14398u, 0x7f767333u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x73f9ebdcu, 0x7f767333u, 0x73f9ebdcu, 0x7f767333u, 0x2369eadcu, 0x00000001u, 0x2369eadcu, 0x00000001u, 0x73f9ebdcu, 0x7f767333u, 0x2369eadcu, 0x00000001u, 0x50d14398u, 0x7f767333u, 0x50d14398u, 0x7f767333u}
+};
+constexpr std::uint32_t Expected64[64][28] = {
+    {0x6a6ae768u, 0x89abcdefu, 0x6a6ae768u, 0x89abcdefu, 0x6a6ae768u, 0x89abcdefu, 0x6a6ae768u, 0x89abcdefu, 0x6a6ae768u, 0x89abcdefu, 0x6a6ae768u, 0x89abcdefu, 0x6e6fe7eeu, 0x89ffcdffu, 0x6e6fe7eeu, 0x89ffcdffu, 0x466de486u, 0x00ff00ffu, 0x466de486u, 0x00ff00ffu, 0x6e6fe7eeu, 0x89ffcdffu, 0x466de486u, 0x00ff00ffu, 0x6e6fe7eeu, 0x89ffcdffu, 0x466de486u, 0x00ff00ffu},
+    {0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x80000000u, 0x2e09e4b8u, 0x80000000u, 0x2e09e4b8u, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x80ff00ffu, 0x2effe4ffu, 0x80ff00ffu, 0x2effe4ffu, 0x80000000u, 0x2e09e4b8u, 0x80000000u, 0x2e09e4b8u, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x80ff00ffu, 0x2effe4ffu, 0x80000000u, 0x2e09e4b8u},
+    {0x2be38eccu, 0xdd1775a9u, 0x2be38eccu, 0xdd1775a9u, 0x2be38eccu, 0xdd1775a9u, 0x2be38eccu, 0xdd1775a9u, 0x2be38eccu, 0xdd1775a9u, 0x2be38eccu, 0xdd1775a9u, 0xabfbdfcfu, 0xfd9f75abu, 0xabfbdfcfu, 0xfd9f75abu, 0xab195b47u, 0xe88f40a2u, 0xab195b47u, 0xe88f40a2u, 0xabfbdfcfu, 0xfd9f75abu, 0xab195b47u, 0xe88f40a2u, 0xabfbdfcfu, 0xfd9f75abu, 0xab195b47u, 0xe88f40a2u},
+    {0xe9a462ceu, 0x00000000u, 0xe9a462ceu, 0x00000000u, 0xe9a462ceu, 0x00000000u, 0xe9a462ceu, 0x00000000u, 0xe9a462ceu, 0x00000000u, 0xe9a462ceu, 0x00000000u, 0xebb563deu, 0xac0e36d5u, 0xebb563deu, 0xac0e36d5u, 0x2b1141d6u, 0xac0e36d5u, 0x2b1141d6u, 0xac0e36d5u, 0xebb563deu, 0xac0e36d5u, 0x2b1141d6u, 0xac0e36d5u, 0xebb563deu, 0xac0e36d5u, 0x2b1141d6u, 0xac0e36d5u},
+    {0x64ffc604u, 0xbcfdc56au, 0x64ffc604u, 0xbcfdc56au, 0x64ffc604u, 0xbcfdc56au, 0x64ffc604u, 0xbcfdc56au, 0x64ffc604u, 0xbcfdc56au, 0x64ffc604u, 0xbcfdc56au, 0x6effc63cu, 0xbffdefebu, 0x6effc63cu, 0xbffdefebu, 0x2af2003cu, 0x8f4c6fc3u, 0x2af2003cu, 0x8f4c6fc3u, 0x6effc63cu, 0xbffdefebu, 0x2af2003cu, 0x8f4c6fc3u, 0x6effc63cu, 0xbffdefebu, 0x2af2003cu, 0x8f4c6fc3u},
+    {0x70ce7993u, 0x00ff00ffu, 0x70ce7993u, 0x00ff00ffu, 0x70ce7993u, 0x00ff00ffu, 0x70ce7993u, 0x00ff00ffu, 0x70ce7993u, 0x00ff00ffu, 0x70ce7993u, 0x00ff00ffu, 0xf0ce7993u, 0x7bffbcffu, 0xf0ce7993u, 0x7bffbcffu, 0x80000000u, 0x7b05bc1au, 0x80000000u, 0x7b05bc1au, 0xf0ce7993u, 0x7bffbcffu, 0x80000000u, 0x7b05bc1au, 0xf0ce7993u, 0x7bffbcffu, 0x80000000u, 0x7b05bc1au},
+    {0x00000001u, 0xba30a818u, 0x00000001u, 0xba30a818u, 0x00000001u, 0xba30a818u, 0x00000001u, 0xba30a818u, 0x00000001u, 0xba30a818u, 0x00000001u, 0xba30a818u, 0x6e325d77u, 0xff77ba9du, 0x6e325d77u, 0xff77ba9du, 0x6e325d76u, 0x7767929du, 0x6e325d76u, 0x7767929du, 0x6e325d77u, 0xff77ba9du, 0x6e325d76u, 0x7767929du, 0x6e325d77u, 0xff77ba9du, 0x6e325d76u, 0x7767929du},
+    {0x7e974317u, 0x00ff00ffu, 0x7e974317u, 0x00ff00ffu, 0x7e974317u, 0x00ff00ffu, 0x7e974317u, 0x00ff00ffu, 0x7e974317u, 0x00ff00ffu, 0x7e974317u, 0x00ff00ffu, 0xffd7d757u, 0xdfffa0ffu, 0xffd7d757u, 0xdfffa0ffu, 0xddd59451u, 0xdf30a0eau, 0xddd59451u, 0xdf30a0eau, 0xffd7d757u, 0xdfffa0ffu, 0xddd59451u, 0xdf30a0eau, 0xffd7d757u, 0xdfffa0ffu, 0xddd59451u, 0xdf30a0eau},
+    {0xffffffffu, 0x159840eau, 0xffffffffu, 0x159840eau, 0xe41c70dbu, 0x88d1f52cu, 0xe41c70dbu, 0x88d1f52cu, 0xffffffffu, 0x159840eau, 0xffffffffu, 0x159840eau, 0xffffffffu, 0x9dd9f5eeu, 0xffffffffu, 0x9dd9f5eeu, 0xe41c70dbu, 0x88d1f52cu, 0xe41c70dbu, 0x88d1f52cu, 0xffffffffu, 0x159840eau, 0xffffffffu, 0x159840eau, 0xffffffffu, 0x9dd9f5eeu, 0xe41c70dbu, 0x88d1f52cu},
+    {0x89abcdefu, 0x00000000u, 0x89abcdefu, 0x00000000u, 0x89abcdefu, 0x00000000u, 0x89abcdefu, 0x00000000u, 0x89abcdefu, 0x00000000u, 0x89abcdefu, 0x00000000u, 0xedfbdfffu, 0x93cd7e6fu, 0xedfbdfffu, 0x93cd7e6fu, 0x6dd9de7bu, 0x93cd7e6fu, 0x6dd9de7bu, 0x93cd7e6fu, 0xedfbdfffu, 0x93cd7e6fu, 0x6dd9de7bu, 0x93cd7e6fu, 0xedfbdfffu, 0x93cd7e6fu, 0x6dd9de7bu, 0x93cd7e6fu},
+    {0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x6d924e06u, 0x7fffffffu, 0x7f976faeu, 0x7fffffffu, 0x7f976faeu, 0x379cf45fu, 0x5a176daau, 0x379cf45fu, 0x5a176daau, 0x7fffffffu, 0x7f976faeu, 0x379cf45fu, 0x5a176daau, 0x7fffffffu, 0x7f976faeu, 0x379cf45fu, 0x5a176daau},
+    {0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xdd343334u, 0x89abcdefu, 0xdd343334u, 0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xfd3eb3bdu, 0x89abcdefu, 0xfd3eb3bdu, 0x89abcdefu, 0xdd343334u, 0x89abcdefu, 0xdd343334u, 0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xfc3eb39du, 0x89abcdefu, 0xfd3eb3bdu, 0x89abcdefu, 0xdd343334u},
+    {0xf4c65508u, 0x774afa8fu, 0xf4c65508u, 0x774afa8fu, 0x4bf01f77u, 0x80000000u, 0x4bf01f77u, 0x80000000u, 0xf4c65508u, 0x774afa8fu, 0xf4c65508u, 0x774afa8fu, 0xfff65f7fu, 0xf74afa8fu, 0xfff65f7fu, 0xf74afa8fu, 0x4bf01f77u, 0x80000000u, 0x4bf01f77u, 0x80000000u, 0xf4c65508u, 0x774afa8fu, 0xf4c65508u, 0x774afa8fu, 0xfff65f7fu, 0xf74afa8fu, 0x4bf01f77u, 0x80000000u},
+    {0x0680d87fu, 0xe2aeba83u, 0x0680d87fu, 0xe2aeba83u, 0x0cae8376u, 0x21027ce8u, 0x0cae8376u, 0x21027ce8u, 0x0680d87fu, 0xe2aeba83u, 0x0680d87fu, 0xe2aeba83u, 0x0eaedb7fu, 0xe3aefeebu, 0x0eaedb7fu, 0xe3aefeebu, 0x0cae8376u, 0x21027ce8u, 0x0cae8376u, 0x21027ce8u, 0x0680d87fu, 0xe2aeba83u, 0x0680d87fu, 0xe2aeba83u, 0x0eaedb7fu, 0xe3aefeebu, 0x0cae8376u, 0x21027ce8u},
+    {0x12345678u, 0x24483dacu, 0x12345678u, 0x24483dacu, 0x12345678u, 0x24483dacu, 0x12345678u, 0x24483dacu, 0x12345678u, 0x24483dacu, 0x12345678u, 0x24483dacu, 0x3634dff9u, 0xfe7abdadu, 0x3634dff9u, 0xfe7abdadu, 0x343089c1u, 0xfe3a9921u, 0x343089c1u, 0xfe3a9921u, 0x3634dff9u, 0xfe7abdadu, 0x343089c1u, 0xfe3a9921u, 0x3634dff9u, 0xfe7abdadu, 0x343089c1u, 0xfe3a9921u},
+    {0x9dd441ebu, 0x7fffffffu, 0x9dd441ebu, 0x7fffffffu, 0x9dd441ebu, 0x7fffffffu, 0x9dd441ebu, 0x7fffffffu, 0x9dd441ebu, 0x7fffffffu, 0x9dd441ebu, 0x7fffffffu, 0xbff54febu, 0x7fffffffu, 0xbff54febu, 0x7fffffffu, 0x3f250fabu, 0x5a25508fu, 0x3f250fabu, 0x5a25508fu, 0xbff54febu, 0x7fffffffu, 0x3f250fabu, 0x5a25508fu, 0xbff54febu, 0x7fffffffu, 0x3f250fabu, 0x5a25508fu},
+    {0x80000000u, 0x940c48d1u, 0x80000000u, 0x940c48d1u, 0x00000001u, 0x12345678u, 0x00000001u, 0x12345678u, 0x80000000u, 0x940c48d1u, 0x80000000u, 0x940c48d1u, 0x80000001u, 0x963c5ef9u, 0x80000001u, 0x963c5ef9u, 0x00000001u, 0x12345678u, 0x00000001u, 0x12345678u, 0x80000000u, 0x940c48d1u, 0x80000000u, 0x940c48d1u, 0x80000001u, 0x963c5ef9u, 0x00000001u, 0x12345678u},
+    {0x06a06e39u, 0xdca4271cu, 0x06a06e39u, 0xdca4271cu, 0x89abcdefu, 0x80000000u, 0x89abcdefu, 0x80000000u, 0x06a06e39u, 0xdca4271cu, 0x06a06e39u, 0xdca4271cu, 0x8fabefffu, 0xdca4271cu, 0x8fabefffu, 0xdca4271cu, 0x89abcdefu, 0x80000000u, 0x89abcdefu, 0x80000000u, 0x06a06e39u, 0xdca4271cu, 0x06a06e39u, 0xdca4271cu, 0x8fabefffu, 0xdca4271cu, 0x89abcdefu, 0x80000000u},
+    {0x00ff00ffu, 0xdadef6e0u, 0x00ff00ffu, 0xdadef6e0u, 0x00ff00ffu, 0xdadef6e0u, 0x00ff00ffu, 0xdadef6e0u, 0x00ff00ffu, 0xdadef6e0u, 0x00ff00ffu, 0xdadef6e0u, 0xdfffb7ffu, 0xffffffffu, 0xdfffb7ffu, 0xffffffffu, 0xdfd0b7a2u, 0xffffffffu, 0xdfd0b7a2u, 0xffffffffu, 0xdfffb7ffu, 0xffffffffu, 0xdfd0b7a2u, 0xffffffffu, 0xdfffb7ffu, 0xffffffffu, 0xdfd0b7a2u, 0xffffffffu},
+    {0x436d4bd7u, 0xb4373875u, 0x436d4bd7u, 0xb4373875u, 0xc71d5b33u, 0x63b433c8u, 0xc71d5b33u, 0x63b433c8u, 0x436d4bd7u, 0xb4373875u, 0x436d4bd7u, 0xb4373875u, 0xc77d5bf7u, 0xf7b73bfdu, 0xc77d5bf7u, 0xf7b73bfdu, 0xc71d5b33u, 0x63b433c8u, 0xc71d5b33u, 0x63b433c8u, 0x436d4bd7u, 0xb4373875u, 0x436d4bd7u, 0xb4373875u, 0xc77d5bf7u, 0xf7b73bfdu, 0xc71d5b33u, 0x63b433c8u},
+    {0xe1a9611du, 0xdd70b68bu, 0xe1a9611du, 0xdd70b68bu, 0xc3e41dfbu, 0x80000000u, 0xc3e41dfbu, 0x80000000u, 0xe1a9611du, 0xdd70b68bu, 0xe1a9611du, 0xdd70b68bu, 0xe3ed7dffu, 0xdd70b68bu, 0xe3ed7dffu, 0xdd70b68bu, 0xc3e41dfbu, 0x80000000u, 0xc3e41dfbu, 0x80000000u, 0xe1a9611du, 0xdd70b68bu, 0xe1a9611du, 0xdd70b68bu, 0xe3ed7dffu, 0xdd70b68bu, 0xc3e41dfbu, 0x80000000u},
+    {0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x00ff00ffu, 0x2aff5fffu, 0xf2ffccffu, 0x2aff5fffu, 0xf2ffccffu, 0x2a2b5f67u, 0xf2d9cc0du, 0x2a2b5f67u, 0xf2d9cc0du, 0x2aff5fffu, 0xf2ffccffu, 0x2a2b5f67u, 0xf2d9cc0du, 0x2aff5fffu, 0xf2ffccffu, 0x2a2b5f67u, 0xf2d9cc0du},
+    {0x89abcdefu, 0x38573260u, 0x89abcdefu, 0x38573260u, 0x89abcdefu, 0x38573260u, 0x89abcdefu, 0x38573260u, 0x89abcdefu, 0x38573260u, 0x89abcdefu, 0x38573260u, 0xdbbbcfffu, 0x79f77772u, 0xdbbbcfffu, 0x79f77772u, 0x5b120ed1u, 0x49e07752u, 0x5b120ed1u, 0x49e07752u, 0xdbbbcfffu, 0x79f77772u, 0x5b120ed1u, 0x49e07752u, 0xdbbbcfffu, 0x79f77772u, 0x5b120ed1u, 0x49e07752u},
+    {0xf60106b7u, 0xd61d1405u, 0xf60106b7u, 0xd61d1405u, 0xf60106b7u, 0xd61d1405u, 0xf60106b7u, 0xd61d1405u, 0xf60106b7u, 0xd61d1405u, 0xf60106b7u, 0xd61d1405u, 0xf78dceffu, 0xfeff3707u, 0xf78dceffu, 0xfeff3707u, 0xc58ccccdu, 0xbce22306u, 0xc58ccccdu, 0xbce22306u, 0xf78dceffu, 0xfeff3707u, 0xc58ccccdu, 0xbce22306u, 0xf78dceffu, 0xfeff3707u, 0xc58ccccdu, 0xbce22306u},
+    {0x7fffffffu, 0x80000000u, 0x7fffffffu, 0x80000000u, 0x15452b7bu, 0xb8054218u, 0x15452b7bu, 0xb8054218u, 0x7fffffffu, 0x80000000u, 0x7fffffffu, 0x80000000u, 0x7fffffffu, 0xb8054218u, 0x7fffffffu, 0xb8054218u, 0x15452b7bu, 0xb8054218u, 0x15452b7bu, 0xb8054218u, 0x7fffffffu, 0x80000000u, 0x7fffffffu, 0x80000000u, 0x7fffffffu, 0xb8054218u, 0x15452b7bu, 0xb8054218u},
+    {0x12345678u, 0x00000001u, 0x12345678u, 0x00000001u, 0x12345678u, 0x00000001u, 0x12345678u, 0x00000001u, 0x12345678u, 0x00000001u, 0x12345678u, 0x00000001u, 0xb73c7e7cu, 0xffffffffu, 0xb73c7e7cu, 0xffffffffu, 0xa7387a4cu, 0xffffffffu, 0xa7387a4cu, 0xffffffffu, 0xb73c7e7cu, 0xffffffffu, 0xa7387a4cu, 0xffffffffu, 0xb73c7e7cu, 0xffffffffu, 0xa7387a4cu, 0xffffffffu},
+    {0x00000000u, 0x90322d5bu, 0x00000000u, 0x90322d5bu, 0x00000000u, 0x90322d5bu, 0x00000000u, 0x90322d5bu, 0x00000000u, 0x90322d5bu, 0x00000000u, 0x90322d5bu, 0x7ee058dau, 0x99bbedffu, 0x7ee058dau, 0x99bbedffu, 0x7ee058dau, 0x89abcdefu, 0x7ee058dau, 0x89abcdefu, 0x7ee058dau, 0x99bbedffu, 0x7ee058dau, 0x89abcdefu, 0x7ee058dau, 0x99bbedffu, 0x7ee058dau, 0x89abcdefu},
+    {0x7fffffffu, 0x93b8f2e0u, 0x7fffffffu, 0x93b8f2e0u, 0x7fffffffu, 0x93b8f2e0u, 0x7fffffffu, 0x93b8f2e0u, 0x7fffffffu, 0x93b8f2e0u, 0x7fffffffu, 0x93b8f2e0u, 0xffffffffu, 0x97bbf7f5u, 0xffffffffu, 0x97bbf7f5u, 0x8245e09du, 0x168b95f5u, 0x8245e09du, 0x168b95f5u, 0xffffffffu, 0x97bbf7f5u, 0x8245e09du, 0x168b95f5u, 0xffffffffu, 0x97bbf7f5u, 0x8245e09du, 0x168b95f5u},
+    {0x80000000u, 0x4ec04618u, 0x80000000u, 0x4ec04618u, 0x80000000u, 0x4ec04618u, 0x80000000u, 0x4ec04618u, 0x80000000u, 0x4ec04618u, 0x80000000u, 0x4ec04618u, 0xc4b6a55du, 0xcfebcfffu, 0xc4b6a55du, 0xcfebcfffu, 0x44b6a55du, 0x89abcdefu, 0x44b6a55du, 0x89abcdefu, 0xc4b6a55du, 0xcfebcfffu, 0x44b6a55du, 0x89abcdefu, 0xc4b6a55du, 0xcfebcfffu, 0x44b6a55du, 0x89abcdefu},
+    {0x80a3acc4u, 0x7fffffffu, 0x80a3acc4u, 0x7fffffffu, 0x80a3acc4u, 0x7fffffffu, 0x80a3acc4u, 0x7fffffffu, 0x80a3acc4u, 0x7fffffffu, 0x80a3acc4u, 0x7fffffffu, 0x9fa3efedu, 0xffffffffu, 0x9fa3efedu, 0xffffffffu, 0x1f016769u, 0xac7c83b9u, 0x1f016769u, 0xac7c83b9u, 0x9fa3efedu, 0xffffffffu, 0x1f016769u, 0xac7c83b9u, 0x9fa3efedu, 0xffffffffu, 0x1f016769u, 0xac7c83b9u},
+    {0x725f6ff6u, 0x00000000u, 0x725f6ff6u, 0x00000000u, 0x725f6ff6u, 0x00000000u, 0x725f6ff6u, 0x00000000u, 0x725f6ff6u, 0x00000000u, 0x725f6ff6u, 0x00000000u, 0x775f6ff7u, 0x00ff00ffu, 0x775f6ff7u, 0x00ff00ffu, 0x45040843u, 0x00ff00ffu, 0x45040843u, 0x00ff00ffu, 0x775f6ff7u, 0x00ff00ffu, 0x45040843u, 0x00ff00ffu, 0x775f6ff7u, 0x00ff00ffu, 0x45040843u, 0x00ff00ffu},
+    {0x50d14398u, 0x7f767333u, 0x50d14398u, 0x7f767333u, 0x50d14398u, 0x7f767333u, 0x50d14398u, 0x7f767333u, 0x50d14398u, 0x7f767333u, 0x50d14398u, 0x7f767333u, 0x73f9ebdcu, 0x7f767333u, 0x73f9ebdcu, 0x7f767333u, 0x2369eadcu, 0x00000001u, 0x2369eadcu, 0x00000001u, 0x73f9ebdcu, 0x7f767333u, 0x2369eadcu, 0x00000001u, 0x73f9ebdcu, 0x7f767333u, 0x2369eadcu, 0x00000001u},
+    {0xac8c599bu, 0x7fffffffu, 0xac8c599bu, 0x7fffffffu, 0xac8c599bu, 0x7fffffffu, 0xac8c599bu, 0x7fffffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfcff79dfu, 0x7fffffffu, 0xfcff79dfu, 0x7fffffffu, 0xd8f36844u, 0x4761f31au, 0xd8f36844u, 0x4761f31au, 0xfcff79dfu, 0x7fffffffu, 0xd8f36844u, 0x4761f31au, 0xac8c599bu, 0x7fffffffu, 0xac8c599bu, 0x7fffffffu},
+    {0xd078019fu, 0x60ac3d38u, 0xd078019fu, 0x60ac3d38u, 0x7fffffffu, 0x20b5d1d4u, 0x7fffffffu, 0x20b5d1d4u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0x60bdfdfcu, 0xffffffffu, 0x60bdfdfcu, 0x7fffffffu, 0x20b5d1d4u, 0x7fffffffu, 0x20b5d1d4u, 0xd078019fu, 0x60ac3d38u, 0xd078019fu, 0x60ac3d38u, 0xd078019fu, 0x60ac3d38u, 0xd078019fu, 0x60ac3d38u},
+    {0x2925feb3u, 0x00000001u, 0x2925feb3u, 0x00000001u, 0x7fffffffu, 0xb16614f4u, 0x7fffffffu, 0xb16614f4u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fffffffu, 0xb16614f5u, 0x7fffffffu, 0xb16614f5u, 0x7fffffffu, 0xb16614f4u, 0x7fffffffu, 0xb16614f4u, 0x2925feb3u, 0x00000001u, 0x2925feb3u, 0x00000001u, 0x2925feb3u, 0x00000001u, 0x2925feb3u, 0x00000001u},
+    {0x86d5360fu, 0xfaf40417u, 0x86d5360fu, 0xfaf40417u, 0x86d5360fu, 0xfaf40417u, 0x86d5360fu, 0xfaf40417u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xd6d77e5fu, 0xfbf71737u, 0xd6d77e5fu, 0xfbf71737u, 0xd4424852u, 0x89631325u, 0xd4424852u, 0x89631325u, 0xd6d77e5fu, 0xfbf71737u, 0xd4424852u, 0x89631325u, 0x86d5360fu, 0xfaf40417u, 0x86d5360fu, 0xfaf40417u},
+    {0xffffffffu, 0xde274210u, 0xffffffffu, 0xde274210u, 0xe96e12c9u, 0x65305d1cu, 0xe96e12c9u, 0x65305d1cu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0xff375f1cu, 0xffffffffu, 0xff375f1cu, 0xe96e12c9u, 0x65305d1cu, 0xe96e12c9u, 0x65305d1cu, 0xffffffffu, 0xde274210u, 0xffffffffu, 0xde274210u, 0xffffffffu, 0xde274210u, 0xffffffffu, 0xde274210u},
+    {0x1043c4f3u, 0x8a860eeau, 0x1043c4f3u, 0x8a860eeau, 0xe9579523u, 0x7563acb0u, 0xe9579523u, 0x7563acb0u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xf957d5f3u, 0xffe7aefau, 0xf957d5f3u, 0xffe7aefau, 0xe9579523u, 0x7563acb0u, 0xe9579523u, 0x7563acb0u, 0x1043c4f3u, 0x8a860eeau, 0x1043c4f3u, 0x8a860eeau, 0x1043c4f3u, 0x8a860eeau, 0x1043c4f3u, 0x8a860eeau},
+    {0x310b55c7u, 0x12345678u, 0x310b55c7u, 0x12345678u, 0x310b55c7u, 0x12345678u, 0x310b55c7u, 0x12345678u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x333f57ffu, 0x963cd77fu, 0x333f57ffu, 0x963cd77fu, 0x12345678u, 0x96188707u, 0x12345678u, 0x96188707u, 0x333f57ffu, 0x963cd77fu, 0x12345678u, 0x96188707u, 0x310b55c7u, 0x12345678u, 0x310b55c7u, 0x12345678u},
+    {0x80000000u, 0xbcabb1bau, 0x80000000u, 0xbcabb1bau, 0x80000000u, 0xbcabb1bau, 0x80000000u, 0xbcabb1bau, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x92345678u, 0xffffffffu, 0x92345678u, 0xffffffffu, 0x12345678u, 0xffffffffu, 0x12345678u, 0xffffffffu, 0x92345678u, 0xffffffffu, 0x12345678u, 0xffffffffu, 0x80000000u, 0xbcabb1bau, 0x80000000u, 0xbcabb1bau},
+    {0x72fdad03u, 0x00000001u, 0x72fdad03u, 0x00000001u, 0x72fdad03u, 0x00000001u, 0x72fdad03u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x72ffbf03u, 0x6c60da43u, 0x72ffbf03u, 0x6c60da43u, 0x028f3b02u, 0x6c60da42u, 0x028f3b02u, 0x6c60da42u, 0x72ffbf03u, 0x6c60da43u, 0x028f3b02u, 0x6c60da42u, 0x72fdad03u, 0x00000001u, 0x72fdad03u, 0x00000001u},
+    {0xe473cdc9u, 0x75eb16c7u, 0xe473cdc9u, 0x75eb16c7u, 0x12345678u, 0xe0d59f3cu, 0x12345678u, 0xe0d59f3cu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xf677dff9u, 0xf5ff9fffu, 0xf677dff9u, 0xf5ff9fffu, 0x12345678u, 0xe0d59f3cu, 0x12345678u, 0xe0d59f3cu, 0xe473cdc9u, 0x75eb16c7u, 0xe473cdc9u, 0x75eb16c7u, 0xe473cdc9u, 0x75eb16c7u, 0xe473cdc9u, 0x75eb16c7u},
+    {0x89abcdefu, 0x74102553u, 0x89abcdefu, 0x74102553u, 0x89abcdefu, 0x74102553u, 0x89abcdefu, 0x74102553u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x9dbbedffu, 0xf536affbu, 0x9dbbedffu, 0xf536affbu, 0x1cb3e931u, 0xe536aae9u, 0x1cb3e931u, 0xe536aae9u, 0x9dbbedffu, 0xf536affbu, 0x1cb3e931u, 0xe536aae9u, 0x89abcdefu, 0x74102553u, 0x89abcdefu, 0x74102553u},
+    {0x4d126f38u, 0x7d42a12au, 0x4d126f38u, 0x7d42a12au, 0x4d126f38u, 0x7d42a12au, 0x4d126f38u, 0x7d42a12au, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x5d7b6ffdu, 0xffc3b3ffu, 0x5d7b6ffdu, 0xffc3b3ffu, 0x50692ce5u, 0xa68132ddu, 0x50692ce5u, 0xa68132ddu, 0x5d7b6ffdu, 0xffc3b3ffu, 0x50692ce5u, 0xa68132ddu, 0x4d126f38u, 0x7d42a12au, 0x4d126f38u, 0x7d42a12au},
+    {0x744e3f0eu, 0x7fffffffu, 0x744e3f0eu, 0x7fffffffu, 0xde298785u, 0x80000000u, 0xde298785u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfe6fbf8fu, 0xffffffffu, 0xfe6fbf8fu, 0xffffffffu, 0xde298785u, 0x80000000u, 0xde298785u, 0x80000000u, 0x744e3f0eu, 0x7fffffffu, 0x744e3f0eu, 0x7fffffffu, 0x744e3f0eu, 0x7fffffffu, 0x744e3f0eu, 0x7fffffffu},
+    {0xad554ba1u, 0x7c5153d0u, 0xad554ba1u, 0x7c5153d0u, 0xad554ba1u, 0x7c5153d0u, 0xad554ba1u, 0x7c5153d0u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xeffdefefu, 0xfcfbfbffu, 0xeffdefefu, 0xfcfbfbffu, 0xeaf8edcfu, 0xc8aab92fu, 0xeaf8edcfu, 0xc8aab92fu, 0xeffdefefu, 0xfcfbfbffu, 0xeaf8edcfu, 0xc8aab92fu, 0xad554ba1u, 0x7c5153d0u, 0xad554ba1u, 0x7c5153d0u},
+    {0xf8f008c5u, 0x0f119987u, 0xf8f008c5u, 0x0f119987u, 0xf8f008c5u, 0x0f119987u, 0xf8f008c5u, 0x0f119987u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfbf39cfdu, 0x5fd5bf97u, 0xfbf39cfdu, 0x5fd5bf97u, 0x1bd39c7cu, 0x5dc52692u, 0x1bd39c7cu, 0x5dc52692u, 0xfbf39cfdu, 0x5fd5bf97u, 0x1bd39c7cu, 0x5dc52692u, 0xf8f008c5u, 0x0f119987u, 0xf8f008c5u, 0x0f119987u},
+    {0xd03b304du, 0x00ff00ffu, 0xd03b304du, 0x00ff00ffu, 0x89abcdefu, 0x991132c0u, 0x89abcdefu, 0x991132c0u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xd9bbfdefu, 0x99ff32ffu, 0xd9bbfdefu, 0x99ff32ffu, 0x89abcdefu, 0x991132c0u, 0x89abcdefu, 0x991132c0u, 0xd03b304du, 0x00ff00ffu, 0xd03b304du, 0x00ff00ffu, 0xd03b304du, 0x00ff00ffu, 0xd03b304du, 0x00ff00ffu},
+    {0xa92c8f72u, 0x07b9077du, 0xa92c8f72u, 0x07b9077du, 0xa92c8f72u, 0x07b9077du, 0xa92c8f72u, 0x07b9077du, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xef7ceff6u, 0xf7bdf7fdu, 0xef7ceff6u, 0xf7bdf7fdu, 0x66546794u, 0xf6a5f7c1u, 0x66546794u, 0xf6a5f7c1u, 0xef7ceff6u, 0xf7bdf7fdu, 0x66546794u, 0xf6a5f7c1u, 0xa92c8f72u, 0x07b9077du, 0xa92c8f72u, 0x07b9077du},
+    {0xb5691c18u, 0x5d6c710au, 0xb5691c18u, 0x5d6c710au, 0x7ae6b997u, 0xbfe4abb0u, 0x7ae6b997u, 0xbfe4abb0u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffefbd9fu, 0xffecfbbau, 0xffefbd9fu, 0xffecfbbau, 0x7ae6b997u, 0xbfe4abb0u, 0x7ae6b997u, 0xbfe4abb0u, 0xb5691c18u, 0x5d6c710au, 0xb5691c18u, 0x5d6c710au, 0xb5691c18u, 0x5d6c710au, 0xb5691c18u, 0x5d6c710au},
+    {0xc9570a1cu, 0xefa96b6eu, 0xc9570a1cu, 0xefa96b6eu, 0xc9570a1cu, 0xefa96b6eu, 0xc9570a1cu, 0xefa96b6eu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xc9570a1cu, 0xefbbffffu, 0xc9570a1cu, 0xefbbffffu, 0x00000000u, 0xcd12bfb3u, 0x00000000u, 0xcd12bfb3u, 0xc9570a1cu, 0xefbbffffu, 0x00000000u, 0xcd12bfb3u, 0xc9570a1cu, 0xefa96b6eu, 0xc9570a1cu, 0xefa96b6eu},
+    {0x00000000u, 0x37eadcd2u, 0x00000000u, 0x37eadcd2u, 0x00000000u, 0x37eadcd2u, 0x00000000u, 0x37eadcd2u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3a6d60aau, 0xbfebddffu, 0x3a6d60aau, 0xbfebddffu, 0x3a6d60aau, 0x89abcdefu, 0x3a6d60aau, 0x89abcdefu, 0x3a6d60aau, 0xbfebddffu, 0x3a6d60aau, 0x89abcdefu, 0x00000000u, 0x37eadcd2u, 0x00000000u, 0x37eadcd2u},
+    {0x114a2d32u, 0x00000000u, 0x114a2d32u, 0x00000000u, 0x89abcdefu, 0x12345678u, 0x89abcdefu, 0x12345678u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x99ebedffu, 0x12345678u, 0x99ebedffu, 0x12345678u, 0x89abcdefu, 0x12345678u, 0x89abcdefu, 0x12345678u, 0x114a2d32u, 0x00000000u, 0x114a2d32u, 0x00000000u, 0x114a2d32u, 0x00000000u, 0x114a2d32u, 0x00000000u},
+    {0x14090b01u, 0xffffffffu, 0x14090b01u, 0xffffffffu, 0x14090b01u, 0xffffffffu, 0x14090b01u, 0xffffffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1eb98bedu, 0xffffffffu, 0x1eb98bedu, 0xffffffffu, 0x0eb083edu, 0x1a0caeefu, 0x0eb083edu, 0x1a0caeefu, 0x1eb98bedu, 0xffffffffu, 0x0eb083edu, 0x1a0caeefu, 0x14090b01u, 0xffffffffu, 0x14090b01u, 0xffffffffu},
+    {0x12345678u, 0x4521d7efu, 0x12345678u, 0x4521d7efu, 0x00000001u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x12345679u, 0xc521d7efu, 0x12345679u, 0xc521d7efu, 0x00000001u, 0x80000000u, 0x00000001u, 0x80000000u, 0x12345678u, 0x4521d7efu, 0x12345678u, 0x4521d7efu, 0x12345678u, 0x4521d7efu, 0x12345678u, 0x4521d7efu},
+    {0xefa94a8eu, 0x561bfb20u, 0xefa94a8eu, 0x561bfb20u, 0xefa94a8eu, 0x561bfb20u, 0xefa94a8eu, 0x561bfb20u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffb9cafeu, 0x7fbbfb3du, 0xffb9cafeu, 0x7fbbfb3du, 0x19b0c2f0u, 0x69b1111du, 0x19b0c2f0u, 0x69b1111du, 0xffb9cafeu, 0x7fbbfb3du, 0x19b0c2f0u, 0x69b1111du, 0xefa94a8eu, 0x561bfb20u, 0xefa94a8eu, 0x561bfb20u},
+    {0xbcf7629fu, 0xaf1de605u, 0xbcf7629fu, 0xaf1de605u, 0x41622456u, 0x00000000u, 0x41622456u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfdf766dfu, 0xaf1de605u, 0xfdf766dfu, 0xaf1de605u, 0x41622456u, 0x00000000u, 0x41622456u, 0x00000000u, 0xbcf7629fu, 0xaf1de605u, 0xbcf7629fu, 0xaf1de605u, 0xbcf7629fu, 0xaf1de605u, 0xbcf7629fu, 0xaf1de605u},
+    {0xd47a46a9u, 0x12da9645u, 0xd47a46a9u, 0x12da9645u, 0x7ded0e8du, 0x00000000u, 0x7ded0e8du, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfdff4eadu, 0x12da9645u, 0xfdff4eadu, 0x12da9645u, 0x7ded0e8du, 0x00000000u, 0x7ded0e8du, 0x00000000u, 0xd47a46a9u, 0x12da9645u, 0xd47a46a9u, 0x12da9645u, 0xd47a46a9u, 0x12da9645u, 0xd47a46a9u, 0x12da9645u},
+    {0x697d50dbu, 0x89abcdefu, 0x697d50dbu, 0x89abcdefu, 0xdeacacaau, 0xcf93dd30u, 0xdeacacaau, 0xcf93dd30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfffdfcfbu, 0xcfbbddffu, 0xfffdfcfbu, 0xcfbbddffu, 0xdeacacaau, 0xcf93dd30u, 0xdeacacaau, 0xcf93dd30u, 0x697d50dbu, 0x89abcdefu, 0x697d50dbu, 0x89abcdefu, 0x697d50dbu, 0x89abcdefu, 0x697d50dbu, 0x89abcdefu},
+    {0x00ff00ffu, 0xb6a48819u, 0x00ff00ffu, 0xb6a48819u, 0x1f752ee6u, 0x80000000u, 0x1f752ee6u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1fff2effu, 0xb6a48819u, 0x1fff2effu, 0xb6a48819u, 0x1f752ee6u, 0x80000000u, 0x1f752ee6u, 0x80000000u, 0x00ff00ffu, 0xb6a48819u, 0x00ff00ffu, 0xb6a48819u, 0x00ff00ffu, 0xb6a48819u, 0x00ff00ffu, 0xb6a48819u},
+    {0x12345678u, 0xbb36bb87u, 0x12345678u, 0xbb36bb87u, 0x12345678u, 0xbb36bb87u, 0x12345678u, 0xbb36bb87u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1abc7e7bu, 0xffb6ffd7u, 0x1abc7e7bu, 0xffb6ffd7u, 0x0888385bu, 0x5cb6c6d7u, 0x0888385bu, 0x5cb6c6d7u, 0x1abc7e7bu, 0xffb6ffd7u, 0x0888385bu, 0x5cb6c6d7u, 0x12345678u, 0xbb36bb87u, 0x12345678u, 0xbb36bb87u},
+    {0xd24df64du, 0x9595f0c7u, 0xd24df64du, 0x9595f0c7u, 0xd24df64du, 0x9595f0c7u, 0xd24df64du, 0x9595f0c7u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xd24df64du, 0xffffffffu, 0xd24df64du, 0xffffffffu, 0x00000000u, 0x7fffffffu, 0x00000000u, 0x7fffffffu, 0xd24df64du, 0xffffffffu, 0x00000000u, 0x7fffffffu, 0xd24df64du, 0x9595f0c7u, 0xd24df64du, 0x9595f0c7u},
+    {0xffffffffu, 0x00ff00ffu, 0xffffffffu, 0x00ff00ffu, 0xffffffffu, 0x00ff00ffu, 0xffffffffu, 0x00ff00ffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0xfefffeffu, 0xffffffffu, 0xfefffeffu, 0x00000001u, 0xfe7ffe21u, 0x00000001u, 0xfe7ffe21u, 0xffffffffu, 0xfefffeffu, 0x00000001u, 0xfe7ffe21u, 0xffffffffu, 0x00ff00ffu, 0xffffffffu, 0x00ff00ffu},
+    {0xdd6537d2u, 0x6c2ab542u, 0xdd6537d2u, 0x6c2ab542u, 0xdd6537d2u, 0x6c2ab542u, 0xdd6537d2u, 0x6c2ab542u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0xfe7fffcau, 0xffffffffu, 0xfe7fffcau, 0xffffffffu, 0xde5ffb8au, 0xffffffffu, 0xde5ffb8au, 0xffffffffu, 0xfe7fffcau, 0xffffffffu, 0xde5ffb8au, 0xdd6537d2u, 0x6c2ab542u, 0xdd6537d2u, 0x6c2ab542u}
+};
+
+constexpr const char* Names[Checked] = {
+    "buffer_atomic_or_x2 returned lo", "buffer_atomic_or_x2 returned hi", "buffer_atomic_swap_x2 returned lo", "buffer_atomic_swap_x2 returned hi",
+    "buffer_atomic_or_x2 returned lo (exec-masked)", "buffer_atomic_or_x2 returned hi (exec-masked)", "buffer_atomic_swap_x2 returned lo (exec-masked)", "buffer_atomic_swap_x2 returned hi (exec-masked)",
+    "buffer_atomic_or_x2 returned lo (V# end)", "buffer_atomic_or_x2 returned hi (V# end)", "buffer_atomic_swap_x2 returned lo (V# end)", "buffer_atomic_swap_x2 returned hi (V# end)",
+    "or_x2 memory lo", "or_x2 memory hi", "or_x2 (no return) memory lo", "or_x2 (no return) memory hi",
+    "swap_x2 memory lo", "swap_x2 memory hi", "swap_x2 (no return) memory lo", "swap_x2 (no return) memory hi",
+    "or_x2 (exec-masked) memory lo", "or_x2 (exec-masked) memory hi", "swap_x2 (exec-masked) memory lo", "swap_x2 (exec-masked) memory hi",
+    "or_x2 (V# end) memory lo", "or_x2 (V# end) memory hi", "swap_x2 (V# end) memory lo", "swap_x2 (V# end) memory hi",
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target) {
+    Input.fill(0u);
+    for (std::uint32_t tid = 0; tid < waveSize; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), waveSize * Inputs * 4u);
+    const auto output = BufferDescriptor(Output.data(), waveSize * Results * 4u);
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{waveSize, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        target,
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+template <std::size_t Lanes>
+void Check(const std::uint32_t (&expected)[Lanes][Checked], const char* const (&names)[Checked], const char* run) {
+    for (std::uint32_t tid = 0; tid < Lanes; ++tid) {
+        for (std::uint32_t index = 0; index < Checked; ++index) {
+            const auto actual = Output[tid * Results + index];
+            Require(actual == expected[tid][index], std::string("buffer 64-bit or/swap atomics ") + run + ": lane " + std::to_string(tid) + " " + names[index] + " is " + Hex(actual) + ", expected " + Hex(expected[tid][index]));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device, Wave32Code, 32, device->Target());
+        Check(Expected32, Names, "wave32");
+        Run(*device, Wave64Code, 64, device->Target());
+        Check(Expected64, Names, "wave64");
+        Run(*device, Wave64Code, 64, device->ComputeTarget(32));
+        Check(Expected64, Names, "wave64 split");
+        std::puts("buffer 64-bit or/swap atomics tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp

@@ -0,0 +1,304 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <iostream>
+#include <limits>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Record = 16;
+constexpr std::uint32_t Forms = 4;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::uint64_t NodeBytes = 64;
+constexpr std::uint32_t LastNode = 6;
+constexpr std::uint32_t Invalid = 0xffffffffu;
+constexpr std::uint32_t BoxSort = 0x80000000u;
+constexpr std::uint32_t TriangleId = 0x00abc123u;
+constexpr std::array<std::uint32_t, 4> Box32Children{0x1000u, 0x1001u, 0x1002u, 0x1003u};
+constexpr std::array<std::uint32_t, 4> Box16Children{0x2000u, 0x2001u, 0x2002u, 0x2003u};
+constexpr std::array<std::uint32_t, 4> PastEndChildren{0x3000u, 0x3001u, 0x3002u, 0x3003u};
+constexpr std::array<const char*, Forms> FormNames{"image_bvh64_intersect_ray", "image_bvh_intersect_ray", "image_bvh64_intersect_ray a16", "image_bvh_intersect_ray a16"};
+
+alignas(256) std::array<std::uint32_t, Threads * Record> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Record> Output{};
+
+constexpr std::array<std::uint32_t, 82> MakeCode() {
+    std::array<std::uint32_t, 82> code{};
+    std::size_t at = 0;
+    code[at++] = 0x34020086u;
+    for (std::uint32_t dword = 0; dword < Record; ++dword) {
+        code[at++] = 0xe0301000u | (dword * 4u);
+        code[at++] = 0x80000001u | ((4u + dword) << 8u);
+    }
+    code[at++] = 0xbf8c3f70u;
+    code[at++] = 0xf19c9f01u;
+    code[at++] = 0x00031405u;
+    code[at++] = 0xf1989f07u;
+    code[at++] = 0x00021804u;
+    code[at++] = 0x0a090807u;
+    code[at++] = 0x0e0d0c0bu;
+    code[at++] = 0x0000100fu;
+    code[at++] = 0xf19c9f05u;
+    code[at++] = 0x40031c05u;
+    code[at++] = 0x09080706u;
+    code[at++] = 0x1312110au;
+    code[at++] = 0xf1989f05u;
+    code[at++] = 0x40022004u;
+    code[at++] = 0x0a090807u;
+    code[at++] = 0x00131211u;
+    for (std::uint32_t dword = 0; dword < Record; ++dword) {
+        code[at++] = 0xe0701000u | (dword * 4u);
+        code[at++] = 0x80010001u | ((20u + dword) << 8u);
+    }
+    code[at++] = 0xbf810000u;
+    return code;
+}
+
+alignas(256) constexpr std::array<std::uint32_t, 82> Code = MakeCode();
+
+struct Ray {
+    float extent;
+    std::array<float, 3> origin;
+    std::array<float, 3> direction;
+    std::array<float, 3> inverse;
+};
+
+struct Case {
+    std::uint32_t node;
+    Ray ray;
+    std::array<std::uint32_t, 4> expected;
+};
+
+constexpr float Infinity = std::numeric_limits<float>::infinity();
+constexpr std::array<float, 3> Diagonal{0.25f, 0.5f, 1.0f};
+constexpr std::array<float, 3> DiagonalInverse{4.0f, 2.0f, 1.0f};
+constexpr Ray Long{100.0f, {0.0f, 0.0f, 0.0f}, Diagonal, DiagonalInverse};
+constexpr Ray Short{2.0f, {0.0f, 0.0f, 0.0f}, Diagonal, DiagonalInverse};
+constexpr Ray Forward{100.0f, {0.25f, 0.25f, 0.0f}, {0.0f, 0.0f, 1.0f}, {Infinity, Infinity, 1.0f}};
+
+constexpr std::uint32_t Pointer(std::uint32_t node, std::uint32_t type) {
+    return (node << 3u) | type;
+}
+
+constexpr std::array<Case, 8> Cases{{
+    {Pointer(2, 5), Long, {Box32Children[2], Box32Children[0], Invalid, Invalid}},
+    {Pointer(4, 4), Long, {Box16Children[2], Box16Children[0], Invalid, Invalid}},
+    {Pointer(1, 0), Forward, {0xc0a00000u, 0xbf800000u, TriangleId, 1u}},
+    {Pointer(1, 1), Forward, {0x7f800000u, 0x3f800000u, TriangleId, 0u}},
+    {Pointer(1, 7), Long, {Invalid, Invalid, Invalid, Invalid}},
+    {Pointer(6, 5), Long, {Invalid, Invalid, Invalid, Invalid}},
+    {Pointer(7, 0), Forward, {Invalid, Invalid, Invalid, Invalid}},
+    {Pointer(2, 5), Short, {Box32Children[2], Invalid, Invalid, Invalid}},
+}};
+
+constexpr std::array<std::array<float, 6>, 4> Boxes{{
+    {1.0f, 2.0f, 4.0f, 2.0f, 4.0f, 8.0f},
+    {10.0f, 0.0f, 0.0f, 11.0f, 1.0f, 1.0f},
+    {0.25f, 0.5f, 1.0f, 0.5f, 1.0f, 2.0f},
+    {-2.0f, -4.0f, -8.0f, -1.0f, -2.0f, -4.0f},
+}};
+
+constexpr std::array<float, 12> TriangleVertices{0.0f, 0.0f, 5.0f, 1.0f, 0.0f, 5.0f, 0.0f, 1.0f, 5.0f, 1.0f, 1.0f, 5.0f};
+
+std::uint32_t Bits(float value) {
+    return std::bit_cast<std::uint32_t>(value);
+}
+
+std::uint32_t Half(float value) {
+    const auto bits = Bits(value);
+    const auto sign = (bits >> 16u) & 0x8000u;
+    const auto exponent = (bits >> 23u) & 0xffu;
+    const auto mantissa = bits & 0x7fffffu;
+    if (exponent == 0u && mantissa == 0u) return sign;
+    if (exponent == 0xffu && mantissa == 0u) return sign | 0x7c00u;
+    Require(exponent >= 113u && exponent <= 142u && (mantissa & 0x1fffu) == 0u, "bvh64 intersect ray: a test value is not a float16");
+    return sign | ((exponent - 112u) << 10u) | (mantissa >> 13u);
+}
+
+class GuestBlock {
+public:
+    GuestBlock() {
+        constexpr std::uintptr_t hint = std::uintptr_t{1} << 40u;
+#ifdef _WIN32
+        for (std::uintptr_t attempt = 0; block == nullptr && attempt < 16u; ++attempt) {
+            block = static_cast<std::uint8_t*>(VirtualAlloc(reinterpret_cast<void*>(hint + (attempt << 32u)), BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+        if (block == nullptr) block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        void* mapped = mmap(reinterpret_cast<void*>(hint), BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        block = mapped == MAP_FAILED ? nullptr : static_cast<std::uint8_t*>(mapped);
+#endif
+        Require(block != nullptr, "bvh64 intersect ray: cannot allocate the guest block");
+        std::memset(block, 0, BlockBytes);
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, false);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        VirtualFree(block, 0, MEM_RELEASE);
+#else
+        munmap(block, BlockBytes);
+#endif
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint64_t Address() const { return static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(block)); }
+
+    void Write(std::uint32_t node, std::uint32_t dword, std::uint32_t value) {
+        std::memcpy(block + node * NodeBytes + dword * 4u, &value, sizeof(value));
+    }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+void WriteBox32(GuestBlock& guest, std::uint32_t node, const std::array<std::uint32_t, 4>& children) {
+    for (std::uint32_t child = 0; child < 4u; ++child) {
+        guest.Write(node, child, children[child]);
+        for (std::uint32_t index = 0; index < 6u; ++index) guest.Write(node, 4u + child * 6u + index, Bits(Boxes[child][index]));
+    }
+}
+
+void WriteBox16(GuestBlock& guest, std::uint32_t node, const std::array<std::uint32_t, 4>& children) {
+    for (std::uint32_t child = 0; child < 4u; ++child) {
+        guest.Write(node, child, children[child]);
+        for (std::uint32_t index = 0; index < 6u; index += 2u) {
+            const auto position = child * 6u + index;
+            guest.Write(node, 4u + position / 2u, Half(Boxes[child][index]) | (Half(Boxes[child][index + 1u]) << 16u));
+        }
+    }
+}
+
+void WriteTriangles(GuestBlock& guest, std::uint32_t node) {
+    for (std::uint32_t index = 0; index < TriangleVertices.size(); ++index) guest.Write(node, index, Bits(TriangleVertices[index]));
+    guest.Write(node, 15u, TriangleId);
+}
+
+void FillNodes(GuestBlock& guest) {
+    WriteTriangles(guest, 1u);
+    WriteBox32(guest, 2u, Box32Children);
+    WriteBox16(guest, 4u, Box16Children);
+    WriteBox32(guest, 6u, PastEndChildren);
+    WriteTriangles(guest, 7u);
+}
+
+void FillInput(std::uint64_t nodeBias) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const auto& test = Cases[tid % Cases.size()];
+        const auto& ray = test.ray;
+        const std::uint64_t node64 = nodeBias + test.node;
+        auto* words = &Input[tid * Record];
+        words[0] = test.node;
+        words[1] = static_cast<std::uint32_t>(node64);
+        words[2] = static_cast<std::uint32_t>(node64 >> 32u);
+        words[3] = Bits(ray.extent);
+        for (std::uint32_t axis = 0; axis < 3u; ++axis) {
+            words[4u + axis] = Bits(ray.origin[axis]);
+            words[7u + axis] = Bits(ray.direction[axis]);
+            words[10u + axis] = Bits(ray.inverse[axis]);
+        }
+        words[13] = Half(ray.direction[0]) | (Half(ray.direction[1]) << 16u);
+        words[14] = Half(ray.direction[2]) | (Half(ray.inverse[0]) << 16u);
+        words[15] = Half(ray.inverse[1]) | (Half(ray.inverse[2]) << 16u);
+    }
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::array<std::uint32_t, 4> BvhDescriptor(std::uint64_t base, std::uint64_t lastNode) {
+    return {static_cast<std::uint32_t>(base >> 8u), static_cast<std::uint32_t>((base >> 40u) & 0xffu) | BoxSort, static_cast<std::uint32_t>(lastNode), static_cast<std::uint32_t>((lastNode >> 32u) & 0x3ffu)};
+}
+
+void Run(AgcDriver::VulkanDevice& device, const GuestBlock& guest, bool addressInPointer) {
+    const auto address = guest.Address();
+    FillInput(addressInPointer ? address >> 3u : 0u);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(16, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    const auto narrow = BvhDescriptor(address, LastNode);
+    const auto wide = addressInPointer ? BvhDescriptor(0u, address / NodeBytes + LastNode) : narrow;
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    std::copy(narrow.begin(), narrow.end(), userData.begin() + 8);
+    std::copy(wide.begin(), wide.end(), userData.begin() + 12);
+    const std::span<const std::uint32_t> code(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check(bool addressInPointer) {
+    const std::string mode = addressInPointer ? "address in the node pointer" : "base in the descriptor";
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const auto& expected = Cases[tid % Cases.size()].expected;
+        for (std::uint32_t form = 0; form < Forms; ++form) {
+            for (std::uint32_t dword = 0; dword < 4u; ++dword) {
+                const auto actual = Output[tid * Record + form * 4u + dword];
+                Require(actual == expected[dword], std::string(FormNames[form]) + ", " + mode + ": thread " + std::to_string(tid) + " dword " + std::to_string(dword) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[dword]));
+            }
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        if (ShaderRecompiler::RayTracingStrict() || ShaderRecompiler::RayTracingMiss()) {
+            std::puts("skipped, APS5_RAYTRACING replaces the node test");
+            return VulkanTestSkipped;
+        }
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock guest;
+        Require((guest.Address() >> 35u) != 0u, "bvh64 intersect ray: the guest block is not above 32 GiB, the node pointer would fit 32 bits");
+        FillNodes(guest);
+        for (const bool addressInPointer : {true, false}) {
+            Run(*device, guest, addressInPointer);
+            Check(addressInPointer);
+        }
+        std::puts("bvh64 intersect ray tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DivisionResultModifiers.cpp

@@ -0,0 +1,206 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 36> Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x36100e81, 0x7d8a1080, 0xd56f800a,
+    0x041a0b04, 0xd55f800b, 0x041a0b04, 0xd55f820c, 0x241a0b04, 0xd56f800d, 0x441a0905, 0xd570800e,
+    0x04120d04, 0xd5700010, 0x0c120d06, 0xd5608012, 0x04120d04, 0xd5600014, 0x1c1a0906, 0xd5608016,
+    0x141a0906, 0xd5608018, 0x24120d04, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xe0781020,
+    0x80011203, 0xe0781030, 0x80011603, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[32][4] = {
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x32780000u, 0x7d2e1601u},
+    {0x00000000u, 0x00800000u, 0x3ec00000u, 0x9ffbf25eu},
+    {0x00000000u, 0x3e8e7000u, 0x8800759cu, 0x7e37e43cu},
+    {0x00800000u, 0x40d00000u, 0xca667aa1u, 0x0f7a9d5bu},
+    {0x32e94401u, 0x5f800000u, 0x3f800000u, 0xded37902u},
+    {0x00000000u, 0x80000000u, 0x3f800000u, 0x0406ef86u},
+    {0x00000000u, 0xfff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff00000u, 0xc2f8f359u, 0x81a56e1fu},
+    {0x3f800000u, 0x2232f1b4u, 0x7f7fffffu, 0x998ad4a5u},
+    {0x40000000u, 0xb6e038d3u, 0xf4787e84u, 0x37112fe1u},
+    {0x40780000u, 0xbf800000u, 0x32680000u, 0x1bdea0a2u},
+    {0x3f800000u, 0x5f800000u, 0xbeb80000u, 0xc009870du},
+    {0x7f800000u, 0xff780000u, 0xc0900000u, 0xde1756c4u},
+    {0x00000000u, 0x3ff00000u, 0x00000000u, 0x00100000u},
+    {0x00000000u, 0x3ff00000u, 0x00000000u, 0xbff00000u},
+    {0x3f680000u, 0x80f80000u, 0x80800000u, 0x9f8fe465u},
+    {0x1f800000u, 0xc0200000u, 0xbf280000u, 0xb1fee08fu},
+    {0x41e3bd75u, 0x7f800000u, 0xbf800000u, 0xece56e40u},
+    {0xbef80000u, 0x40000000u, 0x40400000u, 0xfc972e59u},
+    {0x00000000u, 0x001e1000u, 0x00000000u, 0x3ff00000u},
+    {0x00000000u, 0x3c400000u, 0xbf800000u, 0xc8c6df8eu},
+    {0x00000000u, 0xbff00000u, 0x00000000u, 0x00000000u},
+    {0x47e2cc36u, 0x00800000u, 0x00000000u, 0x66263f9fu},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x7ff80000u},
+    {0x00000000u, 0x00000000u, 0xc2f8f359u, 0x81a56e1fu},
+    {0x00000000u, 0x3f400000u, 0x80000000u, 0xa8333fe6u},
+    {0x00000000u, 0x40080000u, 0xffffffffu, 0x7fefffffu},
+    {0x00000000u, 0x7ff80000u, 0xc2f8f359u, 0x81a56e1fu},
+    {0x00000000u, 0xff7fffffu, 0x80000000u, 0xd6ee400au},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00100000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x06420000u}
+};
+constexpr std::uint32_t Expected[32][16] = {
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x12780000u, 0x3f800000u, 0x3f800000u, 0x12780000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x00000000u},
+    {0x3ec00000u, 0x00000000u, 0x00000000u, 0x3ec00000u, 0x00000000u, 0x00800000u, 0x2abc6ea9u, 0x00801868u, 0x00000000u, 0x00000000u, 0x3ec00000u, 0x9ffbf25eu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x3e8e7000u, 0x8800759cu, 0x7e37e43cu, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3e8e7000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00800000u, 0x48d00000u, 0x00000000u, 0x3ff00000u, 0xca667aa1u, 0x0f7a9d5bu, 0xca667aa1u, 0x0f7a9d5bu, 0x00000000u, 0x3ff00000u},
+    {0x3f800000u, 0x32e94401u, 0x32e94401u, 0x00000000u, 0x00000000u, 0x00000000u, 0x88d0c0dcu, 0x7db7b2f8u, 0x00000000u, 0x00000000u, 0x3f800000u, 0xded37902u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u},
+    {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x1a32f1b4u, 0x3f800000u, 0x1a32f1b4u, 0x00000000u, 0x00000000u, 0x7f7fffffu, 0x998ad4a5u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40000000u, 0xaee038d3u, 0x00000000u, 0x00000000u, 0xf4787e84u, 0xb7112fe1u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x40780000u, 0xbf800000u, 0x00000000u, 0x00000000u, 0x32680000u, 0x9bdea0a2u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x67800000u, 0x00000000u, 0x00000000u, 0xbeb80000u, 0xc009870du, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3ff00000u, 0x7f7ffffeu, 0xff780000u, 0x00000000u, 0x3ff00000u, 0xc0900000u, 0x5e1756c4u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x00100000u, 0x00000000u, 0x00100000u, 0x00000000u, 0x3ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x3f680000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x3f680000u, 0x00f80000u, 0x80800000u, 0x1f8fe465u, 0x80800000u, 0x1f8fe465u, 0x3f680000u, 0x00f80000u},
+    {0x00000000u, 0x1f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f800000u, 0xc8200000u, 0x00000000u, 0x3ff00000u, 0xbf280000u, 0x31fee08fu, 0xbf280000u, 0x31fee08fu, 0x00000000u, 0x3ff00000u},
+    {0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0xbf800000u, 0xece56e40u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x3f800000u, 0x3ef80000u, 0x3ef80000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x40400000u, 0xfc972e59u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x002e1000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x001e1000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x001e1000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1759d00au, 0x51a0597eu, 0x00000000u, 0x00000000u, 0xbf800000u, 0xc8c6df8eu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x63f5b02cu, 0x1eb63f9fu, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x66263f9fu, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f400000u, 0x00000000u, 0x3f400000u, 0x00000000u, 0x00000000u, 0x80000000u, 0xa8333fe6u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x3ff00000u, 0xffffffffu, 0x7fefffffu, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0xff7fffffu, 0x00000000u, 0x3ff00000u, 0x80000000u, 0x56ee400au, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x00000000u}
+};
+constexpr std::uint32_t Checked[32] = {0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xfff0u, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu};
+constexpr const char* Names[16] = {
+    "v_div_fmas_f32 v10, v4, v5, v6 clamp",
+    "v_div_fixup_f32 v11, v4, v5, v6 clamp",
+    "v_div_fixup_f32 v12, -v4, |v5|, v6 clamp",
+    "v_div_fmas_f32 v13, v5, -v4, v6 clamp",
+    "v_div_fmas_f64 v[14:15], v[4:5], v[6:7], v[4:5] clamp lo",
+    "v_div_fmas_f64 v[14:15], v[4:5], v[6:7], v[4:5] clamp hi",
+    "v_div_fmas_f64 v[16:17], v[6:7], v[6:7], v[4:5] mul:2 lo",
+    "v_div_fmas_f64 v[16:17], v[6:7], v[6:7], v[4:5] mul:2 hi",
+    "v_div_fixup_f64 v[18:19], v[4:5], v[6:7], v[4:5] clamp lo",
+    "v_div_fixup_f64 v[18:19], v[4:5], v[6:7], v[4:5] clamp hi",
+    "v_div_fixup_f64 v[20:21], v[6:7], v[4:5], v[6:7] div:2 lo",
+    "v_div_fixup_f64 v[20:21], v[6:7], v[4:5], v[6:7] div:2 hi",
+    "v_div_fixup_f64 v[22:23], v[6:7], v[4:5], v[6:7] clamp mul:4 lo",
+    "v_div_fixup_f64 v[22:23], v[6:7], v[4:5], v[6:7] clamp mul:4 hi",
+    "v_div_fixup_f64 v[24:25], -v[4:5], v[6:7], v[4:5] clamp lo",
+    "v_div_fixup_f64 v[24:25], -v[4:5], v[6:7], v[4:5] clamp hi",
+};
+
+void Fill(std::uint32_t tid, std::uint32_t* words) {
+    std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words);
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
+    Require(actual == expected, std::string("division result modifiers: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
+}
+
+ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code) {
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    return ShaderRecompiler::Recompile(request);
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    const std::span<const std::uint32_t> code(Code);
+    const auto result = Compile(device, code);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void CheckF32OmodRefused(AgcDriver::VulkanDevice& device) {
+    for (const std::uint32_t opcode : {0x15fu, 0x16fu}) {
+        alignas(256) const std::array<std::uint32_t, 3> code{0xd400000au | (opcode << 16u), 0x0c1a0b04u, 0xbf810000u};
+        std::string refusal;
+        try {
+            static_cast<void>(Compile(device, code));
+        } catch (const std::exception& error) {
+            refusal = error.what();
+        }
+        Require(refusal.find("VOP3 source modifiers are not implemented") != std::string::npos, "VOP3 opcode " + Hex(opcode) + " with omod was not refused");
+    }
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* in = &Input[tid * Inputs];
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t i = 0; i < 16; ++i) {
+            if ((Checked[tid] >> i) & 1u) Expect(tid, out[i], Expected[tid][i], Names[i]);
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device);
+        Check();
+        CheckF32OmodRefused(*device);
+        std::puts("division result modifiers tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/Dpp16InactiveSource.cpp

@@ -0,0 +1,287 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t MaxThreads = 64;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+constexpr std::uint32_t Checked = 15;
+alignas(256) std::array<std::uint32_t, MaxThreads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, MaxThreads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 77> Wave32Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x7e1402ff, 0xdeaddead, 0x7e1602ff,
+    0xdeaddead, 0x7e1802ff, 0xdeaddead, 0x7e1a02ff, 0xdeaddead, 0x7e1c02ff, 0xdeaddead, 0x7e1e02ff,
+    0xdeaddead, 0x7e2002ff, 0xdeaddead, 0x7e2202ff, 0xdeaddead, 0x7e2402ff, 0xdeaddead, 0x7e2602ff,
+    0xdeaddead, 0x7e2802ff, 0xdeaddead, 0x7e2a02ff, 0xdeaddead, 0x7e2c02ff, 0xdeaddead, 0x7e2e02ff,
+    0xdeaddead, 0x7e3002ff, 0xdeaddead, 0xbefe03ff, 0x33cc0ff3, 0x7e1402fa, 0xff004b04, 0x7e1602fa,
+    0xff044b04, 0x7e1802fa, 0xff084b04, 0x7e1a02fa, 0xff011104, 0x7e1c02fa, 0xff051104, 0x7e1e02fa,
+    0xff091104, 0x7e2002fa, 0xff0d1104, 0x7e2202fa, 0x5a014004, 0x4a240afa, 0xff004b04, 0x4a260afa,
+    0xff044b04, 0x4a280afa, 0xff084b04, 0x4a2a0afa, 0xff011104, 0x4a2c0afa, 0xff051104, 0x4a2e0afa,
+    0xff091104, 0x4a300afa, 0xff0d1104, 0xbefe03c1, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03,
+    0xe0781020, 0x80011203, 0xe0781030, 0x80011603, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 79> Wave64Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x7e1402ff, 0xdeaddead, 0x7e1602ff,
+    0xdeaddead, 0x7e1802ff, 0xdeaddead, 0x7e1a02ff, 0xdeaddead, 0x7e1c02ff, 0xdeaddead, 0x7e1e02ff,
+    0xdeaddead, 0x7e2002ff, 0xdeaddead, 0x7e2202ff, 0xdeaddead, 0x7e2402ff, 0xdeaddead, 0x7e2602ff,
+    0xdeaddead, 0x7e2802ff, 0xdeaddead, 0x7e2a02ff, 0xdeaddead, 0x7e2c02ff, 0xdeaddead, 0x7e2e02ff,
+    0xdeaddead, 0x7e3002ff, 0xdeaddead, 0xbefe03ff, 0x33cc0ff3, 0xbeff03ff, 0xc3a5960f, 0x7e1402fa,
+    0xff004b04, 0x7e1602fa, 0xff044b04, 0x7e1802fa, 0xff084b04, 0x7e1a02fa, 0xff011104, 0x7e1c02fa,
+    0xff051104, 0x7e1e02fa, 0xff091104, 0x7e2002fa, 0xff0d1104, 0x7e2202fa, 0x5a014004, 0x4a240afa,
+    0xff004b04, 0x4a260afa, 0xff044b04, 0x4a280afa, 0xff084b04, 0x4a2a0afa, 0xff011104, 0x4a2c0afa,
+    0xff051104, 0x4a2e0afa, 0xff091104, 0x4a300afa, 0xff0d1104, 0xbefe04c1, 0xe0781000, 0x80010a03,
+    0xe0781010, 0x80010e03, 0xe0781020, 0x80011203, 0xe0781030, 0x80011603, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[64][4] = {
+    {0x1e7ea419u, 0x51c9bc70u, 0x00000000u, 0x00000000u},
+    {0x80a4df5au, 0xf38b2ffcu, 0x00000000u, 0x00000000u},
+    {0x8306d03bu, 0xa5aec797u, 0x00000000u, 0x00000000u},
+    {0xdc28ff90u, 0xf3f49249u, 0x00000000u, 0x00000000u},
+    {0x1a466884u, 0xe255accbu, 0x00000000u, 0x00000000u},
+    {0x39292d22u, 0xe5121482u, 0x00000000u, 0x00000000u},
+    {0x99dd251du, 0x9f199504u, 0x00000000u, 0x00000000u},
+    {0x8e7aa6e9u, 0x6bad6be2u, 0x00000000u, 0x00000000u},
+    {0xc88b2875u, 0x9293de8fu, 0x00000000u, 0x00000000u},
+    {0x8c3d5f16u, 0xd7a7a3ccu, 0x00000000u, 0x00000000u},
+    {0xbb049a79u, 0xc6cd75e9u, 0x00000000u, 0x00000000u},
+    {0xc4a334bfu, 0x7dabe929u, 0x00000000u, 0x00000000u},
+    {0xc0433cbdu, 0xc5e818fau, 0x00000000u, 0x00000000u},
+    {0x96263ae6u, 0x70eb9a0au, 0x00000000u, 0x00000000u},
+    {0x3d6c51e3u, 0x00a61f93u, 0x00000000u, 0x00000000u},
+    {0x9d3c7decu, 0x14aa4e71u, 0x00000000u, 0x00000000u},
+    {0x1c593af5u, 0x49889310u, 0x00000000u, 0x00000000u},
+    {0xd11745adu, 0x1919e93au, 0x00000000u, 0x00000000u},
+    {0x731c9452u, 0x02f0ee99u, 0x00000000u, 0x00000000u},
+    {0xd0944996u, 0xe4163207u, 0x00000000u, 0x00000000u},
+    {0xaf67d461u, 0x7d836e77u, 0x00000000u, 0x00000000u},
+    {0xadec6f11u, 0x5071950eu, 0x00000000u, 0x00000000u},
+    {0x35d14880u, 0x65b00a2du, 0x00000000u, 0x00000000u},
+    {0x406329bcu, 0x59001ac9u, 0x00000000u, 0x00000000u},
+    {0xfadb8908u, 0xffb88309u, 0x00000000u, 0x00000000u},
+    {0xede7bd0cu, 0x5b4c8012u, 0x00000000u, 0x00000000u},
+    {0xce1d62e0u, 0x60581ccau, 0x00000000u, 0x00000000u},
+    {0xffe976abu, 0xbef7e268u, 0x00000000u, 0x00000000u},
+    {0x837a7935u, 0xa341738cu, 0x00000000u, 0x00000000u},
+    {0x1389b372u, 0xb91dddd9u, 0x00000000u, 0x00000000u},
+    {0x57241955u, 0x16f44881u, 0x00000000u, 0x00000000u},
+    {0x8eb5140fu, 0x89be9c1cu, 0x00000000u, 0x00000000u},
+    {0x4ac87fc0u, 0x49a047dcu, 0x00000000u, 0x00000000u},
+    {0x751b4c83u, 0x24114258u, 0x00000000u, 0x00000000u},
+    {0xe060a724u, 0xa51149bbu, 0x00000000u, 0x00000000u},
+    {0xb437bdb5u, 0xb766ff10u, 0x00000000u, 0x00000000u},
+    {0x93d20470u, 0x4f1a8a1du, 0x00000000u, 0x00000000u},
+    {0x06555096u, 0xd9090b89u, 0x00000000u, 0x00000000u},
+    {0xb5fb12e0u, 0x5e6c9992u, 0x00000000u, 0x00000000u},
+    {0x5d111a9du, 0x7608d942u, 0x00000000u, 0x00000000u},
+    {0x6c167229u, 0x17371472u, 0x00000000u, 0x00000000u},
+    {0x66160227u, 0xe41686cdu, 0x00000000u, 0x00000000u},
+    {0x95f19a51u, 0x8df05f25u, 0x00000000u, 0x00000000u},
+    {0x7f9ac098u, 0xde0279e1u, 0x00000000u, 0x00000000u},
+    {0x1dcf884cu, 0x6ce9da66u, 0x00000000u, 0x00000000u},
+    {0x81a0ba05u, 0xcd73de00u, 0x00000000u, 0x00000000u},
+    {0xc0099ebau, 0xdde3a361u, 0x00000000u, 0x00000000u},
+    {0x99a46df0u, 0xe01c683eu, 0x00000000u, 0x00000000u},
+    {0x7db4304du, 0x65ed0de4u, 0x00000000u, 0x00000000u},
+    {0xeab06e9bu, 0x864fa3f3u, 0x00000000u, 0x00000000u},
+    {0x423674a6u, 0xd82fb0f1u, 0x00000000u, 0x00000000u},
+    {0x6b82e6c9u, 0x90624fe3u, 0x00000000u, 0x00000000u},
+    {0xed1d9d16u, 0xfb759e0fu, 0x00000000u, 0x00000000u},
+    {0x7aac319fu, 0x8350a73fu, 0x00000000u, 0x00000000u},
+    {0x849acfb5u, 0xce1fd3d9u, 0x00000000u, 0x00000000u},
+    {0x067f0cfcu, 0x93550840u, 0x00000000u, 0x00000000u},
+    {0x397a7623u, 0xc40d31b5u, 0x00000000u, 0x00000000u},
+    {0x2106119eu, 0xd1711cbdu, 0x00000000u, 0x00000000u},
+    {0x0c83e786u, 0xf675299bu, 0x00000000u, 0x00000000u},
+    {0xb464c554u, 0x85f184e0u, 0x00000000u, 0x00000000u},
+    {0xe306f0c4u, 0xaba4c70eu, 0x00000000u, 0x00000000u},
+    {0x1a42721eu, 0xb7a1774fu, 0x00000000u, 0x00000000u},
+    {0x9f4fb02bu, 0x6dedc86au, 0x00000000u, 0x00000000u},
+    {0x78549014u, 0x2475263cu, 0x00000000u, 0x00000000u}
+};
+
+constexpr std::uint32_t Expected32[32][15] = {
+    {0xdeaddeadu, 0xdc28ff90u, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0x00000000u, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0x2df2bc00u, 0x51c9bc70u, 0xdeaddeadu, 0xdeaddeadu, 0x51c9bc70u, 0x51c9bc70u},
+    {0xdeaddeadu, 0x8306d03bu, 0x00000000u, 0x1e7ea419u, 0x1e7ea419u, 0x1e7ea419u, 0x1e7ea419u, 0xdeaddeadu, 0xdeaddeadu, 0x76920037u, 0xf38b2ffcu, 0x1209d415u, 0x1209d415u, 0x1209d415u, 0x1209d415u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x8e7aa6e9u, 0x8e7aa6e9u, 0x8e7aa6e9u, 0xdeaddeadu, 0xdc28ff90u, 0x00000000u, 0xdc28ff90u, 0xc4a334bfu, 0x70d053b4u, 0x70d053b4u, 0x70d053b4u, 0xdeaddeadu, 0xbe7eac5bu, 0xe255accbu, 0xbe7eac5bu},
+    {0x99dd251du, 0x99dd251du, 0x99dd251du, 0x1a466884u, 0x1a466884u, 0x1a466884u, 0x1a466884u, 0xbb049a79u, 0x7eef399fu, 0x7eef399fu, 0x7eef399fu, 0xff587d06u, 0xff587d06u, 0xff587d06u, 0xff587d06u},
+    {0x1a466884u, 0x1a466884u, 0x1a466884u, 0x39292d22u, 0x39292d22u, 0x39292d22u, 0x39292d22u, 0x8c3d5f16u, 0xb95ffd88u, 0xb95ffd88u, 0xb95ffd88u, 0xd842c226u, 0xd842c226u, 0xd842c226u, 0xd842c226u},
+    {0x39292d22u, 0x39292d22u, 0x39292d22u, 0x99dd251du, 0x99dd251du, 0x99dd251du, 0x99dd251du, 0xc88b2875u, 0xa4d69904u, 0xa4d69904u, 0xa4d69904u, 0x058a90ffu, 0x058a90ffu, 0x058a90ffu, 0x058a90ffu},
+    {0xc4a334bfu, 0xc4a334bfu, 0xc4a334bfu, 0x8e7aa6e9u, 0x8e7aa6e9u, 0x8e7aa6e9u, 0x8e7aa6e9u, 0xdeaddeadu, 0x5737134eu, 0x5737134eu, 0x5737134eu, 0x210e8578u, 0x210e8578u, 0x210e8578u, 0x210e8578u},
+    {0xbb049a79u, 0xbb049a79u, 0xbb049a79u, 0xc88b2875u, 0xc88b2875u, 0xc88b2875u, 0xc88b2875u, 0xdeaddeadu, 0x92ac3e45u, 0x92ac3e45u, 0x92ac3e45u, 0xa032cc41u, 0xa032cc41u, 0xa032cc41u, 0xa032cc41u},
+    {0xc88b2875u, 0xc88b2875u, 0xc88b2875u, 0x8c3d5f16u, 0x8c3d5f16u, 0x8c3d5f16u, 0x8c3d5f16u, 0xdeaddeadu, 0x8f589e5eu, 0x8f589e5eu, 0x8f589e5eu, 0x530ad4ffu, 0x530ad4ffu, 0x530ad4ffu, 0x530ad4ffu},
+    {0x8c3d5f16u, 0x8c3d5f16u, 0x8c3d5f16u, 0xbb049a79u, 0xbb049a79u, 0xbb049a79u, 0xbb049a79u, 0xdeaddeadu, 0x09e9483fu, 0x09e9483fu, 0x09e9483fu, 0x38b083a2u, 0x38b083a2u, 0x38b083a2u, 0x38b083a2u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0x1c593af5u, 0x00000000u, 0xdeaddeadu, 0xd11745adu, 0x00000000u, 0xd11745adu, 0xdeaddeadu, 0xdeaddeadu, 0x1f4a298eu, 0x02f0ee99u, 0xdeaddeadu, 0xd4083446u, 0x02f0ee99u, 0xd4083446u},
+    {0xdeaddeadu, 0xd11745adu, 0x00000000u, 0x731c9452u, 0x731c9452u, 0x731c9452u, 0x731c9452u, 0xdeaddeadu, 0xdeaddeadu, 0xb52d77b4u, 0xe4163207u, 0x5732c659u, 0x5732c659u, 0x5732c659u, 0x5732c659u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xaf67d461u, 0x00000000u, 0xdeaddeadu, 0xadec6f11u, 0x00000000u, 0xadec6f11u, 0xdeaddeadu, 0xdeaddeadu, 0x1517de8eu, 0x65b00a2du, 0xdeaddeadu, 0x139c793eu, 0x65b00a2du, 0x139c793eu},
+    {0xdeaddeadu, 0xadec6f11u, 0x00000000u, 0x35d14880u, 0x35d14880u, 0x35d14880u, 0x35d14880u, 0xdeaddeadu, 0xdeaddeadu, 0x06ec89dau, 0x59001ac9u, 0x8ed16349u, 0x8ed16349u, 0x8ed16349u, 0x8ed16349u},
+    {0xdeaddeadu, 0xffe976abu, 0x00000000u, 0x406329bcu, 0x406329bcu, 0x406329bcu, 0x406329bcu, 0xdeaddeadu, 0xdeaddeadu, 0xffa1f9b4u, 0xffb88309u, 0x401bacc5u, 0x401bacc5u, 0x401bacc5u, 0x401bacc5u},
+    {0xdeaddeadu, 0xce1d62e0u, 0x00000000u, 0xfadb8908u, 0xfadb8908u, 0xfadb8908u, 0xfadb8908u, 0xdeaddeadu, 0xdeaddeadu, 0x2969e2f2u, 0x5b4c8012u, 0x5628091au, 0x5628091au, 0x5628091au, 0x5628091au},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0x8eb5140fu, 0x00000000u, 0xdeaddeadu, 0xffe976abu, 0x00000000u, 0xffe976abu, 0xdeaddeadu, 0xdeaddeadu, 0x31f6879bu, 0xa341738cu, 0xdeaddeadu, 0xa32aea37u, 0xa341738cu, 0xa32aea37u},
+    {0xdeaddeadu, 0x57241955u, 0x00000000u, 0x837a7935u, 0x837a7935u, 0x837a7935u, 0x837a7935u, 0xdeaddeadu, 0xdeaddeadu, 0x1041f72eu, 0xb91dddd9u, 0x3c98570eu, 0x3c98570eu, 0x3c98570eu, 0x3c98570eu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu}
+};
+constexpr std::uint32_t Expected64[64][15] = {
+    {0xdeaddeadu, 0xdc28ff90u, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0x00000000u, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0x2df2bc00u, 0x51c9bc70u, 0xdeaddeadu, 0xdeaddeadu, 0x51c9bc70u, 0x51c9bc70u},
+    {0xdeaddeadu, 0x8306d03bu, 0x00000000u, 0x1e7ea419u, 0x1e7ea419u, 0x1e7ea419u, 0x1e7ea419u, 0xdeaddeadu, 0xdeaddeadu, 0x76920037u, 0xf38b2ffcu, 0x1209d415u, 0x1209d415u, 0x1209d415u, 0x1209d415u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x8e7aa6e9u, 0x8e7aa6e9u, 0x8e7aa6e9u, 0xdeaddeadu, 0xdc28ff90u, 0x00000000u, 0xdc28ff90u, 0xc4a334bfu, 0x70d053b4u, 0x70d053b4u, 0x70d053b4u, 0xdeaddeadu, 0xbe7eac5bu, 0xe255accbu, 0xbe7eac5bu},
+    {0x99dd251du, 0x99dd251du, 0x99dd251du, 0x1a466884u, 0x1a466884u, 0x1a466884u, 0x1a466884u, 0xbb049a79u, 0x7eef399fu, 0x7eef399fu, 0x7eef399fu, 0xff587d06u, 0xff587d06u, 0xff587d06u, 0xff587d06u},
+    {0x1a466884u, 0x1a466884u, 0x1a466884u, 0x39292d22u, 0x39292d22u, 0x39292d22u, 0x39292d22u, 0x8c3d5f16u, 0xb95ffd88u, 0xb95ffd88u, 0xb95ffd88u, 0xd842c226u, 0xd842c226u, 0xd842c226u, 0xd842c226u},
+    {0x39292d22u, 0x39292d22u, 0x39292d22u, 0x99dd251du, 0x99dd251du, 0x99dd251du, 0x99dd251du, 0xc88b2875u, 0xa4d69904u, 0xa4d69904u, 0xa4d69904u, 0x058a90ffu, 0x058a90ffu, 0x058a90ffu, 0x058a90ffu},
+    {0xc4a334bfu, 0xc4a334bfu, 0xc4a334bfu, 0x8e7aa6e9u, 0x8e7aa6e9u, 0x8e7aa6e9u, 0x8e7aa6e9u, 0xdeaddeadu, 0x5737134eu, 0x5737134eu, 0x5737134eu, 0x210e8578u, 0x210e8578u, 0x210e8578u, 0x210e8578u},
+    {0xbb049a79u, 0xbb049a79u, 0xbb049a79u, 0xc88b2875u, 0xc88b2875u, 0xc88b2875u, 0xc88b2875u, 0xdeaddeadu, 0x92ac3e45u, 0x92ac3e45u, 0x92ac3e45u, 0xa032cc41u, 0xa032cc41u, 0xa032cc41u, 0xa032cc41u},
+    {0xc88b2875u, 0xc88b2875u, 0xc88b2875u, 0x8c3d5f16u, 0x8c3d5f16u, 0x8c3d5f16u, 0x8c3d5f16u, 0xdeaddeadu, 0x8f589e5eu, 0x8f589e5eu, 0x8f589e5eu, 0x530ad4ffu, 0x530ad4ffu, 0x530ad4ffu, 0x530ad4ffu},
+    {0x8c3d5f16u, 0x8c3d5f16u, 0x8c3d5f16u, 0xbb049a79u, 0xbb049a79u, 0xbb049a79u, 0xbb049a79u, 0xdeaddeadu, 0x09e9483fu, 0x09e9483fu, 0x09e9483fu, 0x38b083a2u, 0x38b083a2u, 0x38b083a2u, 0x38b083a2u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0x1c593af5u, 0x00000000u, 0xdeaddeadu, 0xd11745adu, 0x00000000u, 0xd11745adu, 0xdeaddeadu, 0xdeaddeadu, 0x1f4a298eu, 0x02f0ee99u, 0xdeaddeadu, 0xd4083446u, 0x02f0ee99u, 0xd4083446u},
+    {0xdeaddeadu, 0xd11745adu, 0x00000000u, 0x731c9452u, 0x731c9452u, 0x731c9452u, 0x731c9452u, 0xdeaddeadu, 0xdeaddeadu, 0xb52d77b4u, 0xe4163207u, 0x5732c659u, 0x5732c659u, 0x5732c659u, 0x5732c659u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xaf67d461u, 0x00000000u, 0xdeaddeadu, 0xadec6f11u, 0x00000000u, 0xadec6f11u, 0xdeaddeadu, 0xdeaddeadu, 0x1517de8eu, 0x65b00a2du, 0xdeaddeadu, 0x139c793eu, 0x65b00a2du, 0x139c793eu},
+    {0xdeaddeadu, 0xadec6f11u, 0x00000000u, 0x35d14880u, 0x35d14880u, 0x35d14880u, 0x35d14880u, 0xdeaddeadu, 0xdeaddeadu, 0x06ec89dau, 0x59001ac9u, 0x8ed16349u, 0x8ed16349u, 0x8ed16349u, 0x8ed16349u},
+    {0xdeaddeadu, 0xffe976abu, 0x00000000u, 0x406329bcu, 0x406329bcu, 0x406329bcu, 0x406329bcu, 0xdeaddeadu, 0xdeaddeadu, 0xffa1f9b4u, 0xffb88309u, 0x401bacc5u, 0x401bacc5u, 0x401bacc5u, 0x401bacc5u},
+    {0xdeaddeadu, 0xce1d62e0u, 0x00000000u, 0xfadb8908u, 0xfadb8908u, 0xfadb8908u, 0xfadb8908u, 0xdeaddeadu, 0xdeaddeadu, 0x2969e2f2u, 0x5b4c8012u, 0x5628091au, 0x5628091au, 0x5628091au, 0x5628091au},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0x8eb5140fu, 0x00000000u, 0xdeaddeadu, 0xffe976abu, 0x00000000u, 0xffe976abu, 0xdeaddeadu, 0xdeaddeadu, 0x31f6879bu, 0xa341738cu, 0xdeaddeadu, 0xa32aea37u, 0xa341738cu, 0xa32aea37u},
+    {0xdeaddeadu, 0x57241955u, 0x00000000u, 0x837a7935u, 0x837a7935u, 0x837a7935u, 0x837a7935u, 0xdeaddeadu, 0xdeaddeadu, 0x1041f72eu, 0xb91dddd9u, 0x3c98570eu, 0x3c98570eu, 0x3c98570eu, 0x3c98570eu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xb437bdb5u, 0xb437bdb5u, 0xb437bdb5u, 0xdeaddeadu, 0xdeaddeadu, 0x00000000u, 0x00000000u, 0xdeaddeadu, 0xfdd80591u, 0xfdd80591u, 0xfdd80591u, 0xdeaddeadu, 0xdeaddeadu, 0x49a047dcu, 0x49a047dcu},
+    {0xe060a724u, 0xe060a724u, 0xe060a724u, 0x4ac87fc0u, 0x4ac87fc0u, 0x4ac87fc0u, 0x4ac87fc0u, 0xdeaddeadu, 0x0471e97cu, 0x0471e97cu, 0x0471e97cu, 0x6ed9c218u, 0x6ed9c218u, 0x6ed9c218u, 0x6ed9c218u},
+    {0x4ac87fc0u, 0x4ac87fc0u, 0x4ac87fc0u, 0x751b4c83u, 0x751b4c83u, 0x751b4c83u, 0x751b4c83u, 0xdeaddeadu, 0xefd9c97bu, 0xefd9c97bu, 0xefd9c97bu, 0x1a2c963eu, 0x1a2c963eu, 0x1a2c963eu, 0x1a2c963eu},
+    {0x751b4c83u, 0x751b4c83u, 0x751b4c83u, 0xe060a724u, 0xe060a724u, 0xe060a724u, 0xe060a724u, 0xdeaddeadu, 0x2c824b93u, 0x2c824b93u, 0x2c824b93u, 0x97c7a634u, 0x97c7a634u, 0x97c7a634u, 0x97c7a634u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x95f19a51u, 0x95f19a51u, 0x95f19a51u, 0xdeaddeadu, 0x6c167229u, 0x00000000u, 0x6c167229u, 0xdeaddeadu, 0x7a08211eu, 0x7a08211eu, 0x7a08211eu, 0xdeaddeadu, 0x502cf8f6u, 0xe41686cdu, 0x502cf8f6u},
+    {0xdeaddeadu, 0x6c167229u, 0x00000000u, 0x66160227u, 0x66160227u, 0x66160227u, 0x66160227u, 0xdeaddeadu, 0xdeaddeadu, 0xfa06d14eu, 0x8df05f25u, 0xf406614cu, 0xf406614cu, 0xf406614cu, 0xf406614cu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x99a46df0u, 0x99a46df0u, 0x99a46df0u, 0xdeaddeadu, 0x7f9ac098u, 0x00000000u, 0x7f9ac098u, 0xb437bdb5u, 0x068e4856u, 0x068e4856u, 0x068e4856u, 0xdeaddeadu, 0xec849afeu, 0x6ce9da66u, 0xec849afeu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0x81a0ba05u, 0x00000000u, 0xdeaddeadu, 0xc0099ebau, 0x00000000u, 0xc0099ebau, 0x4ac87fc0u, 0xdeaddeadu, 0x61bd2243u, 0xe01c683eu, 0xdeaddeadu, 0xa02606f8u, 0xe01c683eu, 0xa02606f8u},
+    {0xdeaddeadu, 0x6b82e6c9u, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0x00000000u, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0xd16ff4adu, 0x65ed0de4u, 0xdeaddeadu, 0xdeaddeadu, 0x65ed0de4u, 0x65ed0de4u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x7db4304du, 0x7db4304du, 0x7db4304du, 0xdeaddeadu, 0xeab06e9bu, 0x00000000u, 0xeab06e9bu, 0xdeaddeadu, 0x55e3e13eu, 0x55e3e13eu, 0x55e3e13eu, 0xdeaddeadu, 0xc2e01f8cu, 0xd82fb0f1u, 0xc2e01f8cu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0x849acfb5u, 0x00000000u, 0xdeaddeadu, 0xed1d9d16u, 0x00000000u, 0xed1d9d16u, 0xdeaddeadu, 0xdeaddeadu, 0x07eb76f4u, 0x8350a73fu, 0xdeaddeadu, 0x706e4455u, 0x8350a73fu, 0x706e4455u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x7aac319fu, 0x7aac319fu, 0x7aac319fu, 0xdeaddeadu, 0x849acfb5u, 0x00000000u, 0x849acfb5u, 0xdeaddeadu, 0x0e0139dfu, 0x0e0139dfu, 0x0e0139dfu, 0xdeaddeadu, 0x17efd7f5u, 0x93550840u, 0x17efd7f5u},
+    {0xdeaddeadu, 0xb464c554u, 0x00000000u, 0x067f0cfcu, 0x067f0cfcu, 0x067f0cfcu, 0x067f0cfcu, 0xdeaddeadu, 0xdeaddeadu, 0x7871f709u, 0xc40d31b5u, 0xca8c3eb1u, 0xca8c3eb1u, 0xca8c3eb1u, 0xca8c3eb1u},
+    {0xdeaddeadu, 0x0c83e786u, 0x00000000u, 0x397a7623u, 0x397a7623u, 0x397a7623u, 0x397a7623u, 0xdeaddeadu, 0xdeaddeadu, 0xddf50443u, 0xd1711cbdu, 0x0aeb92e0u, 0x0aeb92e0u, 0x0aeb92e0u, 0x0aeb92e0u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xe306f0c4u, 0x00000000u, 0xdeaddeadu, 0x1a42721eu, 0x00000000u, 0x1a42721eu, 0xdeaddeadu, 0xdeaddeadu, 0x50f4b92eu, 0x6dedc86au, 0xdeaddeadu, 0x88303a88u, 0x6dedc86au, 0x88303a88u},
+    {0xdeaddeadu, 0x1a42721eu, 0x00000000u, 0x9f4fb02bu, 0x9f4fb02bu, 0x9f4fb02bu, 0x9f4fb02bu, 0xdeaddeadu, 0xdeaddeadu, 0x3eb7985au, 0x2475263cu, 0xc3c4d667u, 0xc3c4d667u, 0xc3c4d667u, 0xc3c4d667u}
+};
+
+constexpr const char* Names[Checked] = {"mov_quad_perm", "mov_quad_perm_fi", "mov_quad_perm_bc", "mov_row_shr", "mov_row_shr_fi", "mov_row_shr_bc", "mov_row_shr_bc_fi", "mov_row_mirror_masks", "add_quad_perm", "add_quad_perm_fi", "add_quad_perm_bc", "add_row_shr", "add_row_shr_fi", "add_row_shr_bc", "add_row_shr_bc_fi"};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target) {
+    for (std::uint32_t tid = 0; tid < MaxThreads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{waveSize, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        target,
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+template<std::size_t Lanes>
+void Check(const std::uint32_t (&expected)[Lanes][Checked]) {
+    for (std::uint32_t tid = 0; tid < Lanes; ++tid) {
+        for (std::uint32_t index = 0; index < Checked; ++index) {
+            const auto actual = Output[tid * Results + index];
+            Require(actual == expected[tid][index], "dpp16 inactive source wave" + std::to_string(Lanes) + ": lane " + std::to_string(tid) + " " + Names[index] + " is " + Hex(actual) + ", expected " + Hex(expected[tid][index]));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (device->Target().subgroupSize < 32) {
+            std::printf("skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize);
+            return VulkanTestSkipped;
+        }
+        Run(*device, Wave32Code, 32, device->ComputeTarget(32));
+        Check(Expected32);
+        Run(*device, Wave64Code, 64, device->Target());
+        Check(Expected64);
+        Run(*device, Wave64Code, 64, device->ComputeTarget(32));
+        Check(Expected64);
+        std::puts("dpp16 inactive source tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/Dpp16Operands.cpp

@@ -0,0 +1,182 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 122> Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x7e500304, 0x7e520305, 0x7e140306,
+    0x7e1602ff, 0xdeaddead, 0x7e5802ff, 0xdeaddead, 0x7e5a02ff, 0xdeaddead, 0x7e5c02ff, 0xdeaddead,
+    0x7e5e02ff, 0xdeaddead, 0x7e6002ff, 0xdeaddead, 0x7e6202ff, 0xdeaddead, 0xbefe03c1, 0xbefc0381,
+    0x78140afa, 0xff004b04, 0x7e1686fa, 0xff004b28, 0x7e5888fa, 0xff004b28, 0x7e5c84fa, 0xff004b28,
+    0xbefc03ff, 0x00010001, 0x7e6090fa, 0xff004b28, 0xbefe03c1, 0x7e18032d, 0x7e1a032f, 0x7e1c0331,
+    0x7e500304, 0x7e520305, 0x7e1e0306, 0x7e2002ff, 0xdeaddead, 0x7e5802ff, 0xdeaddead, 0x7e5a02ff,
+    0xdeaddead, 0x7e5c02ff, 0xdeaddead, 0x7e5e02ff, 0xdeaddead, 0x7e6002ff, 0xdeaddead, 0x7e6202ff,
+    0xdeaddead, 0xbefe03c1, 0xbefc0381, 0x781e0afa, 0xff011104, 0x7e2086fa, 0xff011128, 0x7e5888fa,
+    0xff011128, 0x7e5c84fa, 0xff011128, 0xbefc03ff, 0x00010001, 0x7e6090fa, 0xff011128, 0xbefe03c1,
+    0x7e22032d, 0x7e24032f, 0x7e260331, 0x7e500304, 0x7e520305, 0x7e280306, 0x7e2a02ff, 0xdeaddead,
+    0x7e5802ff, 0xdeaddead, 0x7e5a02ff, 0xdeaddead, 0x7e5c02ff, 0xdeaddead, 0x7e5e02ff, 0xdeaddead,
+    0x7e6002ff, 0xdeaddead, 0x7e6202ff, 0xdeaddead, 0xbefe03ff, 0x33cc0ff3, 0xbefc0381, 0x78280afa,
+    0x5a00b104, 0x7e2a86fa, 0x5a00b128, 0x7e5888fa, 0x5a00b128, 0x7e5c84fa, 0x5a00b128, 0xbefc03ff,
+    0x00010001, 0x7e6090fa, 0x5a00b128, 0xbefe03c1, 0x7e2c032d, 0x7e2e032f, 0x7e300331, 0xd5780019,
+    0x04c25d2c, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xe0781020, 0x80011203, 0xe0781030,
+    0x80011603, 0xbf810000,
+};
+
+constexpr const char* Names[] = {"quad_perm_pk_fmac_f16", "quad_perm_movrels", "quad_perm_movrelsd", "quad_perm_movreld", "quad_perm_movrelsd_2", "row_shr_pk_fmac_f16", "row_shr_movrels", "row_shr_movrelsd", "row_shr_movreld", "row_shr_movrelsd_2", "masks_exec_pk_fmac_f16", "masks_exec_movrels", "masks_exec_movrelsd", "masks_exec_movreld", "masks_exec_movrelsd_2", "relative_bases"};
+constexpr std::uint32_t Rows[32][4] = {
+    {0x4000c4abu, 0x00000000u, 0x4a3d4000u, 0xe255accbu},
+    {0x35558000u, 0x80000000u, 0x2e668000u, 0x8c3d5f16u},
+    {0xb1c73f01u, 0x80003f01u, 0x35553c00u, 0x9d3c7decu},
+    {0x40004000u, 0x42484000u, 0x3f013c00u, 0xd0944996u},
+    {0x4a3d3f01u, 0x4a3dc4abu, 0x35552e66u, 0x406329bcu},
+    {0xc4abc4abu, 0x2e66b1c7u, 0x00004a3du, 0x1389b372u},
+    {0xb1c7c4abu, 0x40000000u, 0x00004248u, 0x49a047dcu},
+    {0x3f01b800u, 0x4a3db1c7u, 0xb1c78000u, 0x4f1a8a1du},
+    {0x3c00b1c7u, 0xc4abc4abu, 0x3f012e66u, 0x17371472u},
+    {0x2e668000u, 0x00003f01u, 0x40002e66u, 0x81a0ba05u},
+    {0x80003f01u, 0x2e660000u, 0x42482e66u, 0x90624fe3u},
+    {0x3f010000u, 0x00003c00u, 0x80003555u, 0xc40d31b5u},
+    {0xb8003c00u, 0xb1c70000u, 0x4a3d4000u, 0xb7a1774fu},
+    {0x80002e66u, 0x3f01b800u, 0x3f013555u, 0xd9814559u},
+    {0x40004a3du, 0xb1c73f01u, 0x4a3d3f01u, 0xb6eafff5u},
+    {0x00004248u, 0x00002e66u, 0x4a3d4248u, 0x8c53765fu},
+    {0x4a3d0000u, 0xb1c73f01u, 0x2e662e66u, 0x33d91808u},
+    {0x42484248u, 0x3c003c00u, 0xb8004a3du, 0xe1cd7be8u},
+    {0x80003f01u, 0x00003f01u, 0xc4ab3555u, 0x42969a50u},
+    {0xb1c73f01u, 0x3f010000u, 0x3f012e66u, 0x8ece78b0u},
+    {0x4a3d3c00u, 0x4248c4abu, 0x3f014000u, 0x703e53cfu},
+    {0xb1c7b1c7u, 0x00004248u, 0x3f01b800u, 0xf79edc82u},
+    {0x2e66b1c7u, 0x3f014000u, 0x4a3d4248u, 0xe73c49eau},
+    {0x3f01c4abu, 0x40003555u, 0xb1c72e66u, 0x2577c324u},
+    {0x3f013c00u, 0x40003c00u, 0x3f018000u, 0x8bed5b65u},
+    {0x40008000u, 0x35553555u, 0x40008000u, 0x0b6ec6f3u},
+    {0x00004248u, 0x4a3d4a3du, 0xc4abb800u, 0x01504585u},
+    {0xc4ab4a3du, 0x80008000u, 0x3f01b1c7u, 0xb870e4e1u},
+    {0xb8000000u, 0x35553f01u, 0x8000b800u, 0xf551abfdu},
+    {0x8000b1c7u, 0x00004248u, 0x40002e66u, 0xab266981u},
+    {0xb800b800u, 0x80004248u, 0x00002e66u, 0x63f1bbbcu},
+    {0xb8000000u, 0x35552e66u, 0x3c004000u, 0x63ac4f14u}
+};
+constexpr std::uint32_t Expected[32][16] = {
+    {0x4a3d4000u, 0x42484000u, 0x42484000u, 0x40004000u, 0x42484000u, 0x4a3d4000u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0x4a3d4000u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x2e660000u, 0x80003f01u, 0x80003f01u, 0xb1c73f01u, 0x80003f01u, 0x2e668000u, 0x00000000u, 0x00000000u, 0x4000c4abu, 0x00000000u, 0x2e668000u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x3555c72cu, 0x00000000u, 0x00000000u, 0x4000c4abu, 0x00000000u, 0x35553c00u, 0x80000000u, 0x80000000u, 0x35558000u, 0x80000000u, 0x35553c00u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x41983c00u, 0x80000000u, 0x80000000u, 0x35558000u, 0x80000000u, 0x3cbc4480u, 0x80003f01u, 0x80003f01u, 0xb1c73f01u, 0x80003f01u, 0x3f013c00u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x4d8b40deu, 0x4a3db1c7u, 0x4a3db1c7u, 0x3f01b800u, 0x4a3db1c7u, 0x4e52c89eu, 0x42484000u, 0x42484000u, 0x40004000u, 0x42484000u, 0xd33d4d79u, 0x2e66b1c7u, 0x2e66b1c7u, 0xc4abc4abu, 0x2e66b1c7u, 0xdeaddeadu},
+    {0xa49f4aa9u, 0x40000000u, 0x40000000u, 0xb1c7c4abu, 0x40000000u, 0x3cfd4a15u, 0x4a3dc4abu, 0x4a3dc4abu, 0x4a3d3f01u, 0x4a3dc4abu, 0x3cfd4a15u, 0x4a3dc4abu, 0x4a3dc4abu, 0x4a3d3f01u, 0x4a3dc4abu, 0xdeaddeadu},
+    {0x4e3d4248u, 0x4a3dc4abu, 0x4a3dc4abu, 0x4a3d3f01u, 0x4a3dc4abu, 0xc8ab4248u, 0x2e66b1c7u, 0x2e66b1c7u, 0xc4abc4abu, 0x2e66b1c7u, 0x43014248u, 0x4a3db1c7u, 0x4a3db1c7u, 0x3f01b800u, 0x4a3db1c7u, 0xdeaddeadu},
+    {0xd34d3abeu, 0x2e66b1c7u, 0x2e66b1c7u, 0xc4abc4abu, 0x2e66b1c7u, 0xc0de3abeu, 0x40000000u, 0x40000000u, 0xb1c7c4abu, 0x40000000u, 0xc0de3abeu, 0x40000000u, 0x40000000u, 0xb1c7c4abu, 0x40000000u, 0xdeaddeadu},
+    {0xc66c2e66u, 0x00003c00u, 0x00003c00u, 0x3f010000u, 0x00003c00u, 0xc66c40deu, 0x4a3db1c7u, 0x4a3db1c7u, 0x3f01b800u, 0x4a3db1c7u, 0x3f012e66u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x40004255u, 0x2e660000u, 0x2e660000u, 0x80003f01u, 0x2e660000u, 0x4000b2ebu, 0xc4abc4abu, 0xc4abc4abu, 0x3c00b1c7u, 0xc4abc4abu, 0x40002e66u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x427b2e66u, 0xc4abc4abu, 0xc4abc4abu, 0x3c00b1c7u, 0xc4abc4abu, 0x424d2e66u, 0x00003f01u, 0x00003f01u, 0x2e668000u, 0x00003f01u, 0x42482e66u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x00003555u, 0x00003f01u, 0x00003f01u, 0x2e668000u, 0x00003f01u, 0x8000402bu, 0x2e660000u, 0x2e660000u, 0x80003f01u, 0x2e660000u, 0x80003555u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x4a3d4000u, 0x00002e66u, 0x00002e66u, 0x00004248u, 0x00002e66u, 0x4a154000u, 0x00003c00u, 0x00003c00u, 0x3f010000u, 0x00003c00u, 0x4a3d4000u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x4541c5e8u, 0xb1c73f01u, 0xb1c73f01u, 0x40004a3du, 0xb1c73f01u, 0x3b01b156u, 0xb1c70000u, 0xb1c70000u, 0xb8003c00u, 0xb1c70000u, 0x3f013555u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x4a494301u, 0xb1c70000u, 0xb1c70000u, 0xb8003c00u, 0xb1c70000u, 0x4a3d3fb4u, 0x3f01b800u, 0x3f01b800u, 0x80002e66u, 0x3f01b800u, 0x4a3d3f01u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x4a3d424du, 0x3f01b800u, 0x3f01b800u, 0x80002e66u, 0x3f01b800u, 0x4a3d4463u, 0xb1c73f01u, 0xb1c73f01u, 0x40004a3du, 0xb1c73f01u, 0x4a3d4248u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x303e4255u, 0x3f010000u, 0x3f010000u, 0xb1c73f01u, 0x3f010000u, 0x2e662e66u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0x2e662e66u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xb8004b1du, 0x00003f01u, 0x00003f01u, 0x80003f01u, 0x00003f01u, 0x49fd4a3du, 0xb1c73f01u, 0xb1c73f01u, 0x4a3d0000u, 0xb1c73f01u, 0xb8004a3du, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xc4ab3555u, 0xb1c73f01u, 0xb1c73f01u, 0x4a3d0000u, 0xb1c73f01u, 0xc4ab45d5u, 0x3c003c00u, 0x3c003c00u, 0x42484248u, 0x3c003c00u, 0xc4ab3555u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x47402e66u, 0x3c003c00u, 0x3c003c00u, 0x42484248u, 0x3c003c00u, 0x3f012e66u, 0x00003f01u, 0x00003f01u, 0x80003f01u, 0x00003f01u, 0x3f012e66u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x47404df3u, 0x40003555u, 0x40003555u, 0x3f01c4abu, 0x40003555u, 0x3cbcc62cu, 0x3f010000u, 0x3f010000u, 0xb1c73f01u, 0x3f010000u, 0x3f014000u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x3f01bc45u, 0x3f014000u, 0x3f014000u, 0x2e66b1c7u, 0x3f014000u, 0x3f014148u, 0x4248c4abu, 0x4248c4abu, 0x4a3d3c00u, 0x4248c4abu, 0x3f01b800u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x504a4524u, 0x4248c4abu, 0x4248c4abu, 0x4a3d3c00u, 0x4248c4abu, 0x4a15418fu, 0x00004248u, 0x00004248u, 0xb1c7b1c7u, 0x00004248u, 0x4a3d4248u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xb8552918u, 0x00004248u, 0x00004248u, 0xb1c7b1c7u, 0x00004248u, 0x24f82918u, 0x3f014000u, 0x3f014000u, 0x2e66b1c7u, 0x3f014000u, 0xb1c72e66u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xc7964a3du, 0x80008000u, 0x80008000u, 0xc4ab4a3du, 0x80008000u, 0x4541c4abu, 0x40003555u, 0x40003555u, 0x3f01c4abu, 0x40003555u, 0x3f018000u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x40003c30u, 0x4a3d4a3du, 0x4a3d4a3du, 0x00004248u, 0x4a3d4a3du, 0x412b3555u, 0x40003c00u, 0x40003c00u, 0x3f013c00u, 0x40003c00u, 0x40008000u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x4c4b49fdu, 0x40003c00u, 0x40003c00u, 0x3f013c00u, 0x40003c00u, 0x4d12b800u, 0x35553555u, 0x35553555u, 0x40008000u, 0x35553555u, 0xc4abb800u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x3f01b1c7u, 0x35553555u, 0x35553555u, 0x40008000u, 0x35553555u, 0x3f01b1c7u, 0x4a3d4a3du, 0x4a3d4a3du, 0x00004248u, 0x4a3d4a3du, 0x3f01b1c7u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xb155b800u, 0x35552e66u, 0x35552e66u, 0xb8000000u, 0x35552e66u, 0xbe394d56u, 0x80008000u, 0x80008000u, 0xc4ab4a3du, 0x80008000u, 0x8000b800u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x4000bde2u, 0x80004248u, 0x80004248u, 0xb800b800u, 0x80004248u, 0x40002e66u, 0x35553f01u, 0x35553f01u, 0xb8000000u, 0x35553f01u, 0x40002e66u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x00002e66u, 0x35553f01u, 0x35553f01u, 0xb8000000u, 0x35553f01u, 0x0000b779u, 0x00004248u, 0x00004248u, 0x8000b1c7u, 0x00004248u, 0x00002e66u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x3c003feeu, 0x00004248u, 0x00004248u, 0x8000b1c7u, 0x00004248u, 0x3aab3fcdu, 0x80004248u, 0x80004248u, 0xb800b800u, 0x80004248u, 0x3c004000u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu}
+};
+void Fill(std::uint32_t tid, std::uint32_t* words) {
+    for (std::uint32_t i = 0; i < 4; ++i) words[i] = Rows[tid][i];
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
+    Require(actual == expected, std::string("dpp16 operands: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* in = &Input[tid * Inputs];
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t i = 0; i < 16; ++i) Expect(tid, out[i], Expected[tid][i], Names[i]);
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (device->Target().subgroupSize < Threads) {
+            std::printf("skipped, the device's subgroups are narrower than a wave (%u lanes)\n", device->Target().subgroupSize);
+            return VulkanTestSkipped;
+        }
+        Run(*device);
+        Check();
+        std::puts("dpp16 operands tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/Dpp16RowShareXmask.cpp

@@ -0,0 +1,289 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t MaxThreads = 64;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+constexpr std::uint32_t Checked = 16;
+alignas(256) std::array<std::uint32_t, MaxThreads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, MaxThreads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 81> Wave32Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x7e1402ff, 0xdeaddead, 0x7e1602ff,
+    0xdeaddead, 0x7e1802ff, 0xdeaddead, 0x7e1a02ff, 0xdeaddead, 0x7e1c02ff, 0xdeaddead, 0x7e1e02ff,
+    0xdeaddead, 0x7e2002ff, 0xdeaddead, 0x7e2202ff, 0xdeaddead, 0x7e2402ff, 0xdeaddead, 0x7e2602ff,
+    0xdeaddead, 0x7e2802ff, 0xdeaddead, 0x7e2a02ff, 0xdeaddead, 0x7e2c02ff, 0xdeaddead, 0x7e2e02ff,
+    0xdeaddead, 0x7e3002ff, 0xdeaddead, 0x7e3202ff, 0xdeaddead, 0xbefe03ff, 0x33cc0ff3, 0x7e1402fa,
+    0xff015004, 0x7e1602fa, 0xff055504, 0x7e1802fa, 0xff095c04, 0x7e1a02fa, 0xff0d5804, 0x7e1c02fa,
+    0x5a095f04, 0x7e1e02fa, 0xff016104, 0x7e2002fa, 0xff056604, 0x7e2202fa, 0xff096f04, 0x7e2402fa,
+    0xa5016a04, 0x4a260afa, 0xff015304, 0x4a280afa, 0xff095c04, 0x4a2a0afa, 0x69055404, 0x4a2c0afa,
+    0xff016404, 0x4c2e0afa, 0xff0d6d04, 0x20300efa, 0xff095406, 0x20320efa, 0xff056906, 0xbefe03c1,
+    0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xe0781020, 0x80011203, 0xe0781030, 0x80011603,
+    0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 83> Wave64Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x7e1402ff, 0xdeaddead, 0x7e1602ff,
+    0xdeaddead, 0x7e1802ff, 0xdeaddead, 0x7e1a02ff, 0xdeaddead, 0x7e1c02ff, 0xdeaddead, 0x7e1e02ff,
+    0xdeaddead, 0x7e2002ff, 0xdeaddead, 0x7e2202ff, 0xdeaddead, 0x7e2402ff, 0xdeaddead, 0x7e2602ff,
+    0xdeaddead, 0x7e2802ff, 0xdeaddead, 0x7e2a02ff, 0xdeaddead, 0x7e2c02ff, 0xdeaddead, 0x7e2e02ff,
+    0xdeaddead, 0x7e3002ff, 0xdeaddead, 0x7e3202ff, 0xdeaddead, 0xbefe03ff, 0x33cc0ff3, 0xbeff03ff,
+    0xc3a5960f, 0x7e1402fa, 0xff015004, 0x7e1602fa, 0xff055504, 0x7e1802fa, 0xff095c04, 0x7e1a02fa,
+    0xff0d5804, 0x7e1c02fa, 0x5a095f04, 0x7e1e02fa, 0xff016104, 0x7e2002fa, 0xff056604, 0x7e2202fa,
+    0xff096f04, 0x7e2402fa, 0xa5016a04, 0x4a260afa, 0xff015304, 0x4a280afa, 0xff095c04, 0x4a2a0afa,
+    0x69055404, 0x4a2c0afa, 0xff016404, 0x4c2e0afa, 0xff0d6d04, 0x20300efa, 0xff095406, 0x20320efa,
+    0xff056906, 0xbefe04c1, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xe0781020, 0x80011203,
+    0xe0781030, 0x80011603, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[64][4] = {
+    {0x6ecd1cd2u, 0x4bef5ab2u, 0xc1f00000u, 0xc0c00000u},
+    {0x6a003be9u, 0xe12916b9u, 0x422a0000u, 0x41fc0000u},
+    {0xe8ace817u, 0x43f615feu, 0x42600000u, 0xc21e0000u},
+    {0x4e3aa244u, 0x2d47d909u, 0x42540000u, 0xc2220000u},
+    {0x68060c75u, 0xd69d14d5u, 0x41940000u, 0xc1d40000u},
+    {0x5af9fb5au, 0x54393673u, 0x42440000u, 0xc1080000u},
+    {0xad52f58fu, 0x3c3ff249u, 0x3fc00000u, 0xc26c0000u},
+    {0xcb826265u, 0xd06f4fc2u, 0x425c0000u, 0xc2420000u},
+    {0xc99d74ceu, 0xcb3eacf5u, 0x41ac0000u, 0x42320000u},
+    {0x9c7fb5dbu, 0xa5e8a4bau, 0xc2340000u, 0xc24a0000u},
+    {0x1e7cc79eu, 0x3197b5a3u, 0x41c00000u, 0x42420000u},
+    {0x099b18b9u, 0xc7d8a2a5u, 0xc26a0000u, 0xc1980000u},
+    {0x1504efdfu, 0x7e04a9d5u, 0xc2520000u, 0x422a0000u},
+    {0xf200add6u, 0xd20e47a3u, 0xc1680000u, 0xc1380000u},
+    {0xe31908c0u, 0x60de8b85u, 0xc20c0000u, 0x421a0000u},
+    {0xf1c81bb0u, 0xa39b139du, 0x40000000u, 0xc2300000u},
+    {0xdd7dd010u, 0x93d28ae8u, 0xc1f80000u, 0xc2460000u},
+    {0xea239e60u, 0x1136bab0u, 0xc0900000u, 0xc1840000u},
+    {0x4a248cbdu, 0x2cde59ccu, 0xc1380000u, 0xc1c40000u},
+    {0x06734af7u, 0xfc5d75e7u, 0x40b00000u, 0xc2200000u},
+    {0x76646c94u, 0x33cb3f0cu, 0xc25e0000u, 0xc2080000u},
+    {0xc3f02221u, 0xba35f124u, 0xc2600000u, 0x424e0000u},
+    {0xb955e2e5u, 0xb0a01d78u, 0xc1a40000u, 0x41300000u},
+    {0xad47bf7du, 0x7e37ee41u, 0xc2380000u, 0x41e80000u},
+    {0x4467b3a9u, 0xc243faa9u, 0x42400000u, 0x42280000u},
+    {0xee39a32fu, 0x5d27404bu, 0xc1e00000u, 0xc2020000u},
+    {0x362b62e4u, 0x40e07905u, 0x425e0000u, 0xc2640000u},
+    {0x3597987bu, 0x9f2dda53u, 0xc1280000u, 0xc0200000u},
+    {0x8cdfd404u, 0xaa825860u, 0x42700000u, 0x41940000u},
+    {0x2883e9fdu, 0x6fcb4509u, 0xc1bc0000u, 0xc2700000u},
+    {0xa64c74f6u, 0x6c225e76u, 0x41c40000u, 0xc22a0000u},
+    {0xc9c4946du, 0x27f8b552u, 0x41300000u, 0xbf000000u},
+    {0xec1dd6b9u, 0x37eae67cu, 0x42300000u, 0xc1fc0000u},
+    {0x080a718au, 0xdd3a87b9u, 0xc1480000u, 0x426c0000u},
+    {0x4d7d2f79u, 0x242b5b96u, 0xc1b80000u, 0xc25e0000u},
+    {0xda0d5bd5u, 0xbd1eab24u, 0x41f40000u, 0xc2060000u},
+    {0x30aa4ea5u, 0x908d264eu, 0xc0800000u, 0xc0c00000u},
+    {0x4d4b8ec9u, 0x59f9bbd9u, 0xc1c40000u, 0x40c00000u},
+    {0x008d7334u, 0x81e7a48bu, 0x3f800000u, 0x423a0000u},
+    {0xa985a1d6u, 0x6454c697u, 0xc2300000u, 0xc22a0000u},
+    {0x6945a98eu, 0x56efdfffu, 0x425e0000u, 0x41200000u},
+    {0x543a17e0u, 0x9680cbfau, 0xc2360000u, 0xc20e0000u},
+    {0x0e1e5311u, 0x8dbd6564u, 0xc2480000u, 0xc24c0000u},
+    {0xd6c814a2u, 0x7f2c1289u, 0xc0b00000u, 0x41e00000u},
+    {0x248db44au, 0xc60fd986u, 0x42400000u, 0xc2160000u},
+    {0xd6c5e906u, 0x10117cecu, 0xc1f80000u, 0x41a80000u},
+    {0x324909c5u, 0xdcd7b8c2u, 0x42080000u, 0x42220000u},
+    {0xf995627fu, 0xc6e555f6u, 0xc2260000u, 0xc1800000u},
+    {0x735c003eu, 0xc54e7632u, 0x41900000u, 0xc25a0000u},
+    {0xfc3effbbu, 0x46c20a66u, 0xc1ec0000u, 0xc20a0000u},
+    {0x0986feb5u, 0x040b21dfu, 0xc2180000u, 0x3f800000u},
+    {0xacca236du, 0x7cfef2f2u, 0x40f00000u, 0x41cc0000u},
+    {0x00bd61d8u, 0x5654123eu, 0x41400000u, 0xc1100000u},
+    {0xe319a18bu, 0x5d8bc32du, 0x41100000u, 0xc2480000u},
+    {0x6583147du, 0x72ce7a51u, 0x41780000u, 0x426e0000u},
+    {0x49981da9u, 0x9b4ba109u, 0xc1a40000u, 0xc1e00000u},
+    {0xe341cf98u, 0x3ec5433cu, 0xc0000000u, 0xc23c0000u},
+    {0xc861e665u, 0xda64e2ddu, 0xc1f00000u, 0x41900000u},
+    {0x8c31f3d4u, 0x807eed38u, 0xc1300000u, 0xc24e0000u},
+    {0xf5411da7u, 0x6886f7e2u, 0xc1580000u, 0x420e0000u},
+    {0x56aecd4cu, 0x7cfe2052u, 0xc23a0000u, 0x423a0000u},
+    {0x29fc3f6du, 0x843d8b71u, 0x41d40000u, 0xbfc00000u},
+    {0xf14c9634u, 0x6de904e9u, 0xc1c40000u, 0xc1180000u},
+    {0x0524b659u, 0xb07d24c5u, 0xc22e0000u, 0xc23a0000u}
+};
+
+constexpr std::uint32_t Expected32[32][16] = {
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0x6a003be9u, 0xad52f58fu, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0x4bef5ab2u, 0xdeaddeadu, 0xb3f56727u, 0xa6115324u, 0x41940000u, 0xc0c00000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0x6ecd1cd2u, 0xcb826265u, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0xe12916b9u, 0xdeaddeadu, 0x3c231213u, 0x33dbd926u, 0x41fc0000u, 0x41fc0000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0x00000000u, 0x5af9fb5au, 0xe8ace817u, 0x099b18b9u, 0xdeaddeadu, 0xdeaddeadu, 0xd69d14d5u, 0xdeaddeadu, 0x456a31a7u, 0xc5e2a106u, 0x41940000u, 0xc1680000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0x00000000u, 0x68060c75u, 0x4e3aa244u, 0x1e7cc79eu, 0xdeaddeadu, 0xdeaddeadu, 0x54393673u, 0xdeaddeadu, 0xbe39725cu, 0x75643e5bu, 0x41940000u, 0xc1080000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0x00000000u, 0xcb826265u, 0x6ecd1cd2u, 0x9c7fb5dbu, 0xdeaddeadu, 0xdeaddeadu, 0x3c3ff249u, 0xdeaddeadu, 0xdeaddeadu, 0xcd5b2670u, 0x41940000u, 0x40000000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0x00000000u, 0xad52f58fu, 0x6a003be9u, 0xc99d74ceu, 0xdeaddeadu, 0xdeaddeadu, 0xd06f4fc2u, 0xdeaddeadu, 0xdeaddeadu, 0x4e0d77dcu, 0x41940000u, 0xc20c0000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0x9c7fb5dbu, 0xe31908c0u, 0xcb826265u, 0xdeaddeadu, 0xdeaddeadu, 0xcb3eacf5u, 0xdeaddeadu, 0xdeaddeadu, 0x8fbb4e65u, 0x42320000u, 0x42320000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0xc99d74ceu, 0xf1c81bb0u, 0xad52f58fu, 0xdeaddeadu, 0xdeaddeadu, 0xa5e8a4bau, 0xdeaddeadu, 0xdeaddeadu, 0xc21d67bbu, 0x41940000u, 0xc1f00000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0x099b18b9u, 0x1504efdfu, 0x5af9fb5au, 0xdeaddeadu, 0xdeaddeadu, 0x3197b5a3u, 0xdeaddeadu, 0xdeaddeadu, 0x99eaacc2u, 0x42420000u, 0x42540000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0x1e7cc79eu, 0xf200add6u, 0x68060c75u, 0xdeaddeadu, 0xdeaddeadu, 0xc7d8a2a5u, 0xdeaddeadu, 0xdeaddeadu, 0xe57a52eau, 0x41940000u, 0x42600000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0x06734af7u, 0x76646c94u, 0x2883e9fdu, 0x4467b3a9u, 0x3351a4c3u, 0xb9be2dd0u, 0xa342c660u, 0xe6343cb1u, 0x9ce63aa1u, 0x00000000u, 0xc1280000u},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0x4a248cbdu, 0xc3f02221u, 0x8cdfd404u, 0xee39a32fu, 0x02d0c0deu, 0x893d49ebu, 0x72c1e27bu, 0xa9a53564u, 0xa9eeff0fu, 0x00000000u, 0x425e0000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0xad47bf7du, 0xdd7dd010u, 0xee39a32fu, 0xdeaddeadu, 0xb713686fu, 0x3d7ff17cu, 0xdeaddeadu, 0xfac4aa35u, 0x84f77b03u, 0x41300000u, 0x41300000u},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0xb955e2e5u, 0xea239e60u, 0x4467b3a9u, 0xdeaddeadu, 0x84ab3938u, 0x0b17c245u, 0xdeaddeadu, 0x84ab3938u, 0xb7f374a3u, 0x41e80000u, 0x41e80000u},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0xee39a32fu, 0xa64c74f6u, 0xad47bf7du, 0x4a248cbdu, 0xc8b745a0u, 0x4f23ceadu, 0xdeaddeadu, 0x4f23ceadu, 0x01ac2778u, 0x42280000u, 0x42280000u},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0x4467b3a9u, 0xc9c4946du, 0xb955e2e5u, 0x06734af7u, 0x639a8b42u, 0xea07144fu, 0xdeaddeadu, 0x85ab2a48u, 0x193d2c49u, 0x00000000u, 0xc1f80000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0x2883e9fdu, 0x362b62e4u, 0x06734af7u, 0xdeaddeadu, 0xb0f5a357u, 0x37622c64u, 0x20e6c4f4u, 0xeeea0c09u, 0x3fa14600u, 0x41940000u, 0x41940000u},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0x8cdfd404u, 0x3597987bu, 0x4a248cbdu, 0xdeaddeadu, 0x763e9000u, 0xfcab190du, 0xe62fb19du, 0x5e04e838u, 0x6db28b07u, 0x00000000u, 0xc25e0000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu}
+};
+constexpr std::uint32_t Expected64[64][16] = {
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0x6a003be9u, 0xad52f58fu, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0x4bef5ab2u, 0xdeaddeadu, 0xb3f56727u, 0xa6115324u, 0x41940000u, 0xc0c00000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0x6ecd1cd2u, 0xcb826265u, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0xe12916b9u, 0xdeaddeadu, 0x3c231213u, 0x33dbd926u, 0x41fc0000u, 0x41fc0000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0x00000000u, 0x5af9fb5au, 0xe8ace817u, 0x099b18b9u, 0xdeaddeadu, 0xdeaddeadu, 0xd69d14d5u, 0xdeaddeadu, 0x456a31a7u, 0xc5e2a106u, 0x41940000u, 0xc1680000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0x00000000u, 0x68060c75u, 0x4e3aa244u, 0x1e7cc79eu, 0xdeaddeadu, 0xdeaddeadu, 0x54393673u, 0xdeaddeadu, 0xbe39725cu, 0x75643e5bu, 0x41940000u, 0xc1080000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0x00000000u, 0xcb826265u, 0x6ecd1cd2u, 0x9c7fb5dbu, 0xdeaddeadu, 0xdeaddeadu, 0x3c3ff249u, 0xdeaddeadu, 0xdeaddeadu, 0xcd5b2670u, 0x41940000u, 0x40000000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0x00000000u, 0xad52f58fu, 0x6a003be9u, 0xc99d74ceu, 0xdeaddeadu, 0xdeaddeadu, 0xd06f4fc2u, 0xdeaddeadu, 0xdeaddeadu, 0x4e0d77dcu, 0x41940000u, 0xc20c0000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0x9c7fb5dbu, 0xe31908c0u, 0xcb826265u, 0xdeaddeadu, 0xdeaddeadu, 0xcb3eacf5u, 0xdeaddeadu, 0xdeaddeadu, 0x8fbb4e65u, 0x42320000u, 0x42320000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0xc99d74ceu, 0xf1c81bb0u, 0xad52f58fu, 0xdeaddeadu, 0xdeaddeadu, 0xa5e8a4bau, 0xdeaddeadu, 0xdeaddeadu, 0xc21d67bbu, 0x41940000u, 0xc1f00000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0x099b18b9u, 0x1504efdfu, 0x5af9fb5au, 0xdeaddeadu, 0xdeaddeadu, 0x3197b5a3u, 0xdeaddeadu, 0xdeaddeadu, 0x99eaacc2u, 0x42420000u, 0x42540000u},
+    {0x6ecd1cd2u, 0x5af9fb5au, 0x00000000u, 0xc99d74ceu, 0xdeaddeadu, 0x1e7cc79eu, 0xf200add6u, 0x68060c75u, 0xdeaddeadu, 0xdeaddeadu, 0xc7d8a2a5u, 0xdeaddeadu, 0xdeaddeadu, 0xe57a52eau, 0x41940000u, 0x42600000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0x06734af7u, 0x76646c94u, 0x2883e9fdu, 0x4467b3a9u, 0x3351a4c3u, 0xb9be2dd0u, 0xa342c660u, 0xe6343cb1u, 0x9ce63aa1u, 0x00000000u, 0xc1280000u},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0x4a248cbdu, 0xc3f02221u, 0x8cdfd404u, 0xee39a32fu, 0x02d0c0deu, 0x893d49ebu, 0x72c1e27bu, 0xa9a53564u, 0xa9eeff0fu, 0x00000000u, 0x425e0000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0xad47bf7du, 0xdd7dd010u, 0xee39a32fu, 0xdeaddeadu, 0xb713686fu, 0x3d7ff17cu, 0xdeaddeadu, 0xfac4aa35u, 0x84f77b03u, 0x41300000u, 0x41300000u},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0xb955e2e5u, 0xea239e60u, 0x4467b3a9u, 0xdeaddeadu, 0x84ab3938u, 0x0b17c245u, 0xdeaddeadu, 0x84ab3938u, 0xb7f374a3u, 0x41e80000u, 0x41e80000u},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0xee39a32fu, 0xa64c74f6u, 0xad47bf7du, 0x4a248cbdu, 0xc8b745a0u, 0x4f23ceadu, 0xdeaddeadu, 0x4f23ceadu, 0x01ac2778u, 0x42280000u, 0x42280000u},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0x4467b3a9u, 0xc9c4946du, 0xb955e2e5u, 0x06734af7u, 0x639a8b42u, 0xea07144fu, 0xdeaddeadu, 0x85ab2a48u, 0x193d2c49u, 0x00000000u, 0xc1f80000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0x2883e9fdu, 0x362b62e4u, 0x06734af7u, 0xdeaddeadu, 0xb0f5a357u, 0x37622c64u, 0x20e6c4f4u, 0xeeea0c09u, 0x3fa14600u, 0x41940000u, 0x41940000u},
+    {0xdeaddeadu, 0xc3f02221u, 0x8cdfd404u, 0x4467b3a9u, 0xdeaddeadu, 0x8cdfd404u, 0x3597987bu, 0x4a248cbdu, 0xdeaddeadu, 0x763e9000u, 0xfcab190du, 0xe62fb19du, 0x5e04e838u, 0x6db28b07u, 0x00000000u, 0xc25e0000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xec1dd6b9u, 0x4d4b8ec9u, 0x248db44au, 0x6945a98eu, 0xdeaddeadu, 0x080a718au, 0x008d7334u, 0xf995627fu, 0xdeaddeadu, 0x11f84251u, 0x5c789ac6u, 0x68953521u, 0xdeaddeadu, 0x9edb028au, 0x00000000u, 0xc1fc0000u},
+    {0xec1dd6b9u, 0x4d4b8ec9u, 0x248db44au, 0x6945a98eu, 0xdeaddeadu, 0xec1dd6b9u, 0xa985a1d6u, 0x00000000u, 0xdeaddeadu, 0xb747e38eu, 0x01c83c03u, 0x0de4d65eu, 0xdeaddeadu, 0x47532c91u, 0x426c0000u, 0x426c0000u},
+    {0xec1dd6b9u, 0x4d4b8ec9u, 0x248db44au, 0x6945a98eu, 0xdeaddeadu, 0xda0d5bd5u, 0x30aa4ea5u, 0x00000000u, 0xdeaddeadu, 0xfe38b76bu, 0x48b90fe0u, 0x54d5aa3bu, 0xdeaddeadu, 0xd56a06e9u, 0x00000000u, 0xc0b00000u},
+    {0xec1dd6b9u, 0x4d4b8ec9u, 0x248db44au, 0x6945a98eu, 0xdeaddeadu, 0x4d7d2f79u, 0x4d4b8ec9u, 0x248db44au, 0xdeaddeadu, 0x972c06f9u, 0xe1ac5f6eu, 0xedc8f9c9u, 0xdeaddeadu, 0x752a5ea1u, 0x00000000u, 0xc2060000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xec1dd6b9u, 0x4d4b8ec9u, 0x248db44au, 0x6945a98eu, 0xdeaddeadu, 0xdeaddeadu, 0xf995627fu, 0x00000000u, 0xdeaddeadu, 0x708e27cfu, 0xbb0e8044u, 0xdeaddeadu, 0xdeaddeadu, 0x9a2982abu, 0x00000000u, 0x42300000u},
+    {0xec1dd6b9u, 0x4d4b8ec9u, 0x248db44au, 0x6945a98eu, 0xdeaddeadu, 0xdeaddeadu, 0x248db44au, 0x00000000u, 0xdeaddeadu, 0x67cac139u, 0xb24b19aeu, 0xdeaddeadu, 0xdeaddeadu, 0x1bc83c72u, 0x00000000u, 0x41f40000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xec1dd6b9u, 0x4d4b8ec9u, 0x248db44au, 0x6945a98eu, 0xf995627fu, 0xdeaddeadu, 0x0e1e5311u, 0xda0d5bd5u, 0xdeaddeadu, 0xa01d355bu, 0xea9d8dd0u, 0xf6ba282bu, 0xdeaddeadu, 0x41fa9804u, 0x00000000u, 0xc1c40000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xec1dd6b9u, 0x4d4b8ec9u, 0x248db44au, 0x6945a98eu, 0xf995627fu, 0xdeaddeadu, 0x543a17e0u, 0xec1dd6b9u, 0xdeaddeadu, 0xa0f2b1cbu, 0xeb730a40u, 0xf78fa49bu, 0xdeaddeadu, 0x8697d983u, 0x00000000u, 0x3f800000u},
+    {0x735c003eu, 0xe319a18bu, 0x00000000u, 0xe341cf98u, 0xdeaddeadu, 0xdeaddeadu, 0x6583147du, 0x0524b659u, 0xdeaddeadu, 0xdeaddeadu, 0xc54e7632u, 0xdeaddeadu, 0xdeaddeadu, 0x64adc93bu, 0x00000000u, 0xc1f00000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x735c003eu, 0xe319a18bu, 0x00000000u, 0xe341cf98u, 0xdeaddeadu, 0xdeaddeadu, 0x00bd61d8u, 0x00000000u, 0xe341cf98u, 0xdeaddeadu, 0x040b21dfu, 0xdeaddeadu, 0xdeaddeadu, 0x0119947au, 0x3f800000u, 0x3f800000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x735c003eu, 0xe319a18bu, 0x00000000u, 0xe341cf98u, 0xdeaddeadu, 0xdeaddeadu, 0xacca236du, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0x5d8bc32du, 0xdeaddeadu, 0xdeaddeadu, 0x85b60c6bu, 0x00000000u, 0xc23a0000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x735c003eu, 0xe319a18bu, 0x00000000u, 0xe341cf98u, 0xdeaddeadu, 0xdeaddeadu, 0xfc3effbbu, 0xe341cf98u, 0xdeaddeadu, 0xdeaddeadu, 0x9b4ba109u, 0xdeaddeadu, 0xdeaddeadu, 0xf0e652cbu, 0x00000000u, 0xc1c40000u},
+    {0x735c003eu, 0xe319a18bu, 0x00000000u, 0xe341cf98u, 0xdeaddeadu, 0xc861e665u, 0xf14c9634u, 0x49981da9u, 0x0986feb5u, 0xdeaddeadu, 0x3ec5433cu, 0xdeaddeadu, 0xdeaddeadu, 0xa4545e4fu, 0x00000000u, 0xc1ec0000u},
+    {0x735c003eu, 0xe319a18bu, 0x00000000u, 0xe341cf98u, 0xdeaddeadu, 0xe341cf98u, 0x0524b659u, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0xda64e2ddu, 0xdeaddeadu, 0xdeaddeadu, 0x26587efbu, 0x41900000u, 0x41900000u},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x735c003eu, 0xe319a18bu, 0x00000000u, 0xe341cf98u, 0xdeaddeadu, 0x0524b659u, 0xe341cf98u, 0x00000000u, 0xdeaddeadu, 0xdeaddeadu, 0x6de904e9u, 0xdeaddeadu, 0xdeaddeadu, 0x3ee11e84u, 0x00000000u, 0xc1180000u},
+    {0x735c003eu, 0xe319a18bu, 0x00000000u, 0xe341cf98u, 0xdeaddeadu, 0xf14c9634u, 0xc861e665u, 0x735c003eu, 0xdeaddeadu, 0xdeaddeadu, 0xb07d24c5u, 0xdeaddeadu, 0xdeaddeadu, 0x5909d9f0u, 0x00000000u, 0x41780000u}
+};
+
+constexpr const char* Names[Checked] = {"mov_row_share0", "mov_row_share5_fi", "mov_row_share12_bc", "mov_row_share8_bc_fi", "mov_row_share15_masks", "mov_row_xmask1", "mov_row_xmask6_fi", "mov_row_xmask15_bc", "mov_row_xmask10_masks", "add_row_share3", "add_row_share12_bc", "add_row_share4_fi_masks", "add_row_xmask4", "sub_row_xmask13_bc_fi", "max_f32_row_share4_bc", "max_f32_row_xmask9_fi"};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target) {
+    for (std::uint32_t tid = 0; tid < MaxThreads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{waveSize, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        target,
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+template<std::size_t Lanes>
+void Check(const std::uint32_t (&expected)[Lanes][Checked]) {
+    for (std::uint32_t tid = 0; tid < Lanes; ++tid) {
+        for (std::uint32_t index = 0; index < Checked; ++index) {
+            const auto actual = Output[tid * Results + index];
+            Require(actual == expected[tid][index], "dpp16 row share xmask wave" + std::to_string(Lanes) + ": lane " + std::to_string(tid) + " " + Names[index] + " is " + Hex(actual) + ", expected " + Hex(expected[tid][index]));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (device->Target().subgroupSize < 32) {
+            std::printf("skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize);
+            return VulkanTestSkipped;
+        }
+        Run(*device, Wave32Code, 32, device->ComputeTarget(32));
+        Check(Expected32);
+        Run(*device, Wave64Code, 64, device->Target());
+        Check(Expected64);
+        Run(*device, Wave64Code, 64, device->ComputeTarget(32));
+        Check(Expected64);
+        std::puts("dpp16 row share xmask tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/Dpp8.cpp

@@ -0,0 +1,285 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t MaxThreads = 64;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+constexpr std::uint32_t Checked = 12;
+alignas(256) std::array<std::uint32_t, MaxThreads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, MaxThreads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 67> Wave32Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x7e1402ff, 0xdeaddead, 0x7e1602ff,
+    0xdeaddead, 0x7e1802ff, 0xdeaddead, 0x7e1a02ff, 0xdeaddead, 0x7e1c02ff, 0xdeaddead, 0x7e1e02ff,
+    0xdeaddead, 0x7e2002ff, 0xdeaddead, 0x7e2202ff, 0xdeaddead, 0x7e2402ff, 0xdeaddead, 0x7e2602ff,
+    0xdeaddead, 0x7e2802ff, 0xdeaddead, 0x7e2a02ff, 0xdeaddead, 0x7e1402e9, 0x78afc504, 0x7e1602ea,
+    0x05397704, 0x4a180ae9, 0x00024904, 0x061a0ee9, 0x97705306, 0x7e1c0306, 0x561c0ee9, 0xa5602406,
+    0x7d880b04, 0x021e0ae9, 0x78afc504, 0x7e200ae9, 0x97705304, 0x50220ae9, 0x05397704, 0x7e1202ff,
+    0x12345678, 0xbefe03ff, 0x5a5af0f3, 0x7e2402e9, 0x78afc504, 0x7e2602ea, 0x78afc504, 0x4c280ae9,
+    0xa5602404, 0x7e2a02e9, 0x05397709, 0xbefe03c1, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03,
+    0xe0781020, 0x80011203, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 69> Wave64Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x7e1402ff, 0xdeaddead, 0x7e1602ff,
+    0xdeaddead, 0x7e1802ff, 0xdeaddead, 0x7e1a02ff, 0xdeaddead, 0x7e1c02ff, 0xdeaddead, 0x7e1e02ff,
+    0xdeaddead, 0x7e2002ff, 0xdeaddead, 0x7e2202ff, 0xdeaddead, 0x7e2402ff, 0xdeaddead, 0x7e2602ff,
+    0xdeaddead, 0x7e2802ff, 0xdeaddead, 0x7e2a02ff, 0xdeaddead, 0x7e1402e9, 0x78afc504, 0x7e1602ea,
+    0x05397704, 0x4a180ae9, 0x00024904, 0x061a0ee9, 0x97705306, 0x7e1c0306, 0x561c0ee9, 0xa5602406,
+    0x7d880b04, 0x021e0ae9, 0x78afc504, 0x7e200ae9, 0x97705304, 0x50220ae9, 0x05397704, 0x7e1202ff,
+    0x12345678, 0xbefe03ff, 0x5a5af0f3, 0xbeff03ff, 0xc3a5960f, 0x7e2402e9, 0x78afc504, 0x7e2602ea,
+    0x78afc504, 0x4c280ae9, 0xa5602404, 0x7e2a02e9, 0x05397709, 0xbefe04c1, 0xe0781000, 0x80010a03,
+    0xe0781010, 0x80010e03, 0xe0781020, 0x80011203, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[64][4] = {
+    {0x1e7ea419u, 0x51c9bc70u, 0x42800000u, 0xc2180000u},
+    {0xe255accbu, 0x39292d22u, 0x422c0000u, 0x42740000u},
+    {0xc0433cbdu, 0xc5e818fau, 0x42400000u, 0xc0400000u},
+    {0x00a61f93u, 0x9d3c7decu, 0xc2300000u, 0xc2100000u},
+    {0x49889310u, 0xd11745adu, 0xc21c0000u, 0x424c0000u},
+    {0x02f0ee99u, 0xd0944996u, 0x42740000u, 0x41800000u},
+    {0x35d14880u, 0x65b00a2du, 0x00000000u, 0x41c80000u},
+    {0xfadb8908u, 0xffb88309u, 0x41d80000u, 0x42000000u},
+    {0xffe976abu, 0xbef7e268u, 0xc2340000u, 0x41b80000u},
+    {0x16f44881u, 0x8eb5140fu, 0x41200000u, 0x41100000u},
+    {0x751b4c83u, 0x24114258u, 0x41700000u, 0xc2680000u},
+    {0xd9090b89u, 0xb5fb12e0u, 0x41f00000u, 0x41e80000u},
+    {0x7608d942u, 0x6c167229u, 0xc2240000u, 0x42180000u},
+    {0xe41686cdu, 0x95f19a51u, 0x427c0000u, 0xc20c0000u},
+    {0x6ce9da66u, 0x81a0ba05u, 0x42740000u, 0x42140000u},
+    {0xeab06e9bu, 0x864fa3f3u, 0x40000000u, 0x422c0000u},
+    {0x90624fe3u, 0xed1d9d16u, 0x42680000u, 0xc2680000u},
+    {0x93550840u, 0x397a7623u, 0xc1f80000u, 0xc2500000u},
+    {0xf675299bu, 0xb464c554u, 0xc2180000u, 0x42340000u},
+    {0x78549014u, 0x2475263cu, 0x426c0000u, 0xc0c00000u},
+    {0xd9814559u, 0xe2cbbb93u, 0xc2040000u, 0x42500000u},
+    {0xac9f21dfu, 0x782afe6bu, 0x00000000u, 0x42180000u},
+    {0xab74fe57u, 0xff8b2a6au, 0x41600000u, 0x42800000u},
+    {0xb2c55523u, 0x7a98b9acu, 0x42180000u, 0x42380000u},
+    {0x33d91808u, 0x4b5c1a95u, 0x40800000u, 0xc2780000u},
+    {0x0879d955u, 0x265c28eau, 0x42540000u, 0x42740000u},
+    {0x5c24c1aeu, 0xebe80fa9u, 0xc1500000u, 0x40000000u},
+    {0xb1515fffu, 0x7c4e4248u, 0x425c0000u, 0x427c0000u},
+    {0xced66b44u, 0x6f29a9f1u, 0xc2600000u, 0x41200000u},
+    {0x511070a7u, 0x7528ea79u, 0xc21c0000u, 0x42400000u},
+    {0xb07822dbu, 0xb76dc0c3u, 0x40000000u, 0x424c0000u},
+    {0xec0643b1u, 0xf71bd135u, 0xc1900000u, 0x42300000u},
+    {0xd7aa2208u, 0xba5c875cu, 0x42680000u, 0xc2400000u},
+    {0xa60741f3u, 0x44371ff3u, 0x427c0000u, 0x41f00000u},
+    {0x18bee074u, 0x31d06d9cu, 0x42240000u, 0xc1d80000u},
+    {0x730b89dcu, 0xea9e7ab5u, 0xc2740000u, 0xc2240000u},
+    {0x000e05d0u, 0x7383d496u, 0xc22c0000u, 0xc1600000u},
+    {0x3f724926u, 0x1eedfcfcu, 0xc2540000u, 0x40800000u},
+    {0xab215122u, 0xc1f6192bu, 0x41d80000u, 0xc1c80000u},
+    {0x01504585u, 0x503dc86bu, 0x42740000u, 0xc1f00000u},
+    {0x8bb9c912u, 0x3911c9b0u, 0x42640000u, 0xc1f80000u},
+    {0xf551abfdu, 0x991f77c9u, 0x41200000u, 0xc22c0000u},
+    {0x6af38f6cu, 0xab266981u, 0xc1c00000u, 0xc1f00000u},
+    {0xc88f63d6u, 0xc84c1884u, 0x41600000u, 0x42240000u},
+    {0x63f1bbbcu, 0xced8c284u, 0xc1d00000u, 0xc1600000u},
+    {0x6041f10eu, 0x039e257au, 0xc2140000u, 0x420c0000u},
+    {0xe344f1c3u, 0x12cc86e3u, 0xc24c0000u, 0x41c00000u},
+    {0xa5721c6au, 0x5a789b32u, 0xc2680000u, 0xc2680000u},
+    {0x82a8c712u, 0x79dcbc81u, 0x41980000u, 0x41500000u},
+    {0x733c6676u, 0x5377899au, 0xc2040000u, 0x427c0000u},
+    {0xf2e36355u, 0x7771f894u, 0xc1a80000u, 0xc1c80000u},
+    {0x975cef7au, 0x850567c2u, 0xc1400000u, 0x423c0000u},
+    {0x9f93693fu, 0x561f8bfeu, 0x42540000u, 0xc0a00000u},
+    {0xadb21095u, 0xb5fc684au, 0xc23c0000u, 0x42180000u},
+    {0x08be3b07u, 0x320be3f8u, 0xc1a00000u, 0xc2540000u},
+    {0xa5f27419u, 0x8b75d02au, 0xc2500000u, 0x424c0000u},
+    {0xde3b7dd3u, 0xc134a79bu, 0xc1800000u, 0x426c0000u},
+    {0xb600860cu, 0xadbafed0u, 0xc26c0000u, 0xc1f80000u},
+    {0xa5497863u, 0x87a9ec13u, 0xc1100000u, 0xc1e80000u},
+    {0x290f5d0fu, 0xa945e7bbu, 0xc1880000u, 0x420c0000u},
+    {0x45d6ac42u, 0x6ce4048fu, 0x42280000u, 0xc2300000u},
+    {0x62186f4bu, 0x4e0e8896u, 0xc1880000u, 0x41c00000u},
+    {0x2cb8cefcu, 0x43710bafu, 0x42580000u, 0x427c0000u},
+    {0xffce6973u, 0xc7be5613u, 0x42280000u, 0x41100000u}
+};
+
+constexpr std::uint32_t Expected32[32][12] = {
+    {0x02f0ee99u, 0xfadb8908u, 0x341f693bu, 0xc2a40000u, 0x44c14000u, 0x02f0ee99u, 0x4b261f93u, 0x4ca54578u, 0x02f0ee99u, 0x02f0ee99u, 0xf7bed6a0u, 0x12345678u},
+    {0x1e7ea419u, 0x35d14880u, 0x1b7ed9edu, 0x42da0000u, 0xc5120000u, 0x39292d22u, 0xce7ef30du, 0x6efa75a3u, 0x1e7ea419u, 0x1e7ea419u, 0x105f65eeu, 0x12345678u},
+    {0xfadb8908u, 0x02f0ee99u, 0xa83dc5c5u, 0x42200000u, 0xc3100000u, 0xfadb8908u, 0xcded529au, 0xc8d90793u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xfadb8908u, 0x49889310u, 0x7f922ab7u, 0x41e00000u, 0xc512c000u, 0xfadb8908u, 0x4df3f521u, 0xe6c510fcu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xc0433cbdu, 0x00a61f93u, 0xef95e9c6u, 0x429c0000u, 0xc21c0000u, 0xc0433cbdu, 0xcca48edfu, 0xd1bd6540u, 0x00000000u, 0xc0433cbdu, 0x64ba02d3u, 0x00000000u},
+    {0xe255accbu, 0xc0433cbdu, 0xef12edafu, 0x41800000u, 0x444f4000u, 0xe255accbu, 0x4e574522u, 0x90d78653u, 0xe255accbu, 0xe255accbu, 0x2f6bb66au, 0x00000000u},
+    {0x35d14880u, 0xe255accbu, 0x842eae46u, 0x42ac0000u, 0x44866000u, 0x35d14880u, 0x4c3c3ba6u, 0x4805b6f8u, 0x35d14880u, 0x35d14880u, 0x7ca5a29eu, 0x12345678u},
+    {0x00a61f93u, 0x1e7ea419u, 0x1e372722u, 0xc0e00000u, 0x44f76000u, 0x00a61f93u, 0x4e931126u, 0x1e372722u, 0x00000000u, 0x00a61f93u, 0x03386b90u, 0x12345678u},
+    {0xe41686cdu, 0xeab06e9bu, 0xd5ec2ae9u, 0x42540000u, 0xc4770000u, 0xbef7e268u, 0xce1bdbd2u, 0xa9a85104u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xffe976abu, 0x6ce9da66u, 0xa5a95c90u, 0x41c00000u, 0xc3b38000u, 0xffe976abu, 0x4eea3699u, 0xfb9eee75u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xeab06e9bu, 0xe41686cdu, 0x3b058ad9u, 0xc2400000u, 0x45241000u, 0x24114258u, 0x4db7a244u, 0x0827c926u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xeab06e9bu, 0x7608d942u, 0xccef5b61u, 0xc1800000u, 0xc49f6000u, 0xb5fb12e0u, 0xc9b44aa8u, 0x2c03ec23u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x751b4c83u, 0xd9090b89u, 0x6bffe8d4u, 0x42200000u, 0x450e5000u, 0x6c167229u, 0xcdaa7c8bu, 0x451f7db3u, 0x00000000u, 0x751b4c83u, 0x00d3683du, 0x00000000u},
+    {0x16f44881u, 0x751b4c83u, 0x95db10fcu, 0x41d00000u, 0xc3e70000u, 0x95f19a51u, 0x4ed9d3b5u, 0x0b0ce6d5u, 0x00000000u, 0x16f44881u, 0x6a0e65afu, 0x00000000u},
+    {0x6ce9da66u, 0x16f44881u, 0x818a30b0u, 0x42c80000u, 0x43d78000u, 0x6ce9da66u, 0xcddf4bcau, 0x98950286u, 0x6ce9da66u, 0x6ce9da66u, 0x7e5f45fbu, 0x00000000u},
+    {0xd9090b89u, 0xffe976abu, 0x86391a9eu, 0x40000000u, 0x45297000u, 0x864fa3f3u, 0x4eec11b3u, 0x86391a9fu, 0x00000000u, 0xd9090b89u, 0x5dc6e2dau, 0x00000000u},
+    {0xac9f21dfu, 0xb2c55523u, 0x8072a556u, 0x3f800000u, 0x44f68000u, 0xac9f21dfu, 0x4ef0a920u, 0x9fe2f239u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x90624fe3u, 0xab74fe57u, 0xcccf7e63u, 0xc2b40000u, 0x44d2a000u, 0x397a7623u, 0xcd18ad66u, 0xe4ef747bu, 0x00000000u, 0x90624fe3u, 0xa006cf36u, 0x12345678u},
+    {0xb2c55523u, 0xac9f21dfu, 0x47b9cd94u, 0x41600000u, 0x4520c000u, 0xb464c554u, 0xced955f0u, 0x6103e734u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xb2c55523u, 0xd9814559u, 0xb7ca2e7cu, 0x42500000u, 0xc3908000u, 0x2475263cu, 0xcedf3b60u, 0xfdf66b96u, 0x00000000u, 0xb2c55523u, 0xdb8ad9c4u, 0x12345678u},
+    {0xf675299bu, 0x78549014u, 0x732e0b76u, 0x42b40000u, 0x442dc000u, 0xf675299bu, 0xce9a7556u, 0x5b204ba7u, 0x00000000u, 0xf675299bu, 0xc8a942c4u, 0x12345678u},
+    {0x93550840u, 0xf675299bu, 0x088d4e4eu, 0x42500000u, 0xc4b48000u, 0x782afe6bu, 0xcea91603u, 0x6ea02807u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xab74fe57u, 0x93550840u, 0x8fed7a4du, 0x42800000u, 0xc4f64000u, 0xab74fe57u, 0xcea6c1bcu, 0x92e032aau, 0xab74fe57u, 0xab74fe57u, 0x93c9ddd6u, 0x12345678u},
+    {0x78549014u, 0x90624fe3u, 0x0afb098fu, 0x41500000u, 0x42180000u, 0x7a98b9acu, 0xce19faebu, 0x0afb0990u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x511070a7u, 0xec0643b1u, 0x53d5f3eau, 0xc0e00000u, 0x45594000u, 0x511070a7u, 0xce9d5d40u, 0x37625e46u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x33d91808u, 0xb07822dbu, 0x2ed6023fu, 0x42400000u, 0xc5523000u, 0x33d91808u, 0x4eb84983u, 0xd6d44bc5u, 0x00000000u, 0x33d91808u, 0xa87a425au, 0x12345678u},
+    {0xec0643b1u, 0x511070a7u, 0xf461e8feu, 0x425c0000u, 0xc0a00000u, 0xec0643b1u, 0x4d079d95u, 0x3cf88050u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xec0643b1u, 0xced66b44u, 0x84c81b9du, 0x42860000u, 0x43998000u, 0x7c4e4248u, 0x4e4f6460u, 0x4b24ad8du, 0x00000000u, 0xec0643b1u, 0x83b1bdb8u, 0x12345678u},
+    {0x5c24c1aeu, 0xb1515fffu, 0xa302c1f9u, 0xc1000000u, 0xc2100000u, 0x6f29a9f1u, 0xcd9fcde2u, 0x207b09f1u, 0x00000000u, 0x5c24c1aeu, 0x414e78eau, 0x12345678u},
+    {0x0879d955u, 0x5c24c1aeu, 0xa9020281u, 0x42480000u, 0xc425c000u, 0x0879d955u, 0xce9f0fbau, 0xd14dac27u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xb07822dbu, 0x0879d955u, 0xeb46d8cbu, 0x41400000u, 0x45291000u, 0xb07822dbu, 0x4ea220e1u, 0xbfe79a18u, 0xb07822dbu, 0xb07822dbu, 0x510c1892u, 0x12345678u},
+    {0xb1515fffu, 0x33d91808u, 0x2af4e93du, 0xc1400000u, 0xc4d8c000u, 0xb1515fffu, 0xce44a653u, 0x2af4e93du, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu}
+};
+constexpr std::uint32_t Expected64[64][12] = {
+    {0x02f0ee99u, 0xfadb8908u, 0x341f693bu, 0xc2a40000u, 0x44c14000u, 0x02f0ee99u, 0x4b261f93u, 0x4ca54578u, 0x02f0ee99u, 0x02f0ee99u, 0xf7bed6a0u, 0x12345678u},
+    {0x1e7ea419u, 0x35d14880u, 0x1b7ed9edu, 0x42da0000u, 0xc5120000u, 0x39292d22u, 0xce7ef30du, 0x6efa75a3u, 0x1e7ea419u, 0x1e7ea419u, 0x105f65eeu, 0x12345678u},
+    {0xfadb8908u, 0x02f0ee99u, 0xa83dc5c5u, 0x42200000u, 0xc3100000u, 0xfadb8908u, 0xcded529au, 0xc8d90793u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xfadb8908u, 0x49889310u, 0x7f922ab7u, 0x41e00000u, 0xc512c000u, 0xfadb8908u, 0x4df3f521u, 0xe6c510fcu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xc0433cbdu, 0x00a61f93u, 0xef95e9c6u, 0x429c0000u, 0xc21c0000u, 0xc0433cbdu, 0xcca48edfu, 0xd1bd6540u, 0x00000000u, 0xc0433cbdu, 0x64ba02d3u, 0x00000000u},
+    {0xe255accbu, 0xc0433cbdu, 0xef12edafu, 0x41800000u, 0x444f4000u, 0xe255accbu, 0x4e574522u, 0x90d78653u, 0xe255accbu, 0xe255accbu, 0x2f6bb66au, 0x00000000u},
+    {0x35d14880u, 0xe255accbu, 0x842eae46u, 0x42ac0000u, 0x44866000u, 0x35d14880u, 0x4c3c3ba6u, 0x4805b6f8u, 0x35d14880u, 0x35d14880u, 0x7ca5a29eu, 0x12345678u},
+    {0x00a61f93u, 0x1e7ea419u, 0x1e372722u, 0xc0e00000u, 0x44f76000u, 0x00a61f93u, 0x4e931126u, 0x1e372722u, 0x00000000u, 0x00a61f93u, 0x03386b90u, 0x12345678u},
+    {0xe41686cdu, 0xeab06e9bu, 0xd5ec2ae9u, 0x42540000u, 0xc4770000u, 0xbef7e268u, 0xce1bdbd2u, 0xa9a85104u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xffe976abu, 0x6ce9da66u, 0xa5a95c90u, 0x41c00000u, 0xc3b38000u, 0xffe976abu, 0x4eea3699u, 0xfb9eee75u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xeab06e9bu, 0xe41686cdu, 0x3b058ad9u, 0xc2400000u, 0x45241000u, 0x24114258u, 0x4db7a244u, 0x0827c926u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xeab06e9bu, 0x7608d942u, 0xccef5b61u, 0xc1800000u, 0xc49f6000u, 0xb5fb12e0u, 0xc9b44aa8u, 0x2c03ec23u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x751b4c83u, 0xd9090b89u, 0x6bffe8d4u, 0x42200000u, 0x450e5000u, 0x6c167229u, 0xcdaa7c8bu, 0x451f7db3u, 0x00000000u, 0x751b4c83u, 0x00d3683du, 0x00000000u},
+    {0x16f44881u, 0x751b4c83u, 0x95db10fcu, 0x41d00000u, 0xc3e70000u, 0x95f19a51u, 0x4ed9d3b5u, 0x0b0ce6d5u, 0x00000000u, 0x16f44881u, 0x6a0e65afu, 0x00000000u},
+    {0x6ce9da66u, 0x16f44881u, 0x818a30b0u, 0x42c80000u, 0x43d78000u, 0x6ce9da66u, 0xcddf4bcau, 0x98950286u, 0x6ce9da66u, 0x6ce9da66u, 0x7e5f45fbu, 0x00000000u},
+    {0xd9090b89u, 0xffe976abu, 0x86391a9eu, 0x40000000u, 0x45297000u, 0x864fa3f3u, 0x4eec11b3u, 0x86391a9fu, 0x00000000u, 0xd9090b89u, 0x5dc6e2dau, 0x00000000u},
+    {0xac9f21dfu, 0xb2c55523u, 0x8072a556u, 0x3f800000u, 0x44f68000u, 0xac9f21dfu, 0x4ef0a920u, 0x9fe2f239u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x90624fe3u, 0xab74fe57u, 0xcccf7e63u, 0xc2b40000u, 0x44d2a000u, 0x397a7623u, 0xcd18ad66u, 0xe4ef747bu, 0x00000000u, 0x90624fe3u, 0xa006cf36u, 0x12345678u},
+    {0xb2c55523u, 0xac9f21dfu, 0x47b9cd94u, 0x41600000u, 0x4520c000u, 0xb464c554u, 0xced955f0u, 0x6103e734u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xb2c55523u, 0xd9814559u, 0xb7ca2e7cu, 0x42500000u, 0xc3908000u, 0x2475263cu, 0xcedf3b60u, 0xfdf66b96u, 0x00000000u, 0xb2c55523u, 0xdb8ad9c4u, 0x12345678u},
+    {0xf675299bu, 0x78549014u, 0x732e0b76u, 0x42b40000u, 0x442dc000u, 0xf675299bu, 0xce9a7556u, 0x5b204ba7u, 0x00000000u, 0xf675299bu, 0xc8a942c4u, 0x12345678u},
+    {0x93550840u, 0xf675299bu, 0x088d4e4eu, 0x42500000u, 0xc4b48000u, 0x782afe6bu, 0xcea91603u, 0x6ea02807u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xab74fe57u, 0x93550840u, 0x8fed7a4du, 0x42800000u, 0xc4f64000u, 0xab74fe57u, 0xcea6c1bcu, 0x92e032aau, 0xab74fe57u, 0xab74fe57u, 0x93c9ddd6u, 0x12345678u},
+    {0x78549014u, 0x90624fe3u, 0x0afb098fu, 0x41500000u, 0x42180000u, 0x7a98b9acu, 0xce19faebu, 0x0afb0990u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x511070a7u, 0xec0643b1u, 0x53d5f3eau, 0xc0e00000u, 0x45594000u, 0x511070a7u, 0xce9d5d40u, 0x37625e46u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x33d91808u, 0xb07822dbu, 0x2ed6023fu, 0x42400000u, 0xc5523000u, 0x33d91808u, 0x4eb84983u, 0xd6d44bc5u, 0x00000000u, 0x33d91808u, 0xa87a425au, 0x12345678u},
+    {0xec0643b1u, 0x511070a7u, 0xf461e8feu, 0x425c0000u, 0xc0a00000u, 0xec0643b1u, 0x4d079d95u, 0x3cf88050u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xec0643b1u, 0xced66b44u, 0x84c81b9du, 0x42860000u, 0x43998000u, 0x7c4e4248u, 0x4e4f6460u, 0x4b24ad8du, 0x00000000u, 0xec0643b1u, 0x83b1bdb8u, 0x12345678u},
+    {0x5c24c1aeu, 0xb1515fffu, 0xa302c1f9u, 0xc1000000u, 0xc2100000u, 0x6f29a9f1u, 0xcd9fcde2u, 0x207b09f1u, 0x00000000u, 0x5c24c1aeu, 0x414e78eau, 0x12345678u},
+    {0x0879d955u, 0x5c24c1aeu, 0xa9020281u, 0x42480000u, 0xc425c000u, 0x0879d955u, 0xce9f0fbau, 0xd14dac27u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xb07822dbu, 0x0879d955u, 0xeb46d8cbu, 0x41400000u, 0x45291000u, 0xb07822dbu, 0x4ea220e1u, 0xbfe79a18u, 0xb07822dbu, 0xb07822dbu, 0x510c1892u, 0x12345678u},
+    {0xb1515fffu, 0x33d91808u, 0x2af4e93du, 0xc1400000u, 0xc4d8c000u, 0xb1515fffu, 0xce44a653u, 0x2af4e93du, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x3f724926u, 0x01504585u, 0x6063c94fu, 0xc2da0000u, 0x4504a000u, 0xba5c875cu, 0x4ee61714u, 0xbbaccce2u, 0x00000000u, 0x3f724926u, 0x45a378a4u, 0x00000000u},
+    {0xd7aa2208u, 0xab215122u, 0xea3e61e6u, 0x428e0000u, 0xc4996000u, 0x44371ff3u, 0x4dc5f704u, 0xef587116u, 0xd7aa2208u, 0xd7aa2208u, 0xbbc8e00du, 0x00000000u},
+    {0x01504585u, 0x3f724926u, 0xd7d7af8fu, 0x42100000u, 0xc4bea000u, 0x01504585u, 0xceb3f17cu, 0x7142b6c2u, 0x00000000u, 0x01504585u, 0xa5d9b46cu, 0x00000000u},
+    {0x01504585u, 0x000e05d0u, 0x90a5bca8u, 0x41880000u, 0xc5187000u, 0x01504585u, 0xce215778u, 0xeaac8085u, 0x00000000u, 0x01504585u, 0xed0ba753u, 0x00000000u},
+    {0x18bee074u, 0x730b89dcu, 0x4b2df69eu, 0x423c0000u, 0xc3d28000u, 0x18bee074u, 0x4ba822c2u, 0xe68f5e72u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xa60741f3u, 0x18bee074u, 0xf6981f04u, 0x41f80000u, 0x42de0000u, 0x1eedfcfcu, 0xcea9bd5eu, 0x37acdd71u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xab215122u, 0xa60741f3u, 0x99a03b33u, 0xc29c0000u, 0xc4c18000u, 0xab215122u, 0x4e7dc925u, 0x67fd5b1eu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x730b89dcu, 0xd7aa2208u, 0x27e7ea73u, 0xc2920000u, 0x44ce6000u, 0x730b89dcu, 0x49605d00u, 0x27e7ea73u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x6041f10eu, 0xa5721c6au, 0x2e6375adu, 0xc1880000u, 0x4457c000u, 0x3911c9b0u, 0xce5dc271u, 0xde83e61bu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x8bb9c912u, 0xe344f1c3u, 0x8e7123c6u, 0xc2860000u, 0x448d0000u, 0x991f77c9u, 0x4ed5e71fu, 0x7c64698du, 0x00000000u, 0x8bb9c912u, 0xcad243f3u, 0x00000000u},
+    {0xa5721c6au, 0x6041f10eu, 0xa078157eu, 0xc1a00000u, 0xc4d8c000u, 0xa5721c6au, 0xcd2ae540u, 0x0b685a8fu, 0xa5721c6au, 0xa5721c6au, 0x54d9967fu, 0x00000000u},
+    {0xa5721c6au, 0x63f1bbbcu, 0xbd9dc481u, 0x42c40000u, 0x4512f000u, 0xc84c1884u, 0xcee88c6eu, 0x2c3dd441u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x6af38f6cu, 0xc88f63d6u, 0x5a928b96u, 0xc2900000u, 0x442c0000u, 0x6af38f6cu, 0xceb51bc7u, 0x9768265au, 0x6af38f6cu, 0x6af38f6cu, 0x31273d7cu, 0x00000000u},
+    {0xf551abfdu, 0x6af38f6cu, 0x8f57ee8cu, 0xc1800000u, 0xc45b4000u, 0x039e257au, 0xcde5d872u, 0x6e91b4e7u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xe344f1c3u, 0xf551abfdu, 0x9e864ff5u, 0xc1500000u, 0x433d0000u, 0x12cc86e3u, 0x4ec083e2u, 0x081e32e1u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xc88f63d6u, 0x8bb9c912u, 0xe6326444u, 0xc2a80000u, 0x45028000u, 0x5a789b32u, 0x4ec7e377u, 0xe6326445u, 0x00000000u, 0xc88f63d6u, 0xa58764ceu, 0x00000000u},
+    {0xadb21095u, 0xa5f27419u, 0xed1922f7u, 0x3f800000u, 0x44310000u, 0x79dcbc81u, 0xced14621u, 0x1fcf309bu, 0xadb21095u, 0xadb21095u, 0x8623437fu, 0x12345678u},
+    {0x82a8c712u, 0x08be3b07u, 0xc6b3f010u, 0x42280000u, 0x454ea000u, 0x5377899au, 0xcd51c9cbu, 0x5c35c4a2u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xa5f27419u, 0xadb21095u, 0xeaae5f0au, 0xc2680000u, 0xc3f80000u, 0x7771f894u, 0x4ee678cdu, 0x2524092au, 0xa5f27419u, 0xa5f27419u, 0x0b36ce7eu, 0x12345678u},
+    {0xa5f27419u, 0x9f93693fu, 0xf841ce38u, 0x42840000u, 0x445c4000u, 0x850567c2u, 0xcefaae72u, 0x2498d102u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xf2e36355u, 0x975cef7au, 0xd8c85310u, 0xc2640000u, 0x43190000u, 0x561f8bfeu, 0xceb41b18u, 0xed7c7b79u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x733c6676u, 0xf2e36355u, 0x38a52f5cu, 0x41900000u, 0xc4534000u, 0x733c6676u, 0x4d0be3b0u, 0xa8dfcb9fu, 0x00000000u, 0x733c6676u, 0x3ce6fb0bu, 0x12345678u},
+    {0x08be3b07u, 0x733c6676u, 0xb4b4ab0au, 0xc2c80000u, 0x44d82000u, 0x08be3b07u, 0xcea49bdfu, 0xa5484a6eu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x975cef7au, 0x82a8c712u, 0x0e1e973cu, 0x42d00000u, 0xc5191000u, 0x8b75d02au, 0xcec0d92eu, 0x0e1e973du, 0x00000000u, 0x975cef7au, 0x223c406bu, 0x12345678u},
+    {0x62186f4bu, 0xffce6973u, 0x77352da7u, 0x42280000u, 0x4519e000u, 0xc134a79bu, 0x4e243d74u, 0xc103110fu, 0x00000000u, 0x62186f4bu, 0x3ecb5865u, 0x12345678u},
+    {0xde3b7dd3u, 0x2cb8cefcu, 0x63bb84dcu, 0xc2200000u, 0xc4aa2000u, 0xadbafed0u, 0xceb56d0fu, 0xda73cdcdu, 0xde3b7dd3u, 0xde3b7dd3u, 0x52450130u, 0x12345678u},
+    {0xffce6973u, 0x62186f4bu, 0x3daa721fu, 0xc2b00000u, 0x43e38000u, 0x87a9ec13u, 0xce93fef4u, 0xe9c25b5fu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xffce6973u, 0x45d6ac42u, 0x5f466dc7u, 0x41980000u, 0xc4104000u, 0xffce6973u, 0xce071209u, 0xef1c93fdu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xa5497863u, 0x290f5d0fu, 0x4b1f8262u, 0xc0000000u, 0xc511e000u, 0xa5497863u, 0xca465a34u, 0x95f3619eu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xb600860cu, 0xa5497863u, 0x2c4a0669u, 0x429c0000u, 0xc3690000u, 0x4e0e8896u, 0x4e32e33cu, 0xf35800fau, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x2cb8cefcu, 0xb600860cu, 0x21ac8982u, 0x42380000u, 0xc564f000u, 0x2cb8cefcu, 0x4ec430dfu, 0xf97191bbu, 0x2cb8cefcu, 0x2cb8cefcu, 0x728f7a5du, 0x12345678u},
+    {0x290f5d0fu, 0xde3b7dd3u, 0xa5f9d3e6u, 0x424c0000u, 0xc2de0000u, 0xc7be5613u, 0x4e8bad59u, 0xa5f9d3e7u, 0x00000000u, 0x290f5d0fu, 0x3841a9edu, 0x12345678u}
+};
+
+constexpr const char* Names[Checked] = {"mov", "mov_fi", "add_nc_u32", "add_f32", "fmac_f32", "cndmask", "cvt_f32_i32", "add_co_ci_u32", "mov_inactive", "mov_inactive_fi", "sub_nc_u32_inactive", "mov_uniform_inactive"};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target) {
+    for (std::uint32_t tid = 0; tid < MaxThreads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{waveSize, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        target,
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+template<std::size_t Lanes>
+void Check(const std::uint32_t (&expected)[Lanes][Checked]) {
+    for (std::uint32_t tid = 0; tid < Lanes; ++tid) {
+        for (std::uint32_t index = 0; index < Checked; ++index) {
+            const auto actual = Output[tid * Results + index];
+            Require(actual == expected[tid][index], "dpp8 wave" + std::to_string(Lanes) + ": lane " + std::to_string(tid) + " " + Names[index] + " is " + Hex(actual) + ", expected " + Hex(expected[tid][index]));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (device->Target().subgroupSize < 32) {
+            std::printf("skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize);
+            return VulkanTestSkipped;
+        }
+        Run(*device, Wave32Code, 32, device->ComputeTarget(32));
+        Check(Expected32);
+        Run(*device, Wave64Code, 64, device->Target());
+        Check(Expected64);
+        Run(*device, Wave64Code, 64, device->ComputeTarget(32));
+        Check(Expected64);
+        std::puts("dpp8 tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DppNoForm.cpp

@@ -0,0 +1,97 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+struct NoFormOpcode {
+    bool vop2;
+    std::uint32_t opcode;
+    const char* name;
+};
+
+constexpr std::array<NoFormOpcode, 23> NoFormOpcodes{{
+    {false, 0x02u, "v_readfirstlane_b32"}, {false, 0x03u, "v_cvt_i32_f64"}, {false, 0x04u, "v_cvt_f64_i32"},
+    {false, 0x0fu, "v_cvt_f32_f64"}, {false, 0x10u, "v_cvt_f64_f32"}, {false, 0x15u, "v_cvt_u32_f64"},
+    {false, 0x16u, "v_cvt_f64_u32"}, {false, 0x17u, "v_trunc_f64"}, {false, 0x18u, "v_ceil_f64"},
+    {false, 0x19u, "v_rndne_f64"}, {false, 0x1au, "v_floor_f64"}, {false, 0x3cu, "v_frexp_exp_i32_f64"},
+    {false, 0x3du, "v_frexp_mant_f64"}, {false, 0x3eu, "v_fract_f64"}, {false, 0x65u, "v_swap_b32"},
+    {false, 0x68u, "v_swaprel_b32"}, {true, 0x06u, "v_mac_legacy_f32"}, {true, 0x20u, "v_madmk_f32"},
+    {true, 0x21u, "v_madak_f32"}, {true, 0x2cu, "v_fmamk_f32"}, {true, 0x2du, "v_fmaak_f32"},
+    {true, 0x37u, "v_fmamk_f16"}, {true, 0x38u, "v_fmaak_f16"},
+}};
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+auto Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code) {
+    std::vector<std::uint32_t> userData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{32, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    return ShaderRecompiler::Recompile(request);
+}
+
+void CheckRefused(AgcDriver::VulkanDevice& device, std::uint32_t word0, std::uint32_t word1, const std::string& reason, const std::string& what) {
+    alignas(256) const std::array<std::uint32_t, 4> code{word0, word1, 0x3c003c00u, 0xbf810000u};
+    std::string refusal;
+    try {
+        static_cast<void>(Compile(device, code));
+    } catch (const std::exception& error) {
+        refusal = error.what();
+    }
+    Require(refusal.find(reason) != std::string::npos, what + " " + Hex(word0) + " was not refused");
+}
+
+void CheckAccepted(AgcDriver::VulkanDevice& device, std::uint32_t word0, std::uint32_t word1) {
+    alignas(256) const std::array<std::uint32_t, 3> code{word0, word1, 0xbf810000u};
+    static_cast<void>(Compile(device, code));
+}
+
+std::uint32_t Encode(const NoFormOpcode& op, std::uint32_t form) {
+    return op.vop2 ? (op.opcode << 25u) | (10u << 17u) | (5u << 9u) | form : 0x7e000000u | (10u << 17u) | (op.opcode << 9u) | form;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        constexpr std::uint32_t Dpp16 = 0xff00b104u;
+        constexpr std::uint32_t Dpp8 = 0x00fac604u;
+        for (const auto& op : NoFormOpcodes) {
+            const std::string name = op.name;
+            CheckRefused(*device, Encode(op, 0xfau), Dpp16, "DPP modifier is not supported for opcode", name + " with DPP16");
+            CheckRefused(*device, Encode(op, 0xe9u), Dpp8, "DPP8 modifier is not supported for opcode", name + " with DPP8");
+            CheckRefused(*device, Encode(op, 0xeau), Dpp8, "DPP8 modifier is not supported for opcode", name + " with DPP8 fi:1");
+        }
+        CheckAccepted(*device, 0x7e1402fau, Dpp16);
+        CheckAccepted(*device, 0x4a140afau, Dpp16);
+        std::puts("dpp no form tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DsSwizzle.cpp

@@ -0,0 +1,289 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t MaxThreads = 64;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+constexpr std::uint32_t Checked = 16;
+alignas(256) std::array<std::uint32_t, MaxThreads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, MaxThreads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 82> Wave32Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x7e1402ff, 0xdeaddead, 0x7e1602ff,
+    0xdeaddead, 0x7e1802ff, 0xdeaddead, 0x7e1a02ff, 0xdeaddead, 0x7e1c02ff, 0xdeaddead, 0x7e1e02ff,
+    0xdeaddead, 0x7e2002ff, 0xdeaddead, 0x7e2202ff, 0xdeaddead, 0x7e2402ff, 0xdeaddead, 0x7e2602ff,
+    0xdeaddead, 0x7e2802ff, 0xdeaddead, 0x7e2a02ff, 0xdeaddead, 0x7e2c02ff, 0xdeaddead, 0x7e2e02ff,
+    0xdeaddead, 0x7e3002ff, 0xdeaddead, 0x7e3202ff, 0xdeaddead, 0xd8d4001f, 0x0a000004, 0xd8d4201f,
+    0x0b000004, 0xd8d400b8, 0x0c000004, 0xd8d47c1f, 0x0d000004, 0xd8d42493, 0x0e000004, 0xd8d4804b,
+    0x0f000004, 0xd8d4bf4b, 0x10000004, 0xd8d4c020, 0x11000004, 0xd8d4c460, 0x12000004, 0xd8d4c3e0,
+    0x13000004, 0xd8d4c4ac, 0x14000004, 0xd8d4d8e0, 0x15000004, 0xbefe03ff, 0x33cc0ff3, 0xd8d4201f,
+    0x16000004, 0xd8d4804b, 0x17000004, 0xd8d4c460, 0x18000004, 0xd8d400b8, 0x19000004, 0xbf8cc07f,
+    0xbefe03c1, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xe0781020, 0x80011203, 0xe0781030,
+    0x80011603, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 84> Wave64Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x7e1402ff, 0xdeaddead, 0x7e1602ff,
+    0xdeaddead, 0x7e1802ff, 0xdeaddead, 0x7e1a02ff, 0xdeaddead, 0x7e1c02ff, 0xdeaddead, 0x7e1e02ff,
+    0xdeaddead, 0x7e2002ff, 0xdeaddead, 0x7e2202ff, 0xdeaddead, 0x7e2402ff, 0xdeaddead, 0x7e2602ff,
+    0xdeaddead, 0x7e2802ff, 0xdeaddead, 0x7e2a02ff, 0xdeaddead, 0x7e2c02ff, 0xdeaddead, 0x7e2e02ff,
+    0xdeaddead, 0x7e3002ff, 0xdeaddead, 0x7e3202ff, 0xdeaddead, 0xd8d4001f, 0x0a000004, 0xd8d4201f,
+    0x0b000004, 0xd8d400b8, 0x0c000004, 0xd8d47c1f, 0x0d000004, 0xd8d42493, 0x0e000004, 0xd8d4804b,
+    0x0f000004, 0xd8d4bf4b, 0x10000004, 0xd8d4c020, 0x11000004, 0xd8d4c460, 0x12000004, 0xd8d4c3e0,
+    0x13000004, 0xd8d4c4ac, 0x14000004, 0xd8d4d8e0, 0x15000004, 0xbefe03ff, 0x33cc0ff3, 0xbeff03ff,
+    0xc3a5960f, 0xd8d4201f, 0x16000004, 0xd8d4804b, 0x17000004, 0xd8d4c460, 0x18000004, 0xd8d400b8,
+    0x19000004, 0xbf8cc07f, 0xbefe04c1, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xe0781020,
+    0x80011203, 0xe0781030, 0x80011603, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[64][4] = {
+    {0x1e7ea419u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x51c9bc70u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x80a4df5au, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xf38b2ffcu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x8306d03bu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xa5aec797u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xdc28ff90u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xf3f49249u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x1a466884u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xe255accbu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x39292d22u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xe5121482u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x99dd251du, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x9f199504u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x8e7aa6e9u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x6bad6be2u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xc88b2875u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x9293de8fu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x8c3d5f16u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xd7a7a3ccu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xbb049a79u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xc6cd75e9u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xc4a334bfu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x7dabe929u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xc0433cbdu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xc5e818fau, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x96263ae6u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x70eb9a0au, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x3d6c51e3u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00a61f93u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x9d3c7decu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x14aa4e71u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x1c593af5u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x49889310u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xd11745adu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x1919e93au, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x731c9452u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x02f0ee99u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xd0944996u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xe4163207u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xaf67d461u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x7d836e77u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xadec6f11u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x5071950eu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x35d14880u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x65b00a2du, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x406329bcu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x59001ac9u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xfadb8908u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xffb88309u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xede7bd0cu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x5b4c8012u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xce1d62e0u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x60581ccau, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xffe976abu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xbef7e268u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x837a7935u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xa341738cu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x1389b372u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xb91dddd9u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x57241955u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x16f44881u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x8eb5140fu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x89be9c1cu, 0x00000000u, 0x00000000u, 0x00000000u}
+};
+
+constexpr std::uint32_t Expected32[32][16] = {
+    {0x1e7ea419u, 0x1a466884u, 0xa5aec797u, 0x14aa4e71u, 0x9f199504u, 0xf38b2ffcu, 0xf38b2ffcu, 0x51c9bc70u, 0x00a61f93u, 0x14aa4e71u, 0xd7a7a3ccu, 0xf3f49249u, 0x1a466884u, 0x00000000u, 0x00a61f93u, 0xa5aec797u},
+    {0x51c9bc70u, 0xe255accbu, 0xa5aec797u, 0x9d3c7decu, 0x99dd251du, 0x80a4df5au, 0x80a4df5au, 0x80a4df5au, 0x9d3c7decu, 0x1e7ea419u, 0xc88b2875u, 0x1a466884u, 0xe255accbu, 0x00000000u, 0x00000000u, 0xa5aec797u},
+    {0x80a4df5au, 0x39292d22u, 0xa5aec797u, 0x00a61f93u, 0x6bad6be2u, 0x1e7ea419u, 0x1e7ea419u, 0xf38b2ffcu, 0x14aa4e71u, 0x51c9bc70u, 0x9293de8fu, 0xe255accbu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xf38b2ffcu, 0xe5121482u, 0xa5aec797u, 0x3d6c51e3u, 0x8e7aa6e9u, 0x51c9bc70u, 0x51c9bc70u, 0x8306d03bu, 0x1e7ea419u, 0x80a4df5au, 0x8c3d5f16u, 0x39292d22u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x8306d03bu, 0x99dd251du, 0xa5aec797u, 0x70eb9a0au, 0x9f199504u, 0xf3f49249u, 0xf3f49249u, 0xa5aec797u, 0x51c9bc70u, 0xf38b2ffcu, 0x7dabe929u, 0xe5121482u, 0x00000000u, 0xf3f49249u, 0x51c9bc70u, 0xa5aec797u},
+    {0xa5aec797u, 0x9f199504u, 0xa5aec797u, 0x96263ae6u, 0x99dd251du, 0xdc28ff90u, 0xdc28ff90u, 0xdc28ff90u, 0x80a4df5au, 0x8306d03bu, 0x8306d03bu, 0x99dd251du, 0x00000000u, 0xdc28ff90u, 0x00000000u, 0xa5aec797u},
+    {0xdc28ff90u, 0x8e7aa6e9u, 0xa5aec797u, 0xc5e818fau, 0x6bad6be2u, 0x8306d03bu, 0x8306d03bu, 0xf3f49249u, 0xf38b2ffcu, 0xa5aec797u, 0xa5aec797u, 0x9f199504u, 0x00000000u, 0x8306d03bu, 0x00000000u, 0xa5aec797u},
+    {0xf3f49249u, 0x6bad6be2u, 0xa5aec797u, 0xc0433cbdu, 0x8e7aa6e9u, 0xa5aec797u, 0xa5aec797u, 0x1a466884u, 0x8306d03bu, 0xdc28ff90u, 0xdc28ff90u, 0x8e7aa6e9u, 0x00000000u, 0xa5aec797u, 0x8306d03bu, 0xa5aec797u},
+    {0x1a466884u, 0x1e7ea419u, 0x9f199504u, 0x7dabe929u, 0x9f199504u, 0xe5121482u, 0xe5121482u, 0xe255accbu, 0xa5aec797u, 0xf3f49249u, 0xe5121482u, 0x6bad6be2u, 0x1e7ea419u, 0xe5121482u, 0xa5aec797u, 0x00000000u},
+    {0xe255accbu, 0x51c9bc70u, 0x9f199504u, 0xc4a334bfu, 0x99dd251du, 0x39292d22u, 0x39292d22u, 0x39292d22u, 0xdc28ff90u, 0x1a466884u, 0x1a466884u, 0xc88b2875u, 0x51c9bc70u, 0x39292d22u, 0xdc28ff90u, 0x00000000u},
+    {0x39292d22u, 0x80a4df5au, 0x9f199504u, 0xc6cd75e9u, 0x6bad6be2u, 0x1a466884u, 0x1a466884u, 0xe5121482u, 0xf3f49249u, 0xe255accbu, 0xe255accbu, 0x9293de8fu, 0x00000000u, 0x1a466884u, 0xf3f49249u, 0x00000000u},
+    {0xe5121482u, 0xf38b2ffcu, 0x9f199504u, 0xbb049a79u, 0x8e7aa6e9u, 0xe255accbu, 0xe255accbu, 0x99dd251du, 0x1a466884u, 0x39292d22u, 0x39292d22u, 0x8c3d5f16u, 0x00000000u, 0xe255accbu, 0x1a466884u, 0x00000000u},
+    {0x99dd251du, 0x8306d03bu, 0x9f199504u, 0xd7a7a3ccu, 0x9f199504u, 0x6bad6be2u, 0x6bad6be2u, 0x9f199504u, 0xe255accbu, 0xe5121482u, 0x6bad6be2u, 0xd7a7a3ccu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x9f199504u, 0xa5aec797u, 0x9f199504u, 0x8c3d5f16u, 0x99dd251du, 0x8e7aa6e9u, 0x8e7aa6e9u, 0x8e7aa6e9u, 0x39292d22u, 0x99dd251du, 0x99dd251du, 0xbb049a79u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x8e7aa6e9u, 0xdc28ff90u, 0x9f199504u, 0x9293de8fu, 0x6bad6be2u, 0x99dd251du, 0x99dd251du, 0x6bad6be2u, 0xe5121482u, 0x9f199504u, 0x9f199504u, 0xc6cd75e9u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x6bad6be2u, 0xf3f49249u, 0x9f199504u, 0xc88b2875u, 0x8e7aa6e9u, 0x9f199504u, 0x9f199504u, 0xc88b2875u, 0x99dd251du, 0x8e7aa6e9u, 0x8e7aa6e9u, 0xc4a334bfu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xc88b2875u, 0xc0433cbdu, 0xc6cd75e9u, 0x6bad6be2u, 0x00a61f93u, 0xd7a7a3ccu, 0xd7a7a3ccu, 0x9293de8fu, 0x9f199504u, 0x6bad6be2u, 0xf38b2ffcu, 0x7dabe929u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x9293de8fu, 0xc5e818fau, 0xc6cd75e9u, 0x8e7aa6e9u, 0x3d6c51e3u, 0x8c3d5f16u, 0x8c3d5f16u, 0x8c3d5f16u, 0x8e7aa6e9u, 0xc88b2875u, 0x1e7ea419u, 0xc0433cbdu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x8c3d5f16u, 0x96263ae6u, 0xc6cd75e9u, 0x9f199504u, 0x14aa4e71u, 0xc88b2875u, 0xc88b2875u, 0xd7a7a3ccu, 0x6bad6be2u, 0x9293de8fu, 0x51c9bc70u, 0xc5e818fau, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xd7a7a3ccu, 0x70eb9a0au, 0xc6cd75e9u, 0x99dd251du, 0x9d3c7decu, 0x9293de8fu, 0x9293de8fu, 0xbb049a79u, 0xc88b2875u, 0x8c3d5f16u, 0x80a4df5au, 0x96263ae6u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xbb049a79u, 0x3d6c51e3u, 0xc6cd75e9u, 0xe5121482u, 0x00a61f93u, 0x7dabe929u, 0x7dabe929u, 0xc6cd75e9u, 0x9293de8fu, 0xd7a7a3ccu, 0xf3f49249u, 0x70eb9a0au, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xc6cd75e9u, 0x00a61f93u, 0xc6cd75e9u, 0x39292d22u, 0x3d6c51e3u, 0xc4a334bfu, 0xc4a334bfu, 0xc4a334bfu, 0x8c3d5f16u, 0xbb049a79u, 0xbb049a79u, 0x3d6c51e3u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xc4a334bfu, 0x9d3c7decu, 0xc6cd75e9u, 0xe255accbu, 0x14aa4e71u, 0xbb049a79u, 0xbb049a79u, 0x7dabe929u, 0xd7a7a3ccu, 0xc6cd75e9u, 0xc6cd75e9u, 0x00a61f93u, 0x00000000u, 0x00000000u, 0xd7a7a3ccu, 0x00000000u},
+    {0x7dabe929u, 0x14aa4e71u, 0xc6cd75e9u, 0x1a466884u, 0x9d3c7decu, 0xc6cd75e9u, 0xc6cd75e9u, 0xc0433cbdu, 0xbb049a79u, 0xc4a334bfu, 0xc4a334bfu, 0x9d3c7decu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xc0433cbdu, 0xc88b2875u, 0x00a61f93u, 0xf3f49249u, 0x00a61f93u, 0x70eb9a0au, 0x70eb9a0au, 0xc5e818fau, 0xc6cd75e9u, 0x7dabe929u, 0x70eb9a0au, 0x14aa4e71u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00a61f93u},
+    {0xc5e818fau, 0x9293de8fu, 0x00a61f93u, 0xdc28ff90u, 0x3d6c51e3u, 0x96263ae6u, 0x96263ae6u, 0x96263ae6u, 0xc4a334bfu, 0xc0433cbdu, 0xc0433cbdu, 0x1e7ea419u, 0x00000000u, 0x00000000u, 0xc4a334bfu, 0x00a61f93u},
+    {0x96263ae6u, 0x8c3d5f16u, 0x00a61f93u, 0xa5aec797u, 0x14aa4e71u, 0xc0433cbdu, 0xc0433cbdu, 0x70eb9a0au, 0x7dabe929u, 0xc5e818fau, 0xc5e818fau, 0x51c9bc70u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x70eb9a0au, 0xd7a7a3ccu, 0x00a61f93u, 0x8306d03bu, 0x9d3c7decu, 0xc5e818fau, 0xc5e818fau, 0x3d6c51e3u, 0xc0433cbdu, 0x96263ae6u, 0x96263ae6u, 0x80a4df5au, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x3d6c51e3u, 0xbb049a79u, 0x00a61f93u, 0xf38b2ffcu, 0x00a61f93u, 0x14aa4e71u, 0x14aa4e71u, 0x00a61f93u, 0xc5e818fau, 0x70eb9a0au, 0x14aa4e71u, 0xf38b2ffcu, 0x00000000u, 0x00000000u, 0xc5e818fau, 0x00a61f93u},
+    {0x00a61f93u, 0xc6cd75e9u, 0x00a61f93u, 0x80a4df5au, 0x3d6c51e3u, 0x9d3c7decu, 0x9d3c7decu, 0x9d3c7decu, 0x96263ae6u, 0x3d6c51e3u, 0x3d6c51e3u, 0x8306d03bu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00a61f93u},
+    {0x9d3c7decu, 0xc4a334bfu, 0x00a61f93u, 0x51c9bc70u, 0x14aa4e71u, 0x3d6c51e3u, 0x3d6c51e3u, 0x14aa4e71u, 0x70eb9a0au, 0x00a61f93u, 0x00a61f93u, 0xa5aec797u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x14aa4e71u, 0x7dabe929u, 0x00a61f93u, 0x1e7ea419u, 0x9d3c7decu, 0x00a61f93u, 0x00a61f93u, 0x1e7ea419u, 0x3d6c51e3u, 0x9d3c7decu, 0x9d3c7decu, 0xdc28ff90u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu}
+};
+constexpr std::uint32_t Expected64[64][16] = {
+    {0x1e7ea419u, 0x1a466884u, 0xa5aec797u, 0x14aa4e71u, 0x9f199504u, 0xf38b2ffcu, 0xf38b2ffcu, 0x51c9bc70u, 0x00a61f93u, 0x14aa4e71u, 0xd7a7a3ccu, 0xf3f49249u, 0x1a466884u, 0x00000000u, 0x00a61f93u, 0xa5aec797u},
+    {0x51c9bc70u, 0xe255accbu, 0xa5aec797u, 0x9d3c7decu, 0x99dd251du, 0x80a4df5au, 0x80a4df5au, 0x80a4df5au, 0x9d3c7decu, 0x1e7ea419u, 0xc88b2875u, 0x1a466884u, 0xe255accbu, 0x00000000u, 0x00000000u, 0xa5aec797u},
+    {0x80a4df5au, 0x39292d22u, 0xa5aec797u, 0x00a61f93u, 0x6bad6be2u, 0x1e7ea419u, 0x1e7ea419u, 0xf38b2ffcu, 0x14aa4e71u, 0x51c9bc70u, 0x9293de8fu, 0xe255accbu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xf38b2ffcu, 0xe5121482u, 0xa5aec797u, 0x3d6c51e3u, 0x8e7aa6e9u, 0x51c9bc70u, 0x51c9bc70u, 0x8306d03bu, 0x1e7ea419u, 0x80a4df5au, 0x8c3d5f16u, 0x39292d22u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x8306d03bu, 0x99dd251du, 0xa5aec797u, 0x70eb9a0au, 0x9f199504u, 0xf3f49249u, 0xf3f49249u, 0xa5aec797u, 0x51c9bc70u, 0xf38b2ffcu, 0x7dabe929u, 0xe5121482u, 0x00000000u, 0xf3f49249u, 0x51c9bc70u, 0xa5aec797u},
+    {0xa5aec797u, 0x9f199504u, 0xa5aec797u, 0x96263ae6u, 0x99dd251du, 0xdc28ff90u, 0xdc28ff90u, 0xdc28ff90u, 0x80a4df5au, 0x8306d03bu, 0x8306d03bu, 0x99dd251du, 0x00000000u, 0xdc28ff90u, 0x00000000u, 0xa5aec797u},
+    {0xdc28ff90u, 0x8e7aa6e9u, 0xa5aec797u, 0xc5e818fau, 0x6bad6be2u, 0x8306d03bu, 0x8306d03bu, 0xf3f49249u, 0xf38b2ffcu, 0xa5aec797u, 0xa5aec797u, 0x9f199504u, 0x00000000u, 0x8306d03bu, 0x00000000u, 0xa5aec797u},
+    {0xf3f49249u, 0x6bad6be2u, 0xa5aec797u, 0xc0433cbdu, 0x8e7aa6e9u, 0xa5aec797u, 0xa5aec797u, 0x1a466884u, 0x8306d03bu, 0xdc28ff90u, 0xdc28ff90u, 0x8e7aa6e9u, 0x00000000u, 0xa5aec797u, 0x8306d03bu, 0xa5aec797u},
+    {0x1a466884u, 0x1e7ea419u, 0x9f199504u, 0x7dabe929u, 0x9f199504u, 0xe5121482u, 0xe5121482u, 0xe255accbu, 0xa5aec797u, 0xf3f49249u, 0xe5121482u, 0x6bad6be2u, 0x1e7ea419u, 0xe5121482u, 0xa5aec797u, 0x00000000u},
+    {0xe255accbu, 0x51c9bc70u, 0x9f199504u, 0xc4a334bfu, 0x99dd251du, 0x39292d22u, 0x39292d22u, 0x39292d22u, 0xdc28ff90u, 0x1a466884u, 0x1a466884u, 0xc88b2875u, 0x51c9bc70u, 0x39292d22u, 0xdc28ff90u, 0x00000000u},
+    {0x39292d22u, 0x80a4df5au, 0x9f199504u, 0xc6cd75e9u, 0x6bad6be2u, 0x1a466884u, 0x1a466884u, 0xe5121482u, 0xf3f49249u, 0xe255accbu, 0xe255accbu, 0x9293de8fu, 0x00000000u, 0x1a466884u, 0xf3f49249u, 0x00000000u},
+    {0xe5121482u, 0xf38b2ffcu, 0x9f199504u, 0xbb049a79u, 0x8e7aa6e9u, 0xe255accbu, 0xe255accbu, 0x99dd251du, 0x1a466884u, 0x39292d22u, 0x39292d22u, 0x8c3d5f16u, 0x00000000u, 0xe255accbu, 0x1a466884u, 0x00000000u},
+    {0x99dd251du, 0x8306d03bu, 0x9f199504u, 0xd7a7a3ccu, 0x9f199504u, 0x6bad6be2u, 0x6bad6be2u, 0x9f199504u, 0xe255accbu, 0xe5121482u, 0x6bad6be2u, 0xd7a7a3ccu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x9f199504u, 0xa5aec797u, 0x9f199504u, 0x8c3d5f16u, 0x99dd251du, 0x8e7aa6e9u, 0x8e7aa6e9u, 0x8e7aa6e9u, 0x39292d22u, 0x99dd251du, 0x99dd251du, 0xbb049a79u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x8e7aa6e9u, 0xdc28ff90u, 0x9f199504u, 0x9293de8fu, 0x6bad6be2u, 0x99dd251du, 0x99dd251du, 0x6bad6be2u, 0xe5121482u, 0x9f199504u, 0x9f199504u, 0xc6cd75e9u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x6bad6be2u, 0xf3f49249u, 0x9f199504u, 0xc88b2875u, 0x8e7aa6e9u, 0x9f199504u, 0x9f199504u, 0xc88b2875u, 0x99dd251du, 0x8e7aa6e9u, 0x8e7aa6e9u, 0xc4a334bfu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xc88b2875u, 0xc0433cbdu, 0xc6cd75e9u, 0x6bad6be2u, 0x00a61f93u, 0xd7a7a3ccu, 0xd7a7a3ccu, 0x9293de8fu, 0x9f199504u, 0x6bad6be2u, 0xf38b2ffcu, 0x7dabe929u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x9293de8fu, 0xc5e818fau, 0xc6cd75e9u, 0x8e7aa6e9u, 0x3d6c51e3u, 0x8c3d5f16u, 0x8c3d5f16u, 0x8c3d5f16u, 0x8e7aa6e9u, 0xc88b2875u, 0x1e7ea419u, 0xc0433cbdu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x8c3d5f16u, 0x96263ae6u, 0xc6cd75e9u, 0x9f199504u, 0x14aa4e71u, 0xc88b2875u, 0xc88b2875u, 0xd7a7a3ccu, 0x6bad6be2u, 0x9293de8fu, 0x51c9bc70u, 0xc5e818fau, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xd7a7a3ccu, 0x70eb9a0au, 0xc6cd75e9u, 0x99dd251du, 0x9d3c7decu, 0x9293de8fu, 0x9293de8fu, 0xbb049a79u, 0xc88b2875u, 0x8c3d5f16u, 0x80a4df5au, 0x96263ae6u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xbb049a79u, 0x3d6c51e3u, 0xc6cd75e9u, 0xe5121482u, 0x00a61f93u, 0x7dabe929u, 0x7dabe929u, 0xc6cd75e9u, 0x9293de8fu, 0xd7a7a3ccu, 0xf3f49249u, 0x70eb9a0au, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xc6cd75e9u, 0x00a61f93u, 0xc6cd75e9u, 0x39292d22u, 0x3d6c51e3u, 0xc4a334bfu, 0xc4a334bfu, 0xc4a334bfu, 0x8c3d5f16u, 0xbb049a79u, 0xbb049a79u, 0x3d6c51e3u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xc4a334bfu, 0x9d3c7decu, 0xc6cd75e9u, 0xe255accbu, 0x14aa4e71u, 0xbb049a79u, 0xbb049a79u, 0x7dabe929u, 0xd7a7a3ccu, 0xc6cd75e9u, 0xc6cd75e9u, 0x00a61f93u, 0x00000000u, 0x00000000u, 0xd7a7a3ccu, 0x00000000u},
+    {0x7dabe929u, 0x14aa4e71u, 0xc6cd75e9u, 0x1a466884u, 0x9d3c7decu, 0xc6cd75e9u, 0xc6cd75e9u, 0xc0433cbdu, 0xbb049a79u, 0xc4a334bfu, 0xc4a334bfu, 0x9d3c7decu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xc0433cbdu, 0xc88b2875u, 0x00a61f93u, 0xf3f49249u, 0x00a61f93u, 0x70eb9a0au, 0x70eb9a0au, 0xc5e818fau, 0xc6cd75e9u, 0x7dabe929u, 0x70eb9a0au, 0x14aa4e71u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00a61f93u},
+    {0xc5e818fau, 0x9293de8fu, 0x00a61f93u, 0xdc28ff90u, 0x3d6c51e3u, 0x96263ae6u, 0x96263ae6u, 0x96263ae6u, 0xc4a334bfu, 0xc0433cbdu, 0xc0433cbdu, 0x1e7ea419u, 0x00000000u, 0x00000000u, 0xc4a334bfu, 0x00a61f93u},
+    {0x96263ae6u, 0x8c3d5f16u, 0x00a61f93u, 0xa5aec797u, 0x14aa4e71u, 0xc0433cbdu, 0xc0433cbdu, 0x70eb9a0au, 0x7dabe929u, 0xc5e818fau, 0xc5e818fau, 0x51c9bc70u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x70eb9a0au, 0xd7a7a3ccu, 0x00a61f93u, 0x8306d03bu, 0x9d3c7decu, 0xc5e818fau, 0xc5e818fau, 0x3d6c51e3u, 0xc0433cbdu, 0x96263ae6u, 0x96263ae6u, 0x80a4df5au, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x3d6c51e3u, 0xbb049a79u, 0x00a61f93u, 0xf38b2ffcu, 0x00a61f93u, 0x14aa4e71u, 0x14aa4e71u, 0x00a61f93u, 0xc5e818fau, 0x70eb9a0au, 0x14aa4e71u, 0xf38b2ffcu, 0x00000000u, 0x00000000u, 0xc5e818fau, 0x00a61f93u},
+    {0x00a61f93u, 0xc6cd75e9u, 0x00a61f93u, 0x80a4df5au, 0x3d6c51e3u, 0x9d3c7decu, 0x9d3c7decu, 0x9d3c7decu, 0x96263ae6u, 0x3d6c51e3u, 0x3d6c51e3u, 0x8306d03bu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00a61f93u},
+    {0x9d3c7decu, 0xc4a334bfu, 0x00a61f93u, 0x51c9bc70u, 0x14aa4e71u, 0x3d6c51e3u, 0x3d6c51e3u, 0x14aa4e71u, 0x70eb9a0au, 0x00a61f93u, 0x00a61f93u, 0xa5aec797u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x14aa4e71u, 0x7dabe929u, 0x00a61f93u, 0x1e7ea419u, 0x9d3c7decu, 0x00a61f93u, 0x00a61f93u, 0x1e7ea419u, 0x3d6c51e3u, 0x9d3c7decu, 0x9d3c7decu, 0xdc28ff90u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x1c593af5u, 0xaf67d461u, 0x02f0ee99u, 0x89be9c1cu, 0x65b00a2du, 0x1919e93au, 0x1919e93au, 0x49889310u, 0x16f44881u, 0x89be9c1cu, 0x5b4c8012u, 0xe4163207u, 0x00000000u, 0x1919e93au, 0x00000000u, 0x00000000u},
+    {0x49889310u, 0x7d836e77u, 0x02f0ee99u, 0x8eb5140fu, 0x35d14880u, 0xd11745adu, 0xd11745adu, 0xd11745adu, 0x8eb5140fu, 0x1c593af5u, 0xfadb8908u, 0xaf67d461u, 0x7d836e77u, 0xd11745adu, 0x8eb5140fu, 0x00000000u},
+    {0xd11745adu, 0xadec6f11u, 0x02f0ee99u, 0x16f44881u, 0x59001ac9u, 0x1c593af5u, 0x1c593af5u, 0x1919e93au, 0x89be9c1cu, 0x49889310u, 0xffb88309u, 0x7d836e77u, 0xadec6f11u, 0x1c593af5u, 0x89be9c1cu, 0x00000000u},
+    {0x1919e93au, 0x5071950eu, 0x02f0ee99u, 0x57241955u, 0x406329bcu, 0x49889310u, 0x49889310u, 0x731c9452u, 0x1c593af5u, 0xd11745adu, 0xede7bd0cu, 0xadec6f11u, 0x00000000u, 0x49889310u, 0x1c593af5u, 0x00000000u},
+    {0x731c9452u, 0x35d14880u, 0x02f0ee99u, 0xb91dddd9u, 0x65b00a2du, 0xe4163207u, 0xe4163207u, 0x02f0ee99u, 0x49889310u, 0x1919e93au, 0xbef7e268u, 0x5071950eu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x02f0ee99u, 0x65b00a2du, 0x02f0ee99u, 0x1389b372u, 0x35d14880u, 0xd0944996u, 0xd0944996u, 0xd0944996u, 0xd11745adu, 0x731c9452u, 0x731c9452u, 0x35d14880u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xd0944996u, 0x406329bcu, 0x02f0ee99u, 0xa341738cu, 0x59001ac9u, 0x731c9452u, 0x731c9452u, 0xe4163207u, 0x1919e93au, 0x02f0ee99u, 0x02f0ee99u, 0x65b00a2du, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xe4163207u, 0x59001ac9u, 0x02f0ee99u, 0x837a7935u, 0x406329bcu, 0x02f0ee99u, 0x02f0ee99u, 0xaf67d461u, 0x731c9452u, 0xd0944996u, 0xd0944996u, 0x406329bcu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xaf67d461u, 0x1c593af5u, 0x65b00a2du, 0xbef7e268u, 0x65b00a2du, 0x5071950eu, 0x5071950eu, 0x7d836e77u, 0x02f0ee99u, 0xe4163207u, 0x5071950eu, 0x59001ac9u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x7d836e77u, 0x49889310u, 0x65b00a2du, 0xffe976abu, 0x35d14880u, 0xadec6f11u, 0xadec6f11u, 0xadec6f11u, 0xd0944996u, 0xaf67d461u, 0xaf67d461u, 0xfadb8908u, 0x49889310u, 0xadec6f11u, 0x00000000u, 0x00000000u},
+    {0xadec6f11u, 0xd11745adu, 0x65b00a2du, 0x60581ccau, 0x59001ac9u, 0xaf67d461u, 0xaf67d461u, 0x5071950eu, 0xe4163207u, 0x7d836e77u, 0x7d836e77u, 0xffb88309u, 0xd11745adu, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x5071950eu, 0x1919e93au, 0x65b00a2du, 0xce1d62e0u, 0x406329bcu, 0x7d836e77u, 0x7d836e77u, 0x35d14880u, 0xaf67d461u, 0xadec6f11u, 0xadec6f11u, 0xede7bd0cu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x35d14880u, 0x731c9452u, 0x65b00a2du, 0x5b4c8012u, 0x65b00a2du, 0x59001ac9u, 0x59001ac9u, 0x65b00a2du, 0x7d836e77u, 0x5071950eu, 0x59001ac9u, 0x5b4c8012u, 0x00000000u, 0x59001ac9u, 0x7d836e77u, 0x00000000u},
+    {0x65b00a2du, 0x02f0ee99u, 0x65b00a2du, 0xede7bd0cu, 0x35d14880u, 0x406329bcu, 0x406329bcu, 0x406329bcu, 0xadec6f11u, 0x35d14880u, 0x35d14880u, 0xce1d62e0u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x406329bcu, 0xd0944996u, 0x65b00a2du, 0xffb88309u, 0x59001ac9u, 0x35d14880u, 0x35d14880u, 0x59001ac9u, 0x5071950eu, 0x65b00a2du, 0x65b00a2du, 0x60581ccau, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x59001ac9u, 0xe4163207u, 0x65b00a2du, 0xfadb8908u, 0x406329bcu, 0x65b00a2du, 0x65b00a2du, 0xfadb8908u, 0x35d14880u, 0x406329bcu, 0x406329bcu, 0xffe976abu, 0x00000000u, 0x00000000u, 0x35d14880u, 0x00000000u},
+    {0xfadb8908u, 0x837a7935u, 0x60581ccau, 0x59001ac9u, 0x16f44881u, 0x5b4c8012u, 0x5b4c8012u, 0xffb88309u, 0x65b00a2du, 0x59001ac9u, 0x1919e93au, 0xbef7e268u, 0x837a7935u, 0x00000000u, 0x00000000u, 0x60581ccau},
+    {0xffb88309u, 0xa341738cu, 0x60581ccau, 0x406329bcu, 0x57241955u, 0xede7bd0cu, 0xede7bd0cu, 0xede7bd0cu, 0x406329bcu, 0xfadb8908u, 0x1c593af5u, 0x837a7935u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xede7bd0cu, 0x1389b372u, 0x60581ccau, 0x65b00a2du, 0x89be9c1cu, 0xfadb8908u, 0xfadb8908u, 0x5b4c8012u, 0x59001ac9u, 0xffb88309u, 0x49889310u, 0xa341738cu, 0x00000000u, 0xfadb8908u, 0x59001ac9u, 0x60581ccau},
+    {0x5b4c8012u, 0xb91dddd9u, 0x60581ccau, 0x35d14880u, 0x8eb5140fu, 0xffb88309u, 0xffb88309u, 0xce1d62e0u, 0xfadb8908u, 0xede7bd0cu, 0xd11745adu, 0x1389b372u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xce1d62e0u, 0x57241955u, 0x60581ccau, 0x5071950eu, 0x16f44881u, 0xbef7e268u, 0xbef7e268u, 0x60581ccau, 0xffb88309u, 0x5b4c8012u, 0xe4163207u, 0xb91dddd9u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x60581ccau, 0x16f44881u, 0x60581ccau, 0xadec6f11u, 0x57241955u, 0xffe976abu, 0xffe976abu, 0xffe976abu, 0xede7bd0cu, 0xce1d62e0u, 0xce1d62e0u, 0x57241955u, 0x00000000u, 0x00000000u, 0xede7bd0cu, 0x60581ccau},
+    {0xffe976abu, 0x8eb5140fu, 0x60581ccau, 0x7d836e77u, 0x89be9c1cu, 0xce1d62e0u, 0xce1d62e0u, 0xbef7e268u, 0x5b4c8012u, 0x60581ccau, 0x60581ccau, 0x16f44881u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xbef7e268u, 0x89be9c1cu, 0x60581ccau, 0xaf67d461u, 0x8eb5140fu, 0x60581ccau, 0x60581ccau, 0x837a7935u, 0xce1d62e0u, 0xffe976abu, 0xffe976abu, 0x8eb5140fu, 0x89be9c1cu, 0x60581ccau, 0x00000000u, 0x60581ccau},
+    {0x837a7935u, 0xfadb8908u, 0x16f44881u, 0xe4163207u, 0x16f44881u, 0xb91dddd9u, 0xb91dddd9u, 0xa341738cu, 0x60581ccau, 0xbef7e268u, 0xb91dddd9u, 0x89be9c1cu, 0xfadb8908u, 0x00000000u, 0x60581ccau, 0x00000000u},
+    {0xa341738cu, 0xffb88309u, 0x16f44881u, 0xd0944996u, 0x57241955u, 0x1389b372u, 0x1389b372u, 0x1389b372u, 0xffe976abu, 0x837a7935u, 0x837a7935u, 0x1c593af5u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x1389b372u, 0xede7bd0cu, 0x16f44881u, 0x02f0ee99u, 0x89be9c1cu, 0x837a7935u, 0x837a7935u, 0xb91dddd9u, 0xbef7e268u, 0xa341738cu, 0xa341738cu, 0x49889310u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0xb91dddd9u, 0x5b4c8012u, 0x16f44881u, 0x731c9452u, 0x8eb5140fu, 0xa341738cu, 0xa341738cu, 0x57241955u, 0x837a7935u, 0x1389b372u, 0x1389b372u, 0xd11745adu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x57241955u, 0xce1d62e0u, 0x16f44881u, 0x1919e93au, 0x16f44881u, 0x89be9c1cu, 0x89be9c1cu, 0x16f44881u, 0xa341738cu, 0xb91dddd9u, 0x89be9c1cu, 0x1919e93au, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x16f44881u, 0x60581ccau, 0x16f44881u, 0xd11745adu, 0x57241955u, 0x8eb5140fu, 0x8eb5140fu, 0x8eb5140fu, 0x1389b372u, 0x57241955u, 0x57241955u, 0x731c9452u, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu, 0xdeaddeadu},
+    {0x8eb5140fu, 0xffe976abu, 0x16f44881u, 0x49889310u, 0x89be9c1cu, 0x57241955u, 0x57241955u, 0x89be9c1cu, 0xb91dddd9u, 0x16f44881u, 0x16f44881u, 0x02f0ee99u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x89be9c1cu, 0xbef7e268u, 0x16f44881u, 0x1c593af5u, 0x8eb5140fu, 0x16f44881u, 0x16f44881u, 0x1c593af5u, 0x57241955u, 0x8eb5140fu, 0x8eb5140fu, 0xd0944996u, 0xbef7e268u, 0x00000000u, 0x00000000u, 0x00000000u}
+};
+
+constexpr const char* Names[Checked] = {"bitmask_identity", "bitmask_swap8", "bitmask_broadcast", "bitmask_reverse", "bitmask_mixed", "quad_perm", "quad_perm_high_bits", "rotate_left_1", "rotate_right_3", "rotate_left_31", "rotate_masked", "rotate_high_bits", "inactive_swap8", "inactive_quad_perm", "inactive_rotate", "inactive_broadcast"};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target) {
+    for (std::uint32_t tid = 0; tid < MaxThreads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{waveSize, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        target,
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+template<std::size_t Lanes>
+void Check(const std::uint32_t (&expected)[Lanes][Checked]) {
+    for (std::uint32_t tid = 0; tid < Lanes; ++tid) {
+        for (std::uint32_t index = 0; index < Checked; ++index) {
+            const auto actual = Output[tid * Results + index];
+            Require(actual == expected[tid][index], "ds_swizzle wave" + std::to_string(Lanes) + ": lane " + std::to_string(tid) + " " + Names[index] + " is " + Hex(actual) + ", expected " + Hex(expected[tid][index]));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (device->Target().subgroupSize < 32) {
+            std::printf("skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize);
+            return VulkanTestSkipped;
+        }
+        Run(*device, Wave32Code, 32, device->ComputeTarget(32));
+        Check(Expected32);
+        Run(*device, Wave64Code, 64, device->Target());
+        Check(Expected64);
+        Run(*device, Wave64Code, 64, device->ComputeTarget(32));
+        Check(Expected64);
+        std::puts("ds_swizzle tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/F32MinMaxNan.cpp

@@ -0,0 +1,372 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 22> Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x1e140b04, 0x20160b04, 0xd551000c,
+    0x041a0b04, 0xd554000d, 0x041a0b04, 0xd557000e, 0x041a0b04, 0xd50f000f, 0x02020b04, 0xd5100010,
+    0x02020b04, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[128][4] = {
+    {0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u},
+    {0x00000000u, 0x80000000u, 0x7f800001u, 0x00000000u},
+    {0x00000000u, 0x3f800000u, 0xff800000u, 0x00000000u},
+    {0x00000000u, 0xc0400000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x7f800000u, 0xff812345u, 0x00000000u},
+    {0x00000000u, 0xff800000u, 0x7fc12345u, 0x00000000u},
+    {0x00000000u, 0x7fc12345u, 0xc0400000u, 0x00000000u},
+    {0x00000000u, 0xffd54321u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x7f800001u, 0xffd54321u, 0x00000000u},
+    {0x00000000u, 0xff812345u, 0x7f800000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0x7f800000u, 0x00000000u},
+    {0x80000000u, 0x80000000u, 0x80000000u, 0x00000000u},
+    {0x80000000u, 0x3f800000u, 0x7f800001u, 0x00000000u},
+    {0x80000000u, 0xc0400000u, 0xff800000u, 0x00000000u},
+    {0x80000000u, 0x7f800000u, 0x3f800000u, 0x00000000u},
+    {0x80000000u, 0xff800000u, 0xff812345u, 0x00000000u},
+    {0x80000000u, 0x7fc12345u, 0x7fc12345u, 0x00000000u},
+    {0x80000000u, 0xffd54321u, 0xc0400000u, 0x00000000u},
+    {0x80000000u, 0x7f800001u, 0x00000000u, 0x00000000u},
+    {0x80000000u, 0xff812345u, 0xffd54321u, 0x00000000u},
+    {0x3f800000u, 0x00000000u, 0xffd54321u, 0x00000000u},
+    {0x3f800000u, 0x80000000u, 0x7f800000u, 0x00000000u},
+    {0x3f800000u, 0x3f800000u, 0x80000000u, 0x00000000u},
+    {0x3f800000u, 0xc0400000u, 0x7f800001u, 0x00000000u},
+    {0x3f800000u, 0x7f800000u, 0xff800000u, 0x00000000u},
+    {0x3f800000u, 0xff800000u, 0x3f800000u, 0x00000000u},
+    {0x3f800000u, 0x7fc12345u, 0xff812345u, 0x00000000u},
+    {0x3f800000u, 0xffd54321u, 0x7fc12345u, 0x00000000u},
+    {0x3f800000u, 0x7f800001u, 0xc0400000u, 0x00000000u},
+    {0x3f800000u, 0xff812345u, 0x00000000u, 0x00000000u},
+    {0xc0400000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xc0400000u, 0x80000000u, 0xffd54321u, 0x00000000u},
+    {0xc0400000u, 0x3f800000u, 0x7f800000u, 0x00000000u},
+    {0xc0400000u, 0xc0400000u, 0x80000000u, 0x00000000u},
+    {0xc0400000u, 0x7f800000u, 0x7f800001u, 0x00000000u},
+    {0xc0400000u, 0xff800000u, 0xff800000u, 0x00000000u},
+    {0xc0400000u, 0x7fc12345u, 0x3f800000u, 0x00000000u},
+    {0xc0400000u, 0xffd54321u, 0xff812345u, 0x00000000u},
+    {0xc0400000u, 0x7f800001u, 0x7fc12345u, 0x00000000u},
+    {0xc0400000u, 0xff812345u, 0xc0400000u, 0x00000000u},
+    {0x7f800000u, 0x00000000u, 0xc0400000u, 0x00000000u},
+    {0x7f800000u, 0x80000000u, 0x00000000u, 0x00000000u},
+    {0x7f800000u, 0x3f800000u, 0xffd54321u, 0x00000000u},
+    {0x7f800000u, 0xc0400000u, 0x7f800000u, 0x00000000u},
+    {0x7f800000u, 0x7f800000u, 0x80000000u, 0x00000000u},
+    {0x7f800000u, 0xff800000u, 0x7f800001u, 0x00000000u},
+    {0x7f800000u, 0x7fc12345u, 0xff800000u, 0x00000000u},
+    {0x7f800000u, 0xffd54321u, 0x3f800000u, 0x00000000u},
+    {0x7f800000u, 0x7f800001u, 0xff812345u, 0x00000000u},
+    {0x7f800000u, 0xff812345u, 0x7fc12345u, 0x00000000u},
+    {0xff800000u, 0x00000000u, 0x7fc12345u, 0x00000000u},
+    {0xff800000u, 0x80000000u, 0xc0400000u, 0x00000000u},
+    {0xff800000u, 0x3f800000u, 0x00000000u, 0x00000000u},
+    {0xff800000u, 0xc0400000u, 0xffd54321u, 0x00000000u},
+    {0xff800000u, 0x7f800000u, 0x7f800000u, 0x00000000u},
+    {0xff800000u, 0xff800000u, 0x80000000u, 0x00000000u},
+    {0xff800000u, 0x7fc12345u, 0x7f800001u, 0x00000000u},
+    {0xff800000u, 0xffd54321u, 0xff800000u, 0x00000000u},
+    {0xff800000u, 0x7f800001u, 0x3f800000u, 0x00000000u},
+    {0xff800000u, 0xff812345u, 0xff812345u, 0x00000000u},
+    {0x7fc12345u, 0x00000000u, 0xff812345u, 0x00000000u},
+    {0x7fc12345u, 0x80000000u, 0x7fc12345u, 0x00000000u},
+    {0x7fc12345u, 0x3f800000u, 0xc0400000u, 0x00000000u},
+    {0x7fc12345u, 0xc0400000u, 0x00000000u, 0x00000000u},
+    {0x7fc12345u, 0x7f800000u, 0xffd54321u, 0x00000000u},
+    {0x7fc12345u, 0xff800000u, 0x7f800000u, 0x00000000u},
+    {0x7fc12345u, 0x7fc12345u, 0x80000000u, 0x00000000u},
+    {0x7fc12345u, 0xffd54321u, 0x7f800001u, 0x00000000u},
+    {0x7fc12345u, 0x7f800001u, 0xff800000u, 0x00000000u},
+    {0x7fc12345u, 0xff812345u, 0x3f800000u, 0x00000000u},
+    {0xffd54321u, 0x00000000u, 0x3f800000u, 0x00000000u},
+    {0xffd54321u, 0x80000000u, 0xff812345u, 0x00000000u},
+    {0xffd54321u, 0x3f800000u, 0x7fc12345u, 0x00000000u},
+    {0xffd54321u, 0xc0400000u, 0xc0400000u, 0x00000000u},
+    {0xffd54321u, 0x7f800000u, 0x00000000u, 0x00000000u},
+    {0xffd54321u, 0xff800000u, 0xffd54321u, 0x00000000u},
+    {0xffd54321u, 0x7fc12345u, 0x7f800000u, 0x00000000u},
+    {0xffd54321u, 0xffd54321u, 0x80000000u, 0x00000000u},
+    {0xffd54321u, 0x7f800001u, 0x7f800001u, 0x00000000u},
+    {0xffd54321u, 0xff812345u, 0xff800000u, 0x00000000u},
+    {0x7f800001u, 0x00000000u, 0xff800000u, 0x00000000u},
+    {0x7f800001u, 0x80000000u, 0x3f800000u, 0x00000000u},
+    {0x7f800001u, 0x3f800000u, 0xff812345u, 0x00000000u},
+    {0x7f800001u, 0xc0400000u, 0x7fc12345u, 0x00000000u},
+    {0x7f800001u, 0x7f800000u, 0xc0400000u, 0x00000000u},
+    {0x7f800001u, 0xff800000u, 0x00000000u, 0x00000000u},
+    {0x7f800001u, 0x7fc12345u, 0xffd54321u, 0x00000000u},
+    {0x7f800001u, 0xffd54321u, 0x7f800000u, 0x00000000u},
+    {0x7f800001u, 0x7f800001u, 0x80000000u, 0x00000000u},
+    {0x7f800001u, 0xff812345u, 0x7f800001u, 0x00000000u},
+    {0xff812345u, 0x00000000u, 0x7f800001u, 0x00000000u},
+    {0xff812345u, 0x80000000u, 0xff800000u, 0x00000000u},
+    {0xff812345u, 0x3f800000u, 0x3f800000u, 0x00000000u},
+    {0xff812345u, 0xc0400000u, 0xff812345u, 0x00000000u},
+    {0xff812345u, 0x7f800000u, 0x7fc12345u, 0x00000000u},
+    {0xff812345u, 0xff800000u, 0xc0400000u, 0x00000000u},
+    {0xff812345u, 0x7fc12345u, 0x00000000u, 0x00000000u},
+    {0xff812345u, 0xffd54321u, 0xffd54321u, 0x00000000u},
+    {0xff812345u, 0x7f800001u, 0x7f800000u, 0x00000000u},
+    {0xff812345u, 0xff812345u, 0x80000000u, 0x00000000u},
+    {0x3f800000u, 0x40200000u, 0x00000000u, 0x00000000u},
+    {0x3f800000u, 0x00800000u, 0x3f800000u, 0x00000000u},
+    {0xbf800000u, 0xc0400000u, 0x00000000u, 0x00000000u},
+    {0xc0400000u, 0xbf800000u, 0x3f800000u, 0x00000000u},
+    {0x7fc00000u, 0x7fc12345u, 0x00000000u, 0x00000000u},
+    {0xffc00000u, 0xffd54321u, 0x00000000u, 0x00000000u},
+    {0x7f800001u, 0x7fa00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xbf800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x7f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xff800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x7fc00000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xffc00000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x7f800001u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xff812345u, 0x00000000u},
+    {0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u},
+    {0x00000000u, 0x80000000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x80000000u, 0xbf800000u, 0x00000000u},
+    {0x00000000u, 0x80000000u, 0x7f800000u, 0x00000000u},
+    {0x00000000u, 0x80000000u, 0xff800000u, 0x00000000u},
+    {0x00000000u, 0x80000000u, 0x7fc00000u, 0x00000000u},
+    {0x00000000u, 0x80000000u, 0xffc00000u, 0x00000000u},
+    {0x00000000u, 0x80000000u, 0xff812345u, 0x00000000u},
+    {0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x3f800000u, 0x80000000u, 0x00000000u},
+    {0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u}
+};
+constexpr std::uint32_t Expected[128][7] = {
+    {0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u},
+    {0x00000000u, 0x3f800000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u},
+    {0xc0400000u, 0x00000000u, 0xc0400000u, 0x3f800000u, 0x00000000u, 0xc0400000u, 0x00000000u},
+    {0x00000000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x7f800000u},
+    {0xff800000u, 0x00000000u, 0xff800000u, 0x00000000u, 0xff800000u, 0xff800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xc0400000u, 0x00000000u, 0xc0400000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0x80000000u, 0x7f800000u, 0x00000000u, 0x80000000u, 0x00000000u},
+    {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u},
+    {0x80000000u, 0x3f800000u, 0x80000000u, 0x3f800000u, 0x80000000u, 0x80000000u, 0x3f800000u},
+    {0xc0400000u, 0x80000000u, 0xff800000u, 0x80000000u, 0xc0400000u, 0xc0400000u, 0x80000000u},
+    {0x80000000u, 0x7f800000u, 0x80000000u, 0x7f800000u, 0x3f800000u, 0x80000000u, 0x7f800000u},
+    {0xff800000u, 0x80000000u, 0xff800000u, 0x80000000u, 0xff800000u, 0xff800000u, 0x80000000u},
+    {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u},
+    {0x80000000u, 0x80000000u, 0xc0400000u, 0x80000000u, 0xc0400000u, 0x80000000u, 0x80000000u},
+    {0x80000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u},
+    {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u},
+    {0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u},
+    {0x80000000u, 0x3f800000u, 0x80000000u, 0x7f800000u, 0x3f800000u, 0x80000000u, 0x3f800000u},
+    {0x3f800000u, 0x3f800000u, 0x80000000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u},
+    {0xc0400000u, 0x3f800000u, 0xc0400000u, 0x3f800000u, 0xc0400000u, 0xc0400000u, 0x3f800000u},
+    {0x3f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x3f800000u, 0x3f800000u, 0x7f800000u},
+    {0xff800000u, 0x3f800000u, 0xff800000u, 0x3f800000u, 0x3f800000u, 0xff800000u, 0x3f800000u},
+    {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u},
+    {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u},
+    {0x3f800000u, 0x3f800000u, 0xc0400000u, 0x3f800000u, 0xc0400000u, 0x3f800000u, 0x3f800000u},
+    {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u},
+    {0xc0400000u, 0x00000000u, 0xc0400000u, 0x00000000u, 0x00000000u, 0xc0400000u, 0x00000000u},
+    {0xc0400000u, 0x80000000u, 0xc0400000u, 0x80000000u, 0xc0400000u, 0xc0400000u, 0x80000000u},
+    {0xc0400000u, 0x3f800000u, 0xc0400000u, 0x7f800000u, 0x3f800000u, 0xc0400000u, 0x3f800000u},
+    {0xc0400000u, 0xc0400000u, 0xc0400000u, 0x80000000u, 0xc0400000u, 0xc0400000u, 0xc0400000u},
+    {0xc0400000u, 0x7f800000u, 0xc0400000u, 0x7f800000u, 0xc0400000u, 0xc0400000u, 0x7f800000u},
+    {0xff800000u, 0xc0400000u, 0xff800000u, 0xc0400000u, 0xff800000u, 0xff800000u, 0xc0400000u},
+    {0xc0400000u, 0xc0400000u, 0xc0400000u, 0x3f800000u, 0xc0400000u, 0xc0400000u, 0xc0400000u},
+    {0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u},
+    {0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u},
+    {0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u},
+    {0x00000000u, 0x7f800000u, 0xc0400000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x7f800000u},
+    {0x80000000u, 0x7f800000u, 0x80000000u, 0x7f800000u, 0x00000000u, 0x80000000u, 0x7f800000u},
+    {0x3f800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, 0x3f800000u, 0x3f800000u, 0x7f800000u},
+    {0xc0400000u, 0x7f800000u, 0xc0400000u, 0x7f800000u, 0x7f800000u, 0xc0400000u, 0x7f800000u},
+    {0x7f800000u, 0x7f800000u, 0x80000000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u},
+    {0xff800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0xff800000u, 0xff800000u, 0x7f800000u},
+    {0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u},
+    {0x7f800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, 0x7f800000u},
+    {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u},
+    {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u},
+    {0xff800000u, 0x00000000u, 0xff800000u, 0x00000000u, 0xff800000u, 0xff800000u, 0x00000000u},
+    {0xff800000u, 0x80000000u, 0xff800000u, 0x80000000u, 0xc0400000u, 0xff800000u, 0x80000000u},
+    {0xff800000u, 0x3f800000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0xff800000u, 0x3f800000u},
+    {0xff800000u, 0xc0400000u, 0xff800000u, 0xc0400000u, 0xff800000u, 0xff800000u, 0xc0400000u},
+    {0xff800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0x80000000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0x3f800000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u},
+    {0x3f800000u, 0x3f800000u, 0xc0400000u, 0x3f800000u, 0xc0400000u, 0x3f800000u, 0x3f800000u},
+    {0xc0400000u, 0xc0400000u, 0xc0400000u, 0x00000000u, 0xc0400000u, 0xc0400000u, 0xc0400000u},
+    {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0x7f800000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0x7fc12345u, 0x7fc12345u, 0x80000000u, 0x80000000u, 0x80000000u, 0x7fc12345u, 0x7fc12345u},
+    {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u},
+    {0x7fc12345u, 0x7fc12345u, 0xff800000u, 0xff800000u, 0xff800000u, 0x7fc12345u, 0x7fc12345u},
+    {0x7fc12345u, 0x7fc12345u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x7fc12345u, 0x7fc12345u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u},
+    {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u},
+    {0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u},
+    {0x7f800000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x7f800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0xffd54321u, 0xffd54321u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xffd54321u, 0xffd54321u},
+    {0xffd54321u, 0xffd54321u, 0x80000000u, 0x80000000u, 0x80000000u, 0xffd54321u, 0xffd54321u},
+    {0xffd54321u, 0xffd54321u, 0xffd54321u, 0xffd54321u, 0xffd54321u, 0xffd54321u, 0xffd54321u},
+    {0xffd54321u, 0xffd54321u, 0xff800000u, 0xff800000u, 0xff800000u, 0xffd54321u, 0xffd54321u},
+    {0x00000000u, 0x00000000u, 0xff800000u, 0x00000000u, 0xff800000u, 0x00000000u, 0x00000000u},
+    {0x80000000u, 0x80000000u, 0x80000000u, 0x3f800000u, 0x80000000u, 0x80000000u, 0x80000000u},
+    {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u},
+    {0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u},
+    {0x7f800000u, 0x7f800000u, 0xc0400000u, 0x7f800000u, 0xc0400000u, 0x7f800000u, 0x7f800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0x00000000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u},
+    {0x7f800001u, 0x7f800001u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800001u, 0x7f800001u},
+    {0x7f800001u, 0x7f800001u, 0x80000000u, 0x80000000u, 0x80000000u, 0x7f800001u, 0x7f800001u},
+    {0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x80000000u, 0x80000000u, 0xff800000u, 0x80000000u, 0xff800000u, 0x80000000u, 0x80000000u},
+    {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u},
+    {0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u},
+    {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0xc0400000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0xff812345u, 0xff812345u, 0x00000000u, 0x00000000u, 0x00000000u, 0xff812345u, 0xff812345u},
+    {0xff812345u, 0xff812345u, 0xff812345u, 0xff812345u, 0xff812345u, 0xff812345u, 0xff812345u},
+    {0xff812345u, 0xff812345u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff812345u, 0xff812345u},
+    {0xff812345u, 0xff812345u, 0x80000000u, 0x80000000u, 0x80000000u, 0xff812345u, 0xff812345u},
+    {0x3f800000u, 0x40200000u, 0x00000000u, 0x40200000u, 0x3f800000u, 0x3f800000u, 0x40200000u},
+    {0x00800000u, 0x3f800000u, 0x00800000u, 0x3f800000u, 0x3f800000u, 0x00800000u, 0x3f800000u},
+    {0xc0400000u, 0xbf800000u, 0xc0400000u, 0x00000000u, 0xbf800000u, 0xc0400000u, 0xbf800000u},
+    {0xc0400000u, 0xbf800000u, 0xc0400000u, 0x3f800000u, 0xbf800000u, 0xc0400000u, 0xbf800000u},
+    {0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u, 0x7fc00000u},
+    {0xffc00000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u, 0xffc00000u},
+    {0x7f800001u, 0x7f800001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7f800001u, 0x7f800001u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xbf800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xff800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0x80000000u, 0x3f800000u, 0x00000000u, 0x80000000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0xbf800000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0x80000000u, 0x7f800000u, 0x00000000u, 0x80000000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0xff800000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u},
+    {0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u},
+    {0x00000000u, 0x3f800000u, 0x80000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u},
+    {0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u}
+};
+constexpr std::uint32_t Chunks = 4;
+std::uint32_t Chunk = 0;
+constexpr const char* Names[7] = {
+    "v_min_f32 v10, v4, v5",
+    "v_max_f32 v11, v4, v5",
+    "v_min3_f32 v12, v4, v5, v6",
+    "v_max3_f32 v13, v4, v5, v6",
+    "v_med3_f32 v14, v4, v5, v6",
+    "v_min_f32_e64 v15, v4, v5",
+    "v_max_f32_e64 v16, v4, v5",
+};
+
+void Fill(std::uint32_t tid, std::uint32_t* words) {
+    std::copy(std::begin(Rows[Chunk * Threads + tid]), std::end(Rows[Chunk * Threads + tid]), words);
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
+    Require(actual == expected, std::string("f32 min max nan: row ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* in = &Input[tid * Inputs];
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t i = 0; i < 7; ++i) {
+            Expect(Chunk * Threads + tid, out[i], Expected[Chunk * Threads + tid][i], Names[i]);
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        for (Chunk = 0; Chunk < Chunks; ++Chunk) {
+            Run(*device);
+            Check();
+        }
+        std::puts("f32 min max nan tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/F32OutputModifier.cpp

@@ -0,0 +1,191 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 46> Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xd508000a, 0x0a020b04, 0xd508000b,
+    0x1a020b04, 0xd508000c, 0x12020b04, 0xd503000d, 0x0a020b04, 0xd503000e, 0x1a020b04, 0xd504000f,
+    0x32020b04, 0xd54b0010, 0x0c1a0b04, 0xd54b0011, 0x1c1a0b04, 0xd50f0012, 0x1a020b04, 0xd5100013,
+    0x0a020d04, 0xd5a40014, 0x1a010104, 0xd5850015, 0x12010104, 0xd5088016, 0x0a020b04, 0xd5088017,
+    0x1a020b04, 0xd5038018, 0x12020d04, 0xd54b0019, 0x341a0b04, 0xe0781000, 0x80010a03, 0xe0781010,
+    0x80010e03, 0xe0781020, 0x80011203, 0xe0781030, 0x80011603, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[32][4] = {
+    {0x80000000u, 0x3f800000u, 0x80000000u, 0x00000000u},
+    {0x80800000u, 0x3f800000u, 0x80800000u, 0x00000000u},
+    {0x00ffffffu, 0x3f800000u, 0x80ffffffu, 0x00000000u},
+    {0x7f7fffffu, 0x40000000u, 0x7f7fffffu, 0x00000000u},
+    {0x7fc12345u, 0xffc00000u, 0xff800001u, 0x00000000u},
+    {0x7fc12345u, 0x3f800000u, 0x7fc12345u, 0x00000000u},
+    {0x7f800001u, 0x40000000u, 0x7f800001u, 0x00000000u},
+    {0xff812345u, 0xbf800000u, 0xff812345u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xff800000u, 0x00000000u},
+    {0x3f800000u, 0xff000000u, 0x00000000u, 0x00000000u},
+    {0x3f000000u, 0xbf800000u, 0x7f800000u, 0x00000000u},
+    {0xff800000u, 0x3f000000u, 0x7fc00000u, 0x00000000u},
+    {0xff000000u, 0xbf800000u, 0xbf800000u, 0x00000000u},
+    {0x3f800000u, 0x01000000u, 0x3f000000u, 0x00000000u},
+    {0x00000000u, 0x00800000u, 0x00800000u, 0x00000000u},
+    {0x00000000u, 0x80800000u, 0x80800000u, 0x00000000u},
+    {0x7f7fffffu, 0x3f000000u, 0x00000000u, 0x00000000u},
+    {0x7f800000u, 0x7f800000u, 0x00000000u, 0x00000000u},
+    {0x3f000000u, 0xff000000u, 0x00000000u, 0x00000000u},
+    {0x80000000u, 0x80000000u, 0x80000000u, 0x00000000u},
+    {0xff000000u, 0x7f7fffffu, 0x7fc00000u, 0x00000000u},
+    {0x00000000u, 0x01000000u, 0x01000000u, 0x00000000u},
+    {0x7fc00000u, 0x7f7fffffu, 0x00000000u, 0x00000000u},
+    {0xbf800000u, 0x01000000u, 0x00000000u, 0x00000000u},
+    {0x80800000u, 0x80800000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x7fc00000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xffc00000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x7f800001u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x7fc12345u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x7f800000u, 0x00000000u}
+};
+constexpr std::uint32_t Expected[32][16] = {
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x40000000u, 0x3f000000u, 0xc0800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xd0000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x81000000u, 0x80000000u, 0x81800000u, 0x40000000u, 0x3f000000u, 0xc0800000u, 0x81800000u, 0x80800000u, 0x80000000u, 0x81000000u, 0xbf000000u, 0xcfff0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x017fffffu, 0x00000000u, 0x01ffffffu, 0x40000000u, 0x3f000000u, 0xc0800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x017fffffu, 0x00000000u, 0x4c7fffffu, 0x017fffffu, 0x00000000u, 0x00000000u, 0x827fffffu},
+    {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7effffffu, 0xff800000u, 0x7f800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, 0x7effffffu, 0x4fff0000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xff800000u},
+    {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x4fff8247u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc12345u},
+    {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x3f000000u, 0x7fc12345u, 0x7fc12345u, 0x4fff8247u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc12345u},
+    {0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0xff800001u, 0x7f800001u, 0x7f800001u, 0x3f800000u, 0x7f800001u, 0x7f800001u, 0x4fff0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xff800001u},
+    {0xff812345u, 0xff812345u, 0xff812345u, 0xff812345u, 0xff812345u, 0x7f812345u, 0xff812345u, 0xff812345u, 0xbf000000u, 0xff812345u, 0xff812345u, 0xcbfdb976u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7f812345u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xff800000u, 0xff800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xff800000u},
+    {0xff800000u, 0xfe800000u, 0xff800000u, 0xff800000u, 0xfe800000u, 0x7f800000u, 0xff800000u, 0xfe800000u, 0xfe800000u, 0x40000000u, 0x3f000000u, 0x4f7e0000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f800000u},
+    {0xbf800000u, 0xbe800000u, 0xc0000000u, 0xbf800000u, 0xbe800000u, 0x40000000u, 0x7f800000u, 0x7f800000u, 0xbf000000u, 0x7f800000u, 0x00000000u, 0x4f7c0000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0x7f800000u, 0x7fc00000u, 0x7fc00000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xcc000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u},
+    {0x7f800000u, 0x7e800000u, 0x7f800000u, 0xff800000u, 0xfe800000u, 0x7f800000u, 0x7f800000u, 0x7e800000u, 0xfe800000u, 0xc0000000u, 0xfe800000u, 0xcc800000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0xff800000u},
+    {0x01800000u, 0x00800000u, 0x02000000u, 0x40000000u, 0x3f000000u, 0xc0800000u, 0x3f800000u, 0x3e800000u, 0x00800000u, 0x40000000u, 0x3f000000u, 0x4f7e0000u, 0x01800000u, 0x00800000u, 0x3f800000u, 0x40000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00000000u, 0x81800000u, 0x01000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01800000u, 0x01800000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x81000000u, 0x80000000u, 0x01800000u, 0x81000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x81800000u},
+    {0x7f7fffffu, 0x7e7fffffu, 0x7f800000u, 0x7f800000u, 0x7effffffu, 0xff800000u, 0x7f7fffffu, 0x7e7fffffu, 0x3e800000u, 0x7f800000u, 0x7effffffu, 0x4fff0000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xff800000u},
+    {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x4fff0000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xff800000u},
+    {0xff000000u, 0xfe000000u, 0xff800000u, 0xff800000u, 0xfe800000u, 0x7f800000u, 0xff000000u, 0xfe000000u, 0xfe800000u, 0x3f800000u, 0x00000000u, 0x4f7c0000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f800000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xd0000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0x7f7ffffeu, 0x7e7ffffeu, 0xff800000u, 0x7fc00000u, 0x7fc00000u, 0xfe800000u, 0xff800000u, 0xfe800000u, 0xcc800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x01800000u, 0x00800000u, 0x82000000u, 0x01800000u, 0x00800000u, 0x00000000u, 0x01800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x02000000u, 0x02000000u},
+    {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0x7fc00000u, 0x7effffffu, 0x00000000u, 0x7fc00000u, 0x4fff8000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u},
+    {0x81800000u, 0x80800000u, 0x82000000u, 0xc0000000u, 0xbf000000u, 0x40800000u, 0x81800000u, 0x80800000u, 0xbf000000u, 0x00000000u, 0xbf000000u, 0xcf810000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x02000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x81800000u, 0x80800000u, 0x02000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0xbf000000u, 0xcfff0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7f800001u, 0x7f800001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7f800001u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc12345u, 0x7fc12345u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc12345u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7f800000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f800000u}
+};
+constexpr std::uint32_t Checked[32] = {0xffffu, 0xffffu, 0xffffu, 0x7f3fu, 0x7800u, 0x7b00u, 0x7b00u, 0x7b00u, 0xffffu, 0xffffu, 0xffffu, 0x7f3fu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0x7f3fu, 0xffffu, 0xffffu, 0x7f3fu, 0xffffu, 0x7b00u, 0xffffu, 0x7f3fu, 0xffffu, 0xffffu, 0x7f3fu, 0x7f3fu, 0x7f3fu, 0x7f3fu, 0xffffu};
+constexpr std::uint32_t AnyNan[32] = {0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x84ffu, 0x84ffu, 0x84ffu, 0x84ffu, 0x0000u, 0x0000u, 0x0000u, 0x80c0u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x80c0u, 0x0000u, 0x84ffu, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x80c0u, 0x80c0u, 0x80c0u, 0x80c0u, 0x0000u};
+constexpr const char* Names[16] = {
+    "v_mul_f32_e64 v10, v4, v5 mul:2",
+    "v_mul_f32_e64 v11, v4, v5 div:2",
+    "v_mul_f32_e64 v12, v4, v5 mul:4",
+    "v_add_f32_e64 v13, v4, v5 mul:2",
+    "v_add_f32_e64 v14, v4, v5 div:2",
+    "v_sub_f32_e64 v15, -v4, v5 mul:4",
+    "v_fma_f32 v16, v4, v5, v6 mul:2",
+    "v_fma_f32 v17, v4, v5, v6 div:2",
+    "v_min_f32_e64 v18, v4, v5 div:2",
+    "v_max_f32_e64 v19, v4, v6 mul:2",
+    "v_floor_f32_e64 v20, v4 div:2",
+    "v_cvt_f32_i32_e64 v21, v4 mul:4",
+    "v_mul_f32_e64 v22, v4, v5 clamp mul:2",
+    "v_mul_f32_e64 v23, v4, v5 clamp div:2",
+    "v_add_f32_e64 v24, v4, v6 clamp mul:4",
+    "v_fma_f32 v25, -v4, v5, v6 mul:4",
+};
+
+void Fill(std::uint32_t tid, std::uint32_t* words) {
+    std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words);
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
+    Require(actual == expected, std::string("f32 output modifier: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* in = &Input[tid * Inputs];
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t i = 0; i < 16; ++i) {
+            if ((Checked[tid] >> i) & 1u) Expect(tid, out[i], Expected[tid][i], Names[i]);
+            if ((AnyNan[tid] >> i) & 1u) Require((out[i] & 0x7fffffffu) > 0x7f800000u, std::string("f32 output modifier: lane ") + std::to_string(tid) + " " + Names[i] + " is " + Hex(out[i]) + ", expected a NaN");
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device);
+        Check();
+        std::puts("f32 output modifier tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/F32ResultClamp.cpp

@@ -0,0 +1,189 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 47> Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x7e1e0306, 0xd503800a, 0x02020b04,
+    0xd504800b, 0x02020b04, 0xd508800c, 0x02020b04, 0xd508800d, 0x22020b04, 0xd54b800e, 0x041a0b04,
+    0xd52b800f, 0x02020b04, 0xd50f8010, 0x02020b04, 0xd5108011, 0x22020b04, 0xd5578012, 0x041a0b04,
+    0xd5a48013, 0x02010104, 0xd5a08014, 0x02010104, 0xd5858015, 0x02010104, 0xd5a18016, 0x22010104,
+    0xd5a38017, 0x02010104, 0xd5548018, 0x041a0b04, 0xd5a28019, 0x22010104, 0xe0781000, 0x80010a03,
+    0xe0781010, 0x80010e03, 0xe0781020, 0x80011203, 0xe0781030, 0x80011603, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[32][4] = {
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x7f800000u, 0x80000000u, 0x80000000u, 0x00000000u},
+    {0xbf000000u, 0x3f800000u, 0xff800000u, 0x00000000u},
+    {0xff800000u, 0xff800000u, 0x80000000u, 0x00000000u},
+    {0x3f800000u, 0x3f000000u, 0x3f800001u, 0x00000000u},
+    {0xc0200000u, 0x00000000u, 0x3f000000u, 0x00000000u},
+    {0x40000000u, 0xbf800000u, 0x3f000000u, 0x00000000u},
+    {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x00000000u},
+    {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u},
+    {0x40000000u, 0x3f800000u, 0xbf800000u, 0x00000000u},
+    {0xbf800000u, 0x7f800000u, 0x7f800000u, 0x00000000u},
+    {0x7fc00000u, 0xff800000u, 0x7fc00000u, 0x00000000u},
+    {0x7f800000u, 0x3f800001u, 0x00000000u, 0x00000000u},
+    {0xbf800000u, 0xbf800000u, 0x7fc00000u, 0x00000000u},
+    {0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u},
+    {0x7fc00000u, 0x7f800000u, 0x00000000u, 0x00000000u},
+    {0x3f800000u, 0xbf800000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x7fc00000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xffc00000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x7f800001u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x7fc12345u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x7f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xff800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x3f000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x3f800001u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xbf000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x3f7fffffu, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x40000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xbf800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0xc0200000u, 0x00000000u}
+};
+constexpr std::uint32_t Expected[32][16] = {
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u},
+    {0x3f000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u},
+    {0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u},
+    {0x3f800000u, 0x3f000000u, 0x3f000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x3f000000u, 0x3f000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3f000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f000000u, 0x3f800000u},
+    {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u},
+    {0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f7fffffu, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f7fffffu, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}
+};
+constexpr std::uint32_t Checked[32] = {0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xfeffu, 0xffffu, 0xffffu, 0xffffu, 0xfeffu, 0xffffu, 0xfeffu, 0xffffu, 0xfeffu, 0xffffu, 0xffffu, 0xfeffu, 0xfeffu, 0xbeffu, 0xfeffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu};
+constexpr const char* Names[16] = {
+    "v_add_f32_e64 v10, v4, v5 clamp",
+    "v_sub_f32_e64 v11, v4, v5 clamp",
+    "v_mul_f32_e64 v12, v4, v5 clamp",
+    "v_mul_f32_e64 v13, -v4, v5 clamp",
+    "v_fma_f32 v14, v4, v5, v6 clamp",
+    "v_fmac_f32_e64 v15, v4, v5 clamp",
+    "v_min_f32_e64 v16, v4, v5 clamp",
+    "v_max_f32_e64 v17, -v4, v5 clamp",
+    "v_med3_f32 v18, v4, v5, v6 clamp",
+    "v_floor_f32_e64 v19, v4 clamp",
+    "v_fract_f32_e64 v20, v4 clamp",
+    "v_cvt_f32_i32_e64 v21, v4 clamp",
+    "v_trunc_f32_e64 v22, -v4 clamp",
+    "v_rndne_f32_e64 v23, v4 clamp",
+    "v_max3_f32 v24, v4, v5, v6 clamp",
+    "v_ceil_f32_e64 v25, -v4 clamp",
+};
+
+void Fill(std::uint32_t tid, std::uint32_t* words) {
+    std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words);
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
+    Require(actual == expected, std::string("f32 result clamp: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* in = &Input[tid * Inputs];
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t i = 0; i < 16; ++i) {
+            if ((Checked[tid] >> i) & 1u) Expect(tid, out[i], Expected[tid][i], Names[i]);
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device);
+        Check();
+        std::puts("f32 result clamp tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/F64Division.cpp

@@ -0,0 +1,172 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 36> Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xd56e6a0a, 0x041a0904, 0xd5010018,
+    0x01a90280, 0xd56e6a0c, 0x041a0906, 0xd5010019, 0x01a90480, 0x38303318, 0xbeea03c1, 0xd570000e,
+    0x04120d04, 0xd5700010, 0x041a0d04, 0xd5600012, 0x04120d04, 0xd5600014, 0x041a0906, 0xd5600016,
+    0x041a08ff, 0x7ff80000, 0x7e320280, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xe0781020,
+    0x80011203, 0xe0781030, 0x80011603, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[32][4] = {
+    {0x00000001u, 0x7ff00000u, 0x00000000u, 0x3ff00000u},
+    {0x00000000u, 0x7ff40000u, 0x00000000u, 0x3ff00000u},
+    {0xe61541b6u, 0xcdde1df6u, 0x652b0ed7u, 0xfddc36e5u},
+    {0xc3d4a267u, 0x03d31de5u, 0x08426379u, 0xb3cb953cu},
+    {0xceb52fc3u, 0xb3648a3fu, 0xd8fe4338u, 0x8367c969u},
+    {0x131c99fcu, 0x7f65f30fu, 0x888c6e8bu, 0xcf72c7f8u},
+    {0x00000000u, 0x7fd00000u, 0x00000000u, 0x3ca00000u},
+    {0xffffffffu, 0x7fefffffu, 0x00000000u, 0x3ca00000u},
+    {0x00000000u, 0x7fd00000u, 0xffffffffu, 0x7fefffffu},
+    {0x00000000u, 0x7fc00000u, 0xffffffffu, 0x7fefffffu},
+    {0x00000000u, 0x00100000u, 0x00000000u, 0x03500000u},
+    {0x00000000u, 0x00100000u, 0x00000000u, 0x03600000u},
+    {0x8a41dfb1u, 0xbffc3c7cu, 0xffffffffu, 0x7fefffffu},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0xfff00000u},
+    {0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xbff00000u, 0x00000000u, 0xfff00000u},
+    {0x00000000u, 0x7ff00000u, 0xffffffffu, 0x7fefffffu},
+    {0x00000001u, 0xfff80000u, 0x00000000u, 0x7ff80000u},
+    {0x00000000u, 0x00000000u, 0x00000001u, 0xfff80000u},
+    {0x00000000u, 0x3ff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0xbff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00100000u},
+    {0x00000000u, 0x00000000u, 0xffffffffu, 0x7fefffffu},
+    {0x00000000u, 0x00000000u, 0xffffffffu, 0xffefffffu},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x7ff80000u},
+    {0x00000000u, 0x00000000u, 0x00000001u, 0x7ff00000u}
+};
+constexpr std::uint32_t Expected[32][16] = {
+    {0x00000001u, 0x7ff00000u, 0x00000000u, 0x3ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0x7ff40000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000003u, 0x00000000u},
+    {0xe61541b6u, 0xd5de1df6u, 0x652b0ed7u, 0xfddc36e5u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0xe61541b6u, 0x4dde1df6u, 0x652b0ed7u, 0x7ddc36e5u, 0x00000000u, 0x7ff00000u, 0x00000003u, 0x00000000u},
+    {0xc3d4a267u, 0x03d31de5u, 0x08426379u, 0xb3cb953cu, 0x00000000u, 0x00000000u, 0x08426379u, 0xabcb953cu, 0xc3d4a267u, 0x83d31de5u, 0x08426379u, 0xb3cb953cu, 0x00000000u, 0xfff00000u, 0x00000000u, 0x00000000u},
+    {0xceb52fc3u, 0xb3648a3fu, 0xd8fe4338u, 0x8b67c969u, 0xceb52fc3u, 0xab648a3fu, 0x00000000u, 0x80000000u, 0xceb52fc3u, 0x33648a3fu, 0xd8fe4338u, 0x0367c969u, 0x00000000u, 0x7ff00000u, 0x00000003u, 0x00000000u},
+    {0x131c99fcu, 0x7f65f30fu, 0x888c6e8bu, 0xcf72c7f8u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x131c99fcu, 0xff65f30fu, 0x888c6e8bu, 0xcf72c7f8u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x77d00000u, 0x00000000u, 0x3ca00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x74800000u, 0x00000000u, 0x7fd00000u, 0x00000000u, 0x3ca00000u, 0x00000000u, 0x7ff00000u, 0x00000003u, 0x00000000u},
+    {0xffffffffu, 0x77efffffu, 0x00000000u, 0x3ca00000u, 0x00000000u, 0x7ff00000u, 0xffffffffu, 0x749fffffu, 0xffffffffu, 0x7fefffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0x77d00000u, 0xffffffffu, 0x77efffffu, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7fd00000u, 0xffffffffu, 0x7fefffffu, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x7fc00000u, 0xffffffffu, 0x7fefffffu, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7fc00000u, 0xffffffffu, 0x7fefffffu, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x08100000u, 0x00000000u, 0x0b500000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00100000u, 0x00000000u, 0x03500000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00100000u, 0x00000000u, 0x03600000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00100000u, 0x00000000u, 0x03600000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u},
+    {0x8a41dfb1u, 0xc7fc3c7cu, 0xffffffffu, 0x7fefffffu, 0x8a41dfb0u, 0xf7fc3c7cu, 0x00000000u, 0xfff00000u, 0x8a41dfb1u, 0xbffc3c7cu, 0xffffffffu, 0xffefffffu, 0x00000000u, 0xfff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xc7f00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0x7ff00000u, 0xffffffffu, 0x77efffffu, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000001u, 0xfff80000u, 0x00000000u, 0x7ff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x37f00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x37f00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xb7f00000u, 0x00000000u, 0x80000000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x80000000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000003u, 0x00000000u}
+};
+constexpr const char* Names[7] = {"div_scale denominator", "div_scale numerator", "div_fmas a*b+a", "div_fmas a*b+b", "div_fixup a", "div_fixup b", "div_fixup nan"};
+
+void Fill(std::uint32_t tid, std::uint32_t* words) {
+    std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words);
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
+    Require(actual == expected, std::string("f64 division: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* in = &Input[tid * Inputs];
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t i = 0; i < 7; ++i) {
+            Expect(tid, out[2 * i], Expected[tid][2 * i], Names[i]);
+            Expect(tid, out[2 * i + 1], Expected[tid][2 * i + 1], Names[i]);
+        }
+        Expect(tid, out[14], Expected[tid][14], "div_scale vcc");
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device);
+        Check();
+        std::puts("f64 division tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/F64DivisionModifiers.cpp

@@ -0,0 +1,172 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 38> Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xd56e6a0a, 0x641a0904, 0xd5010018,
+    0x01a90280, 0xd56e6a0c, 0x841a0906, 0xd5010019, 0x01a90480, 0x38303318, 0xbeea03c1, 0xd570060e,
+    0xa4120d04, 0xd5700010, 0x04120d06, 0xbeea0380, 0xd5700012, 0x041a0d04, 0xd5600514, 0x64120d04,
+    0x36320881, 0x7d8a3280, 0xd5700016, 0x041a0906, 0x7e320280, 0xe0781000, 0x80010a03, 0xe0781010,
+    0x80010e03, 0xe0781020, 0x80011203, 0xe0781030, 0x80011603, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[32][4] = {
+    {0x00000001u, 0x7ff00000u, 0x00000000u, 0x3ff00000u},
+    {0x00000000u, 0x7ff40000u, 0x00000000u, 0x3ff00000u},
+    {0xe61541b6u, 0xcdde1df6u, 0x652b0ed7u, 0xfddc36e5u},
+    {0xc3d4a267u, 0x03d31de5u, 0x08426379u, 0xb3cb953cu},
+    {0xceb52fc3u, 0xb3648a3fu, 0xd8fe4338u, 0x8367c969u},
+    {0x131c99fcu, 0x7f65f30fu, 0x888c6e8bu, 0xcf72c7f8u},
+    {0x00000000u, 0x7fd00000u, 0x00000000u, 0x3ca00000u},
+    {0xffffffffu, 0x7fefffffu, 0x00000000u, 0x3ca00000u},
+    {0x00000000u, 0x7fd00000u, 0xffffffffu, 0x7fefffffu},
+    {0x00000000u, 0x7fc00000u, 0xffffffffu, 0x7fefffffu},
+    {0x00000000u, 0x00100000u, 0x00000000u, 0x03500000u},
+    {0x00000000u, 0x00100000u, 0x00000000u, 0x03600000u},
+    {0x00000000u, 0x7ff00000u, 0x00000001u, 0xfff80000u},
+    {0xffffffffu, 0xffefffffu, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0xfff00000u},
+    {0x00000000u, 0x80000000u, 0x00000000u, 0x3ff00000u},
+    {0x00000000u, 0x00100000u, 0x00000000u, 0x80000000u},
+    {0x00000001u, 0xfff80000u, 0x00000000u, 0x7ff80000u},
+    {0x00000000u, 0x3ff00000u, 0x00000000u, 0xfff00000u},
+    {0x00000000u, 0xbff00000u, 0xffffffffu, 0x7fefffffu},
+    {0xb0894f5au, 0x80834975u, 0x4c867062u, 0x0376ec15u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0xbff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00100000u},
+    {0x00000000u, 0x00000000u, 0xffffffffu, 0x7fefffffu},
+    {0x00000000u, 0x00000000u, 0xffffffffu, 0xffefffffu},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x7ff80000u},
+    {0x00000000u, 0x00000000u, 0x00000001u, 0xfff80000u}
+};
+constexpr std::uint32_t Expected[32][16] = {
+    {0x00000001u, 0xfff00000u, 0x00000000u, 0x3ff00000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff40000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0xfffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000003u, 0x00000000u},
+    {0xe61541b6u, 0x55de1df6u, 0x652b0ed7u, 0xfddc36e5u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0xe61541b6u, 0x4dde1df6u, 0x00000000u, 0x7ff00000u, 0x00000003u, 0x00000000u},
+    {0xc3d4a267u, 0x83d31de5u, 0x08426379u, 0xb3cb953cu, 0x00000000u, 0x80000000u, 0x456d9116u, 0x1fa7c68du, 0x08426379u, 0xb3cb953cu, 0xc3d4a267u, 0x03d31de5u, 0x08426379u, 0xabcb953cu, 0x00000000u, 0x00000000u},
+    {0xceb52fc3u, 0x33648a3fu, 0xd8fe4338u, 0x8b67c969u, 0xceb52fc3u, 0xab648a3fu, 0xceb52fc3u, 0xab648a3fu, 0xd8fe4338u, 0x8367c969u, 0xceb52fc3u, 0x33648a3fu, 0x00000000u, 0x80000000u, 0x00000003u, 0x00000000u},
+    {0x131c99fcu, 0xff65f30fu, 0x888c6e8bu, 0xcf72c7f8u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff00000u, 0x131c99fcu, 0x7f65f30fu, 0x00000000u, 0xfff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xf7d00000u, 0x00000000u, 0x3ca00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7c800000u, 0x00000000u, 0xffd00000u, 0x00000000u, 0x7c800000u, 0x00000003u, 0x00000000u},
+    {0xffffffffu, 0xf7efffffu, 0x00000000u, 0x3ca00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0xffffffffu, 0x7c9fffffu, 0xffffffffu, 0xffefffffu, 0xffffffffu, 0x749fffffu, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xf7d00000u, 0xffffffffu, 0x77efffffu, 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xffd00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xffc00000u, 0xffffffffu, 0x7fefffffu, 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xffc00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x88100000u, 0x00000000u, 0x0b500000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x03500000u, 0x00000000u, 0x80100000u, 0x00000000u, 0x03500000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x80100000u, 0x00000000u, 0x03600000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x03600000u, 0x00000000u, 0x80100000u, 0x00000000u, 0x03600000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff00000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff80000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x00000000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff80000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x37f00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x3ff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u},
+    {0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xc7f00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0x47f00000u, 0xffffffffu, 0x7fefffffu, 0xffffffffu, 0x77efffffu, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0x00000000u, 0x00000003u, 0x00000000u},
+    {0xb0894f5au, 0x00834975u, 0x4c867062u, 0x0376ec15u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x4c867062u, 0x0376ec15u, 0xb0894f5au, 0x80834975u, 0x4c867062u, 0x0376ec15u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x37f00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x3ff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x37f00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbff00000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00100000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00100000u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x7ff00000u, 0xffffffffu, 0x7fefffffu, 0x00000000u, 0x80000000u, 0xffffffffu, 0x7fefffffu, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x7ff00000u, 0xffffffffu, 0xffefffffu, 0x00000000u, 0x00000000u, 0xffffffffu, 0xffefffffu, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x80000000u, 0x00000000u, 0xfff80000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x7ff80000u, 0x00000003u, 0x00000000u},
+    {0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff80000u, 0x00000003u, 0x00000000u}
+};
+constexpr const char* Names[7] = {"div_scale -a", "div_scale -b", "div_fmas modifiers", "div_fmas b*b+a", "div_fmas vcc 0", "div_fixup modifiers", "div_fmas vcc per lane"};
+
+void Fill(std::uint32_t tid, std::uint32_t* words) {
+    std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words);
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
+    Require(actual == expected, std::string("f64 division modifiers: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* in = &Input[tid * Inputs];
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t i = 0; i < 7; ++i) {
+            Expect(tid, out[2 * i], Expected[tid][2 * i], Names[i]);
+            Expect(tid, out[2 * i + 1], Expected[tid][2 * i + 1], Names[i]);
+        }
+        Expect(tid, out[14], Expected[tid][14], "div_scale vcc");
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device);
+        Check();
+        std::puts("f64 division modifiers tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/F64Transcendental.cpp

@@ -0,0 +1,190 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 27> Code{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0x7e145f04, 0x7e186304, 0x7e1c6904,
+    0xd5740010, 0x02020d04, 0xd5740012, 0x02010104, 0xd5740114, 0x2a020d04, 0xd5af8016, 0x02010104,
+    0xd5b40118, 0x22010104, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xe0781020, 0x80011203,
+    0xe0781030, 0x80011603, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[32][4] = {
+    {0x00000001u, 0x7ff00000u, 0xcad921b4u, 0x00000000u},
+    {0x00000000u, 0x7ff40000u, 0x0000001fu, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00000025u, 0x80000000u},
+    {0xe0000000u, 0x47efffffu, 0x0000001eu, 0x00000000u},
+    {0x00000000u, 0x80000000u, 0x0000001bu, 0x00000000u},
+    {0x00000000u, 0xbff00000u, 0x0000001au, 0x00000000u},
+    {0x00000000u, 0x7ff00000u, 0x01c32149u, 0x00000000u},
+    {0x00000000u, 0xfff00000u, 0x00000012u, 0x00000000u},
+    {0x00000001u, 0xfff80000u, 0x00000012u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x3ceb3ffdu, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x8b529b4au, 0x3ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000008u, 0xbff00000u},
+    {0x00000000u, 0x00000000u, 0x5eb561a4u, 0x3fe00000u},
+    {0x00000000u, 0x00000000u, 0x00000026u, 0xbfe00000u},
+    {0x00000000u, 0x00000000u, 0x795b929eu, 0x3ff80000u},
+    {0x00000000u, 0x00000000u, 0x00000028u, 0x40040000u},
+    {0x00000000u, 0x00000000u, 0x94b2b8fdu, 0xc0040000u},
+    {0x00000000u, 0x00000000u, 0x00000004u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x9b08923du, 0x800fffffu},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00100000u},
+    {0x00000000u, 0x00000000u, 0xe8a8529fu, 0x7fefffffu},
+    {0x00000000u, 0x00000000u, 0x0000001eu, 0xffefffffu},
+    {0x00000000u, 0x00000000u, 0x42650644u, 0x7ff00000u},
+    {0x00000000u, 0x00000000u, 0x00000023u, 0xfff00000u},
+    {0x00000000u, 0x00000000u, 0x3bfd1d33u, 0x7ff80000u},
+    {0x00000000u, 0x00000000u, 0x0000000cu, 0xfff80000u},
+    {0x00000000u, 0x00000000u, 0xfee29476u, 0x7ff00000u},
+    {0x00000000u, 0x00000000u, 0x0000001eu, 0x7ff40000u},
+    {0x00000000u, 0x00000000u, 0x8a7d43b5u, 0x41e00000u},
+    {0x00000000u, 0x00000000u, 0x00000023u, 0xc1e00000u},
+    {0x00000000u, 0x00000000u, 0x79f248b0u, 0xc1e00000u},
+    {0x00000000u, 0x00000000u, 0x00000019u, 0x41dfffffu}
+};
+constexpr std::uint32_t Expected[32][16] = {
+    {0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0xf5f2f8bdu, 0x0b43dd63u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0xfff80000u},
+    {0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x00000000u, 0xf5f2f8bdu, 0x0b43dd63u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfffc0000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x2126e970u, 0x2f42371du, 0x6dc9c882u, 0x3fe45f30u, 0x2126e970u, 0x2f42371du, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x1af80000u, 0x37f00000u, 0x039e0000u, 0x3bf00000u, 0xed080000u, 0x43efffffu, 0x00000000u, 0x00000000u, 0xfa9a6ee0u, 0x3b43abe8u, 0x00000000u, 0x00000000u, 0x1af80000u, 0x37f00000u, 0x00000000u, 0xfff80000u},
+    {0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0xbff00000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff80000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0xf5f2f8bdu, 0x0b43dd63u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff80000u},
+    {0x00000000u, 0x80000000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0xf5f2f8bdu, 0x0b43dd63u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xfff80000u},
+    {0x00000001u, 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000000u, 0x00000000u, 0xf5f2f8bdu, 0x0b43dd63u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0xfff80000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x2fbf209cu, 0x1ec8135au, 0x6dc9c882u, 0x3fe45f30u, 0x2fbf209cu, 0x1ec8135au, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x74411af8u, 0x2544d39fu, 0x6dc9c882u, 0x3fe45f30u, 0x74411af8u, 0x2544d39fu, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0xde2b0db8u, 0x328d5ef5u, 0x6dc9c882u, 0x3fe45f30u, 0xde2b0db8u, 0x328d5ef5u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0xa8274600u, 0x2b9924bbu, 0x6dc9c882u, 0x3fe45f30u, 0xa8274600u, 0x2b9924bbu, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x74411af8u, 0x2544d39fu, 0x6dc9c882u, 0x3fe45f30u, 0x74411af8u, 0x2544d39fu, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0xde2b0db8u, 0x328d5ef5u, 0x6dc9c882u, 0x3fe45f30u, 0xde2b0db8u, 0x328d5ef5u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x6dc9c882u, 0x3fe45f30u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0xde2b0db8u, 0x328d5ef5u, 0x6dc9c882u, 0x3fe45f30u, 0xde2b0db8u, 0x328d5ef5u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x041fe516u, 0x35f3c439u, 0x6dc9c882u, 0x3fe45f30u, 0x041fe516u, 0x35f3c439u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0xd0739f78u, 0x00ff17b3u, 0x6dc9c882u, 0x3fe45f30u, 0xd0739f78u, 0x00ff17b3u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x7e2ef7e4u, 0x18127211u, 0x6dc9c882u, 0x3fe45f30u, 0x7e2ef7e4u, 0x18127211u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x041fe516u, 0x35f3c439u, 0x6dc9c882u, 0x3fe45f30u, 0x041fe516u, 0x35f3c439u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0xf2c26dd2u, 0x0adfb3c9u, 0x6dc9c882u, 0x3fe45f30u, 0xf2c26dd2u, 0x0adfb3c9u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u},
+    {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x6dc9c882u, 0x3fe45f30u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x80000000u}
+};
+constexpr const char* Names[8] = {"rcp", "rsq", "sqrt", "trig preop", "trig preop inline", "trig preop abs neg omod", "rcp clamp", "sqrt abs neg"};
+constexpr bool Approximate[8] = {true, true, true, false, false, false, true, true};
+
+void Fill(std::uint32_t tid, std::uint32_t* words) {
+    std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words);
+}
+
+double F64(const std::uint32_t* words) {
+    return std::bit_cast<double>(static_cast<std::uint64_t>(words[1]) << 32u | words[0]);
+}
+
+bool Normal(const std::uint32_t* words) {
+    const auto exponent = (words[1] >> 20u) & 0x7ffu;
+    return exponent != 0u && exponent != 0x7ffu;
+}
+
+void ExpectNear(std::uint32_t tid, const std::uint32_t* actual, const std::uint32_t* expected, const char* name) {
+    const auto error = std::abs(F64(actual) - F64(expected)) / std::abs(F64(expected));
+    Require(error <= 0x1p-23, std::string("f64 transcendental: lane ") + std::to_string(tid) + " " + name + " is " + std::to_string(F64(actual)) + ", expected " + std::to_string(F64(expected)));
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
+    Require(actual == expected, std::string("f64 transcendental: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* in = &Input[tid * Inputs];
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t i = 0; i < 8; ++i) {
+            const std::uint32_t* expected = &Expected[tid][2 * i];
+            if (Approximate[i] && Normal(expected)) {
+                ExpectNear(tid, out + 2 * i, expected, Names[i]);
+                continue;
+            }
+            Expect(tid, out[2 * i], expected[0], Names[i]);
+            Expect(tid, out[2 * i + 1], expected[1], Names[i]);
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device);
+        Check();
+        std::puts("f64 transcendental tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/FlatCacheBits.cpp

@@ -0,0 +1,177 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Results = 4;
+constexpr std::uint32_t Sentinel = 0x05e471e1;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::uint8_t Fill = 0xcd;
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 24> FlatCacheBitsCode{
+    0x34020084, 0x34040084, 0xd70f6a1e, 0x02020208, 0x7e3e0209, 0x503e3e80, 0xd70f6a1e, 0x02023c8c,
+    0x503e3e80, 0xdc379000, 0x0a080001, 0xdc331000, 0x0d7d001e, 0xbf8c0070, 0xdc739004, 0x00080d01,
+    0xdccb9008, 0x0c080a01, 0xdc731000, 0x007d0b1e, 0xbf8c0070, 0xe0781000, 0x80010a02, 0xbf810000
+};
+
+struct Row {
+    std::array<std::uint32_t, 4> memory;
+    std::array<std::uint32_t, Results> results;
+    std::array<std::uint32_t, 4> final;
+};
+
+constexpr std::array<Row, 32> Rows{{
+    {{0x00000001, 0x0000007f, 0x00000080, 0x7fffffff}, {0x00000001, 0x0000007f, 0x00000080, 0x7fffffff}, {0x00000001, 0x7fffffff, 0x00000081, 0x0000007f}},
+    {{0x0000007f, 0x000000ff, 0x80000000, 0x80ff7f01}, {0x0000007f, 0x000000ff, 0x80000000, 0x80ff7f01}, {0x0000007f, 0x80ff7f01, 0x8000007f, 0x000000ff}},
+    {{0x00000080, 0x80000000, 0xdeadbeef, 0x000000ff}, {0x00000080, 0x80000000, 0xdeadbeef, 0x000000ff}, {0x00000080, 0x000000ff, 0xdeadbf6f, 0x80000000}},
+    {{0x000000ff, 0x12345678, 0x00000001, 0xdeadbeef}, {0x000000ff, 0x12345678, 0x00000001, 0xdeadbeef}, {0x000000ff, 0xdeadbeef, 0x00000100, 0x12345678}},
+    {{0x7fffffff, 0x80ff7f01, 0x000000ff, 0x00000080}, {0x7fffffff, 0x80ff7f01, 0x000000ff, 0x00000080}, {0x7fffffff, 0x00000080, 0x800000fe, 0x80ff7f01}},
+    {{0x80000000, 0x00000001, 0xffffffff, 0x12345678}, {0x80000000, 0x00000001, 0xffffffff, 0x12345678}, {0x80000000, 0x12345678, 0x7fffffff, 0x00000001}},
+    {{0xffffffff, 0x00000080, 0x80ff7f01, 0x0000007f}, {0xffffffff, 0x00000080, 0x80ff7f01, 0x0000007f}, {0xffffffff, 0x0000007f, 0x80ff7f00, 0x00000080}},
+    {{0x12345678, 0x7fffffff, 0x0000007f, 0xffffffff}, {0x12345678, 0x7fffffff, 0x0000007f, 0xffffffff}, {0x12345678, 0xffffffff, 0x123456f7, 0x7fffffff}},
+    {{0xdeadbeef, 0xffffffff, 0x7fffffff, 0x00000001}, {0xdeadbeef, 0xffffffff, 0x7fffffff, 0x00000001}, {0xdeadbeef, 0x00000001, 0x5eadbeee, 0xffffffff}},
+    {{0x80ff7f01, 0xdeadbeef, 0x12345678, 0x80000000}, {0x80ff7f01, 0xdeadbeef, 0x12345678, 0x80000000}, {0x80ff7f01, 0x80000000, 0x9333d579, 0xdeadbeef}},
+    {{0x00000000, 0x00000000, 0x00000000, 0x00000000}, {0x00000000, 0x00000000, 0x00000000, 0x00000000}, {0x00000000, 0x00000000, 0x00000000, 0x00000000}},
+    {{0x00000001, 0x0000007f, 0x00000080, 0x7fffffff}, {0x00000001, 0x0000007f, 0x00000080, 0x7fffffff}, {0x00000001, 0x7fffffff, 0x00000081, 0x0000007f}},
+    {{0x0000007f, 0x000000ff, 0x80000000, 0x80ff7f01}, {0x0000007f, 0x000000ff, 0x80000000, 0x80ff7f01}, {0x0000007f, 0x80ff7f01, 0x8000007f, 0x000000ff}},
+    {{0x00000080, 0x80000000, 0xdeadbeef, 0x000000ff}, {0x00000080, 0x80000000, 0xdeadbeef, 0x000000ff}, {0x00000080, 0x000000ff, 0xdeadbf6f, 0x80000000}},
+    {{0x000000ff, 0x12345678, 0x00000001, 0xdeadbeef}, {0x000000ff, 0x12345678, 0x00000001, 0xdeadbeef}, {0x000000ff, 0xdeadbeef, 0x00000100, 0x12345678}},
+    {{0x7fffffff, 0x80ff7f01, 0x000000ff, 0x00000080}, {0x7fffffff, 0x80ff7f01, 0x000000ff, 0x00000080}, {0x7fffffff, 0x00000080, 0x800000fe, 0x80ff7f01}},
+    {{0x2265b1f5, 0x91b7584a, 0xd8f16adf, 0xcd613e30}, {0x2265b1f5, 0x91b7584a, 0xd8f16adf, 0xcd613e30}, {0x2265b1f5, 0xcd613e30, 0xfb571cd4, 0x91b7584a}},
+    {{0xc386bbc4, 0x1027c4d1, 0x414c343c, 0x1e2feb89}, {0xc386bbc4, 0x1027c4d1, 0x414c343c, 0x1e2feb89}, {0xc386bbc4, 0x1e2feb89, 0x04d2f000, 0x1027c4d1}},
+    {{0x7ed4d57b, 0xc2ce6f44, 0x7311d8a3, 0x78e51061}, {0x7ed4d57b, 0xc2ce6f44, 0x7311d8a3, 0x78e51061}, {0x7ed4d57b, 0x78e51061, 0xf1e6ae1e, 0xc2ce6f44}},
+    {{0xa6cecc1b, 0x612e7696, 0xc9e9c616, 0x35bf992d}, {0xa6cecc1b, 0x612e7696, 0xc9e9c616, 0x35bf992d}, {0xa6cecc1b, 0x35bf992d, 0x70b89231, 0x612e7696}},
+    {{0x18072e8c, 0x7ce42c82, 0x0741c7a8, 0xe4b06ce6}, {0x18072e8c, 0x7ce42c82, 0x0741c7a8, 0xe4b06ce6}, {0x18072e8c, 0xe4b06ce6, 0x1f48f634, 0x7ce42c82}},
+    {{0xd5f4b3b2, 0x63ca828d, 0x6ec9d286, 0x9b810e76}, {0xd5f4b3b2, 0x63ca828d, 0x6ec9d286, 0x9b810e76}, {0xd5f4b3b2, 0x9b810e76, 0x44be8638, 0x63ca828d}},
+    {{0xc324c985, 0xc4647159, 0x008a05a6, 0xb2221a58}, {0xc324c985, 0xc4647159, 0x008a05a6, 0xb2221a58}, {0xc324c985, 0xb2221a58, 0xc3aecf2b, 0xc4647159}},
+    {{0x7204e52d, 0x442e3d43, 0xb8b6d8fe, 0xcd447e35}, {0x7204e52d, 0x442e3d43, 0xb8b6d8fe, 0xcd447e35}, {0x7204e52d, 0xcd447e35, 0x2abbbe2b, 0x442e3d43}},
+    {{0x3a902931, 0x9755d4c1, 0xf1fd42a2, 0x1a2b8f1f}, {0x3a902931, 0x9755d4c1, 0xf1fd42a2, 0x1a2b8f1f}, {0x3a902931, 0x1a2b8f1f, 0x2c8d6bd3, 0x9755d4c1}},
+    {{0xe6c3f339, 0x51431193, 0x07d4bedc, 0x05b6e6e3}, {0xe6c3f339, 0x51431193, 0x07d4bedc, 0x05b6e6e3}, {0xe6c3f339, 0x05b6e6e3, 0xee98b215, 0x51431193}},
+    {{0x06839eb9, 0xa648a7dd, 0x8a9a021e, 0x025b413f}, {0x06839eb9, 0xa648a7dd, 0x8a9a021e, 0x025b413f}, {0x06839eb9, 0x025b413f, 0x911da0d7, 0xa648a7dd}},
+    {{0xf06c144a, 0xe1988ad9, 0x619699cf, 0xafbd67f9}, {0xf06c144a, 0xe1988ad9, 0x619699cf, 0xafbd67f9}, {0xf06c144a, 0xafbd67f9, 0x5202ae19, 0xe1988ad9}},
+    {{0x37730edf, 0xf8130c42, 0x6c0fd4f5, 0xb9d179e0}, {0x37730edf, 0xf8130c42, 0x6c0fd4f5, 0xb9d179e0}, {0x37730edf, 0xb9d179e0, 0xa382e3d4, 0xf8130c42}},
+    {{0x076f3787, 0x8712b8bc, 0x38c0c8fd, 0xc381e88f}, {0x076f3787, 0x8712b8bc, 0x38c0c8fd, 0xc381e88f}, {0x076f3787, 0xc381e88f, 0x40300084, 0x8712b8bc}},
+    {{0x701966a0, 0xf06d3fef, 0x7eed8d14, 0x8d88348a}, {0x701966a0, 0xf06d3fef, 0x7eed8d14, 0x8d88348a}, {0x701966a0, 0x8d88348a, 0xef06f3b4, 0xf06d3fef}},
+    {{0x3bab6c39, 0x587fd280, 0x3b1a11df, 0xad45f23d}, {0x3bab6c39, 0x587fd280, 0x3b1a11df, 0xad45f23d}, {0x3bab6c39, 0xad45f23d, 0x76c57e18, 0x587fd280}}
+}};
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
+#endif
+        Require(block != nullptr, "flat cache bits: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, true);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%x", value);
+    return text;
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, const std::uint8_t* base) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(base));
+    std::vector<std::uint32_t> userData(10, 0u);
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    userData[8] = static_cast<std::uint32_t>(address);
+    userData[9] = static_cast<std::uint32_t>(address >> 32u);
+    const std::span<const std::uint32_t> code(FlatCacheBitsCode);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t waveSize) {
+    std::memset(guest.Data(), Fill, BlockBytes);
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        std::memcpy(guest.Data() + tid * 16u, Rows[tid % Rows.size()].memory.data(), 16u);
+    }
+    Output.fill(Sentinel);
+    Dispatch(device, waveSize, guest.Data());
+    const auto wave = "flat cache bits: wave" + std::to_string(waveSize) + " lane ";
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const auto& row = Rows[tid % Rows.size()];
+        for (std::uint32_t result = 0; result < Results; ++result) {
+            const auto actual = Output[tid * Results + result];
+            Require(actual == row.results[result], wave + std::to_string(tid) + " v" + std::to_string(10u + result) + " is " + Hex(actual) + ", expected " + Hex(row.results[result]));
+        }
+        std::array<std::uint32_t, 4> memory{};
+        std::memcpy(memory.data(), guest.Data() + tid * 16u, 16u);
+        for (std::uint32_t dword = 0; dword < 4u; ++dword) {
+            Require(memory[dword] == row.final[dword], wave + std::to_string(tid) + " memory dword " + std::to_string(dword) + " is " + Hex(memory[dword]) + ", expected " + Hex(row.final[dword]));
+        }
+    }
+    for (std::size_t offset = Threads * 16u; offset < BlockBytes; ++offset) {
+        Require(guest.Data()[offset] == Fill, wave + "store outside the rows changed byte " + std::to_string(offset));
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock guest;
+        Run(*device, guest, 32);
+        std::puts("flat cache bits tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/FlatD16Loads.cpp

@@ -31,31 +31,18 @@ constexpr std::size_t BlockBytes = 65536;
 constexpr std::size_t RegionOffset = Threads * 16u;
 alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
 
-alignas(256) constexpr std::array<std::uint32_t, 84> GlobalLoadsCode{
+alignas(256) constexpr std::array<std::uint32_t, 92> FlatD16LoadsCode{
     0x34020084, 0x34040086, 0xdc388000, 0x04080001, 0xbf8c3f70, 0x7e1402ff, 0x5a5a0000, 0x7e1602ff,
-    0x5a5a0001, 0x7e1802ff, 0x5a5a0002, 0x7e1a02ff, 0x5a5a0003, 0x7e1c02ff, 0x5a5a0004, 0x7e1e02ff,
-    0x5a5a0005, 0x7e2002ff, 0x5a5a0006, 0x7e2202ff, 0x5a5a0007, 0x7e2402ff, 0x5a5a0008, 0x7e2602ff,
-    0x5a5a0009, 0x7e2802ff, 0x5a5a000a, 0x7e2a02ff, 0x5a5a000b, 0x7e2c02ff, 0x5a5a000c, 0x7e2e02ff,
-    0x5a5a000d, 0x7e3002ff, 0x5a5a000e, 0x7e3202ff, 0x5a5a000f, 0x4a4008ff, 0x00000a00, 0x4a420eff,
+    0x5a5a0101, 0x7e1802ff, 0x5a5a0202, 0x7e1a02ff, 0x5a5a0303, 0x7e1c02ff, 0x5a5a0404, 0x7e1e02ff,
+    0x5a5a0505, 0x7e2002ff, 0x5a5a0606, 0x7e2202ff, 0x5a5a0707, 0x7e2402ff, 0x5a5a0808, 0x7e2602ff,
+    0x5a5a0909, 0x7e2802ff, 0x5a5a0a0a, 0x7e2a02ff, 0x5a5a0b0b, 0x7e2c02ff, 0x5a5a0c0c, 0x7e2e02ff,
+    0x5a5a0d0d, 0x7e3002ff, 0x5a5a0e0e, 0x7e3202ff, 0x5a5a0f0f, 0x4a4008ff, 0x00000a00, 0x4a420eff,
     0x00000200, 0xd70f6a22, 0x02024008, 0x7e460209, 0x50464680, 0x4a4c0aff, 0x00000200, 0xd70f6a24,
-    0x02024c08, 0x7e4a0209, 0x504a4a80, 0x36500c81, 0x7daa5080, 0xdc208800, 0x0a080020, 0xdc2487ff,
-    0x0b080020, 0xdc288ffd, 0x0c080020, 0xdc2c8001, 0x0d7d0022, 0xdc308fff, 0x0e080020, 0xdc348ffa,
-    0x0f7d0022, 0xdc3c8003, 0x11080020, 0xdc3887f8, 0x147d0022, 0xdc308000, 0x18080021, 0xdc208800,
-    0x197d0022, 0xbf8c0070, 0xbefe04c1, 0xe0781000, 0x80010a02, 0xe0781010, 0x80010e02, 0xe0781020,
-    0x80011202, 0xe0781030, 0x80011602, 0xbf810000
-};
-
-alignas(256) constexpr std::array<std::uint32_t, 84> FlatLoadsCode{
-    0x34020084, 0x34040086, 0xdc388000, 0x04080001, 0xbf8c3f70, 0x7e1402ff, 0x5a5a0000, 0x7e1602ff,
-    0x5a5a0001, 0x7e1802ff, 0x5a5a0002, 0x7e1a02ff, 0x5a5a0003, 0x7e1c02ff, 0x5a5a0004, 0x7e1e02ff,
-    0x5a5a0005, 0x7e2002ff, 0x5a5a0006, 0x7e2202ff, 0x5a5a0007, 0x7e2402ff, 0x5a5a0008, 0x7e2602ff,
-    0x5a5a0009, 0x7e2802ff, 0x5a5a000a, 0x7e2a02ff, 0x5a5a000b, 0x7e2c02ff, 0x5a5a000c, 0x7e2e02ff,
-    0x5a5a000d, 0x7e3002ff, 0x5a5a000e, 0x7e3202ff, 0x5a5a000f, 0x4a4008ff, 0x00000a00, 0x4a420eff,
-    0x00000200, 0xd70f6a22, 0x02024008, 0x7e460209, 0x50464680, 0x4a4c0aff, 0x00000200, 0xd70f6a24,
-    0x02024c08, 0x7e4a0209, 0x504a4a80, 0x36500c81, 0x7daa5080, 0xdc200000, 0x0a7d0024, 0xdc2407ff,
-    0x0b7d0024, 0xdc280001, 0x0c7d0024, 0xdc2c0003, 0x0d7d0024, 0xdc300002, 0x0e7d0024, 0xdc340001,
-    0x0f7d0024, 0xdc3c0007, 0x117d0024, 0xdc3807f1, 0x147d0024, 0xdc300400, 0x187d0024, 0xdc2807ff,
-    0x197d0024, 0xbf8c0070, 0xbefe04c1, 0xe0781000, 0x80010a02, 0xe0781010, 0x80010e02, 0xe0781020,
+    0x02024c08, 0x7e4a0209, 0x504a4a80, 0x36500c81, 0x7daa5080, 0xdc800000, 0x0a7d0024, 0xdc8407ff,
+    0x0b7d0024, 0xdc880001, 0x0c7d0024, 0xdc8c0003, 0x0d7d0024, 0xdc900001, 0x0e7d0024, 0xdc9407fe,
+    0x0f7d0024, 0xdc900005, 0x107d0024, 0xdc8c0407, 0x107d0024, 0xdc840400, 0x117d0024, 0xdc880009,
+    0x117d0024, 0xdc940007, 0x127d0024, 0xdc8007f8, 0x127d0024, 0xdc8807ff, 0x137d0024, 0xdc90000c,
+    0x137d0024, 0xbf8c0070, 0xbefe04c1, 0xe0781000, 0x80010a02, 0xe0781010, 0x80010e02, 0xe0781020,
     0x80011202, 0xe0781030, 0x80011602, 0xbf810000
 };
 
@@ -448,43 +435,42 @@ constexpr std::array<std::uint8_t, 6144> Region{
 
 struct Row {
     std::array<std::uint32_t, 4> input;
-    std::array<std::uint32_t, Results> global;
-    std::array<std::uint32_t, Results> flat;
+    std::array<std::uint32_t, Results> expected;
 };
 
 constexpr std::array<Row, Threads> Rows{{
-    {{0x00000000, 0x00000000, 0x3ceb3ffd, 0x00000000}, {0x000000d2, 0x00000029, 0x0000ebf6, 0x00004397, 0x4397357a, 0xf6804124, 0x97357aeb, 0x513c819d, 0x2e32095f, 0xefa22967, 0x6595f862, 0x295aec03, 0x523df3dc, 0x457d538e, 0x6526f2d2, 0x000000d2}, {0x000000d2, 0x0000007a, 0x000026f2, 0xffffa665, 0x0ca66526, 0xa66526f2, 0x89d2120c, 0x5d1889d2, 0x81e80e95, 0x6f160936, 0xe5522840, 0xd0e7469d, 0x80412474, 0x357aebf6, 0xfe1ec477, 0x0000357a}},
-    {{0x00000001, 0x00000001, 0x97b75093, 0x00000001}, {0x000000f2, 0xffffffdc, 0x00007aeb, 0xffff9d43, 0x9d439735, 0xebf68041, 0x4397357a, 0x5f513c81, 0x672e3209, 0x47efa229, 0x036595f8, 0xdc295aec, 0x8e523df3, 0x48457d53, 0xa66526f2, 0x000000f2}, {0x000000f2, 0x00000035, 0x00006526, 0x00000ca6, 0x120ca665, 0x0ca66526, 0x1889d212, 0x955d1889, 0x3681e80e, 0x6b6f1609, 0x9de55228, 0x74d0e746, 0xf6804124, 0x97357aeb, 0xe4fe1ec4, 0x00009735}},
-    {{0x00000002, 0x00000fa0, 0x8b529b4b, 0x00000002}, {0x00000026, 0xfffffff3, 0x0000357a, 0xffff819d, 0x819d4397, 0x7aebf680, 0x9d439735, 0x095f513c, 0x29672e32, 0xad47efa2, 0xec036595, 0xf3dc295a, 0x538e523d, 0xe048457d, 0x0ca66526, 0x00000026}, {0x000000a4, 0x0000005a, 0x0000449e, 0xffffadea, 0x9fadea44, 0xadea449e, 0x8aa0459f, 0xecce8aa0, 0x40199f09, 0x6f03851f, 0x97578278, 0x3c61fe8b, 0xbecaa1d3, 0xb65a60de, 0x37307a70, 0x0000b65a}},
-    {{0x00000003, 0x00000f9f, 0x21636369, 0x00000003}, {0x00000065, 0x0000003d, 0x00009735, 0x00003c81, 0x3c819d43, 0x357aebf6, 0x819d4397, 0x32095f51, 0xa229672e, 0x53ad47ef, 0x5aec0365, 0x3df3dc29, 0x7d538e52, 0xfce04845, 0x120ca665, 0x00000065}, {0x000000dd, 0x00000060, 0x00009ea4, 0xffffea44, 0xadea449e, 0xea449ea4, 0xa0459fad, 0xce8aa045, 0x199f09ec, 0x03851f40, 0x578278da, 0x61fe8b97, 0xcaa1d33c, 0x5a60debe, 0x307a70d9, 0x00005a60}},
-    {{0x000003ff, 0x00000002, 0x5eb561a5, 0x000017fc}, {0x00000077, 0x00000072, 0x00007ea7, 0x00003154, 0x31541eca, 0xa734f958, 0x541eca7e, 0x204cb651, 0x16a2f996, 0x660affc8, 0x30d907a2, 0x72bdad38, 0xa92515b0, 0x948e5f94, 0xa2731cb1, 0x00000077}, {0x00000026, 0xffffff97, 0x0000a665, 0x0000120c, 0xd2120ca6, 0x120ca665, 0x5d1889d2, 0x0e955d18, 0x093681e8, 0x116b6f16, 0x469de552, 0x2474d0e7, 0xebf68041, 0x4397357a, 0x8ce4fe1e, 0x00004397}},
-    {{0x000003fe, 0x00000003, 0xea7b5bf4, 0x000017fb}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
-    {{0x000003fd, 0x000007ff, 0x9a9a80fd, 0x000017fa}, {0x00000049, 0xffffffad, 0x000034f9, 0x00001eca, 0x1eca7ea7, 0xf95881df, 0xca7ea734, 0xb6513154, 0xf996204c, 0xffc816a2, 0x07a2e287, 0xad3830d9, 0x15b072bd, 0x5f94a925, 0x1cb172ff, 0x00000049}, {0x0000007a, 0x0000005a, 0x00009735, 0xffff9d43, 0x819d4397, 0x9d439735, 0x5f513c81, 0x32095f51, 0xa229672e, 0x53ad47ef, 0x553e074f, 0x8b833038, 0x6595f862, 0x295aec03, 0x5131541e, 0x0000295a}},
-    {{0x00000201, 0x00000801, 0x795b929f, 0x00000ffd}, {0x000000c6, 0xffffff91, 0x00002075, 0xffffb0c1, 0xb0c19888, 0x75f13ba1, 0xc1988820, 0x08a59ac0, 0x85539459, 0x73d7de27, 0x62c3c5bc, 0x91b7a807, 0x63034f25, 0x2db116b5, 0xdc295aec, 0x000000c6}, {0x00000097, 0xffffffdc, 0x00009d43, 0x00003c81, 0x513c819d, 0x3c819d43, 0x32095f51, 0x672e3209, 0x47efa229, 0x60e553ad, 0x3038553e, 0xf8628b83, 0xec036595, 0xf3dc295a, 0x4cb65131, 0x0000f3dc}},
-    {{0x00000086, 0x000009b0, 0xe8a8529f, 0x0000006b}, {0x000000f1, 0x00000054, 0x00001873, 0xffffdf25, 0xdf25a310, 0x7326b53e, 0x25a31018, 0x6645c8aa, 0x02f743cf, 0x8fd2a50e, 0x3b946093, 0x54ebd216, 0x9bd7f852, 0x53f53ed6, 0x7dee2657, 0x000000f1}, {0x00000000, 0x00000058, 0x00003386, 0xffffc349, 0xc0c34933, 0xc3493386, 0x010dfac0, 0x7d59010d, 0xcbb17d18, 0x7ef72fd3, 0x1ff610a6, 0x59e9d36c, 0x30633e8d, 0x3c584877, 0xb4de5e8e, 0x00003c58}},
-    {{0x000003c0, 0x00000426, 0x3bfd1d33, 0x000011a0}, {0x000000a8, 0x0000002d, 0x0000a25c, 0xffff9c05, 0x9c0507a5, 0x5ca32e13, 0x0507a5a2, 0xeebcae0b, 0xfbcf54ff, 0x7c7b8218, 0x02701056, 0x2dd3f4aa, 0x602ab9e7, 0xcd71014b, 0xf610a667, 0x000000a8}, {0x00000021, 0xffffff8b, 0x0000d19a, 0xffff82a1, 0xc782a1d1, 0x82a1d19a, 0xb41ce3c7, 0x3b5db41c, 0x7ce0e57f, 0x00280664, 0x300b011b, 0x9b79ebd2, 0xc90fa8c4, 0x9c8be880, 0x401ae5e1, 0x00009c8b}},
-    {{0x00000188, 0x00000b79, 0x8a7d43b5, 0x00000f0c}, {0x000000d7, 0xffffffb6, 0x00004159, 0xffffb70c, 0xb70c48ee, 0x5905426c, 0x0c48ee41, 0x52e95ec2, 0x809af6c4, 0x9e49d979, 0x693895b7, 0xb6e0664f, 0xdeca057c, 0x2b2c163e, 0x3e052289, 0x000000d7}, {0x000000fa, 0x0000003d, 0x000017c8, 0xffffe6ab, 0xe4e6ab17, 0xe6ab17c8, 0xa2c98ce4, 0x4cd6a2c9, 0x36b17e32, 0xd6bd1487, 0x5bd3a52f, 0x225e3a51, 0xd65eeb8d, 0x0e3d40d4, 0x3f5aefae, 0x00000e3d}},
-    {{0x000003cf, 0x0000065a, 0xdc6bf1e1, 0x00001473}, {0x0000003b, 0x00000015, 0x00007b82, 0x000024e5, 0x24e5c17c, 0x8218fbcf, 0xe5c17c7b, 0x6ab73608, 0x185620a0, 0x575da8dc, 0x59ac90cd, 0x15783291, 0x7584528a, 0xe888f86d, 0xdebaea1d, 0x0000003b}, {0x0000008d, 0x00000074, 0x0000538b, 0xfffffa29, 0x6dfa2953, 0xfa29538b, 0xfc1ae26d, 0x4312fc1a, 0x1835159f, 0xb5fd7f6b, 0x4506273f, 0x554bd692, 0x7d422acd, 0xe774511b, 0xda660308, 0x0000e774}},
-    {{0x00000134, 0x000003b5, 0x26d0b944, 0x00001450}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
-    {{0x0000031e, 0x00000bdc, 0xabe19f59, 0x0000007c}, {0x00000021, 0x00000071, 0x00007769, 0xffff8cc2, 0x8cc24bff, 0x69d0cbd1, 0xc24bff77, 0xd5c720a6, 0xd9c85a78, 0x60b4443a, 0xf6724fb3, 0x7153291c, 0x4a5c9177, 0x0b128e2b, 0x7f9859b1, 0x00000021}, {0x0000005d, 0x00000066, 0x00007957, 0xffff86c7, 0x8d86c779, 0x86c77957, 0x35e9c58d, 0x6f4835e9, 0x8d406c57, 0x4a4ad30d, 0x528a1578, 0xf86d7584, 0xdda0e888, 0xf666f927, 0xb19c74d6, 0x0000f666}},
-    {{0x00000083, 0x0000028c, 0x0af438d3, 0x000012ea}, {0x00000011, 0x00000016, 0x0000b53e, 0x00001018, 0x10187326, 0x3ecdef03, 0x187326b5, 0xaadf25a3, 0xcf6645c8, 0x0e02f743, 0x936f25cd, 0x163b9460, 0x5254ebd2, 0xd69bd7f8, 0x224cb259, 0x00000011}, {0x00000010, 0x0000006e, 0x0000fc1c, 0xffffc9eb, 0x3ac9ebfc, 0xc9ebfc1c, 0x1ae0f83a, 0x43151ae0, 0xc7e70a45, 0x21c1452e, 0xc6e0e1a7, 0xda95b3a4, 0xa42ead3a, 0x506e741f, 0x3c3e2f3f, 0x0000506e}},
-    {{0x00000268, 0x00000c7b, 0xd2d58443, 0x000000fe}, {0x00000065, 0xffffff90, 0x0000764b, 0xffff9a41, 0x9a412405, 0x4ba1fb80, 0x41240576, 0xbd0167bc, 0x6edae83e, 0xbf9692b3, 0xa14f8d0a, 0x902df1a3, 0x7f913ad6, 0xec4185b7, 0x99571c29, 0x00000065}, {0x00000015, 0xffffffbe, 0x000078a4, 0xffffe536, 0x26e53678, 0xe53678a4, 0x6d03a026, 0x02016d03, 0xfc1fabaf, 0x3716dbf7, 0xfd9f16e3, 0x1d9033df, 0x5adebaea, 0xd7be5d2b, 0x936f25cd, 0x0000d7be}},
-    {{0x00000227, 0x00000790, 0xb804d821, 0x00001308}, {0x00000053, 0x00000067, 0x00004431, 0xffffd3b4, 0xd3b4c14d, 0x31813cb2, 0xb4c14d44, 0xb9279ed7, 0xb93ff927, 0x59859a53, 0xb7d12049, 0x675b4866, 0xc676e8d8, 0xdca0e1a0, 0x034ab01d, 0x00000053}, {0x000000a0, 0x0000006b, 0x00004ec7, 0xffffeba6, 0x4eeba64e, 0xeba64ec7, 0x6c3f954e, 0xa8636c3f, 0x7080725e, 0x9e00052d, 0x871b5d02, 0x6e53112f, 0x05ab81b3, 0xf86b2339, 0x6b431e8e, 0x0000f86b}},
-    {{0x00000319, 0x00000b6d, 0x651c5253, 0x00000da9}, {0x000000fd, 0x00000072, 0x0000d4a3, 0x000069d0, 0x69d0cbd1, 0xa3aa5a25, 0xd0cbd1d4, 0xc24bff77, 0xc720a68c, 0xc85a78d5, 0x324d47e5, 0x724fb31c, 0x53291cf6, 0x5c917771, 0x15602efe, 0x000000fd}, {0x000000c8, 0xffffffd3, 0x00003527, 0x000067e7, 0xca67e735, 0x67e73527, 0x9c2a88ca, 0xb0e49c2a, 0x17c8fa9b, 0x8ce4e6ab, 0x1a2baeed, 0x4fd4b8c8, 0x61539dbe, 0x5bd3a52f, 0xd052e9fc, 0x00005bd3}},
-    {{0x0000038e, 0x00000f6c, 0xe0fd67dd, 0x0000044a}, {0x00000086, 0xfffffffe, 0x000070d6, 0xffff8e1d, 0x8e1de1ab, 0xd6bd1487, 0x1de1ab70, 0x3b6b431e, 0x7e7923d3, 0x777b0e8e, 0x6c7e37f1, 0xfe28bb7f, 0xa0949a4c, 0x3ab02414, 0x65d656c0, 0x00000086}, {0x000000cd, 0xfffffffd, 0x0000e9fc, 0xffffd052, 0x66d052e9, 0xd052e9fc, 0x0d8fd866, 0x84530d8f, 0xefaef525, 0x6cde3f5a, 0x68e162c2, 0xffe524cd, 0x9b3d01a2, 0x41fded80, 0x3a515bd3, 0x000041fd}},
-    {{0x000002ec, 0x0000018f, 0x22cedafa, 0x00000125}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
-    {{0x000003f5, 0x00000378, 0xf729b4c9, 0x00000841}, {0x00000019, 0x00000057, 0x00005567, 0xffffdf45, 0xdf45fb80, 0x677ed35a, 0x45fb8055, 0x34f95881, 0x1eca7ea7, 0xb6513154, 0xa5a87b5e, 0x57dd4c23, 0x07a2e287, 0xad3830d9, 0xd14b0cf5, 0x00000019}, {0x000000fc, 0xffffffb0, 0x0000267e, 0xffffb948, 0x9eb94826, 0xb948267e, 0x0b25a49e, 0xd5d30b25, 0xa083e4b7, 0xcfb3bb6d, 0x5873c62f, 0xe73527c8, 0x2a88ca67, 0x9bb0e49c, 0xf83f261f, 0x00009bb0}},
-    {{0x0000037d, 0x00000c76, 0xdaf0105b, 0x0000140d}, {0x0000009e, 0xffffff91, 0x000017c8, 0xffff8ce4, 0x8ce4e6ab, 0xc8fa9bb0, 0xe4e6ab17, 0x4cd6a2c9, 0x36b17e32, 0xd6bd1487, 0x3d40d4d6, 0x911b0a0e, 0xd1eba0cd, 0xe767b4ff, 0x1d426570, 0x0000009e}, {0x00000079, 0xffffffba, 0x000069c5, 0x00001a8c, 0x151a8c69, 0x1a8c69c5, 0x3678a415, 0x26e53678, 0x016d03a0, 0x1fabaf02, 0x6d22b51e, 0x9f16e3e8, 0x9033dffd, 0xdebaea1d, 0x892f4c71, 0x0000deba}},
-    {{0x00000268, 0x000006bd, 0xd55ec1a5, 0x0000103b}, {0x00000065, 0xffffff90, 0x0000764b, 0xffff9a41, 0x9a412405, 0x4ba1fb80, 0x41240576, 0xbd0167bc, 0x6edae83e, 0xbf9692b3, 0xa14f8d0a, 0x902df1a3, 0x7f913ad6, 0xec4185b7, 0x79fec2ec, 0x00000065}, {0x00000013, 0x00000033, 0x0000835f, 0x00002ddd, 0x722ddd83, 0x2ddd835f, 0xc6429a72, 0xaac7c642, 0xa31b01f2, 0x599eb598, 0x2c46c27e, 0xab9c2334, 0x31cfa0b5, 0x02334e95, 0x19598433, 0x00000233}},
-    {{0x00000316, 0x0000092f, 0x88bafad9, 0x00000b3a}, {0x000000a2, 0x0000001c, 0x00005a25, 0xffffd1d4, 0xd1d4a3aa, 0x25f6d039, 0xd4a3aa5a, 0x7769d0cb, 0x8cc24bff, 0xd5c720a6, 0xe50f0b34, 0x1c324d47, 0xf6724fb3, 0x7153291c, 0x524da73e, 0x000000a2}, {0x000000ca, 0x0000001b, 0x00006436, 0xffff959f, 0x13959f64, 0x959f6436, 0x2be99013, 0x08252be9, 0x9f1b1c06, 0xfa5829ed, 0x24ff8388, 0x9a693293, 0x8428251f, 0x071b82c2, 0x1588587f, 0x0000071b}},
-    {{0x00000342, 0x00000959, 0xe779c471, 0x0000076f}, {0x0000008c, 0x00000017, 0x00006d7c, 0xffffa815, 0xa815a39f, 0x7c7a524d, 0x15a39f6d, 0xed275ce5, 0x0e62da4d, 0xe790d315, 0x528bc830, 0x17eb5b02, 0x9da049a4, 0xb3a7bbef, 0x660a3790, 0x0000008c}, {0x00000088, 0x0000000d, 0x0000aae4, 0xffffe110, 0xdee110aa, 0xe110aae4, 0xb6eac1de, 0x62f1b6ea, 0x34343f1b, 0xf308081c, 0xdb19a044, 0x1095f2e7, 0x9f733159, 0x380d0562, 0xe242e585, 0x0000380d}},
-    {{0x000002b1, 0x00000ae9, 0xdb3d1151, 0x000000ea}, {0x00000047, 0x0000000d, 0x0000d463, 0x00001bf9, 0x1bf9e87d, 0x6302c753, 0xf9e87dd4, 0x378b4009, 0xf3c8b729, 0x598433f0, 0xa0d6d4b0, 0x0d8273a0, 0x6d34b2ad, 0xedd883ac, 0xaedff3c7, 0x00000047}, {0x00000013, 0xffffffef, 0x0000ea6b, 0xffffa06b, 0xb2a06bea, 0xa06bea6b, 0x945aacb2, 0x1b43945a, 0x66bd4d39, 0xf886f4f0, 0x7cb4af22, 0xb43c7bab, 0xb183013d, 0xa4ef2271, 0x72ee855b, 0x0000a4ef}},
-    {{0x0000023c, 0x000009b1, 0xb2109306, 0x0000157b}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
-    {{0x0000014e, 0x00000b2d, 0xf6f22f41, 0x00000a71}, {0x000000c7, 0xffffffdb, 0x0000b20a, 0x0000100b, 0x100bbc11, 0x0aa2a4b1, 0x0bbc11b2, 0x5dc397db, 0xd1f4d133, 0x10aae488, 0x082b465a, 0xdb19a044, 0x1095f2e7, 0x9f733159, 0x92b36eda, 0x000000c7}, {0x000000af, 0x00000015, 0x0000fb40, 0xffffad6d, 0x2fad6dfb, 0xad6dfb40, 0xce007b2f, 0xccb8ce00, 0xa73e5b47, 0x7c7a524d, 0x4a5c9177, 0x0b128e2b, 0xfafd7702, 0xbf157cc0, 0xde591a71, 0x0000bf15}},
-    {{0x000000d5, 0x00000b6b, 0x360c4979, 0x000014f9}, {0x00000025, 0x0000002b, 0x00006fd8, 0x00001477, 0x1477e9be, 0xd8600189, 0x77e9be6f, 0x2c73a7bd, 0x421aff39, 0x1f09a43b, 0x87b15796, 0x2b17d0bf, 0xfa5d515c, 0x834fd313, 0x63779354, 0x00000025}, {0x00000073, 0x0000002f, 0x0000c858, 0x00003527, 0xe73527c8, 0x3527c858, 0x88ca67e7, 0x9c2a88ca, 0xfa9bb0e4, 0xe6ab17c8, 0xaeede5d8, 0xb8c81a2b, 0x9dbe4fd4, 0xa52f6153, 0xe9fccd90, 0x0000a52f}},
-    {{0x00000222, 0x0000048f, 0x103ef3c3, 0x000003fb}, {0x000000c2, 0xffffffd1, 0x0000e2a4, 0x00003181, 0x31813cb2, 0xa4765520, 0x813cb2e2, 0xb4c14d44, 0x279ed7d3, 0x3ff927b9, 0x164fb710, 0xd1204939, 0x5b4866b7, 0x76e8d867, 0x764984b5, 0x000000c2}, {0x0000009d, 0xffffffde, 0x000023c0, 0xffffdea4, 0x49dea423, 0xdea423c0, 0xe90c7c49, 0x8cede90c, 0x6a782b20, 0x414c4857, 0x6d03a026, 0xabaf0201, 0xdbf7fc1f, 0x1fde3716, 0xa50e02f7, 0x00001fde}},
-    {{0x000003db, 0x00000da9, 0xfd63ed5b, 0x00001470}, {0x00000049, 0xfffffff9, 0x00001856, 0x0000575d, 0x575da8dc, 0x5620a06a, 0x5da8dc18, 0x8d86c779, 0x4835e9c5, 0x406c576f, 0xe888f86d, 0xf927dda0, 0x9e9bf666, 0x0fcfcf14, 0x1d9033df, 0x00000049}, {0x000000fe, 0xffffff9a, 0x0000602e, 0x00000715, 0x0d071560, 0x0715602e, 0x5e8e080d, 0xb4de5e8e, 0xd8f27c75, 0xdc10e5e8, 0xa02ac205, 0x05b43c44, 0x72660c37, 0x489ae433, 0x30633e8d, 0x0000489a}},
-    {{0x000003de, 0x0000016a, 0xccf3d0b2, 0x00000b02}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}}
+    {{0x00000000, 0x00000000, 0x3ceb3ffd, 0x00000000}, {0x5a5a00d2, 0x007a0101, 0x5a5afff2, 0x00650303, 0x5a5a26f2, 0x7aeb0505, 0x004f120c, 0x00770018, 0x89d200d0, 0x5a5ae80e, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000001, 0x00000001, 0x97b75093, 0x00000001}, {0x5a5a00f2, 0x00350101, 0x5a5a0026, 0xffa60303, 0x5a5a6526, 0x357a0505, 0xfffad212, 0x00c4005d, 0x18890074, 0x5a5a81e8, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000002, 0x00000fa0, 0x8b529b4b, 0x00000002}, {0x5a5a00a4, 0x005a0101, 0x5a5aff9e, 0xffea0303, 0x5a5a449e, 0x5a600505, 0x0008459f, 0x0070ffce, 0x8aa0003c, 0x5a5a199f, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000003, 0x00000f9f, 0x21636369, 0x00000003}, {0x5a5a00dd, 0x00600101, 0x5a5affa4, 0x00440303, 0x5a5a9ea4, 0x60de0505, 0x00009fad, 0x00d9ff8a, 0xa0450061, 0x5a5a9f09, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x000003ff, 0x00000002, 0x5eb561a5, 0x000017fc}, {0x5a5a0026, 0x00970101, 0x5a5a0065, 0x000c0303, 0x5a5aa665, 0x97350505, 0x001589d2, 0x001eff95, 0x5d180024, 0x5a5a3681, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x000003fe, 0x00000003, 0xea7b5bf4, 0x000017fb}, {0x5a5a0000, 0x5a5a0101, 0x5a5a0202, 0x5a5a0303, 0x5a5a0404, 0x5a5a0505, 0x5a5a0606, 0x5a5a0707, 0x5a5a0808, 0x5a5a0909, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x000003fd, 0x000007ff, 0x9a9a80fd, 0x000017fa}, {0x5a5a007a, 0x005a0101, 0x5a5a0035, 0x00430303, 0x5a5a9735, 0x5aec0505, 0xff963c81, 0x001e0009, 0x5f51008b, 0x5a5a2967, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000201, 0x00000801, 0x795b929f, 0x00000ffd}, {0x5a5a0097, 0x00dc0101, 0x5a5a0043, 0xff810303, 0x5a5a9d43, 0xdc290505, 0xffa25f51, 0x0031002e, 0x320900f8, 0x5a5aefa2, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000086, 0x000009b0, 0xe8a8529f, 0x0000006b}, {0x5a5a0000, 0x00580101, 0x5a5aff86, 0x00490303, 0x5a5a3386, 0x58480505, 0xffd8fac0, 0x008e0059, 0x010d0059, 0x5a5ab17d, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x000003c0, 0x00000426, 0x3bfd1d33, 0x000011a0}, {0x5a5a0021, 0x008b0101, 0x5a5aff9a, 0xffa10303, 0x5a5ad19a, 0x8be80505, 0xffdbe3c7, 0x00e1005d, 0xb41c009b, 0x5a5ae0e5, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000188, 0x00000b79, 0x8a7d43b5, 0x00000f0c}, {0x5a5a00fa, 0x003d0101, 0x5a5affc8, 0xffab0303, 0x5a5a17c8, 0x3d400505, 0xffa18ce4, 0x00aeffd6, 0xa2c90022, 0x5a5ab17e, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x000003cf, 0x0000065a, 0xdc6bf1e1, 0x00001473}, {0x5a5a008d, 0x00740101, 0x5a5aff8b, 0x00290303, 0x5a5a538b, 0x74510505, 0x0038e26d, 0x00080012, 0xfc1a0055, 0x5a5a3515, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000134, 0x000003b5, 0x26d0b944, 0x00001450}, {0x5a5a0000, 0x5a5a0101, 0x5a5a0202, 0x5a5a0303, 0x5a5a0404, 0x5a5a0505, 0x5a5a0606, 0x5a5a0707, 0x5a5a0808, 0x5a5a0909, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x0000031e, 0x00000bdc, 0xabe19f59, 0x0000007c}, {0x5a5a005d, 0x00660101, 0x5a5a0057, 0xffc70303, 0x5a5a7957, 0x66f90505, 0x0062c58d, 0x00d60048, 0x35e900f8, 0x5a5a406c, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000083, 0x0000028c, 0x0af438d3, 0x000012ea}, {0x5a5a0010, 0x006e0101, 0x5a5a001c, 0xffeb0303, 0x5a5afc1c, 0x6e740505, 0xffe8f83a, 0x003f0015, 0x1ae000da, 0x5a5ae70a, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000268, 0x00000c7b, 0xd2d58443, 0x000000fe}, {0x5a5a0015, 0x00be0101, 0x5a5affa4, 0x00360303, 0x5a5a78a4, 0xbe5d0505, 0x0016a026, 0x00cd0001, 0x6d03001d, 0x5a5a1fab, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000227, 0x00000790, 0xb804d821, 0x00001308}, {0x5a5a00a0, 0x006b0101, 0x5a5affc7, 0xffa60303, 0x5a5a4ec7, 0x6b230505, 0x0079954e, 0x008e0063, 0x6c3f006e, 0x5a5a8072, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000319, 0x00000b6d, 0x651c5253, 0x00000da9}, {0x5a5a00c8, 0x00d30101, 0x5a5a0027, 0xffe70303, 0x5a5a3527, 0xd3a50505, 0x000d88ca, 0x00fcffe4, 0x9c2a004f, 0x5a5ac8fa, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x0000038e, 0x00000f6c, 0xe0fd67dd, 0x0000044a}, {0x5a5a00cd, 0x00fd0101, 0x5a5afffc, 0x00520303, 0x5a5ae9fc, 0xfded0505, 0xffebd866, 0x00d30053, 0x0d8f00ff, 0x5a5aaef5, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x000002ec, 0x0000018f, 0x22cedafa, 0x00000125}, {0x5a5a0000, 0x5a5a0101, 0x5a5a0202, 0x5a5a0303, 0x5a5a0404, 0x5a5a0505, 0x5a5a0606, 0x5a5a0707, 0x5a5a0808, 0x5a5a0909, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x000003f5, 0x00000378, 0xf729b4c9, 0x00000841}, {0x5a5a00fc, 0x00b00101, 0x5a5a007e, 0x00480303, 0x5a5a267e, 0xb0e40505, 0x0031a49e, 0x001fffd3, 0x0b2500e7, 0x5a5a83e4, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x0000037d, 0x00000c76, 0xdaf0105b, 0x0000140d}, {0x5a5a0079, 0x00ba0101, 0x5a5affc5, 0xff8c0303, 0x5a5a69c5, 0xbaea0505, 0x006fa415, 0x0071ffe5, 0x3678009f, 0x5a5a6d03, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000268, 0x000006bd, 0xd55ec1a5, 0x0000103b}, {0x5a5a0013, 0x00330101, 0x5a5a005f, 0xffdd0303, 0x5a5a835f, 0x334e0505, 0xff8a9a72, 0x0033ffc7, 0xc64200ab, 0x5a5a1b01, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000316, 0x0000092f, 0x88bafad9, 0x00000b3a}, {0x5a5a00ca, 0x001b0101, 0x5a5a0036, 0xff9f0303, 0x5a5a6436, 0x1b820505, 0xffcc9013, 0x007f0025, 0x2be9009a, 0x5a5a1b1c, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000342, 0x00000959, 0xe779c471, 0x0000076f}, {0x5a5a0088, 0x000d0101, 0x5a5affe4, 0x00100303, 0x5a5aaae4, 0x0d050505, 0x007ac1de, 0x0085fff1, 0xb6ea0010, 0x5a5a343f, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x000002b1, 0x00000ae9, 0xdb3d1151, 0x000000ea}, {0x5a5a0013, 0x00ef0101, 0x5a5a006b, 0x006b0303, 0x5a5aea6b, 0xef220505, 0x001eacb2, 0x005b0043, 0x945a00b4, 0x5a5abd4d, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x0000023c, 0x000009b1, 0xb2109306, 0x0000157b}, {0x5a5a0000, 0x5a5a0101, 0x5a5a0202, 0x5a5a0303, 0x5a5a0404, 0x5a5a0505, 0x5a5a0606, 0x5a5a0707, 0x5a5a0808, 0x5a5a0909, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x0000014e, 0x00000b2d, 0xf6f22f41, 0x00000a71}, {0x5a5a00af, 0x00150101, 0x5a5a0040, 0x006d0303, 0x5a5afb40, 0x157c0505, 0x000b7b2f, 0x0071ffb8, 0xce00000b, 0x5a5a3e5b, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x000000d5, 0x00000b6b, 0x360c4979, 0x000014f9}, {0x5a5a0073, 0x002f0101, 0x5a5a0058, 0x00270303, 0x5a5ac858, 0x2f610505, 0xffd867e7, 0x0090002a, 0x88ca00b8, 0x5a5a9bb0, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x00000222, 0x0000048f, 0x103ef3c3, 0x000003fb}, {0x5a5a009d, 0x00de0101, 0x5a5affc0, 0xffa40303, 0x5a5a23c0, 0xde370505, 0x00597c49, 0x00f7ffed, 0xe90c00ab, 0x5a5a782b, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x000003db, 0x00000da9, 0xfd63ed5b, 0x00001470}, {0x5a5a00fe, 0x009a0101, 0x5a5a002e, 0x00150303, 0x5a5a602e, 0x9ae40505, 0x003c080d, 0x008dffde, 0x5e8e0005, 0x5a5af27c, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}},
+    {{0x000003de, 0x0000016a, 0xccf3d0b2, 0x00000b02}, {0x5a5a0000, 0x5a5a0101, 0x5a5a0202, 0x5a5a0303, 0x5a5a0404, 0x5a5a0505, 0x5a5a0606, 0x5a5a0707, 0x5a5a0808, 0x5a5a0909, 0x5a5a0a0a, 0x5a5a0b0b, 0x5a5a0c0c, 0x5a5a0d0d, 0x5a5a0e0e, 0x5a5a0f0f}}
 }};
 
 class GuestBlock {
@@ -495,7 +481,7 @@ public:
 #else
         block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
 #endif
-        Require(block != nullptr, "flat global loads: cannot allocate the guest block");
+        Require(block != nullptr, "flat d16 loads: cannot allocate the guest block");
         GuestAllocations::Mutation().Add(block, BlockBytes, true, false);
     }
 
@@ -549,12 +535,12 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> co
     device.WaitIdle();
 }
 
-void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::span<const std::uint32_t> code, bool flat) {
+void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest) {
     Output.fill(Sentinel);
-    Dispatch(device, code, guest.Data());
-    const auto wave = std::string(flat ? "flat" : "global") + " loads: lane ";
+    Dispatch(device, FlatD16LoadsCode, guest.Data());
+    const std::string wave = "flat d16 loads: lane ";
     for (std::uint32_t tid = 0; tid < Threads; ++tid) {
-        const auto& expected = flat ? Rows[tid].flat : Rows[tid].global;
+        const auto& expected = Rows[tid].expected;
         for (std::uint32_t result = 0; result < Results; ++result) {
             const auto actual = Output[tid * Results + result];
             Require(actual == expected[result], wave + std::to_string(tid) + " v" + std::to_string(10u + result) + " is " + Hex(actual) + ", expected " + Hex(expected[result]));
@@ -574,9 +560,8 @@ int main() {
             std::memcpy(guest.Data() + tid * 16u, Rows[tid].input.data(), 16u);
         }
         std::memcpy(guest.Data() + RegionOffset, Region.data(), Region.size());
-        Run(*device, guest, GlobalLoadsCode, false);
-        Run(*device, guest, FlatLoadsCode, true);
-        std::puts("flat global loads tests passed");
+        Run(*device, guest);
+        std::puts("flat d16 loads tests passed");
         return 0;
     } catch (const std::exception& error) {
         std::cerr << error.what() << '\n';

core/libs/prx/libSceAgcDriver/tests/execution/FlatGlobalLoadsRegion.hpp

@@ -0,0 +1,394 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_EXECUTION_FLATGLOBALLOADSREGION_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_EXECUTION_FLATGLOBALLOADSREGION_HPP
+
+#include <array>
+#include <cstdint>
+
+inline constexpr std::array<std::uint8_t, 6144> FlatGlobalLoadsRegion{
+    0xd2, 0xf2, 0x26, 0x65, 0xa6, 0x0c, 0x12, 0xd2, 0x89, 0x18, 0x5d, 0x95, 0x0e, 0xe8, 0x81, 0x36,
+    0x09, 0x16, 0x6f, 0x6b, 0x11, 0x3d, 0x17, 0x8d, 0x6c, 0x0f, 0xd3, 0x90, 0x1f, 0xf2, 0x39, 0xa1,
+    0xa0, 0x95, 0xf2, 0x0f, 0x93, 0x95, 0x65, 0x0c, 0xf9, 0x38, 0x0b, 0x8e, 0xdb, 0x22, 0x4a, 0x6b,
+    0x24, 0x8a, 0x1e, 0x92, 0x4e, 0x8f, 0xd0, 0xae, 0x2e, 0x1a, 0x94, 0x92, 0xa3, 0x30, 0x5f, 0x18,
+    0x8c, 0xb6, 0x10, 0x90, 0x0f, 0x9e, 0x34, 0x7f, 0xae, 0x88, 0x6d, 0xc6, 0x50, 0x77, 0x95, 0xec,
+    0x74, 0x5c, 0x4c, 0x3f, 0xcb, 0x2e, 0xb2, 0xc7, 0x3e, 0x14, 0x93, 0x4c, 0x86, 0x7e, 0xe0, 0x57,
+    0xba, 0xf2, 0x49, 0x9b, 0xfa, 0x12, 0x1e, 0x83, 0x6b, 0x2a, 0xc1, 0x57, 0x26, 0xee, 0x7d, 0x6b,
+    0x0a, 0xf6, 0xab, 0x13, 0xc3, 0x8e, 0x92, 0xca, 0xe0, 0xd1, 0x50, 0x57, 0xb1, 0x59, 0x98, 0x7f,
+    0x94, 0xcc, 0x74, 0x11, 0xd7, 0x17, 0xf1, 0x45, 0x79, 0xb2, 0xaa, 0x10, 0x0f, 0xbb, 0xb3, 0x4f,
+    0xa5, 0x93, 0xfe, 0xae, 0xd2, 0x72, 0x48, 0xb7, 0x62, 0xe3, 0xab, 0x58, 0x05, 0xf0, 0x76, 0x5a,
+    0x2b, 0x9c, 0x1d, 0x7e, 0x0f, 0x37, 0xc4, 0x49, 0x21, 0xbd, 0x3f, 0x65, 0x64, 0xea, 0xdf, 0x7f,
+    0x14, 0x2a, 0x72, 0x66, 0x8c, 0x47, 0xe2, 0x23, 0xd1, 0x6e, 0xdd, 0x8c, 0x47, 0xb4, 0x6a, 0xfc,
+    0x5b, 0xae, 0xe2, 0x61, 0xf5, 0x3b, 0x26, 0x15, 0x2d, 0x26, 0x3b, 0xa8, 0x3b, 0x03, 0x7c, 0xd4,
+    0x96, 0x2e, 0x43, 0x48, 0x01, 0x25, 0x6b, 0x88, 0x5e, 0x9c, 0x90, 0x51, 0xf3, 0x20, 0xb0, 0xdb,
+    0x83, 0xf3, 0x9e, 0xa7, 0xad, 0xbd, 0x0d, 0x74, 0xe6, 0xde, 0xc7, 0xf3, 0xdf, 0xae, 0xcc, 0x8f,
+    0x64, 0x65, 0x66, 0x64, 0x1a, 0x7b, 0xa2, 0x66, 0x0f, 0x30, 0x11, 0xfc, 0x35, 0x70, 0x29, 0x1c,
+    0x57, 0x99, 0x0d, 0x1a, 0x00, 0x91, 0x26, 0x89, 0x19, 0xf2, 0x5d, 0x9d, 0x06, 0x12, 0xdf, 0x35,
+    0x9d, 0x60, 0x26, 0xa2, 0x40, 0xf4, 0x58, 0x9a, 0x5d, 0x79, 0x1f, 0x1d, 0xd9, 0x7c, 0xfe, 0xfa,
+    0x77, 0x7a, 0x7b, 0xcf, 0x15, 0x24, 0x1a, 0xbf, 0x57, 0xbd, 0x43, 0x7a, 0xd4, 0xb1, 0x29, 0x84,
+    0x05, 0x34, 0xf3, 0xf3, 0x87, 0x5c, 0x25, 0xb0, 0x8b, 0xea, 0x06, 0xc2, 0x87, 0x4c, 0xfa, 0xa4,
+    0xdd, 0x17, 0xb2, 0xd8, 0x42, 0x84, 0x5d, 0xe8, 0x2a, 0x5b, 0xc5, 0x39, 0x88, 0x8a, 0xc7, 0x80,
+    0x54, 0xa2, 0x39, 0x9c, 0xcf, 0xc9, 0xfc, 0xc2, 0xda, 0x31, 0xce, 0x3d, 0xd1, 0x66, 0xbd, 0xcd,
+    0x3a, 0x33, 0x84, 0x7e, 0x5b, 0xbb, 0x07, 0xfd, 0x07, 0xca, 0x47, 0x78, 0x42, 0x31, 0xb1, 0x9a,
+    0xf4, 0x58, 0x72, 0xce, 0xef, 0xb9, 0xfc, 0x59, 0xf4, 0xf9, 0x5d, 0x14, 0x38, 0x1a, 0x3a, 0x78,
+    0x32, 0x56, 0x34, 0x7b, 0x9f, 0xfc, 0xe6, 0x9c, 0xd7, 0x00, 0x7a, 0xe8, 0xa7, 0x58, 0xcc, 0xa4,
+    0x15, 0xd5, 0xa9, 0x1e, 0xe8, 0x63, 0xc8, 0xb6, 0xc0, 0x33, 0x7a, 0xe3, 0x2d, 0x6f, 0xca, 0xa2,
+    0x55, 0x16, 0xcd, 0xf2, 0xf8, 0xb8, 0x65, 0x76, 0x66, 0xbe, 0xf2, 0x15, 0xb9, 0x28, 0x2b, 0xfe,
+    0x20, 0x07, 0x26, 0x97, 0xe7, 0x77, 0xce, 0xa7, 0x25, 0x9c, 0xd3, 0x98, 0xfa, 0x79, 0xa8, 0xef,
+    0x59, 0x27, 0x8c, 0x8c, 0x21, 0x05, 0x03, 0xcc, 0xf8, 0xb9, 0xa6, 0x1a, 0x86, 0xbf, 0xef, 0x23,
+    0x6f, 0xfc, 0xdf, 0x31, 0xd3, 0xdf, 0x36, 0x07, 0x40, 0x36, 0x4a, 0x80, 0x3d, 0xc3, 0x96, 0x53,
+    0x42, 0x8b, 0x6b, 0xd5, 0x21, 0x8f, 0xe8, 0xbd, 0x5a, 0xe5, 0x75, 0xa9, 0x95, 0xd0, 0xe7, 0x84,
+    0x6b, 0xd3, 0xea, 0xe0, 0x80, 0x21, 0x88, 0x26, 0x86, 0x82, 0x04, 0xdf, 0x70, 0xc6, 0x2e, 0x9b,
+    0x01, 0xc6, 0xcc, 0x26, 0x2c, 0x24, 0x79, 0x9e, 0xb9, 0x1e, 0x8e, 0x0f, 0x53, 0xae, 0x84, 0x87,
+    0x8e, 0x7b, 0xc8, 0xc6, 0x1b, 0xe2, 0x8f, 0x0e, 0x3f, 0x30, 0x46, 0x0a, 0xc5, 0x19, 0x81, 0x73,
+    0x8f, 0x07, 0xc2, 0xe4, 0xe9, 0x10, 0x71, 0x53, 0x9c, 0xf9, 0x81, 0x9b, 0x83, 0x33, 0xb1, 0x46,
+    0x73, 0x82, 0x88, 0xce, 0x7a, 0x81, 0xf1, 0x3f, 0xb2, 0x85, 0xe0, 0xe0, 0xf1, 0xed, 0x42, 0xec,
+    0x8f, 0xe4, 0xf1, 0x33, 0xd7, 0x72, 0xa3, 0x6a, 0x1f, 0x64, 0x71, 0x50, 0x12, 0xab, 0x3d, 0x6d,
+    0x12, 0x36, 0xab, 0x4d, 0xc8, 0x1f, 0xe5, 0xc6, 0x27, 0xf0, 0xb7, 0xa4, 0xa9, 0x5d, 0x24, 0x40,
+    0xe2, 0x23, 0xf7, 0x77, 0x38, 0xbf, 0xf3, 0x18, 0x65, 0xe2, 0x7c, 0x29, 0xfd, 0xaa, 0xd5, 0x39,
+    0x29, 0xb4, 0x6e, 0xfe, 0x83, 0x67, 0x56, 0x6b, 0x32, 0x5b, 0x51, 0x17, 0xb8, 0x5d, 0x04, 0x56,
+    0x8d, 0x75, 0x70, 0xb4, 0x04, 0x62, 0x54, 0x84, 0x9f, 0x4b, 0x83, 0xf5, 0x10, 0x1c, 0xfc, 0xeb,
+    0xc9, 0x3a, 0xf8, 0xe0, 0x1a, 0x15, 0x43, 0x45, 0x0a, 0xe7, 0xc7, 0x2e, 0x45, 0xc1, 0x21, 0xd1,
+    0x6c, 0xd9, 0xe9, 0xad, 0xd1, 0xf2, 0x42, 0xe7, 0x26, 0x89, 0xeb, 0x83, 0x92, 0x7e, 0xb3, 0x53,
+    0x16, 0x47, 0x0e, 0xcc, 0xb0, 0x2e, 0x6c, 0xe5, 0x12, 0x44, 0xf0, 0x04, 0xa2, 0x16, 0xcd, 0x42,
+    0x15, 0x9b, 0xdb, 0x38, 0x11, 0x43, 0xdc, 0x1f, 0x74, 0x02, 0x56, 0xfe, 0x8d, 0x6a, 0xed, 0xea,
+    0x44, 0x9f, 0x21, 0x0b, 0x86, 0xb5, 0x3d, 0xf0, 0x1c, 0xf8, 0x29, 0x43, 0x0c, 0x2e, 0x33, 0xee,
+    0x4f, 0xa0, 0x4e, 0x87, 0xc2, 0x34, 0x4a, 0x72, 0x80, 0xac, 0x2d, 0x45, 0x58, 0xcd, 0x04, 0xfe,
+    0x40, 0x09, 0x03, 0x04, 0xbb, 0x81, 0x8d, 0xfa, 0x30, 0x83, 0x79, 0x3e, 0xef, 0x72, 0x1b, 0xa8,
+    0xd1, 0xa6, 0x6e, 0xa8, 0x7e, 0x8b, 0xd5, 0xe3, 0xe4, 0xf8, 0x81, 0x4e, 0xb0, 0x37, 0xfb, 0x3a,
+    0x57, 0x32, 0xd5, 0xe1, 0xb4, 0xba, 0xa2, 0x23, 0x67, 0xfd, 0x58, 0xfb, 0x0d, 0xd6, 0x21, 0x03,
+    0x12, 0xa0, 0xbd, 0xe1, 0x41, 0x6e, 0x29, 0x0e, 0x15, 0xaa, 0xd7, 0x61, 0xde, 0x81, 0xab, 0xf8,
+    0x48, 0x99, 0x3e, 0xb1, 0x4b, 0x0b, 0x75, 0x2f, 0x28, 0x44, 0x72, 0x00, 0x43, 0x5d, 0xf6, 0x54,
+    0xf8, 0xfc, 0x8c, 0x52, 0x3e, 0x08, 0xf7, 0xe1, 0x4f, 0x37, 0x5b, 0x2e, 0x00, 0x55, 0x61, 0x15,
+    0x79, 0x47, 0x80, 0xa7, 0x33, 0x3f, 0x81, 0xc6, 0x01, 0x17, 0x43, 0xd1, 0x16, 0x24, 0x66, 0x96,
+    0x0a, 0x64, 0x05, 0x4c, 0x4d, 0xa1, 0x3b, 0x15, 0x95, 0xf5, 0x87, 0xda, 0xc0, 0x27, 0xa8, 0xe4,
+    0xb7, 0xc8, 0xe1, 0x98, 0x63, 0xc3, 0x53, 0xb8, 0xfc, 0x7e, 0x26, 0x48, 0xb9, 0x9e, 0xa4, 0x25,
+    0x0b, 0xd3, 0xd5, 0xb7, 0xe4, 0x83, 0xa0, 0x6d, 0xbb, 0xb3, 0xcf, 0x81, 0x23, 0xe8, 0x86, 0xc0,
+    0x81, 0x91, 0xd5, 0xd0, 0xcd, 0x04, 0xd3, 0xaf, 0x95, 0xcc, 0xe4, 0xb6, 0xae, 0xf4, 0xb1, 0xa4,
+    0x3a, 0x15, 0x07, 0x0a, 0x22, 0xa3, 0x5c, 0xf5, 0x1a, 0x60, 0xd5, 0x73, 0x8e, 0x0c, 0xa0, 0x04,
+    0xa0, 0x88, 0xae, 0x3e, 0x7d, 0x43, 0x00, 0x74, 0xcc, 0x11, 0xbf, 0xee, 0x80, 0xe5, 0x89, 0x17,
+    0xa8, 0x86, 0x10, 0xbe, 0xbc, 0x79, 0x40, 0xcf, 0x13, 0xd8, 0xc3, 0x3c, 0xba, 0xc1, 0x34, 0x3b,
+    0xbd, 0xa6, 0xf9, 0x75, 0x7e, 0xd8, 0x61, 0x13, 0x7a, 0xe9, 0xaf, 0x49, 0xc4, 0x0b, 0x9d, 0xa1,
+    0xa4, 0x32, 0x13, 0x99, 0x25, 0x54, 0x41, 0xa6, 0xbe, 0xb1, 0x4d, 0x9f, 0x91, 0x22, 0x03, 0x7b,
+    0x0f, 0x7c, 0x44, 0xf8, 0xac, 0x19, 0xb1, 0x37, 0xac, 0x7d, 0x4a, 0xb5, 0x84, 0x49, 0x76, 0x77,
+    0x77, 0xc4, 0x1e, 0xfe, 0xe4, 0x8c, 0x33, 0x4f, 0xfa, 0x15, 0xef, 0x79, 0x04, 0x4a, 0x75, 0x13,
+    0xd1, 0x81, 0xf7, 0xfe, 0x73, 0xfe, 0x44, 0x63, 0x35, 0xea, 0xf2, 0xee, 0x35, 0x13, 0x94, 0x17,
+    0x24, 0xbf, 0x86, 0x43, 0xf3, 0x5c, 0x21, 0x9a, 0xd1, 0xa1, 0x82, 0xc7, 0xe3, 0x1c, 0xb4, 0x5d,
+    0x3b, 0x7f, 0xe5, 0xe0, 0x7c, 0x64, 0x06, 0x28, 0x00, 0xf3, 0x7d, 0xae, 0x73, 0x67, 0x4d, 0xba,
+    0x24, 0x6a, 0x58, 0x60, 0x50, 0x1e, 0xd7, 0x54, 0x00, 0x53, 0xc0, 0x56, 0xd6, 0x65, 0x1e, 0xf0,
+    0xed, 0x32, 0xb6, 0x03, 0xe6, 0xbd, 0x4a, 0x40, 0x5f, 0x10, 0x64, 0x63, 0xff, 0xde, 0x96, 0x13,
+    0x5c, 0xec, 0x6d, 0xc1, 0x46, 0xda, 0x0c, 0x47, 0x1a, 0x0d, 0xd5, 0xa9, 0x49, 0xa2, 0xef, 0x26,
+    0x3f, 0xf8, 0x44, 0x6f, 0x82, 0x50, 0x30, 0xc5, 0x5f, 0xc8, 0xf4, 0x6d, 0xe2, 0x07, 0xcf, 0xc2,
+    0xa1, 0x66, 0xe9, 0xe0, 0xf0, 0x8d, 0x8c, 0x34, 0xb8, 0x14, 0x0c, 0xee, 0xbb, 0x69, 0x73, 0x9d,
+    0xc0, 0x23, 0xa4, 0xde, 0x49, 0x7c, 0x0c, 0xe9, 0xed, 0x8c, 0x20, 0x2b, 0x78, 0x6a, 0x57, 0x48,
+    0x4c, 0x41, 0xbd, 0xbd, 0xf9, 0xa7, 0x42, 0x67, 0xa7, 0x3d, 0x4d, 0x7b, 0x8e, 0xab, 0x64, 0x1e,
+    0x2a, 0xa4, 0x29, 0x13, 0x35, 0x80, 0xe7, 0xcf, 0x7f, 0x8c, 0x38, 0x73, 0xe8, 0x55, 0xff, 0xc2,
+    0x73, 0x6d, 0x23, 0x8c, 0x31, 0x3e, 0x17, 0x2c, 0x57, 0x8e, 0x17, 0x51, 0x3d, 0x5e, 0x42, 0xcf,
+    0x91, 0x33, 0xe3, 0x05, 0xbf, 0xde, 0x69, 0x62, 0x69, 0xbe, 0x86, 0x35, 0x60, 0x45, 0x56, 0xc0,
+    0x0f, 0x7f, 0x47, 0x93, 0xf7, 0x5c, 0x20, 0xaf, 0x80, 0x87, 0xa1, 0xca, 0xdc, 0xd9, 0x37, 0x17,
+    0x45, 0xe5, 0x3f, 0x62, 0x66, 0xa5, 0x72, 0x6e, 0xf4, 0x4f, 0xd9, 0xd0, 0xdf, 0xf7, 0x05, 0x20,
+    0x08, 0x6c, 0xb5, 0xc3, 0xe5, 0xcd, 0x79, 0xf7, 0x96, 0x7d, 0x00, 0x12, 0x64, 0xee, 0xed, 0xed,
+    0xd3, 0x87, 0xda, 0x77, 0xf8, 0x72, 0x3f, 0xc8, 0x1b, 0x39, 0x27, 0x26, 0x85, 0xf8, 0xae, 0x1b,
+    0xf1, 0xd3, 0xb8, 0xb3, 0xa5, 0xd8, 0xc3, 0xe5, 0x75, 0x15, 0x8d, 0xc6, 0x0a, 0x00, 0xc8, 0x20,
+    0x3b, 0x91, 0xeb, 0x09, 0xa5, 0xb7, 0x4d, 0xf6, 0x20, 0xa0, 0x40, 0x87, 0xa2, 0x6f, 0xb2, 0xc3,
+    0x1c, 0x19, 0x12, 0x4c, 0x86, 0xf1, 0x95, 0x31, 0x63, 0x42, 0x39, 0xca, 0x99, 0x00, 0x82, 0x89,
+    0x4d, 0xff, 0x75, 0x47, 0xf5, 0x50, 0xa5, 0xd6, 0xe2, 0x3e, 0x79, 0x86, 0x3c, 0x8c, 0x3f, 0x07,
+    0xf5, 0x69, 0xb4, 0xa6, 0x4e, 0x0e, 0x05, 0x31, 0x7f, 0xe2, 0xac, 0xa5, 0x6b, 0x14, 0x41, 0x3a,
+    0xaa, 0x6c, 0xec, 0x5e, 0x3a, 0x7e, 0x08, 0xb2, 0x56, 0xb7, 0x6b, 0x5c, 0xae, 0x65, 0x32, 0x01,
+    0xcc, 0x4a, 0xbd, 0xd8, 0x81, 0x11, 0x34, 0x7e, 0xf8, 0x33, 0x4f, 0xc4, 0xd1, 0x31, 0x3b, 0x77,
+    0x38, 0x43, 0xc2, 0xe3, 0x4b, 0x1b, 0xf3, 0x9f, 0x7e, 0x9c, 0x2f, 0xe5, 0x39, 0x7c, 0x6a, 0xe9,
+    0xaa, 0x0e, 0xf2, 0x98, 0x25, 0xec, 0x64, 0x0d, 0x36, 0x06, 0xf9, 0x98, 0x24, 0x6a, 0x0d, 0xb5,
+    0x0f, 0x2f, 0x64, 0x73, 0xe5, 0xb6, 0xe2, 0x50, 0xbb, 0x1c, 0xff, 0x14, 0xee, 0x2a, 0x54, 0x30,
+    0x2f, 0xa7, 0xef, 0x86, 0xbf, 0x77, 0x08, 0x4f, 0xaa, 0xb9, 0x60, 0xd6, 0x5f, 0xfc, 0x54, 0x71,
+    0x2b, 0x1b, 0x00, 0x14, 0x47, 0x14, 0x59, 0x6b, 0xf4, 0xe2, 0x1f, 0x8f, 0xf6, 0xc2, 0x35, 0x61,
+    0x5b, 0xc4, 0xd2, 0x4f, 0xd2, 0xcd, 0x6e, 0x16, 0x0c, 0xb4, 0x79, 0x32, 0x5f, 0x8a, 0xeb, 0x72,
+    0x31, 0x52, 0x5d, 0xbc, 0xe5, 0x79, 0x07, 0xa1, 0x69, 0x3f, 0xcf, 0xa0, 0xc4, 0x67, 0x0a, 0x60,
+    0x08, 0x76, 0x10, 0xcd, 0xeb, 0x0f, 0x41, 0x31, 0xbf, 0x10, 0xe6, 0x9b, 0x56, 0x5c, 0x45, 0x55,
+    0xf5, 0xf4, 0x9d, 0x0b, 0x43, 0xbf, 0xb7, 0xb0, 0x51, 0xec, 0x46, 0x4c, 0x00, 0xb8, 0xc1, 0x98,
+    0xea, 0xce, 0xa2, 0xf2, 0xf1, 0x10, 0x06, 0xd3, 0x3b, 0x1b, 0x79, 0xb7, 0xf4, 0x77, 0xf4, 0xc6,
+    0x62, 0xca, 0x40, 0xe9, 0x6e, 0xd0, 0x7e, 0x21, 0xed, 0x7f, 0x2e, 0x02, 0xcd, 0xee, 0xbd, 0x4d,
+    0xd2, 0xb1, 0xc5, 0x26, 0x9b, 0x3c, 0x53, 0xdc, 0x51, 0x75, 0x5c, 0xc8, 0xc8, 0x98, 0x14, 0x83,
+    0x32, 0x64, 0xc0, 0x28, 0x3f, 0x68, 0x10, 0xa6, 0x08, 0x7b, 0x8d, 0x8b, 0x53, 0x29, 0xfa, 0x6d,
+    0xe2, 0x1a, 0xfc, 0x12, 0x43, 0x9f, 0x15, 0x35, 0x18, 0x6b, 0x7f, 0xfd, 0xb5, 0xf8, 0x72, 0x2c,
+    0x3b, 0xa2, 0x6a, 0x75, 0x9e, 0xe4, 0xac, 0x3c, 0xbf, 0x89, 0xd8, 0xc6, 0xaa, 0xc2, 0x1f, 0xc7,
+    0xd7, 0x4b, 0x4b, 0x47, 0x91, 0x44, 0x5f, 0x41, 0xbc, 0x42, 0x32, 0x70, 0x3f, 0x2f, 0x3e, 0x3c,
+    0x27, 0x48, 0xe2, 0xe8, 0x94, 0x30, 0x53, 0x10, 0x65, 0x40, 0xfe, 0x3e, 0x81, 0x86, 0x3b, 0xa6,
+    0xce, 0x19, 0xa7, 0x76, 0xfd, 0x09, 0x1a, 0x01, 0x79, 0xe2, 0xd1, 0x3b, 0xd7, 0x72, 0xea, 0x5f,
+    0x0a, 0xe0, 0x4b, 0x3b, 0x1e, 0x0c, 0x30, 0x99, 0xf9, 0xd3, 0x95, 0x31, 0xee, 0x13, 0x5f, 0x83,
+    0xdd, 0x2d, 0x72, 0x9a, 0x42, 0xc6, 0xc7, 0xaa, 0xf2, 0x01, 0x1b, 0xa3, 0x98, 0xb5, 0x9e, 0x59,
+    0x37, 0x09, 0xde, 0x57, 0x24, 0x0b, 0x34, 0xff, 0x41, 0x09, 0x99, 0xbb, 0xa6, 0xe9, 0x34, 0xd0,
+    0x02, 0xd1, 0x53, 0x68, 0xad, 0x5f, 0x2f, 0x9e, 0x4f, 0x13, 0x34, 0x08, 0xcb, 0x7e, 0x8c, 0x7b,
+    0x10, 0x68, 0x19, 0xcb, 0x65, 0xa9, 0x8c, 0x27, 0xa3, 0x88, 0x17, 0xa7, 0x29, 0x65, 0xb2, 0x45,
+    0x68, 0xfc, 0x48, 0xaa, 0x4e, 0x6a, 0xf4, 0x0d, 0x4f, 0xbe, 0x91, 0xe2, 0x5b, 0x6a, 0x6a, 0x04,
+    0xdd, 0xc4, 0xff, 0xcd, 0x5d, 0xa4, 0x32, 0x64, 0xba, 0x67, 0x34, 0xf1, 0x01, 0x6f, 0xe6, 0x28,
+    0x6c, 0x1d, 0xd2, 0x17, 0x67, 0x93, 0xe2, 0x5d, 0x75, 0xc5, 0x29, 0x21, 0x03, 0x0d, 0x8d, 0x24,
+    0xa4, 0xce, 0xe8, 0xe5, 0x16, 0x92, 0x9f, 0xed, 0x5e, 0xbc, 0x81, 0x2b, 0x25, 0x59, 0x48, 0x29,
+    0x85, 0x2b, 0xec, 0x11, 0x1b, 0x62, 0x7d, 0xc0, 0xce, 0xca, 0xf7, 0xce, 0x32, 0x4d, 0x20, 0xd6,
+    0xf1, 0x0b, 0xf9, 0xe9, 0x7b, 0x50, 0x0d, 0x9b, 0xed, 0xa2, 0x63, 0x16, 0xe7, 0xb6, 0x9e, 0xb0,
+    0xd3, 0xe4, 0x29, 0xa3, 0xc9, 0xdb, 0x38, 0x9e, 0x67, 0x9d, 0xd8, 0x32, 0xd4, 0x79, 0x2e, 0x90,
+    0x37, 0x0a, 0x66, 0xf0, 0x84, 0x28, 0x62, 0x5b, 0x1f, 0x26, 0x3f, 0xf8, 0xb9, 0xd0, 0xe5, 0x31,
+    0x0a, 0xe2, 0x8f, 0xd7, 0xc1, 0xac, 0x09, 0xaa, 0xd6, 0x52, 0x1e, 0x63, 0x99, 0x74, 0x8c, 0xd9,
+    0xa0, 0xc7, 0x4e, 0xa6, 0xeb, 0x4e, 0x95, 0x3f, 0x6c, 0x63, 0xa8, 0x5e, 0x72, 0x80, 0x70, 0x2d,
+    0x05, 0x00, 0x9e, 0xfc, 0x7d, 0x77, 0x3c, 0x72, 0xc3, 0x9e, 0xc7, 0xd1, 0x75, 0xd6, 0x2d, 0xcf,
+    0x79, 0x66, 0x1b, 0x11, 0x20, 0x5b, 0x6e, 0x5d, 0x17, 0xcd, 0x71, 0x81, 0x82, 0xa8, 0x0a, 0x0a,
+    0xa2, 0x21, 0x15, 0xec, 0xbb, 0x50, 0xc7, 0xb8, 0x82, 0x14, 0x0d, 0xc0, 0x81, 0xe5, 0x60, 0xa7,
+    0xf3, 0xc8, 0x22, 0x06, 0xdb, 0x10, 0xff, 0x9d, 0xbb, 0xb1, 0xd0, 0x1c, 0x31, 0x21, 0xfb, 0xe2,
+    0x7d, 0x49, 0xf4, 0xcf, 0xea, 0xcb, 0x2a, 0xaf, 0xc9, 0xb8, 0xee, 0x38, 0x10, 0xd5, 0x59, 0x9c,
+    0xc1, 0x40, 0x28, 0x52, 0xe5, 0x9d, 0x46, 0xe7, 0xd0, 0x74, 0x24, 0x41, 0x80, 0xf6, 0xeb, 0x7a,
+    0x35, 0x97, 0x43, 0x9d, 0x81, 0x3c, 0x51, 0x5f, 0x09, 0x32, 0x2e, 0x67, 0x29, 0xa2, 0xef, 0x47,
+    0xad, 0x53, 0xe5, 0x60, 0x2b, 0xca, 0xc8, 0x43, 0x1d, 0xc4, 0x87, 0x0c, 0xa2, 0xdb, 0x5c, 0xf7,
+    0xdf, 0x73, 0x8e, 0x85, 0x94, 0xb0, 0xe1, 0xe5, 0x1a, 0x40, 0xfe, 0x89, 0xa1, 0xdb, 0x64, 0xbc,
+    0xcc, 0x5f, 0x43, 0x60, 0xfd, 0x5e, 0x93, 0x25, 0x5c, 0x54, 0xc3, 0x14, 0x71, 0x3a, 0x2d, 0x9d,
+    0xbe, 0xf5, 0x0c, 0x4b, 0xd1, 0x84, 0x40, 0x4f, 0xa3, 0xf7, 0xfb, 0xde, 0x95, 0xed, 0xa9, 0xe5,
+    0x50, 0xbb, 0x00, 0xbf, 0x08, 0x38, 0xa6, 0x4a, 0x9d, 0xa0, 0x6e, 0x6a, 0x83, 0x5d, 0xe5, 0x0c,
+    0x21, 0x7d, 0x3a, 0x9c, 0xa7, 0x0b, 0x05, 0x0d, 0x00, 0x91, 0x5a, 0x4d, 0x1b, 0x85, 0x5b, 0x88,
+    0x39, 0x69, 0x95, 0x4d, 0x96, 0x22, 0x34, 0x5d, 0x9f, 0xd4, 0x79, 0x28, 0x22, 0x03, 0xef, 0xcd,
+    0x3e, 0xb5, 0x26, 0x73, 0x18, 0x10, 0xa3, 0x25, 0xdf, 0xaa, 0xc8, 0x45, 0x66, 0xcf, 0x43, 0xf7,
+    0x02, 0x0e, 0xa5, 0xd2, 0x8f, 0xe4, 0x59, 0x98, 0xa5, 0x94, 0x71, 0x9a, 0xef, 0x84, 0xbb, 0x7e,
+    0x3f, 0x2a, 0xe7, 0x00, 0x0b, 0x0f, 0x88, 0x06, 0x67, 0x2f, 0x3c, 0x28, 0x0e, 0xe9, 0xc7, 0x1a,
+    0x03, 0x9c, 0x8d, 0xa8, 0xf0, 0x32, 0x24, 0xe9, 0x33, 0x84, 0x9b, 0xa4, 0x81, 0xa5, 0xa4, 0x6a,
+    0xd0, 0x9c, 0x2c, 0x82, 0x4f, 0x10, 0x4c, 0xa0, 0x0c, 0xfe, 0xe3, 0xb9, 0xc8, 0x7a, 0xb7, 0x89,
+    0x01, 0x60, 0xd8, 0x6f, 0xbe, 0xe9, 0x77, 0x14, 0xbd, 0xa7, 0x73, 0x2c, 0x39, 0xff, 0x1a, 0x42,
+    0x3b, 0xa4, 0x09, 0x1f, 0x55, 0xe4, 0xbf, 0xec, 0xb1, 0xf1, 0xd8, 0x43, 0xb6, 0x0d, 0x44, 0xa2,
+    0x8d, 0xad, 0x6f, 0xaf, 0xc9, 0xea, 0x85, 0xf8, 0x43, 0x4b, 0xa4, 0xed, 0xf7, 0xe4, 0x37, 0x15,
+    0xe1, 0x81, 0x03, 0x2b, 0x42, 0xe7, 0x3c, 0xd7, 0xbe, 0x33, 0xf1, 0x28, 0xbf, 0xea, 0x53, 0x31,
+    0xe1, 0x63, 0x54, 0x99, 0x3d, 0x61, 0xe8, 0xda, 0xa1, 0xeb, 0xb1, 0xfb, 0xaa, 0xd7, 0xfa, 0x89,
+    0x78, 0x78, 0xd6, 0x87, 0xb2, 0x01, 0xdb, 0x06, 0x6f, 0xf4, 0xb9, 0x3b, 0x92, 0xe2, 0x4e, 0xca,
+    0x36, 0x64, 0x9f, 0x95, 0x13, 0x90, 0xe9, 0x2b, 0x25, 0x08, 0x06, 0x1c, 0x1b, 0x9f, 0xed, 0x29,
+    0x58, 0xfa, 0x24, 0xb3, 0x07, 0x07, 0x0a, 0x23, 0xb1, 0xa4, 0xa2, 0x0a, 0xb2, 0x11, 0xbc, 0x0b,
+    0x10, 0xdb, 0x97, 0xc3, 0x5d, 0x33, 0xd1, 0xf4, 0xd1, 0x88, 0xe4, 0xaa, 0x10, 0xe1, 0xde, 0xc1,
+    0xea, 0xb6, 0xf1, 0x62, 0x1b, 0x3f, 0x34, 0x34, 0x1c, 0x08, 0x08, 0xf3, 0xd9, 0xe9, 0xcf, 0xc0,
+    0xa2, 0x16, 0xd3, 0xc0, 0xa1, 0xa1, 0x49, 0x7a, 0x19, 0xa1, 0x19, 0xca, 0xc1, 0xa5, 0x34, 0x4b,
+    0x51, 0x56, 0x6c, 0x42, 0x05, 0x59, 0x41, 0xee, 0x48, 0x0c, 0xb7, 0xc2, 0x5e, 0xe9, 0x52, 0xc4,
+    0xf6, 0x9a, 0x80, 0x79, 0xd9, 0x49, 0x9e, 0xbe, 0x07, 0xc9, 0x69, 0x07, 0x6f, 0x84, 0xc5, 0x19,
+    0x58, 0x78, 0xb4, 0x0c, 0x89, 0x90, 0x37, 0xb6, 0xdc, 0xd3, 0x17, 0x93, 0xd1, 0x49, 0x2b, 0x6f,
+    0x00, 0x86, 0x33, 0x49, 0xc3, 0xc0, 0xfa, 0x0d, 0x01, 0x59, 0x7d, 0x18, 0x7d, 0xb1, 0xcb, 0xd3,
+    0x2f, 0xf7, 0x7e, 0x97, 0x58, 0xf5, 0xd4, 0x83, 0x42, 0x93, 0xf1, 0x28, 0x48, 0xd0, 0x36, 0xf0,
+    0xb3, 0x3b, 0x7f, 0x2a, 0x1c, 0xf0, 0xa2, 0xc4, 0x14, 0x7d, 0xc9, 0xfd, 0xb2, 0x8f, 0xc9, 0x1a,
+    0xa0, 0x53, 0x5b, 0x18, 0x66, 0xed, 0x65, 0xe4, 0xe3, 0xbe, 0x16, 0x6c, 0xe3, 0xa5, 0x06, 0x5f,
+    0x34, 0x4d, 0x43, 0x6d, 0xe6, 0x8b, 0x80, 0x2b, 0x61, 0xfb, 0xe2, 0xa1, 0x3b, 0xf1, 0x75, 0x20,
+    0x88, 0x98, 0xc1, 0xb0, 0xc0, 0x9a, 0xa5, 0x08, 0x59, 0x94, 0x53, 0x85, 0x27, 0xde, 0xd7, 0x73,
+    0xa9, 0x8d, 0xbd, 0x52, 0x2b, 0x76, 0x70, 0xb0, 0xc5, 0x41, 0x94, 0x3b, 0x20, 0x55, 0x76, 0xa4,
+    0xe2, 0xb2, 0x3c, 0x81, 0x31, 0x44, 0x4d, 0xc1, 0xb4, 0xd3, 0xd7, 0x9e, 0x27, 0xb9, 0x27, 0xf9,
+    0x3f, 0xb9, 0x53, 0x9a, 0x85, 0x59, 0x29, 0x3c, 0x53, 0xf4, 0x30, 0xc2, 0xf9, 0xf4, 0xba, 0xfe,
+    0x1a, 0x2a, 0xf6, 0xa8, 0x1a, 0x32, 0x62, 0x26, 0xfb, 0x25, 0xcb, 0x4d, 0xbb, 0x4c, 0x6f, 0x46,
+    0x32, 0x1b, 0xa3, 0xe9, 0x1b, 0x47, 0x34, 0xe2, 0x63, 0x76, 0x08, 0x03, 0x66, 0xda, 0xca, 0x6f,
+    0xb1, 0x38, 0x80, 0xfb, 0xa1, 0x4b, 0x76, 0x05, 0x24, 0x41, 0x9a, 0xbc, 0x67, 0x01, 0xbd, 0x3e,
+    0xe8, 0xda, 0x6e, 0xb3, 0x92, 0x96, 0xbf, 0xa5, 0x6b, 0xd8, 0x3a, 0xaa, 0xb8, 0xa7, 0xe1, 0xe0,
+    0xc6, 0xa4, 0xb3, 0x95, 0xda, 0x3a, 0xad, 0x2e, 0xa4, 0x1f, 0x74, 0x6e, 0x50, 0x42, 0xa0, 0xb3,
+    0x19, 0xe5, 0x6b, 0x3e, 0xc8, 0x66, 0xb6, 0xb6, 0xa1, 0x28, 0x40, 0xd9, 0xec, 0x7b, 0x74, 0x05,
+    0x9f, 0xdb, 0x68, 0x84, 0xac, 0xa9, 0xee, 0xdf, 0x2e, 0xe4, 0xa7, 0x53, 0xc7, 0x02, 0x63, 0xd4,
+    0x7d, 0xe8, 0xf9, 0x1b, 0x09, 0x40, 0x8b, 0x37, 0x29, 0xb7, 0xc8, 0xf3, 0xf0, 0x33, 0x84, 0x59,
+    0x19, 0xd8, 0x93, 0x74, 0x8a, 0x34, 0xb7, 0x79, 0x83, 0x04, 0xa3, 0xca, 0xd4, 0x5e, 0x85, 0x57,
+    0x69, 0xbd, 0xf2, 0x74, 0x35, 0xfd, 0xaf, 0x2f, 0x64, 0x83, 0xc3, 0xee, 0x1f, 0xba, 0xfc, 0x9d,
+    0x5b, 0xa3, 0x0e, 0x40, 0x46, 0x61, 0x66, 0x0f, 0x03, 0x13, 0x6b, 0xea, 0x6b, 0xa0, 0xb2, 0xac,
+    0x5a, 0x94, 0x43, 0x1b, 0x39, 0x4d, 0xbd, 0x66, 0xf0, 0xf4, 0x86, 0xf8, 0x38, 0xfe, 0xcd, 0xf5,
+    0x64, 0x76, 0x36, 0x2a, 0x21, 0xed, 0xc6, 0x11, 0xcf, 0xcc, 0xa2, 0x31, 0x78, 0xa4, 0x8f, 0xb8,
+    0x39, 0xd0, 0xf6, 0x25, 0x5a, 0xaa, 0xa3, 0xd4, 0xd1, 0xcb, 0xd0, 0x69, 0x77, 0xff, 0x4b, 0xc2,
+    0x8c, 0xa6, 0x20, 0xc7, 0xd5, 0x78, 0x5a, 0xc8, 0xd9, 0x3a, 0x44, 0xb4, 0x60, 0xaf, 0x40, 0xfb,
+    0x6d, 0xad, 0x2f, 0x7b, 0x00, 0xce, 0xb8, 0xcc, 0x47, 0x5b, 0x3e, 0xa7, 0x4d, 0x52, 0x7a, 0x7c,
+    0x6d, 0x9f, 0xa3, 0x15, 0xa8, 0xe5, 0x5c, 0x27, 0xed, 0x4d, 0xda, 0x62, 0x0e, 0x15, 0xd3, 0x90,
+    0xe7, 0x53, 0xc8, 0xf1, 0x23, 0x87, 0xd4, 0x58, 0xa2, 0x95, 0x03, 0xa8, 0x02, 0x35, 0xf3, 0x12,
+    0xa7, 0x4b, 0x40, 0x9b, 0x19, 0x94, 0x24, 0xda, 0x3b, 0x2f, 0xc6, 0x73, 0x58, 0xc8, 0x27, 0x35,
+    0xe7, 0x67, 0xca, 0x88, 0x2a, 0x9c, 0xe4, 0xb0, 0x9b, 0xfa, 0xc8, 0x17, 0xab, 0xe6, 0xe4, 0x8c,
+    0xc9, 0xa2, 0xd6, 0x4c, 0x32, 0x7e, 0xb1, 0x36, 0x87, 0x14, 0xbd, 0xd6, 0x70, 0xab, 0xe1, 0x1d,
+    0x8e, 0x1e, 0x43, 0x6b, 0x3b, 0xd3, 0x23, 0x79, 0x7e, 0x8e, 0x0e, 0x7b, 0x77, 0xe7, 0x24, 0xb3,
+    0x7d, 0x3f, 0x7f, 0x2a, 0x8a, 0x99, 0xdc, 0xbc, 0x01, 0x29, 0xd7, 0x52, 0x77, 0xb2, 0x90, 0x7f,
+    0xaa, 0x4b, 0xd7, 0x77, 0x5f, 0x6d, 0x6b, 0xff, 0xf5, 0xad, 0x13, 0x2e, 0xa3, 0x5c, 0xa2, 0xa5,
+    0x07, 0x05, 0x9c, 0x0b, 0xae, 0xbc, 0xee, 0xff, 0x54, 0xcf, 0xfb, 0x18, 0x82, 0x7b, 0x7c, 0xc1,
+    0xe5, 0x24, 0x08, 0x36, 0xb7, 0x6a, 0xa0, 0x20, 0x56, 0x18, 0xdc, 0xa8, 0x5d, 0x57, 0x79, 0xc7,
+    0x86, 0x8d, 0xc5, 0xe9, 0x35, 0x48, 0x6f, 0x57, 0x6c, 0x40, 0x8d, 0x0d, 0xd3, 0x4a, 0x4a, 0x5a,
+    0xd3, 0x7e, 0x67, 0x55, 0x80, 0xfb, 0x45, 0xdf, 0x81, 0x58, 0xf9, 0x34, 0xa7, 0x7e, 0xca, 0x1e,
+    0x54, 0x31, 0x51, 0xb6, 0x4c, 0x20, 0x96, 0xf9, 0xa2, 0x16, 0xc8, 0xff, 0x0a, 0x66, 0xb9, 0x8d,
+    0xe2, 0x67, 0x8b, 0x92, 0x0c, 0x66, 0x4c, 0x1b, 0x01, 0x0b, 0x30, 0xd2, 0xeb, 0x79, 0x9b, 0xc4,
+    0xa8, 0x0f, 0xc9, 0x80, 0xe8, 0x8b, 0x9c, 0x60, 0x9d, 0x25, 0xa0, 0xac, 0xb2, 0xb0, 0x98, 0xe0,
+    0xae, 0x15, 0x36, 0x0a, 0xaa, 0xa2, 0x75, 0xa0, 0xc3, 0x2c, 0x19, 0xa9, 0x2e, 0xde, 0x09, 0x6b,
+    0xc6, 0x19, 0xea, 0xee, 0xa7, 0x03, 0x5e, 0xdf, 0xd2, 0x23, 0xc9, 0x4f, 0x8f, 0xb5, 0x42, 0xdc,
+    0x4d, 0x2f, 0x6b, 0x08, 0x51, 0x05, 0x6e, 0x90, 0xa4, 0x94, 0xef, 0xe9, 0x0d, 0x7f, 0x91, 0x85,
+    0x0a, 0xd3, 0x1e, 0xc6, 0xcf, 0x6b, 0x93, 0xb2, 0xeb, 0x67, 0x72, 0x11, 0x03, 0xae, 0x63, 0x98,
+    0x97, 0xfe, 0xf0, 0xa8, 0xfb, 0x27, 0x79, 0xc5, 0x69, 0x8c, 0x1a, 0x15, 0xa4, 0x78, 0x36, 0xe5,
+    0x26, 0xa0, 0x03, 0x6d, 0x01, 0x02, 0xaf, 0xab, 0x1f, 0xfc, 0xf7, 0xdb, 0x16, 0x37, 0xde, 0x1f,
+    0x21, 0x78, 0x04, 0x46, 0xb8, 0x91, 0x3e, 0x73, 0xbb, 0xbe, 0x2f, 0xec, 0x0c, 0x5d, 0xc6, 0xbf,
+    0xb6, 0xb1, 0xdb, 0x25, 0xba, 0xc2, 0x15, 0x4b, 0xa0, 0x8e, 0xb5, 0x7f, 0x75, 0xab, 0xee, 0xe3,
+    0x41, 0xe9, 0xf6, 0x0d, 0xb7, 0x08, 0x02, 0x0f, 0x03, 0xe2, 0xa6, 0xaf, 0xd1, 0x9e, 0x14, 0x63,
+    0x4f, 0x4f, 0xba, 0x99, 0x2a, 0xf5, 0xdc, 0xd5, 0x7c, 0x9b, 0x0f, 0x50, 0x5e, 0xf2, 0x93, 0xba,
+    0x70, 0x78, 0xad, 0x2a, 0x25, 0xf7, 0xcc, 0x1d, 0x5c, 0xf4, 0xa5, 0x29, 0xa1, 0xcd, 0x6a, 0x7a,
+    0x62, 0xc7, 0xc9, 0x73, 0xf1, 0x45, 0xc8, 0xc1, 0x91, 0x55, 0x4a, 0x47, 0x0f, 0x9f, 0xf9, 0xa6,
+    0xb4, 0xcd, 0xd3, 0x99, 0x55, 0xde, 0x9b, 0xb9, 0xfa, 0x03, 0xd4, 0x26, 0x99, 0xd5, 0x4f, 0x95,
+    0x6d, 0xf9, 0xe3, 0x3f, 0x60, 0x63, 0xaf, 0x60, 0x9a, 0xc5, 0xe5, 0x3b, 0xce, 0x73, 0x48, 0xb0,
+    0x00, 0x52, 0x43, 0x44, 0x6c, 0x28, 0x96, 0xeb, 0xd0, 0xc3, 0xe3, 0xc8, 0x0a, 0x49, 0xd5, 0x24,
+    0xcf, 0xe3, 0xde, 0xfe, 0x92, 0x25, 0x46, 0xf9, 0xd9, 0xcc, 0xce, 0x8c, 0xaf, 0xc6, 0xe9, 0x7f,
+    0x58, 0x88, 0x15, 0x8a, 0x8d, 0x7c, 0xcc, 0x61, 0x33, 0xc9, 0xc0, 0xb8, 0xee, 0xfb, 0x3b, 0x4f,
+    0x9b, 0x0e, 0xad, 0xe5, 0x77, 0xb5, 0x34, 0xed, 0x41, 0x96, 0xc0, 0x02, 0xca, 0x62, 0x75, 0x8a,
+    0x16, 0x89, 0xce, 0x5a, 0xc5, 0x10, 0x3b, 0x65, 0x94, 0x85, 0xe5, 0x42, 0xe2, 0xd5, 0x85, 0x52,
+    0x7a, 0x81, 0x96, 0x33, 0x30, 0x36, 0x31, 0x17, 0x2e, 0xce, 0xb3, 0x4a, 0x5c, 0x93, 0x90, 0x5b,
+    0x67, 0xc7, 0x84, 0xdb, 0x26, 0x3f, 0x0b, 0xec, 0xff, 0x7e, 0x5f, 0xdd, 0x1b, 0x5f, 0xa1, 0x76,
+    0xc9, 0x14, 0x27, 0x50, 0x98, 0x07, 0x58, 0x47, 0x84, 0x9b, 0x05, 0x18, 0x08, 0x34, 0xfd, 0xde,
+    0xdd, 0x90, 0x7c, 0x96, 0x91, 0x36, 0x42, 0xec, 0xc7, 0x47, 0x6d, 0x18, 0xf2, 0x72, 0xc4, 0x97,
+    0xd1, 0x9b, 0xf6, 0x21, 0xc1, 0xd7, 0x09, 0x56, 0x33, 0xfe, 0x2e, 0x60, 0x15, 0x07, 0x0d, 0x08,
+    0x8e, 0x5e, 0xde, 0xb4, 0x75, 0x7c, 0xf2, 0xd8, 0xe8, 0xe5, 0x10, 0xdc, 0x99, 0xa3, 0x65, 0xec,
+    0x1e, 0xb4, 0xf5, 0x17, 0x41, 0x51, 0x90, 0x3b, 0xa4, 0x16, 0xf4, 0xeb, 0xab, 0x81, 0x64, 0x2e,
+    0x72, 0xd9, 0x28, 0x5e, 0xf7, 0x3c, 0xfd, 0xb8, 0x38, 0x2c, 0x09, 0xf1, 0x41, 0xf0, 0x5a, 0x0f,
+    0xe7, 0x8d, 0xe7, 0x07, 0xd6, 0xeb, 0x0c, 0x42, 0xc9, 0x83, 0xb5, 0xbd, 0xa5, 0xc2, 0xfc, 0x7b,
+    0x0e, 0x19, 0x25, 0x51, 0xc1, 0x01, 0xf0, 0x32, 0xad, 0xbf, 0x4c, 0x96, 0x97, 0x70, 0xc2, 0xa7,
+    0x1a, 0x78, 0x52, 0x5f, 0x41, 0xe3, 0x1f, 0x5f, 0x7b, 0x61, 0x2b, 0x70, 0x3d, 0xce, 0x24, 0xea,
+    0xad, 0xe4, 0x03, 0x77, 0xb7, 0xe9, 0x31, 0xcc, 0x09, 0x28, 0xed, 0xd5, 0x38, 0x13, 0xef, 0x9e,
+    0xdd, 0x5f, 0xe3, 0xbf, 0x23, 0xc7, 0x72, 0xf5, 0x18, 0xed, 0xed, 0x62, 0xd7, 0x05, 0xa0, 0x13,
+    0x73, 0xf8, 0x56, 0x52, 0xd2, 0x3b, 0x7a, 0x1d, 0xa0, 0x5d, 0x24, 0x54, 0x38, 0xbc, 0x0e, 0x2e,
+    0xb6, 0x73, 0x8d, 0xe3, 0x25, 0x70, 0xde, 0x26, 0x44, 0x6b, 0x69, 0x3f, 0x27, 0x06, 0x45, 0x92,
+    0xd6, 0x4b, 0x55, 0xcd, 0x2a, 0x42, 0x7d, 0x1b, 0x51, 0x74, 0xe7, 0x7b, 0x1d, 0x27, 0xfa, 0x83,
+    0x0e, 0xa1, 0xe5, 0xc9, 0xab, 0xec, 0xb6, 0x8f, 0x7a, 0xd5, 0x49, 0x1e, 0x41, 0xc1, 0x33, 0xf8,
+    0x5d, 0x6e, 0xfd, 0x42, 0xff, 0x3d, 0xec, 0x3c, 0x18, 0x63, 0x4a, 0x6a, 0xe5, 0x29, 0x0e, 0xd5,
+    0xb9, 0xfa, 0x4b, 0x24, 0xfa, 0xa3, 0x04, 0x71, 0xce, 0x81, 0x57, 0x82, 0x23, 0x71, 0x00, 0xca,
+    0xd5, 0xf1, 0x86, 0x49, 0x2f, 0x5c, 0x6f, 0x0a, 0xe9, 0x68, 0x37, 0x46, 0x92, 0x2e, 0x23, 0xd7,
+    0x2e, 0x85, 0xc5, 0x3a, 0xb6, 0x2c, 0x32, 0x99, 0x14, 0xd4, 0x16, 0xe3, 0x9b, 0xbb, 0x7e, 0xc2,
+    0x46, 0x2c, 0x34, 0x23, 0x9c, 0xab, 0xb5, 0xa0, 0xcf, 0x31, 0x95, 0x4e, 0x33, 0x02, 0x10, 0xb1,
+    0xbb, 0x85, 0x68, 0xd7, 0xb8, 0xea, 0x0e, 0x84, 0xcf, 0x58, 0x55, 0x48, 0xd7, 0xa3, 0xdd, 0xf2,
+    0x7e, 0x17, 0x03, 0x68, 0xe9, 0xc3, 0x7a, 0x22, 0xdf, 0xaa, 0x44, 0x3f, 0x2f, 0x90, 0xd4, 0xfc,
+    0x5d, 0x09, 0x29, 0xb3, 0x5f, 0x93, 0x98, 0xdb, 0x01, 0x5b, 0x85, 0xee, 0x72, 0xf7, 0x84, 0x12,
+    0x1e, 0x5b, 0xb6, 0x3e, 0xd1, 0xd4, 0xdd, 0xe9, 0x52, 0xc7, 0xb6, 0xde, 0x61, 0x93, 0xc0, 0xe5,
+    0x0f, 0x4a, 0xdf, 0x1b, 0xf4, 0xbb, 0x7e, 0x72, 0x83, 0x06, 0x87, 0xcd, 0x89, 0x22, 0x05, 0x3e,
+    0xf7, 0x16, 0x39, 0x9e, 0x2e, 0x2a, 0x1a, 0x4f, 0x40, 0x8e, 0xd1, 0xf4, 0x07, 0x04, 0x18, 0xed,
+    0xb2, 0xbd, 0x31, 0x42, 0x04, 0xd6, 0x99, 0xa3, 0x93, 0x76, 0x85, 0x3d, 0xb3, 0x71, 0x1a, 0x59,
+    0xde, 0x18, 0xb7, 0x2d, 0x0b, 0x45, 0x1f, 0x77, 0x7e, 0x95, 0x80, 0xc2, 0x47, 0x1c, 0x1f, 0x1f,
+    0x67, 0xe2, 0x23, 0x8a, 0x97, 0x3a, 0xdc, 0x3a, 0x25, 0xab, 0x92, 0x76, 0xbf, 0x65, 0x2a, 0xf2,
+    0xd3, 0x04, 0xf0, 0xa2, 0x63, 0xb1, 0x6b, 0x98, 0xd6, 0x9a, 0x86, 0x09, 0x65, 0xf8, 0xf0, 0x0d,
+    0xc6, 0x5c, 0x56, 0x66, 0x3d, 0xd6, 0x55, 0xb7, 0x6f, 0xd7, 0xfb, 0x90, 0xcd, 0xfc, 0xe9, 0x52,
+    0xd0, 0x66, 0xd8, 0x8f, 0x0d, 0x53, 0x84, 0x25, 0xf5, 0xae, 0xef, 0x5a, 0x3f, 0xde, 0x6c, 0xa9,
+    0xa1, 0x02, 0x5d, 0x1b, 0x87, 0x2f, 0x11, 0x53, 0x6e, 0xb3, 0x81, 0xab, 0x05, 0x39, 0x23, 0x6b,
+    0xf8, 0x65, 0xc6, 0xff, 0xef, 0x74, 0xa2, 0x0b, 0xcf, 0xfa, 0xe2, 0xf9, 0xe2, 0x0a, 0x08, 0xdd,
+    0xa4, 0x9e, 0x44, 0xea, 0xad, 0x9f, 0x45, 0xa0, 0x8a, 0xce, 0xec, 0x09, 0x9f, 0x19, 0x40, 0x1f,
+    0x85, 0x03, 0x6f, 0x3c, 0xf3, 0x0a, 0x49, 0x1c, 0x4e, 0x58, 0xa5, 0x2a, 0x1e, 0x0f, 0x98, 0xf5,
+    0xf4, 0xeb, 0x83, 0xe6, 0x44, 0x15, 0x77, 0x97, 0x88, 0xee, 0x25, 0x70, 0x1f, 0x82, 0x21, 0xe2,
+    0x4b, 0xea, 0x68, 0x93, 0x49, 0x46, 0x3e, 0xbc, 0x16, 0xbd, 0x8b, 0x49, 0xd6, 0x74, 0x9c, 0xb1,
+    0x91, 0x38, 0xa6, 0x62, 0x33, 0x8c, 0xb5, 0x5d, 0x75, 0xe4, 0x8c, 0x4d, 0x9c, 0x7a, 0x78, 0xd1,
+    0x4f, 0x07, 0x3e, 0x55, 0x38, 0x30, 0x83, 0x8b, 0x62, 0xf8, 0x95, 0x65, 0x03, 0xec, 0x5a, 0x29,
+    0xdc, 0xf3, 0x3d, 0x52, 0x8e, 0x53, 0x7d, 0x45, 0x48, 0xe0, 0xfc, 0x37, 0x4b, 0x0e, 0xc5, 0x05,
+    0x28, 0x8d, 0x11, 0x9b, 0xdf, 0x59, 0x70, 0xa8, 0x0f, 0x84, 0x63, 0xd5, 0x70, 0x5a, 0xbc, 0xc3,
+    0x1b, 0x85, 0x39, 0xfd, 0xf5, 0xad, 0xbd, 0xef, 0x27, 0x6a, 0x56, 0xab, 0x5a, 0x23, 0xac, 0x33,
+    0x9d, 0x9c, 0xd9, 0x46, 0xd2, 0xd6, 0x84, 0x18, 0xbd, 0xdb, 0xbe, 0xec, 0xc2, 0xfe, 0x79, 0x44,
+    0xc8, 0xa1, 0xb5, 0xa1, 0xea, 0xb4, 0x20, 0x69, 0xde, 0x1a, 0x01, 0xe9, 0xc4, 0x8c, 0x95, 0x1e,
+    0x7f, 0x65, 0xf6, 0xfe, 0x92, 0x26, 0x6a, 0xd9, 0xc8, 0x47, 0xdf, 0x9f, 0x9b, 0x1c, 0x61, 0xda,
+    0x73, 0xb1, 0x75, 0x49, 0xb9, 0x5a, 0x4a, 0x5a, 0x64, 0x86, 0x8e, 0x98, 0x62, 0xa5, 0x52, 0x01,
+    0xc9, 0xbe, 0xd9, 0xfd, 0x7f, 0x61, 0x71, 0x4c, 0x2f, 0x89, 0x4d, 0xcd, 0x25, 0x6f, 0x93, 0x60,
+    0x94, 0x3b, 0x16, 0xd2, 0xeb, 0x54, 0x52, 0xf8, 0xd7, 0x9b, 0xd6, 0x3e, 0xf5, 0x53, 0x34, 0xf8,
+    0x6d, 0xe4, 0xe9, 0xf4, 0x02, 0x06, 0x0c, 0x41, 0x90, 0xe5, 0x7f, 0x4c, 0xeb, 0x89, 0xc6, 0x4f,
+    0x89, 0x9e, 0xff, 0x6f, 0x84, 0xd3, 0x84, 0xba, 0xaf, 0x6e, 0x63, 0x76, 0xdb, 0x0a, 0x98, 0xad,
+    0x59, 0x73, 0xf2, 0x02, 0xad, 0x11, 0x86, 0x3a, 0x19, 0x68, 0x5f, 0x80, 0x66, 0xa6, 0x8f, 0xed,
+    0x92, 0x27, 0xe1, 0x30, 0xf6, 0x6b, 0x7c, 0x66, 0x70, 0xc4, 0x9f, 0xe6, 0xff, 0x96, 0x57, 0xb1,
+    0x87, 0xbf, 0xd0, 0x17, 0x2b, 0x5c, 0x51, 0x5d, 0xfa, 0x13, 0xd3, 0x4f, 0x83, 0x2c, 0x1c, 0xa7,
+    0xe4, 0x4b, 0xb0, 0x57, 0xd2, 0xef, 0xfd, 0x82, 0xe3, 0xf8, 0x6b, 0xa1, 0x28, 0x86, 0x4a, 0xd0,
+    0x82, 0x35, 0x81, 0xe4, 0x30, 0x69, 0x2e, 0x0f, 0xa1, 0x90, 0x9a, 0x1b, 0x5a, 0x91, 0xfe, 0xa1,
+    0xa2, 0xb9, 0x0a, 0xb1, 0x69, 0x02, 0xc9, 0x00, 0x4e, 0xb5, 0xb0, 0x8d, 0x01, 0xea, 0x4d, 0x65,
+    0xd7, 0x19, 0x96, 0x03, 0xab, 0x07, 0x32, 0x2c, 0x7f, 0xc4, 0x8d, 0x91, 0x44, 0xdf, 0xa5, 0xe5,
+    0x88, 0x83, 0xff, 0x24, 0x93, 0x32, 0x69, 0x9a, 0x1f, 0x25, 0x28, 0x84, 0xc2, 0x82, 0x1b, 0x07,
+    0x19, 0x13, 0x2b, 0xf2, 0x85, 0x7d, 0xd2, 0x77, 0x9c, 0x6e, 0xce, 0xcc, 0x0f, 0xa6, 0x03, 0xaf,
+    0xc5, 0x94, 0x52, 0x24, 0xb7, 0x3c, 0x5a, 0x46, 0x2b, 0x08, 0x44, 0xa0, 0x19, 0xdb, 0xe7, 0xf2,
+    0x95, 0x10, 0x59, 0x31, 0x73, 0x9f, 0x62, 0x05, 0x0d, 0x38, 0xe3, 0x65, 0x95, 0xc3, 0xf5, 0x0b,
+    0x70, 0x0d, 0x9e, 0x3d, 0x3f, 0x39, 0x0b, 0x28, 0xee, 0x96, 0xda, 0x2c, 0x50, 0x01, 0xe6, 0xdd,
+    0xd0, 0x74, 0x4d, 0x6b, 0x9a, 0x40, 0xf5, 0xe3, 0x7e, 0xfa, 0xf3, 0x11, 0x3e, 0xad, 0x63, 0xac,
+    0xb7, 0x95, 0x38, 0x69, 0x4f, 0x66, 0xe0, 0xb6, 0x7c, 0x05, 0xca, 0xde, 0x3e, 0x16, 0x2c, 0x2b,
+    0x5b, 0x61, 0x2f, 0x01, 0xf8, 0xe1, 0x4a, 0x65, 0x8f, 0x5c, 0x1d, 0x55, 0x88, 0xdf, 0x62, 0x55,
+    0x67, 0xa6, 0x10, 0xf6, 0x1f, 0x6c, 0xd3, 0xe9, 0x59, 0x8d, 0x3e, 0x63, 0x30, 0x77, 0x48, 0x58,
+    0x3c, 0x6f, 0x08, 0x47, 0xaa, 0x06, 0x57, 0xce, 0x27, 0x3d, 0xb4, 0x21, 0x17, 0x32, 0x45, 0x8b,
+    0xd5, 0xc9, 0x20, 0x8e, 0x71, 0x77, 0xd6, 0xcb, 0xce, 0x3d, 0x28, 0x5e, 0x5a, 0x37, 0xb8, 0xe7,
+    0x60, 0xa1, 0xf5, 0x94, 0x35, 0x4c, 0xf3, 0x79, 0x81, 0x34, 0x3a, 0xdb, 0x73, 0xac, 0x21, 0xf1,
+    0xb4, 0xff, 0x42, 0x98, 0xe6, 0x70, 0x96, 0xfd, 0x5e, 0x88, 0x3f, 0x67, 0x9b, 0x82, 0x36, 0x20,
+    0xdf, 0xc0, 0x1f, 0xad, 0x83, 0x17, 0x8a, 0xda, 0x45, 0xbc, 0xc5, 0xc3, 0x62, 0x07, 0xa8, 0xb7,
+    0x91, 0x25, 0x4f, 0x03, 0x63, 0xb5, 0x16, 0xb1, 0x2d, 0xc6, 0xd9, 0x3b, 0x52, 0x30, 0xa9, 0xe4,
+    0x1b, 0x11, 0x8f, 0xe9, 0x5c, 0xce, 0x80, 0xc2, 0x4c, 0x31, 0x10, 0xb7, 0x4f, 0x16, 0x39, 0x49,
+    0x20, 0xd1, 0xb7, 0x66, 0x48, 0x5b, 0x67, 0xd8, 0xe8, 0x76, 0xc6, 0xa0, 0xe1, 0xa0, 0xdc, 0xdc,
+    0xa1, 0xef, 0x46, 0x2d, 0x07, 0x5d, 0xad, 0xcc, 0xa9, 0xb0, 0x59, 0xe5, 0x69, 0x06, 0xa8, 0xb4,
+    0xb3, 0x76, 0x3f, 0xff, 0xd8, 0x66, 0x5a, 0xe7, 0xa0, 0x19, 0x2e, 0x4a, 0x1d, 0x45, 0xe9, 0x9b,
+    0xbb, 0x38, 0xb6, 0xad, 0x0a, 0x67, 0x0a, 0x9b, 0x29, 0x6e, 0x32, 0xc1, 0x4d, 0x27, 0x61, 0xbd,
+    0x0a, 0x8d, 0x4f, 0xa1, 0xa3, 0xf1, 0x2d, 0x90, 0xd6, 0x3a, 0x91, 0x7f, 0xb7, 0x85, 0x41, 0xec,
+    0x6f, 0xab, 0xaf, 0x93, 0x59, 0xef, 0x00, 0x1c, 0xd5, 0xc3, 0xc6, 0xa7, 0x49, 0xe6, 0x0a, 0xe0,
+    0xda, 0x95, 0x9b, 0xb2, 0x0c, 0xf9, 0x3e, 0xae, 0x1c, 0x09, 0xca, 0x51, 0x35, 0xc6, 0xea, 0x58,
+    0xbf, 0xe9, 0x16, 0x6a, 0xb1, 0xbe, 0x64, 0xff, 0xbf, 0x9d, 0xd4, 0x38, 0x47, 0x86, 0x17, 0x59,
+    0xf2, 0xf3, 0x6c, 0x71, 0xee, 0x57, 0xb1, 0x80, 0xbd, 0xb0, 0xd4, 0xd6, 0xa0, 0xa0, 0x73, 0x82,
+    0x0d, 0xad, 0xb2, 0x34, 0x6d, 0xac, 0x83, 0xd8, 0xed, 0xc7, 0x20, 0x7d, 0xc3, 0x30, 0x0b, 0xf3,
+    0xb3, 0xd3, 0xce, 0x8f, 0x42, 0x2c, 0x8b, 0x29, 0xf8, 0xc7, 0xa3, 0x3c, 0x8b, 0x42, 0x3f, 0xf6,
+    0x0f, 0x2b, 0x5b, 0x58, 0x69, 0x17, 0x33, 0xa2, 0x4f, 0x23, 0x22, 0xaf, 0xb4, 0x7c, 0xab, 0x7b,
+    0x3c, 0xb4, 0x3d, 0x01, 0x83, 0xb1, 0x71, 0x22, 0xef, 0xa4, 0x59, 0xb2, 0x4c, 0x22, 0xe2, 0xb5,
+    0x24, 0x96, 0x90, 0x3d, 0x55, 0xa1, 0xd0, 0x1e, 0x8c, 0x6c, 0xc2, 0xf0, 0x2b, 0xad, 0xaa, 0x27,
+    0x99, 0xfa, 0x76, 0xd6, 0xc4, 0x67, 0xd4, 0x34, 0x1d, 0xb0, 0x4a, 0x03, 0x5c, 0x7c, 0x34, 0x0b,
+    0x0f, 0xe5, 0x47, 0x4d, 0x32, 0x1c, 0xb3, 0x4f, 0x72, 0xf6, 0x1c, 0x29, 0x53, 0x71, 0x77, 0x91,
+    0x5c, 0x4a, 0x2b, 0x8e, 0x12, 0x0b, 0x02, 0x77, 0xfd, 0xfa, 0xc0, 0x7c, 0x15, 0xbf, 0xb7, 0x54,
+    0xfa, 0xbd, 0x90, 0x43, 0x1b, 0xa5, 0x7d, 0xf4, 0x6f, 0x7d, 0x30, 0xc8, 0x8b, 0x52, 0x02, 0x5b,
+    0xeb, 0x17, 0xa4, 0x49, 0xa0, 0x9d, 0xef, 0xbb, 0xa7, 0xb3, 0x40, 0xa7, 0x3e, 0x14, 0x23, 0xbf,
+    0x07, 0x06, 0xc6, 0xe5, 0xd6, 0x25, 0x4b, 0x5e, 0x2f, 0xf6, 0xa3, 0x86, 0xd8, 0xe5, 0xed, 0xae,
+    0x2b, 0x1a, 0xc8, 0xb8, 0xd4, 0x4f, 0xbe, 0x9d, 0x53, 0x61, 0x2f, 0xa5, 0xd3, 0x5b, 0x51, 0x3a,
+    0x5e, 0x22, 0x8d, 0xeb, 0x5e, 0xd6, 0xd4, 0x40, 0x3d, 0x0e, 0x0a, 0x1b, 0x91, 0xcd, 0xa0, 0xeb,
+    0xd1, 0xff, 0xb4, 0x67, 0xe7, 0x0c, 0xf1, 0x37, 0x7e, 0x6c, 0x7f, 0xbb, 0x28, 0xfe, 0x4c, 0x9a,
+    0x94, 0xa0, 0x14, 0x24, 0xb0, 0x3a, 0x29, 0x23, 0x71, 0xa3, 0xf8, 0x66, 0x16, 0xfa, 0x0a, 0xd9,
+    0x70, 0x7a, 0x30, 0x37, 0xb9, 0x5f, 0x00, 0x08, 0xd7, 0x9c, 0xda, 0xd5, 0xc9, 0x82, 0x6c, 0x24,
+    0x48, 0x12, 0xa9, 0x0e, 0x83, 0xb5, 0x6b, 0xe3, 0x56, 0x10, 0x70, 0x02, 0xaa, 0xf4, 0xd3, 0x2d,
+    0xe7, 0xb9, 0x2a, 0x60, 0x4b, 0x01, 0x71, 0xcd, 0x90, 0xac, 0x59, 0x91, 0x32, 0x78, 0x15, 0x8a,
+    0x52, 0x84, 0x75, 0x6d, 0xf8, 0x88, 0xe8, 0xa0, 0xdd, 0x27, 0xf9, 0x66, 0xf6, 0x9b, 0x9e, 0x14,
+    0xcf, 0xcf, 0x0f, 0xb9, 0xad, 0x54, 0x9b, 0xa8, 0x4c, 0x90, 0x92, 0x6b, 0xf3, 0x5e, 0x7b, 0xa8,
+    0xa5, 0x23, 0x4c, 0xdd, 0x57, 0x87, 0xe2, 0xa2, 0x07, 0xd9, 0x30, 0x38, 0xad, 0xbd, 0x72, 0xb0,
+    0x15, 0x25, 0xa9, 0x94, 0x5f, 0x8e, 0x94, 0xf1, 0x6a, 0x5c, 0x87, 0x3d, 0x90, 0x70, 0x65, 0x42,
+    0x1d, 0x3a, 0x2e, 0xf7, 0xe3, 0xb3, 0x8c, 0xbf, 0x1c, 0x38, 0xdc, 0xd6, 0x40, 0xa6, 0x18, 0x30,
+    0x87, 0xab, 0x40, 0xb5, 0x7d, 0x3a, 0x8d, 0x75, 0x39, 0x8a, 0x92, 0xb2, 0x1c, 0xbc, 0x83, 0xe8,
+    0x96, 0x91, 0x14, 0xd9, 0x68, 0xad, 0x12, 0xcc, 0x70, 0x22, 0xdd, 0x80, 0x8c, 0x81, 0xb6, 0xd6,
+    0xc1, 0xf2, 0x1d, 0xa0, 0xfd, 0xf5, 0xb8, 0x83, 0x1a, 0x75, 0xd4, 0xaf, 0x64, 0x8b, 0x2b, 0xf7,
+    0xf5, 0x31, 0x90, 0x79, 0xc6, 0x17, 0x23, 0x5f, 0xc6, 0x9e, 0x0e, 0x67, 0x3c, 0x0c, 0x5f, 0x0a,
+    0x03, 0xb3, 0x98, 0xf4, 0x36, 0x75, 0x4c, 0x1e, 0xb5, 0x22, 0x6d, 0xe8, 0xe3, 0x16, 0x9f, 0xfd,
+    0xdf, 0x33, 0x90, 0x1d, 0xea, 0xba, 0xde, 0x5a, 0x2b, 0x5d, 0xbe, 0xd7, 0x57, 0xcd, 0xc3, 0xbc,
+    0xae, 0x02, 0xd3, 0x41, 0x1f, 0x3d, 0x5f, 0x83, 0xbc, 0x86, 0xf2, 0x5b, 0xb8, 0x7d, 0x0b, 0xd1,
+    0x9a, 0x5a, 0x19, 0x5b, 0x8c, 0x53, 0xcd, 0x9a, 0x1c, 0x08, 0xec, 0xe9, 0xac, 0x3e, 0x41, 0x5a,
+    0x31, 0xb1, 0x72, 0x05, 0xd6, 0xfd, 0x94, 0x70, 0x1d, 0xca, 0x05, 0x7c, 0x1c, 0x12, 0xcc, 0x42,
+    0x2f, 0x26, 0x8d, 0xee, 0x4a, 0xdf, 0xaf, 0xab, 0x61, 0xd6, 0x24, 0x96, 0xe0, 0x40, 0x89, 0xff,
+    0xb0, 0xc2, 0xce, 0x44, 0xf2, 0x71, 0x03, 0x06, 0x57, 0xfe, 0x26, 0x7c, 0x80, 0x7b, 0xdf, 0x08,
+    0xcc, 0xd6, 0x09, 0x13, 0x2e, 0x9e, 0xd1, 0xa5, 0xad, 0x99, 0x64, 0xd7, 0x79, 0xf7, 0x28, 0xb1,
+    0xd8, 0x72, 0x64, 0x3a, 0xdf, 0xf5, 0x9c, 0x84, 0x13, 0x5c, 0x54, 0x87, 0x37, 0x4f, 0xe4, 0x21,
+    0x96, 0x9f, 0x0b, 0x36, 0x2b, 0xd1, 0x5c, 0xba, 0x77, 0x54, 0x93, 0x77, 0x63, 0xef, 0x5a, 0x50,
+    0x01, 0x55, 0x94, 0x7b, 0x55, 0x3a, 0x05, 0x3f, 0x75, 0xe0, 0xfc, 0x9b, 0x0b, 0xa1, 0x25, 0xba,
+    0xab, 0x24, 0x45, 0x62, 0x45, 0x10, 0x80, 0xfd, 0x43, 0x5b, 0x91, 0x92, 0x87, 0x95, 0xf4, 0x23,
+    0xfd, 0xb2, 0x08, 0xea, 0x8f, 0xe7, 0xc5, 0x18, 0xdf, 0x33, 0xc6, 0x6d, 0xa2, 0x92, 0xa2, 0x19,
+    0x5c, 0xca, 0x48, 0xcb, 0xcb, 0x3c, 0xdf, 0xcb, 0xf0, 0x24, 0xae, 0x12, 0x4d, 0xf6, 0xc3, 0x57,
+    0xbd, 0x5c, 0x82, 0xda, 0xa2, 0x3e, 0x59, 0xdf, 0x8c, 0xb7, 0x67, 0x55, 0x0f, 0xb4, 0x56, 0xab,
+    0x52, 0xe2, 0xfd, 0xc8, 0x7b, 0x80, 0x5e, 0xe4, 0x3e, 0xcf, 0x3c, 0xff, 0x59, 0x26, 0x22, 0x34,
+    0x01, 0xe3, 0xde, 0xab, 0x74, 0x67, 0x72, 0x65, 0x91, 0xc5, 0x4d, 0xed, 0x2b, 0x96, 0x10, 0x24,
+    0x4d, 0xb8, 0x4e, 0x40, 0xba, 0x92, 0x8d, 0xa8, 0xef, 0xf7, 0x57, 0x12, 0xeb, 0x30, 0x95, 0xec,
+    0x14, 0x95, 0x2d, 0x4d, 0x94, 0x5a, 0xfc, 0x77, 0x5b, 0xf8, 0xc6, 0xb0, 0x6d, 0xb8, 0xde, 0xec,
+    0x11, 0xd6, 0x7c, 0x51, 0xe6, 0x2c, 0x46, 0xe5, 0x41, 0x8b, 0x05, 0xc2, 0x2a, 0xa0, 0x44, 0x3c,
+    0xb4, 0x05, 0x37, 0x0c, 0x66, 0x72, 0x33, 0xe4, 0x9a, 0x48, 0xdd, 0x80, 0xa5, 0x19, 0x32, 0x3d,
+    0xbb, 0x0e, 0xf6, 0x21, 0x99, 0x0c, 0x14, 0x12, 0xcf, 0xd0, 0xe0, 0x93, 0x57, 0xb8, 0x22, 0x01,
+    0x30, 0x45, 0x89, 0xa4, 0xe0, 0x03, 0xa3, 0x52, 0xec, 0x07, 0x36, 0x52, 0x53, 0xde, 0xbf, 0x06,
+    0xa6, 0x7c, 0x67, 0x9c, 0xad, 0xcc, 0x56, 0x2c, 0x0e, 0xdd, 0x6a, 0xcb, 0x0b, 0x16, 0xa0, 0x9c,
+    0x55, 0xc6, 0x7e, 0xfc, 0x99, 0x66, 0x41, 0xf0, 0x76, 0xdf, 0x03, 0x06, 0xec, 0x51, 0x90, 0xa7,
+    0xfc, 0x50, 0x0e, 0x6a, 0x9d, 0xb5, 0xb9, 0xd5, 0x54, 0x28, 0x97, 0x04, 0x27, 0x35, 0x24, 0x87,
+    0xc4, 0xd7, 0x17, 0x5b, 0xd0, 0x5c, 0x6c, 0x58, 0x89, 0xae, 0x96, 0xdd, 0x8e, 0x27, 0xa8, 0xfb,
+    0x9a, 0x93, 0x54, 0x3a, 0xbd, 0x9e, 0x42, 0xd0, 0xb6, 0x7a, 0xc3, 0x08, 0xc6, 0xa5, 0x4f, 0xa6,
+    0xc5, 0x8c, 0xfa, 0xb4, 0x74, 0x8f, 0x47, 0x5c, 0x85, 0x87, 0xf0, 0x46, 0x21, 0x40, 0x02, 0x8e,
+    0x79, 0x19, 0xa7, 0xcf, 0xc6, 0xfa, 0x5c, 0x26, 0xfd, 0xa0, 0x3a, 0x66, 0xc1, 0xfa, 0x17, 0xef,
+    0x07, 0x9f, 0x22, 0x1f, 0x0f, 0x8b, 0x80, 0x34, 0x8e, 0xc7, 0x2e, 0x42, 0xf0, 0x9b, 0x5d, 0xbc,
+    0x26, 0xe7, 0x2d, 0xde, 0xbc, 0xdb, 0xeb, 0xc7, 0x29, 0x87, 0x07, 0xd9, 0xc7, 0xb5, 0x3e, 0x71,
+    0xfb, 0xdc, 0x7f, 0x36, 0xa2, 0xe9, 0x58, 0xe6, 0xcc, 0x63, 0x75, 0x36, 0x52, 0xca, 0xe7, 0x06,
+    0x1b, 0xa8, 0xbb, 0x03, 0x10, 0xce, 0xa5, 0xe9, 0x66, 0xac, 0xdd, 0x59, 0x0f, 0x3a, 0x90, 0x60,
+    0x68, 0xe8, 0xeb, 0x60, 0xf1, 0xa8, 0xa0, 0xdc, 0x39, 0x07, 0x40, 0x05, 0x43, 0xb5, 0x6f, 0x3d,
+    0x3b, 0x5a, 0x34, 0x53, 0xc2, 0x6c, 0xa4, 0x47, 0x4c, 0xe1, 0xfe, 0x7f, 0x37, 0xfb, 0x91, 0xca,
+    0x28, 0x7a, 0xdc, 0xef, 0xde, 0xc4, 0x44, 0xf4, 0xc0, 0x22, 0xd2, 0x4c, 0x48, 0x16, 0x54, 0x01,
+    0x7c, 0xdf, 0xe4, 0x3f, 0x29, 0x51, 0xae, 0x9c, 0x98, 0xf4, 0x73, 0x36, 0x94, 0x0d, 0xe2, 0xc8,
+    0x35, 0xd9, 0xe2, 0xbc, 0x5c, 0x0b, 0xc7, 0xc6, 0xdd, 0x70, 0x2e, 0x6f, 0xdd, 0x23, 0xfe, 0xef,
+    0x4c, 0xaf, 0x06, 0xce, 0x1c, 0x26, 0xf9, 0xe9, 0x02, 0x22, 0xe9, 0x4d, 0x26, 0x80, 0xbc, 0x5a,
+    0x18, 0xc0, 0x2b, 0x76, 0xae, 0x65, 0x17, 0x6a, 0x56, 0xa4, 0xeb, 0xaa, 0xb7, 0x65, 0xe1, 0x55,
+    0xfa, 0xe5, 0x08, 0x95, 0x3c, 0x33, 0xca, 0xa0, 0xb0, 0x03, 0x09, 0x22, 0x81, 0x98, 0x3b, 0x93,
+    0x6e, 0xb2, 0x1a, 0xba, 0x05, 0x0c, 0xfd, 0xe4, 0x51, 0x10, 0xe0, 0x1c, 0x1e, 0xf5, 0x7c, 0xf8,
+    0x22, 0x86, 0x6d, 0x00, 0x2d, 0x39, 0xaf, 0x8a, 0x25, 0xa2, 0xbc, 0x8b, 0x80, 0xfe, 0x1c, 0x87,
+    0x5a, 0xd6, 0x7f, 0xf5, 0xeb, 0x13, 0x59, 0xf8, 0x37, 0xda, 0xf7, 0xf8, 0xe2, 0x39, 0xbb, 0x12,
+    0x45, 0xb4, 0x2d, 0x03, 0x43, 0x44, 0x11, 0xf7, 0x0b, 0x32, 0x82, 0x0c, 0x68, 0xca, 0x8e, 0xf3,
+    0x5c, 0x44, 0x02, 0x53, 0xb0, 0x0a, 0xa7, 0x74, 0x8b, 0x48, 0x8c, 0x54, 0xb0, 0x69, 0xfb, 0xfe,
+    0xdf, 0xbe, 0xb7, 0x44, 0x66, 0x6c, 0x51, 0x8a, 0x6b, 0x62, 0xf9, 0x26, 0x63, 0xc2, 0x62, 0xe1,
+    0x68, 0xcd, 0x24, 0xe5, 0xff, 0xa2, 0x01, 0x3d, 0x9b, 0x80, 0xed, 0xfd, 0x41, 0xb1, 0x9c, 0xba,
+    0x60, 0xfd, 0x3d, 0xd3, 0x32, 0xa9, 0x1d, 0x16, 0xd7, 0x9e, 0xc8, 0x08, 0xe8, 0xb7, 0x8c, 0x67,
+    0xb1, 0x8e, 0x53, 0xaf, 0xa5, 0x71, 0x8c, 0xab, 0x50, 0x74, 0xf8, 0x93, 0x00, 0x79, 0xbf, 0xa5,
+    0xda, 0x78, 0x82, 0x57, 0x97, 0x8b, 0xfe, 0x61, 0x3c, 0xd3, 0xa1, 0xca, 0xbe, 0xde, 0x60, 0x5a,
+    0xb6, 0x10, 0x64, 0xf9, 0x86, 0x44, 0x9c, 0xa8, 0xad, 0xd3, 0x52, 0x12, 0xa0, 0xcc, 0x8b, 0xaa,
+    0x39, 0xec, 0x9c, 0xc3, 0x43, 0x43, 0xe8, 0xd7, 0x79, 0xdb, 0xb8, 0x59, 0x85, 0x96, 0x7a, 0x92,
+    0x38, 0xff, 0x24, 0x10, 0xed, 0xc1, 0x87, 0x5d, 0x86, 0x34, 0x87, 0x2b, 0xd0, 0x5d, 0x3d, 0xac,
+    0x2c, 0x27, 0xd2, 0xa9, 0x75, 0x2d, 0xa3, 0xf2, 0xd3, 0xdb, 0xe4, 0xa6, 0xde, 0xe9, 0x0b, 0xd2,
+    0x61, 0x5c, 0xd5, 0xdd, 0xd1, 0x6d, 0x1f, 0x68, 0x27, 0xb3, 0x40, 0x60, 0x1a, 0x5d, 0x5b, 0xa9,
+    0xcd, 0x85, 0x85, 0x4d, 0x73, 0xa9, 0x16, 0x46, 0x65, 0x4a, 0xff, 0x72, 0xb1, 0x1c, 0x73, 0xa2
+};
+
+#endif

core/libs/prx/libSceAgcDriver/tests/execution/FlatLoads.cpp

@@ -0,0 +1,182 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "FlatGlobalLoadsRegion.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Results = 16;
+constexpr std::uint32_t Sentinel = 0x05e471e1;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t RegionOffset = Threads * 16u;
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 78> LoadsCode{
+    0x34020084, 0x34040086, 0xdc388000, 0x04080001, 0xbf8c3f70, 0x7e1402ff, 0x5a5a0000, 0x7e1602ff,
+    0x5a5a0001, 0x7e1802ff, 0x5a5a0002, 0x7e1a02ff, 0x5a5a0003, 0x7e1c02ff, 0x5a5a0004, 0x7e1e02ff,
+    0x5a5a0005, 0x7e2002ff, 0x5a5a0006, 0x7e2202ff, 0x5a5a0007, 0x7e2402ff, 0x5a5a0008, 0x7e2602ff,
+    0x5a5a0009, 0x7e2802ff, 0x5a5a000a, 0x7e2a02ff, 0x5a5a000b, 0x7e2c02ff, 0x5a5a000c, 0x7e2e02ff,
+    0x5a5a000d, 0x7e3002ff, 0x5a5a000e, 0x7e3202ff, 0x5a5a000f, 0x4a4008ff, 0x00000a00, 0x4a420eff,
+    0x00000200, 0xd70f6a22, 0x02024008, 0x7e460209, 0x50464680, 0x4a4c0aff, 0x00000200, 0xd70f6a24,
+    0x02024c08, 0x7e4a0209, 0x504a4a80, 0x36500c81, 0x7daa5080, 0xdc200000, 0x0a7d0024, 0xdc2407ff,
+    0x0b7d0024, 0xdc280001, 0x0c7d0024, 0xdc2c0003, 0x0d7d0024, 0xdc300002, 0x0e7d0024, 0xdc300400,
+    0x187d0024, 0xdc2807ff, 0x197d0024, 0xbf8c0070, 0xbefe04c1, 0xe0781000, 0x80010a02, 0xe0781010,
+    0x80010e02, 0xe0781020, 0x80011202, 0xe0781030, 0x80011602, 0xbf810000
+};
+
+struct Row {
+    std::array<std::uint32_t, 4> input;
+    std::array<std::uint32_t, Results> results;
+};
+
+constexpr std::array<Row, Threads> Rows{{
+    {{0x00000000, 0x00000000, 0x3ceb3ffd, 0x00000000}, {0x000000d2, 0x0000007a, 0x000026f2, 0xffffa665, 0x0ca66526, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0xfe1ec477, 0x0000357a}},
+    {{0x00000001, 0x00000001, 0x97b75093, 0x00000001}, {0x000000f2, 0x00000035, 0x00006526, 0x00000ca6, 0x120ca665, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0xe4fe1ec4, 0x00009735}},
+    {{0x00000002, 0x00000fa0, 0x8b529b4b, 0x00000002}, {0x000000a4, 0x0000005a, 0x0000449e, 0xffffadea, 0x9fadea44, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x37307a70, 0x0000b65a}},
+    {{0x00000003, 0x00000f9f, 0x21636369, 0x00000003}, {0x000000dd, 0x00000060, 0x00009ea4, 0xffffea44, 0xadea449e, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x307a70d9, 0x00005a60}},
+    {{0x000003ff, 0x00000002, 0x5eb561a5, 0x000017fc}, {0x00000026, 0xffffff97, 0x0000a665, 0x0000120c, 0xd2120ca6, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x8ce4fe1e, 0x00004397}},
+    {{0x000003fe, 0x00000003, 0xea7b5bf4, 0x000017fb}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003fd, 0x000007ff, 0x9a9a80fd, 0x000017fa}, {0x0000007a, 0x0000005a, 0x00009735, 0xffff9d43, 0x819d4397, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5131541e, 0x0000295a}},
+    {{0x00000201, 0x00000801, 0x795b929f, 0x00000ffd}, {0x00000097, 0xffffffdc, 0x00009d43, 0x00003c81, 0x513c819d, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x4cb65131, 0x0000f3dc}},
+    {{0x00000086, 0x000009b0, 0xe8a8529f, 0x0000006b}, {0x00000000, 0x00000058, 0x00003386, 0xffffc349, 0xc0c34933, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0xb4de5e8e, 0x00003c58}},
+    {{0x000003c0, 0x00000426, 0x3bfd1d33, 0x000011a0}, {0x00000021, 0xffffff8b, 0x0000d19a, 0xffff82a1, 0xc782a1d1, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x401ae5e1, 0x00009c8b}},
+    {{0x00000188, 0x00000b79, 0x8a7d43b5, 0x00000f0c}, {0x000000fa, 0x0000003d, 0x000017c8, 0xffffe6ab, 0xe4e6ab17, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x3f5aefae, 0x00000e3d}},
+    {{0x000003cf, 0x0000065a, 0xdc6bf1e1, 0x00001473}, {0x0000008d, 0x00000074, 0x0000538b, 0xfffffa29, 0x6dfa2953, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0xda660308, 0x0000e774}},
+    {{0x00000134, 0x000003b5, 0x26d0b944, 0x00001450}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000031e, 0x00000bdc, 0xabe19f59, 0x0000007c}, {0x0000005d, 0x00000066, 0x00007957, 0xffff86c7, 0x8d86c779, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0xb19c74d6, 0x0000f666}},
+    {{0x00000083, 0x0000028c, 0x0af438d3, 0x000012ea}, {0x00000010, 0x0000006e, 0x0000fc1c, 0xffffc9eb, 0x3ac9ebfc, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x3c3e2f3f, 0x0000506e}},
+    {{0x00000268, 0x00000c7b, 0xd2d58443, 0x000000fe}, {0x00000015, 0xffffffbe, 0x000078a4, 0xffffe536, 0x26e53678, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x936f25cd, 0x0000d7be}},
+    {{0x00000227, 0x00000790, 0xb804d821, 0x00001308}, {0x000000a0, 0x0000006b, 0x00004ec7, 0xffffeba6, 0x4eeba64e, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x6b431e8e, 0x0000f86b}},
+    {{0x00000319, 0x00000b6d, 0x651c5253, 0x00000da9}, {0x000000c8, 0xffffffd3, 0x00003527, 0x000067e7, 0xca67e735, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0xd052e9fc, 0x00005bd3}},
+    {{0x0000038e, 0x00000f6c, 0xe0fd67dd, 0x0000044a}, {0x000000cd, 0xfffffffd, 0x0000e9fc, 0xffffd052, 0x66d052e9, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x3a515bd3, 0x000041fd}},
+    {{0x000002ec, 0x0000018f, 0x22cedafa, 0x00000125}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003f5, 0x00000378, 0xf729b4c9, 0x00000841}, {0x000000fc, 0xffffffb0, 0x0000267e, 0xffffb948, 0x9eb94826, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0xf83f261f, 0x00009bb0}},
+    {{0x0000037d, 0x00000c76, 0xdaf0105b, 0x0000140d}, {0x00000079, 0xffffffba, 0x000069c5, 0x00001a8c, 0x151a8c69, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x892f4c71, 0x0000deba}},
+    {{0x00000268, 0x000006bd, 0xd55ec1a5, 0x0000103b}, {0x00000013, 0x00000033, 0x0000835f, 0x00002ddd, 0x722ddd83, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x19598433, 0x00000233}},
+    {{0x00000316, 0x0000092f, 0x88bafad9, 0x00000b3a}, {0x000000ca, 0x0000001b, 0x00006436, 0xffff959f, 0x13959f64, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x1588587f, 0x0000071b}},
+    {{0x00000342, 0x00000959, 0xe779c471, 0x0000076f}, {0x00000088, 0x0000000d, 0x0000aae4, 0xffffe110, 0xdee110aa, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0xe242e585, 0x0000380d}},
+    {{0x000002b1, 0x00000ae9, 0xdb3d1151, 0x000000ea}, {0x00000013, 0xffffffef, 0x0000ea6b, 0xffffa06b, 0xb2a06bea, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x72ee855b, 0x0000a4ef}},
+    {{0x0000023c, 0x000009b1, 0xb2109306, 0x0000157b}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000014e, 0x00000b2d, 0xf6f22f41, 0x00000a71}, {0x000000af, 0x00000015, 0x0000fb40, 0xffffad6d, 0x2fad6dfb, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0xde591a71, 0x0000bf15}},
+    {{0x000000d5, 0x00000b6b, 0x360c4979, 0x000014f9}, {0x00000073, 0x0000002f, 0x0000c858, 0x00003527, 0xe73527c8, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0xe9fccd90, 0x0000a52f}},
+    {{0x00000222, 0x0000048f, 0x103ef3c3, 0x000003fb}, {0x0000009d, 0xffffffde, 0x000023c0, 0xffffdea4, 0x49dea423, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0xa50e02f7, 0x00001fde}},
+    {{0x000003db, 0x00000da9, 0xfd63ed5b, 0x00001470}, {0x000000fe, 0xffffff9a, 0x0000602e, 0x00000715, 0x0d071560, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x30633e8d, 0x0000489a}},
+    {{0x000003de, 0x0000016a, 0xccf3d0b2, 0x00000b02}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}}
+}};
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
+#endif
+        Require(block != nullptr, "flat loads: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, false);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        VirtualFree(block, 0, MEM_RELEASE);
+#else
+        std::free(block);
+#endif
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%x", value);
+    return text;
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const std::uint8_t* base) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(base));
+    std::vector<std::uint32_t> userData(10, 0u);
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    userData[8] = static_cast<std::uint32_t>(address);
+    userData[9] = static_cast<std::uint32_t>(address >> 32u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::span<const std::uint32_t> code) {
+    Output.fill(Sentinel);
+    Dispatch(device, code, guest.Data());
+    const std::string wave = "flat loads: lane ";
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const auto& expected = Rows[tid].results;
+        for (std::uint32_t result = 0; result < Results; ++result) {
+            const auto actual = Output[tid * Results + result];
+            Require(actual == expected[result], wave + std::to_string(tid) + " v" + std::to_string(10u + result) + " is " + Hex(actual) + ", expected " + Hex(expected[result]));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock guest;
+        std::memset(guest.Data(), 0, BlockBytes);
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            std::memcpy(guest.Data() + tid * 16u, Rows[tid].input.data(), 16u);
+        }
+        std::memcpy(guest.Data() + RegionOffset, FlatGlobalLoadsRegion.data(), FlatGlobalLoadsRegion.size());
+        Run(*device, guest, LoadsCode);
+        std::puts("flat loads tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX2X3.cpp

@@ -0,0 +1,181 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "FlatGlobalLoadsRegion.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Results = 16;
+constexpr std::uint32_t Sentinel = 0x05e471e1;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t RegionOffset = Threads * 16u;
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 68> LoadsCode{
+    0x34020084, 0x34040086, 0xdc388000, 0x04080001, 0xbf8c3f70, 0x7e1402ff, 0x5a5a0000, 0x7e1602ff,
+    0x5a5a0001, 0x7e1802ff, 0x5a5a0002, 0x7e1a02ff, 0x5a5a0003, 0x7e1c02ff, 0x5a5a0004, 0x7e1e02ff,
+    0x5a5a0005, 0x7e2002ff, 0x5a5a0006, 0x7e2202ff, 0x5a5a0007, 0x7e2402ff, 0x5a5a0008, 0x7e2602ff,
+    0x5a5a0009, 0x7e2802ff, 0x5a5a000a, 0x7e2a02ff, 0x5a5a000b, 0x7e2c02ff, 0x5a5a000c, 0x7e2e02ff,
+    0x5a5a000d, 0x7e3002ff, 0x5a5a000e, 0x7e3202ff, 0x5a5a000f, 0x4a4008ff, 0x00000a00, 0x4a420eff,
+    0x00000200, 0xd70f6a22, 0x02024008, 0x7e460209, 0x50464680, 0x4a4c0aff, 0x00000200, 0xd70f6a24,
+    0x02024c08, 0x7e4a0209, 0x504a4a80, 0x36500c81, 0x7daa5080, 0xdc340001, 0x0f7d0024, 0xdc3c0007,
+    0x117d0024, 0xbf8c0070, 0xbefe04c1, 0xe0781000, 0x80010a02, 0xe0781010, 0x80010e02, 0xe0781020,
+    0x80011202, 0xe0781030, 0x80011602, 0xbf810000
+};
+
+struct Row {
+    std::array<std::uint32_t, 4> input;
+    std::array<std::uint32_t, Results> results;
+};
+
+constexpr std::array<Row, Threads> Rows{{
+    {{0x00000000, 0x00000000, 0x3ceb3ffd, 0x00000000}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xa66526f2, 0x89d2120c, 0x5d1889d2, 0x81e80e95, 0x6f160936, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000001, 0x00000001, 0x97b75093, 0x00000001}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x0ca66526, 0x1889d212, 0x955d1889, 0x3681e80e, 0x6b6f1609, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000002, 0x00000fa0, 0x8b529b4b, 0x00000002}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xadea449e, 0x8aa0459f, 0xecce8aa0, 0x40199f09, 0x6f03851f, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000003, 0x00000f9f, 0x21636369, 0x00000003}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xea449ea4, 0xa0459fad, 0xce8aa045, 0x199f09ec, 0x03851f40, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003ff, 0x00000002, 0x5eb561a5, 0x000017fc}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x120ca665, 0x5d1889d2, 0x0e955d18, 0x093681e8, 0x116b6f16, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003fe, 0x00000003, 0xea7b5bf4, 0x000017fb}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003fd, 0x000007ff, 0x9a9a80fd, 0x000017fa}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x9d439735, 0x5f513c81, 0x32095f51, 0xa229672e, 0x53ad47ef, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000201, 0x00000801, 0x795b929f, 0x00000ffd}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x3c819d43, 0x32095f51, 0x672e3209, 0x47efa229, 0x60e553ad, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000086, 0x000009b0, 0xe8a8529f, 0x0000006b}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xc3493386, 0x010dfac0, 0x7d59010d, 0xcbb17d18, 0x7ef72fd3, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003c0, 0x00000426, 0x3bfd1d33, 0x000011a0}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x82a1d19a, 0xb41ce3c7, 0x3b5db41c, 0x7ce0e57f, 0x00280664, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000188, 0x00000b79, 0x8a7d43b5, 0x00000f0c}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xe6ab17c8, 0xa2c98ce4, 0x4cd6a2c9, 0x36b17e32, 0xd6bd1487, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003cf, 0x0000065a, 0xdc6bf1e1, 0x00001473}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xfa29538b, 0xfc1ae26d, 0x4312fc1a, 0x1835159f, 0xb5fd7f6b, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000134, 0x000003b5, 0x26d0b944, 0x00001450}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000031e, 0x00000bdc, 0xabe19f59, 0x0000007c}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x86c77957, 0x35e9c58d, 0x6f4835e9, 0x8d406c57, 0x4a4ad30d, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000083, 0x0000028c, 0x0af438d3, 0x000012ea}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xc9ebfc1c, 0x1ae0f83a, 0x43151ae0, 0xc7e70a45, 0x21c1452e, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000268, 0x00000c7b, 0xd2d58443, 0x000000fe}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xe53678a4, 0x6d03a026, 0x02016d03, 0xfc1fabaf, 0x3716dbf7, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000227, 0x00000790, 0xb804d821, 0x00001308}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xeba64ec7, 0x6c3f954e, 0xa8636c3f, 0x7080725e, 0x9e00052d, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000319, 0x00000b6d, 0x651c5253, 0x00000da9}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x67e73527, 0x9c2a88ca, 0xb0e49c2a, 0x17c8fa9b, 0x8ce4e6ab, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000038e, 0x00000f6c, 0xe0fd67dd, 0x0000044a}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xd052e9fc, 0x0d8fd866, 0x84530d8f, 0xefaef525, 0x6cde3f5a, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000002ec, 0x0000018f, 0x22cedafa, 0x00000125}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003f5, 0x00000378, 0xf729b4c9, 0x00000841}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xb948267e, 0x0b25a49e, 0xd5d30b25, 0xa083e4b7, 0xcfb3bb6d, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000037d, 0x00000c76, 0xdaf0105b, 0x0000140d}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x1a8c69c5, 0x3678a415, 0x26e53678, 0x016d03a0, 0x1fabaf02, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000268, 0x000006bd, 0xd55ec1a5, 0x0000103b}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x2ddd835f, 0xc6429a72, 0xaac7c642, 0xa31b01f2, 0x599eb598, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000316, 0x0000092f, 0x88bafad9, 0x00000b3a}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x959f6436, 0x2be99013, 0x08252be9, 0x9f1b1c06, 0xfa5829ed, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000342, 0x00000959, 0xe779c471, 0x0000076f}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xe110aae4, 0xb6eac1de, 0x62f1b6ea, 0x34343f1b, 0xf308081c, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000002b1, 0x00000ae9, 0xdb3d1151, 0x000000ea}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xa06bea6b, 0x945aacb2, 0x1b43945a, 0x66bd4d39, 0xf886f4f0, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000023c, 0x000009b1, 0xb2109306, 0x0000157b}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000014e, 0x00000b2d, 0xf6f22f41, 0x00000a71}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xad6dfb40, 0xce007b2f, 0xccb8ce00, 0xa73e5b47, 0x7c7a524d, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000000d5, 0x00000b6b, 0x360c4979, 0x000014f9}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x3527c858, 0x88ca67e7, 0x9c2a88ca, 0xfa9bb0e4, 0xe6ab17c8, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000222, 0x0000048f, 0x103ef3c3, 0x000003fb}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0xdea423c0, 0xe90c7c49, 0x8cede90c, 0x6a782b20, 0x414c4857, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003db, 0x00000da9, 0xfd63ed5b, 0x00001470}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x0715602e, 0x5e8e080d, 0xb4de5e8e, 0xd8f27c75, 0xdc10e5e8, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003de, 0x0000016a, 0xccf3d0b2, 0x00000b02}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}}
+}};
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
+#endif
+        Require(block != nullptr, "flat loads x2 x3: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, false);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        VirtualFree(block, 0, MEM_RELEASE);
+#else
+        std::free(block);
+#endif
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%x", value);
+    return text;
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const std::uint8_t* base) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(base));
+    std::vector<std::uint32_t> userData(10, 0u);
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    userData[8] = static_cast<std::uint32_t>(address);
+    userData[9] = static_cast<std::uint32_t>(address >> 32u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::span<const std::uint32_t> code) {
+    Output.fill(Sentinel);
+    Dispatch(device, code, guest.Data());
+    const std::string wave = "flat loads x2 x3: lane ";
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const auto& expected = Rows[tid].results;
+        for (std::uint32_t result = 0; result < Results; ++result) {
+            const auto actual = Output[tid * Results + result];
+            Require(actual == expected[result], wave + std::to_string(tid) + " v" + std::to_string(10u + result) + " is " + Hex(actual) + ", expected " + Hex(expected[result]));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock guest;
+        std::memset(guest.Data(), 0, BlockBytes);
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            std::memcpy(guest.Data() + tid * 16u, Rows[tid].input.data(), 16u);
+        }
+        std::memcpy(guest.Data() + RegionOffset, FlatGlobalLoadsRegion.data(), FlatGlobalLoadsRegion.size());
+        Run(*device, guest, LoadsCode);
+        std::puts("flat loads x2 x3 tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX4.cpp

@@ -0,0 +1,181 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "FlatGlobalLoadsRegion.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Results = 16;
+constexpr std::uint32_t Sentinel = 0x05e471e1;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t RegionOffset = Threads * 16u;
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 66> LoadsCode{
+    0x34020084, 0x34040086, 0xdc388000, 0x04080001, 0xbf8c3f70, 0x7e1402ff, 0x5a5a0000, 0x7e1602ff,
+    0x5a5a0001, 0x7e1802ff, 0x5a5a0002, 0x7e1a02ff, 0x5a5a0003, 0x7e1c02ff, 0x5a5a0004, 0x7e1e02ff,
+    0x5a5a0005, 0x7e2002ff, 0x5a5a0006, 0x7e2202ff, 0x5a5a0007, 0x7e2402ff, 0x5a5a0008, 0x7e2602ff,
+    0x5a5a0009, 0x7e2802ff, 0x5a5a000a, 0x7e2a02ff, 0x5a5a000b, 0x7e2c02ff, 0x5a5a000c, 0x7e2e02ff,
+    0x5a5a000d, 0x7e3002ff, 0x5a5a000e, 0x7e3202ff, 0x5a5a000f, 0x4a4008ff, 0x00000a00, 0x4a420eff,
+    0x00000200, 0xd70f6a22, 0x02024008, 0x7e460209, 0x50464680, 0x4a4c0aff, 0x00000200, 0xd70f6a24,
+    0x02024c08, 0x7e4a0209, 0x504a4a80, 0x36500c81, 0x7daa5080, 0xdc3807f1, 0x147d0024, 0xbf8c0070,
+    0xbefe04c1, 0xe0781000, 0x80010a02, 0xe0781010, 0x80010e02, 0xe0781020, 0x80011202, 0xe0781030,
+    0x80011602, 0xbf810000
+};
+
+struct Row {
+    std::array<std::uint32_t, 4> input;
+    std::array<std::uint32_t, Results> results;
+};
+
+constexpr std::array<Row, Threads> Rows{{
+    {{0x00000000, 0x00000000, 0x3ceb3ffd, 0x00000000}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0xe5522840, 0xd0e7469d, 0x80412474, 0x357aebf6, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000001, 0x00000001, 0x97b75093, 0x00000001}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x9de55228, 0x74d0e746, 0xf6804124, 0x97357aeb, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000002, 0x00000fa0, 0x8b529b4b, 0x00000002}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x97578278, 0x3c61fe8b, 0xbecaa1d3, 0xb65a60de, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000003, 0x00000f9f, 0x21636369, 0x00000003}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x578278da, 0x61fe8b97, 0xcaa1d33c, 0x5a60debe, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003ff, 0x00000002, 0x5eb561a5, 0x000017fc}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x469de552, 0x2474d0e7, 0xebf68041, 0x4397357a, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003fe, 0x00000003, 0xea7b5bf4, 0x000017fb}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003fd, 0x000007ff, 0x9a9a80fd, 0x000017fa}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x553e074f, 0x8b833038, 0x6595f862, 0x295aec03, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000201, 0x00000801, 0x795b929f, 0x00000ffd}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x3038553e, 0xf8628b83, 0xec036595, 0xf3dc295a, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000086, 0x000009b0, 0xe8a8529f, 0x0000006b}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x1ff610a6, 0x59e9d36c, 0x30633e8d, 0x3c584877, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003c0, 0x00000426, 0x3bfd1d33, 0x000011a0}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x300b011b, 0x9b79ebd2, 0xc90fa8c4, 0x9c8be880, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000188, 0x00000b79, 0x8a7d43b5, 0x00000f0c}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5bd3a52f, 0x225e3a51, 0xd65eeb8d, 0x0e3d40d4, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003cf, 0x0000065a, 0xdc6bf1e1, 0x00001473}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x4506273f, 0x554bd692, 0x7d422acd, 0xe774511b, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000134, 0x000003b5, 0x26d0b944, 0x00001450}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000031e, 0x00000bdc, 0xabe19f59, 0x0000007c}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x528a1578, 0xf86d7584, 0xdda0e888, 0xf666f927, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000083, 0x0000028c, 0x0af438d3, 0x000012ea}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0xc6e0e1a7, 0xda95b3a4, 0xa42ead3a, 0x506e741f, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000268, 0x00000c7b, 0xd2d58443, 0x000000fe}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0xfd9f16e3, 0x1d9033df, 0x5adebaea, 0xd7be5d2b, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000227, 0x00000790, 0xb804d821, 0x00001308}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x871b5d02, 0x6e53112f, 0x05ab81b3, 0xf86b2339, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000319, 0x00000b6d, 0x651c5253, 0x00000da9}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x1a2baeed, 0x4fd4b8c8, 0x61539dbe, 0x5bd3a52f, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000038e, 0x00000f6c, 0xe0fd67dd, 0x0000044a}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x68e162c2, 0xffe524cd, 0x9b3d01a2, 0x41fded80, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000002ec, 0x0000018f, 0x22cedafa, 0x00000125}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003f5, 0x00000378, 0xf729b4c9, 0x00000841}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5873c62f, 0xe73527c8, 0x2a88ca67, 0x9bb0e49c, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000037d, 0x00000c76, 0xdaf0105b, 0x0000140d}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x6d22b51e, 0x9f16e3e8, 0x9033dffd, 0xdebaea1d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000268, 0x000006bd, 0xd55ec1a5, 0x0000103b}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x2c46c27e, 0xab9c2334, 0x31cfa0b5, 0x02334e95, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000316, 0x0000092f, 0x88bafad9, 0x00000b3a}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x24ff8388, 0x9a693293, 0x8428251f, 0x071b82c2, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000342, 0x00000959, 0xe779c471, 0x0000076f}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0xdb19a044, 0x1095f2e7, 0x9f733159, 0x380d0562, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000002b1, 0x00000ae9, 0xdb3d1151, 0x000000ea}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x7cb4af22, 0xb43c7bab, 0xb183013d, 0xa4ef2271, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000023c, 0x000009b1, 0xb2109306, 0x0000157b}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}},
+    {{0x0000014e, 0x00000b2d, 0xf6f22f41, 0x00000a71}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x4a5c9177, 0x0b128e2b, 0xfafd7702, 0xbf157cc0, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000000d5, 0x00000b6b, 0x360c4979, 0x000014f9}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0xaeede5d8, 0xb8c81a2b, 0x9dbe4fd4, 0xa52f6153, 0x5a5a000e, 0x5a5a000f}},
+    {{0x00000222, 0x0000048f, 0x103ef3c3, 0x000003fb}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x6d03a026, 0xabaf0201, 0xdbf7fc1f, 0x1fde3716, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003db, 0x00000da9, 0xfd63ed5b, 0x00001470}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0xa02ac205, 0x05b43c44, 0x72660c37, 0x489ae433, 0x5a5a000e, 0x5a5a000f}},
+    {{0x000003de, 0x0000016a, 0xccf3d0b2, 0x00000b02}, {0x5a5a0000, 0x5a5a0001, 0x5a5a0002, 0x5a5a0003, 0x5a5a0004, 0x5a5a0005, 0x5a5a0006, 0x5a5a0007, 0x5a5a0008, 0x5a5a0009, 0x5a5a000a, 0x5a5a000b, 0x5a5a000c, 0x5a5a000d, 0x5a5a000e, 0x5a5a000f}}
+}};
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
+#endif
+        Require(block != nullptr, "flat loads x4: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, false);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        VirtualFree(block, 0, MEM_RELEASE);
+#else
+        std::free(block);
+#endif
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%x", value);
+    return text;
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const std::uint8_t* base) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(base));
+    std::vector<std::uint32_t> userData(10, 0u);
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    userData[8] = static_cast<std::uint32_t>(address);
+    userData[9] = static_cast<std::uint32_t>(address >> 32u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::span<const std::uint32_t> code) {
+    Output.fill(Sentinel);
+    Dispatch(device, code, guest.Data());
+    const std::string wave = "flat loads x4: lane ";
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const auto& expected = Rows[tid].results;
+        for (std::uint32_t result = 0; result < Results; ++result) {
+            const auto actual = Output[tid * Results + result];
+            Require(actual == expected[result], wave + std::to_string(tid) + " v" + std::to_string(10u + result) + " is " + Hex(actual) + ", expected " + Hex(expected[result]));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock guest;
+        std::memset(guest.Data(), 0, BlockBytes);
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            std::memcpy(guest.Data() + tid * 16u, Rows[tid].input.data(), 16u);
+        }
+        std::memcpy(guest.Data() + RegionOffset, FlatGlobalLoadsRegion.data(), FlatGlobalLoadsRegion.size());
+        Run(*device, guest, LoadsCode);
+        std::puts("flat loads x4 tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp

@@ -0,0 +1,143 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 64;
+constexpr std::uint32_t MaskedThreads = 16;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::uint32_t CheckedBytes = 0x2000;
+constexpr std::uint8_t Fill = 0xcd;
+
+alignas(256) constexpr std::array<std::uint32_t, 45> FlatStoreCode{
+    0x34020082, 0x34040084, 0x340c0081, 0x340e0083, 0x4a060200, 0x7e080201, 0x4a1806ff, 0x00000204,
+    0x7e1a0201, 0x4a1c04ff, 0x00001000, 0x4a1000ff, 0xa1b2c300, 0x4a1202ff, 0x51000000, 0x4a1404ff,
+    0x00c0ffee, 0x4a160eff, 0x7e570000, 0xdc708000, 0x00000801, 0xdc700100, 0x007d0903, 0xdc708ffc,
+    0x007d0a0c, 0xdc608400, 0x00000800, 0xdc688480, 0x00000906, 0xdc708601, 0x00000b07, 0xdc788000,
+    0x0000080e, 0xdc748400, 0x0000080e, 0xdc7c8800, 0x0000090e, 0xdc308000, 0x0f000001, 0xdc708500,
+    0x00000f01, 0x7da80090, 0xdc708300, 0x00000a01, 0xbf810000,
+};
+
+class GuestBlock {
+public:
+    explicit GuestBlock(bool writable) {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
+#endif
+        Require(block != nullptr, "flat store: cannot allocate the guest block");
+        Clear();
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, writable);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    void Clear() { std::memset(block, Fill, BlockBytes); }
+    const std::uint8_t* Data() const { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+std::vector<std::uint8_t> Expected() {
+    std::vector<std::uint8_t> image(CheckedBytes, Fill);
+    const auto put = [&](std::uint32_t offset, std::uint32_t value, std::uint32_t bytes) {
+        for (std::uint32_t byte = 0; byte < bytes; ++byte) image.at(offset + byte) = static_cast<std::uint8_t>(value >> (byte * 8u));
+    };
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::array<std::uint32_t, 4> data{0xa1b2c300u + tid, 0x51000000u + tid * 4u, 0x00c0ffeeu + tid * 16u, 0x7e570000u + tid * 8u};
+        put(0x000u + tid * 4u, data[0], 4);
+        put(0x100u + tid * 4u, data[1], 4);
+        put(0x200u + tid * 4u, data[2], 4);
+        if (tid < MaskedThreads) put(0x300u + tid * 4u, data[2], 4);
+        put(0x400u + tid, data[0], 1);
+        put(0x480u + tid * 2u, data[1], 2);
+        put(0x500u + tid * 4u, data[0], 4);
+        put(0x601u + tid * 8u, data[3], 4);
+        for (std::uint32_t dword = 0; dword < 3; ++dword) put(0x800u + tid * 16u + dword * 4u, data[dword + 1u], 4);
+        for (std::uint32_t dword = 0; dword < 4; ++dword) put(0x1000u + tid * 16u + dword * 4u, data[dword], 4);
+        for (std::uint32_t dword = 0; dword < 2; ++dword) put(0x1400u + tid * 16u + dword * 4u, data[dword], 4);
+    }
+    return image;
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, const std::uint8_t* base) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(base));
+    std::vector<std::uint32_t> userData(8, 0u);
+    userData[0] = static_cast<std::uint32_t>(address);
+    userData[1] = static_cast<std::uint32_t>(address >> 32u);
+    const std::span<const std::uint32_t> code(FlatStoreCode);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void RunStores(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t waveSize) {
+    guest.Clear();
+    Dispatch(device, waveSize, guest.Data());
+    const auto expected = Expected();
+    for (std::uint32_t offset = 0; offset < CheckedBytes; ++offset) {
+        const auto actual = guest.Data()[offset];
+        Require(actual == expected[offset], "flat store: wave" + std::to_string(waveSize) + " byte " + std::to_string(offset) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[offset]));
+    }
+}
+
+void RunReadOnly(AgcDriver::VulkanDevice& device, const GuestBlock& guest) {
+    Dispatch(device, 32, guest.Data());
+    for (std::uint32_t offset = 0; offset < CheckedBytes; ++offset) {
+        Require(guest.Data()[offset] == Fill, "flat store: a store into a read-only range changed byte " + std::to_string(offset));
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock writable(true);
+        const GuestBlock readOnly(false);
+        RunStores(*device, writable, 32);
+        RunStores(*device, writable, 64);
+        RunReadOnly(*device, readOnly);
+        std::puts("flat store tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/Float16Dot.cpp

@@ -0,0 +1,162 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 64;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 8;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 26> DotCode{
+    0x34020084, 0x34060085, 0xe0301000, 0x80000401, 0xe0301004, 0x80000501, 0xe0301008, 0x80000601,
+    0xbf8c3f70, 0xcc13400a, 0x1c1a0b04, 0xcc13480b, 0x141a0b04, 0xcc13420c, 0xbc1a0b04, 0xcc13c00d,
+    0x1c1a0b04, 0xe0701000, 0x80010a03, 0xe0701004, 0x80010b03, 0xe0701008, 0x80010c03, 0xe070100c,
+    0x80010d03, 0xbf810000,
+};
+
+constexpr std::array<std::array<std::uint32_t, 3>, 8> Edges{{
+    {0x3c003c00u, 0x3c003c00u, 0x00000000u}, {0x7c000000u, 0x3c000000u, 0x3f800000u}, {0x7c003c00u, 0x00003c00u, 0x3f800000u},
+    {0xfc007c00u, 0x3c003c00u, 0x40000000u}, {0x80000000u, 0x00008000u, 0x80000000u}, {0x7bff7bffu, 0x7bff7bffu, 0xcf800000u},
+    {0x38003800u, 0xb800b800u, 0x3e800000u}, {0x7e003c00u, 0x3c003c00u, 0x00000000u},
+}};
+
+float Half(std::uint32_t bits) {
+    const std::uint32_t sign = (bits & 0x8000u) << 16u;
+    const std::uint32_t exponent = (bits >> 10u) & 0x1fu;
+    const std::uint32_t mantissa = bits & 0x3ffu;
+    if (exponent == 0x1fu) return std::bit_cast<float>(sign | 0x7f800000u | (mantissa << 13u));
+    if (exponent == 0u) return std::bit_cast<float>(sign) + (sign != 0u ? -1.0f : 1.0f) * std::ldexp(static_cast<float>(mantissa), -24);
+    return std::bit_cast<float>(sign | ((exponent + 112u) << 23u) | (mantissa << 13u));
+}
+
+std::uint32_t RandomHalf(std::uint64_t& state) {
+    state = state * 6364136223846793005ull + 1442695040888963407ull;
+    const auto bits = static_cast<std::uint32_t>(state >> 33u);
+    return (bits & 0x8000u) | ((8u + (bits >> 16u) % 15u) << 10u) | (bits & 0x3ffu);
+}
+
+void FillInput() {
+    std::uint64_t state = 0x9e3779b97f4a7c15ull;
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        auto* words = &Input[tid * Inputs];
+        if (tid < Edges.size()) {
+            std::copy(Edges[tid].begin(), Edges[tid].end(), words);
+            continue;
+        }
+        words[0] = RandomHalf(state) | (RandomHalf(state) << 16u);
+        words[1] = RandomHalf(state) | (RandomHalf(state) << 16u);
+        words[2] = std::bit_cast<std::uint32_t>(Half(RandomHalf(state)) * Half(RandomHalf(state)));
+    }
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+auto Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code) {
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    return ShaderRecompiler::Recompile(request);
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    Output.fill(0xdeadbeefu);
+    const std::span<const std::uint32_t> code(DotCode);
+    const auto result = Compile(device, code);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void CheckRefused(AgcDriver::VulkanDevice& device, std::uint32_t word0, const std::string& what) {
+    alignas(256) const std::array<std::uint32_t, 3> code{word0, 0x1c1a0b04u, 0xbf810000u};
+    std::string refusal;
+    try {
+        static_cast<void>(Compile(device, code));
+    } catch (const std::exception& error) {
+        refusal = error.what();
+    }
+    Require(refusal.find("v_dot2_f32_f16 accumulator op_sel, op_sel_hi and neg_hi are not implemented") != std::string::npos, what + " was not refused");
+}
+
+float Dot(float aLow, float aHigh, float bLow, float bHigh, float c) {
+    return std::fma(aHigh, bHigh, std::fma(aLow, bLow, c));
+}
+
+void Expect(std::uint32_t tid, std::uint32_t actual, float expected, const char* name) {
+    const std::uint32_t bits = std::bit_cast<std::uint32_t>(expected);
+    const bool nan = std::isnan(expected) && std::isnan(std::bit_cast<float>(actual));
+    Require(nan || actual == bits, std::string("v_dot2_f32_f16: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(bits));
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const auto* in = &Input[tid * Inputs];
+        const auto* out = &Output[tid * Results];
+        const float aLow = Half(in[0] & 0xffffu);
+        const float aHigh = Half(in[0] >> 16u);
+        const float bLow = Half(in[1] & 0xffffu);
+        const float bHigh = Half(in[1] >> 16u);
+        const float c = std::bit_cast<float>(in[2]);
+        const float plain = Dot(aLow, aHigh, bLow, bHigh, c);
+        Expect(tid, out[0], plain, "plain");
+        Expect(tid, out[1], Dot(aHigh, aLow, bLow, bHigh, c), "op_sel swapping the halves of src0");
+        Expect(tid, out[2], Dot(-aLow, aHigh, bLow, -bHigh, -c), "neg_lo src0, neg_hi src1 and neg src2");
+        if (!std::isnan(plain)) Expect(tid, out[3], plain > 0.0f ? std::min(plain, 1.0f) : 0.0f, "clamp");
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        FillInput();
+        Run(*device);
+        Check();
+        CheckRefused(*device, 0xcc13600au, "op_sel on the accumulator");
+        CheckRefused(*device, 0xcc13440au, "neg_hi on the accumulator");
+        CheckRefused(*device, 0xcc13000au, "op_sel_hi cleared on the accumulator");
+        std::puts("v_dot2_f32_f16 tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/Float16ResultModifiers.cpp

@@ -0,0 +1,202 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 92> Code{
+    0x34020084, 0x34060086, 0xe0301000, 0x80000401, 0xe0301004, 0x80000501, 0xe0301008, 0x80000601,
+    0xe030100c, 0x80000701, 0x7e140280, 0x7e160280, 0x7e180280, 0x7e1a0280, 0x7e1c0280, 0x7e1e0280,
+    0x7e200280, 0x7e220280, 0x7e240280, 0x7e260280, 0x7e280280, 0x7e2a0280, 0x7e2c0280, 0x7e2e0280,
+    0x7e300280, 0x7e320280, 0xbf8c3f70, 0xd5d4800a, 0x00000104, 0xd5d4000b, 0x08000104, 0xd5d5800c,
+    0x08000104, 0xd5e0000d, 0x08000104, 0xd5e0800e, 0x00000105, 0xd58a800f, 0x00000107, 0xd58a0010,
+    0x18000107, 0xd5d08011, 0x00000106, 0xd74b8012, 0x041a0b04, 0xd74b0013, 0x0c1a0b04, 0x64280af9,
+    0x04042404, 0x6a2a0af9, 0x04044404, 0xd52f8016, 0x00020d07, 0xd52f0017, 0x18020d07, 0xd5db8018,
+    0x00000105, 0x64320af9, 0x04043504, 0xe0701000, 0x80010a03, 0xe0701004, 0x80010b03, 0xe0701008,
+    0x80010c03, 0xe070100c, 0x80010d03, 0xe0701010, 0x80010e03, 0xe0701014, 0x80010f03, 0xe0701018,
+    0x80011003, 0xe070101c, 0x80011103, 0xe0701020, 0x80011203, 0xe0701024, 0x80011303, 0xe0701028,
+    0x80011403, 0xe070102c, 0x80011503, 0xe0701030, 0x80011603, 0xe0701034, 0x80011703, 0xe0701038,
+    0x80011803, 0xe070103c, 0x80011903, 0xbf810000,
+};
+
+constexpr std::uint32_t Rows[32][4] = {
+    {0x00003400u, 0x00003800u, 0x00003000u, 0x3e99999au},
+    {0x00003e00u, 0x0000b400u, 0x00004000u, 0xbfd9999au},
+    {0x00007b53u, 0x000070e2u, 0x00007bffu, 0x47ffdc00u},
+    {0x00000400u, 0x00003800u, 0x00000000u, 0x33000000u},
+    {0x00007e00u, 0x00003c00u, 0x00003c00u, 0x7fc00000u},
+    {0x00007c00u, 0x00003c00u, 0x0000bc00u, 0x7f800000u},
+    {0x0000c000u, 0x0000c200u, 0x00004400u, 0x80000000u},
+    {0x00000001u, 0x00003a00u, 0x00008001u, 0x477ff000u},
+    {0x00002265u, 0x000091b7u, 0x0000d8f1u, 0xcd613e30u},
+    {0x0000c386u, 0x00001027u, 0x0000414cu, 0xc2ce6f44u},
+    {0x00007311u, 0x000078e5u, 0x0000a6ceu, 0xc9e9c616u},
+    {0x000035bfu, 0x00001807u, 0x00000741u, 0xc324c985u},
+    {0x0000c464u, 0x0000b222u, 0x00007204u, 0x442e3d43u},
+    {0x0000b8b6u, 0x0000cd44u, 0x00003a90u, 0x37730edfu},
+    {0x0000f813u, 0x00006c0fu, 0x0000b9d1u, 0x38c0c8fdu},
+    {0x0000c381u, 0x00007019u, 0x0000f06du, 0x3bab6c39u},
+    {0x0000587fu, 0x00003b1au, 0x0000ad45u, 0x380208a9u},
+    {0x0000c2cdu, 0x000075a8u, 0x0000f3c6u, 0x4a2f20aau},
+    {0x0000ed2fu, 0x00000580u, 0x00006a8au, 0xb94067edu},
+    {0x0000dc25u, 0x00004be0u, 0x00001ef2u, 0xbe3edc0au},
+    {0x0000552bu, 0x0000e544u, 0x0000b8b3u, 0xb610a9f7u},
+    {0x0000efbau, 0x0000f79bu, 0x00006c0fu, 0x4da98f1du},
+    {0x000048beu, 0x0000966bu, 0x0000f934u, 0x3e2434e3u},
+    {0x0000be65u, 0x0000cc22u, 0x0000677fu, 0xb3fa7aa7u},
+    {0x0000c69du, 0x0000acabu, 0x0000bcfbu, 0xc74803e3u},
+    {0x000029e8u, 0x0000855cu, 0x0000d707u, 0xbb968a43u},
+    {0x00000792u, 0x00007825u, 0x00000b21u, 0x4efbc8d6u},
+    {0x0000b410u, 0x0000d92au, 0x0000fbb2u, 0x3a1890c7u},
+    {0x0000fb69u, 0x0000c541u, 0x00003313u, 0x3b6fe507u},
+    {0x0000678au, 0x00005804u, 0x0000f3d4u, 0x44ef7febu},
+    {0x0000a8c2u, 0x00008c49u, 0x00009be3u, 0xbab9f87fu},
+    {0x00006239u, 0x0000c89du, 0x0000db61u, 0xbd91a1b7u},
+};
+constexpr std::uint32_t Expected[32][16] = {
+    {0x00003c00u, 0x00004400u, 0x00003800u, 0x00003c00u, 0x00000000u, 0x000034cdu, 0x000034cdu, 0x00003c00u, 0x00003400u, 0x00003400u, 0x00003a00u, 0x00003000u, 0x000034ccu, 0x000034ccu, 0x00000000u, 0x3a000000u},
+    {0x00003955u, 0x00003955u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000becdu, 0x00003c00u, 0x00003c00u, 0x00003e80u, 0x00003c00u, 0x0000b600u, 0x0000beccu, 0x0000beccu, 0x00000000u, 0x3c000000u},
+    {0x00000118u, 0x00000118u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00007c00u, 0x00007bffu, 0x00007bffu, 0x00003c00u, 0x3c000000u},
+    {0x00003c00u, 0x00007400u, 0x00002000u, 0x00000e48u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000200u, 0x00000200u, 0x00003800u, 0x00000200u, 0x00000000u, 0x00000000u, 0x00000000u, 0x38000000u},
+    {0x00000000u, 0x00007e00u, 0x00000000u, 0x00007e00u, 0x00000000u, 0x00000000u, 0x00007e00u, 0x00003c00u, 0x00000000u, 0x00007e00u, 0x00000000u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00003c00u, 0x00000000u},
+    {0x00000000u, 0x00000000u, 0x00003c00u, 0x0000fe00u, 0x00000000u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00003c00u, 0x3c000000u},
+    {0x00000000u, 0x0000b800u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00003c00u, 0x00003c00u, 0x00004900u, 0x00000000u, 0x00004600u, 0x00008000u, 0x00008000u, 0x00000000u, 0x00000000u},
+    {0x00003c00u, 0x00007c00u, 0x00000c00u, 0x00000006u, 0x00000000u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00000000u, 0x00008000u, 0x00003a00u, 0x00000001u, 0x00007bffu, 0x00007bffu, 0x00000000u, 0x3a000000u},
+    {0x00003c00u, 0x00005501u, 0x00002f27u, 0x00002d04u, 0x00000000u, 0x00000000u, 0x0000fc00u, 0x00003c00u, 0x00000000u, 0x0000d8f1u, 0x0000220au, 0x00008092u, 0x0000fbffu, 0x0000fbffu, 0x00000000u, 0x220a0000u},
+    {0x00000000u, 0x0000b441u, 0x00000000u, 0x00003bfau, 0x00001a86u, 0x00000000u, 0x0000d673u, 0x00003c00u, 0x00003c00u, 0x0000414bu, 0x00000000u, 0x000097cfu, 0x0000d673u, 0x0000d673u, 0x00000000u, 0x00000000u},
+    {0x00000487u, 0x00000487u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000fc00u, 0x00003c00u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00007c00u, 0x0000fbffu, 0x0000fbffu, 0x00003c00u, 0x3c000000u},
+    {0x00003c00u, 0x00004192u, 0x000038cbu, 0x00003a31u, 0x00002253u, 0x00000000u, 0x0000d926u, 0x00003c00u, 0x000012b1u, 0x000012b1u, 0x000035c7u, 0x000011c9u, 0x0000d926u, 0x0000d926u, 0x00000000u, 0x35c70000u},
+    {0x00000000u, 0x0000b34au, 0x00000000u, 0x0000b913u, 0x00000000u, 0x00003c00u, 0x00006172u, 0x00003c00u, 0x00003c00u, 0x00007204u, 0x00000000u, 0x00003abbu, 0x00006171u, 0x00006171u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x0000becbu, 0x00000000u, 0x0000383du, 0x00000000u, 0x000000f3u, 0x000000f3u, 0x00003c00u, 0x00003c00u, 0x00004a9du, 0x00000000u, 0x00004a34u, 0x000000f3u, 0x000000f3u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x000081f7u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000606u, 0x00000606u, 0x00003c00u, 0x00000000u, 0x0000fc00u, 0x00000000u, 0x0000fc00u, 0x00000606u, 0x00000606u, 0x00003c00u, 0x00000000u},
+    {0x00000000u, 0x0000b444u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00001d5bu, 0x00001d5bu, 0x00003c00u, 0x00000000u, 0x0000f8f3u, 0x00003c00u, 0x0000f7b0u, 0x00001d5bu, 0x00001d5bu, 0x00003c00u, 0x3c000000u},
+    {0x00001f1eu, 0x00001f1eu, 0x00003c00u, 0x0000b9a8u, 0x00000000u, 0x00000208u, 0x00000208u, 0x00003c00u, 0x00003c00u, 0x000057fau, 0x00003c00u, 0x000057fbu, 0x00000208u, 0x00000208u, 0x00000000u, 0x3c000000u},
+    {0x00000000u, 0x0000b4b5u, 0x00000000u, 0x0000b8b0u, 0x00000000u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00000000u, 0x0000fc00u, 0x00003c00u, 0x0000fc00u, 0x00007bffu, 0x00007bffu, 0x00003c00u, 0x3c000000u},
+    {0x00000000u, 0x00008a2cu, 0x00000000u, 0x00000000u, 0x00001052u, 0x00000000u, 0x00008a03u, 0x00003c00u, 0x00003c00u, 0x00006a8au, 0x00000000u, 0x0000b721u, 0x00008a03u, 0x00008a03u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00009bb9u, 0x00000000u, 0x0000bc00u, 0x00000000u, 0x00000000u, 0x0000b1f7u, 0x00003c00u, 0x00000000u, 0x0000ec14u, 0x00000000u, 0x0000ec14u, 0x0000b1f6u, 0x0000b1f6u, 0x00003c00u, 0x00000000u},
+    {0x00002231u, 0x00002231u, 0x00003c00u, 0x0000bb64u, 0x00000000u, 0x00000000u, 0x00008024u, 0x00003c00u, 0x00000000u, 0x0000fc00u, 0x00000000u, 0x0000fc00u, 0x00008024u, 0x00008024u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00008824u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00003c00u, 0x00007c00u, 0x00000000u, 0x00007c00u, 0x00007bffu, 0x00007bffu, 0x00000000u, 0x00000000u},
+    {0x00002ebfu, 0x00002ebfu, 0x00003c00u, 0x00002e46u, 0x00000000u, 0x00003122u, 0x00003122u, 0x00003c00u, 0x00000000u, 0x0000f934u, 0x00003c00u, 0x0000a39cu, 0x00003121u, 0x00003121u, 0x00000000u, 0x3c000000u},
+    {0x00000000u, 0x0000b901u, 0x00000000u, 0x000038a6u, 0x0000323eu, 0x00000000u, 0x00008002u, 0x00003c00u, 0x00003c00u, 0x00006799u, 0x00000000u, 0x00004e9bu, 0x00008001u, 0x00008001u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x0000b0d7u, 0x00000000u, 0x0000393au, 0x00000000u, 0x00000000u, 0x0000fa40u, 0x00003c00u, 0x00000000u, 0x0000ba1au, 0x00000000u, 0x000037b8u, 0x0000fa40u, 0x0000fa40u, 0x00000000u, 0x00000000u},
+    {0x00003c00u, 0x00004d6bu, 0x000032e0u, 0x00003493u, 0x00000000u, 0x00000000u, 0x00009cb4u, 0x00003c00u, 0x00000000u, 0x0000d707u, 0x000029e5u, 0x0000803fu, 0x00009cb4u, 0x00009cb4u, 0x00000000u, 0x29e50000u},
+    {0x00003c00u, 0x0000703au, 0x00002181u, 0x000011f2u, 0x00000000u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00003c00u, 0x000043d8u, 0x00003c00u, 0x000043d8u, 0x00007bffu, 0x00007bffu, 0x00003c00u, 0x3c000000u},
+    {0x00000000u, 0x0000c3e0u, 0x00000000u, 0x0000bbffu, 0x00000000u, 0x000010c5u, 0x000010c5u, 0x00003c00u, 0x00000000u, 0x0000fbb1u, 0x00000000u, 0x0000513fu, 0x000010c4u, 0x000010c4u, 0x00000000u, 0x00000000u},
+    {0x00000000u, 0x00008114u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00001b7fu, 0x00001b7fu, 0x00003c00u, 0x00003c00u, 0x00007c00u, 0x00000000u, 0x00007c00u, 0x00001b7fu, 0x00001b7fu, 0x00000000u, 0x00000000u},
+    {0x0000103fu, 0x0000103fu, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x0000677cu, 0x00003c00u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00007c00u, 0x0000677bu, 0x0000677bu, 0x00003c00u, 0x3c000000u},
+    {0x00000000u, 0x0000cebau, 0x00000000u, 0x0000b368u, 0x00000000u, 0x00000000u, 0x000095d0u, 0x00003c00u, 0x00000000u, 0x00009bdeu, 0x00000000u, 0x000000a3u, 0x000095cfu, 0x000095cfu, 0x00000000u, 0x00000000u},
+    {0x00001524u, 0x00001524u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000ac8du, 0x00003c00u, 0x00000000u, 0x0000ef68u, 0x00003c00u, 0x0000ef2du, 0x0000ac8du, 0x0000ac8du, 0x00000000u, 0x3c000000u},
+};
+constexpr const char* Names[16] = {"rcp_clamp", "rcp_mul2", "sqrt_clamp_mul2", "sin_mul2", "sin_clamp", "cvt_f16_f32_clamp", "cvt_f16_f32_div2", "cvt_f16_u16_clamp", "fma_clamp", "fma_mul2", "add_sdwa_clamp", "mul_sdwa_mul2", "pkrtz_clamp", "pkrtz_div2", "floor_clamp", "add_sdwa_hi_clamp"};
+
+void Fill(std::uint32_t tid, std::uint32_t* words) {
+    std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words);
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+bool IsNan16(std::uint32_t bits) {
+    return (bits & 0x7fffu) > 0x7c00u;
+}
+
+bool SameHalf(std::uint32_t actual, std::uint32_t expected) {
+    return actual == expected || (IsNan16(actual) && IsNan16(expected));
+}
+
+double HalfValue(std::uint32_t bits) {
+    const double magnitude = (bits & 0x7c00u) == 0u ? std::ldexp(static_cast<double>(bits & 0x3ffu), -24) : std::ldexp(static_cast<double>((bits & 0x3ffu) | 0x400u), static_cast<int>((bits >> 10u) & 0x1fu) - 25);
+    return (bits & 0x8000u) != 0u ? -magnitude : magnitude;
+}
+
+bool NearHalf(std::uint32_t actual, std::uint32_t expected, bool clamped) {
+    if (clamped && (actual & 0x8000u) != 0u) return false;
+    if (!clamped && (expected & 0x7fffu) == 0u) return actual == expected;
+    if (IsNan16(actual) || IsNan16(expected) || (actual & 0x7fffu) == 0x7c00u || (expected & 0x7fffu) == 0x7c00u) return SameHalf(actual, expected);
+    return std::abs(HalfValue(actual) - HalfValue(expected)) <= 0x1p-10;
+}
+
+void ExpectNear(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name, bool clamped) {
+    Require((actual >> 16u) == (expected >> 16u) && NearHalf(actual & 0xffffu, expected & 0xffffu, clamped), std::string("float16 result modifiers: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected near " + Hex(expected));
+}
+
+void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
+    Require(SameHalf(actual & 0xffffu, expected & 0xffffu) && SameHalf(actual >> 16u, expected >> 16u), std::string("float16 result modifiers: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* in = &Input[tid * Inputs];
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t index = 0; index < 16; ++index) {
+            if (index == 3u || index == 4u) ExpectNear(tid, out[index], Expected[tid][index], Names[index], index == 4u);
+            else Expect(tid, out[index], Expected[tid][index], Names[index]);
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device);
+        Check();
+        std::puts("float16 result modifiers tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/FmaLegacy.cpp

@@ -0,0 +1,144 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 8;
+constexpr std::uint32_t Results = 8;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 30> FmaLegacyCode{
+    0x34020083, 0x34060083, 0xe0302000, 0x80000401, 0xe0302004, 0x80000501, 0xe0302008, 0x80000601,
+    0xbf8c3f70, 0xd540000a, 0x041a0b04, 0xd540000b, 0x241a0b04, 0xd540010c, 0x841a0b04, 0xd540000d,
+    0x041a0a80, 0xd540020e, 0x641a0b04, 0xe0702000, 0x80010a03, 0xe0702004, 0x80010b03, 0xe0702008,
+    0x80010c03, 0xe070200c, 0x80010d03, 0xe0702010, 0x80010e03, 0xbf810000,
+};
+
+struct Row {
+    std::uint32_t a, b, c;
+};
+
+constexpr std::array<Row, 20> Rows{{
+    {0x00000000u, 0x7f800000u, 0x3f800000u},
+    {0x80000000u, 0x7f800000u, 0xc0200000u},
+    {0x7f800000u, 0x00000000u, 0x3f400000u},
+    {0xff800000u, 0x80000000u, 0x40400000u},
+    {0x00000000u, 0x7fc00000u, 0x3fc00000u},
+    {0x7fc00000u, 0x80000000u, 0xc0800000u},
+    {0x00000000u, 0x40000000u, 0x7f800000u},
+    {0x80000000u, 0xc0400000u, 0xff800000u},
+    {0x00000000u, 0x3f800000u, 0x7fc00000u},
+    {0x3fc00000u, 0x40100000u, 0xbf400000u},
+    {0xc0c00000u, 0x3f000000u, 0x41200000u},
+    {0x7f800000u, 0x40000000u, 0x3f800000u},
+    {0x7f800000u, 0xbf800000u, 0x7f800000u},
+    {0x7fc00000u, 0x3f800000u, 0x3f800000u},
+    {0x40400000u, 0x40800000u, 0xc1400000u},
+    {0x7f7fffffu, 0x40000000u, 0x3f800000u},
+    {0x3ec00000u, 0xc1480000u, 0x42c88000u},
+    {0xc4800000u, 0xbd800000u, 0xc2800000u},
+    {0x40e00000u, 0x41100000u, 0x3f000000u},
+    {0x80000000u, 0x80000000u, 0x40a00000u},
+}};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu};
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    Input.fill(0u);
+    for (std::uint32_t tid = 0; tid < Rows.size(); ++tid) {
+        const auto& row = Rows[tid];
+        const std::array<std::uint32_t, 3> words{row.a, row.b, row.c};
+        std::copy(words.begin(), words.end(), Input.begin() + tid * Inputs);
+    }
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size()));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size()));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(FmaLegacyCode);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+bool IsNan(std::uint32_t bits) {
+    return (bits & 0x7fffffffu) > 0x7f800000u;
+}
+
+std::uint32_t FmaLegacy(float a, float b, float c) {
+    if (a == 0.0f || b == 0.0f) {
+        return std::bit_cast<std::uint32_t>(0.0f + c);
+    }
+    return std::bit_cast<std::uint32_t>(std::fma(a, b, c));
+}
+
+void Check() {
+    constexpr std::array<const char*, 5> names{
+        "v_fma_legacy_f32", "v_fma_legacy_f32 -src0", "v_fma_legacy_f32 |src0| -src2", "v_fma_legacy_f32 src0=0", "v_fma_legacy_f32 -src0 -|src1|",
+    };
+    for (std::uint32_t tid = 0; tid < Rows.size(); ++tid) {
+        const auto& row = Rows[tid];
+        const float a = std::bit_cast<float>(row.a);
+        const float b = std::bit_cast<float>(row.b);
+        const float c = std::bit_cast<float>(row.c);
+        const std::array<std::uint32_t, 5> expected{
+            FmaLegacy(a, b, c), FmaLegacy(-a, b, c), FmaLegacy(std::fabs(a), b, -c), FmaLegacy(0.0f, b, c), FmaLegacy(-a, -std::fabs(b), c),
+        };
+        for (std::uint32_t j = 0; j < expected.size(); ++j) {
+            const auto actual = Output[tid * Results + j];
+            const bool matches = actual == expected[j] || (IsNan(actual) && IsNan(expected[j]));
+            Require(matches, std::string("fma legacy: lane ") + std::to_string(tid) + " (" + Hex(row.a) + ", " + Hex(row.b) + ", " + Hex(row.c) + ") " + names[j] + " is " + Hex(actual) + ", expected " + Hex(expected[j]));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device);
+        Check();
+        std::puts("fma legacy tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicCsub.cpp

@@ -0,0 +1,249 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 64;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 4;
+constexpr std::uint32_t Written = 3;
+constexpr std::uint32_t SharedOffset = Threads * 4u;
+constexpr std::uint32_t Untouched = 0xdeadbeefu;
+constexpr std::uint32_t SharedPartial = 0x800186a0u;
+constexpr std::uint32_t SharedExhausted = 1000u;
+constexpr std::size_t BlockBytes = 4096;
+constexpr std::uint8_t Fill = 0xcd;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 25> Code{
+    0x34020084, 0x34060082, 0xe0301000, 0x80000a01, 0xe0301004, 0x80000b01, 0xe0301008, 0x80000c01,
+    0x7e0e02ff, 0x00000100, 0xbf8c3f70, 0xdcd18000, 0x14080a03, 0xdcd18000, 0x15080b07, 0xdcd18004,
+    0x16080c07, 0xbf8c3f70, 0xe0701000, 0x80011401, 0xe0701004, 0x80011501, 0xe0701008, 0x80011601,
+    0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 3> NoReturnCode{
+    0xdcd08000, 0x00080a03, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 3> FlatSegmentCode{
+    0xdcd10000, 0x147f0a03, 0xbf810000,
+};
+
+struct Row {
+    std::uint32_t memory;
+    std::uint32_t subtrahend;
+};
+
+constexpr std::array<Row, 16> Edges{{
+    {0u, 0u}, {0u, 1u}, {1u, 0u}, {1u, 1u},
+    {5u, 7u}, {7u, 5u}, {0xffffffffu, 1u}, {1u, 0xffffffffu},
+    {0x80000000u, 0x7fffffffu}, {0x7fffffffu, 0x80000000u}, {0xffffffffu, 0xffffffffu}, {0x80000000u, 0x80000000u},
+    {0x80000001u, 0x80000000u}, {0u, 0xffffffffu}, {0xffffffffu, 0u}, {0x12345678u, 0x12345679u},
+}};
+
+Row RowOf(std::uint32_t tid) {
+    if (tid < Edges.size()) return Edges[tid];
+    const std::uint32_t memory = (tid + 1u) * 0x9e3779b9u;
+    const std::uint32_t subtrahend = (tid + 1u) * 0x85ebca6bu;
+    return {memory, (tid & 3u) == 0u ? subtrahend >> 8u : subtrahend};
+}
+
+std::uint32_t SharedPartialStep(std::uint32_t tid) { return tid + 1u; }
+std::uint32_t SharedExhaustedStep(std::uint32_t tid) { return tid + 100u; }
+
+std::uint32_t SubtractClamped(std::uint32_t memory, std::uint32_t subtrahend) {
+    return memory >= subtrahend ? memory - subtrahend : 0u;
+}
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
+#endif
+        Require(block != nullptr, "global atomic csub: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, true);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+void Put(std::vector<std::uint8_t>& image, std::uint32_t offset, std::uint32_t value) {
+    for (std::uint32_t byte = 0; byte < 4u; ++byte) image.at(offset + byte) = static_cast<std::uint8_t>(value >> (byte * 8u));
+}
+
+std::vector<std::uint8_t> Initial() {
+    std::vector<std::uint8_t> image(BlockBytes, Fill);
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        Put(image, tid * 4u, RowOf(tid).memory);
+    }
+    Put(image, SharedOffset, SharedPartial);
+    Put(image, SharedOffset + 4u, SharedExhausted);
+    return image;
+}
+
+std::vector<std::uint8_t> Expected() {
+    auto image = Initial();
+    std::uint32_t partial = SharedPartial;
+    std::uint32_t exhausted = SharedExhausted;
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const Row row = RowOf(tid);
+        Put(image, tid * 4u, SubtractClamped(row.memory, row.subtrahend));
+        partial = SubtractClamped(partial, SharedPartialStep(tid));
+        exhausted = SubtractClamped(exhausted, SharedExhaustedStep(tid));
+    }
+    Require(partial != 0u && exhausted == 0u, "global atomic csub: the shared dwords do not cover both outcomes");
+    Put(image, SharedOffset, partial);
+    Put(image, SharedOffset + 4u, exhausted);
+    return image;
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+std::vector<std::uint32_t> UserData(const std::uint8_t* base) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(base));
+    std::vector<std::uint32_t> userData(10, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    userData[8] = static_cast<std::uint32_t>(address);
+    userData[9] = static_cast<std::uint32_t>(address >> 32u);
+    return userData;
+}
+
+ShaderRecompiler::RecompileResult Recompile(const AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::uint32_t waveSize, std::span<const std::uint32_t> userData) {
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    return ShaderRecompiler::Recompile(request);
+}
+
+void CheckChain(const std::string& where, std::uint32_t column, std::uint32_t initial, std::uint32_t (*step)(std::uint32_t)) {
+    std::vector<std::pair<std::uint32_t, std::uint32_t>> order;
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        order.emplace_back(Output[tid * Results + column], tid);
+    }
+    std::sort(order.begin(), order.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
+    std::uint32_t running = initial;
+    for (const auto& [returned, tid] : order) {
+        Require(returned == running, where + " returned " + Hex(returned) + " to thread " + std::to_string(tid) + ", expected " + Hex(running));
+        running = SubtractClamped(running, step(tid));
+    }
+}
+
+void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t waveSize) {
+    const auto initial = Initial();
+    std::memcpy(guest.Data(), initial.data(), BlockBytes);
+    Input.fill(0u);
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        Input[tid * Inputs] = RowOf(tid).subtrahend;
+        Input[tid * Inputs + 1u] = SharedPartialStep(tid);
+        Input[tid * Inputs + 2u] = SharedExhaustedStep(tid);
+    }
+    Output.fill(Untouched);
+    const auto userData = UserData(guest.Data());
+    const std::span<const std::uint32_t> code(Code);
+    const auto result = Recompile(device, code, waveSize, userData);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+
+    const auto wave = "global atomic csub: wave" + std::to_string(waveSize) + " ";
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const Row row = RowOf(tid);
+        const auto returned = Output[tid * Results];
+        Require(returned == row.memory, wave + "thread " + std::to_string(tid) + " returned " + Hex(returned) + ", expected " + Hex(row.memory));
+        for (std::uint32_t j = Written; j < Results; ++j) {
+            Require(Output[tid * Results + j] == Untouched, wave + "thread " + std::to_string(tid) + " result " + std::to_string(j) + " was written");
+        }
+    }
+    CheckChain(wave + "dword shared by all lanes", 1u, SharedPartial, SharedPartialStep);
+    CheckChain(wave + "dword exhausted by all lanes", 2u, SharedExhausted, SharedExhaustedStep);
+    const auto expected = Expected();
+    for (std::uint32_t offset = 0; offset < BlockBytes; ++offset) {
+        const auto actual = guest.Data()[offset];
+        Require(actual == expected[offset], wave + "byte " + std::to_string(offset) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[offset]));
+    }
+}
+
+void CheckRejected(const AgcDriver::VulkanDevice& device, GuestBlock& guest, std::span<const std::uint32_t> code, const std::string& reason) {
+    const auto userData = UserData(guest.Data());
+    std::string failure;
+    try {
+        static_cast<void>(Recompile(device, code, 32, userData));
+    } catch (const std::exception& error) {
+        failure = error.what();
+    }
+    Require(failure.find(reason) != std::string::npos, "global atomic csub: expected '" + reason + "', got '" + failure + "'");
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock guest;
+        Run(*device, guest, 32);
+        Run(*device, guest, 64);
+        CheckRejected(*device, guest, NoReturnCode, "global_atomic_csub without glc is not supported");
+        CheckRejected(*device, guest, FlatSegmentCode, "global_atomic_csub is available only in the global segment");
+        std::puts("global atomic csub tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics.cpp

@@ -0,0 +1,436 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 64;
+constexpr std::uint32_t RowsPerOp = 16;
+constexpr std::uint32_t OpCount = 13;
+constexpr std::uint32_t NarrowOps = 13;
+constexpr std::uint32_t OpStride = 0x200;
+constexpr std::uint32_t Special = OpCount * OpStride;
+constexpr std::uint32_t Sentinel = 0x05e471e1;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::uint8_t Fill = 0xcd;
+alignas(256) std::array<std::uint32_t, OpCount * Threads * 4> Input{};
+alignas(256) std::array<std::uint32_t, (OpCount + 2) * Threads * 2> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 187> GlobalAtomicsCode{
+    0x34020083, 0x34040084, 0x340a0083, 0x7e2802ff, 0x05e471e1, 0x7e2a02ff, 0x05e471e1, 0x4a060280,
+    0x4a080480, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0a80, 0xdcc18000, 0x14080a03, 0xbf8c3f70,
+    0xe0701000, 0x80011406, 0x4a0602ff, 0x00000200, 0x4a0804ff, 0x00000400, 0xe0381000, 0x80000a04,
+    0xbf8c3f70, 0x4a0c0aff, 0x00000200, 0xdcc58000, 0x14080a03, 0xbf8c3f70, 0xe0701000, 0x80011406,
+    0x4a0602ff, 0x00000400, 0x4a0804ff, 0x00000800, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff,
+    0x00000400, 0xdcc98000, 0x14080a03, 0xbf8c3f70, 0xe0701000, 0x80011406, 0x4a0602ff, 0x00000600,
+    0x4a0804ff, 0x00000c00, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00000600, 0xdccd8000,
+    0x14080a03, 0xbf8c3f70, 0xe0701000, 0x80011406, 0x4a0602ff, 0x00000800, 0x4a0804ff, 0x00001000,
+    0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00000800, 0xdcd58000, 0x14080a03, 0xbf8c3f70,
+    0xe0701000, 0x80011406, 0x4a0602ff, 0x00000a00, 0x4a0804ff, 0x00001400, 0xe0381000, 0x80000a04,
+    0xbf8c3f70, 0x4a0c0aff, 0x00000a00, 0xdcd98000, 0x14080a03, 0xbf8c3f70, 0xe0701000, 0x80011406,
+    0x4a0602ff, 0x00000c00, 0x4a0804ff, 0x00001800, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff,
+    0x00000c00, 0xdcdd8000, 0x14080a03, 0xbf8c3f70, 0xe0701000, 0x80011406, 0x4a0602ff, 0x00000e00,
+    0x4a0804ff, 0x00001c00, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00000e00, 0xdce18000,
+    0x14080a03, 0xbf8c3f70, 0xe0701000, 0x80011406, 0x4a0602ff, 0x00001000, 0x4a0804ff, 0x00002000,
+    0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00001000, 0xdce58000, 0x14080a03, 0xbf8c3f70,
+    0xe0701000, 0x80011406, 0x4a0602ff, 0x00001200, 0x4a0804ff, 0x00002400, 0xe0381000, 0x80000a04,
+    0xbf8c3f70, 0x4a0c0aff, 0x00001200, 0xdce98000, 0x14080a03, 0xbf8c3f70, 0xe0701000, 0x80011406,
+    0x4a0602ff, 0x00001400, 0x4a0804ff, 0x00002800, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff,
+    0x00001400, 0xdced8000, 0x14080a03, 0xbf8c3f70, 0xe0701000, 0x80011406, 0x4a0602ff, 0x00001600,
+    0x4a0804ff, 0x00002c00, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00001600, 0xdcf18000,
+    0x14080a03, 0xbf8c3f70, 0xe0701000, 0x80011406, 0x4a0602ff, 0x00001800, 0x4a0804ff, 0x00003000,
+    0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00001800, 0xdcf58000, 0x14080a03, 0xbf8c3f70,
+    0xe0701000, 0x80011406, 0xbf810000
+};
+
+struct Row {
+    std::uint32_t op;
+    std::uint64_t memory;
+    std::uint64_t data;
+    std::uint64_t comparator;
+    std::uint64_t returned;
+    std::uint64_t final;
+};
+
+constexpr std::array<Row, 208> Rows{{
+    {0, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {0, 0x1ull, 0x80000000ull, 0xf9677975ull, 0x1ull, 0x80000000ull},
+    {0, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0xffffffffull},
+    {0, 0x7fffffffull, 0x0ull, 0xfc5039f7ull, 0x7fffffffull, 0x0ull},
+    {0, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0x7fffffffull},
+    {0, 0x80000001ull, 0xfffffffeull, 0x1098ad65ull, 0x80000001ull, 0xfffffffeull},
+    {0, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0xdeadbeefull},
+    {0, 0xffffffffull, 0x2ull, 0x19211eull, 0xffffffffull, 0x2ull},
+    {0, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x80000001ull},
+    {0, 0xdeadbeefull, 0x12345678ull, 0xcdbc2be9ull, 0xdeadbeefull, 0x12345678ull},
+    {0, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {0, 0x1ull, 0x80000000ull, 0xa3639922ull, 0x1ull, 0x80000000ull},
+    {0, 0x46e9a894ull, 0xd107eea9ull, 0x46e9a894ull, 0x46e9a894ull, 0xd107eea9ull},
+    {0, 0xbc50bcc8ull, 0x58ffc3ecull, 0x4cd661ull, 0xbc50bcc8ull, 0x58ffc3ecull},
+    {0, 0xb9093c7eull, 0x665932a4ull, 0xb9093c7eull, 0xb9093c7eull, 0x665932a4ull},
+    {0, 0xd88449d9ull, 0xfe358db5ull, 0xba80659full, 0xd88449d9ull, 0xfe358db5ull},
+    {1, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {1, 0x1ull, 0x80000000ull, 0x109949f9ull, 0x1ull, 0x1ull},
+    {1, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0xffffffffull},
+    {1, 0x7fffffffull, 0x0ull, 0x22d348c2ull, 0x7fffffffull, 0x7fffffffull},
+    {1, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0x7fffffffull},
+    {1, 0x80000001ull, 0xfffffffeull, 0x42872e08ull, 0x80000001ull, 0x80000001ull},
+    {1, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0xdeadbeefull},
+    {1, 0xffffffffull, 0x2ull, 0x675c2152ull, 0xffffffffull, 0xffffffffull},
+    {1, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x80000001ull},
+    {1, 0xdeadbeefull, 0x12345678ull, 0x2f1ed38ull, 0xdeadbeefull, 0xdeadbeefull},
+    {1, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {1, 0x1ull, 0x80000000ull, 0xd6be8bc1ull, 0x1ull, 0x1ull},
+    {1, 0x283d5502ull, 0xfc565b67ull, 0x283d5502ull, 0x283d5502ull, 0xfc565b67ull},
+    {1, 0xe503313aull, 0x1722ca72ull, 0xdd44c959ull, 0xe503313aull, 0xe503313aull},
+    {1, 0x2a841e2ull, 0x52a450e2ull, 0x2a841e2ull, 0x2a841e2ull, 0x52a450e2ull},
+    {1, 0xcc440ac9ull, 0x46313f70ull, 0xee4a57bbull, 0xcc440ac9ull, 0xcc440ac9ull},
+    {2, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {2, 0x1ull, 0x80000000ull, 0x452f2ffcull, 0x1ull, 0x80000001ull},
+    {2, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0x1ull},
+    {2, 0x7fffffffull, 0x0ull, 0xf5091963ull, 0x7fffffffull, 0x7fffffffull},
+    {2, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0xffffffffull},
+    {2, 0x80000001ull, 0xfffffffeull, 0xeb165c4ull, 0x80000001ull, 0x7fffffffull},
+    {2, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0xdeadbeedull},
+    {2, 0xffffffffull, 0x2ull, 0x1a438e52ull, 0xffffffffull, 0x1ull},
+    {2, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x92345679ull},
+    {2, 0xdeadbeefull, 0x12345678ull, 0x64638c03ull, 0xdeadbeefull, 0xf0e21567ull},
+    {2, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {2, 0x1ull, 0x80000000ull, 0xe6d2996dull, 0x1ull, 0x80000001ull},
+    {2, 0xa9bdd75aull, 0x70734273ull, 0xa9bdd75aull, 0xa9bdd75aull, 0x1a3119cdull},
+    {2, 0x8aa692d0ull, 0x74d3a033ull, 0x5a7fd942ull, 0x8aa692d0ull, 0xff7a3303ull},
+    {2, 0x201fe536ull, 0xf8ddb196ull, 0x201fe536ull, 0x201fe536ull, 0x18fd96ccull},
+    {2, 0x15f79627ull, 0xb2346d27ull, 0x8a6b642dull, 0x15f79627ull, 0xc82c034eull},
+    {3, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0xffffffffull},
+    {3, 0x1ull, 0x80000000ull, 0x69639719ull, 0x1ull, 0x80000001ull},
+    {3, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0x3ull},
+    {3, 0x7fffffffull, 0x0ull, 0x89e34c92ull, 0x7fffffffull, 0x7fffffffull},
+    {3, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0x1ull},
+    {3, 0x80000001ull, 0xfffffffeull, 0x73229920ull, 0x80000001ull, 0x80000003ull},
+    {3, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0x2152410full},
+    {3, 0xffffffffull, 0x2ull, 0x66310986ull, 0xffffffffull, 0xfffffffdull},
+    {3, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x92345677ull},
+    {3, 0xdeadbeefull, 0x12345678ull, 0x8fa4a357ull, 0xdeadbeefull, 0xcc796877ull},
+    {3, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0xffffffffull},
+    {3, 0x1ull, 0x80000000ull, 0x64bdad88ull, 0x1ull, 0x80000001ull},
+    {3, 0xd5b148e7ull, 0x91d4c68ull, 0xd5b148e7ull, 0xd5b148e7ull, 0xcc93fc7full},
+    {3, 0xb6cd96e4ull, 0xa145e36full, 0x6efd9be0ull, 0xb6cd96e4ull, 0x1587b375ull},
+    {3, 0xf123997cull, 0x12d3d18aull, 0xf123997cull, 0xf123997cull, 0xde4fc7f2ull},
+    {3, 0x62bfff89ull, 0x2106fea7ull, 0x22db0452ull, 0x62bfff89ull, 0x41b900e2ull},
+    {4, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x0ull},
+    {4, 0x1ull, 0x80000000ull, 0xdbbb69abull, 0x1ull, 0x80000000ull},
+    {4, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0xffffffffull},
+    {4, 0x7fffffffull, 0x0ull, 0xdffce4e3ull, 0x7fffffffull, 0x0ull},
+    {4, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0x80000000ull},
+    {4, 0x80000001ull, 0xfffffffeull, 0x77ac152aull, 0x80000001ull, 0x80000001ull},
+    {4, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0xdeadbeefull},
+    {4, 0xffffffffull, 0x2ull, 0x24880f60ull, 0xffffffffull, 0xffffffffull},
+    {4, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x80000001ull},
+    {4, 0xdeadbeefull, 0x12345678ull, 0x6adc5e6full, 0xdeadbeefull, 0xdeadbeefull},
+    {4, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x0ull},
+    {4, 0x1ull, 0x80000000ull, 0x5c5e447eull, 0x1ull, 0x80000000ull},
+    {4, 0xbc193f27ull, 0x5b2c838cull, 0xbc193f27ull, 0xbc193f27ull, 0xbc193f27ull},
+    {4, 0xa69c5ce5ull, 0xbf61575eull, 0x341a640bull, 0xa69c5ce5ull, 0xa69c5ce5ull},
+    {4, 0x1836851cull, 0x12fef585ull, 0x1836851cull, 0x1836851cull, 0x12fef585ull},
+    {4, 0xaa5c538dull, 0x618e48b9ull, 0x56002b6cull, 0xaa5c538dull, 0xaa5c538dull},
+    {5, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x0ull},
+    {5, 0x1ull, 0x80000000ull, 0x21d3075eull, 0x1ull, 0x1ull},
+    {5, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0x2ull},
+    {5, 0x7fffffffull, 0x0ull, 0x16b60422ull, 0x7fffffffull, 0x0ull},
+    {5, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0x7fffffffull},
+    {5, 0x80000001ull, 0xfffffffeull, 0xfd31cfd7ull, 0x80000001ull, 0x80000001ull},
+    {5, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0xdeadbeefull},
+    {5, 0xffffffffull, 0x2ull, 0x6c5db104ull, 0xffffffffull, 0x2ull},
+    {5, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x12345678ull},
+    {5, 0xdeadbeefull, 0x12345678ull, 0xca2b6cbeull, 0xdeadbeefull, 0x12345678ull},
+    {5, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x0ull},
+    {5, 0x1ull, 0x80000000ull, 0xdc32d1aaull, 0x1ull, 0x1ull},
+    {5, 0xfd11be0eull, 0x30eaba72ull, 0xfd11be0eull, 0xfd11be0eull, 0x30eaba72ull},
+    {5, 0x4f696046ull, 0xed71a68aull, 0x59f1d18dull, 0x4f696046ull, 0x4f696046ull},
+    {5, 0x2e192541ull, 0x1f6c7f26ull, 0x2e192541ull, 0x2e192541ull, 0x1f6c7f26ull},
+    {5, 0x7a1ff8aeull, 0xdae2a240ull, 0xd604dd68ull, 0x7a1ff8aeull, 0x7a1ff8aeull},
+    {6, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {6, 0x1ull, 0x80000000ull, 0xadf8324cull, 0x1ull, 0x1ull},
+    {6, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0x2ull},
+    {6, 0x7fffffffull, 0x0ull, 0x9f75da08ull, 0x7fffffffull, 0x7fffffffull},
+    {6, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0x7fffffffull},
+    {6, 0x80000001ull, 0xfffffffeull, 0xcd48b9bdull, 0x80000001ull, 0xfffffffeull},
+    {6, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0xfffffffeull},
+    {6, 0xffffffffull, 0x2ull, 0x40591809ull, 0xffffffffull, 0x2ull},
+    {6, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x12345678ull},
+    {6, 0xdeadbeefull, 0x12345678ull, 0x951a2694ull, 0xdeadbeefull, 0x12345678ull},
+    {6, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {6, 0x1ull, 0x80000000ull, 0x55ff16b5ull, 0x1ull, 0x1ull},
+    {6, 0xadb9fc15ull, 0x9263cee2ull, 0xadb9fc15ull, 0xadb9fc15ull, 0xadb9fc15ull},
+    {6, 0xa055a0b2ull, 0xe06373c3ull, 0x406ffc43ull, 0xa055a0b2ull, 0xe06373c3ull},
+    {6, 0xe442a859ull, 0xbcdeb639ull, 0xe442a859ull, 0xe442a859ull, 0xe442a859ull},
+    {6, 0x238c6bf4ull, 0xfe8fedc6ull, 0x771aaf91ull, 0x238c6bf4ull, 0x238c6bf4ull},
+    {7, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {7, 0x1ull, 0x80000000ull, 0x237244c9ull, 0x1ull, 0x80000000ull},
+    {7, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0xffffffffull},
+    {7, 0x7fffffffull, 0x0ull, 0xf62e5e4full, 0x7fffffffull, 0x7fffffffull},
+    {7, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0x80000000ull},
+    {7, 0x80000001ull, 0xfffffffeull, 0x90bfe553ull, 0x80000001ull, 0xfffffffeull},
+    {7, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0xfffffffeull},
+    {7, 0xffffffffull, 0x2ull, 0x44b4cc99ull, 0xffffffffull, 0xffffffffull},
+    {7, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x80000001ull},
+    {7, 0xdeadbeefull, 0x12345678ull, 0x99faa521ull, 0xdeadbeefull, 0xdeadbeefull},
+    {7, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {7, 0x1ull, 0x80000000ull, 0x5136e276ull, 0x1ull, 0x80000000ull},
+    {7, 0x64a7abb4ull, 0x146d75bcull, 0x64a7abb4ull, 0x64a7abb4ull, 0x64a7abb4ull},
+    {7, 0x2ae5498eull, 0x63a37ae1ull, 0xde40e42bull, 0x2ae5498eull, 0x63a37ae1ull},
+    {7, 0x99206560ull, 0x2a0494fdull, 0x99206560ull, 0x99206560ull, 0x99206560ull},
+    {7, 0x634e77acull, 0xbfe9ef2bull, 0x70824c54ull, 0x634e77acull, 0xbfe9ef2bull},
+    {8, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x0ull},
+    {8, 0x1ull, 0x80000000ull, 0x5b03f923ull, 0x1ull, 0x0ull},
+    {8, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0x2ull},
+    {8, 0x7fffffffull, 0x0ull, 0x32c25127ull, 0x7fffffffull, 0x0ull},
+    {8, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0x0ull},
+    {8, 0x80000001ull, 0xfffffffeull, 0x1197fd88ull, 0x80000001ull, 0x80000000ull},
+    {8, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0xdeadbeeeull},
+    {8, 0xffffffffull, 0x2ull, 0x7aa29c55ull, 0xffffffffull, 0x2ull},
+    {8, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x0ull},
+    {8, 0xdeadbeefull, 0x12345678ull, 0x4dc23c90ull, 0xdeadbeefull, 0x12241668ull},
+    {8, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x0ull},
+    {8, 0x1ull, 0x80000000ull, 0xfa62c0f8ull, 0x1ull, 0x0ull},
+    {8, 0xfd7537d1ull, 0x72643b5aull, 0xfd7537d1ull, 0xfd7537d1ull, 0x70643350ull},
+    {8, 0xd8dd1ff5ull, 0x8007f94cull, 0x1c7c7b29ull, 0xd8dd1ff5ull, 0x80051944ull},
+    {8, 0xadaa9f69ull, 0x542db28dull, 0xadaa9f69ull, 0xadaa9f69ull, 0x4289209ull},
+    {8, 0xeba8ad6dull, 0xaf3f49e4ull, 0xbf7eef4eull, 0xeba8ad6dull, 0xab280964ull},
+    {9, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {9, 0x1ull, 0x80000000ull, 0xd2dc3da8ull, 0x1ull, 0x80000001ull},
+    {9, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0xffffffffull},
+    {9, 0x7fffffffull, 0x0ull, 0xcbcf2e9cull, 0x7fffffffull, 0x7fffffffull},
+    {9, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0xffffffffull},
+    {9, 0x80000001ull, 0xfffffffeull, 0xf1616b9dull, 0x80000001ull, 0xffffffffull},
+    {9, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0xffffffffull},
+    {9, 0xffffffffull, 0x2ull, 0xaf3ff23eull, 0xffffffffull, 0xffffffffull},
+    {9, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x92345679ull},
+    {9, 0xdeadbeefull, 0x12345678ull, 0xd2b04560ull, 0xdeadbeefull, 0xdebdfeffull},
+    {9, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {9, 0x1ull, 0x80000000ull, 0xef07f743ull, 0x1ull, 0x80000001ull},
+    {9, 0x24c74e5ull, 0x5084dec1ull, 0x24c74e5ull, 0x24c74e5ull, 0x52ccfee5ull},
+    {9, 0x8c6705a2ull, 0x1e525d48ull, 0xab05ab28ull, 0x8c6705a2ull, 0x9e775deaull},
+    {9, 0xa36a05e5ull, 0x3a793efcull, 0xa36a05e5ull, 0xa36a05e5ull, 0xbb7b3ffdull},
+    {9, 0xd5f31646ull, 0xa296c3e7ull, 0x213632ddull, 0xd5f31646ull, 0xf7f7d7e7ull},
+    {10, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {10, 0x1ull, 0x80000000ull, 0x11e1dbaaull, 0x1ull, 0x80000001ull},
+    {10, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0xfffffffdull},
+    {10, 0x7fffffffull, 0x0ull, 0x2cd60e1aull, 0x7fffffffull, 0x7fffffffull},
+    {10, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0xffffffffull},
+    {10, 0x80000001ull, 0xfffffffeull, 0x1f025ad0ull, 0x80000001ull, 0x7fffffffull},
+    {10, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0x21524111ull},
+    {10, 0xffffffffull, 0x2ull, 0x5d9eb5b7ull, 0xffffffffull, 0xfffffffdull},
+    {10, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x92345679ull},
+    {10, 0xdeadbeefull, 0x12345678ull, 0xf167a808ull, 0xdeadbeefull, 0xcc99e897ull},
+    {10, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {10, 0x1ull, 0x80000000ull, 0x9f4e6575ull, 0x1ull, 0x80000001ull},
+    {10, 0x8bd3aa1cull, 0x44b01dcdull, 0x8bd3aa1cull, 0x8bd3aa1cull, 0xcf63b7d1ull},
+    {10, 0xc2c143f7ull, 0x51bc9443ull, 0xd65e2124ull, 0xc2c143f7ull, 0x937dd7b4ull},
+    {10, 0xb0cb286eull, 0xec5194a4ull, 0xb0cb286eull, 0xb0cb286eull, 0x5c9abccaull},
+    {10, 0xe17a51ecull, 0xe0bfc82aull, 0x44dde368ull, 0xe17a51ecull, 0x1c599c6ull},
+    {11, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {11, 0x1ull, 0x80000000ull, 0x20d071afull, 0x1ull, 0x2ull},
+    {11, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0x3ull},
+    {11, 0x7fffffffull, 0x0ull, 0x7a347eebull, 0x7fffffffull, 0x0ull},
+    {11, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0x0ull},
+    {11, 0x80000001ull, 0xfffffffeull, 0x56e21801ull, 0x80000001ull, 0x80000002ull},
+    {11, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0x0ull},
+    {11, 0xffffffffull, 0x2ull, 0x7a390ac1ull, 0xffffffffull, 0x0ull},
+    {11, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x12345679ull},
+    {11, 0xdeadbeefull, 0x12345678ull, 0xe457cd1full, 0xdeadbeefull, 0x0ull},
+    {11, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {11, 0x1ull, 0x80000000ull, 0x460a51e2ull, 0x1ull, 0x2ull},
+    {11, 0xe3202a2aull, 0xe3202a2bull, 0xe3202a2aull, 0xe3202a2aull, 0xe3202a2bull},
+    {11, 0x9ec2949ull, 0x9ec2949ull, 0x947613e8ull, 0x9ec2949ull, 0x0ull},
+    {11, 0x8f8118afull, 0x8f8118b0ull, 0x8f8118afull, 0x8f8118afull, 0x8f8118b0ull},
+    {11, 0x761757b6ull, 0x761757b6ull, 0x8be15c75ull, 0x761757b6ull, 0x0ull},
+    {12, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {12, 0x1ull, 0x80000000ull, 0x7c5a8c15ull, 0x1ull, 0x0ull},
+    {12, 0x2ull, 0xffffffffull, 0x2ull, 0x2ull, 0x1ull},
+    {12, 0x7fffffffull, 0x0ull, 0x29769614ull, 0x7fffffffull, 0x0ull},
+    {12, 0x80000000ull, 0x7fffffffull, 0x80000000ull, 0x80000000ull, 0x7fffffffull},
+    {12, 0x80000001ull, 0xfffffffeull, 0x537a54d3ull, 0x80000001ull, 0x80000000ull},
+    {12, 0xfffffffeull, 0xdeadbeefull, 0xfffffffeull, 0xfffffffeull, 0xdeadbeefull},
+    {12, 0xffffffffull, 0x2ull, 0x6add45d9ull, 0xffffffffull, 0x2ull},
+    {12, 0x12345678ull, 0x80000001ull, 0x12345678ull, 0x12345678ull, 0x12345677ull},
+    {12, 0xdeadbeefull, 0x12345678ull, 0x978c0812ull, 0xdeadbeefull, 0x12345678ull},
+    {12, 0x0ull, 0x1ull, 0x0ull, 0x0ull, 0x1ull},
+    {12, 0x1ull, 0x80000000ull, 0xf67e5e26ull, 0x1ull, 0x0ull},
+    {12, 0xebea6cf1ull, 0xebea6cf2ull, 0xebea6cf1ull, 0xebea6cf1ull, 0xebea6cf0ull},
+    {12, 0xf9a9360ull, 0xf9a9360ull, 0xf85b81cdull, 0xf9a9360ull, 0xf9a935full},
+    {12, 0x6e723d5cull, 0x6e723d5dull, 0x6e723d5cull, 0x6e723d5cull, 0x6e723d5bull},
+    {12, 0x1f609b4full, 0x1f609b4full, 0x69d177f8ull, 0x1f609b4full, 0x1f609b4eull},
+}};
+
+class GuestBlock {
+public:
+    explicit GuestBlock(bool writable) {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
+#endif
+        Require(block != nullptr, "global atomics: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, writable);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+    const std::uint8_t* Data() const { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+void Put(std::vector<std::uint8_t>& image, std::uint32_t offset, std::uint64_t value, std::uint32_t bytes) {
+    for (std::uint32_t byte = 0; byte < bytes; ++byte) image.at(offset + byte) = static_cast<std::uint8_t>(value >> (byte * 8u));
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint64_t value) {
+    char text[24];
+    std::snprintf(text, sizeof(text), "0x%llx", static_cast<unsigned long long>(value));
+    return text;
+}
+
+const Row& RowOf(std::uint32_t op, std::uint32_t tid) {
+    return Rows.at(op * RowsPerOp + tid % RowsPerOp);
+}
+
+std::uint32_t OpBytes(std::uint32_t op) {
+    return op < NarrowOps ? 4u : 8u;
+}
+
+std::vector<std::uint8_t> Initial() {
+    std::vector<std::uint8_t> image(BlockBytes, Fill);
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            Put(image, op * OpStride + tid * 8u, RowOf(op, tid).memory, OpBytes(op));
+        }
+    }
+    return image;
+}
+
+std::vector<std::uint8_t> Expected() {
+    auto image = Initial();
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            Put(image, op * OpStride + tid * 8u, RowOf(op, tid).final, OpBytes(op));
+        }
+    }
+    return image;
+}
+
+void FillInput() {
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            const auto& row = RowOf(op, tid);
+            auto* words = &Input[(op * Threads + tid) * 4u];
+            if (OpBytes(op) == 4u) {
+                words[0] = static_cast<std::uint32_t>(row.data);
+                words[1] = static_cast<std::uint32_t>(row.comparator);
+                words[2] = 0u;
+                words[3] = 0u;
+            } else {
+                words[0] = static_cast<std::uint32_t>(row.data);
+                words[1] = static_cast<std::uint32_t>(row.data >> 32u);
+                words[2] = static_cast<std::uint32_t>(row.comparator);
+                words[3] = static_cast<std::uint32_t>(row.comparator >> 32u);
+            }
+        }
+    }
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, const std::uint8_t* base) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(base));
+    std::vector<std::uint32_t> userData(10, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    userData[8] = static_cast<std::uint32_t>(address);
+    userData[9] = static_cast<std::uint32_t>(address >> 32u);
+    const std::span<const std::uint32_t> code(GlobalAtomicsCode);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void RunAtomics(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t waveSize) {
+    const auto initial = Initial();
+    std::memcpy(guest.Data(), initial.data(), BlockBytes);
+    FillInput();
+    Output.fill(Sentinel);
+    Dispatch(device, waveSize, guest.Data());
+    const auto wave = "global atomics: wave" + std::to_string(waveSize) + " ";
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            const auto& row = RowOf(op, tid);
+            const auto* words = &Output[(op * Threads + tid) * 2u];
+            const auto returned = OpBytes(op) == 4u ? static_cast<std::uint64_t>(words[0]) : (static_cast<std::uint64_t>(words[1]) << 32u) | words[0];
+            Require(returned == row.returned, wave + "op " + std::to_string(op) + " lane " + std::to_string(tid) + " returned " + Hex(returned) + ", expected " + Hex(row.returned));
+        }
+    }
+    const auto expected = Expected();
+    for (std::uint32_t offset = 0; offset < BlockBytes; ++offset) {
+        const auto actual = guest.Data()[offset];
+        Require(actual == expected[offset], wave + "byte " + std::to_string(offset) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[offset]));
+    }
+}
+
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock writable(true);
+        RunAtomics(*device, writable, 32);
+        std::puts("global atomics tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics64.cpp

@@ -0,0 +1,400 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 64;
+constexpr std::uint32_t RowsPerOp = 16;
+constexpr std::uint32_t OpCount = 11;
+constexpr std::uint32_t NarrowOps = 0;
+constexpr std::uint32_t OpStride = 0x200;
+constexpr std::uint32_t Special = OpCount * OpStride;
+constexpr std::uint32_t Sentinel = 0x05e471e1;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::uint8_t Fill = 0xcd;
+alignas(256) std::array<std::uint32_t, OpCount * Threads * 4> Input{};
+alignas(256) std::array<std::uint32_t, (OpCount + 2) * Threads * 2> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 159> GlobalAtomicsCode{
+    0x34020083, 0x34040084, 0x340a0083, 0x7e2802ff, 0x05e471e1, 0x7e2a02ff, 0x05e471e1, 0x4a060280,
+    0x4a080480, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0a80, 0xdd418000, 0x14080a03, 0xbf8c3f70,
+    0xe0741000, 0x80011406, 0x4a0602ff, 0x00000200, 0x4a0804ff, 0x00000400, 0xe0381000, 0x80000a04,
+    0xbf8c3f70, 0x4a0c0aff, 0x00000200, 0xdd458000, 0x14080a03, 0xbf8c3f70, 0xe0741000, 0x80011406,
+    0x4a0602ff, 0x00000400, 0x4a0804ff, 0x00000800, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff,
+    0x00000400, 0xdd498000, 0x14080a03, 0xbf8c3f70, 0xe0741000, 0x80011406, 0x4a0602ff, 0x00000600,
+    0x4a0804ff, 0x00000c00, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00000600, 0xdd4d8000,
+    0x14080a03, 0xbf8c3f70, 0xe0741000, 0x80011406, 0x4a0602ff, 0x00000800, 0x4a0804ff, 0x00001000,
+    0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00000800, 0xdd558000, 0x14080a03, 0xbf8c3f70,
+    0xe0741000, 0x80011406, 0x4a0602ff, 0x00000a00, 0x4a0804ff, 0x00001400, 0xe0381000, 0x80000a04,
+    0xbf8c3f70, 0x4a0c0aff, 0x00000a00, 0xdd598000, 0x14080a03, 0xbf8c3f70, 0xe0741000, 0x80011406,
+    0x4a0602ff, 0x00000c00, 0x4a0804ff, 0x00001800, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff,
+    0x00000c00, 0xdd5d8000, 0x14080a03, 0xbf8c3f70, 0xe0741000, 0x80011406, 0x4a0602ff, 0x00000e00,
+    0x4a0804ff, 0x00001c00, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00000e00, 0xdd618000,
+    0x14080a03, 0xbf8c3f70, 0xe0741000, 0x80011406, 0x4a0602ff, 0x00001000, 0x4a0804ff, 0x00002000,
+    0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00001000, 0xdd658000, 0x14080a03, 0xbf8c3f70,
+    0xe0741000, 0x80011406, 0x4a0602ff, 0x00001200, 0x4a0804ff, 0x00002400, 0xe0381000, 0x80000a04,
+    0xbf8c3f70, 0x4a0c0aff, 0x00001200, 0xdd698000, 0x14080a03, 0xbf8c3f70, 0xe0741000, 0x80011406,
+    0x4a0602ff, 0x00001400, 0x4a0804ff, 0x00002800, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff,
+    0x00001400, 0xdd6d8000, 0x14080a03, 0xbf8c3f70, 0xe0741000, 0x80011406, 0xbf810000
+};
+
+struct Row {
+    std::uint32_t op;
+    std::uint64_t memory;
+    std::uint64_t data;
+    std::uint64_t comparator;
+    std::uint64_t returned;
+    std::uint64_t final;
+};
+
+constexpr std::array<Row, 176> Rows{{
+    {0, 0x0ull, 0x7fffffffffffffffull, 0x0ull, 0x0ull, 0x7fffffffffffffffull},
+    {0, 0x1ull, 0xfffffffeffffffffull, 0x3ull, 0x1ull, 0xfffffffeffffffffull},
+    {0, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x8000000000000000ull},
+    {0, 0x8000000000000000ull, 0x123456789abcdef0ull, 0xb210ee8d643f54e6ull, 0x8000000000000000ull, 0x123456789abcdef0ull},
+    {0, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull},
+    {0, 0xffffffffull, 0x0ull, 0xffffffdfull, 0xffffffffull, 0x0ull},
+    {0, 0x100000000ull, 0xffffffffull, 0x100000000ull, 0x100000000ull, 0xffffffffull},
+    {0, 0xfffffffeffffffffull, 0x1ull, 0xed5fb21be38948d7ull, 0xfffffffeffffffffull, 0x1ull},
+    {0, 0x123456789abcdef0ull, 0x100000000ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull, 0x100000000ull},
+    {0, 0x0ull, 0x7fffffffffffffffull, 0x200ull, 0x0ull, 0x7fffffffffffffffull},
+    {0, 0x1ull, 0xfffffffeffffffffull, 0x1ull, 0x1ull, 0xfffffffeffffffffull},
+    {0, 0x7fffffffffffffffull, 0x8000000000000000ull, 0xb346daefa27ab170ull, 0x7fffffffffffffffull, 0x8000000000000000ull},
+    {0, 0x53bb9629497dfb3full, 0xa4abfed10ad2a242ull, 0x53bb9629497dfb3full, 0x53bb9629497dfb3full, 0xa4abfed10ad2a242ull},
+    {0, 0x6c12816965391baaull, 0xce7cd7b44b45eaf3ull, 0x6c12816967391baaull, 0x6c12816965391baaull, 0xce7cd7b44b45eaf3ull},
+    {0, 0xdba996ccffffffffull, 0x10bbe64100000001ull, 0xdba996ccffffffffull, 0xdba996ccffffffffull, 0x10bbe64100000001ull},
+    {0, 0x5a7c138cffffffffull, 0x9c15fed00000001ull, 0x5a7c138cdfffffffull, 0x5a7c138cffffffffull, 0x9c15fed00000001ull},
+    {1, 0x0ull, 0x7fffffffffffffffull, 0x0ull, 0x0ull, 0x7fffffffffffffffull},
+    {1, 0x1ull, 0xfffffffeffffffffull, 0x3ull, 0x1ull, 0x1ull},
+    {1, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x8000000000000000ull},
+    {1, 0x8000000000000000ull, 0x123456789abcdef0ull, 0x166c48c6e9c1b01cull, 0x8000000000000000ull, 0x8000000000000000ull},
+    {1, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull},
+    {1, 0xffffffffull, 0x0ull, 0xffffffdfull, 0xffffffffull, 0xffffffffull},
+    {1, 0x100000000ull, 0xffffffffull, 0x100000000ull, 0x100000000ull, 0xffffffffull},
+    {1, 0xfffffffeffffffffull, 0x1ull, 0x866b08e67ea9d02cull, 0xfffffffeffffffffull, 0xfffffffeffffffffull},
+    {1, 0x123456789abcdef0ull, 0x100000000ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull, 0x100000000ull},
+    {1, 0x0ull, 0x7fffffffffffffffull, 0x200ull, 0x0ull, 0x0ull},
+    {1, 0x1ull, 0xfffffffeffffffffull, 0x1ull, 0x1ull, 0xfffffffeffffffffull},
+    {1, 0x7fffffffffffffffull, 0x8000000000000000ull, 0xa54604e5b9b1311full, 0x7fffffffffffffffull, 0x7fffffffffffffffull},
+    {1, 0xb9a56d03ead2890full, 0x242f2f3b525af861ull, 0xb9a56d03ead2890full, 0xb9a56d03ead2890full, 0x242f2f3b525af861ull},
+    {1, 0x65c783e7e26c0130ull, 0x19a0106761c2f186ull, 0x65c783e7e06c0130ull, 0x65c783e7e26c0130ull, 0x65c783e7e26c0130ull},
+    {1, 0x7bebc240ffffffffull, 0x4da1ba4b00000001ull, 0x7bebc240ffffffffull, 0x7bebc240ffffffffull, 0x4da1ba4b00000001ull},
+    {1, 0x57e3536bffffffffull, 0xd8d033bd00000001ull, 0x57e3536bdfffffffull, 0x57e3536bffffffffull, 0x57e3536bffffffffull},
+    {2, 0x0ull, 0x7fffffffffffffffull, 0x0ull, 0x0ull, 0x7fffffffffffffffull},
+    {2, 0x1ull, 0xfffffffeffffffffull, 0x3ull, 0x1ull, 0xffffffff00000000ull},
+    {2, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0xffffffffffffffffull},
+    {2, 0x8000000000000000ull, 0x123456789abcdef0ull, 0xd4db9193123f6ebaull, 0x8000000000000000ull, 0x923456789abcdef0ull},
+    {2, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xfffffffffffffffeull},
+    {2, 0xffffffffull, 0x0ull, 0xffffffdfull, 0xffffffffull, 0xffffffffull},
+    {2, 0x100000000ull, 0xffffffffull, 0x100000000ull, 0x100000000ull, 0x1ffffffffull},
+    {2, 0xfffffffeffffffffull, 0x1ull, 0x25fd502c82facbc0ull, 0xfffffffeffffffffull, 0xffffffff00000000ull},
+    {2, 0x123456789abcdef0ull, 0x100000000ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull, 0x123456799abcdef0ull},
+    {2, 0x0ull, 0x7fffffffffffffffull, 0x200ull, 0x0ull, 0x7fffffffffffffffull},
+    {2, 0x1ull, 0xfffffffeffffffffull, 0x1ull, 0x1ull, 0xffffffff00000000ull},
+    {2, 0x7fffffffffffffffull, 0x8000000000000000ull, 0xf9124f0e4aae24c5ull, 0x7fffffffffffffffull, 0xffffffffffffffffull},
+    {2, 0xc3e86469baffc1dull, 0xc22f0e9d6d69338dull, 0xc3e86469baffc1dull, 0xc3e86469baffc1dull, 0xce6d94e409192faaull},
+    {2, 0xe499e6544c91e53ull, 0x4647904bf55b8053ull, 0xe499e6546c91e53ull, 0xe499e6544c91e53ull, 0x54912eb13a249ea6ull},
+    {2, 0x7a4a48e2ffffffffull, 0x7ccc4b5300000001ull, 0x7a4a48e2ffffffffull, 0x7a4a48e2ffffffffull, 0xf716943600000000ull},
+    {2, 0xa925179dffffffffull, 0xb2b1ac1800000001ull, 0xa925179ddfffffffull, 0xa925179dffffffffull, 0x5bd6c3b600000000ull},
+    {3, 0x0ull, 0x7fffffffffffffffull, 0x0ull, 0x0ull, 0x8000000000000001ull},
+    {3, 0x1ull, 0xfffffffeffffffffull, 0x3ull, 0x1ull, 0x100000002ull},
+    {3, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0xffffffffffffffffull},
+    {3, 0x8000000000000000ull, 0x123456789abcdef0ull, 0x6835bf86a066a97eull, 0x8000000000000000ull, 0x6dcba98765432110ull},
+    {3, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0x0ull},
+    {3, 0xffffffffull, 0x0ull, 0xffffffdfull, 0xffffffffull, 0xffffffffull},
+    {3, 0x100000000ull, 0xffffffffull, 0x100000000ull, 0x100000000ull, 0x1ull},
+    {3, 0xfffffffeffffffffull, 0x1ull, 0xbcbca7d9810b83c1ull, 0xfffffffeffffffffull, 0xfffffffefffffffeull},
+    {3, 0x123456789abcdef0ull, 0x100000000ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull, 0x123456779abcdef0ull},
+    {3, 0x0ull, 0x7fffffffffffffffull, 0x200ull, 0x0ull, 0x8000000000000001ull},
+    {3, 0x1ull, 0xfffffffeffffffffull, 0x1ull, 0x1ull, 0x100000002ull},
+    {3, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x663c553a9a16e35ull, 0x7fffffffffffffffull, 0xffffffffffffffffull},
+    {3, 0x63fbf4e8e453a4c5ull, 0xb7a15652045c43f6ull, 0x63fbf4e8e453a4c5ull, 0x63fbf4e8e453a4c5ull, 0xac5a9e96dff760cfull},
+    {3, 0x1603b25645bdd8aaull, 0x1f87b96dd545e1f1ull, 0x1603b25647bdd8aaull, 0x1603b25645bdd8aaull, 0xf67bf8e87077f6b9ull},
+    {3, 0x757fd168ffffffffull, 0x2356488500000001ull, 0x757fd168ffffffffull, 0x757fd168ffffffffull, 0x522988e3fffffffeull},
+    {3, 0x752b2a35ffffffffull, 0x869e617c00000001ull, 0x752b2a35dfffffffull, 0x752b2a35ffffffffull, 0xee8cc8b9fffffffeull},
+    {4, 0x0ull, 0x7fffffffffffffffull, 0x0ull, 0x0ull, 0x0ull},
+    {4, 0x1ull, 0xfffffffeffffffffull, 0x3ull, 0x1ull, 0xfffffffeffffffffull},
+    {4, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x8000000000000000ull},
+    {4, 0x8000000000000000ull, 0x123456789abcdef0ull, 0xd286c038892fd9d4ull, 0x8000000000000000ull, 0x8000000000000000ull},
+    {4, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull},
+    {4, 0xffffffffull, 0x0ull, 0xffffffdfull, 0xffffffffull, 0x0ull},
+    {4, 0x100000000ull, 0xffffffffull, 0x100000000ull, 0x100000000ull, 0xffffffffull},
+    {4, 0xfffffffeffffffffull, 0x1ull, 0x80b34c5cb5907e40ull, 0xfffffffeffffffffull, 0xfffffffeffffffffull},
+    {4, 0x123456789abcdef0ull, 0x100000000ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull, 0x100000000ull},
+    {4, 0x0ull, 0x7fffffffffffffffull, 0x200ull, 0x0ull, 0x0ull},
+    {4, 0x1ull, 0xfffffffeffffffffull, 0x1ull, 0x1ull, 0xfffffffeffffffffull},
+    {4, 0x7fffffffffffffffull, 0x8000000000000000ull, 0xe7cd1f2260293767ull, 0x7fffffffffffffffull, 0x8000000000000000ull},
+    {4, 0x8372abbda96ba8eeull, 0x5e71be928ba29363ull, 0x8372abbda96ba8eeull, 0x8372abbda96ba8eeull, 0x8372abbda96ba8eeull},
+    {4, 0xce77c3cdcc677910ull, 0xdf44f6f934647f0eull, 0xce77c3cdce677910ull, 0xce77c3cdcc677910ull, 0xce77c3cdcc677910ull},
+    {4, 0x9b3edbb1ffffffffull, 0xf4f16cf100000001ull, 0x9b3edbb1ffffffffull, 0x9b3edbb1ffffffffull, 0x9b3edbb1ffffffffull},
+    {4, 0xadf3fbf0ffffffffull, 0x8b29592900000001ull, 0xadf3fbf0dfffffffull, 0xadf3fbf0ffffffffull, 0x8b29592900000001ull},
+    {5, 0x0ull, 0x7fffffffffffffffull, 0x0ull, 0x0ull, 0x0ull},
+    {5, 0x1ull, 0xfffffffeffffffffull, 0x3ull, 0x1ull, 0x1ull},
+    {5, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x7fffffffffffffffull},
+    {5, 0x8000000000000000ull, 0x123456789abcdef0ull, 0xff63cf82d01938e9ull, 0x8000000000000000ull, 0x123456789abcdef0ull},
+    {5, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull},
+    {5, 0xffffffffull, 0x0ull, 0xffffffdfull, 0xffffffffull, 0x0ull},
+    {5, 0x100000000ull, 0xffffffffull, 0x100000000ull, 0x100000000ull, 0xffffffffull},
+    {5, 0xfffffffeffffffffull, 0x1ull, 0x2a88358605f1254ull, 0xfffffffeffffffffull, 0x1ull},
+    {5, 0x123456789abcdef0ull, 0x100000000ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull, 0x100000000ull},
+    {5, 0x0ull, 0x7fffffffffffffffull, 0x200ull, 0x0ull, 0x0ull},
+    {5, 0x1ull, 0xfffffffeffffffffull, 0x1ull, 0x1ull, 0x1ull},
+    {5, 0x7fffffffffffffffull, 0x8000000000000000ull, 0xf078418740b4f6d3ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull},
+    {5, 0x8d4a240ceafb92f2ull, 0xd789c424b5282b1dull, 0x8d4a240ceafb92f2ull, 0x8d4a240ceafb92f2ull, 0x8d4a240ceafb92f2ull},
+    {5, 0x64b3620ed7a9434full, 0x930b9894c3741be0ull, 0x64b3620ed5a9434full, 0x64b3620ed7a9434full, 0x64b3620ed7a9434full},
+    {5, 0x62a12f63ffffffffull, 0x4a1df58400000001ull, 0x62a12f63ffffffffull, 0x62a12f63ffffffffull, 0x4a1df58400000001ull},
+    {5, 0xd0826a0cffffffffull, 0x14fe08aa00000001ull, 0xd0826a0cdfffffffull, 0xd0826a0cffffffffull, 0x14fe08aa00000001ull},
+    {6, 0x0ull, 0x7fffffffffffffffull, 0x0ull, 0x0ull, 0x7fffffffffffffffull},
+    {6, 0x1ull, 0xfffffffeffffffffull, 0x3ull, 0x1ull, 0x1ull},
+    {6, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x7fffffffffffffffull},
+    {6, 0x8000000000000000ull, 0x123456789abcdef0ull, 0x35f6b71611838017ull, 0x8000000000000000ull, 0x123456789abcdef0ull},
+    {6, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull},
+    {6, 0xffffffffull, 0x0ull, 0xffffffdfull, 0xffffffffull, 0xffffffffull},
+    {6, 0x100000000ull, 0xffffffffull, 0x100000000ull, 0x100000000ull, 0x100000000ull},
+    {6, 0xfffffffeffffffffull, 0x1ull, 0x884dcfec36f43da6ull, 0xfffffffeffffffffull, 0x1ull},
+    {6, 0x123456789abcdef0ull, 0x100000000ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull},
+    {6, 0x0ull, 0x7fffffffffffffffull, 0x200ull, 0x0ull, 0x7fffffffffffffffull},
+    {6, 0x1ull, 0xfffffffeffffffffull, 0x1ull, 0x1ull, 0x1ull},
+    {6, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x97e21514d87b299dull, 0x7fffffffffffffffull, 0x7fffffffffffffffull},
+    {6, 0x7ae9f54bc19e56ddull, 0x4a6ac7aeb830ed76ull, 0x7ae9f54bc19e56ddull, 0x7ae9f54bc19e56ddull, 0x7ae9f54bc19e56ddull},
+    {6, 0x5b1c8aa1eb9a258cull, 0xf4a0ea8b3badac72ull, 0x5b1c8aa1e99a258cull, 0x5b1c8aa1eb9a258cull, 0x5b1c8aa1eb9a258cull},
+    {6, 0x869f35deffffffffull, 0xdeffa68100000001ull, 0x869f35deffffffffull, 0x869f35deffffffffull, 0xdeffa68100000001ull},
+    {6, 0x5752b4dfffffffffull, 0x2a51ee7e00000001ull, 0x5752b4dfdfffffffull, 0x5752b4dfffffffffull, 0x5752b4dfffffffffull},
+    {7, 0x0ull, 0x7fffffffffffffffull, 0x0ull, 0x0ull, 0x7fffffffffffffffull},
+    {7, 0x1ull, 0xfffffffeffffffffull, 0x3ull, 0x1ull, 0xfffffffeffffffffull},
+    {7, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x8000000000000000ull},
+    {7, 0x8000000000000000ull, 0x123456789abcdef0ull, 0xd316f470e2d8993bull, 0x8000000000000000ull, 0x8000000000000000ull},
+    {7, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull},
+    {7, 0xffffffffull, 0x0ull, 0xffffffdfull, 0xffffffffull, 0xffffffffull},
+    {7, 0x100000000ull, 0xffffffffull, 0x100000000ull, 0x100000000ull, 0x100000000ull},
+    {7, 0xfffffffeffffffffull, 0x1ull, 0xfc2b7ace1072e7b9ull, 0xfffffffeffffffffull, 0xfffffffeffffffffull},
+    {7, 0x123456789abcdef0ull, 0x100000000ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull},
+    {7, 0x0ull, 0x7fffffffffffffffull, 0x200ull, 0x0ull, 0x7fffffffffffffffull},
+    {7, 0x1ull, 0xfffffffeffffffffull, 0x1ull, 0x1ull, 0xfffffffeffffffffull},
+    {7, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x142c1e683d47a123ull, 0x7fffffffffffffffull, 0x8000000000000000ull},
+    {7, 0x5ea0e74f0d30cb76ull, 0xecd62560f1407a13ull, 0x5ea0e74f0d30cb76ull, 0x5ea0e74f0d30cb76ull, 0xecd62560f1407a13ull},
+    {7, 0x6f8664b99db31c8bull, 0x77417f9d1c387d00ull, 0x6f8664b99fb31c8bull, 0x6f8664b99db31c8bull, 0x77417f9d1c387d00ull},
+    {7, 0x1e0ce078ffffffffull, 0xc130cf3c00000001ull, 0x1e0ce078ffffffffull, 0x1e0ce078ffffffffull, 0xc130cf3c00000001ull},
+    {7, 0xd160d889ffffffffull, 0x2b456f6800000001ull, 0xd160d889dfffffffull, 0xd160d889ffffffffull, 0xd160d889ffffffffull},
+    {8, 0x0ull, 0x7fffffffffffffffull, 0x0ull, 0x0ull, 0x0ull},
+    {8, 0x1ull, 0xfffffffeffffffffull, 0x3ull, 0x1ull, 0x1ull},
+    {8, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x0ull},
+    {8, 0x8000000000000000ull, 0x123456789abcdef0ull, 0x691beca72ac31615ull, 0x8000000000000000ull, 0x0ull},
+    {8, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull},
+    {8, 0xffffffffull, 0x0ull, 0xffffffdfull, 0xffffffffull, 0x0ull},
+    {8, 0x100000000ull, 0xffffffffull, 0x100000000ull, 0x100000000ull, 0x0ull},
+    {8, 0xfffffffeffffffffull, 0x1ull, 0xc89594109911783aull, 0xfffffffeffffffffull, 0x1ull},
+    {8, 0x123456789abcdef0ull, 0x100000000ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull, 0x0ull},
+    {8, 0x0ull, 0x7fffffffffffffffull, 0x200ull, 0x0ull, 0x0ull},
+    {8, 0x1ull, 0xfffffffeffffffffull, 0x1ull, 0x1ull, 0x1ull},
+    {8, 0x7fffffffffffffffull, 0x8000000000000000ull, 0xa69bfe93d6efb4ddull, 0x7fffffffffffffffull, 0x0ull},
+    {8, 0xd5e7cc579d21963aull, 0x93826a1ee8ab2ed1ull, 0xd5e7cc579d21963aull, 0xd5e7cc579d21963aull, 0x9182481688210610ull},
+    {8, 0x657c382c4e1cfcddull, 0x837a0dc63ac35f32ull, 0x657c382c4c1cfcddull, 0x657c382c4e1cfcddull, 0x17808040a005c10ull},
+    {8, 0x7552c284ffffffffull, 0x995c2cf200000001ull, 0x7552c284ffffffffull, 0x7552c284ffffffffull, 0x1150008000000001ull},
+    {8, 0xe20d4f8dffffffffull, 0x4864cdbc00000001ull, 0xe20d4f8ddfffffffull, 0xe20d4f8dffffffffull, 0x40044d8c00000001ull},
+    {9, 0x0ull, 0x7fffffffffffffffull, 0x0ull, 0x0ull, 0x7fffffffffffffffull},
+    {9, 0x1ull, 0xfffffffeffffffffull, 0x3ull, 0x1ull, 0xfffffffeffffffffull},
+    {9, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0xffffffffffffffffull},
+    {9, 0x8000000000000000ull, 0x123456789abcdef0ull, 0xbad3dfdbb007d4ecull, 0x8000000000000000ull, 0x923456789abcdef0ull},
+    {9, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull},
+    {9, 0xffffffffull, 0x0ull, 0xffffffdfull, 0xffffffffull, 0xffffffffull},
+    {9, 0x100000000ull, 0xffffffffull, 0x100000000ull, 0x100000000ull, 0x1ffffffffull},
+    {9, 0xfffffffeffffffffull, 0x1ull, 0xedecea3c6fa0d23full, 0xfffffffeffffffffull, 0xfffffffeffffffffull},
+    {9, 0x123456789abcdef0ull, 0x100000000ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull, 0x123456799abcdef0ull},
+    {9, 0x0ull, 0x7fffffffffffffffull, 0x200ull, 0x0ull, 0x7fffffffffffffffull},
+    {9, 0x1ull, 0xfffffffeffffffffull, 0x1ull, 0x1ull, 0xfffffffeffffffffull},
+    {9, 0x7fffffffffffffffull, 0x8000000000000000ull, 0xf7926675f45d1971ull, 0x7fffffffffffffffull, 0xffffffffffffffffull},
+    {9, 0x360443a82569e20cull, 0x3800d9ff899231faull, 0x360443a82569e20cull, 0x360443a82569e20cull, 0x3e04dbffadfbf3feull},
+    {9, 0xc265df77a6c71a0dull, 0x5e94d6ccc50557f8ull, 0xc265df77a4c71a0dull, 0xc265df77a6c71a0dull, 0xdef5dfffe7c75ffdull},
+    {9, 0x41a90d6bffffffffull, 0x4bb3092400000001ull, 0x41a90d6bffffffffull, 0x41a90d6bffffffffull, 0x4bbb0d6fffffffffull},
+    {9, 0xc54e9269ffffffffull, 0xf536f34000000001ull, 0xc54e9269dfffffffull, 0xc54e9269ffffffffull, 0xf57ef369ffffffffull},
+    {10, 0x0ull, 0x7fffffffffffffffull, 0x0ull, 0x0ull, 0x7fffffffffffffffull},
+    {10, 0x1ull, 0xfffffffeffffffffull, 0x3ull, 0x1ull, 0xfffffffefffffffeull},
+    {10, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0xffffffffffffffffull},
+    {10, 0x8000000000000000ull, 0x123456789abcdef0ull, 0xe3b411de678ee5c0ull, 0x8000000000000000ull, 0x923456789abcdef0ull},
+    {10, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0x0ull},
+    {10, 0xffffffffull, 0x0ull, 0xffffffdfull, 0xffffffffull, 0xffffffffull},
+    {10, 0x100000000ull, 0xffffffffull, 0x100000000ull, 0x100000000ull, 0x1ffffffffull},
+    {10, 0xfffffffeffffffffull, 0x1ull, 0xe4e171d83c299defull, 0xfffffffeffffffffull, 0xfffffffefffffffeull},
+    {10, 0x123456789abcdef0ull, 0x100000000ull, 0x123456789abcdef0ull, 0x123456789abcdef0ull, 0x123456799abcdef0ull},
+    {10, 0x0ull, 0x7fffffffffffffffull, 0x200ull, 0x0ull, 0x7fffffffffffffffull},
+    {10, 0x1ull, 0xfffffffeffffffffull, 0x1ull, 0x1ull, 0xfffffffefffffffeull},
+    {10, 0x7fffffffffffffffull, 0x8000000000000000ull, 0xaa23249b400d9ebdull, 0x7fffffffffffffffull, 0xffffffffffffffffull},
+    {10, 0xc1f06114a97ad060ull, 0x679f7fe64c950a15ull, 0xc1f06114a97ad060ull, 0xc1f06114a97ad060ull, 0xa66f1ef2e5efda75ull},
+    {10, 0x59bdb13042bcfe48ull, 0x1669010b3ea903bbull, 0x59bdb13040bcfe48ull, 0x59bdb13042bcfe48ull, 0x4fd4b03b7c15fdf3ull},
+    {10, 0x701d03a9ffffffffull, 0xf96a081600000001ull, 0x701d03a9ffffffffull, 0x701d03a9ffffffffull, 0x89770bbffffffffeull},
+    {10, 0x9b817862ffffffffull, 0x3c3e50fe00000001ull, 0x9b817862dfffffffull, 0x9b817862ffffffffull, 0xa7bf289cfffffffeull},
+}};
+
+class GuestBlock {
+public:
+    explicit GuestBlock(bool writable) {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
+#endif
+        Require(block != nullptr, "global atomics: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, writable);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+    const std::uint8_t* Data() const { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+void Put(std::vector<std::uint8_t>& image, std::uint32_t offset, std::uint64_t value, std::uint32_t bytes) {
+    for (std::uint32_t byte = 0; byte < bytes; ++byte) image.at(offset + byte) = static_cast<std::uint8_t>(value >> (byte * 8u));
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint64_t value) {
+    char text[24];
+    std::snprintf(text, sizeof(text), "0x%llx", static_cast<unsigned long long>(value));
+    return text;
+}
+
+const Row& RowOf(std::uint32_t op, std::uint32_t tid) {
+    return Rows.at(op * RowsPerOp + tid % RowsPerOp);
+}
+
+std::uint32_t OpBytes(std::uint32_t op) {
+    return op < NarrowOps ? 4u : 8u;
+}
+
+std::vector<std::uint8_t> Initial() {
+    std::vector<std::uint8_t> image(BlockBytes, Fill);
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            Put(image, op * OpStride + tid * 8u, RowOf(op, tid).memory, OpBytes(op));
+        }
+    }
+    return image;
+}
+
+std::vector<std::uint8_t> Expected() {
+    auto image = Initial();
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            Put(image, op * OpStride + tid * 8u, RowOf(op, tid).final, OpBytes(op));
+        }
+    }
+    return image;
+}
+
+void FillInput() {
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            const auto& row = RowOf(op, tid);
+            auto* words = &Input[(op * Threads + tid) * 4u];
+            if (OpBytes(op) == 4u) {
+                words[0] = static_cast<std::uint32_t>(row.data);
+                words[1] = static_cast<std::uint32_t>(row.comparator);
+                words[2] = 0u;
+                words[3] = 0u;
+            } else {
+                words[0] = static_cast<std::uint32_t>(row.data);
+                words[1] = static_cast<std::uint32_t>(row.data >> 32u);
+                words[2] = static_cast<std::uint32_t>(row.comparator);
+                words[3] = static_cast<std::uint32_t>(row.comparator >> 32u);
+            }
+        }
+    }
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, const std::uint8_t* base) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(base));
+    std::vector<std::uint32_t> userData(10, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    userData[8] = static_cast<std::uint32_t>(address);
+    userData[9] = static_cast<std::uint32_t>(address >> 32u);
+    const std::span<const std::uint32_t> code(GlobalAtomicsCode);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void RunAtomics(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t waveSize) {
+    const auto initial = Initial();
+    std::memcpy(guest.Data(), initial.data(), BlockBytes);
+    FillInput();
+    Output.fill(Sentinel);
+    Dispatch(device, waveSize, guest.Data());
+    const auto wave = "global atomics: wave" + std::to_string(waveSize) + " ";
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            const auto& row = RowOf(op, tid);
+            const auto* words = &Output[(op * Threads + tid) * 2u];
+            const auto returned = OpBytes(op) == 4u ? static_cast<std::uint64_t>(words[0]) : (static_cast<std::uint64_t>(words[1]) << 32u) | words[0];
+            Require(returned == row.returned, wave + "op " + std::to_string(op) + " lane " + std::to_string(tid) + " returned " + Hex(returned) + ", expected " + Hex(row.returned));
+        }
+    }
+    const auto expected = Expected();
+    for (std::uint32_t offset = 0; offset < BlockBytes; ++offset) {
+        const auto actual = guest.Data()[offset];
+        Require(actual == expected[offset], wave + "byte " + std::to_string(offset) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[offset]));
+    }
+}
+
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock writable(true);
+        RunAtomics(*device, writable, 32);
+        std::puts("global atomics 64 tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsFloat.cpp

@@ -0,0 +1,750 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 64;
+constexpr std::uint32_t OpCount = 8;
+constexpr std::uint32_t NarrowOps = 3;
+constexpr std::uint32_t OpStride = 0x200;
+constexpr std::uint32_t Sentinel = 0x05e471e1;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::uint8_t Fill = 0xcd;
+alignas(256) std::array<std::uint32_t, OpCount * Threads * 4> Input{};
+alignas(256) std::array<std::uint32_t, OpCount * Threads * 2> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 129> GlobalAtomicsFloatCode{
+    0x34020083, 0x34040084, 0x340a0083, 0x7e2802ff, 0x05e471e1, 0x7e2a02ff, 0x05e471e1, 0x4a060280,
+    0x4a080480, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0a80, 0xdcf98000, 0x14080a03, 0xbf8c0070,
+    0xe0701000, 0x80011406, 0x4a0602ff, 0x00000200, 0x4a0804ff, 0x00000400, 0xe0381000, 0x80000a04,
+    0xbf8c3f70, 0x4a0c0aff, 0x00000200, 0xd70f6a1e, 0x02020608, 0x7e3e0209, 0x503e3e80, 0xdcfd0000,
+    0x147d0a1e, 0xbf8c0070, 0xe0701000, 0x80011406, 0x4a0602ff, 0x00000400, 0x4a0804ff, 0x00000800,
+    0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00000400, 0xdd018000, 0x14080a03, 0xbf8c0070,
+    0xe0701000, 0x80011406, 0x4a0602ff, 0x00000600, 0x4a0804ff, 0x00000c00, 0xe0381000, 0x80000a04,
+    0xbf8c3f70, 0x4a0c0aff, 0x00000600, 0xdd798000, 0x14080a03, 0xbf8c0070, 0xe0741000, 0x80011406,
+    0x4a0602ff, 0x00000800, 0x4a0804ff, 0x00001000, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff,
+    0x00000800, 0xdd7d8000, 0x14080a03, 0xbf8c0070, 0xe0741000, 0x80011406, 0x4a0602ff, 0x00000a00,
+    0x4a0804ff, 0x00001400, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00000a00, 0xd70f6a1e,
+    0x02020608, 0x7e3e0209, 0x503e3e80, 0xdd810000, 0x147d0a1e, 0xbf8c0070, 0xe0741000, 0x80011406,
+    0x4a0602ff, 0x00000c00, 0x4a0804ff, 0x00001800, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff,
+    0x00000c00, 0xdd718000, 0x14080a03, 0xbf8c0070, 0xe0741000, 0x80011406, 0x4a0602ff, 0x00000e00,
+    0x4a0804ff, 0x00001c00, 0xe0381000, 0x80000a04, 0xbf8c3f70, 0x4a0c0aff, 0x00000e00, 0xd70f6a1e,
+    0x02020608, 0x7e3e0209, 0x503e3e80, 0xdd750000, 0x147d0a1e, 0xbf8c0070, 0xe0741000, 0x80011406,
+    0xbf810000
+};
+
+struct Row {
+    std::uint32_t op;
+    std::uint64_t memory;
+    std::uint64_t data;
+    std::uint64_t comparator;
+    std::uint64_t final;
+};
+
+constexpr std::array<Row, 512> Rows{{
+    {0, 0x0ull, 0x0ull, 0x0ull, 0x0ull},
+    {0, 0x0ull, 0x80000000ull, 0x80000000ull, 0x80000000ull},
+    {0, 0x0ull, 0x3f800000ull, 0xc0200000ull, 0x0ull},
+    {0, 0x0ull, 0xbf800000ull, 0xbf800000ull, 0x0ull},
+    {0, 0x0ull, 0x3fc00000ull, 0x0ull, 0x3fc00000ull},
+    {0, 0x0ull, 0x7f800000ull, 0xffc12345ull, 0x0ull},
+    {0, 0x0ull, 0xff800000ull, 0xff800000ull, 0x0ull},
+    {0, 0x0ull, 0x7fa00000ull, 0x807fffffull, 0x0ull},
+    {0, 0x0ull, 0xff800001ull, 0xff800001ull, 0x0ull},
+    {0, 0x0ull, 0x800000ull, 0x800000ull, 0x0ull},
+    {0, 0x80000000ull, 0x0ull, 0x80000000ull, 0x0ull},
+    {0, 0x80000000ull, 0x80000000ull, 0x0ull, 0x80000000ull},
+    {0, 0x80000000ull, 0x3f800000ull, 0x7f800000ull, 0x80000000ull},
+    {0, 0x80000000ull, 0xbf800000ull, 0xbf800000ull, 0x80000000ull},
+    {0, 0x80000000ull, 0xc0200000ull, 0x0ull, 0xc0200000ull},
+    {0, 0x80000000ull, 0x7f800000ull, 0x7fa00000ull, 0x80000000ull},
+    {0, 0x80000000ull, 0xff800000ull, 0xff800000ull, 0x80000000ull},
+    {0, 0x80000000ull, 0x7fa00000ull, 0x7f7fffffull, 0x80000000ull},
+    {0, 0x80000000ull, 0xff800001ull, 0xff800001ull, 0x80000000ull},
+    {0, 0x80000000ull, 0x7f7fffffull, 0x3f800000ull, 0x80000000ull},
+    {0, 0x80000000ull, 0x800000ull, 0x800000ull, 0x80000000ull},
+    {0, 0x3f800000ull, 0x0ull, 0x3f800000ull, 0x0ull},
+    {0, 0x3f800000ull, 0x80000000ull, 0xbf800000ull, 0x3f800000ull},
+    {0, 0x3f800000ull, 0x3f800000ull, 0xff800000ull, 0x3f800000ull},
+    {0, 0x3f800000ull, 0x7f800000ull, 0xff800001ull, 0x3f800000ull},
+    {0, 0x3f800000ull, 0x7fa00000ull, 0x800000ull, 0x3f800000ull},
+    {0, 0xbf800000ull, 0x0ull, 0xbf800000ull, 0x0ull},
+    {0, 0xbf800000ull, 0x80000000ull, 0x3f800000ull, 0xbf800000ull},
+    {0, 0xbf800000ull, 0x3f800000ull, 0x7fc00000ull, 0xbf800000ull},
+    {0, 0xbf800000ull, 0xbf800000ull, 0xbf800000ull, 0xbf800000ull},
+    {0, 0xbf800000ull, 0x7f800000ull, 0x1ull, 0xbf800000ull},
+    {0, 0xbf800000ull, 0xff800000ull, 0xff800000ull, 0xbf800000ull},
+    {0, 0xbf800000ull, 0x7fa00000ull, 0x0ull, 0xbf800000ull},
+    {0, 0xbf800000ull, 0xff800001ull, 0xff800001ull, 0xbf800000ull},
+    {0, 0xbf800000ull, 0x7f7fffffull, 0x3fc00000ull, 0xbf800000ull},
+    {0, 0xbf800000ull, 0x800000ull, 0x800000ull, 0xbf800000ull},
+    {0, 0x3fc00000ull, 0x0ull, 0x3fc00000ull, 0x0ull},
+    {0, 0x3fc00000ull, 0x80000000ull, 0xbfc00000ull, 0x3fc00000ull},
+    {0, 0x3fc00000ull, 0x3f800000ull, 0xffc12345ull, 0x3fc00000ull},
+    {0, 0x3fc00000ull, 0xbf800000ull, 0xbf800000ull, 0x3fc00000ull},
+    {0, 0x3fc00000ull, 0x3fc00000ull, 0x3fc00000ull, 0x3fc00000ull},
+    {0, 0x3fc00000ull, 0x7f800000ull, 0x807fffffull, 0x3fc00000ull},
+    {0, 0x3fc00000ull, 0xff800000ull, 0xff800000ull, 0x3fc00000ull},
+    {0, 0x3fc00000ull, 0x7fa00000ull, 0x80000000ull, 0x3fc00000ull},
+    {0, 0x3fc00000ull, 0xff800001ull, 0xff800001ull, 0x3fc00000ull},
+    {0, 0x3fc00000ull, 0x7f7fffffull, 0xc0200000ull, 0x3fc00000ull},
+    {0, 0x3fc00000ull, 0x800000ull, 0x800000ull, 0x3fc00000ull},
+    {0, 0xc0200000ull, 0x0ull, 0xc0200000ull, 0x0ull},
+    {0, 0xc0200000ull, 0x80000000ull, 0x40200000ull, 0xc0200000ull},
+    {0, 0xc0200000ull, 0x3f800000ull, 0x7fa00000ull, 0xc0200000ull},
+    {0, 0xc0200000ull, 0xbf800000ull, 0xbf800000ull, 0xc0200000ull},
+    {0, 0xc0200000ull, 0x7f800000ull, 0x7f7fffffull, 0xc0200000ull},
+    {0, 0xc0200000ull, 0xff800000ull, 0xff800000ull, 0xc0200000ull},
+    {0, 0xc0200000ull, 0x7fa00000ull, 0x3f800000ull, 0xc0200000ull},
+    {0, 0xc0200000ull, 0xff800001ull, 0xff800001ull, 0xc0200000ull},
+    {0, 0xc0200000ull, 0x7f7fffffull, 0x7f800000ull, 0xc0200000ull},
+    {0, 0xc0200000ull, 0x800000ull, 0x800000ull, 0xc0200000ull},
+    {0, 0x7f800000ull, 0x0ull, 0x7f800000ull, 0x0ull},
+    {0, 0x7f800000ull, 0x80000000ull, 0xff800000ull, 0x7f800000ull},
+    {0, 0x7f800000ull, 0x3f800000ull, 0xff800001ull, 0x7f800000ull},
+    {0, 0x7f800000ull, 0xbf800000ull, 0xbf800000ull, 0x7f800000ull},
+    {0, 0x7f800000ull, 0x7f800000ull, 0x800000ull, 0x7f800000ull},
+    {0, 0xff800000ull, 0x0ull, 0xff800000ull, 0x0ull},
+    {0, 0xff800000ull, 0x80000000ull, 0x7f800000ull, 0xff800000ull},
+    {1, 0x0ull, 0x0ull, 0x0ull, 0x0ull},
+    {1, 0x0ull, 0x80000000ull, 0x80000000ull, 0x80000000ull},
+    {1, 0x0ull, 0x3f800000ull, 0xc0200000ull, 0x0ull},
+    {1, 0x0ull, 0xbf800000ull, 0xbf800000ull, 0xbf800000ull},
+    {1, 0x0ull, 0x3fc00000ull, 0x0ull, 0x0ull},
+    {1, 0x0ull, 0xc0200000ull, 0x80000000ull, 0xc0200000ull},
+    {1, 0x0ull, 0x7f800000ull, 0xffc12345ull, 0x0ull},
+    {1, 0x0ull, 0xff800000ull, 0xff800000ull, 0xff800000ull},
+    {1, 0x0ull, 0x7fc00000ull, 0x0ull, 0x0ull},
+    {1, 0x0ull, 0xffc12345ull, 0x80000000ull, 0x0ull},
+    {1, 0x0ull, 0x7fa00000ull, 0x807fffffull, 0x7fe00000ull},
+    {1, 0x0ull, 0xff800001ull, 0xff800001ull, 0xffc00001ull},
+    {1, 0x0ull, 0x1ull, 0x0ull, 0x0ull},
+    {1, 0x0ull, 0x807fffffull, 0x80000000ull, 0x807fffffull},
+    {1, 0x0ull, 0x7f7fffffull, 0x80000000ull, 0x0ull},
+    {1, 0x0ull, 0x800000ull, 0x800000ull, 0x0ull},
+    {1, 0x80000000ull, 0x0ull, 0x80000000ull, 0x80000000ull},
+    {1, 0x80000000ull, 0x80000000ull, 0x0ull, 0x80000000ull},
+    {1, 0x80000000ull, 0x3f800000ull, 0x7f800000ull, 0x80000000ull},
+    {1, 0x80000000ull, 0xbf800000ull, 0xbf800000ull, 0xbf800000ull},
+    {1, 0x80000000ull, 0x3fc00000ull, 0x80000000ull, 0x80000000ull},
+    {1, 0x80000000ull, 0xc0200000ull, 0x0ull, 0xc0200000ull},
+    {1, 0x80000000ull, 0x7f800000ull, 0x7fa00000ull, 0x80000000ull},
+    {1, 0x80000000ull, 0xff800000ull, 0xff800000ull, 0xff800000ull},
+    {1, 0x80000000ull, 0x7fc00000ull, 0x80000000ull, 0x80000000ull},
+    {1, 0x80000000ull, 0xffc12345ull, 0x0ull, 0x80000000ull},
+    {1, 0x80000000ull, 0x7fa00000ull, 0x7f7fffffull, 0x7fe00000ull},
+    {1, 0x80000000ull, 0xff800001ull, 0xff800001ull, 0xffc00001ull},
+    {1, 0x80000000ull, 0x1ull, 0x80000000ull, 0x80000000ull},
+    {1, 0x80000000ull, 0x807fffffull, 0x0ull, 0x807fffffull},
+    {1, 0x80000000ull, 0x7f7fffffull, 0x3f800000ull, 0x80000000ull},
+    {1, 0x80000000ull, 0x800000ull, 0x800000ull, 0x80000000ull},
+    {1, 0x3f800000ull, 0x0ull, 0x3f800000ull, 0x0ull},
+    {1, 0x3f800000ull, 0x80000000ull, 0xbf800000ull, 0x80000000ull},
+    {1, 0x3f800000ull, 0x3f800000ull, 0xff800000ull, 0x3f800000ull},
+    {1, 0x3f800000ull, 0xbf800000ull, 0xbf800000ull, 0xbf800000ull},
+    {1, 0x3f800000ull, 0x3fc00000ull, 0x3f800000ull, 0x3f800000ull},
+    {1, 0x3f800000ull, 0xc0200000ull, 0xbf800000ull, 0xc0200000ull},
+    {1, 0x3f800000ull, 0x7f800000ull, 0xff800001ull, 0x3f800000ull},
+    {1, 0x3f800000ull, 0xff800000ull, 0xff800000ull, 0xff800000ull},
+    {1, 0x3f800000ull, 0x7fc00000ull, 0x3f800000ull, 0x3f800000ull},
+    {1, 0x3f800000ull, 0xffc12345ull, 0xbf800000ull, 0x3f800000ull},
+    {1, 0x3f800000ull, 0x7fa00000ull, 0x800000ull, 0x7fe00000ull},
+    {1, 0x3f800000ull, 0xff800001ull, 0xff800001ull, 0xffc00001ull},
+    {1, 0x3f800000ull, 0x1ull, 0x3f800000ull, 0x1ull},
+    {1, 0x3f800000ull, 0x807fffffull, 0xbf800000ull, 0x807fffffull},
+    {1, 0x3f800000ull, 0x7f7fffffull, 0xbf800000ull, 0x3f800000ull},
+    {1, 0x3f800000ull, 0x800000ull, 0x800000ull, 0x800000ull},
+    {1, 0xbf800000ull, 0x0ull, 0xbf800000ull, 0xbf800000ull},
+    {1, 0xbf800000ull, 0x80000000ull, 0x3f800000ull, 0xbf800000ull},
+    {1, 0xbf800000ull, 0x3f800000ull, 0x7fc00000ull, 0xbf800000ull},
+    {1, 0xbf800000ull, 0xbf800000ull, 0xbf800000ull, 0xbf800000ull},
+    {1, 0xbf800000ull, 0x3fc00000ull, 0xbf800000ull, 0xbf800000ull},
+    {1, 0xbf800000ull, 0xc0200000ull, 0x3f800000ull, 0xc0200000ull},
+    {1, 0xbf800000ull, 0x7f800000ull, 0x1ull, 0xbf800000ull},
+    {1, 0xbf800000ull, 0xff800000ull, 0xff800000ull, 0xff800000ull},
+    {1, 0xbf800000ull, 0x7fc00000ull, 0xbf800000ull, 0xbf800000ull},
+    {1, 0xbf800000ull, 0xffc12345ull, 0x3f800000ull, 0xbf800000ull},
+    {1, 0xbf800000ull, 0x7fa00000ull, 0x0ull, 0x7fe00000ull},
+    {1, 0xbf800000ull, 0xff800001ull, 0xff800001ull, 0xffc00001ull},
+    {1, 0xbf800000ull, 0x1ull, 0xbf800000ull, 0xbf800000ull},
+    {1, 0xbf800000ull, 0x807fffffull, 0x3f800000ull, 0xbf800000ull},
+    {1, 0xbf800000ull, 0x7f7fffffull, 0x3fc00000ull, 0xbf800000ull},
+    {1, 0xbf800000ull, 0x800000ull, 0x800000ull, 0xbf800000ull},
+    {2, 0x0ull, 0x0ull, 0x0ull, 0x0ull},
+    {2, 0x0ull, 0x80000000ull, 0x80000000ull, 0x0ull},
+    {2, 0x0ull, 0x3f800000ull, 0xc0200000ull, 0x3f800000ull},
+    {2, 0x0ull, 0xbf800000ull, 0xbf800000ull, 0x0ull},
+    {2, 0x0ull, 0x3fc00000ull, 0x0ull, 0x3fc00000ull},
+    {2, 0x0ull, 0xc0200000ull, 0x80000000ull, 0x0ull},
+    {2, 0x0ull, 0x7f800000ull, 0xffc12345ull, 0x7f800000ull},
+    {2, 0x0ull, 0xff800000ull, 0xff800000ull, 0x0ull},
+    {2, 0x0ull, 0x7fc00000ull, 0x0ull, 0x0ull},
+    {2, 0x0ull, 0xffc12345ull, 0x80000000ull, 0x0ull},
+    {2, 0x0ull, 0x7fa00000ull, 0x807fffffull, 0x7fe00000ull},
+    {2, 0x0ull, 0xff800001ull, 0xff800001ull, 0xffc00001ull},
+    {2, 0x0ull, 0x1ull, 0x0ull, 0x1ull},
+    {2, 0x0ull, 0x807fffffull, 0x80000000ull, 0x0ull},
+    {2, 0x0ull, 0x7f7fffffull, 0x80000000ull, 0x7f7fffffull},
+    {2, 0x0ull, 0x800000ull, 0x800000ull, 0x800000ull},
+    {2, 0x80000000ull, 0x0ull, 0x80000000ull, 0x0ull},
+    {2, 0x80000000ull, 0x80000000ull, 0x0ull, 0x80000000ull},
+    {2, 0x80000000ull, 0x3f800000ull, 0x7f800000ull, 0x3f800000ull},
+    {2, 0x80000000ull, 0xbf800000ull, 0xbf800000ull, 0x80000000ull},
+    {2, 0x80000000ull, 0x3fc00000ull, 0x80000000ull, 0x3fc00000ull},
+    {2, 0x80000000ull, 0xc0200000ull, 0x0ull, 0x80000000ull},
+    {2, 0x80000000ull, 0x7f800000ull, 0x7fa00000ull, 0x7f800000ull},
+    {2, 0x80000000ull, 0xff800000ull, 0xff800000ull, 0x80000000ull},
+    {2, 0x80000000ull, 0x7fc00000ull, 0x80000000ull, 0x80000000ull},
+    {2, 0x80000000ull, 0xffc12345ull, 0x0ull, 0x80000000ull},
+    {2, 0x80000000ull, 0x7fa00000ull, 0x7f7fffffull, 0x7fe00000ull},
+    {2, 0x80000000ull, 0xff800001ull, 0xff800001ull, 0xffc00001ull},
+    {2, 0x80000000ull, 0x1ull, 0x80000000ull, 0x1ull},
+    {2, 0x80000000ull, 0x807fffffull, 0x0ull, 0x80000000ull},
+    {2, 0x80000000ull, 0x7f7fffffull, 0x3f800000ull, 0x7f7fffffull},
+    {2, 0x80000000ull, 0x800000ull, 0x800000ull, 0x800000ull},
+    {2, 0x3f800000ull, 0x0ull, 0x3f800000ull, 0x3f800000ull},
+    {2, 0x3f800000ull, 0x80000000ull, 0xbf800000ull, 0x3f800000ull},
+    {2, 0x3f800000ull, 0x3f800000ull, 0xff800000ull, 0x3f800000ull},
+    {2, 0x3f800000ull, 0xbf800000ull, 0xbf800000ull, 0x3f800000ull},
+    {2, 0x3f800000ull, 0x3fc00000ull, 0x3f800000ull, 0x3fc00000ull},
+    {2, 0x3f800000ull, 0xc0200000ull, 0xbf800000ull, 0x3f800000ull},
+    {2, 0x3f800000ull, 0x7f800000ull, 0xff800001ull, 0x7f800000ull},
+    {2, 0x3f800000ull, 0xff800000ull, 0xff800000ull, 0x3f800000ull},
+    {2, 0x3f800000ull, 0x7fc00000ull, 0x3f800000ull, 0x3f800000ull},
+    {2, 0x3f800000ull, 0xffc12345ull, 0xbf800000ull, 0x3f800000ull},
+    {2, 0x3f800000ull, 0x7fa00000ull, 0x800000ull, 0x7fe00000ull},
+    {2, 0x3f800000ull, 0xff800001ull, 0xff800001ull, 0xffc00001ull},
+    {2, 0x3f800000ull, 0x1ull, 0x3f800000ull, 0x3f800000ull},
+    {2, 0x3f800000ull, 0x807fffffull, 0xbf800000ull, 0x3f800000ull},
+    {2, 0x3f800000ull, 0x7f7fffffull, 0xbf800000ull, 0x7f7fffffull},
+    {2, 0x3f800000ull, 0x800000ull, 0x800000ull, 0x3f800000ull},
+    {2, 0xbf800000ull, 0x0ull, 0xbf800000ull, 0x0ull},
+    {2, 0xbf800000ull, 0x80000000ull, 0x3f800000ull, 0x80000000ull},
+    {2, 0xbf800000ull, 0x3f800000ull, 0x7fc00000ull, 0x3f800000ull},
+    {2, 0xbf800000ull, 0xbf800000ull, 0xbf800000ull, 0xbf800000ull},
+    {2, 0xbf800000ull, 0x3fc00000ull, 0xbf800000ull, 0x3fc00000ull},
+    {2, 0xbf800000ull, 0xc0200000ull, 0x3f800000ull, 0xbf800000ull},
+    {2, 0xbf800000ull, 0x7f800000ull, 0x1ull, 0x7f800000ull},
+    {2, 0xbf800000ull, 0xff800000ull, 0xff800000ull, 0xbf800000ull},
+    {2, 0xbf800000ull, 0x7fc00000ull, 0xbf800000ull, 0xbf800000ull},
+    {2, 0xbf800000ull, 0xffc12345ull, 0x3f800000ull, 0xbf800000ull},
+    {2, 0xbf800000ull, 0x7fa00000ull, 0x0ull, 0x7fe00000ull},
+    {2, 0xbf800000ull, 0xff800001ull, 0xff800001ull, 0xffc00001ull},
+    {2, 0xbf800000ull, 0x1ull, 0xbf800000ull, 0x1ull},
+    {2, 0xbf800000ull, 0x807fffffull, 0x3f800000ull, 0x807fffffull},
+    {2, 0xbf800000ull, 0x7f7fffffull, 0x3fc00000ull, 0x7f7fffffull},
+    {2, 0xbf800000ull, 0x800000ull, 0x800000ull, 0x800000ull},
+    {3, 0x0ull, 0x0ull, 0x0ull, 0x0ull},
+    {3, 0x0ull, 0x8000000000000000ull, 0x8000000000000000ull, 0x8000000000000000ull},
+    {3, 0x0ull, 0x3ff0000000000000ull, 0xc004000000000000ull, 0x0ull},
+    {3, 0x0ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0x0ull},
+    {3, 0x0ull, 0x3ff8000000000000ull, 0x0ull, 0x3ff8000000000000ull},
+    {3, 0x0ull, 0x7ff0000000000000ull, 0xfff8000012345678ull, 0x0ull},
+    {3, 0x0ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0x0ull},
+    {3, 0x0ull, 0x7ff4000000000000ull, 0x800fffffffffffffull, 0x0ull},
+    {3, 0x0ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0x0ull},
+    {3, 0x0ull, 0x10000000000000ull, 0x10000000000000ull, 0x0ull},
+    {3, 0x8000000000000000ull, 0x0ull, 0x8000000000000000ull, 0x0ull},
+    {3, 0x8000000000000000ull, 0x8000000000000000ull, 0x0ull, 0x8000000000000000ull},
+    {3, 0x8000000000000000ull, 0x3ff0000000000000ull, 0x7ff0000000000000ull, 0x8000000000000000ull},
+    {3, 0x8000000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0x8000000000000000ull},
+    {3, 0x8000000000000000ull, 0xc004000000000000ull, 0x0ull, 0xc004000000000000ull},
+    {3, 0x8000000000000000ull, 0x7ff0000000000000ull, 0x7ff4000000000000ull, 0x8000000000000000ull},
+    {3, 0x8000000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0x8000000000000000ull},
+    {3, 0x8000000000000000ull, 0x7ff4000000000000ull, 0x7fefffffffffffffull, 0x8000000000000000ull},
+    {3, 0x8000000000000000ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0x8000000000000000ull},
+    {3, 0x8000000000000000ull, 0x7fefffffffffffffull, 0x3ff0000000000000ull, 0x8000000000000000ull},
+    {3, 0x8000000000000000ull, 0x10000000000000ull, 0x10000000000000ull, 0x8000000000000000ull},
+    {3, 0x3ff0000000000000ull, 0x0ull, 0x3ff0000000000000ull, 0x0ull},
+    {3, 0x3ff0000000000000ull, 0x8000000000000000ull, 0xbff0000000000000ull, 0x3ff0000000000000ull},
+    {3, 0x3ff0000000000000ull, 0x3ff0000000000000ull, 0xfff0000000000000ull, 0x3ff0000000000000ull},
+    {3, 0x3ff0000000000000ull, 0x7ff0000000000000ull, 0xfff0000000000001ull, 0x3ff0000000000000ull},
+    {3, 0x3ff0000000000000ull, 0x7ff4000000000000ull, 0x10000000000000ull, 0x3ff0000000000000ull},
+    {3, 0xbff0000000000000ull, 0x0ull, 0xbff0000000000000ull, 0x0ull},
+    {3, 0xbff0000000000000ull, 0x8000000000000000ull, 0x3ff0000000000000ull, 0xbff0000000000000ull},
+    {3, 0xbff0000000000000ull, 0x3ff0000000000000ull, 0x7ff8000000000000ull, 0xbff0000000000000ull},
+    {3, 0xbff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull},
+    {3, 0xbff0000000000000ull, 0x7ff0000000000000ull, 0x1ull, 0xbff0000000000000ull},
+    {3, 0xbff0000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0xbff0000000000000ull},
+    {3, 0xbff0000000000000ull, 0x7ff4000000000000ull, 0x0ull, 0xbff0000000000000ull},
+    {3, 0xbff0000000000000ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0xbff0000000000000ull},
+    {3, 0xbff0000000000000ull, 0x7fefffffffffffffull, 0x3ff8000000000000ull, 0xbff0000000000000ull},
+    {3, 0xbff0000000000000ull, 0x10000000000000ull, 0x10000000000000ull, 0xbff0000000000000ull},
+    {3, 0x3ff8000000000000ull, 0x0ull, 0x3ff8000000000000ull, 0x0ull},
+    {3, 0x3ff8000000000000ull, 0x8000000000000000ull, 0xbff8000000000000ull, 0x3ff8000000000000ull},
+    {3, 0x3ff8000000000000ull, 0x3ff0000000000000ull, 0xfff8000012345678ull, 0x3ff8000000000000ull},
+    {3, 0x3ff8000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0x3ff8000000000000ull},
+    {3, 0x3ff8000000000000ull, 0x3ff8000000000000ull, 0x3ff8000000000000ull, 0x3ff8000000000000ull},
+    {3, 0x3ff8000000000000ull, 0x7ff0000000000000ull, 0x800fffffffffffffull, 0x3ff8000000000000ull},
+    {3, 0x3ff8000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0x3ff8000000000000ull},
+    {3, 0x3ff8000000000000ull, 0x7ff4000000000000ull, 0x8000000000000000ull, 0x3ff8000000000000ull},
+    {3, 0x3ff8000000000000ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0x3ff8000000000000ull},
+    {3, 0x3ff8000000000000ull, 0x7fefffffffffffffull, 0xc004000000000000ull, 0x3ff8000000000000ull},
+    {3, 0x3ff8000000000000ull, 0x10000000000000ull, 0x10000000000000ull, 0x3ff8000000000000ull},
+    {3, 0xc004000000000000ull, 0x0ull, 0xc004000000000000ull, 0x0ull},
+    {3, 0xc004000000000000ull, 0x8000000000000000ull, 0x4004000000000000ull, 0xc004000000000000ull},
+    {3, 0xc004000000000000ull, 0x3ff0000000000000ull, 0x7ff4000000000000ull, 0xc004000000000000ull},
+    {3, 0xc004000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0xc004000000000000ull},
+    {3, 0xc004000000000000ull, 0x7ff0000000000000ull, 0x7fefffffffffffffull, 0xc004000000000000ull},
+    {3, 0xc004000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0xc004000000000000ull},
+    {3, 0xc004000000000000ull, 0x7ff4000000000000ull, 0x3ff0000000000000ull, 0xc004000000000000ull},
+    {3, 0xc004000000000000ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0xc004000000000000ull},
+    {3, 0xc004000000000000ull, 0x7fefffffffffffffull, 0x7ff0000000000000ull, 0xc004000000000000ull},
+    {3, 0xc004000000000000ull, 0x10000000000000ull, 0x10000000000000ull, 0xc004000000000000ull},
+    {3, 0x7ff0000000000000ull, 0x0ull, 0x7ff0000000000000ull, 0x0ull},
+    {3, 0x7ff0000000000000ull, 0x8000000000000000ull, 0xfff0000000000000ull, 0x7ff0000000000000ull},
+    {3, 0x7ff0000000000000ull, 0x3ff0000000000000ull, 0xfff0000000000001ull, 0x7ff0000000000000ull},
+    {3, 0x7ff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0x7ff0000000000000ull},
+    {3, 0x7ff0000000000000ull, 0x7ff0000000000000ull, 0x10000000000000ull, 0x7ff0000000000000ull},
+    {3, 0xfff0000000000000ull, 0x0ull, 0xfff0000000000000ull, 0x0ull},
+    {3, 0xfff0000000000000ull, 0x8000000000000000ull, 0x7ff0000000000000ull, 0xfff0000000000000ull},
+    {4, 0x0ull, 0x0ull, 0x0ull, 0x0ull},
+    {4, 0x0ull, 0x8000000000000000ull, 0x8000000000000000ull, 0x8000000000000000ull},
+    {4, 0x0ull, 0x3ff0000000000000ull, 0xc004000000000000ull, 0x0ull},
+    {4, 0x0ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull},
+    {4, 0x0ull, 0x3ff8000000000000ull, 0x0ull, 0x0ull},
+    {4, 0x0ull, 0xc004000000000000ull, 0x8000000000000000ull, 0xc004000000000000ull},
+    {4, 0x0ull, 0x7ff0000000000000ull, 0xfff8000012345678ull, 0x0ull},
+    {4, 0x0ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull},
+    {4, 0x0ull, 0x7ff8000000000000ull, 0x0ull, 0x0ull},
+    {4, 0x0ull, 0xfff8000012345678ull, 0x8000000000000000ull, 0x0ull},
+    {4, 0x0ull, 0x7ff4000000000000ull, 0x800fffffffffffffull, 0x7ffc000000000000ull},
+    {4, 0x0ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0xfff8000000000001ull},
+    {4, 0x0ull, 0x1ull, 0x0ull, 0x0ull},
+    {4, 0x0ull, 0x800fffffffffffffull, 0x8000000000000000ull, 0x800fffffffffffffull},
+    {4, 0x0ull, 0x7fefffffffffffffull, 0x8000000000000000ull, 0x0ull},
+    {4, 0x0ull, 0x10000000000000ull, 0x10000000000000ull, 0x0ull},
+    {4, 0x8000000000000000ull, 0x0ull, 0x8000000000000000ull, 0x8000000000000000ull},
+    {4, 0x8000000000000000ull, 0x8000000000000000ull, 0x0ull, 0x8000000000000000ull},
+    {4, 0x8000000000000000ull, 0x3ff0000000000000ull, 0x7ff0000000000000ull, 0x8000000000000000ull},
+    {4, 0x8000000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull},
+    {4, 0x8000000000000000ull, 0x3ff8000000000000ull, 0x8000000000000000ull, 0x8000000000000000ull},
+    {4, 0x8000000000000000ull, 0xc004000000000000ull, 0x0ull, 0xc004000000000000ull},
+    {4, 0x8000000000000000ull, 0x7ff0000000000000ull, 0x7ff4000000000000ull, 0x8000000000000000ull},
+    {4, 0x8000000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull},
+    {4, 0x8000000000000000ull, 0x7ff8000000000000ull, 0x8000000000000000ull, 0x8000000000000000ull},
+    {4, 0x8000000000000000ull, 0xfff8000012345678ull, 0x0ull, 0x8000000000000000ull},
+    {4, 0x8000000000000000ull, 0x7ff4000000000000ull, 0x7fefffffffffffffull, 0x7ffc000000000000ull},
+    {4, 0x8000000000000000ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0xfff8000000000001ull},
+    {4, 0x8000000000000000ull, 0x1ull, 0x8000000000000000ull, 0x8000000000000000ull},
+    {4, 0x8000000000000000ull, 0x800fffffffffffffull, 0x0ull, 0x800fffffffffffffull},
+    {4, 0x8000000000000000ull, 0x7fefffffffffffffull, 0x3ff0000000000000ull, 0x8000000000000000ull},
+    {4, 0x8000000000000000ull, 0x10000000000000ull, 0x10000000000000ull, 0x8000000000000000ull},
+    {4, 0x3ff0000000000000ull, 0x0ull, 0x3ff0000000000000ull, 0x0ull},
+    {4, 0x3ff0000000000000ull, 0x8000000000000000ull, 0xbff0000000000000ull, 0x8000000000000000ull},
+    {4, 0x3ff0000000000000ull, 0x3ff0000000000000ull, 0xfff0000000000000ull, 0x3ff0000000000000ull},
+    {4, 0x3ff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull},
+    {4, 0x3ff0000000000000ull, 0x3ff8000000000000ull, 0x3ff0000000000000ull, 0x3ff0000000000000ull},
+    {4, 0x3ff0000000000000ull, 0xc004000000000000ull, 0xbff0000000000000ull, 0xc004000000000000ull},
+    {4, 0x3ff0000000000000ull, 0x7ff0000000000000ull, 0xfff0000000000001ull, 0x3ff0000000000000ull},
+    {4, 0x3ff0000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull},
+    {4, 0x3ff0000000000000ull, 0x7ff8000000000000ull, 0x3ff0000000000000ull, 0x3ff0000000000000ull},
+    {4, 0x3ff0000000000000ull, 0xfff8000012345678ull, 0xbff0000000000000ull, 0x3ff0000000000000ull},
+    {4, 0x3ff0000000000000ull, 0x7ff4000000000000ull, 0x10000000000000ull, 0x7ffc000000000000ull},
+    {4, 0x3ff0000000000000ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0xfff8000000000001ull},
+    {4, 0x3ff0000000000000ull, 0x1ull, 0x3ff0000000000000ull, 0x1ull},
+    {4, 0x3ff0000000000000ull, 0x800fffffffffffffull, 0xbff0000000000000ull, 0x800fffffffffffffull},
+    {4, 0x3ff0000000000000ull, 0x7fefffffffffffffull, 0xbff0000000000000ull, 0x3ff0000000000000ull},
+    {4, 0x3ff0000000000000ull, 0x10000000000000ull, 0x10000000000000ull, 0x10000000000000ull},
+    {4, 0xbff0000000000000ull, 0x0ull, 0xbff0000000000000ull, 0xbff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0x8000000000000000ull, 0x3ff0000000000000ull, 0xbff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0x3ff0000000000000ull, 0x7ff8000000000000ull, 0xbff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0x3ff8000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0xc004000000000000ull, 0x3ff0000000000000ull, 0xc004000000000000ull},
+    {4, 0xbff0000000000000ull, 0x7ff0000000000000ull, 0x1ull, 0xbff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0x7ff8000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0xfff8000012345678ull, 0x3ff0000000000000ull, 0xbff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0x7ff4000000000000ull, 0x0ull, 0x7ffc000000000000ull},
+    {4, 0xbff0000000000000ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0xfff8000000000001ull},
+    {4, 0xbff0000000000000ull, 0x1ull, 0xbff0000000000000ull, 0xbff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0x800fffffffffffffull, 0x3ff0000000000000ull, 0xbff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0x7fefffffffffffffull, 0x3ff8000000000000ull, 0xbff0000000000000ull},
+    {4, 0xbff0000000000000ull, 0x10000000000000ull, 0x10000000000000ull, 0xbff0000000000000ull},
+    {5, 0x0ull, 0x0ull, 0x0ull, 0x0ull},
+    {5, 0x0ull, 0x8000000000000000ull, 0x8000000000000000ull, 0x0ull},
+    {5, 0x0ull, 0x3ff0000000000000ull, 0xc004000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x0ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0x0ull},
+    {5, 0x0ull, 0x3ff8000000000000ull, 0x0ull, 0x3ff8000000000000ull},
+    {5, 0x0ull, 0xc004000000000000ull, 0x8000000000000000ull, 0x0ull},
+    {5, 0x0ull, 0x7ff0000000000000ull, 0xfff8000012345678ull, 0x7ff0000000000000ull},
+    {5, 0x0ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0x0ull},
+    {5, 0x0ull, 0x7ff8000000000000ull, 0x0ull, 0x0ull},
+    {5, 0x0ull, 0xfff8000012345678ull, 0x8000000000000000ull, 0x0ull},
+    {5, 0x0ull, 0x7ff4000000000000ull, 0x800fffffffffffffull, 0x7ffc000000000000ull},
+    {5, 0x0ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0xfff8000000000001ull},
+    {5, 0x0ull, 0x1ull, 0x0ull, 0x1ull},
+    {5, 0x0ull, 0x800fffffffffffffull, 0x8000000000000000ull, 0x0ull},
+    {5, 0x0ull, 0x7fefffffffffffffull, 0x8000000000000000ull, 0x7fefffffffffffffull},
+    {5, 0x0ull, 0x10000000000000ull, 0x10000000000000ull, 0x10000000000000ull},
+    {5, 0x8000000000000000ull, 0x0ull, 0x8000000000000000ull, 0x0ull},
+    {5, 0x8000000000000000ull, 0x8000000000000000ull, 0x0ull, 0x8000000000000000ull},
+    {5, 0x8000000000000000ull, 0x3ff0000000000000ull, 0x7ff0000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x8000000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0x8000000000000000ull},
+    {5, 0x8000000000000000ull, 0x3ff8000000000000ull, 0x8000000000000000ull, 0x3ff8000000000000ull},
+    {5, 0x8000000000000000ull, 0xc004000000000000ull, 0x0ull, 0x8000000000000000ull},
+    {5, 0x8000000000000000ull, 0x7ff0000000000000ull, 0x7ff4000000000000ull, 0x7ff0000000000000ull},
+    {5, 0x8000000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0x8000000000000000ull},
+    {5, 0x8000000000000000ull, 0x7ff8000000000000ull, 0x8000000000000000ull, 0x8000000000000000ull},
+    {5, 0x8000000000000000ull, 0xfff8000012345678ull, 0x0ull, 0x8000000000000000ull},
+    {5, 0x8000000000000000ull, 0x7ff4000000000000ull, 0x7fefffffffffffffull, 0x7ffc000000000000ull},
+    {5, 0x8000000000000000ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0xfff8000000000001ull},
+    {5, 0x8000000000000000ull, 0x1ull, 0x8000000000000000ull, 0x1ull},
+    {5, 0x8000000000000000ull, 0x800fffffffffffffull, 0x0ull, 0x8000000000000000ull},
+    {5, 0x8000000000000000ull, 0x7fefffffffffffffull, 0x3ff0000000000000ull, 0x7fefffffffffffffull},
+    {5, 0x8000000000000000ull, 0x10000000000000ull, 0x10000000000000ull, 0x10000000000000ull},
+    {5, 0x3ff0000000000000ull, 0x0ull, 0x3ff0000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x3ff0000000000000ull, 0x8000000000000000ull, 0xbff0000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x3ff0000000000000ull, 0x3ff0000000000000ull, 0xfff0000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x3ff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x3ff0000000000000ull, 0x3ff8000000000000ull, 0x3ff0000000000000ull, 0x3ff8000000000000ull},
+    {5, 0x3ff0000000000000ull, 0xc004000000000000ull, 0xbff0000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x3ff0000000000000ull, 0x7ff0000000000000ull, 0xfff0000000000001ull, 0x7ff0000000000000ull},
+    {5, 0x3ff0000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x3ff0000000000000ull, 0x7ff8000000000000ull, 0x3ff0000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x3ff0000000000000ull, 0xfff8000012345678ull, 0xbff0000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x3ff0000000000000ull, 0x7ff4000000000000ull, 0x10000000000000ull, 0x7ffc000000000000ull},
+    {5, 0x3ff0000000000000ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0xfff8000000000001ull},
+    {5, 0x3ff0000000000000ull, 0x1ull, 0x3ff0000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x3ff0000000000000ull, 0x800fffffffffffffull, 0xbff0000000000000ull, 0x3ff0000000000000ull},
+    {5, 0x3ff0000000000000ull, 0x7fefffffffffffffull, 0xbff0000000000000ull, 0x7fefffffffffffffull},
+    {5, 0x3ff0000000000000ull, 0x10000000000000ull, 0x10000000000000ull, 0x3ff0000000000000ull},
+    {5, 0xbff0000000000000ull, 0x0ull, 0xbff0000000000000ull, 0x0ull},
+    {5, 0xbff0000000000000ull, 0x8000000000000000ull, 0x3ff0000000000000ull, 0x8000000000000000ull},
+    {5, 0xbff0000000000000ull, 0x3ff0000000000000ull, 0x7ff8000000000000ull, 0x3ff0000000000000ull},
+    {5, 0xbff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull},
+    {5, 0xbff0000000000000ull, 0x3ff8000000000000ull, 0xbff0000000000000ull, 0x3ff8000000000000ull},
+    {5, 0xbff0000000000000ull, 0xc004000000000000ull, 0x3ff0000000000000ull, 0xbff0000000000000ull},
+    {5, 0xbff0000000000000ull, 0x7ff0000000000000ull, 0x1ull, 0x7ff0000000000000ull},
+    {5, 0xbff0000000000000ull, 0xfff0000000000000ull, 0xfff0000000000000ull, 0xbff0000000000000ull},
+    {5, 0xbff0000000000000ull, 0x7ff8000000000000ull, 0xbff0000000000000ull, 0xbff0000000000000ull},
+    {5, 0xbff0000000000000ull, 0xfff8000012345678ull, 0x3ff0000000000000ull, 0xbff0000000000000ull},
+    {5, 0xbff0000000000000ull, 0x7ff4000000000000ull, 0x0ull, 0x7ffc000000000000ull},
+    {5, 0xbff0000000000000ull, 0xfff0000000000001ull, 0xfff0000000000001ull, 0xfff8000000000001ull},
+    {5, 0xbff0000000000000ull, 0x1ull, 0xbff0000000000000ull, 0x1ull},
+    {5, 0xbff0000000000000ull, 0x800fffffffffffffull, 0x3ff0000000000000ull, 0x800fffffffffffffull},
+    {5, 0xbff0000000000000ull, 0x7fefffffffffffffull, 0x3ff8000000000000ull, 0x7fefffffffffffffull},
+    {5, 0xbff0000000000000ull, 0x10000000000000ull, 0x10000000000000ull, 0x10000000000000ull},
+    {6, 0x0ull, 0x0ull, 0x0ull, 0x0ull},
+    {6, 0x0ull, 0x1ull, 0x8000000000000000ull, 0x1ull},
+    {6, 0x0ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x1ull},
+    {6, 0x1ull, 0x0ull, 0x1ull, 0x0ull},
+    {6, 0x1ull, 0x1ull, 0x8000000000000001ull, 0x0ull},
+    {6, 0x1ull, 0x2ull, 0xfffffffffffffffeull, 0x2ull},
+    {6, 0x1ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x2ull},
+    {6, 0x7fffffffffffffffull, 0x0ull, 0x7fffffffffffffffull, 0x0ull},
+    {6, 0x0ull, 0x2ull, 0xffffffffffffffffull, 0x1ull},
+    {6, 0x0ull, 0x8000000000000000ull, 0x0ull, 0x1ull},
+    {6, 0x0ull, 0xffffffffffffffffull, 0x8000000000000000ull, 0x1ull},
+    {6, 0x0ull, 0xfffffffffffffffeull, 0x100000001ull, 0x1ull},
+    {6, 0x0ull, 0xffffffffull, 0xffffffffull, 0x1ull},
+    {6, 0x0ull, 0x100000000ull, 0x0ull, 0x1ull},
+    {6, 0x0ull, 0x100000001ull, 0x8000000000000000ull, 0x1ull},
+    {6, 0x0ull, 0x123456789abcdef0ull, 0x3ull, 0x1ull},
+    {6, 0x0ull, 0xfffffffeffffffffull, 0xfffffffeffffffffull, 0x1ull},
+    {6, 0x0ull, 0xfffffffeull, 0x0ull, 0x1ull},
+    {6, 0x0ull, 0x3ull, 0x8000000000000000ull, 0x1ull},
+    {6, 0x0ull, 0x80000000ull, 0x1ull, 0x1ull},
+    {6, 0x0ull, 0x7fffffffull, 0x7fffffffull, 0x1ull},
+    {6, 0x1ull, 0x8000000000000000ull, 0x1ull, 0x2ull},
+    {6, 0x1ull, 0xffffffffffffffffull, 0x8000000000000001ull, 0x2ull},
+    {6, 0x1ull, 0xfffffffffffffffeull, 0x123456789abcdef0ull, 0x2ull},
+    {6, 0x1ull, 0xffffffffull, 0xffffffffull, 0x2ull},
+    {6, 0x1ull, 0x100000000ull, 0x1ull, 0x2ull},
+    {6, 0x1ull, 0x100000001ull, 0x8000000000000001ull, 0x2ull},
+    {6, 0x1ull, 0x123456789abcdef0ull, 0x80000000ull, 0x2ull},
+    {6, 0x1ull, 0xfffffffeffffffffull, 0xfffffffeffffffffull, 0x2ull},
+    {6, 0x1ull, 0xfffffffeull, 0x1ull, 0x2ull},
+    {6, 0x1ull, 0x3ull, 0x8000000000000001ull, 0x2ull},
+    {6, 0x1ull, 0x80000000ull, 0x2ull, 0x2ull},
+    {6, 0x1ull, 0x7fffffffull, 0x7fffffffull, 0x2ull},
+    {6, 0x2ull, 0x0ull, 0x2ull, 0x0ull},
+    {6, 0x2ull, 0x1ull, 0x8000000000000002ull, 0x0ull},
+    {6, 0x2ull, 0x2ull, 0xffffffffull, 0x0ull},
+    {6, 0x2ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x3ull},
+    {6, 0x2ull, 0x8000000000000000ull, 0x2ull, 0x3ull},
+    {6, 0x2ull, 0xffffffffffffffffull, 0x8000000000000002ull, 0x3ull},
+    {6, 0x2ull, 0xfffffffffffffffeull, 0xfffffffeffffffffull, 0x3ull},
+    {6, 0x2ull, 0xffffffffull, 0xffffffffull, 0x3ull},
+    {6, 0x2ull, 0x100000000ull, 0x2ull, 0x3ull},
+    {6, 0x2ull, 0x100000001ull, 0x8000000000000002ull, 0x3ull},
+    {6, 0x2ull, 0x123456789abcdef0ull, 0x7fffffffull, 0x3ull},
+    {6, 0x2ull, 0xfffffffeffffffffull, 0xfffffffeffffffffull, 0x3ull},
+    {6, 0x2ull, 0xfffffffeull, 0x2ull, 0x3ull},
+    {6, 0x2ull, 0x3ull, 0x8000000000000002ull, 0x3ull},
+    {6, 0x2ull, 0x80000000ull, 0x7fffffffffffffffull, 0x3ull},
+    {6, 0x2ull, 0x7fffffffull, 0x7fffffffull, 0x3ull},
+    {6, 0x7fffffffffffffffull, 0x1ull, 0xffffffffffffffffull, 0x0ull},
+    {6, 0x7fffffffffffffffull, 0x2ull, 0x100000000ull, 0x0ull},
+    {6, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x0ull},
+    {6, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x8000000000000000ull},
+    {6, 0x7fffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0x8000000000000000ull},
+    {6, 0x7fffffffffffffffull, 0xfffffffffffffffeull, 0xfffffffeull, 0x8000000000000000ull},
+    {6, 0x7fffffffffffffffull, 0xffffffffull, 0xffffffffull, 0x0ull},
+    {6, 0x7fffffffffffffffull, 0x100000000ull, 0x7fffffffffffffffull, 0x0ull},
+    {6, 0x7fffffffffffffffull, 0x100000001ull, 0xffffffffffffffffull, 0x0ull},
+    {6, 0x7fffffffffffffffull, 0x123456789abcdef0ull, 0x0ull, 0x0ull},
+    {6, 0x7fffffffffffffffull, 0xfffffffeffffffffull, 0xfffffffeffffffffull, 0x8000000000000000ull},
+    {6, 0x7fffffffffffffffull, 0xfffffffeull, 0x7fffffffffffffffull, 0x0ull},
+    {6, 0x7fffffffffffffffull, 0x3ull, 0xffffffffffffffffull, 0x0ull},
+    {6, 0x7fffffffffffffffull, 0x80000000ull, 0x8000000000000000ull, 0x0ull},
+    {6, 0x7fffffffffffffffull, 0x7fffffffull, 0x7fffffffull, 0x0ull},
+    {7, 0x0ull, 0x0ull, 0x0ull, 0x0ull},
+    {7, 0x0ull, 0x1ull, 0x8000000000000000ull, 0x1ull},
+    {7, 0x0ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x7fffffffffffffffull},
+    {7, 0x1ull, 0x0ull, 0x1ull, 0x0ull},
+    {7, 0x1ull, 0x1ull, 0x8000000000000001ull, 0x0ull},
+    {7, 0x1ull, 0x2ull, 0xfffffffffffffffeull, 0x0ull},
+    {7, 0x1ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x0ull},
+    {7, 0x7fffffffffffffffull, 0x0ull, 0x7fffffffffffffffull, 0x0ull},
+    {7, 0x0ull, 0x2ull, 0xffffffffffffffffull, 0x2ull},
+    {7, 0x0ull, 0x8000000000000000ull, 0x0ull, 0x8000000000000000ull},
+    {7, 0x0ull, 0xffffffffffffffffull, 0x8000000000000000ull, 0xffffffffffffffffull},
+    {7, 0x0ull, 0xfffffffffffffffeull, 0x100000001ull, 0xfffffffffffffffeull},
+    {7, 0x0ull, 0xffffffffull, 0xffffffffull, 0xffffffffull},
+    {7, 0x0ull, 0x100000000ull, 0x0ull, 0x100000000ull},
+    {7, 0x0ull, 0x100000001ull, 0x8000000000000000ull, 0x100000001ull},
+    {7, 0x0ull, 0x123456789abcdef0ull, 0x3ull, 0x123456789abcdef0ull},
+    {7, 0x0ull, 0xfffffffeffffffffull, 0xfffffffeffffffffull, 0xfffffffeffffffffull},
+    {7, 0x0ull, 0xfffffffeull, 0x0ull, 0xfffffffeull},
+    {7, 0x0ull, 0x3ull, 0x8000000000000000ull, 0x3ull},
+    {7, 0x0ull, 0x80000000ull, 0x1ull, 0x80000000ull},
+    {7, 0x0ull, 0x7fffffffull, 0x7fffffffull, 0x7fffffffull},
+    {7, 0x1ull, 0x8000000000000000ull, 0x1ull, 0x0ull},
+    {7, 0x1ull, 0xffffffffffffffffull, 0x8000000000000001ull, 0x0ull},
+    {7, 0x1ull, 0xfffffffffffffffeull, 0x123456789abcdef0ull, 0x0ull},
+    {7, 0x1ull, 0xffffffffull, 0xffffffffull, 0x0ull},
+    {7, 0x1ull, 0x100000000ull, 0x1ull, 0x0ull},
+    {7, 0x1ull, 0x100000001ull, 0x8000000000000001ull, 0x0ull},
+    {7, 0x1ull, 0x123456789abcdef0ull, 0x80000000ull, 0x0ull},
+    {7, 0x1ull, 0xfffffffeffffffffull, 0xfffffffeffffffffull, 0x0ull},
+    {7, 0x1ull, 0xfffffffeull, 0x1ull, 0x0ull},
+    {7, 0x1ull, 0x3ull, 0x8000000000000001ull, 0x0ull},
+    {7, 0x1ull, 0x80000000ull, 0x2ull, 0x0ull},
+    {7, 0x1ull, 0x7fffffffull, 0x7fffffffull, 0x0ull},
+    {7, 0x2ull, 0x0ull, 0x2ull, 0x0ull},
+    {7, 0x2ull, 0x1ull, 0x8000000000000002ull, 0x1ull},
+    {7, 0x2ull, 0x2ull, 0xffffffffull, 0x1ull},
+    {7, 0x2ull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x1ull},
+    {7, 0x2ull, 0x8000000000000000ull, 0x2ull, 0x1ull},
+    {7, 0x2ull, 0xffffffffffffffffull, 0x8000000000000002ull, 0x1ull},
+    {7, 0x2ull, 0xfffffffffffffffeull, 0xfffffffeffffffffull, 0x1ull},
+    {7, 0x2ull, 0xffffffffull, 0xffffffffull, 0x1ull},
+    {7, 0x2ull, 0x100000000ull, 0x2ull, 0x1ull},
+    {7, 0x2ull, 0x100000001ull, 0x8000000000000002ull, 0x1ull},
+    {7, 0x2ull, 0x123456789abcdef0ull, 0x7fffffffull, 0x1ull},
+    {7, 0x2ull, 0xfffffffeffffffffull, 0xfffffffeffffffffull, 0x1ull},
+    {7, 0x2ull, 0xfffffffeull, 0x2ull, 0x1ull},
+    {7, 0x2ull, 0x3ull, 0x8000000000000002ull, 0x1ull},
+    {7, 0x2ull, 0x80000000ull, 0x7fffffffffffffffull, 0x1ull},
+    {7, 0x2ull, 0x7fffffffull, 0x7fffffffull, 0x1ull},
+    {7, 0x7fffffffffffffffull, 0x1ull, 0xffffffffffffffffull, 0x1ull},
+    {7, 0x7fffffffffffffffull, 0x2ull, 0x100000000ull, 0x2ull},
+    {7, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x7fffffffffffffffull, 0x7ffffffffffffffeull},
+    {7, 0x7fffffffffffffffull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x7ffffffffffffffeull},
+    {7, 0x7fffffffffffffffull, 0xffffffffffffffffull, 0xffffffffffffffffull, 0x7ffffffffffffffeull},
+    {7, 0x7fffffffffffffffull, 0xfffffffffffffffeull, 0xfffffffeull, 0x7ffffffffffffffeull},
+    {7, 0x7fffffffffffffffull, 0xffffffffull, 0xffffffffull, 0xffffffffull},
+    {7, 0x7fffffffffffffffull, 0x100000000ull, 0x7fffffffffffffffull, 0x100000000ull},
+    {7, 0x7fffffffffffffffull, 0x100000001ull, 0xffffffffffffffffull, 0x100000001ull},
+    {7, 0x7fffffffffffffffull, 0x123456789abcdef0ull, 0x0ull, 0x123456789abcdef0ull},
+    {7, 0x7fffffffffffffffull, 0xfffffffeffffffffull, 0xfffffffeffffffffull, 0x7ffffffffffffffeull},
+    {7, 0x7fffffffffffffffull, 0xfffffffeull, 0x7fffffffffffffffull, 0xfffffffeull},
+    {7, 0x7fffffffffffffffull, 0x3ull, 0xffffffffffffffffull, 0x3ull},
+    {7, 0x7fffffffffffffffull, 0x80000000ull, 0x8000000000000000ull, 0x80000000ull},
+    {7, 0x7fffffffffffffffull, 0x7fffffffull, 0x7fffffffull, 0x7fffffffull}
+}};
+
+class GuestBlock {
+public:
+    explicit GuestBlock(bool writable) {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
+#endif
+        Require(block != nullptr, "global float atomics: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, writable);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+    const std::uint8_t* Data() const { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+void Put(std::vector<std::uint8_t>& image, std::uint32_t offset, std::uint64_t value, std::uint32_t bytes) {
+    for (std::uint32_t byte = 0; byte < bytes; ++byte) image.at(offset + byte) = static_cast<std::uint8_t>(value >> (byte * 8u));
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint64_t value) {
+    char text[24];
+    std::snprintf(text, sizeof(text), "0x%llx", static_cast<unsigned long long>(value));
+    return text;
+}
+
+const Row& RowOf(std::uint32_t op, std::uint32_t tid) {
+    return Rows.at(op * Threads + tid);
+}
+
+std::uint32_t OpBytes(std::uint32_t op) {
+    return op < NarrowOps ? 4u : 8u;
+}
+
+std::vector<std::uint8_t> Initial() {
+    std::vector<std::uint8_t> image(BlockBytes, Fill);
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            Put(image, op * OpStride + tid * 8u, RowOf(op, tid).memory, OpBytes(op));
+        }
+    }
+    return image;
+}
+
+std::vector<std::uint8_t> Expected() {
+    auto image = Initial();
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            Put(image, op * OpStride + tid * 8u, RowOf(op, tid).final, OpBytes(op));
+        }
+    }
+    return image;
+}
+
+void FillInput() {
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            const auto& row = RowOf(op, tid);
+            auto* words = &Input[(op * Threads + tid) * 4u];
+            if (OpBytes(op) == 4u) {
+                words[0] = static_cast<std::uint32_t>(row.data);
+                words[1] = static_cast<std::uint32_t>(row.comparator);
+                words[2] = 0u;
+                words[3] = 0u;
+            } else {
+                words[0] = static_cast<std::uint32_t>(row.data);
+                words[1] = static_cast<std::uint32_t>(row.data >> 32u);
+                words[2] = static_cast<std::uint32_t>(row.comparator);
+                words[3] = static_cast<std::uint32_t>(row.comparator >> 32u);
+            }
+        }
+    }
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, const std::uint8_t* base) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(base));
+    std::vector<std::uint32_t> userData(10, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    userData[8] = static_cast<std::uint32_t>(address);
+    userData[9] = static_cast<std::uint32_t>(address >> 32u);
+    const std::span<const std::uint32_t> code(GlobalAtomicsFloatCode);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void RunAtomics(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t waveSize) {
+    const auto initial = Initial();
+    std::memcpy(guest.Data(), initial.data(), BlockBytes);
+    FillInput();
+    Output.fill(Sentinel);
+    Dispatch(device, waveSize, guest.Data());
+    const auto wave = "global float atomics: wave" + std::to_string(waveSize) + " ";
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            const auto& row = RowOf(op, tid);
+            const auto* words = &Output[(op * Threads + tid) * 2u];
+            const auto returned = OpBytes(op) == 4u ? static_cast<std::uint64_t>(words[0]) : (static_cast<std::uint64_t>(words[1]) << 32u) | words[0];
+            Require(returned == row.memory, wave + "op " + std::to_string(op) + " lane " + std::to_string(tid) + " returned " + Hex(returned) + ", expected " + Hex(row.memory));
+        }
+    }
+    const auto expected = Expected();
+    for (std::uint32_t op = 0; op < OpCount; ++op) {
+        for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+            const auto offset = op * OpStride + tid * 8u;
+            std::uint64_t actual = 0;
+            std::memcpy(&actual, guest.Data() + offset, OpBytes(op));
+            const auto& row = RowOf(op, tid);
+            Require(actual == row.final, wave + "op " + std::to_string(op) + " lane " + std::to_string(tid) + " memory " + Hex(row.memory) + " data " + Hex(row.data) + " comparator " + Hex(row.comparator) + " is " + Hex(actual) + ", expected " + Hex(row.final));
+        }
+    }
+    for (std::uint32_t offset = 0; offset < BlockBytes; ++offset) {
+        const auto actual = guest.Data()[offset];
+        Require(actual == expected[offset], wave + "byte " + std::to_string(offset) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[offset]));
+    }
+}
+
+void RunReadOnly(AgcDriver::VulkanDevice& device, GuestBlock& guest) {
+    const auto initial = Initial();
+    std::memcpy(guest.Data(), initial.data(), BlockBytes);
+    FillInput();
+    Dispatch(device, 32, guest.Data());
+    for (std::uint32_t offset = 0; offset < BlockBytes; ++offset) {
+        Require(guest.Data()[offset] == initial[offset], "global float atomics: an atomic into a read-only range changed byte " + std::to_string(offset));
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock writable(true);
+        GuestBlock readOnly(false);
+        RunAtomics(*device, writable, 32);
+        RunAtomics(*device, writable, 64);
+        RunReadOnly(*device, readOnly);
+        std::puts("global float atomics tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}