#include "common/assert.h"
#include "common/emulatorConfig.h"
#include "common/file.h"
#include "common/hostException.h"
#include "common/logging/log.h"
#include "common/subsystems.h"
#include "common/threads.h"
#include "gpu_test_shaders/gpu_test_ms_depth_spv.h"
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/command_processor/pm4Dispatch.h"
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/guest_gpu/gpu_format.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/guest_gpu/hardwareContext.h"
#include "graphics/guest_gpu/pm4.h"
#include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/pageManager.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "graphics/host_gpu/renderer/colorRenderTarget.h"
#include "graphics/host_gpu/renderer/depthRenderTarget.h"
#include "graphics/host_gpu/renderer/image/blitHelper.h"
#include "graphics/host_gpu/renderer/image/image.h"
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "graphics/host_gpu/renderer/image/textureCommon.h"
#include "graphics/host_gpu/renderer/image/tiler.h"
#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderDraw.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
#include "graphics/host_gpu/renderer/sync.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include "graphics/presentation/window/windowInternal.h"
#include "graphics/shader/recompiler/BufferFormat.h"
#include "graphics/shader/recompiler/ShaderRecompiler.h"
#include "graphics/shader/recompiler/Tessellation.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
#include "graphics/shader/rectListShader.h"
#include "graphics/shader/shader.h"
#include "graphics/shader/shaderCompiler.h"
#include "kernel/eventQueue.h"
#include "kernel/memory.h"
#include "libs/agc.h"
#include "libs/dialog.h"
#include "libs/errno.h"
#include "loader/symbolDatabase.h"
#include "spirv-tools/libspirv.hpp"

#if __has_include("graphics/host_gpu/renderer/renderTargetBarriers.h")
#error "legacy render-target barrier API must remain deleted"
#endif

#if __has_include("graphics/host_gpu/transfer.h")
#error "legacy transfer facade must remain deleted"
#endif

#if __has_include("graphics/host_gpu/renderer/framebufferCache.h")
#error "classic framebuffer/render-pass cache must remain deleted"
#endif

#include <algorithm>
#include <array>
#include <atomic>
#include <bit>
#include <chrono>
#include <cinttypes>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <initializer_list>
#include <limits>
#include <memory>
#include <mutex>
#include <numeric>
#include <optional>
#include <semaphore>
#include <set>
#include <span>
#include <sstream>
#include <string>
#include <string_view>
#include <tuple>
#include <vector>

#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
#include <cerrno>
#include <sys/wait.h>
#include <unistd.h>
#endif

#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#undef min
#undef max
#endif

namespace Libs {
void InitAgcDriver_1(Loader::SymbolDatabase *symbols);
}

namespace Libs::Graphics {

template <typename Cache>
concept HasGetDownloadBuffer =
    requires(Cache &cache) { cache.GetDownloadBuffer(uint64_t{1}); };
static_assert(!HasGetDownloadBuffer<BufferCache>);
static_assert(std::same_as<decltype(&BufferCache::GetBuffer),
                           Buffer &(BufferCache::*)(BufferId)>);
static_assert(std::same_as<decltype(&TextureCache::GetImage),
                           Image &(TextureCache::*)(ImageId)>);

template <typename Cache>
concept HasSynchronizeImageToBuffer = requires(Cache &cache) {
  cache.SynchronizeImageToBuffer(uint64_t{1}, uint64_t{1});
};
template <typename Cache>
concept HasObtainBufferForImageCopy = requires(Cache &cache) {
  cache.ObtainBufferForImageCopy(uint64_t{1}, uint64_t{1});
};
template <typename Cache>
concept HasObtainBufferForImageWrite = requires(Cache &cache) {
  cache.ObtainBufferForImageWrite(uint64_t{1}, uint64_t{1});
};
template <typename Cache>
concept HasDiscardGpuDirtyBytes = requires(Cache &cache) {
  cache.DiscardGpuDirtyBytes(uint64_t{1}, uint64_t{1});
};
static_assert(!HasSynchronizeImageToBuffer<TextureCache>);
static_assert(!HasObtainBufferForImageCopy<BufferCache>);
static_assert(!HasObtainBufferForImageWrite<BufferCache>);
static_assert(!HasDiscardGpuDirtyBytes<BufferCache>);

template <typename Backing>
concept HasLegacyImageLayout = requires(Backing &backing) { backing.layout; };
static_assert(!HasLegacyImageLayout<VulkanImage>);

template <typename Manager>
concept HasPolicyDetileImage = requires { &Manager::DetileImage; };
template <typename Manager>
concept HasPolicyTileImageBacking = requires { &Manager::TileImageBacking; };
static_assert(!HasPolicyDetileImage<TileManager>);
static_assert(!HasPolicyTileImageBacking<TileManager>);
static_assert(BlitHelper::ColorToMsDepthLayout ==
              vk::ImageLayout::eDepthStencilAttachmentOptimal);

struct BufferCacheTestAccess {
  static_assert(std::same_as<decltype(BufferCache::m_slot_buffers),
                             Common::SlotVector<Buffer>>);

  static uint64_t GcTick(const BufferCache &cache) { return cache.m_gc_tick; }

  static void SetGarbageCollectionThresholds(BufferCache &cache,
                                             uint64_t trigger,
                                             uint64_t critical) {
    cache.m_trigger_gc_memory = trigger;
    cache.m_critical_gc_memory = critical;
  }

  static StreamBuffer &DownloadBuffer(BufferCache &cache) {
    return cache.m_download_buffer;
  }

  static void SetUnusedStagingBufferSize(BufferCache &cache, uint64_t size) {
    // Called immediately after cache construction, before any staging reservation.
    std::destroy_at(&cache.m_staging_buffer);
    std::construct_at(&cache.m_staging_buffer, cache.m_graphics, cache.m_scheduler,
                      MemoryUsage::Upload, size);
  }

  static bool SynchronizeBufferFromImage(BufferCache &cache, Buffer &buffer,
                                         uint64_t address, uint64_t size) {
    return cache.SynchronizeBufferFromImage(buffer, address, size);
  }

  static BufferId PageOwner(const BufferCache &cache, uint64_t address) {
    const auto *owner = cache.m_page_table.Find(
        address >> BufferCache::PageTable::kPageBits);
    return owner == nullptr ? BufferId{} : *owner;
  }

  static bool IsBufferAllocated(const BufferCache &cache, BufferId id) {
    return cache.m_slot_buffers.is_allocated(id);
  }
};

struct StreamBufferTestAccess {
  static bool NormalizeReservation(bool coherent, uint64_t atom, uint64_t &size,
                                   uint64_t &alignment) {
    return StreamBuffer::NormalizeReservation(coherent, atom, size, alignment);
  }
};

struct ImageTestAccess {
  static uint32_t CopyRows(uint64_t row_size, uint32_t rows,
                           uint64_t capacity) {
    return Image::CopyRows(row_size, rows, capacity);
  }
};

struct TileManagerTestAccess {
  static uint32_t ConversionRows(uint64_t offset, uint64_t row_stride,
                                 uint64_t active, uint32_t remaining,
                                 uint64_t alignment, uint64_t max_range,
                                 uint32_t max_groups) {
    return TileManager::ConversionRows(offset, row_stride, active, remaining,
                                       alignment, max_range, max_groups);
  }
};

struct TextureCacheTestAccess {
  static_assert(std::same_as<decltype(TextureCache::m_slot_images),
                             Common::SlotVector<Image>>);
  static_assert(TextureCache::ImagePageTable::kPageBits == 20);
  static_assert(TextureCache::ImagePageTable::kAddressSpaceBits == 44);
  static_assert(TextureCache::ImagePageTable::kFirstLevelBits == 14);

  static uint64_t GcTick(const TextureCache &cache) { return cache.m_gc_tick; }

  static std::unique_lock<TrackingSpinLock> Lock(TextureCache &cache) {
    return std::unique_lock(cache.m_lock);
  }

  static void ClearImage(TextureCache &cache, CommandBuffer &command, ImageId id,
                         const vk::ImageSubresourceRange &range,
                         const vk::ClearValue &clear) {
    auto lock = Lock(cache);
    cache.ClearImage(command, id, cache.GetImage(id).backing.format, range, clear);
  }

  static void ConfigureGarbageCollection(TextureCache &cache,
                                         std::span<const ImageId> oldest,
                                         uint64_t tick, uint64_t pressure) {
    cache.m_trigger_gc_memory = 0;
    cache.m_pressure_gc_memory = pressure;
    cache.m_critical_gc_memory = UINT64_MAX;
    cache.m_gc_tick = tick;
    std::vector<ImageId> live;
    cache.m_lru_cache = {};
    cache.m_slot_images.ForEach([&](ImageId id, Image &image) {
      if (image.registered) {
        image.tick_accessed_last = cache.m_scheduler.CurrentTick();
        live.push_back(id);
      }
    });
    for (const auto id : oldest) {
      const auto owner = cache.m_slot_images.try_get(id);
      if (owner != nullptr && owner->registered) {
        owner->tick_accessed_last = 0;
        owner->lru_id = cache.m_lru_cache.Insert(id, 0);
      }
    }
    for (const auto id : live) {
      if (std::ranges::find(oldest, id) == oldest.end()) {
        cache.m_slot_images[id].lru_id = cache.m_lru_cache.Insert(id, tick);
      }
    }
  }

  static bool Contains(const TextureCache &cache, ImageId id) {
    const auto owner = cache.m_slot_images.try_get(id);
    return owner != nullptr && owner->registered;
  }

  static std::vector<ImageId> FindImages(TextureCache &cache, uint64_t address,
                                         uint64_t size, bool page_overlap) {
    std::lock_guard lock(cache.m_lock);
    const auto found = cache.FindImagesInRegion(address, size, page_overlap);
    std::vector<ImageId> result;
    result.reserve(found.size());
    for (const auto id : found) {
      result.push_back(id);
    }
    return result;
  }

  static size_t PageOwnerCount(TextureCache &cache, uint64_t address) {
    std::lock_guard lock(cache.m_lock);
    const auto *owners = cache.m_image_page_table.Find(static_cast<size_t>(
        address >> TextureCache::ImagePageTable::kPageBits));
    return owners == nullptr ? 0 : owners->size();
  }

  static size_t OwnedPageCount(TextureCache &cache, uint64_t address,
                               uint64_t size, ImageId id) {
    std::lock_guard lock(cache.m_lock);
    TextureCache::ImagePageTable::PageRange pages{};
    if (!TextureCache::ImagePageTable::TryGetPageRange(address, size, pages)) {
      return 0;
    }
    size_t count = 0;
    for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
      const auto *owners = cache.m_image_page_table.Find(page);
      count += owners != nullptr && owners->Contains(id) ? 1 : 0;
    }
    return count;
  }

  static void AddPageOwner(TextureCache &cache, uint64_t address, ImageId id) {
    std::lock_guard lock(cache.m_lock);
    cache
        .m_image_page_table[static_cast<size_t>(
            address >> TextureCache::ImagePageTable::kPageBits)]
        .push_back(id);
  }

  static bool RemovePageOwner(TextureCache &cache, uint64_t address,
                              ImageId id) {
    std::lock_guard lock(cache.m_lock);
    auto *owners = cache.m_image_page_table.Find(static_cast<size_t>(
        address >> TextureCache::ImagePageTable::kPageBits));
    return owners != nullptr && owners->Erase(id);
  }

  static void SetQueryEpoch(TextureCache &cache, uint32_t epoch) {
    std::lock_guard lock(cache.m_lock);
    cache.m_image_query_epoch = epoch;
  }

  static uint32_t QueryEpoch(TextureCache &cache) {
    std::lock_guard lock(cache.m_lock);
    return cache.m_image_query_epoch;
  }

  static ImageId InsertImage(TextureCache &cache, const ImageInfo &info) {
    std::lock_guard lock(cache.m_lock);
    return cache.InsertImage(info);
  }

  static void DeleteImage(TextureCache &cache, ImageId id) {
    std::lock_guard lock(cache.m_lock);
    cache.DeleteImage(id);
  }

  static const Image *Owner(const TextureCache &cache, ImageId id) {
    return cache.m_slot_images.try_get(id);
  }

  static bool PendingDownload(const TextureCache &cache, ImageId id) {
    return cache.m_download_images.contains(id);
  }

  static size_t NullImageCount(const TextureCache &cache) {
    return cache.m_null_images.size();
  }

  static void TrackDownload(TextureCache &cache, ImageId id) {
    std::lock_guard lock(cache.m_lock);
    cache.TrackImageDownload(id, cache.m_slot_images[id]);
  }

  static void AssociateStencil(TextureCache &cache, ImageId depth,
                               GuestRange stencil) {
    std::lock_guard lock(cache.m_lock);
    (void)cache.AssociateStencil(depth, stencil);
  }

  static void SetLinearReadback(TextureCache &cache, bool enabled) {
    cache.m_readback_linear_images = enabled;
  }

  static std::pair<uint8_t *, uint64_t>
  MapDownload(TextureCache &cache, uint64_t size, uint64_t alignment) {
    auto &download =
        cache.m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download);
    auto mapping = download.Map(size, std::max<uint64_t>(alignment, 4));
    if (mapping.first != nullptr) {
      download.Commit();
    }
    return mapping;
  }

  static bool TryDownload(TextureCache &cache, ImageId id) {
    return cache.DownloadImageMemory(id);
  }

  static void RegisterHtileMeta(TextureCache &cache, uint64_t address) {
    std::lock_guard lock(cache.m_lock);
    auto &metadata = cache.m_surface_metas[address];
    metadata.type = TextureCache::MetaDataInfo::Type::HTile;
    metadata.clear_mask = 0;
  }

  static TileManager &Tiler(TextureCache &cache) { return cache.m_tiler; }
};

struct RenderExecutorTestAccess {
  static void DrawIndex(RenderExecutor &executor, CommandBuffer &command,
                        const DrawIndexArgs &args) {
    executor.DrawIndex(0, command, args);
  }

  static void DrawAuto(RenderExecutor &executor, CommandBuffer &command,
                       const DrawAutoArgs &args) {
    executor.DrawAuto(0, command, args);
  }

  static bool TryConsumeComputeImageClear(RenderExecutor &executor,
      const ShaderComputeInputInfo &input, CommandBuffer &command,
      uint32_t x, uint32_t y, uint32_t z, uint32_t mode) {
    return executor.TryConsumeComputeImageClear(input, command, x, y, z, mode);
  }

  static bool TryConsumeComputeMetaClear(RenderExecutor &executor,
                                         const ShaderComputeInputInfo &input,
                                         const CommandBuffer &buffer) {
    return executor.TryConsumeComputeMetaClear(input, buffer);
  }

  static TextureBinding
  ResolveTexture(RenderExecutor &executor,
                 const ShaderRecompiler::IR::ImageResource &resource,
                 const ShaderRecompiler::IR::DescriptorValue &value) {
    return executor.ResolveTexture(resource, value);
  }

  static auto PrepareGraphicsBindings(RenderExecutor &executor,
                                      const ShaderStageRuntime &vertex,
                                      const ShaderStageRuntime &pixel,
                                      bool pixel_active) {
    RenderExecutor::GraphicsBindings result;
    executor.PrepareBindings(vertex, result.vertex[0]);
    std::array<PreparedBindings *, 2> stages{&result.vertex[0], nullptr};
    if (pixel_active) {
      result.pixel.emplace();
      executor.PrepareBindings(pixel, *result.pixel);
      stages[1] = &*result.pixel;
    }
    executor.PrepareGraphicsBindings(
        std::span{stages.data(), pixel_active ? 2u : 1u}, {});
    return result;
  }

  static void PrepareGraphicsBindings(RenderExecutor &executor,
                                      std::span<PreparedBindings *const> stages,
                                      std::span<RenderColorInfo> colors) {
    executor.PrepareGraphicsBindings(stages, colors);
  }

  static PipelineCache::Pipeline
  CreateDescriptorPipeline(RenderExecutor &executor,
                           std::span<PreparedBindings *const> stages) {
    std::vector<vk::DescriptorSetLayoutBinding> layout_bindings;
    bool compute = false;
    for (const auto *prepared : stages) {
      const auto &program = *prepared->runtime->program;
      compute |= program.stage == ShaderType::Compute;
      const auto shader_stage = program.stage == ShaderType::Vertex
                                    ? vk::ShaderStageFlagBits::eVertex
                                : program.stage == ShaderType::Pixel
                                    ? vk::ShaderStageFlagBits::eFragment
                                    : vk::ShaderStageFlagBits::eCompute;
      for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) {
        const auto index = std::countr_zero(mask);
        const auto kind = static_cast<ShaderRecompiler::IR::DescriptorBindingKind>(index);
        layout_bindings.push_back(
            {ShaderRecompiler::IR::NativeBinding(program.stage, kind),
             NativeDescriptorType(kind), program.bindings.descriptor_counts[index], shader_stage});
      }
    }
    vk::DescriptorSetLayoutCreateInfo descriptor_info{};
    descriptor_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
    descriptor_info.flags =
        vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR;
    descriptor_info.bindingCount =
        static_cast<uint32_t>(layout_bindings.size());
    descriptor_info.pBindings = layout_bindings.data();
    PipelineCache::Pipeline pipeline{};
    auto &device = executor.m_context.GetGraphics().device;
    EXIT_IF(device.createDescriptorSetLayout(&descriptor_info, nullptr,
                                             &pipeline.descriptor_set_layout) !=
            vk::Result::eSuccess);

    vk::PipelineLayoutCreateInfo pipeline_info{};
    pipeline_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
    pipeline_info.setLayoutCount = 1;
    pipeline_info.pSetLayouts = &pipeline.descriptor_set_layout;
    const vk::PushConstantRange push_range {
        compute ? vk::ShaderStageFlags {vk::ShaderStageFlagBits::eCompute}
                : vk::ShaderStageFlags {vk::ShaderStageFlagBits::eVertex |
                                       vk::ShaderStageFlagBits::eFragment},
        0, ShaderRecompiler::IR::NativePushConstantSize};
    pipeline_info.pushConstantRangeCount = 1;
    pipeline_info.pPushConstantRanges = &push_range;
    EXIT_IF(device.createPipelineLayout(&pipeline_info, nullptr,
                                        &pipeline.pipeline_layout) !=
            vk::Result::eSuccess);
    pipeline.uses_push_descriptors = true;
    return pipeline;
  }

  static PipelineCache::Pipeline CommitBindings(RenderExecutor &executor,
                                                CommandBuffer &buffer,
                                                PreparedBindings &bindings) {
    std::array<PreparedBindings *, 1> stages{&bindings};
    auto pipeline = CreateDescriptorPipeline(executor, stages);
    const auto bind_point = bindings.runtime->program->stage == ShaderType::Compute
                                ? vk::PipelineBindPoint::eCompute
                                : vk::PipelineBindPoint::eGraphics;
    executor.CommitBindings(buffer, bind_point, pipeline, stages);
    return pipeline;
  }

  static PipelineCache::Pipeline CommitBindings(RenderExecutor &executor,
                                                CommandBuffer &buffer,
                                                PreparedBindings &vertex,
                                                PreparedBindings &pixel) {
    std::array<PreparedBindings *, 2> stages{&vertex, &pixel};
    auto pipeline = CreateDescriptorPipeline(executor, stages);
    executor.CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline,
                            stages);
    return pipeline;
  }

  static void CommitBindings(RenderExecutor &executor, CommandBuffer &buffer,
                             const PipelineCache::Pipeline &pipeline,
                             PreparedBindings &vertex, PreparedBindings &pixel) {
    std::array<PreparedBindings *, 2> stages{&vertex, &pixel};
    executor.CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline,
                            stages);
  }

  static void DestroyDescriptorPipelines(
      RenderExecutor &executor,
      std::span<const PipelineCache::Pipeline> pipelines) {
    auto &device = executor.m_context.GetGraphics().device;
    for (const auto &pipeline : pipelines) {
      device.destroyPipelineLayout(pipeline.pipeline_layout, nullptr);
      device.destroyDescriptorSetLayout(pipeline.descriptor_set_layout,
                                        nullptr);
    }
  }

  static void ResolveRenderDepthTarget(RenderExecutor &executor,
                                       CommandBuffer &buffer,
                                       RenderDepthInfo &depth) {
    executor.ResolveRenderDepthTarget(buffer, depth);
  }

  static bool DepthStencilCopy(RenderExecutor &executor, CommandBuffer &buffer) {
    return executor.DepthStencilCopy(buffer);
  }

  static void ResolveRenderColorTarget(RenderExecutor &executor,
                                       CommandBuffer &buffer,
                                       RenderColorInfo &color, uint32_t slot) {
    executor.ResolveRenderColorTarget(buffer, color, 0, slot);
  }

  static void BindRenderTarget(RenderExecutor &executor, ImageId id) {
    executor.BindRenderTarget(id);
  }

  static RenderState AcquireRenderTargets(RenderExecutor &executor,
                                          CommandBuffer &buffer,
                                          RenderColorInfo *colors,
                                          uint32_t color_count,
                                          RenderDepthInfo &depth,
                                          std::span<PreparedBindings *const> stages = {},
                                          vk::ImageAspectFlags *feedback_out = nullptr) {
    vk::ImageAspectFlags feedback;
    auto state = executor.AcquireRenderTargets(buffer, colors, color_count, depth,
                                               feedback, stages);
    if (feedback_out != nullptr) {
      *feedback_out = feedback;
    }
    return state;
  }

  static void ResetBindings(RenderExecutor &executor) {
    executor.ResetBindings();
  }

  static bool BoundImagesInOrder(const RenderExecutor &executor, ImageId first,
                                 ImageId second) {
    return executor.m_bound_images.size() == 2 &&
           executor.m_bound_images[0] == first &&
           executor.m_bound_images[1] == second;
  }
};

namespace {

using u32 = uint32_t;
using BindingType = TextureCache::BindingType;
using ImageDesc = TextureCache::ImageDesc;
using ShaderOpcode = ShaderRecompiler::Decoder::Opcode;

ImageInfo MakeTilingImage(Prospero::BufferFormat format, uint32_t width,
                          uint32_t height, uint32_t levels, uint32_t depth,
                          Prospero::TileMode tile, uint64_t guest_size,
                          bool volume) {
  ImageInfo info{};
  info.guest_format = format;
  info.type =
      volume ? Prospero::ImageType::kColor3D : Prospero::ImageType::kColor2D;
  info.extent = {width, height, volume ? depth : 1u};
  info.resources = {levels, volume ? 1u : depth};
  info.data.size = guest_size;
  info.tile_mode = tile;
  info.UpdateSize();
  return info;
}

constexpr u32 InlineU32(u32 value) { return 128u + value; }

constexpr u32 Vgpr(u32 reg) { return 256u + reg; }

constexpr u32 EncodeSMovB32(u32 dst, u32 src) {
  return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | (0x03u << 8u) |
         (src & 0xffu);
}

constexpr u32 EncodeSop1(u32 opcode, u32 dst, u32 src) {
  return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) |
         ((opcode & 0xffu) << 8u) | (src & 0xffu);
}

constexpr u32 EncodeSop2(u32 opcode, u32 dst, u32 src0, u32 src1) {
  return 0x80000000u | ((opcode & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) |
         ((src1 & 0xffu) << 8u) | (src0 & 0xffu);
}

constexpr u32 EncodeSopc(u32 opcode, u32 src0, u32 src1) {
  return 0x80000000u | (0x7eu << 23u) | ((opcode & 0x7fu) << 16u) |
         ((src1 & 0xffu) << 8u) | (src0 & 0xffu);
}

constexpr u32 EncodeSopp(u32 opcode, u32 simm = 0) {
  return 0x80000000u | (0x7fu << 23u) | ((opcode & 0x7fu) << 16u) |
         (simm & 0xffffu);
}

constexpr u32 EncodeSopk(u32 opcode, u32 dst, u32 imm) {
  return 0x80000000u | (((opcode + 0x60u) & 0x7fu) << 23u) |
         ((dst & 0x7fu) << 16u) | (imm & 0xffffu);
}

constexpr u32 EncodeVop1(u32 opcode, u32 dst, u32 src0) {
  return (0x3fu << 25u) | ((dst & 0xffu) << 17u) | ((opcode & 0xffu) << 9u) |
         (src0 & 0x1ffu);
}

constexpr u32 EncodeVop1Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0,
                             u32 src0_sel = 6, u32 src0_sext = 0,
                             u32 src0_neg = 0, u32 src0_abs = 0, u32 s0 = 0,
                             u32 clamp = 0, u32 omod = 0) {
  return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) |
         ((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) |
         ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) |
         ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
         ((s0 & 0x1u) << 23u);
}

constexpr u32 EncodeVop1Dpp(u32 src0, u32 dpp_ctrl = 0, u32 row_mask = 0xf,
                            u32 bank_mask = 0xf) {
  return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) |
         ((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u);
}

constexpr u32 EncodeVop2(u32 opcode, u32 dst, u32 src0, u32 src1) {
  return ((opcode & 0x3fu) << 25u) | ((dst & 0xffu) << 17u) |
         ((src1 & 0xffu) << 9u) | (src0 & 0x1ffu);
}

constexpr u32 EncodeVop2Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0,
                             u32 src0_sel = 6, u32 src1_sel = 6,
                             u32 src0_sext = 0, u32 src1_sext = 0,
                             u32 src0_neg = 0, u32 src0_abs = 0,
                             u32 src1_neg = 0, u32 src1_abs = 0, u32 s0 = 0,
                             u32 s1 = 0, u32 clamp = 0, u32 omod = 0) {
  return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) |
         ((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) |
         ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) |
         ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
         ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) |
         ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) |
         ((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u);
}

constexpr u32 EncodeVop2Dpp(u32 src0, u32 dpp_ctrl = 0, u32 row_mask = 0xf,
                            u32 bank_mask = 0xf, u32 src0_neg = 0,
                            u32 src0_abs = 0, u32 src1_neg = 0,
                            u32 src1_abs = 0) {
  return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) |
         ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
         ((src1_neg & 0x1u) << 22u) | ((src1_abs & 0x1u) << 23u) |
         ((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u);
}

constexpr u32 EncodeVop3Word0(u32 opcode, u32 dst, u32 abs = 0, u32 op_sel = 0,
                              bool clamp = false) {
  return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((abs & 0x7u) << 8u) |
         ((op_sel & 0xfu) << 11u) | (clamp ? (1u << 15u) : 0u) | (dst & 0xffu);
}

constexpr u32 EncodeVop3BWord0(u32 opcode, u32 vdst, u32 sdst) {
  return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((sdst & 0x7fu) << 8u) |
         (vdst & 0xffu);
}

constexpr u32 EncodeVop3Word1(u32 src0, u32 src1, u32 src2 = 0, u32 omod = 0,
                              u32 neg = 0) {
  return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) |
         ((omod & 0x3u) << 27u) | ((neg & 0x7u) << 29u);
}

constexpr u32 EncodeVop3pWord0(u32 opcode, u32 dst, u32 op_sel_hi = 0,
                               u32 op_sel = 0, u32 neg_hi = 0,
                               bool clamp = false) {
  return (0x33u << 26u) | ((opcode & 0x7fu) << 16u) | ((neg_hi & 0x7u) << 8u) |
         ((op_sel & 0x7u) << 11u) | (((op_sel_hi >> 2u) & 0x1u) << 14u) |
         (clamp ? (1u << 15u) : 0u) | (dst & 0xffu);
}

constexpr u32 EncodeVop3pWord1(u32 src0, u32 src1, u32 src2 = 0,
                               u32 op_sel_hi = 0, u32 neg = 0) {
  return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) |
         ((op_sel_hi & 0x3u) << 27u) | ((neg & 0x7u) << 29u);
}

constexpr u32 EncodeVopc(u32 opcode, u32 src0, u32 src1) {
  return (0x3eu << 25u) | ((opcode & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) |
         (src0 & 0x1ffu);
}

constexpr u32 EncodeVopcSdwa(u32 src0, u32 sdst = 0, u32 sd = 0,
                             u32 src0_sel = 6, u32 src1_sel = 6,
                             u32 src0_sext = 0, u32 src1_sext = 0,
                             u32 src0_neg = 0, u32 src0_abs = 0,
                             u32 src1_neg = 0, u32 src1_abs = 0, u32 s0 = 0,
                             u32 s1 = 0) {
  return (src0 & 0xffu) | ((sdst & 0x7fu) << 8u) | ((sd & 0x1u) << 15u) |
         ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) |
         ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
         ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) |
         ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) |
         ((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u);
}

constexpr u32 EncodeMubuf0(u32 opcode, u32 offset = 0, bool idxen = false,
                           bool offen = true, bool glc = false) {
  return (0x38u << 26u) | ((opcode & 0xffu) << 18u) |
         (offen ? (1u << 12u) : 0u) | (idxen ? (1u << 13u) : 0u) |
         (glc ? (1u << 14u) : 0u) | (offset & 0xfffu);
}

constexpr u32 EncodeMubuf1(u32 vdata, u32 srsrc, u32 vaddr, u32 soffset = 128) {
  return ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) |
         ((vdata & 0xffu) << 8u) | (vaddr & 0xffu);
}

constexpr u32 EncodeMtbuf0(u32 opcode, u32 dfmt, u32 nfmt, u32 offset = 0,
                           bool idxen = false, bool offen = true) {
  return (0x3au << 26u) | (offset & 0xfffu) | (offen ? (1u << 12u) : 0u) |
         (idxen ? (1u << 13u) : 0u) | ((opcode & 0x7u) << 16u) |
         ((dfmt & 0xfu) << 19u) | ((nfmt & 0x7u) << 23u);
}

constexpr u32 EncodeMtbuf1(u32 opcode, u32 vdata, u32 srsrc, u32 vaddr,
                           u32 soffset = 128) {
  return (((opcode >> 3u) & 1u) << 21u) | ((soffset & 0xffu) << 24u) |
         ((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | (vaddr & 0xffu);
}

constexpr u32 EncodeSmem0(u32 opcode, u32 dst, u32 sbase = 0) {
  return (0x3du << 26u) | ((opcode & 0xffu) << 18u) | ((dst & 0x7fu) << 6u) |
         (sbase & 0x3fu);
}

constexpr u32 EncodeSmem1(u32 offset, u32 soffset = 0) {
  return (offset & 0x1fffffu) | ((soffset & 0x7fu) << 25u);
}

constexpr u32 EncodeMimg0(u32 opcode, u32 dmask, u32 nsa_dwords = 0,
                          bool glc = false, u32 dim = 1, bool r128 = false) {
  return (0x3cu << 26u) | ((opcode >> 7u) & 0x1u) |
         ((nsa_dwords & 0x3u) << 1u) | ((dim & 0x7u) << 3u) |
         ((dmask & 0xfu) << 8u) | (glc ? (1u << 13u) : 0u) |
         (r128 ? (1u << 15u) : 0u) | ((opcode & 0x7fu) << 18u);
}

constexpr u32 EncodeMimg1(u32 vdata, u32 vaddr, u32 srsrc = 0, u32 ssamp = 0,
                          bool a16 = false, bool d16 = false) {
  return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) |
         ((vdata & 0xffu) << 8u) | (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u) |
         (d16 ? (1u << 31u) : 0u);
}

constexpr u32 EncodeVintrp(u32 opcode, u32 dst, u32 attr, u32 chan, u32 src) {
  return (0x32u << 26u) | ((dst & 0xffu) << 18u) | ((opcode & 0x3u) << 16u) |
         ((attr & 0x3fu) << 10u) | ((chan & 0x3u) << 8u) | (src & 0xffu);
}

constexpr u32 EncodeExp0(u32 target, u32 en, bool done = true,
                         bool compr = false, bool vm = false) {
  return (0x3eu << 26u) | ((target & 0x3fu) << 4u) | (en & 0xfu) |
         (compr ? (1u << 10u) : 0u) | (done ? (1u << 11u) : 0u) |
         (vm ? (1u << 12u) : 0u);
}

constexpr u32 EncodeExp1(u32 src0, u32 src1, u32 src2, u32 src3) {
  return (src0 & 0xffu) | ((src1 & 0xffu) << 8u) | ((src2 & 0xffu) << 16u) |
         ((src3 & 0xffu) << 24u);
}

constexpr u32 EncodeFlat0(u32 opcode, u32 segment, u32 offset = 0) {
  return (0x37u << 26u) | ((opcode & 0x7fu) << 18u) |
         ((segment & 0x3u) << 14u) | (offset & 0xfffu);
}

constexpr u32 EncodeFlat1(u32 vdst, u32 saddr, u32 data, u32 addr) {
  return ((vdst & 0xffu) << 24u) | ((saddr & 0x7fu) << 16u) |
         ((data & 0xffu) << 8u) | (addr & 0xffu);
}

constexpr u32 EncodeDs0(u32 opcode, u32 offset = 0, bool gds = false) {
  return (0x36u << 26u) | ((opcode & 0xffu) << 18u) | (gds ? (1u << 17u) : 0u) |
         (offset & 0xffffu);
}

constexpr u32 EncodeDs1Ex(u32 vdst, u32 data1, u32 data0, u32 addr) {
  return ((vdst & 0xffu) << 24u) | ((data1 & 0xffu) << 16u) |
         ((data0 & 0xffu) << 8u) | (addr & 0xffu);
}

constexpr u32 EncodeDs1(u32 vdst, u32 data0, u32 addr) {
  return EncodeDs1Ex(vdst, 0, data0, addr);
}

void AppendSMovLiteral(std::vector<u32> *code, u32 dst, u32 literal) {
  code->push_back(EncodeSMovB32(dst, 255u));
  code->push_back(literal);
}

void AppendVMovLiteral(std::vector<u32> *code, u32 dst, u32 literal) {
  code->push_back(EncodeVop1(0x01, dst, 255u));
  code->push_back(literal);
}

void AppendVMovU32(std::vector<u32> *code, u32 dst, u32 value) {
  if (value <= 64u) {
    code->push_back(EncodeVop1(0x01, dst, InlineU32(value)));
    return;
  }
  AppendVMovLiteral(code, dst, value);
}

void AppendVop3(std::vector<u32> *code, u32 opcode, u32 dst, u32 src0, u32 src1,
                u32 src2 = 0, u32 abs = 0, u32 op_sel = 0, bool clamp = false,
                u32 omod = 0, u32 neg = 0) {
  code->push_back(EncodeVop3Word0(opcode, dst, abs, op_sel, clamp));
  code->push_back(EncodeVop3Word1(src0, src1, src2, omod, neg));
}

void AppendVop3B(std::vector<u32> *code, u32 opcode, u32 vdst, u32 sdst,
                 u32 src0, u32 src1, u32 src2 = 0) {
  code->push_back(EncodeVop3BWord0(opcode, vdst, sdst));
  code->push_back(EncodeVop3Word1(src0, src1, src2));
}

void AppendVop3p(std::vector<u32> *code, u32 opcode, u32 dst, u32 src0,
                 u32 src1, u32 src2 = 0, u32 op_sel_hi = 0, u32 op_sel = 0,
                 u32 neg_hi = 0, u32 neg = 0) {
  code->push_back(EncodeVop3pWord0(opcode, dst, op_sel_hi, op_sel, neg_hi));
  code->push_back(EncodeVop3pWord1(src0, src1, src2, op_sel_hi, neg));
}

void AppendBufferLoadDword(std::vector<u32> *code, u32 dst_vgpr,
                           u32 address_vgpr) {
  code->push_back(EncodeMubuf0(0x0cu));
  code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr));
}

void AppendBufferLoadOpcode(std::vector<u32> *code, u32 opcode, u32 dst_vgpr,
                            u32 address_vgpr) {
  code->push_back(EncodeMubuf0(opcode));
  code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr));
}

void AppendBufferStoreDword(std::vector<u32> *code, u32 value_vgpr,
                            u32 address_vgpr) {
  code->push_back(EncodeMubuf0(0x1cu));
  code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr));
}

void AppendBufferStoreOpcode(std::vector<u32> *code, u32 opcode, u32 value_vgpr,
                             u32 address_vgpr, bool glc = false) {
  code->push_back(EncodeMubuf0(opcode, 0, false, true, glc));
  code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr));
}

void AppendTBufferLoadOpcode(std::vector<u32> *code, u32 opcode, u32 dst_vgpr,
                             u32 address_vgpr) {
  code->push_back(EncodeMtbuf0(opcode, 14, 7));
  code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr));
}

constexpr u32 BufferFormat(Prospero::BufferFormat format) {
  return static_cast<uint32_t>(format);
}

void AppendTBufferLoadFormatOpcode(std::vector<u32> *code, u32 opcode,
                                   u32 dst_vgpr, u32 address_vgpr,
                                   Prospero::BufferFormat format) {
  const auto value = BufferFormat(format);
  code->push_back(EncodeMtbuf0(opcode, value & 0xfu, (value >> 4u) & 0x7u));
  code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr));
}

void AppendTBufferStoreOpcode(std::vector<u32> *code, u32 opcode,
                              u32 value_vgpr, u32 address_vgpr) {
  code->push_back(EncodeMtbuf0(opcode, 14, 7));
  code->push_back(EncodeMtbuf1(opcode, value_vgpr, 0, address_vgpr));
}

void AppendSmemLoadOpcode(std::vector<u32> *code, u32 opcode, u32 dst_sgpr,
                          u32 byte_offset) {
  code->push_back(EncodeSmem0(opcode, dst_sgpr));
  code->push_back(EncodeSmem1(byte_offset));
}

void AppendStoreVgpr(std::vector<u32> *code, u32 value_vgpr, u32 dword_index) {
  AppendVMovU32(code, 31, dword_index * 4u);
  AppendBufferStoreDword(code, value_vgpr, 31);
}

void AppendStoreVgprAtLaneDwordOffset(std::vector<u32> *code, u32 value_vgpr,
                                      u32 lane_vgpr, u32 dword_offset) {
  if (dword_offset == 0u) {
    code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), lane_vgpr));
  } else {
    AppendVMovU32(code, 31, dword_offset);
    code->push_back(EncodeVop2(0x25, 31, Vgpr(lane_vgpr), 31));
    code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), 31));
  }
  AppendBufferStoreDword(code, value_vgpr, 31);
}

void AppendStoreSgpr(std::vector<u32> *code, u32 value_sgpr, u32 dword_index) {
  code->push_back(EncodeVop1(0x01, 30, value_sgpr));
  AppendStoreVgpr(code, 30, dword_index);
}

void AppendStoreSgprAtLaneDwordOffset(std::vector<u32> *code, u32 value_sgpr,
                                      u32 lane_vgpr, u32 dword_offset) {
  code->push_back(EncodeVop1(0x01, 30, value_sgpr));
  AppendStoreVgprAtLaneDwordOffset(code, 30, lane_vgpr, dword_offset);
}

void AppendStoreSgprPair(std::vector<u32> *code, u32 value_sgpr,
                         u32 first_dword_index) {
  AppendStoreSgpr(code, value_sgpr, first_dword_index);
  AppendStoreSgpr(code, value_sgpr + 1u, first_dword_index + 1u);
}

void AppendEnd(std::vector<u32> *code) { code->push_back(0xbf810000u); }

std::string Hex(u32 value) {
  char buffer[32] = {};
  std::snprintf(buffer, sizeof(buffer), "0x%08" PRIx32, value);
  return buffer;
}

[[noreturn]] void Fail(const char *shader_name, const char *stage,
                       const std::string &message) {
  std::fprintf(stderr, "ShaderRecompilerComputeTests: %s failed at %s: %s\n",
               shader_name, stage, message.c_str());
  std::abort();
}

void Require(const char *shader_name, const char *stage, bool value,
             const std::string &message);

void EnsureConfigInitialized() {
  static bool config_initialized = false;
  if (!config_initialized) {
    static Common::Subsystems subsystems;
    Common::InitializeThreads();
    subsystems.Initialize<Config::Lifecycle>();
    Config::ConfigOptions options;
    options.printf_direction = Config::LogDirection::Silent;
    Config::Load(options);
    subsystems.Initialize<Log::Lifecycle>();
    subsystems.Initialize<Libs::LibKernel::Memory::Lifecycle>();
    ShaderInit();
    config_initialized = true;
  }
}

void RequireVk(const char *shader_name, const char *stage, vk::Result result,
               const char *action) {
  if (result != vk::Result::eSuccess) {
    Fail(shader_name, stage,
         std::string(action) + " returned " + vk::to_string(result));
  }
}

void Require(const char *shader_name, const char *stage, bool value,
             const std::string &message) {
  if (!value) {
    Fail(shader_name, stage, message);
  }
}

void CheckErrorDialogLifecycle() {
  namespace ErrorDialog = Libs::Dialog::ErrorDialog;
  namespace SystemDialog = Libs::Dialog::SystemDialog;
  struct Param {
    int32_t size = 16;
    int32_t error_code = static_cast<int32_t>(0x80550006u);
    int32_t user_id = 1;
    int32_t reserved = 0;
  } param;
  static_assert(sizeof(Param) == 16);
  constexpr int invalid_state = static_cast<int>(0x80ed0005u);
  constexpr int invalid_param = static_cast<int>(0x80ed0003u);
  static std::vector<SystemDialog::VisualState> notifications;
  notifications.clear();
  SystemDialog::SetVisibilityCallback([] {
    const auto visual = SystemDialog::GetVisualState();
    SystemDialog::HostSnapshot snapshot{};
    Require("ErrorDialog", "callback reentry",
            SystemDialog::GetHostSnapshot(&snapshot) == visual.active &&
                (ErrorDialog::ErrorDialogGetStatus() == 2) == visual.active,
            "visibility callback observed inconsistent dialog state");
    notifications.push_back(visual);
  });
  const auto initial_revision = SystemDialog::GetVisualState().revision;
  const auto check_state = [&](const char *stage, int status,
                               size_t visibility_changes) {
    const auto visual = SystemDialog::GetVisualState();
    SystemDialog::HostSnapshot snapshot{};
    Require("ErrorDialog", stage,
            ErrorDialog::ErrorDialogGetStatus() == status &&
                ErrorDialog::ErrorDialogUpdateStatus() == status &&
                visual.active == (status == 2) && !visual.background &&
                SystemDialog::GetHostSnapshot(&snapshot) == visual.active &&
                visual.revision == initial_revision + visibility_changes &&
                notifications.size() == visibility_changes &&
                (notifications.empty() ||
                 (notifications.back().active == visual.active &&
                  notifications.back().revision == visual.revision)),
            "status, host visibility, or visibility notification changed "
            "unexpectedly");
  };
  check_state("uninitialized", 0, 0);
  Require("ErrorDialog", "uninitialized operations",
          ErrorDialog::ErrorDialogTerminate() ==
                  static_cast<int>(0x80ed0001u) &&
              ErrorDialog::ErrorDialogClose() == invalid_state,
          "uninitialized dialog accepted termination or close");
  Require("ErrorDialog", "initialize",
          ErrorDialog::ErrorDialogInitialize() == 0 &&
              ErrorDialog::ErrorDialogInitialize() ==
                  static_cast<int>(0x80ed0002u),
          "initialization did not reject a second initialization");
  Param invalid = param;
  invalid.size--;
  Require("ErrorDialog", "invalid open",
          ErrorDialog::ErrorDialogOpen(nullptr) == invalid_param &&
              ErrorDialog::ErrorDialogOpen(&invalid) == invalid_param &&
              ErrorDialog::ErrorDialogClose() == invalid_state,
          "invalid parameters or close were accepted before opening");
  check_state("failed open preserves initialized state", 1, 0);
  Require("ErrorDialog", "open", ErrorDialog::ErrorDialogOpen(&param) == 0,
          "valid error dialog did not open");
  SystemDialog::HostSnapshot first{};
  Require("ErrorDialog", "copied error code",
          SystemDialog::GetHostSnapshot(&first) &&
              first.error_code == param.error_code,
          "host dialog did not expose the guest error code");
  param.error_code = static_cast<int32_t>(0x8055000au);
  for (int poll = 0; poll < 8; ++poll) {
    check_state("polling waits for acknowledgement", 2, 1);
  }
  Require(
      "ErrorDialog", "running failures",
      ErrorDialog::ErrorDialogOpen(&param) == invalid_state &&
          !SystemDialog::HostClose(first.generation + 1),
      "an invalid open or stale acknowledgement replaced the active dialog");
  SystemDialog::HostSnapshot current{};
  Require("ErrorDialog", "failed open preserves active dialog",
          SystemDialog::GetHostSnapshot(&current) &&
              current.generation == first.generation &&
              current.error_code == first.error_code,
          "active error code or generation changed after a failed open");
  check_state("failed operations preserve running state", 2, 1);
  Require("ErrorDialog", "acknowledge",
          SystemDialog::HostClose(first.generation),
          "current host acknowledgement did not finish the dialog");
  check_state("acknowledged", 3, 2);
  Require("ErrorDialog", "finished operations",
          !SystemDialog::HostClose(first.generation) &&
              ErrorDialog::ErrorDialogClose() == invalid_state &&
              ErrorDialog::ErrorDialogOpen(&invalid) == invalid_param,
          "finished dialog accepted duplicate completion or an invalid open");
  check_state("failed operations preserve finished state", 3, 2);
  Require("ErrorDialog", "reopen and stale acknowledgement",
          ErrorDialog::ErrorDialogOpen(&param) == 0 &&
              SystemDialog::GetHostSnapshot(&current) &&
              current.generation != first.generation &&
              current.error_code == param.error_code &&
              !SystemDialog::HostClose(first.generation),
          "reopened dialog accepted acknowledgement from its predecessor");
  check_state("reopened", 2, 3);
  Require("ErrorDialog", "guest close", ErrorDialog::ErrorDialogClose() == 0,
          "guest close did not finish the running dialog");
  check_state("closed", 3, 4);
  Require("ErrorDialog", "terminate running dialog",
          ErrorDialog::ErrorDialogOpen(&param) == 0 &&
              SystemDialog::GetHostSnapshot(&current) &&
              ErrorDialog::ErrorDialogTerminate() == 0 &&
              !SystemDialog::HostClose(current.generation),
          "termination failed to remove the active dialog");
  check_state("terminated", 0, 6);
  Require("ErrorDialog", "reinitialize and stale acknowledgement",
          ErrorDialog::ErrorDialogInitialize() == 0 &&
              ErrorDialog::ErrorDialogOpen(&param) == 0 &&
              !SystemDialog::HostClose(current.generation),
          "reinitialization reused an old dialog generation");
  check_state("reinitialized", 2, 7);
  Require("ErrorDialog", "final termination",
          ErrorDialog::ErrorDialogTerminate() == 0 &&
              ErrorDialog::ErrorDialogTerminate() ==
                  static_cast<int>(0x80ed0001u),
          "termination did not restore the uninitialized state");
  check_state("final state", 0, 8);
  SystemDialog::SetVisibilityCallback(nullptr);
  Require("ErrorDialog", "lifecycle without a visibility listener",
          ErrorDialog::ErrorDialogInitialize() == 0 &&
              ErrorDialog::ErrorDialogOpen(&param) == 0 &&
              ErrorDialog::ErrorDialogClose() == 0 &&
              ErrorDialog::ErrorDialogTerminate() == 0 &&
              ErrorDialog::ErrorDialogGetStatus() == 0 &&
              !SystemDialog::GetVisualState().active &&
              SystemDialog::GetVisualState().revision ==
                  initial_revision + 10 &&
              notifications.size() == 8,
          "guest lifecycle depended on a registered host visibility listener");
  std::printf("[host]    %-32s ok\n", "ErrorDialogLifecycle");
}

void CheckSigninDialogLifecycle() {
  namespace Signin = Libs::Dialog::SigninDialog;
  namespace Error = Libs::Dialog::ErrorDialog;
  namespace Host = Libs::Dialog::SystemDialog;
  struct Param {
    int32_t size = 16;
    int32_t user_id = Config::GetUserId();
    int32_t reserved[2] = {};
  } param;
  struct Result {
    int32_t result = -1;
    int32_t reserved[3] = {-1, -1, -1};
  } result;
  constexpr int not_initialized = static_cast<int>(0x81350001u);
  constexpr int already_initialized = static_cast<int>(0x81350002u);
  constexpr int invalid_param = static_cast<int>(0x81350003u);
  constexpr int invalid_state = static_cast<int>(0x81350005u);
  Require("SigninDialog", "uninitialized",
          Signin::SigninDialogOpen(&param) == not_initialized &&
              Signin::SigninDialogClose() == not_initialized &&
              Signin::SigninDialogGetResult(&result) == not_initialized,
          "uninitialized operations returned the wrong error");
  Require("SigninDialog", "initialize and close",
          Signin::SigninDialogInitialize() == 0 &&
              Signin::SigninDialogClose() == 0 &&
              Signin::SigninDialogUpdateStatus() == 3,
          "close must also succeed from INITIALIZED");
  Param invalid = param;
  invalid.reserved[0] = 1;
  Require("SigninDialog", "parameter validation",
          Signin::SigninDialogOpen(nullptr) == invalid_param &&
              Signin::SigninDialogOpen(&invalid) == invalid_param,
          "invalid parameters were accepted");
  invalid = param;
  invalid.user_id = -1;
  Require("SigninDialog", "user validation",
          Signin::SigninDialogOpen(&invalid) == static_cast<int>(0x81350007u),
          "an invalid user was accepted");
  Host::SetVisibilityCallback([] {
    const auto visual = Host::GetVisualState();
    Require("SigninDialog", "callback reentry",
            visual.background == (Signin::SigninDialogGetStatus() == 2),
            "foreground state did not follow the dialog lifetime");
  });
  Require("SigninDialog", "open", Signin::SigninDialogOpen(&param) == 0,
          "valid sign-in dialog did not open");
  Host::HostSnapshot first{};
  Require(
      "SigninDialog", "visible and backgrounded",
      Host::GetHostSnapshot(&first) && first.kind == Host::Kind::Signin &&
          Host::GetVisualState().active && Host::GetVisualState().background,
      "sign-in dialog was not visible or did not background the application");
  for (int poll = 0; poll < 8; ++poll) {
    Require(
        "SigninDialog", "polling waits for the user",
        Signin::SigninDialogUpdateStatus() == 2 &&
            Signin::SigninDialogGetStatus() == 2 &&
            Signin::SigninDialogInitialize() == already_initialized &&
            Signin::SigninDialogGetResult(&result) == invalid_state,
        "polling silently finished sign-in or lost the active initialization");
  }
  Require(
      "SigninDialog", "user cancellation",
      Host::HostClose(first.generation) &&
          Signin::SigninDialogGetResult(&result) == 0 && result.result == 1 &&
          result.reserved[0] == 0 && result.reserved[1] == 0 &&
          result.reserved[2] == 0 && !Host::GetVisualState().background &&
          !Host::GetVisualState().active && Signin::SigninDialogClose() == 0,
      "user cancellation did not finish sign-in and restore the foreground");
  const int32_t error_param[4] = {16, static_cast<int32_t>(0x80550006u), 1, 0};
  Host::HostSnapshot error{};
  Require("SigninDialog", "shared presentation generations",
          Error::ErrorDialogInitialize() == 0 &&
              Error::ErrorDialogOpen(error_param) == 0 &&
              Host::GetHostSnapshot(&error) &&
              error.kind == Host::Kind::Error &&
              error.generation != first.generation &&
              !Host::HostClose(first.generation),
          "an old sign-in action affected an error dialog");
  Host::HostSnapshot overlay{};
  Require("SigninDialog", "overlapping dialogs",
          Signin::SigninDialogOpen(&param) == 0 &&
              Host::GetHostSnapshot(&overlay) &&
              overlay.kind == Host::Kind::Signin &&
              !Host::HostClose(error.generation) &&
              Host::HostClose(overlay.generation) &&
              Host::GetHostSnapshot(&overlay) &&
              overlay.kind == Host::Kind::Error &&
              overlay.generation == error.generation &&
              !Host::GetVisualState().background &&
              Host::HostClose(error.generation) &&
              Error::ErrorDialogTerminate() == 0,
          "closing sign-in did not restore the pending error dialog");
  Host::HostSnapshot reopened{};
  Require(
      "SigninDialog", "reopen and abort",
      Signin::SigninDialogOpen(&param) == 0 &&
          Host::GetHostSnapshot(&reopened) &&
          !Host::HostClose(first.generation) &&
          !Host::HostClose(error.generation) &&
          Signin::SigninDialogTerminate() == 0 &&
          !Host::HostClose(reopened.generation) &&
          !Host::GetVisualState().active &&
          !Host::GetVisualState().background &&
          Signin::SigninDialogTerminate() == not_initialized,
      "abort did not remove the dialog or a stale host action was accepted");
  Host::SetVisibilityCallback(nullptr);
  std::printf("[host]    %-32s ok\n", "SigninDialogLifecycle");
}

#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
template <typename Function>
void ExpectFatal(const char *name, Function function) {
  const pid_t pid = ::fork();
  Require(name, "fatal subprocess", pid >= 0,
          "fork failed while starting fatal shader test");
  if (pid == 0) {
    function();
    ::_exit(0);
  }

  int status = 0;
  pid_t waited = -1;
  do {
    waited = ::waitpid(pid, &status, 0);
  } while (waited < 0 && errno == EINTR);
  Require(name, "fatal subprocess", waited == pid,
          "waitpid failed while collecting fatal shader test");
  Require(name, "fatal exit",
          WIFEXITED(status) && WEXITSTATUS(status) == (321 & 0xff),
          "fatal shader path did not terminate with status 321");
  std::printf("[host]    %-32s ok\n", name);
}
#endif

void CheckLeastRecentlyUsedCacheOrdering() {
  Common::LeastRecentlyUsedCache<uint32_t, uint64_t> cache;
  const auto first = cache.Insert(1, 1);
  const auto middle = cache.Insert(2, 2);
  const auto last = cache.Insert(3, 3);
  (void)first;
  (void)last;
  cache.Touch(middle, 4);
  std::vector<uint32_t> visited;
  cache.ForEachItemBelow(4, [&](uint32_t value) {
    visited.push_back(value);
    return visited.size() > 3;
  });
  Require("LeastRecentlyUsedCache", "non-tail touch traversal",
          visited == std::vector<uint32_t>{1, 3, 2},
          "touching a non-tail item left a cycle or changed LRU order");
  std::printf("[host]    %-32s ok\n", "LeastRecentlyUsedCache");
}

struct BdaMapping {
  uint64_t guest_base = 0;
  u32 backing_offset = 0;
};

struct TestCase {
  const char *name = "";
  std::vector<u32> code;
  std::vector<u32> initial;
  std::vector<u32> expected;
  std::vector<ShaderOpcode> opcodes;
  u32 image_width = 4;
  u32 image_height = 4;
  std::vector<u32> sampled_image_rgba;
  std::vector<std::vector<u32>> sampled_image_rgba_mips;
  vk::Format sampled_image_format = vk::Format::eR32G32B32A32Sfloat;
  vk::Filter sampler_filter = vk::Filter::eNearest;
  u32 sampled_image_dwords_per_pixel = 4;
  vk::ImageType sampled_image_type = vk::ImageType::e2D;
  vk::ImageViewType sampled_image_view_type = vk::ImageViewType::e2D;
  u32 sampled_image_layers = 1;
  u32 sampled_image_view_base_mip = 0;
  u32 sampled_image_view_base_layer = 0;
  u32 sampled_image_view_layers = 0;
  std::vector<u32> storage_image_rgba;
  std::vector<u32> expected_storage_image_rgba;
  vk::Format storage_image_format = vk::Format::eR32G32B32A32Sfloat;
  u32 storage_image_dwords_per_pixel = 4;
  std::vector<u32> storage_image_r32ui;
  std::vector<u32> expected_storage_image_r32ui;
  std::vector<std::string> required_spirv;
  std::vector<std::string> forbidden_spirv;
  ShaderComputeInputInfo compute_info = [] {
    ShaderComputeInputInfo info{};
    info.lds_size_dwords = 1024;
    return info;
  }();
  bool has_compute_info = false;
  u32 dispatch_x = 1;
  u32 dispatch_y = 1;
  u32 dispatch_z = 1;
  std::array<u32, 64> user_data{};
  bool has_user_data = false;
  u32 image_descriptor_swizzle = DstSel(4, 5, 6, 7);
  bool compile_only = false;
  size_t storage_buffer_range_bytes = 0;
  std::vector<u32> storage_buffer_offsets;
  std::vector<std::pair<u32, u32>> storage_buffer_ranges;
  std::vector<BdaMapping> bda_mappings;
  std::optional<uint64_t> fault_free_address;
  bool expand_shader_data_storage = false;
  bool expected_force_point_sampler = false;
  float expected_float_tolerance = 0.0f;
  std::vector<u32> gds_initial;
  std::vector<u32> expected_gds;
  std::vector<std::pair<std::string, size_t>> decoded_counts;
  std::vector<std::pair<std::string, size_t>> ir_counts;
  u32 expected_mip_descriptors = 0;
  std::optional<std::vector<u32>> expected_buffer_resources;
};

struct GraphicsCase {
  const char *name = "";
  std::vector<u32> fragment_code;
  std::vector<u32> expected_pixel;
  std::vector<ShaderOpcode> opcodes;
  std::array<u32, 64> user_data{};
  bool has_user_data = false;
  std::vector<u32> push_constants;
  std::vector<u32> pixel_interpolator_settings;
  bool pixel_no_perspective = false;
  bool provoking_vtx_last = false;
  std::vector<u32> vertices;
  bool pixel_ancillary = false;
  bool pixel_front_face = false;
  u32 layers = 1;
  vk::SampleCountFlagBits samples = vk::SampleCountFlagBits::e1;
  bool pixel_position_w = false;
  float vertex_clip_w = 1.0f;
  bool pixel_depth_export = false;
  u32 pixel_perspective_sample_vgpr = UINT32_MAX;
  u32 pixel_perspective_centroid_vgpr = UINT32_MAX;
  u32 pixel_custom_interpolation_mask = 0;
};

struct CompiledShader {
  std::vector<u32> spirv;
  ShaderRecompiler::IR::Program program;
  ShaderRecompiler::IR::ResourceSnapshot resources;
  std::vector<u32> packed_user_data;
};

std::array<u32, 64> MakeNativeUserData(const std::array<u32, 64> *source) {
  std::array<u32, 64> data{};
  data[3] = 3u << 28u;
  data[50] = 1u << 20u;
  data[51] = 3u << 28u;
  if (source != nullptr) {
    data = *source;
  }
  return data;
}

bool ReadTestMemory(void *userdata, uint64_t address, std::span<u32> values) {
  const auto *data = static_cast<const std::vector<u32> *>(userdata);
  if (data == nullptr || address % 4u != 0 || values.size() > data->size() ||
      address / 4u > data->size() - values.size()) {
    return false;
  }
  std::copy_n(data->begin() + address / 4u, values.size(), values.begin());
  return true;
}

void ValidateSpirv(const char *shader_name, const std::vector<u32> &spirv) {
  spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
  std::string messages;
  tools.SetMessageConsumer([&messages](spv_message_level_t, const char *,
                                       const spv_position_t &position,
                                       const char *message) {
    char buffer[1024] = {};
    std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line,
                  position.column, message);
    messages += buffer;
  });

  spvtools::ValidatorOptions options;
  options.SetAllowOffsetTextureOperand(true);
  if (!tools.Validate(spirv.data(), spirv.size(), options)) {
    Fail(shader_name, "SPIR-V validation", messages);
  }
}

size_t CountText(const std::string &text, const std::string &needle) {
  size_t count = 0;
  for (size_t offset = 0;
       (offset = text.find(needle, offset)) != std::string::npos;
       offset += needle.size()) {
    count++;
  }
  return count;
}

void CheckPixelParameterAliases() {
  constexpr const char *name = "PixelParameterAliases";
  ShaderPixelInputInfo pixel{};
  pixel.input_num = 2;
  const std::array<uint32_t, 2> pair = {0, 1};
  for (const auto flat : {0u, 0x400u}) {
    pixel.interpolator_settings[0] = flat | 3u;
    pixel.interpolator_settings[1] = flat | 3u;
    Require(name, "same-mode nonidentity alias",
            ShaderPixelParameterLocation(pixel, pair, 0) == 3u &&
                ShaderPixelParameterLocation(pixel, pair, 1) == 3u,
            "aliases of one export with the same interpolation mode must "
            "share that physical location");
  }
  pixel.interpolator_settings[0] = 3u;
  pixel.interpolator_settings[1] = 0x423u;
  pixel.custom_interpolation_mask = 2u;
  Require(name, "smooth/custom alias",
          ShaderPixelParameterLocation(pixel, pair, 0) == 3u &&
              ShaderPixelParameterLocation(pixel, pair, 1) == 3u,
          "custom interpolation must retain its shared vertex export");

  pixel.input_num = 3;
  pixel.custom_interpolation_mask = 0;
  pixel.interpolator_settings[0] = 0x400u;
  pixel.interpolator_settings[1] = 0u;
  pixel.interpolator_settings[2] = 1u;
  const std::array<uint32_t, 3> active = {0, 1, 2};
  Require(name, "ordinary mixed aliases",
          ShaderPixelParameterLocation(pixel, active, 0) == 0u &&
              ShaderPixelParameterLocation(pixel, active, 1) == 0u &&
              ShaderPixelParameterLocation(pixel, active, 2) == 1u,
          "flat/smooth aliases must retain the real vertex export location");
  std::vector<uint32_t> ordinary_key, rectangle_key, provoking_key;
  BuildStageStaticKey(pixel, ordinary_key);
  pixel.parameter_mode = ShaderPixelParameterMode::LastVertex;
  BuildStageStaticKey(pixel, provoking_key);
  pixel.parameter_mode = ShaderPixelParameterMode::Rectangle;
  BuildStageStaticKey(pixel, rectangle_key);
  Require(name, "interpolation specialization",
          ordinary_key != rectangle_key && ordinary_key != provoking_key,
          "rectangle and provoking-vertex modes must participate in the shader key");
  Require(name, "reserved physical locations",
          ShaderPixelParameterLocation(pixel, active, 0) == 0u &&
              ShaderPixelParameterLocation(pixel, active, 1) == 2u &&
              ShaderPixelParameterLocation(pixel, active, 2) == 1u,
          "a synthesized opposite-mode alias must not occupy another "
          "vertex export's physical location");
  std::printf("[host]    %-32s ok\n", name);
}

void CheckRectListShaders() {
  constexpr const char *name = "RectListShaders";

  ShaderRecompiler::IR::CompiledShaderInfo program{};
  program.info.inputs.push_back(
      {ShaderRecompiler::IR::StageInputKind::Parameter, 0, 4, "in_param_0"});
  program.info.inputs.push_back(
      {ShaderRecompiler::IR::StageInputKind::Parameter, 1, 4, "in_param_1"});

  ShaderVertexInputInfo vertex{};
  ShaderRecompiler::IR::CompiledShaderInfo vertex_program{.param_export_mask = 1u};
  ShaderRecompiler::IR::ResourceSnapshot empty_snapshot;
  vertex.stage.program = &vertex_program;
  vertex.stage.resources = &empty_snapshot;
  ShaderPixelInputInfo pixel{};
  pixel.parameter_mode = ShaderPixelParameterMode::Rectangle;
  pixel.input_num = 2;
  pixel.interpolator_settings[0] = 0x400u;
  pixel.interpolator_settings[1] = 0;
  pixel.stage.program = &program;
  pixel.stage.resources = &empty_snapshot;
  std::vector<uint32_t> perspective_specialization;
  BuildStageStaticKey(pixel, perspective_specialization);
  pixel.ps_no_perspective = true;
  std::vector<uint32_t> no_perspective_specialization;
  BuildStageStaticKey(pixel, no_perspective_specialization);
  pixel.ps_no_perspective = false;
  Require(name, "pipeline identity",
          perspective_specialization != no_perspective_specialization,
          "pixel interpolation mode must participate in the shader and "
          "pipeline key");

  const std::array<uint32_t, 2> active_inputs = {0, 1};
  Require(name, "duplicate mapping",
          ShaderPixelParameterLocation(pixel, active_inputs, 0) == 0 &&
              ShaderPixelParameterLocation(pixel, active_inputs, 1) == 1,
          "duplicate pixel mappings must receive distinct effective output "
          "locations");

  const auto shaders = BuildRectListShaders(vertex, &pixel);
  Require(name, "SPIR-V version",
          shaders.control.size() > 1 && shaders.control[1] == 0x00010500u &&
              shaders.evaluation.size() > 1 &&
              shaders.evaluation[1] == 0x00010500u,
          "vector selection requires SPIR-V 1.5");
  ValidateSpirv(name, shaders.control);
  ValidateSpirv(name, shaders.evaluation);

  spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
  std::string control_text;
  std::string evaluation_text;
  Require(name, "control disassembly",
          tools.Disassemble(shaders.control, &control_text),
          "failed to disassemble rectangle-list tessellation control shader");
  Require(
      name, "evaluation disassembly",
      tools.Disassemble(shaders.evaluation, &evaluation_text),
      "failed to disassemble rectangle-list tessellation evaluation shader");
  Require(name, "control execution mode",
          control_text.find("TessellationControl") != std::string::npos &&
              control_text.find("OutputVertices 4") != std::string::npos,
          "rectangle-list control shader must produce four control points");
  Require(name, "evaluation execution modes",
          evaluation_text.find("TessellationEvaluation") != std::string::npos &&
              evaluation_text.find("Quads") != std::string::npos &&
              evaluation_text.find("SpacingEqual") != std::string::npos &&
              evaluation_text.find("VertexOrderCw") != std::string::npos,
          "rectangle-list evaluation shader has the wrong patch modes");
  Require(name, "no geometry stage",
          control_text.find("Geometry") == std::string::npos &&
              evaluation_text.find("Geometry") == std::string::npos,
          "rectangle-list expansion must not use geometry shaders");
  Require(name, "flat broadcast",
          CountText(control_text, "OpVectorTimesScalar") == 6 &&
              CountText(control_text, "OpSelect %v4float") == 2,
          "flat parameters must use guest vertex zero instead of reconstructed "
          "values");
  Require(name, "remapped interface",
          CountText(control_text, " Location 0") == 2 &&
              CountText(control_text, " Location 1") == 1 &&
              CountText(evaluation_text, " Location 0") == 2 &&
              CountText(evaluation_text, " Location 1") == 2,
          "duplicate pixel mappings must share one vertex input and keep "
          "distinct patch outputs");

  for (const auto flat : {0u, 0x400u}) {
    pixel.interpolator_settings[0] = flat;
    pixel.interpolator_settings[1] = flat;
    const auto shared = BuildRectListShaders(vertex, &pixel);
    ValidateSpirv(name, shared.control);
    ValidateSpirv(name, shared.evaluation);
    Require(name, "same-mode alias disassembly",
            tools.Disassemble(shared.control, &control_text) &&
                tools.Disassemble(shared.evaluation, &evaluation_text),
            "failed to disassemble shared rectangle-list parameters");
    Require(name, "same-mode interface deduplication",
            CountText(control_text, " Location 0") == 2 &&
                CountText(evaluation_text, " Location 0") == 2 &&
                control_text.find(" Location 1") == std::string::npos &&
                evaluation_text.find(" Location 1") == std::string::npos,
            "same-mode aliases must produce one input and one output at "
            "the shared location in each rectangle-list stage");
  }

  const auto position_only = BuildRectListShaders(vertex, nullptr);
  ValidateSpirv(name, position_only.control);
  ValidateSpirv(name, position_only.evaluation);
}

void CheckSpirvText(const TestCase &test, const std::vector<u32> &spirv) {
  if (test.required_spirv.empty() && test.forbidden_spirv.empty()) {
    return;
  }

  spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
  std::string text;
  if (!tools.Disassemble(spirv, &text)) {
    Fail(test.name, "SPIR-V disassembly",
         "failed to disassemble emitted SPIR-V");
  }
  for (const auto &required : test.required_spirv) {
    if (text.find(required) == std::string::npos) {
      Fail(test.name, "SPIR-V disassembly",
           std::string("missing required text: ") + required);
    }
  }
  for (const auto &forbidden : test.forbidden_spirv) {
    if (text.find(forbidden) != std::string::npos) {
      Fail(test.name, "SPIR-V disassembly",
           std::string("found forbidden text: ") + forbidden);
    }
  }
}

CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
  auto user_data =
      MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr);
  const auto uses_image =
      std::any_of(test.opcodes.begin(), test.opcodes.end(), [](auto op) {
        return op >= ShaderOpcode::IMAGE_GET_RESINFO &&
               op <= ShaderOpcode::IMAGE_GATHER4H;
      });
  if (uses_image && ((user_data[3] >> 28u) & 0xfu) == 0) {
    user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u;
  }
  if (uses_image) {
    user_data[3] = (user_data[3] & ~0xfffu) | test.image_descriptor_swizzle;
  }
  if (!test.has_user_data) {
    user_data[2] = static_cast<u32>(test.initial.size() * sizeof(u32));
  }
  ShaderRecompiler::CompileOptions options;
  options.stage = ShaderType::Compute;
  options.maintenance8_enabled = true;
  options.dump_ir = true;
  auto compute_info = test.compute_info;
  compute_info.host_subgroup_size = host_subgroup_size;
  options.input_info.compute = &compute_info;
  options.user_data = user_data;

  if (test.has_compute_info) {
    options.wave_size = test.compute_info.wave_size;
  }

  auto translated = ShaderRecompiler::TranslateProgram(test.code, options);
  auto resource_plan =
      ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
  ShaderRecompiler::IR::ResourceSnapshot resources;
  ShaderRecompiler::IR::ResourceSpecialization specialization;
  const ShaderRecompiler::IR::SrtRuntime runtime{
      .user_data = options.user_data,
      .shader_base = reinterpret_cast<uint64_t>(test.code.data()),
      .read_memory = ReadTestMemory,
      .userdata = const_cast<std::vector<u32> *>(&test.initial),
      .read_specialization_memory = ReadTestMemory,
  };
  Require(test.name, "resource materialization",
          ShaderRecompiler::IR::MaterializeResources(
              resource_plan, runtime, resources, specialization),
          "translated resources could not be materialized");
  auto result = ShaderRecompiler::CompileProgram(
      std::move(translated), options, specialization);
  for (const auto &[text, expected] : test.decoded_counts) {
    const auto actual = CountText(result.decoded_dump, text);
    Require(test.name, "decoded RDNA2", actual == expected,
            text + " count=" + std::to_string(actual) +
                ", expected=" + std::to_string(expected));
  }
  for (const auto &[text, expected] : test.ir_counts) {
    const auto actual = CountText(result.ir_dump, text);
    Require(test.name, "typed IR", actual == expected,
            text + " count=" + std::to_string(actual) +
                ", expected=" + std::to_string(expected));
  }
  if (test.expected_mip_descriptors != 0) {
    const auto image = std::ranges::find_if(
        result.program.info.images, [](const auto &resource) {
          return resource.mip_mode ==
                 ShaderRecompiler::IR::ImageMipMode::Dynamic;
        });
    Require(test.name, "dynamic mip specialization",
            image != result.program.info.images.end() &&
                image->mip_count == test.expected_mip_descriptors,
            "runtime descriptor range did not specialize the mip count");
    const auto kind = ShaderRecompiler::IR::DescriptorBindingForImage(*image);
    Require(test.name, "dynamic mip binding",
            kind && image->descriptor_index != UINT32_MAX &&
                image->descriptor_index + image->mip_count <=
                    result.program.bindings.descriptor_counts[static_cast<size_t>(*kind)],
            "dynamic image did not receive one descriptor per mip");
  }
  if (test.expand_shader_data_storage) {
    auto &block = *result.program.blocks.front();
    for (u32 reg = 0; reg < 33; reg++) {
      auto &user_data = block.AppendNewInst(
          ShaderRecompiler::IR::ValueOpcode::GetUserData,
          {ShaderRecompiler::IR::Value(
              static_cast<ShaderRecompiler::IR::ScalarReg>(reg))});
      block.AppendNewInst(ShaderRecompiler::IR::ValueOpcode::ReferenceU32,
                          {ShaderRecompiler::IR::Value(&user_data)});
    }
    result.program.shader_info_complete = false;
    ShaderRecompiler::IR::CollectShaderInfo(result.program, options.input_info);
    result.spirv = ShaderRecompiler::Spirv::EmitProgram(
        result.program, options.input_info, 0u, options.maintenance8_enabled);
    const auto shader_data = result.program.bindings.descriptor_counts[
        static_cast<size_t>(ShaderRecompiler::IR::DescriptorBindingKind::ShaderData)];
    Require(test.name, "binding layout",
            result.program.info.user_data_registers.size() == 33 &&
                !result.program.bindings.UsesPushData() &&
                shader_data != 0,
            "oversized shader data did not use its storage fallback");
  }
  Require(test.name, "SPIR-V emit", !result.spirv.empty(),
          "recompiler returned empty SPIR-V");
  ValidateSpirv(test.name, result.spirv);
  CheckSpirvText(test, result.spirv);
  if (test.expected_buffer_resources) {
    std::vector<u32> resources;
    for (u32 i = 0; i < result.program.info.buffers.size(); ++i) {
      if (result.program.info.buffers[i].descriptor_index != UINT32_MAX) resources.push_back(i);
    }
    Require(test.name, "live buffer bindings",
            resources == *test.expected_buffer_resources,
            "dead buffer changed the surviving descriptor order");
  }
  std::vector<u32> packed_user_data;
  for (const auto reg : result.program.info.user_data_registers) {
    packed_user_data.push_back(
        resources.user_data[reg - result.program.user_data_base]);
  }
  packed_user_data.resize(result.program.bindings.ShaderDataDwords());
  for (u32 resource = 0; resource < result.program.info.buffers.size(); ++resource) {
    const auto index = result.program.info.buffers[resource].descriptor_index;
    if (index == UINT32_MAX) continue;
    const auto offset = resource < test.storage_buffer_offsets.size()
                            ? test.storage_buffer_offsets[resource] : 0u;
    Require(test.name, "shader data", offset < 256,
            "storage buffer offset is not representable");
    packed_user_data[result.program.bindings.memory_offset_dword + index / 4u] |=
        offset << ((index % 4u) * 8u);
  }
  return {std::move(result.spirv), std::move(result.program),
          std::move(resources), std::move(packed_user_data)};
}

std::array<u32, 64> MakeStructuredStorageBufferData(u32 stride_bytes,
                                                    u32 num_records,
                                                    bool add_tid = false,
                                                    u32 format = 0) {
  std::array<u32, 64> data{};
  data[1] = (stride_bytes & 0x3fffu) << 16u;
  data[2] = num_records;
  data[3] = DstSel(4, 5, 6, 7) | (1u << 24u) |
            (stride_bytes == 0 ? 3u << 28u : 0u);
  if (add_tid) {
    data[3] |= 1u << 23u;
  }
  data[3] |= (format & 0x7fu) << 12u;
  return data;
}

std::array<u32, 64> MakeStorageTextureData(Prospero::BufferFormat format) {
  std::array<u32, 64> data{};
  data[0] = 0x1000u;
  data[1] = (static_cast<uint32_t>(format) & 0x1ffu) << 20u;
  data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u;
  return data;
}

std::string StorageUint2DImageBindingName(bool atomic) {
  ShaderRecompiler::IR::ImageResource image{};
  image.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
  image.numeric_class = Prospero::TextureNumericClass::Uint;
  image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
  image.atomic = atomic;
  const auto binding = ShaderRecompiler::IR::DescriptorBindingForImage(image);
  Require("StorageUint2DImageBindingName", "descriptor binding", binding.has_value(),
          "storage image has no descriptor binding");
  return "image_" + std::to_string(static_cast<uint32_t>(*binding));
}

CompiledShader CompileFragmentCase(const GraphicsCase &test) {
  const auto user_data =
      MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr);
  ShaderPixelInputInfo pixel_info{};
  pixel_info.input_num =
      test.pixel_interpolator_settings.empty()
          ? 1u
          : static_cast<u32>(test.pixel_interpolator_settings.size());
  pixel_info.ps_no_perspective = test.pixel_no_perspective;
  pixel_info.parameter_mode = test.provoking_vtx_last ? ShaderPixelParameterMode::LastVertex
                                                    : ShaderPixelParameterMode::FirstVertex;
  pixel_info.ps_ancillary = test.pixel_ancillary;
  pixel_info.ps_front_face = test.pixel_front_face;
  pixel_info.ps_pos_w = test.pixel_position_w;
  pixel_info.ps_depth_export_enable = test.pixel_depth_export;
  pixel_info.ps_system_input_base = 2;
  pixel_info.ps_perspective_sample_vgpr = test.pixel_perspective_sample_vgpr;
  pixel_info.ps_perspective_centroid_vgpr = test.pixel_perspective_centroid_vgpr;
  pixel_info.custom_interpolation_mask = test.pixel_custom_interpolation_mask;
  for (u32 i = 0; i < std::size(pixel_info.interpolator_settings); i++) {
    pixel_info.interpolator_settings[i] = i;
  }
  for (u32 i = 0; i < test.pixel_interpolator_settings.size() &&
                  i < std::size(pixel_info.interpolator_settings);
       i++) {
    pixel_info.interpolator_settings[i] = test.pixel_interpolator_settings[i];
  }

  ShaderRecompiler::CompileOptions options;
  options.stage = ShaderType::Pixel;
  options.maintenance8_enabled = true;
  options.dump_ir = false;
  options.input_info.pixel = &pixel_info;
  options.user_data = user_data;

  auto translated =
      ShaderRecompiler::TranslateProgram(test.fragment_code, options);
  auto resource_plan =
      ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
  ShaderRecompiler::IR::ResourceSnapshot resources;
  ShaderRecompiler::IR::ResourceSpecialization specialization;
  const ShaderRecompiler::IR::SrtRuntime runtime{
      .user_data = options.user_data,
      .shader_base = reinterpret_cast<uint64_t>(test.fragment_code.data()),
  };
  Require(test.name, "resource materialization",
          ShaderRecompiler::IR::MaterializeResources(
              resource_plan, runtime, resources, specialization),
          "translated resources could not be materialized");
  auto result = ShaderRecompiler::CompileProgram(
      std::move(translated), options, specialization);
  Require(test.name, "SPIR-V emit", !result.spirv.empty(),
          "recompiler returned empty SPIR-V");
  ValidateSpirv(test.name, result.spirv);
  std::vector<u32> packed_user_data;
  for (const auto reg : result.program.info.user_data_registers) {
    packed_user_data.push_back(
        resources.user_data[reg - result.program.user_data_base]);
  }
  packed_user_data.resize(result.program.bindings.ShaderDataDwords());
  return {std::move(result.spirv), std::move(result.program),
          std::move(resources), std::move(packed_user_data)};
}

std::array<u32, 64> MakeSampledTextureData(Prospero::BufferFormat format) {
  return MakeStorageTextureData(format);
}

namespace TestSpv {

std::vector<u32> MakePassthroughVertexSpirv(bool layered, float clip_w = 1.0f) {
  using ShaderRecompiler::Spirv::Builder;

  Builder b;
  const auto void_type = b.Type(spv::OpTypeVoid);
  const auto uint_type = b.Type(spv::OpTypeInt, 32, 0);
  const auto int_type = b.Type(spv::OpTypeInt, 32, 1);
  const auto float_type = b.Type(spv::OpTypeFloat, 32);
  const auto vec2_type = b.Type(spv::OpTypeVector, float_type, 2);
  const auto vec4_type = b.Type(spv::OpTypeVector, float_type, 4);
  const auto per_vertex_type =
      b.DecoratedType(spv::OpTypeStruct, {{spv::OpDecorate, {spv::DecorationBlock}},
                       {spv::OpMemberDecorate,
                        {0, spv::DecorationBuiltIn, spv::BuiltInPosition}}}, vec4_type);
  const auto ptr_input_vec2 =
      b.Type(spv::OpTypePointer, spv::StorageClassInput, vec2_type);
  const auto ptr_input_vec4 =
      b.Type(spv::OpTypePointer, spv::StorageClassInput, vec4_type);
  const auto ptr_output_vec4 =
      b.Type(spv::OpTypePointer, spv::StorageClassOutput, vec4_type);
  const auto ptr_output_per_vertex =
      b.Type(spv::OpTypePointer, spv::StorageClassOutput, per_vertex_type);
  const auto func_type = b.Type(spv::OpTypeFunction, void_type);
  const auto const_u32_0 = b.Constant(spv::OpConstant, uint_type, 0);
  const auto const_f32_0 =
      b.Constant(spv::OpConstant, float_type, 0x00000000u);
  const auto const_f32_w =
      b.Constant(spv::OpConstant, float_type, std::bit_cast<u32>(clip_w));
  const auto in_pos =
      b.DefineGlobalVariable(ptr_input_vec2, spv::StorageClassInput);
  const auto in_color =
      b.DefineGlobalVariable(ptr_input_vec4, spv::StorageClassInput);
  const auto out_color =
      b.DefineGlobalVariable(ptr_output_vec4, spv::StorageClassOutput);
  const auto per_vertex =
      b.DefineGlobalVariable(ptr_output_per_vertex, spv::StorageClassOutput);
  u32 instance = 0;
  u32 layer = 0;
  std::vector<u32> interfaces = {in_pos, in_color, per_vertex, out_color};
  if (layered) {
    instance = b.DefineGlobalVariable(
        b.Type(spv::OpTypePointer, spv::StorageClassInput, int_type),
        spv::StorageClassInput);
    layer = b.DefineGlobalVariable(
        b.Type(spv::OpTypePointer, spv::StorageClassOutput, int_type),
        spv::StorageClassOutput);
    b.AddAnnotation(spv::OpDecorate, instance, spv::DecorationBuiltIn,
                     spv::BuiltInInstanceIndex);
    b.AddAnnotation(spv::OpDecorate, layer, spv::DecorationBuiltIn, spv::BuiltInLayer);
    b.RequireVersion(0x00010500u);
    b.RequireCapability(spv::CapabilityShaderLayer);
    interfaces.insert(interfaces.end(), {instance, layer});
  }
  const auto main = b.AllocateId();
  const auto label = b.AllocateId();
  const auto pos2 = b.AllocateId();
  const auto color4 = b.AllocateId();
  const auto pos_x = b.AllocateId();
  const auto pos_y = b.AllocateId();
  const auto position = b.AllocateId();
  const auto position_ptr = b.AllocateId();

  b.RequireCapability(spv::CapabilityShader);
  b.AddMemoryModel(spv::AddressingModelLogical, spv::MemoryModelGLSL450);
  b.AddEntryPoint(spv::ExecutionModelVertex, main, "main", interfaces);
  b.AddAnnotation(spv::OpDecorate, in_pos, spv::DecorationLocation, 0);
  b.AddAnnotation(spv::OpDecorate, in_color, spv::DecorationLocation, 1);
  b.AddAnnotation(spv::OpDecorate, out_color, spv::DecorationLocation, 0);

  b.AddFunction(spv::OpFunction, void_type, main, spv::FunctionControlMaskNone,
                 func_type);
  b.AddFunction(spv::OpLabel, label);
  b.AddFunction(spv::OpLoad, vec2_type, pos2, in_pos);
  b.AddFunction(spv::OpLoad, vec4_type, color4, in_color);
  b.AddFunction(spv::OpCompositeExtract, float_type, pos_x, pos2, 0);
  b.AddFunction(spv::OpCompositeExtract, float_type, pos_y, pos2, 1);
  b.AddFunction(spv::OpCompositeConstruct, vec4_type, position, pos_x, pos_y,
                 const_f32_0, const_f32_w);
  b.AddFunction(spv::OpAccessChain, ptr_output_vec4, position_ptr, per_vertex,
                 const_u32_0);
  b.AddFunction(spv::OpStore, position_ptr, position);
  b.AddFunction(spv::OpStore, out_color, color4);
  if (layered) {
    const auto index = b.AllocateId();
    b.AddFunction(spv::OpLoad, int_type, index, instance);
    b.AddFunction(spv::OpStore, layer, index);
  }
  b.AddFunction(spv::OpReturn);
  b.AddFunction(spv::OpFunctionEnd);
  return b.Build();
}

} // namespace TestSpv

class VulkanHarness {
public:
  VulkanHarness() { Init(); }
  ~VulkanHarness() { Destroy(); }

  VulkanHarness(const VulkanHarness &) = delete;
  VulkanHarness &operator=(const VulkanHarness &) = delete;

  struct Buffer {
    vk::Buffer buffer = nullptr;
    vk::DeviceMemory memory = nullptr;
    vk::DeviceAddress device_address = 0;
    vk::DeviceSize size = 0;
    bool coherent = false;
  };

  struct Image {
    vk::Image image = nullptr;
    vk::DeviceMemory memory = nullptr;
    vk::ImageView view = nullptr;
    vk::Format format = vk::Format::eUndefined;
    vk::ImageLayout layout = vk::ImageLayout::eUndefined;
    u32 width = 0;
    u32 height = 0;
    u32 layers = 1;
    u32 mip_levels = 1;
    u32 dwords_per_pixel = 0;
  };

  [[nodiscard]] vk::Device Device() const { return m_device; }
  [[nodiscard]] u32 SubgroupSize() const {
    vk::PhysicalDeviceSubgroupProperties subgroup{};
    vk::PhysicalDeviceProperties2 properties{};
    properties.pNext = &subgroup;
    m_physical_device.getProperties2(&properties);
    return subgroup.subgroupSize;
  }
  // A consumer GPU may lack the production rasterization features the fragment
  // cases need. The compute cases still run on such a device, so the harness reports
  // the capability instead of failing the whole binary before the first case.
  [[nodiscard]] bool RasterizationSupported() const {
    return m_rasterization_supported;
  }
  void SkipRasterizationCases(u32 count) { m_skipped_cases += count; }
  [[nodiscard]] u32 SkippedCaseCount() const { return m_skipped_cases; }
  [[nodiscard]] GraphicContext &RuntimeContext() {
    EnsureRuntimeContext();
    return m_runtime_context;
  }
  [[nodiscard]] RenderContext &RuntimeRenderer() {
    EnsureRuntimeContext();
    return Renderer();
  }

  void CheckHostImageAllocation() {
    constexpr const char *name = "HostImageAllocation";
    auto &graphics = RuntimeContext();
    VulkanImage image;
    vk::ImageCreateInfo create{};
    create.flags = vk::ImageCreateFlagBits::eMutableFormat;
    create.imageType = vk::ImageType::e2D;
    create.format = vk::Format::eR8G8B8A8Unorm;
    create.extent = {16, 8, 1};
    create.mipLevels = 3;
    create.arrayLayers = 4;
    create.samples = vk::SampleCountFlagBits::e1;
    create.tiling = vk::ImageTiling::eOptimal;
    create.usage = vk::ImageUsageFlagBits::eTransferSrc |
                   vk::ImageUsageFlagBits::eTransferDst |
                   vk::ImageUsageFlagBits::eSampled;
    create.initialLayout = vk::ImageLayout::eUndefined;

    for (uint32_t pass = 0; pass < 2; ++pass) {
      Require(name, "create", graphics.CreateImage(create, image),
              "could not allocate native image");
      Require(name, "allocation metadata",
              image.image != nullptr && image.allocation != nullptr &&
                  image.format == create.format &&
                  image.image_type == create.imageType &&
                  image.extent == create.extent &&
                  image.layers == create.arrayLayers &&
                  image.mip_levels == create.mipLevels &&
                  image.samples == static_cast<uint32_t>(create.samples) &&
                  image.usage == create.usage && image.flags == create.flags,
              "native image metadata did not follow its creation descriptor");
      Require(name, "fresh synchronization state",
              image.state.layout == create.initialLayout &&
                  image.state.access_mask == vk::AccessFlags2{} &&
                  image.state.pl_stage == vk::PipelineStageFlagBits2::eAllCommands &&
                  image.subresource_states.empty(),
              "new allocation retained synchronization state from an old image");
      VkMemoryPropertyFlags properties{};
      vmaGetAllocationMemoryProperties(graphics.allocator, image.allocation,
                                       &properties);
      Require(name, "allocation policy",
              (properties & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0,
              "native image lost its device-local memory requirement");

      // A reused presentation image may still describe its previous recording.
      // No commands use this fixture; seed that bookkeeping before retirement.
      image.state = {vk::PipelineStageFlagBits2::eTransfer,
                     vk::AccessFlagBits2::eTransferWrite,
                     vk::ImageLayout::eTransferDstOptimal};
      image.subresource_states.assign(image.layers * image.mip_levels, image.state);
      graphics.DeleteImage(image);
      Require(name, "retirement",
              image.image == nullptr && image.allocation == nullptr,
              "retired native image retained an allocation handle");

      create.flags = vk::ImageCreateFlagBits::e2DArrayCompatible;
      create.imageType = vk::ImageType::e3D;
      create.format = vk::Format::eR32Uint;
      create.extent = {8, 4, 2};
      create.mipLevels = 2;
      create.arrayLayers = 1;
      create.usage = vk::ImageUsageFlagBits::eTransferSrc |
                     vk::ImageUsageFlagBits::eTransferDst;
    }
    std::printf("[host]    %-32s ok\n", name);
  }

  void CheckDescriptorHeapLargeSet() {
    EnsureRuntimeContext();
    std::array<vk::DescriptorSetLayoutBinding, 3> bindings{};
    constexpr std::array stages{
        vk::ShaderStageFlagBits::eVertex,
        vk::ShaderStageFlagBits::eFragment,
        vk::ShaderStageFlagBits::eCompute,
    };
    for (uint32_t i = 0; i < bindings.size(); i++) {
      bindings[i].binding = i;
      bindings[i].descriptorType = vk::DescriptorType::eSampler;
      bindings[i].descriptorCount = 16;
      bindings[i].stageFlags = stages[i];
    }
    vk::DescriptorSetLayoutCreateInfo create{};
    create.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
    create.bindingCount = static_cast<uint32_t>(bindings.size());
    create.pBindings = bindings.data();
    vk::DescriptorSetLayout layout = nullptr;
    RequireVk("DescriptorHeapLargeSet", "layout",
              m_device.createDescriptorSetLayout(&create, nullptr, &layout),
              "vkCreateDescriptorSetLayout");

    vk::PipelineLayoutCreateInfo pipeline_create{};
    pipeline_create.sType = vk::StructureType::ePipelineLayoutCreateInfo;
    pipeline_create.setLayoutCount = 1;
    pipeline_create.pSetLayouts = &layout;
    vk::PipelineLayout pipeline_layout = nullptr;
    RequireVk("DescriptorHeapLargeSet", "pipeline layout",
              m_device.createPipelineLayout(&pipeline_create, nullptr,
                                            &pipeline_layout),
              "vkCreatePipelineLayout");

    {
      RenderContext context(m_runtime_context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      scheduler.Begin(registers, user_config, shaders);

      const auto commit = [&] {
        const auto set = context.GetDescriptorHeap().Commit(layout);
        Require("DescriptorHeapLargeSet", "allocation", set != nullptr,
                "ordinary descriptor heap could not allocate an oversized sampler set");
        scheduler.Current().Handle().bindDescriptorSets(
            vk::PipelineBindPoint::eCompute, pipeline_layout, 0, 1, &set, 0,
            nullptr);
      };

      for (uint32_t i = 0; i < 22; i++) {
        commit();
      }
      scheduler.Finish();
      for (uint32_t i = 0; i < 21; i++) {
        commit();
      }
    }

    m_device.destroyPipelineLayout(pipeline_layout, nullptr);
    m_device.destroyDescriptorSetLayout(layout, nullptr);
    std::printf("[host]    %-32s ok\n", "DescriptorHeapLargeSet");
  }

  void CheckGraphicsPushConstantBank() {
    constexpr const char *name = "GraphicsPushConstantStages";
    EnsureRuntimeContext();
    RenderContext context(m_runtime_context);
    auto &scheduler = context.GetCommandScheduler();
    HW::Context registers{};
    HW::UserConfig user_config{};
    HW::Shader shaders{};
    scheduler.Begin(registers, user_config, shaders);

    ShaderRecompiler::IR::CompiledShaderInfo vertex_program{};
    vertex_program.stage = ShaderType::Vertex;
    vertex_program.bindings.push_data_start_dword = 2;
    vertex_program.info.user_data_registers = {0, 1};
    vertex_program.bindings.memory_offset_dword = 2;
    ShaderRecompiler::IR::CompiledShaderInfo pixel_program{};
    pixel_program.stage = ShaderType::Pixel;
    pixel_program.bindings.push_data_start_dword = 0;
    pixel_program.info.user_data_registers = {0, 1};
    pixel_program.bindings.memory_offset_dword = 2;
    ShaderRecompiler::IR::ResourceSnapshot vertex_snapshot;
    vertex_snapshot.user_data = {0x11111111u, 0x22222222u};
    ShaderStageRuntime vertex_runtime{&vertex_program, &vertex_snapshot};
    ShaderRecompiler::IR::ResourceSnapshot pixel_snapshot;
    pixel_snapshot.user_data = {0x33333333u, 0x44444444u};
    ShaderStageRuntime pixel_runtime{&pixel_program, &pixel_snapshot};
    PreparedBindings vertex;
    PreparedBindings pixel;
    auto &executor = context.GetRenderExecutor();
    executor.PrepareBindings(vertex_runtime, vertex);
    executor.PrepareBindings(pixel_runtime, pixel);
    const auto *vertex_storage = vertex.shader_data.data();
    const auto *pixel_storage = pixel.shader_data.data();
    vertex.buffer_sources.push_back({});
    vertex.buffers.push_back({});
    vertex.gds.offset = 1;
    vertex.flattened_srt.offset = 1;
    vertex.shader_data_buffer.offset = 1;
    vertex_snapshot.user_data = {0x55555555u, 0x66666666u};
    pixel_snapshot.user_data = {0x77777777u, 0x88888888u};
    executor.PrepareBindings(vertex_runtime, vertex);
    executor.PrepareBindings(pixel_runtime, pixel);
    std::array<PreparedBindings *, 2> stages{&vertex, &pixel};
    RenderExecutorTestAccess::PrepareGraphicsBindings(executor, stages, {});
    Require(name, "prepared binding scratch",
            vertex.shader_data.data() == vertex_storage &&
                pixel.shader_data.data() == pixel_storage &&
                vertex.buffer_sources.empty() && vertex.buffers.empty() &&
                vertex.gds.offset == 0 && vertex.flattened_srt.offset == 0 &&
                vertex.shader_data_buffer.offset == 0,
            "repeated shader binding preparation allocated or retained stale state");

    const auto pipeline = RenderExecutorTestAccess::CommitBindings(
        context.GetRenderExecutor(), scheduler.Current(), vertex, pixel);
    Require(name, "shared push data",
            vertex.shader_data ==
                    std::vector<uint32_t>{0x55555555u, 0x66666666u} &&
                pixel.shader_data ==
                    std::vector<uint32_t>{0x77777777u, 0x88888888u},
            "graphics stages did not commit their shared push data");
    scheduler.Finish();
    RenderExecutorTestAccess::DestroyDescriptorPipelines(
        context.GetRenderExecutor(), std::span {&pipeline, 1u});
    std::printf("[host]    %-32s ok\n", name);
  }

  void CheckSchedulerTimeline() {
    EnsureRuntimeContext();
    CommandScheduler scheduler(Renderer(), m_runtime_context);
    HW::Context registers{};
    HW::UserConfig user_config{};
    HW::Shader shaders{};
    scheduler.Begin(registers, user_config, shaders);

    auto *command = &scheduler.Current();
    const auto original_handle = command->Handle();
    HW::Context next_registers{};
    HW::UserConfig next_user_config{};
    HW::Shader next_shaders{};
    scheduler.Begin(next_registers, next_user_config, next_shaders);
    Require("SchedulerTimeline", "guest context rebind",
            &scheduler.Current() == command &&
                command->Handle() == original_handle &&
                &command->GetRegisters() == &next_registers &&
                &command->GetUserConfig() == &next_user_config &&
                &command->GetShaders() == &next_shaders,
            "rebinding guest state replaced the active command recording");
    scheduler.FlushAndWait();
    Require("SchedulerTimeline", "guest context after recording restart",
            scheduler.Active() && &scheduler.Current() == command &&
                &command->GetRegisters() == &next_registers &&
                &command->GetUserConfig() == &next_user_config &&
                &command->GetShaders() == &next_shaders,
            "restarting command recording lost the current guest context");
    scheduler.Begin(registers, user_config, shaders);

    const auto first_tick = scheduler.CurrentTick();
    uint32_t completed = 0;
    scheduler.DeferOperation([owned = std::make_unique<uint32_t>(1),
                              &completed] { completed = *owned; });
    scheduler.Flush();
    scheduler.Wait(first_tick);
    Require("SchedulerTimeline", "wait boundary",
            scheduler.CurrentTick() == first_tick + 1 &&
                scheduler.IsFree(first_tick) && completed == 0,
            "timeline wait released a resource in the middle of an operation");
    scheduler.PopPendingOperations();
    Require("SchedulerTimeline", "first tick", completed == 1,
            "operation boundary did not release its deferred operation");

    const auto second_tick = scheduler.CurrentTick();
    std::atomic<bool> priority_in_callback{false};
    scheduler.DeferPriorityOperation([owned = std::make_unique<uint32_t>(2),
                                      &completed, &priority_in_callback] {
      priority_in_callback = CommandScheduler::InDeferredOperation();
      completed = *owned;
    });
    scheduler.Finish();
    scheduler.DrainPriorityOperations();
    Require("SchedulerTimeline", "finish",
            scheduler.CurrentTick() == second_tick + 1 &&
                scheduler.IsFree(second_tick) && completed == 2 &&
                priority_in_callback.load() &&
                !CommandScheduler::InDeferredOperation(),
            "scheduler completion drain lost its tick or callback context");

    const auto implicit_flush_tick = scheduler.CurrentTick();
    scheduler.DeferOperation([owned = std::make_unique<uint32_t>(3),
                              &completed] { completed = *owned; });
    scheduler.Wait(implicit_flush_tick);
    Require("SchedulerTimeline", "implicit flush",
            scheduler.CurrentTick() == implicit_flush_tick + 1 &&
                scheduler.IsFree(implicit_flush_tick) && completed == 2,
            "waiting for the current tick released a deferred resource");
    scheduler.PopPendingOperations();
    Require("SchedulerTimeline", "implicit boundary", completed == 3,
            "operation boundary did not release the implicit tick callback");

    vk::SemaphoreTypeCreateInfo timeline_type{};
    timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo;
    timeline_type.semaphoreType = vk::SemaphoreType::eTimeline;
    vk::SemaphoreCreateInfo timeline_create{};
    timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo;
    timeline_create.pNext = &timeline_type;
    vk::Semaphore external_timeline = nullptr;
    Require("SchedulerTimeline", "external timeline create",
            m_runtime_context.device.createSemaphore(&timeline_create, nullptr,
                                                     &external_timeline) ==
                    vk::Result::eSuccess &&
                external_timeline != nullptr,
            "failed to create the external timeline semaphore");

    constexpr uint64_t external_wait_value = 9;
    const auto external_wait_tick = scheduler.CurrentTick();
    SubmitInfo external_wait;
    external_wait.AddWait(external_timeline, external_wait_value);
    constexpr size_t blocked_submission_count = 6;
    std::array<vk::CommandBuffer, blocked_submission_count> blocked_handles{};
    for (size_t i = 0; i < blocked_submission_count; ++i) {
      blocked_handles[i] = scheduler.Current().Handle();
      if (i == 0) {
        scheduler.Flush(external_wait);
      } else {
        scheduler.Flush();
      }
    }
    bool distinct_handles = true;
    for (size_t i = 0; i < blocked_handles.size(); ++i) {
      for (size_t j = i + 1; j < blocked_handles.size(); ++j) {
        distinct_handles &= blocked_handles[i] != blocked_handles[j];
      }
    }
    const auto last_blocked_tick = scheduler.CurrentTick() - 1;
    Require("SchedulerTimeline", "external timeline overflow",
            distinct_handles &&
                last_blocked_tick ==
                    external_wait_tick + blocked_submission_count - 1 &&
                !scheduler.IsFree(last_blocked_tick),
            "blocked submissions did not grow the command-buffer pool");
    vk::SemaphoreSignalInfo signal_info{};
    signal_info.sType = vk::StructureType::eSemaphoreSignalInfo;
    signal_info.semaphore = external_timeline;
    signal_info.value = external_wait_value;
    Require("SchedulerTimeline", "external timeline signal",
            m_runtime_context.device.signalSemaphore(&signal_info) ==
                vk::Result::eSuccess,
            "failed to signal the external timeline semaphore");
    scheduler.Wait(last_blocked_tick);
    m_runtime_context.device.destroySemaphore(external_timeline, nullptr);
    scheduler.Flush();
    scheduler.Flush();
    Require(
        "SchedulerTimeline", "timeline pool reuse",
        std::ranges::find(blocked_handles, scheduler.Current().Handle()) !=
            blocked_handles.end(),
        "completed command buffers were not reused after timeline progress");
    scheduler.Shutdown();

    CommandScheduler draining(Renderer(), m_runtime_context);
    HW::Context drain_registers{};
    HW::UserConfig drain_user_config{};
    HW::Shader drain_shaders{};
    draining.Begin(drain_registers, drain_user_config, drain_shaders);
    uint32_t reentrant = 0;
    draining.DeferOperation([&draining, &reentrant] {
      reentrant++;
      draining.DeferOperation([&reentrant] { reentrant++; });
    });

    std::binary_semaphore priority_entered{0};
    std::binary_semaphore release_priority{0};
    draining.DeferPriorityOperation(
        [&draining, &priority_entered, &release_priority] {
          priority_entered.release();
          draining.Shutdown();
          release_priority.acquire();
        });
    std::jthread shutdown_thread([&draining] { draining.Shutdown(); });
    priority_entered.acquire();
    std::jthread second_shutdown_thread([&draining] { draining.Shutdown(); });

    std::atomic<bool> concurrent_completed{false};
    std::jthread concurrent_defer([&draining, &concurrent_completed] {
      draining.DeferOperation(
          [&concurrent_completed] { concurrent_completed = true; });
    });
    release_priority.release();
    shutdown_thread.join();
    second_shutdown_thread.join();
    concurrent_defer.join();
    Require("SchedulerTimeline", "shutdown drain",
            reentrant == 2 && concurrent_completed.load(),
            "shutdown lost reentrant or concurrent deferred work");
    std::printf("[host]    %-32s ok\n", "SchedulerTimeline");
  }

  void CheckGpuMappedRangeLifecycle() {
    EnsureRuntimeContext();
    RenderContext context(m_runtime_context);
    auto &scheduler = context.GetCommandScheduler();
    HW::Context registers{};
    HW::UserConfig user_config{};
    HW::Shader shaders{};
    scheduler.Begin(registers, user_config, shaders);
    context.InitializeGpu(nullptr);
    auto &resources = context;

    constexpr uint64_t base = 0x0000000200000000ull;
    constexpr uint64_t page = 0x4000;
    const auto cpu_only_tick = scheduler.CurrentTick();
    resources.MapMemory(base, page * 4);
    resources.MapMemory(base + page * 2, page * 4);
    Require("GpuMappedRangeLifecycle", "union",
            resources.IsMapped(base, page * 6) &&
                !resources.IsMapped(base, page * 7),
            "overlapping maps did not form one interval union");

    resources.UnmapMemory(base + page * 2, page * 2);
    Require("GpuMappedRangeLifecycle", "subtract",
            resources.IsMapped(base, page * 2) &&
                resources.IsMapped(base + page * 4, page * 2) &&
                !resources.IsMapped(base, page * 6),
            "partial unmap did not punch the expected interval hole");

    resources.UnmapMemory(base + page * 2, page * 2);
    Require("GpuMappedRangeLifecycle", "idempotent unmap",
            resources.IsMapped(base, page * 2) &&
                resources.IsMapped(base + page * 4, page * 2),
            "unmapping an absent interval changed neighboring mappings");

    resources.UnmapMemory(base, page * 6);
    Require("GpuMappedRangeLifecycle", "clear",
            !resources.IsMapped(base, page * 6),
            "full unmap did not clear the interval union");

    constexpr uint64_t old_prt = base + page * 8;
    constexpr uint64_t new_prt = base + page * 16;
    resources.MapMemory(old_prt, page * 4);
    resources.UnmapMemory(old_prt, page * 4);
    resources.MapMemory(new_prt, page * 6);
    Require("GpuMappedRangeLifecycle", "PRT replacement",
            !resources.IsMapped(old_prt, page * 4) &&
                resources.IsMapped(new_prt, page * 6) &&
                scheduler.CurrentTick() == cpu_only_tick,
            "CPU-only unmaps submitted GPU work or lost full PRT coverage");

    scheduler.Finish();
    context.ShutdownGpu();
    std::printf("[host]    %-32s ok\n", "GpuMappedRangeLifecycle");
  }

  void CheckStreamBufferRing() {
    EnsureRuntimeContext();
    RenderContext context(m_runtime_context);
    auto &scheduler = context.GetCommandScheduler();
    HW::Context registers{};
    HW::UserConfig user_config{};
    HW::Shader shaders{};
    scheduler.Begin(registers, user_config, shaders);
    auto &resources = context;
    auto &cache = resources.GetBufferCache();
    constexpr std::array<std::pair<MemoryUsage, uint64_t>, 4> utilities{{
        {MemoryUsage::Upload, 512ull << 20},
        {MemoryUsage::Stream, 64ull << 20},
        {MemoryUsage::Download, 64ull << 20},
        {MemoryUsage::DeviceLocal, 128ull << 20},
    }};
    std::array<vk::Buffer, utilities.size()> handles{};
    for (size_t i = 0; i < utilities.size(); i++) {
      auto &utility = cache.GetUtilityBuffer(utilities[i].first);
      handles[i] = utility.Handle();
      Require("StreamBufferRing", "utility ownership",
              utility.Usage() == utilities[i].first &&
                  utility.Size() == utilities[i].second &&
                  utility.Handle() != nullptr,
              "BufferCache utility buffer does not match the fixed layout");
    }
    for (size_t i = 0; i < handles.size(); i++) {
      for (size_t j = i + 1; j < handles.size(); j++) {
        Require("StreamBufferRing", "utility identity",
                handles[i] != handles[j],
                "utility usages alias the same native buffer");
      }
    }
    auto *fixed_download = &cache.GetUtilityBuffer(MemoryUsage::Download);
    const auto fixed_handle = fixed_download->Handle();
    const auto [oversized_download, oversized_offset] =
        fixed_download->Map(fixed_download->Size() + 4, 64, false);
    const auto [download_probe, download_probe_offset] =
        fixed_download->Map(64, 64, false);
    Require("StreamBufferRing", "fixed download utility",
            oversized_download == nullptr && oversized_offset == 0 &&
                download_probe != nullptr && download_probe_offset == 0 &&
                &cache.GetUtilityBuffer(MemoryUsage::Download) ==
                    fixed_download &&
                fixed_download->Size() == (64ull << 20) &&
                fixed_download->Handle() == fixed_handle,
            "oversized download replaced or corrupted the fixed shared ring");
    fixed_download->Commit();

    const std::array<uint8_t, 16> overflow_data{
        0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5,
        0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5};
    auto &upload_stream = cache.GetUtilityBuffer(MemoryUsage::Stream);
    const auto stream_offset =
        upload_stream.Copy(overflow_data.data(), overflow_data.size(), 16);
    Require("StreamBufferRing", "cache-owned stream upload",
            upload_stream.Handle() != nullptr &&
                stream_offset < upload_stream.Size() &&
                std::memcmp(upload_stream.Mapped().data() + stream_offset,
                            overflow_data.data(), overflow_data.size()) == 0,
            "cache-owned stream upload lost bytes");

    const auto atom =
        m_runtime_context.physical_device_properties.limits.nonCoherentAtomSize;
    uint64_t policy_size = 17;
    uint64_t policy_alignment = 12;
    uint64_t coherent_size = policy_size;
    uint64_t coherent_alignment = policy_alignment;
    Require("StreamBufferRing", "non-coherent reservation policy",
            StreamBufferTestAccess::NormalizeReservation(
                false, 256, policy_size, policy_alignment) &&
                policy_size == 256 && policy_alignment == 768 &&
                StreamBufferTestAccess::NormalizeReservation(
                    true, 256, coherent_size, coherent_alignment) &&
                coherent_size == 17 && coherent_alignment == 12,
            "non-coherent reservations are not isolated at the atom/texel LCM");
    const auto small_ring_size = std::max<uint64_t>(64, atom * 2);
    StreamBuffer stream(m_runtime_context, scheduler, MemoryUsage::Upload,
                        small_ring_size);
    const auto first_size =
        stream.IsCoherent() ? small_ring_size - 16 : small_ring_size - atom + 1;
    constexpr uint64_t blocked_size = 24;
    const auto [first, first_offset] = stream.Map(first_size, 16, false);
    Require("StreamBufferRing", "first allocation",
            first != nullptr && first_offset == 0,
            "initial stream allocation failed");
    std::memset(first, 0x5a, static_cast<size_t>(first_size));
    stream.Commit();

    const auto [oversize, oversize_offset] =
        stream.Map(small_ring_size + 1, 1, false);
    Require("StreamBufferRing", "oversize",
            oversize == nullptr && oversize_offset == 0,
            "oversize allocation was accepted");
    const auto [blocked, blocked_offset] = stream.Map(blocked_size, 1, false);
    const auto [blocked_again, blocked_again_offset] =
        stream.Map(blocked_size, 1, false);
    Require("StreamBufferRing", "transactional failure",
            blocked == nullptr && blocked_offset == 0 &&
                blocked_again == nullptr && blocked_again_offset == 0,
            "failed wrap mutated stream state");

    scheduler.Flush();
    const auto [wrapped, wrapped_offset] = stream.Map(blocked_size, 1);
    Require("StreamBufferRing", "watched wrap",
            wrapped != nullptr && wrapped_offset == 0,
            "completed watch did not release the wrapped allocation");
    stream.Commit();

    const auto download_alignment = atom / std::gcd<uint64_t>(atom, 12) * 12;
    const auto download_ring_size = std::max<uint64_t>(
        64, download_alignment + std::max<uint64_t>(atom, 16));
    StreamBuffer download(m_runtime_context, scheduler, MemoryUsage::Download,
                          download_ring_size);
    const auto [download_data, download_offset] = download.Map(16, 8);
    Require("StreamBufferRing", "download allocation",
            download_data != nullptr && download_offset == 0,
            "download stream is not host visible");
    download.Commit();
    scheduler.Flush();
    const auto [second_download, second_download_offset] = download.Map(16, 12);
    Require("StreamBufferRing", "download atom isolation",
            second_download != nullptr &&
                (download.IsCoherent() ||
                 (download_offset % atom == 0 &&
                  second_download_offset % atom == 0 &&
                  second_download_offset >= download_offset + atom)) &&
                second_download_offset % 12 == 0,
            "download reservations share a non-coherent atom or lost caller "
            "alignment");
    download.Commit();

    scheduler.Finish();
    download.Invalidate(download_offset, 16);
    std::printf("[host]    %-32s ok\n", "StreamBufferRing");
  }

  void CheckGpuSuspendPoint() {
    constexpr const char *name = "GpuSuspendPoint";
    EnsureRuntimeContext();
    auto &context = Renderer();
    context.InitializeGpu(nullptr);
    auto &gpu = context.GetGpu();
    auto &scheduler = context.GetCommandScheduler();
    CommandProcessor setup(context, 0);

    vk::SemaphoreTypeCreateInfo timeline_type{};
    timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo;
    timeline_type.semaphoreType = vk::SemaphoreType::eTimeline;
    vk::SemaphoreCreateInfo timeline_create{};
    timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo;
    timeline_create.pNext = &timeline_type;
    vk::Semaphore gate = nullptr;
    Require(name, "timeline create",
            m_runtime_context.device.createSemaphore(
                &timeline_create, nullptr, &gate) == vk::Result::eSuccess,
            "failed to create the native completion gate");
    const auto signal = [&](uint64_t value) {
      vk::SemaphoreSignalInfo info{};
      info.sType = vk::StructureType::eSemaphoreSignalInfo;
      info.semaphore = gate;
      info.value = value;
      Require(name, "timeline signal",
              m_runtime_context.device.signalSemaphore(&info) ==
                  vk::Result::eSuccess,
              "failed to release native GPU work");
    };
    const auto finish = [&] {
      gpu.WaitForIdle();
      gpu.SendCommandSync([&] {
        scheduler.Finish();
        scheduler.WaitPriorityOperations(scheduler.CurrentTick() - 1);
      });
    };
    const auto write = [](uint32_t *packet, uint32_t *destination,
                          uint32_t value, bool predicated = false) {
      const auto address = reinterpret_cast<uint64_t>(destination);
      packet[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0) | uint32_t{predicated};
      packet[1] = 0;
      packet[2] = static_cast<uint32_t>(address);
      packet[3] = static_cast<uint32_t>(address >> 32u);
      packet[4] = value;
    };

    const auto cache_ticks = [&] {
      std::pair<uint64_t, uint64_t> ticks;
      gpu.SendCommandSync([&] {
        ticks = {TextureCacheTestAccess::GcTick(context.GetTextureCache()),
                 BufferCacheTestAccess::GcTick(context.GetBufferCache())};
      });
      return ticks;
    };
    const auto initial_cache_ticks = cache_ticks();
    uint32_t graphics_value = 0;
    uint32_t async_value = 0;
    std::array<uint32_t, 5> graphics_list{}, compute_list{};
    write(graphics_list.data(), &graphics_value, 17);
    write(compute_list.data(), &async_value, 29);
    // A frame can contain many lists; completing them must not age either cache.
    for (uint32_t i = 0; i < 32; ++i) {
      gpu.Submit(graphics_list, {});
      gpu.SubmitCompute(0x20, compute_list);
    }
    finish();
    Require(name, "command-list cache lifetime",
            graphics_value == 17 && async_value == 29 &&
                cache_ticks() == initial_cache_ticks,
            "individual graphics or compute lists aged the caches within a frame");

    gpu.SendCommandSync([&] {
      setup.BufferInit();
      SubmitInfo blocked;
      blocked.AddWait(gate, 1);
      scheduler.Flush(blocked);
    });
    std::binary_semaphore first_returned{0};
    std::jthread first([&] {
      gpu.SuspendPoint();
      first_returned.release();
    });
    const bool first_nonblocking =
        first_returned.try_acquire_for(std::chrono::seconds(2));
    if (!first_nonblocking) {
      signal(1);
    }
    first.join();
    Require(name, "first call", first_nonblocking,
            "the first suspend point waited for native GPU completion");

    // Let the marker reach Vulkan while its completion remains gated.
    std::binary_semaphore marker_parsed{0};
    std::jthread parser([&] {
      gpu.WaitForIdle();
      marker_parsed.release();
    });
    const bool parser_nonblocking =
        marker_parsed.try_acquire_for(std::chrono::seconds(2));
    if (!parser_nonblocking) {
      signal(1);
    }
    parser.join();
    Require(name, "marker insertion", parser_nonblocking,
            "inserting a suspend point blocked the GPU command thread");
    Require(name, "frame cache maintenance",
            cache_ticks() == std::pair{initial_cache_ticks.first + 1,
                                      initial_cache_ticks.second + 1},
            "a suspend point did not age both caches exactly once");

    std::binary_semaphore second_entered{0};
    std::binary_semaphore second_returned{0};
    std::jthread second([&] {
      second_entered.release();
      gpu.SuspendPoint();
      second_returned.release();
    });
    second_entered.acquire();
    const bool second_overtook_completion =
        second_returned.try_acquire_for(std::chrono::milliseconds(100));

    uint32_t compute_value = 0;
    std::array<uint32_t, 5> compute_commands{};
    write(compute_commands.data(), &compute_value, 33);
    bool host_lane_responded = false;
    std::binary_semaphore compute_progress{0};
    std::jthread compute([&] {
      gpu.SubmitCompute(0x20, compute_commands);
      gpu.SendCommandSync([&] { host_lane_responded = true; });
      gpu.WaitForIdle();
      compute_progress.release();
    });
    const bool compute_did_not_wait =
        compute_progress.try_acquire_for(std::chrono::seconds(2));
    signal(1);
    second.join();
    compute.join();
    finish();
    Require(name, "one outstanding point", !second_overtook_completion,
            "a second suspend point returned before the first native completion");
    Require(name, "independent queue progress",
            compute_did_not_wait && host_lane_responded && compute_value == 33,
            "waiting for a previous suspend point blocked compute submissions "
            "or the host command lane");
    Require(name, "frame count", gpu.GetFrameNum() == 2,
            "suspend-point insertion did not advance the frame counter once");
    Require(name, "async cache lifetime",
            cache_ticks() == std::pair{initial_cache_ticks.first + 2,
                                      initial_cache_ticks.second + 2},
            "asynchronous work aged the caches between frame boundaries");

    // A marker must stay behind a stalled graphics stream, then clear its
    // predicate before the next stream. Async compute releases that stream.
    alignas(16) uint64_t predicate = 0;
    uint32_t label = 0;
    uint32_t prefix = 0;
    uint32_t skipped = 0;
    uint32_t after_reset = 0;
    const auto predicate_address = reinterpret_cast<uint64_t>(&predicate);
    const auto label_address = reinterpret_cast<uint64_t>(&label);
    std::array<uint32_t, 21> before{};
    before[0] = KYTY_PM4(4, Pm4::IT_SET_PREDICATION, 0);
    before[1] = (3u << 16u) | (1u << 8u);
    before[2] = static_cast<uint32_t>(predicate_address);
    before[3] = static_cast<uint32_t>(predicate_address >> 32u);
    write(before.data() + 4, &skipped, 11, true);
    write(before.data() + 9, &prefix, 22);
    before[14] = KYTY_PM4(7, Pm4::IT_WAIT_REG_MEM, 0);
    before[15] = 0x10u | 3u;
    before[16] = static_cast<uint32_t>(label_address);
    before[17] = static_cast<uint32_t>(label_address >> 32u);
    before[18] = 1;
    before[19] = UINT32_MAX;
    std::array<uint32_t, 5> after{};
    write(after.data(), &after_reset, 44, true);
    std::binary_semaphore lane_entered{0};
    std::binary_semaphore release_lane{0};
    gpu.SendCommand([&] {
      lane_entered.release();
      release_lane.acquire();
    });
    lane_entered.acquire();
    gpu.Submit(before, {});
    std::binary_semaphore queued_returned{0};
    std::jthread queued([&] {
      gpu.SuspendPoint();
      queued_returned.release();
    });
    const bool queued_nonblocking =
        queued_returned.try_acquire_for(std::chrono::seconds(2));
    if (!queued_nonblocking) {
      release_lane.release();
      gpu.SendCommandSync([&] { label = 1; });
      queued.join();
      finish();
      Require(name, "queued insertion", false,
              "suspend-point insertion drained pending guest graphics");
    }
    queued.join();
    gpu.Submit(after, {});
    release_lane.release();
    bool prefix_reached = false;
    bool graphics_ordered = false;
    const auto deadline = std::chrono::steady_clock::now() +
                          std::chrono::seconds(2);
    do {
      gpu.SendCommandSync([&] {
        prefix_reached = prefix == 22;
        graphics_ordered = skipped == 0 && after_reset == 0;
      });
      std::this_thread::yield();
    } while (!prefix_reached && std::chrono::steady_clock::now() < deadline);
    std::array<uint32_t, 5> release_graphics{};
    write(release_graphics.data(), &label, 1);
    gpu.SubmitCompute(0x20, release_graphics);
    finish();
    Require(name, "graphics FIFO",
            prefix_reached && graphics_ordered && skipped == 0,
            "suspend point or later graphics overtook a stalled graphics stream");
    Require(name, "graphics reset", after_reset == 44,
            "suspend point preserved the previous frame's predicate");

    // Delay the completion callback beyond the owner's shutdown when possible.
    // Its credit must remain alive independently of the GuestGpu object.
    std::binary_semaphore priority_entered{0};
    std::binary_semaphore release_priority{0};
    gpu.SendCommandSync([&] {
      scheduler.DeferPriorityOperation([&] {
        priority_entered.release();
        release_priority.acquire();
      });
      SubmitInfo blocked;
      blocked.AddWait(gate, 2);
      scheduler.Flush(blocked);
    });
    gpu.SuspendPoint();
    gpu.WaitForIdle();
    std::binary_semaphore shutdown_returned{0};
    std::jthread shutdown([&] {
      context.ShutdownGpu();
      shutdown_returned.release();
    });
    const bool shutdown_overtook_native =
        shutdown_returned.try_acquire_for(std::chrono::milliseconds(100));
    signal(2);
    priority_entered.acquire();
    if (!shutdown_overtook_native) {
      (void)shutdown_returned.try_acquire_for(std::chrono::seconds(2));
    }
    release_priority.release();
    shutdown.join();
    scheduler.DrainPriorityOperations();
    m_runtime_context.device.destroySemaphore(gate, nullptr);
    Require(name, "shutdown native completion", !shutdown_overtook_native,
            "shutdown returned while the suspend point was still executing");
    std::printf("[host]    %-32s ok\n", name);
  }

  void CheckGpuCommandLane() {
    EnsureRuntimeContext();
    auto &context = Renderer();
    context.InitializeGpu(nullptr);
    auto &gpu = context.GetGpu();

    const auto caller_thread = std::this_thread::get_id();
    std::thread::id gpu_thread;
    std::thread::id nested_thread;
    uint32_t order = 0;
    gpu.SendCommand([&order] { order = 1; });
    gpu.SendCommandSync([&] {
      gpu_thread = std::this_thread::get_id();
      Require("GpuCommandLane", "FIFO", order == 1,
              "synchronous command overtook an older host command");
      Require("GpuCommandLane", "GPU context", GuestGpu::IsGpuThread(),
              "host command did not run on the GPU thread");
      gpu.SendCommandSync(
          [&nested_thread] { nested_thread = std::this_thread::get_id(); });
      order = 2;
    });
    Require(
        "GpuCommandLane", "dispatch thread",
        order == 2 && gpu_thread != caller_thread &&
            nested_thread == gpu_thread,
        "host command did not run on the GPU thread or nested sync deadlocked");

    {
      const auto make_release_mem =
          [](uint32_t data_sel, uint32_t interrupt_selector, void *destination,
             uint64_t value, uint32_t action = 0x28u,
             uint32_t gcr_cntl = 1u << 9u) {
            std::array<uint32_t, 8> packet{};
            const auto address = reinterpret_cast<uint64_t>(destination);
            const auto event_index = (action >= 0x2fu ? 6u : 5u);
            packet[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM);
            packet[1] = action | (event_index << 8u) | (gcr_cntl << 12u);
            packet[2] = (interrupt_selector << 24u) | (data_sel << 29u);
            packet[3] = static_cast<uint32_t>(address);
            packet[4] = static_cast<uint32_t>(address >> 32u);
            packet[5] = static_cast<uint32_t>(value);
            packet[6] = static_cast<uint32_t>(value >> 32u);
            packet[7] = 0x123u;
            return packet;
          };

      vk::SemaphoreTypeCreateInfo timeline_type{};
      timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo;
      timeline_type.semaphoreType = vk::SemaphoreType::eTimeline;
      vk::SemaphoreCreateInfo timeline_create{};
      timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo;
      timeline_create.pNext = &timeline_type;
      vk::Semaphore blocked_timeline = nullptr;
      Require("GpuCommandLane", "blocked timeline create",
              m_runtime_context.device.createSemaphore(
                  &timeline_create, nullptr, &blocked_timeline) ==
                      vk::Result::eSuccess &&
                  blocked_timeline != nullptr,
              "failed to create the blocked GPU timeline");

      constexpr uint64_t release_tick = 1;
      auto &gpu_scheduler = context.GetCommandScheduler();
      auto processor = std::make_unique<CommandProcessor>(context, 0);
      gpu.SendCommandSync([&] {
        processor->BufferInit();
        SubmitInfo blocked_submit;
        blocked_submit.AddWait(blocked_timeline, release_tick);
        gpu_scheduler.Flush(blocked_submit);
      });

      std::binary_semaphore parser_complete{0};
      alignas(uint64_t) uint64_t interrupt_only_gds_label = UINT64_MAX;
      alignas(uint64_t) uint64_t cb_db_release_label = UINT64_MAX;
      bool parser_kept_nonblocking_boundaries = false;
      bool host_resource_retired = false;
      bool cpu_unmap_did_not_submit = false;
      std::jthread no_gpu_wait([&] {
        gpu.SendCommandSync([&] {
          processor->BufferInit();
          const auto tick_before = gpu_scheduler.CurrentTick();
          gpu_scheduler.DeferOperation([&] { host_resource_retired = true; });
          constexpr uint64_t cpu_only_address = 0x0000000200400000ull;
          context.MapMemory(cpu_only_address, 0x4000);
          context.UnmapMemory(cpu_only_address, 0x4000);
          cpu_unmap_did_not_submit =
              gpu_scheduler.CurrentTick() == tick_before &&
              !host_resource_retired &&
              !context.IsMapped(cpu_only_address, 0x4000);
          processor->TriggerEvent(0x16u, 0u);
          auto packet = make_release_mem(0, 0, nullptr, 0);
          Pm4Execution execution;
          const auto result = processor->Process(execution, packet);
          processor->IncrementDe();
          const bool cache_and_counter_did_not_submit =
              result == Pm4ProcessResult::Complete &&
              gpu_scheduler.CurrentTick() == tick_before;

          const auto cb_db_tick = gpu_scheduler.CurrentTick();
          auto cb_db_release =
              make_release_mem(2, 0, &cb_db_release_label, 0, 0x14u, 0);
          Pm4Execution cb_db_execution;
          const auto cb_db_result =
              processor->Process(cb_db_execution, cb_db_release);
          const bool cb_db_release_did_not_submit =
              cb_db_result == Pm4ProcessResult::Complete &&
              cb_db_release_label == 0 &&
              gpu_scheduler.CurrentTick() == cb_db_tick;

          for (const auto gcr : {0u, 1u << 9u}) {
            const auto before = Sync::ReadReferenceClock();
            auto timestamp =
                make_release_mem(3, 0, &cb_db_release_label, UINT64_MAX, 0x14u, gcr);
            Pm4Execution timestamp_execution;
            const auto timestamp_result =
                processor->Process(timestamp_execution, timestamp);
            Require("GpuCommandLane", "timestamp write",
                    timestamp_result == Pm4ProcessResult::Complete &&
                        cb_db_release_label >= before &&
                        cb_db_release_label <= Sync::ReadReferenceClock() &&
                        gpu_scheduler.CurrentTick() == cb_db_tick,
                    "timestamp write failed, returned an invalid time, or "
                    "submitted extra GPU work");
          }

          auto gds_interrupt_only =
              make_release_mem(5, 1, &interrupt_only_gds_label, 1ull << 16u);
          Pm4Execution gds_interrupt_execution;
          const auto interrupt_tick = gpu_scheduler.CurrentTick();
          const auto interrupt_result =
              processor->Process(gds_interrupt_execution, gds_interrupt_only);
          parser_kept_nonblocking_boundaries =
              cache_and_counter_did_not_submit &&
              cb_db_release_did_not_submit &&
              interrupt_result == Pm4ProcessResult::Complete &&
              gpu_scheduler.CurrentTick() == interrupt_tick + 1 &&
              interrupt_only_gds_label == UINT64_MAX;
        });
        parser_complete.release();
      });
      const bool parser_did_not_wait_gpu =
          parser_complete.try_acquire_for(std::chrono::seconds(2));

      vk::SemaphoreSignalInfo signal_info{};
      signal_info.sType = vk::StructureType::eSemaphoreSignalInfo;
      signal_info.semaphore = blocked_timeline;
      signal_info.value = release_tick;
      Require("GpuCommandLane", "blocked timeline signal",
              m_runtime_context.device.signalSemaphore(&signal_info) ==
                  vk::Result::eSuccess,
              "failed to release the blocked GPU timeline");
      no_gpu_wait.join();
      gpu.SendCommandSync([&] { gpu_scheduler.Finish(); });
      m_runtime_context.device.destroySemaphore(blocked_timeline, nullptr);
      Require("GpuCommandLane", "CPU-only unmap with pending host retirement",
              parser_did_not_wait_gpu && cpu_unmap_did_not_submit &&
                  host_resource_retired,
              "CPU-only unmap submitted or waited for unrelated native work");
      Require("GpuCommandLane", "nonblocking GPU packets",
              parser_did_not_wait_gpu && parser_kept_nonblocking_boundaries,
              "a cache event, GL2 writeback, DE counter, or interrupt boundary "
              "used the wrong "
              "submission behavior");

      alignas(uint64_t) uint64_t release_label = 0;
      alignas(uint64_t) uint64_t gds_label = UINT64_MAX;
      bool release_mem_submission_counts = false;
      gpu.SendCommandSync([&] {
        processor->BufferInit();

        const auto clock_before = Sync::ReadReferenceClock();
        processor->WriteAtEndOfPipe64(0, 0, 0x04, 0x38, 5, 4,
                                     &release_label, UINT64_MAX, 2);
        Require("GpuCommandLane", "EOP reference clock with interrupt",
                release_label >= clock_before &&
                    release_label <= Sync::ReadReferenceClock(),
                "clock write with writeback and interrupt lost its data");

        for (const auto interrupt : {0u, 3u, 1u, 2u}) {
          release_label = 0;
          auto immediate = make_release_mem(1, interrupt, &release_label, 0x11223344u);
          Pm4Execution immediate_execution;
          const auto immediate_tick = gpu_scheduler.CurrentTick();
          const auto immediate_result =
              processor->Process(immediate_execution, immediate);
          Require("GpuCommandLane", "32-bit release boundary",
                  immediate_result == Pm4ProcessResult::Complete &&
                      gpu_scheduler.CurrentTick() ==
                          immediate_tick + (interrupt == 1 || interrupt == 2) &&
                      release_label == (interrupt == 1 ? 0 : 0x11223344u),
                  "label-only release submitted work or interrupt release lost its boundary");
        }

        auto gds = make_release_mem(5, 0, &gds_label, 1ull << 16u);
        Pm4Execution gds_execution;
        const auto gds_tick = gpu_scheduler.CurrentTick();
        const auto gds_result = processor->Process(gds_execution, gds);
        const bool gds_waited_once =
            gpu_scheduler.CurrentTick() == gds_tick + 1;

        auto interrupt_only = make_release_mem(0, 4, nullptr, 0);
        Pm4Execution interrupt_execution;
        const auto interrupt_tick = gpu_scheduler.CurrentTick();
        const auto interrupt_result =
            processor->Process(interrupt_execution, interrupt_only);
        const bool interrupt_split_once =
            gpu_scheduler.CurrentTick() == interrupt_tick + 1;

        auto gds_interrupt = make_release_mem(5, 2, &gds_label, 1ull << 16u);
        Pm4Execution gds_interrupt_execution;
        const auto gds_interrupt_tick = gpu_scheduler.CurrentTick();
        const auto gds_interrupt_result =
            processor->Process(gds_interrupt_execution, gds_interrupt);
        const bool gds_interrupt_waited_once =
            gpu_scheduler.CurrentTick() == gds_interrupt_tick + 1;

        release_mem_submission_counts =
            gds_result == Pm4ProcessResult::Complete &&
            gds_waited_once && interrupt_result == Pm4ProcessResult::Complete &&
            interrupt_split_once &&
            gds_interrupt_result == Pm4ProcessResult::Complete &&
            gds_interrupt_waited_once;
      });
      gpu.SendCommandSync([&] {
        gpu_scheduler.Finish();
        gpu_scheduler.WaitPriorityOperations(gpu_scheduler.CurrentTick() - 1);
      });
      Require("GpuCommandLane", "RELEASE_MEM submission counts",
              release_mem_submission_counts &&
                  static_cast<uint32_t>(release_label) == 0x11223344u &&
                  static_cast<uint32_t>(gds_label) == 0,
              "RELEASE_MEM lost its required split/readback or retained a "
              "redundant GPU wait");
    }

    alignas(uint32_t) uint32_t packet_marker_a = 0;
    alignas(uint32_t) uint32_t packet_marker_b = 0;
    const auto write_packet = [](uint32_t *packet, uint32_t *destination,
                                 uint32_t value) {
      const auto destination_address = reinterpret_cast<uint64_t>(destination);
      packet[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
      packet[1] = 0;
      packet[2] = static_cast<uint32_t>(destination_address);
      packet[3] = static_cast<uint32_t>(destination_address >> 32u);
      packet[4] = value;
    };

    uint32_t live_graphics_value = 0;
    std::array<uint32_t, 5> live_graphics_commands{};
    write_packet(live_graphics_commands.data(), &live_graphics_value, 11);
    std::binary_semaphore graphics_gate_entered{0};
    std::binary_semaphore graphics_gate_release{0};
    gpu.SendCommand([&] {
      graphics_gate_entered.release();
      graphics_gate_release.acquire();
    });
    gpu.Submit(live_graphics_commands, {});
    graphics_gate_entered.acquire();
    live_graphics_commands[4] = 22;
    graphics_gate_release.release();
    gpu.WaitForIdle();
    Require("GpuCommandLane", "borrowed graphics commands",
            live_graphics_value == 22,
            "graphics submission executed a copied PM4 stream");

    uint32_t live_compute_value = 0;
    std::array<uint32_t, 5> live_compute_commands{};
    write_packet(live_compute_commands.data(), &live_compute_value, 33);
    std::binary_semaphore compute_gate_entered{0};
    std::binary_semaphore compute_gate_release{0};
    gpu.SendCommand([&] {
      compute_gate_entered.release();
      compute_gate_release.acquire();
    });
    gpu.SubmitCompute(0x20, live_compute_commands);
    compute_gate_entered.acquire();
    live_compute_commands[4] = 44;
    compute_gate_release.release();
    gpu.WaitForIdle();
    Require("GpuCommandLane", "borrowed compute commands",
            live_compute_value == 44,
            "compute submission executed a copied PM4 stream");

    LibKernel::EventQueue::KernelEqueue interrupt_queue =
        LibKernel::EventQueue::KERNEL_EQUEUE_INVALID;
    uint32_t graphics_interrupt_udata = 0;
    uint32_t compute_interrupt_udata = 0;
    uint32_t other_compute_interrupt_udata = 0;
    const bool interrupt_queue_ready =
        LibKernel::EventQueue::KernelCreateEqueue(&interrupt_queue,
                                                  "pm4-interrupts") == 0 &&
        Sync::AddEqEvent(context, interrupt_queue, 0,
                         &graphics_interrupt_udata) == 0 &&
        Sync::AddEqEvent(context, interrupt_queue, 0x20,
                         &compute_interrupt_udata) == 0 &&
        Sync::AddEqEvent(context, interrupt_queue, 0x21,
                         &other_compute_interrupt_udata) == 0;
    Require("GpuCommandLane", "interrupt queue", interrupt_queue_ready,
            "failed to register the PM4 interrupt events");

    const auto make_interrupt_packet =
        [](uint32_t data_selector, uint32_t interrupt_selector,
           void *destination, uint64_t value, uint32_t context_id) {
          std::array<uint32_t, 8> packet{};
          const auto address = reinterpret_cast<uint64_t>(destination);
          packet[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM);
          packet[1] = 0x28u | (5u << 8u);
          packet[2] = (interrupt_selector << 24u) | (data_selector << 29u);
          packet[3] = static_cast<uint32_t>(address);
          packet[4] = static_cast<uint32_t>(address >> 32u);
          packet[5] = static_cast<uint32_t>(value);
          packet[6] = static_cast<uint32_t>(value >> 32u);
          packet[7] = context_id & 0x07ffffffu;
          return packet;
        };
    const auto finish_gpu = [&] {
      gpu.SendCommandSync([&] {
        auto &scheduler = context.GetCommandScheduler();
        scheduler.Finish();
        scheduler.WaitPriorityOperations(scheduler.CurrentTick() - 1);
      });
    };
    const auto wait_for_interrupt =
        [&](LibKernel::EventQueue::KernelEvent &event, int &count) {
          LibKernel::KernelUseconds timeout = 0;
          return LibKernel::EventQueue::KernelWaitEqueue(
              interrupt_queue, &event, 1, &count, &timeout);
        };

    uint32_t no_interrupt_graphics_value = 0;
    std::array<uint32_t, 5> no_interrupt_graphics_commands{};
    write_packet(no_interrupt_graphics_commands.data(),
                 &no_interrupt_graphics_value, 55);
    gpu.Submit(no_interrupt_graphics_commands, {});
    gpu.WaitForIdle();
    finish_gpu();

    LibKernel::EventQueue::KernelEvent interrupt_event{};
    int interrupt_count = 0;
    const auto no_graphics_interrupt_wait =
        wait_for_interrupt(interrupt_event, interrupt_count);
    Require("GpuCommandLane", "no implicit graphics interrupt",
            no_graphics_interrupt_wait == LibKernel::KERNEL_ERROR_ETIMEDOUT &&
                interrupt_count == 0 && no_interrupt_graphics_value == 55,
            "graphics submission completion synthesized an interrupt");

    auto live_interrupt_commands =
        make_interrupt_packet(0, 1, nullptr, 0, 0x234u);
    std::binary_semaphore interrupt_gate_entered{0};
    std::binary_semaphore interrupt_gate_release{0};
    gpu.SendCommand([&] {
      interrupt_gate_entered.release();
      interrupt_gate_release.acquire();
    });
    gpu.SubmitCompute(0x20, live_interrupt_commands);
    interrupt_gate_entered.acquire();
    live_interrupt_commands[2] = 0;
    interrupt_gate_release.release();
    gpu.WaitForIdle();
    finish_gpu();

    interrupt_count = 0;
    const auto no_interrupt_wait =
        wait_for_interrupt(interrupt_event, interrupt_count);
    Require("GpuCommandLane", "no implicit completion interrupt",
            no_interrupt_wait == LibKernel::KERNEL_ERROR_ETIMEDOUT &&
                interrupt_count == 0,
            "submission completion synthesized an interrupt absent from the "
            "executed PM4 stream");

    uint32_t graphics_interrupt_label = 0xa5a5a5a5u;
    auto graphics_interrupt_commands = make_interrupt_packet(
        1, 1, &graphics_interrupt_label, 0x11223344u, 0);
    gpu.Submit(graphics_interrupt_commands, {});
    gpu.WaitForIdle();
    finish_gpu();

    interrupt_count = 0;
    const auto graphics_interrupt_wait =
        wait_for_interrupt(interrupt_event, interrupt_count);
    Require("GpuCommandLane", "graphics interrupt routing",
            graphics_interrupt_wait == 0 && interrupt_count == 1 &&
                interrupt_event.ident == 0 && interrupt_event.data == 0 &&
                interrupt_event.udata == &graphics_interrupt_udata &&
                graphics_interrupt_label == 0xa5a5a5a5u,
            "graphics kOnly interrupt was misrouted or wrote its label");

    uint32_t compute_interrupt_label = 0x5a5a5a5au;
    auto compute_interrupt_commands = make_interrupt_packet(
        1, 0, &compute_interrupt_label, 0x55667788u, 0x456u);
    std::binary_semaphore compute_interrupt_gate_entered{0};
    std::binary_semaphore compute_interrupt_gate_release{0};
    gpu.SendCommand([&] {
      compute_interrupt_gate_entered.release();
      compute_interrupt_gate_release.acquire();
    });
    gpu.SubmitCompute(0x20, compute_interrupt_commands);
    compute_interrupt_gate_entered.acquire();
    compute_interrupt_commands[2] |= 1u << 24u;
    compute_interrupt_gate_release.release();
    gpu.WaitForIdle();
    finish_gpu();

    interrupt_count = 0;
    const auto compute_interrupt_wait =
        wait_for_interrupt(interrupt_event, interrupt_count);
    const bool compute_interrupt_matches =
        compute_interrupt_wait == 0 && interrupt_count == 1 &&
        interrupt_event.ident == 0x20 && interrupt_event.data == 0x456u &&
        interrupt_event.udata == &compute_interrupt_udata &&
        compute_interrupt_label == 0x55667788u;

    uint64_t compute_clock_label = 0;
    auto compute_clock_commands = make_interrupt_packet(
        3, 1, &compute_clock_label, 0, 0x567u);
    gpu.SubmitCompute(0x20, compute_clock_commands);
    gpu.WaitForIdle();
    finish_gpu();

    interrupt_count = 0;
    const auto compute_clock_wait =
        wait_for_interrupt(interrupt_event, interrupt_count);
    const bool compute_clock_matches =
        compute_clock_wait == 0 && interrupt_count == 1 &&
        interrupt_event.ident == 0x20 && interrupt_event.data == 0x567u &&
        interrupt_event.udata == &compute_interrupt_udata &&
        compute_clock_label != 0;

    constexpr uint64_t compute_done_value = 0x1234567887654321ull;
    uint64_t compute_done_label = 0;
    auto compute_done_commands = make_interrupt_packet(
        2, 2, &compute_done_label, compute_done_value, 0x678u);
    compute_done_commands[1] = 0x62fu; // CS_DONE, shader-done index, no GCR action.
    gpu.SubmitCompute(0x20, compute_done_commands);
    gpu.WaitForIdle();
    finish_gpu();

    interrupt_count = 0;
    const auto compute_done_wait =
        wait_for_interrupt(interrupt_event, interrupt_count);
    Require("GpuCommandLane", "CS_DONE write-confirm interrupt",
            compute_done_wait == 0 && interrupt_count == 1 &&
                interrupt_event.ident == 0x20 && interrupt_event.data == 0x678u &&
                interrupt_event.udata == &compute_interrupt_udata &&
                compute_done_label == compute_done_value,
            "CS_DONE lost its full 64-bit label write or write-confirm interrupt");

    interrupt_count = 0;
    const auto extra_interrupt_wait =
        wait_for_interrupt(interrupt_event, interrupt_count);
    const auto delete_other_compute_event =
        Sync::DeleteEqEvent(context, interrupt_queue, 0x21);
    LibKernel::EventQueue::KernelDeleteEqueue(interrupt_queue);
    context.TriggerInterrupt(0, 0);
    context.TriggerInterrupt(0x20, 0);
    context.TriggerInterrupt(0x21, 0);
    Require("GpuCommandLane", "compute interrupt routing",
            compute_interrupt_matches && compute_clock_matches &&
                extra_interrupt_wait == LibKernel::KERNEL_ERROR_ETIMEDOUT &&
                interrupt_count == 0 && delete_other_compute_event == 0,
            "compute kOnly interrupt was misrouted, broadcast, or lost a "
            "label write");

    std::array<uint32_t, 1024> polling_loop{};
    for (size_t i = 0; i < polling_loop.size() - 4; i += 2) {
      polling_loop[i] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
      polling_loop[i + 1] = 0;
    }
    const auto loop_address = reinterpret_cast<uint64_t>(polling_loop.data());
    polling_loop[1020] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
    polling_loop[1021] = static_cast<uint32_t>(loop_address);
    polling_loop[1022] = static_cast<uint32_t>(loop_address >> 32u);
    polling_loop[1023] =
        0x0f200000u | static_cast<uint32_t>(polling_loop.size());

    std::array<uint32_t, 14> polling_commands{};
    write_packet(polling_commands.data(), &packet_marker_a, 11);
    polling_commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
    polling_commands[6] = static_cast<uint32_t>(loop_address);
    polling_commands[7] = static_cast<uint32_t>(loop_address >> 32u);
    polling_commands[8] =
        0x0f200000u | static_cast<uint32_t>(polling_loop.size());
    write_packet(polling_commands.data() + 9, &packet_marker_b, 22);
    std::binary_semaphore stream_gate_entered{0};
    std::binary_semaphore stream_gate_release{0};
    gpu.SendCommand([&] {
      stream_gate_entered.release();
      stream_gate_release.acquire();
    });
    gpu.Submit(polling_commands, {});
    stream_gate_entered.acquire();
    stream_gate_release.release();
    uint32_t packet_marker_a_at_callback = UINT32_MAX;
    uint32_t packet_marker_b_at_callback = UINT32_MAX;
    for (uint32_t attempt = 0; attempt < 8 && packet_marker_a_at_callback != 11;
         attempt++) {
      gpu.SendCommandSync([&] {
        if (packet_marker_a == 11 && packet_marker_b == 0) {
          packet_marker_a_at_callback = packet_marker_a;
          packet_marker_b_at_callback = packet_marker_b;
          polling_commands[13] = 33;
          polling_loop[1020] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
          polling_loop[1021] = 0;
          polling_loop[1022] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
          polling_loop[1023] = 0;
        }
      });
      std::this_thread::yield();
    }
    gpu.WaitForIdle();
    Require("GpuCommandLane", "packet-boundary command polling",
            packet_marker_a_at_callback == 11 &&
                packet_marker_b_at_callback == 0 && packet_marker_a == 11 &&
                packet_marker_b == 33,
            "host command waited for an entire PM4 stream or resumed a copied "
            "stream");

    uint32_t label = 0;
    uint32_t prefix = 0;
    uint32_t suffix = 0;
    const auto address = [](const void *value) {
      return reinterpret_cast<uint64_t>(value);
    };
    std::array<uint32_t, 17> commands{};
    commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
    commands[1] = 0;
    commands[2] = static_cast<uint32_t>(address(&prefix));
    commands[3] = static_cast<uint32_t>(address(&prefix) >> 32u);
    commands[4] = 11;
    commands[5] = KYTY_PM4(7, Pm4::IT_WAIT_REG_MEM, 0);
    commands[6] = 0x10u | 3u;
    commands[7] = static_cast<uint32_t>(address(&label));
    commands[8] = static_cast<uint32_t>(address(&label) >> 32u);
    commands[9] = 1;
    commands[10] = UINT32_MAX;
    commands[12] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
    commands[13] = 0;
    commands[14] = static_cast<uint32_t>(address(&suffix));
    commands[15] = static_cast<uint32_t>(address(&suffix) >> 32u);
    commands[16] = 22;
      gpu.Submit(commands, {});

    std::binary_semaphore ordered_started{0};
    std::atomic<bool> ordered_finished{false};
    uint32_t ordered_suffix = 0;
    std::jthread ordered([&] {
      ordered_started.release();
      gpu.WaitForIdle();
      ordered_suffix = suffix;
      ordered_finished = true;
    });
    ordered_started.acquire();
    gpu.SendCommandSync([&] {
      Require("GpuCommandLane", "guest queue idle fence", !ordered_finished.load(),
              "idle fence returned before a queued submission");
      label = 1;
    });
    ordered.join();
    Require("GpuCommandLane", "ordered completion",
            prefix == 11 && suffix == 22 && ordered_suffix == 22 &&
                ordered_finished.load(),
            "idle fence did not drain prior PM4 work");

    auto &resources = context;
    constexpr uint64_t empty_unmap_base = 0x0000000200400000ull;
    constexpr uint64_t empty_unmap_size = 0x4000;
    resources.MapMemory(empty_unmap_base, empty_unmap_size);
    label = 0;
    prefix = 0;
    suffix = 0;
    gpu.Submit(commands, {});

    std::binary_semaphore unmap_complete{0};
    std::jthread unmap_thread([&] {
      resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
      unmap_complete.release();
    });
    const bool unmap_returned =
        unmap_complete.try_acquire_for(std::chrono::seconds(2));
    gpu.SendCommandSync([&] { label = 1; });
    if (!unmap_returned) {
      unmap_complete.acquire();
    }
    unmap_thread.join();
    gpu.WaitForIdle();
    Require("GpuCommandLane", "unmap queue progress",
            unmap_returned &&
                !resources.IsMapped(empty_unmap_base, empty_unmap_size) &&
                prefix == 11 && suffix == 22,
            "an unrelated unmap waited for a blocked PM4 submission");

    auto &scheduler = context.GetCommandScheduler();
    std::atomic<bool> completion_published{false};
    gpu.SendCommandSync([&] {
      scheduler.DeferPriorityOperation([&] { completion_published = true; });
    });
    resources.MapMemory(empty_unmap_base, empty_unmap_size);
    resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
    Require("GpuCommandLane", "unmap native completion",
            completion_published.load() &&
                !resources.IsMapped(empty_unmap_base, empty_unmap_size),
            "unmap returned before an earlier native guest-memory callback");

    std::binary_semaphore priority_entered{0};
    std::binary_semaphore release_priority{0};
    gpu.SendCommandSync([&] {
      scheduler.DeferPriorityOperation([&] {
        priority_entered.release();
        release_priority.acquire();
      });
      scheduler.Flush();
    });
    priority_entered.acquire();
    resources.MapMemory(empty_unmap_base, empty_unmap_size);

    std::binary_semaphore priority_unmap_entered{0};
    std::binary_semaphore priority_unmap_complete{0};
    std::jthread priority_unmap_thread([&] {
      gpu.SendCommandSync([&] {
        priority_unmap_entered.release();
        resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
      });
      priority_unmap_complete.release();
    });
    priority_unmap_entered.acquire();
    const bool unmap_overtook_priority =
        priority_unmap_complete.try_acquire_for(std::chrono::seconds(1));
    release_priority.release();
    if (!unmap_overtook_priority) {
      priority_unmap_complete.acquire();
    }
    priority_unmap_thread.join();
    Require("GpuCommandLane", "unmap priority ordering",
            !unmap_overtook_priority &&
                !resources.IsMapped(empty_unmap_base, empty_unmap_size),
            "unmap returned before an earlier guest-memory callback");

    constexpr uintptr_t fault_base = 0x0000000200500000ull;
    constexpr uint64_t fault_size = 0x10000;
    int64_t fault_direct_offset = -1;
    Require("GpuCommandLane", "processor fault direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                fault_size, fault_size, 0, &fault_direct_offset) == 0,
            "processor-fault direct-memory allocation failed");
    void *fault_memory = reinterpret_cast<void *>(fault_base);
    Require("GpuCommandLane", "processor fault direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &fault_memory, fault_size, 0x3, 0x10, fault_direct_offset,
                fault_size) == 0,
            "processor-fault fixed direct-memory mapping failed");
    Require("GpuCommandLane", "processor fault allocation",
            fault_memory == reinterpret_cast<void *>(fault_base),
            "fixed processor-fault allocation failed");
    resources.MapMemory(fault_base, fault_size);

    constexpr uint64_t immediate_dst = fault_base + 0x1000;
    constexpr uint64_t immediate_memory_dst = fault_base + 0x2000;
    constexpr uint64_t memory_src = fault_base + 0x4000;
    constexpr uint64_t memory_copy_dst = fault_base + 0x6000;
    constexpr uint64_t memory_dst = fault_base + 0x8000;
    constexpr uint64_t l2_copy_dst = fault_base + 0xa000;
    constexpr uint64_t nowhere_dst = fault_base + 0xc000;
    constexpr uint64_t clean_cached_fill = fault_base + 0x3000;
    constexpr uint64_t clean_cached_copy = fault_base + 0x3008;
    constexpr uint64_t clean_cached_source = fault_base + 0x3010;
    constexpr uint64_t clean_cached_readback = fault_base + 0xb000;
    constexpr std::array<uint32_t, 2> source_words{0x11223344u, 0x55667788u};
    constexpr std::array<uint32_t, 2> nowhere_words{0x89abcdefu, 0x01234567u};
    constexpr std::array<uint32_t, 2> clean_copy_words{0x13579bdfu,
                                                       0x2468ace0u};
    std::memcpy(reinterpret_cast<void *>(memory_src), source_words.data(),
                sizeof(source_words));
    std::memcpy(reinterpret_cast<void *>(nowhere_dst), nowhere_words.data(),
                sizeof(nowhere_words));
    std::memcpy(reinterpret_cast<void *>(clean_cached_source),
                clean_copy_words.data(), sizeof(clean_copy_words));

    std::array<uint32_t, 56> dma_commands{};
    size_t dma_cursor = 0;
    const auto append_dma = [&](uint8_t dst_sel, uint64_t dst, uint8_t src_sel,
                                uint64_t src, uint32_t bytes,
                                uint32_t command_flags = 0) {
      auto *packet = dma_commands.data() + dma_cursor;
      dma_cursor += 7;
      packet[0] = KYTY_PM4(7, Pm4::IT_DMA_DATA, 0);
      packet[1] = (static_cast<uint32_t>(dst_sel) << 20u) |
                  (static_cast<uint32_t>(src_sel) << 29u);
      packet[2] = static_cast<uint32_t>(src);
      packet[3] = static_cast<uint32_t>(src >> 32u);
      packet[4] = static_cast<uint32_t>(dst);
      packet[5] = static_cast<uint32_t>(dst >> 32u);
      packet[6] = bytes | command_flags;
    };
    constexpr uint64_t gds_immediate_offset = 0;
    constexpr uint64_t gds_memory_offset = 0x10;
    constexpr uint32_t immediate_value = 0xa1b2c3d4u;
    append_dma(1, gds_immediate_offset, 2, immediate_value,
               sizeof(source_words));
    append_dma(0, immediate_dst, 1, gds_immediate_offset, sizeof(source_words));
    append_dma(1, gds_memory_offset, 0, memory_src, sizeof(source_words));
    append_dma(0, memory_dst, 1, gds_memory_offset, sizeof(source_words));
    append_dma(0, immediate_memory_dst, 2, immediate_value,
               sizeof(source_words));
    append_dma(0, memory_copy_dst, 0, memory_src, sizeof(source_words));
    append_dma(3, l2_copy_dst, 3, memory_src, sizeof(source_words));
    append_dma(2, nowhere_dst, 3, nowhere_dst, sizeof(nowhere_words),
               1u << 31u);
    Require("GpuCommandLane", "DMA_DATA packet assembly",
            dma_cursor == dma_commands.size(),
            "DMA_DATA packet stream has the wrong size");
    gpu.Submit(dma_commands, {});
    gpu.WaitForIdle();
    constexpr uint32_t clean_fill_value = 0xdecafbad;
    gpu.SendCommandSync([&] {
      auto &buffer_cache = resources.GetBufferCache();
      Require("GpuCommandLane", "clean cached setup",
              buffer_cache.IsRegionRegistered(clean_cached_fill,
                                              sizeof(source_words)) &&
                  !buffer_cache.HasGpuDirtyBytes(nowhere_dst,
                                                 sizeof(nowhere_words)) &&
                  buffer_cache.HasGpuDirtyBytes(immediate_dst,
                                                sizeof(source_words)),
              "DMA prefetch modified ownership or did not establish a dirty "
              "cached page");

      buffer_cache.FillBuffer(clean_cached_fill, sizeof(source_words),
                              clean_fill_value, false);
      buffer_cache.CopyBuffer(clean_cached_copy, clean_cached_source,
                              sizeof(clean_copy_words), false, false);
      Require("GpuCommandLane", "clean cached DMA ownership",
              !buffer_cache.HasGpuDirtyBytes(clean_cached_fill,
                                             sizeof(source_words)) &&
                  !buffer_cache.HasGpuDirtyBytes(clean_cached_copy,
                                                 sizeof(clean_copy_words)) &&
                  buffer_cache.HasGpuDirtyBytes(immediate_dst,
                                                sizeof(source_words)),
              "clean cached DMA acquired ownership or released a byte-disjoint "
              "dirty sibling");

      constexpr uint64_t gds_clean_mirror = 0x20;
      buffer_cache.CopyBuffer(gds_clean_mirror, clean_cached_fill,
                              2 * sizeof(source_words), true, false);
      buffer_cache.CopyBuffer(clean_cached_readback, gds_clean_mirror,
                              2 * sizeof(source_words), false, true);
    });
    Require("GpuCommandLane", "DMA_DATA immediate GDS readback",
            resources.HandleFault(PageFaultAccess::Read, immediate_dst),
            "immediate-to-GDS-to-memory copy did not publish GPU bytes");
    Require("GpuCommandLane", "DMA_DATA memory GDS readback",
            resources.HandleFault(PageFaultAccess::Read, memory_dst),
            "memory-to-GDS-to-memory copy did not publish GPU bytes");
    Require("GpuCommandLane", "DMA_DATA immediate memory readback",
            resources.HandleFault(PageFaultAccess::Read, immediate_memory_dst),
            "immediate-to-memory copy did not publish GPU bytes");
    Require("GpuCommandLane", "DMA_DATA memory copy readback",
            resources.HandleFault(PageFaultAccess::Read, memory_copy_dst),
            "memory-to-memory copy did not publish GPU bytes");
    Require("GpuCommandLane", "DMA_DATA L2 readback",
            resources.HandleFault(PageFaultAccess::Read, l2_copy_dst),
            "MemoryUsingL2 copy did not publish GPU bytes");
    Require("GpuCommandLane", "clean cached mirror readback",
            resources.HandleFault(PageFaultAccess::Read, clean_cached_readback),
            "clean host DMA was not reflected in the cached buffer");
    std::array<uint32_t, 2> immediate_words{};
    std::array<uint32_t, 2> copied_words{};
    std::array<uint32_t, 2> immediate_memory_words{};
    std::array<uint32_t, 2> memory_copy_words{};
    std::array<uint32_t, 2> l2_copy_words{};
    std::array<uint32_t, 4> clean_cached_words{};
    std::memcpy(immediate_words.data(),
                reinterpret_cast<const void *>(immediate_dst),
                sizeof(immediate_words));
    std::memcpy(copied_words.data(), reinterpret_cast<const void *>(memory_dst),
                sizeof(copied_words));
    std::memcpy(immediate_memory_words.data(),
                reinterpret_cast<const void *>(immediate_memory_dst),
                sizeof(immediate_memory_words));
    std::memcpy(memory_copy_words.data(),
                reinterpret_cast<const void *>(memory_copy_dst),
                sizeof(memory_copy_words));
    std::memcpy(l2_copy_words.data(),
                reinterpret_cast<const void *>(l2_copy_dst),
                sizeof(l2_copy_words));
    std::memcpy(clean_cached_words.data(),
                reinterpret_cast<const void *>(clean_cached_readback),
                sizeof(clean_cached_words));
    Require("GpuCommandLane", "DMA_DATA immediate GDS contents",
            immediate_words ==
                std::array<uint32_t, 2>{immediate_value, immediate_value},
            "immediate-to-GDS-to-memory bytes do not match");
    Require("GpuCommandLane", "DMA_DATA memory GDS contents",
            copied_words == source_words,
            "memory-to-GDS-to-memory bytes do not match");
    Require("GpuCommandLane", "DMA_DATA immediate memory contents",
            immediate_memory_words ==
                std::array<uint32_t, 2>{immediate_value, immediate_value},
            "immediate-to-memory bytes do not match");
    Require("GpuCommandLane", "DMA_DATA memory copy contents",
            memory_copy_words == source_words,
            "memory-to-memory bytes do not match");
    Require("GpuCommandLane", "DMA_DATA L2 contents",
            l2_copy_words == source_words, "MemoryUsingL2 bytes do not match");
    Require("GpuCommandLane", "DMA_DATA nowhere contents",
            std::memcmp(reinterpret_cast<const void *>(nowhere_dst),
                        nowhere_words.data(), sizeof(nowhere_words)) == 0,
            "Nowhere destination modified guest memory");
    Require("GpuCommandLane", "clean cached DMA contents",
            clean_cached_words == std::array<uint32_t, 4>{clean_fill_value,
                                                          clean_fill_value,
                                                          clean_copy_words[0],
                                                          clean_copy_words[1]},
            "host-memory fill/copy did not update the clean cached mirror");

    gpu.SendCommandSync([&] {
      Require("GpuCommandLane", "GPU memory invalidation",
              resources.InvalidateMemory(fault_base, sizeof(uint32_t)),
              "GPU memory invalidation did not find its mapped range");
    });

    auto &buffer_cache = resources.GetBufferCache();
    Require("GpuCommandLane", "clean tracked fault setup",
            buffer_cache.IsRegionRegistered(immediate_dst, sizeof(uint32_t)) &&
                !buffer_cache.HasGpuDirtyBytes(immediate_dst, sizeof(uint32_t)),
            "clean write-fault test address is not backed by a clean cached "
            "Buffer");
    constexpr uint64_t dirty_fault_address = fault_base + 0xd000;
    constexpr uint32_t dirty_fault_value = 0x5a17c0deu;
    constexpr uint32_t dirty_fault_stale = 0x0ddba11u;
    std::memcpy(reinterpret_cast<void *>(dirty_fault_address),
                &dirty_fault_stale, sizeof(dirty_fault_stale));
    gpu.SendCommandSync([&] {
      auto dirty = buffer_cache.ObtainBuffer(
          dirty_fault_address, sizeof(dirty_fault_value), true, false);
      Require("GpuCommandLane", "dirty tracked fault setup",
              dirty.first != nullptr,
              "dirty write-fault test could not create a cached Buffer");
      buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value),
                              dirty_fault_value, false);
    });
    Require("GpuCommandLane", "dirty tracked fault ownership",
            buffer_cache.HasGpuDirtyBytes(dirty_fault_address,
                                          sizeof(dirty_fault_value)),
            "dirty write-fault test address did not acquire GPU ownership");

    std::binary_semaphore block_entered{0};
    std::binary_semaphore block_release{0};
    std::binary_semaphore clean_fault_complete{0};
    bool clean_fault_handled = false;
    gpu.SendCommand([&] {
      block_entered.release();
      block_release.acquire();
    });
    block_entered.acquire();
    std::jthread clean_fault_thread([&] {
      clean_fault_handled =
          resources.HandleFault(PageFaultAccess::Write, immediate_dst);
      clean_fault_complete.release();
    });
    const bool clean_fault_was_direct =
        clean_fault_complete.try_acquire_for(std::chrono::seconds(1));
    if (!clean_fault_was_direct) {
      block_release.release();
      clean_fault_complete.acquire();
    }
    clean_fault_thread.join();
    Require("GpuCommandLane", "clean fault direct path",
            clean_fault_was_direct && clean_fault_handled,
            "a clean CPU write fault waited for the blocked GPU command lane");

    std::binary_semaphore dirty_fault_complete{0};
    bool dirty_fault_handled = false;
    std::jthread dirty_fault_thread([&] {
      dirty_fault_handled =
          resources.HandleFault(PageFaultAccess::Write, dirty_fault_address);
      dirty_fault_complete.release();
    });
    const bool dirty_fault_bypassed_lane =
        dirty_fault_complete.try_acquire_for(std::chrono::milliseconds(200));
    block_release.release();
    if (!dirty_fault_bypassed_lane) {
      dirty_fault_complete.acquire();
    }
    dirty_fault_thread.join();
    uint32_t dirty_fault_backing = 0;
    std::memcpy(&dirty_fault_backing,
                reinterpret_cast<const void *>(dirty_fault_address),
                sizeof(dirty_fault_backing));
    Require("GpuCommandLane", "dirty fault synchronization",
            !dirty_fault_bypassed_lane && dirty_fault_handled &&
                dirty_fault_backing == dirty_fault_value &&
                !buffer_cache.HasGpuDirtyBytes(dirty_fault_address,
                                               sizeof(dirty_fault_value)),
            "a GPU-dirty write fault bypassed synchronization or lost the "
            "published bytes");

    constexpr uint64_t empty_copy_fault_address = fault_base + 0xe000;
    constexpr uint32_t empty_copy_fault_value = 0x6b28d1efu;
    gpu.SendCommandSync([&] {
      auto dirty = buffer_cache.ObtainBuffer(empty_copy_fault_address,
                                             sizeof(empty_copy_fault_value),
                                             true, false);
      Require("GpuCommandLane", "empty-copy write setup",
              dirty.first != nullptr,
              "empty-copy write test could not create a cached Buffer");
      buffer_cache.FillBuffer(empty_copy_fault_address,
                              sizeof(empty_copy_fault_value),
                              empty_copy_fault_value, false);
      buffer_cache.ReadMemory(empty_copy_fault_address,
                              sizeof(empty_copy_fault_value));
      const bool gpu_owned = buffer_cache.IsRegionGpuModified(
          empty_copy_fault_address, sizeof(empty_copy_fault_value));
      const bool cpu_owned = buffer_cache.IsRegionCpuModified(
          empty_copy_fault_address, sizeof(empty_copy_fault_value));
      Require("GpuCommandLane", "empty-copy read ownership",
              !gpu_owned && !cpu_owned,
              "readback did not leave clean ownership for the queued write "
              "invalidation");
      buffer_cache.ReadMemory(empty_copy_fault_address,
                              sizeof(empty_copy_fault_value), true);
    });
    uint32_t empty_copy_fault_backing = 0;
    std::memcpy(&empty_copy_fault_backing,
                reinterpret_cast<const void *>(empty_copy_fault_address),
                sizeof(empty_copy_fault_backing));
    const bool empty_copy_cpu_owned = buffer_cache.IsRegionCpuModified(
        empty_copy_fault_address, sizeof(empty_copy_fault_value));
    const bool empty_copy_gpu_owned = buffer_cache.IsRegionGpuModified(
        empty_copy_fault_address, sizeof(empty_copy_fault_value));
    const bool empty_copy_dirty = buffer_cache.HasGpuDirtyBytes(
        empty_copy_fault_address, sizeof(empty_copy_fault_value));
    Require("GpuCommandLane", "empty-copy write ownership",
            empty_copy_fault_backing == empty_copy_fault_value &&
                empty_copy_cpu_owned && !empty_copy_gpu_owned &&
                !empty_copy_dirty,
            "write invalidation without a remaining copy did not establish CPU "
            "ownership");

    resources.UnmapMemory(fault_base, fault_size);
    Require("GpuCommandLane", "processor fault unmap",
            Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0,
            "processor-fault direct-memory mapping release failed");
    Require("GpuCommandLane", "processor fault free",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                fault_direct_offset, fault_size) == 0,
            "processor-fault direct-memory allocation release failed");

    std::binary_semaphore command_entered{0};
    std::binary_semaphore release_command{0};
    std::atomic<bool> shutdown_complete{false};
    bool shutdown_owner_visible = false;
    resources.MapMemory(empty_unmap_base, empty_unmap_size);
    gpu.SendCommand([&] {
      command_entered.release();
      release_command.acquire();
      shutdown_owner_visible = &context.GetGpu() == &gpu;
    });
    command_entered.acquire();
    std::jthread shutdown_thread([&] {
      context.ShutdownGpu();
      shutdown_complete = true;
    });
    while (!gpu.IsStopping()) {
      std::this_thread::yield();
    }
    Require("GpuCommandLane", "owned shutdown", !shutdown_complete.load(),
            "GPU owner returned before an in-flight command completed");
    release_command.release();
    shutdown_thread.join();
    Require(
        "GpuCommandLane", "owned shutdown completion",
        shutdown_complete.load() && shutdown_owner_visible,
        "GPU owner did not remain available while draining queued work");
    resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
    Require("GpuCommandLane", "unmap after shutdown",
            !resources.IsMapped(empty_unmap_base, empty_unmap_size),
            "renderer retained mapped coverage after the GPU command lane stopped");
    context.ShutdownGpu();
    std::printf("[host]    %-32s ok\n", "GpuCommandLane");
  }

  void CheckUnifiedImageViewCache() {
    EnsureRuntimeContext();
    constexpr const char *name = "UnifiedImageViewCache";
    CommandScheduler scheduler(Renderer(), m_runtime_context);

    ImageInfo color_info{};
    color_info.pixel_format = vk::Format::eR8G8B8A8Unorm;
    color_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm;
    color_info.type = Prospero::ImageType::kColor2D;
    color_info.extent = {8, 8, 1};
    color_info.resources = {2, 2};
    color_info.pitch = 8;
    color_info.bytes_per_block = 4;
    color_info.samples = 1;
    color_info.tile_mode = Prospero::TileMode::kLinear;
    color_info.mip_layout[0] = {0, 512, 8, 8};
    color_info.mip_layout[1] = {512, 128, 4, 4};

    Libs::Graphics::Image color(m_runtime_context, scheduler, color_info);
    const vk::ComponentMapping identity{};
    const vk::ComponentMapping bgra{
        vk::ComponentSwizzle::eB, vk::ComponentSwizzle::eG,
        vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eA};

    ImageViewInfo sampled{};
    sampled.format = color_info.pixel_format;
    sampled.type = vk::ImageViewType::e2D;
    sampled.aspect = vk::ImageAspectFlagBits::eColor;
    sampled.base_level = 0;
    sampled.level_count = 1;
    sampled.base_layer = 0;
    sampled.layer_count = 1;
    sampled.mapping = identity;
    sampled.usage = vk::ImageUsageFlagBits::eSampled;

    const auto first = color.FindView(sampled);
    const auto first_again = color.FindView(sampled);
    auto swizzled_info = sampled;
    swizzled_info.mapping = bgra;
    const auto swizzled = color.FindView(swizzled_info);
    auto mip_info = sampled;
    mip_info.base_level = 1;
    const auto mip = color.FindView(mip_info);
    auto layer_info = sampled;
    layer_info.base_layer = 1;
    const auto layer = color.FindView(layer_info);
    auto array_info = sampled;
    array_info.type = vk::ImageViewType::e2DArray;
    array_info.layer_count = 2;
    const auto array = color.FindView(array_info);
    auto reinterpreted_info = sampled;
    reinterpreted_info.format = vk::Format::eR8G8B8A8Uint;
    const auto reinterpreted = color.FindView(reinterpreted_info);
    auto storage_info = sampled;
    storage_info.usage = vk::ImageUsageFlagBits::eStorage;
    const auto storage = color.FindView(storage_info);
    auto attachment_info = sampled;
    attachment_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
    const auto attachment = color.FindView(attachment_info);

    Require(name, "color views",
            first != nullptr && first_again == first && swizzled != nullptr &&
                swizzled != first && mip != nullptr && mip != first &&
                layer != nullptr && layer != first && array != nullptr &&
                array != first && reinterpreted != nullptr &&
                reinterpreted != first && storage != nullptr &&
                storage != first && attachment == first &&
                color.views.size() == 7,
            "dynamic view identity omitted mapping, format, mip, layer, type, "
            "or storage usage");

    auto video_out_info = color_info;
    video_out_info.pixel_format = vk::Format::eA2R10G10B10UnormPack32;
    video_out_info.guest_format = Prospero::BufferFormat::k10_10_10_2UNorm;
    Libs::Graphics::Image video_out(m_runtime_context, scheduler,
                                    video_out_info);
    auto video_out_view_info = sampled;
    video_out_view_info.format = video_out_info.pixel_format;
    video_out_view_info.usage = vk::ImageUsageFlagBits::eTransferSrc;
    const auto video_out_view = video_out.FindView(video_out_view_info);
    auto storage_alias_info = video_out_view_info;
    storage_alias_info.format = vk::Format::eA2B10G10R10UnormPack32;
    storage_alias_info.usage = vk::ImageUsageFlagBits::eStorage;
    const auto storage_alias_view = video_out.FindView(storage_alias_info);
    Require(
        name, "video-out storage alias usage",
        ImageViewOps::FormatsCompatible(video_out_info.pixel_format,
                                        storage_alias_info.format) &&
            static_cast<bool>(video_out.backing.usage &
                              vk::ImageUsageFlagBits::eStorage) &&
            video_out_view != nullptr && storage_alias_view != nullptr,
        "compatible video-out backing was created without storage usage");

    ImageInfo depth_info{};
    depth_info.pixel_format = vk::Format::eD32Sfloat;
    depth_info.guest_format = Prospero::BufferFormat::k32Float;
    depth_info.type = Prospero::ImageType::kColor2D;
    depth_info.extent = {8, 8, 1};
    depth_info.resources = {1, 2};
    depth_info.pitch = 8;
    depth_info.bytes_per_block = 4;
    depth_info.samples = 1;
    depth_info.tile_mode = Prospero::TileMode::kDepth;
    depth_info.mip_layout[0] = {0, 512, 8, 8};

    Libs::Graphics::Image depth(m_runtime_context, scheduler, depth_info);
    ImageViewInfo depth_sampled{};
    depth_sampled.format = depth_info.pixel_format;
    depth_sampled.type = vk::ImageViewType::e2D;
    depth_sampled.aspect = vk::ImageAspectFlagBits::eDepth;
    depth_sampled.base_level = 0;
    depth_sampled.level_count = 1;
    depth_sampled.base_layer = 0;
    depth_sampled.layer_count = 1;
    depth_sampled.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR,
                             vk::ComponentSwizzle::eR,
                             vk::ComponentSwizzle::eR};
    depth_sampled.usage = vk::ImageUsageFlagBits::eSampled;
    const auto depth_first = depth.FindView(depth_sampled);
    auto depth_alias_info = depth_sampled;
    depth_alias_info.format = vk::Format::eR32Uint;
    const auto depth_alias = depth.FindView(depth_alias_info);
    const auto depth_again = depth.FindView(depth_sampled);
    auto depth_array_info = depth_sampled;
    depth_array_info.type = vk::ImageViewType::e2DArray;
    depth_array_info.layer_count = 2;
    const auto depth_array = depth.FindView(depth_array_info);
    auto depth_attachment_info = depth_array_info;
    depth_attachment_info.mapping = {};
    depth_attachment_info.usage =
        vk::ImageUsageFlagBits::eDepthStencilAttachment;
    const auto depth_attachment = depth.FindView(depth_attachment_info);
    Require(name, "depth views",
            depth_first != nullptr && depth_again == depth_first &&
                depth_alias == depth_first && depth_array != nullptr &&
                depth_array != depth_first && depth_attachment != nullptr &&
                depth_attachment != depth_array &&
                depth.views.size() == 3,
            "unified depth view cache lost sampled/attachment identity");

    auto depth_stencil_info = depth_info;
    depth_stencil_info.pixel_format = vk::Format::eD24UnormS8Uint;
    depth_stencil_info.guest_format = Prospero::BufferFormat::k16UNorm;
    depth_stencil_info.bytes_per_block = 2;
    Libs::Graphics::Image depth_stencil(m_runtime_context, scheduler,
                                        depth_stencil_info);
    auto depth_only_info = depth_sampled;
    depth_only_info.format = depth_stencil_info.pixel_format;
    const auto depth_only = depth_stencil.FindView(depth_only_info);
    auto combined_info = depth_only_info;
    combined_info.aspect =
        vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
    combined_info.mapping = {};
    combined_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
    const auto combined = depth_stencil.FindView(combined_info);
    Require(name, "depth/stencil aspects",
            depth_only != nullptr && combined != nullptr &&
                depth_only != combined &&
                depth_stencil.views.size() == 2 &&
                depth_stencil.views[0].info.aspect ==
                    vk::ImageAspectFlagBits::eDepth &&
                depth_stencil.views[1].info.aspect ==
                    (vk::ImageAspectFlagBits::eDepth |
                     vk::ImageAspectFlagBits::eStencil),
            "explicit depth-only and combined aspects shared one image view");
    Require(name, "role-free backing",
            color.backing.image_type == vk::ImageType::e2D &&
                color.backing.layers == 2 &&
                depth.backing.image_type == vk::ImageType::e2D &&
                ImageViewOps::FormatsCompatible(vk::Format::eR8G8B8A8Unorm,
                                                vk::Format::eR8G8B8A8Uint) &&
                !ImageViewOps::FormatsCompatible(vk::Format::eD32Sfloat,
                                                 vk::Format::eR32Sfloat),
            "role-free image backing or host format compatibility diverged");

    auto line_info = color_info;
    line_info.type = Prospero::ImageType::kColor1D;
    line_info.extent = {8, 1, 1};
    line_info.resources = {2, 2};
    line_info.mip_layout[0] = {0, 64, 8, 1};
    line_info.mip_layout[1] = {64, 16, 4, 1};
    Libs::Graphics::Image line(m_runtime_context, scheduler, line_info);
    auto line_view_info = sampled;
    line_view_info.type = vk::ImageViewType::e1D;
    const auto line_view = line.FindView(line_view_info);
    auto line_array_info = line_view_info;
    line_array_info.type = vk::ImageViewType::e1DArray;
    line_array_info.layer_count = 2;
    const auto line_array = line.FindView(line_array_info);
    Require(name, "1D views",
            line_view != nullptr && line_array != nullptr &&
                line_array != line_view &&
                line.backing.image_type == vk::ImageType::e1D &&
                line.backing.layers == 2,
            "1D and 1D-array views did not use a first-class 1D backing");

    auto volume_info = color_info;
    volume_info.type = Prospero::ImageType::kColor3D;
    volume_info.extent = {8, 8, 4};
    volume_info.resources = {2, 1};
    Libs::Graphics::Image volume(m_runtime_context, scheduler, volume_info);
    auto slice_info = sampled;
    slice_info.type = vk::ImageViewType::e2D;
    slice_info.base_level = 1;
    slice_info.base_layer = 1;
    const auto slice = volume.FindView(slice_info);
    auto slice_array_info = slice_info;
    slice_array_info.type = vk::ImageViewType::e2DArray;
    slice_array_info.base_layer = 0;
    slice_array_info.layer_count = 2;
    const auto slice_array = volume.FindView(slice_array_info);
    Require(name, "3D slice views",
            slice != nullptr && slice_array != nullptr &&
                slice_array != slice &&
                static_cast<bool>(volume.backing.flags &
                                  vk::ImageCreateFlagBits::e2DArrayCompatible),
            "2D slice views of a compatible 3D backing were rejected");
    std::printf("[host]    %-32s ok\n", name);
  }

  void CheckBufferCacheDirtyGarbageCollection() {
    constexpr const char *name = "BufferCacheDirtyGarbageCollection";
    constexpr uintptr_t base = 0x0000000200700000ull;
    constexpr uint64_t allocation_size = 0x5000000;
    constexpr uint64_t allocation_alignment = 0x10000;
    constexpr uint64_t first_offset = 0x100;
    constexpr uint64_t second_offset = 0x200;
    constexpr uint64_t clean_offset = 0x300;
    constexpr uint64_t unmap_offset = 0x4100;
    constexpr uint64_t partial_unmap_offset = 0x8100;
    constexpr uint64_t partial_unmap_survivor_offset = 0xc100;
    constexpr uint64_t ring_fault_first_offset = 0x2100;
    constexpr uint64_t ring_fault_second_offset = 0x2110;
    constexpr uint32_t first_value = 0x10203040u;
    constexpr uint32_t second_value = 0x50607080u;
    constexpr uint32_t first_stale = 0x0badf00du;
    constexpr uint32_t second_stale = 0xdeadbeefu;
    constexpr uint32_t clean_value = 0xaabbccddu;
    constexpr uint32_t unmap_value = 0x91a2b3c4u;
    constexpr uint32_t remap_value = 0xd5e6f708u;
    constexpr uint32_t partial_unmap_value = 0x62738495u;
    constexpr uint32_t partial_unmap_survivor_value = 0xa6b7c8d9u;
    constexpr uint32_t ring_fault_first_value = 0x0a1b2c3du;
    constexpr uint32_t ring_fault_second_value = 0x4e5f6071u;

    EnsureRuntimeContext();
    RenderContext context(m_runtime_context);
    auto &scheduler = context.GetCommandScheduler();
    HW::Context registers{};
    HW::UserConfig user_config{};
    HW::Shader shaders{};
    scheduler.Begin(registers, user_config, shaders);
    context.InitializeGpu(nullptr);

    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_alignment, 0, &direct_offset) == 0,
            "dirty-GC direct-memory allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_alignment) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "dirty-GC fixed direct-memory mapping failed");
    auto *memory = static_cast<uint8_t *>(mapped);
    std::memcpy(memory + first_offset, &first_stale, sizeof(first_stale));
    std::memcpy(memory + second_offset, &second_stale, sizeof(second_stale));
    std::memcpy(memory + clean_offset, &clean_value, sizeof(clean_value));

    {
      auto &resources = context;
      auto &cache = resources.GetBufferCache();
      resources.MapMemory(base, allocation_size);

      const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) {
        auto allocation = cache.ObtainBuffer(address, size, true, false);
        Require(name, "dirty allocation", allocation.first != nullptr,
                "dirty-GC buffer allocation failed");
      };
      const auto ReadNativeValue =
          [&](const Libs::Graphics::Buffer &buffer, uint64_t offset) {
        auto readback =
            CreateHostBuffer(name, sizeof(uint32_t),
                             vk::BufferUsageFlagBits::eTransferDst, {0});
        const vk::BufferCopy copy{offset, 0, sizeof(uint32_t)};
        scheduler.Current().Handle().copyBuffer(buffer.Handle(), readback.buffer,
                                                1, &copy);
        vk::BufferMemoryBarrier barrier{};
        barrier.sType = vk::StructureType::eBufferMemoryBarrier;
        barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
        barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
        barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        barrier.buffer = readback.buffer;
        barrier.size = readback.size;
        scheduler.Current().Handle().pipelineBarrier(
            vk::PipelineStageFlagBits::eTransfer,
            vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
            nullptr);
        scheduler.Finish();
        const auto value = ReadBuffer(name, readback, 1)[0];
        DestroyBuffer(&readback);
        return value;
      };

      constexpr uint64_t stream_pages = base + 0x18000;
      constexpr uint64_t stream_address = stream_pages + 0x4000 - sizeof(uint32_t);
      const std::array<uint32_t, 2> stream_values{0x13579bdfu, 0x2468ace0u};
      std::memcpy(reinterpret_cast<void *>(stream_address), stream_values.data(),
                  sizeof(stream_values));
      Require(name, "GPU-readable stream source",
              Libs::LibKernel::Memory::KernelMprotect(
                  reinterpret_cast<void *>(stream_pages), 0x8000, 0x10) == 0 &&
                  cache.IsRegionCpuModified(stream_address, sizeof(stream_values)) &&
                  !cache.IsRegionGpuModified(stream_address, sizeof(stream_values)),
              "cross-page stream source is not CPU-dirty and GPU-clean");
      const auto stream_tick = scheduler.CurrentTick();
      const auto [streamed, streamed_offset] =
          cache.ObtainBuffer(stream_address, sizeof(stream_values), false);
      Require(name, "cross-page read-only stream upload",
              streamed == &cache.GetUtilityBuffer(MemoryUsage::Stream) &&
                  scheduler.CurrentTick() == stream_tick &&
                  std::memcmp(streamed->Mapped().data() + streamed_offset,
                              stream_values.data(), sizeof(stream_values)) == 0,
              "GPU-readable guest bytes were not streamed without submission");
      Require(name, "stream source protection restore",
              Libs::LibKernel::Memory::KernelMprotect(
                  reinterpret_cast<void *>(stream_pages), 0x8000, 0x3) == 0,
              "stream source protection was not restored");
      Require(name, "streamed GPU bytes",
              ReadNativeValue(*streamed, streamed_offset) == stream_values[0] &&
                  ReadNativeValue(*streamed, streamed_offset + sizeof(uint32_t)) == stream_values[1],
              "stream commit did not publish both cross-page words to the GPU");

      constexpr uint64_t large_copy_source_offset = 0x40000;
      constexpr uint64_t large_copy_destination_offset = 0x80000;
      constexpr uint64_t large_copy_size = 96 * 1024;
      std::vector<uint8_t> large_copy_source(large_copy_size);
      for (uint64_t index = 0; index < large_copy_size; ++index) {
        large_copy_source[index] =
            static_cast<uint8_t>((index * 37 + index / 251) & 0xffu);
      }
      std::memcpy(memory + large_copy_source_offset, large_copy_source.data(),
                  large_copy_size);
      std::memset(memory + large_copy_destination_offset, 0, large_copy_size);
      cache.CopyBuffer(base + large_copy_destination_offset,
                       base + large_copy_source_offset, large_copy_size, false,
                       false);
      Require(name, "large host copy",
              std::memcmp(memory + large_copy_destination_offset,
                          large_copy_source.data(), large_copy_size) == 0,
              "chunked host copy lost bytes across its scratch boundary");

      constexpr uint64_t index_offset = 0x180000;
      constexpr uint64_t index_page = BufferCache::CACHING_PAGESIZE;
      constexpr uint64_t index_span = 3 * index_page;
      const uint64_t index_begin = base + index_offset;
      Require(name, "registered-range empty lookup",
              !cache.IsRegionRegistered(index_begin, index_span) &&
                  !BufferCacheTestAccess::PageOwner(cache, index_begin),
              "empty Buffer cache reported a registered range");
      const auto index_left =
          cache.FindBuffer(index_begin + 0x100, sizeof(uint32_t));
      const auto index_right = cache.FindBuffer(
          index_begin + 2 * index_page + 0x100, sizeof(uint32_t));
      Require(name, "registered-range owners",
              index_left && index_right &&
                  cache.FindBuffer(index_begin + 0x200, sizeof(uint32_t)) ==
                      index_left &&
                  BufferCacheTestAccess::PageOwner(cache, index_begin) ==
                      index_left &&
                  !BufferCacheTestAccess::PageOwner(cache,
                                                    index_begin + index_page) &&
                  BufferCacheTestAccess::PageOwner(
                      cache, index_begin + 2 * index_page) == index_right,
              "failed to create disjoint Buffer index owners");
      Require(
          name, "registered-range boundaries",
          cache.IsRegionRegistered(index_begin, 1) &&
              cache.IsRegionRegistered(index_begin + index_page - 1, 1) &&
              !cache.IsRegionRegistered(index_begin - 1, 1) &&
              !cache.IsRegionRegistered(index_begin + index_page, index_page) &&
              cache.IsRegionRegistered(index_begin + index_page,
                                       index_page + 1) &&
              !cache.IsRegionRegistered(index_begin + index_span, 1) &&
              cache.IsRegionRegistered(index_begin - 1, index_span + 2),
          "indexed lookup mishandled a half-open boundary, gap, or broad "
          "overlap");
      const auto clean_unmap_tick = scheduler.CurrentTick();
      resources.UnmapMemory(index_begin, index_page);
      Require(name, "clean cached buffer unmap drain",
              scheduler.CurrentTick() == clean_unmap_tick + 1 &&
                  !resources.IsMapped(index_begin, index_page),
              "unmapping a clean cached buffer did not drain native work");
      resources.MapMemory(index_begin, index_page);
      const auto index_bridge =
          cache.FindBuffer(index_begin + index_page - 1, index_page + 2);
      Require(name, "registered-range merge",
              index_bridge && cache.IsRegionRegistered(index_begin + index_page,
                                                       index_page) &&
                  BufferCacheTestAccess::PageOwner(cache, index_begin) ==
                      index_bridge &&
                  BufferCacheTestAccess::PageOwner(
                      cache, index_begin + index_page) == index_bridge &&
                  BufferCacheTestAccess::PageOwner(
                      cache, index_begin + 2 * index_page) == index_bridge,
              "bridging acquisition did not publish its merged Buffer range");
      scheduler.Finish();
      constexpr uint64_t recycled_offset = index_offset + 4 * index_page;
      const auto recycled = cache.FindBuffer(base + recycled_offset + 0x100,
                                             sizeof(uint32_t));
      Require(name, "slot generation reuse",
              (recycled.index == index_left.index && recycled != index_left) ||
                  (recycled.index == index_right.index &&
                   recycled != index_right),
              "retired Buffer slot was not reused with a new generation");
      const auto [resolved_left, resolved_left_offset] = cache.ObtainBuffer(
          index_begin + 0x100, sizeof(uint32_t), true, false, index_left);
      const auto [resolved_right, resolved_right_offset] =
          cache.ObtainBuffer(index_begin + 2 * index_page + 0x100,
                             sizeof(uint32_t), true, false, index_right);
      Require(name, "two-pass stale id resolution",
              resolved_left == &cache.GetBuffer(index_bridge) &&
                  resolved_right == resolved_left &&
                  resolved_left_offset != resolved_right_offset,
              "saved descriptor IDs did not resolve through the merged owner");

      constexpr uint64_t stream_page = BufferCache::CACHING_PAGESIZE;
      constexpr uint64_t stream_leap = stream_page * 128;
      for (const bool forward : {true, false}) {
        const char *stream_name =
            forward ? "ForwardBufferStream" : "BackwardBufferStream";
        const uint64_t stream_begin = base + (forward ? 0x1800000 : 0x2000000);
        const uint64_t caught_address = forward
                                            ? stream_begin + stream_leap / 2
                                            : stream_begin - stream_leap / 2;
        const uint64_t untouched_address = forward
                                               ? stream_begin - stream_leap / 2
                                               : stream_begin + stream_leap / 2;
        const auto caught = cache.FindBuffer(caught_address, stream_page);
        const auto untouched = cache.FindBuffer(untouched_address, stream_page);
        auto grown = cache.FindBuffer(stream_begin, stream_page * 2);
        constexpr uint32_t stream_value = 0x6a3b92d1u;
        for (const auto address : {stream_begin, caught_address}) {
          MarkGpuWrite(address, sizeof(stream_value));
          cache.FillBuffer(address, sizeof(stream_value), stream_value, false);
        }

        uint64_t requested_begin = stream_begin;
        uint64_t requested_end = stream_begin + stream_page * 2;
        for (uint32_t step = 1; step <= 18; ++step) {
          const auto address = (forward ? requested_end : requested_begin) - 4;
          grown = cache.FindBuffer(address, 8);
          if (forward) {
            requested_end += stream_page;
          } else {
            requested_begin -= stream_page;
          }
          if (step < 18) {
            const auto &owner = cache.GetBuffer(grown);
            Require(stream_name, "growth before threshold",
                    owner.CpuAddress() == requested_begin &&
                        owner.Size() == requested_end - requested_begin &&
                        owner.StreamScore() == static_cast<int>(step),
                    "sequential requests grew beyond their consumed pages too "
                    "early");
          }
        }
        const auto &grown_buffer = cache.GetBuffer(grown);
        Require(stream_name, "growth reserve",
                grown_buffer.CpuAddress() ==
                        (forward ? requested_begin
                                 : requested_begin - stream_leap) &&
                    grown_buffer.Size() ==
                        requested_end - requested_begin + stream_leap &&
                    grown_buffer.StreamScore() == 0 &&
                    cache.GetBuffer(caught).is_deleted &&
                    !cache.GetBuffer(untouched).is_deleted &&
                    BufferCacheTestAccess::PageOwner(cache, caught_address) ==
                        grown &&
                    BufferCacheTestAccess::PageOwner(
                        cache, untouched_address) == untouched,
                "streaming did not reserve the next pages or retain correct "
                "neighboring owners");
        for (uint32_t step = 0; step < 128; ++step) {
          const auto address = forward
                                   ? requested_end + step * stream_page - 4
                                   : requested_begin - step * stream_page - 4;
          Require(stream_name, "reuse reserved pages",
                  cache.FindBuffer(address, 8) == grown,
                  "continued sequential requests replaced the buffer within "
                  "its growth reserve");
        }
        for (const auto address : {stream_begin, caught_address}) {
          Require(
              stream_name, "preserve GPU contents",
              ReadNativeValue(grown_buffer, grown_buffer.Offset(address)) ==
                  stream_value,
              "stream growth lost GPU-written contents from a merged owner");
        }
      }

      resources.UnmapMemory(index_begin, index_span);
      resources.UnmapMemory(base + recycled_offset, index_page);
      Require(name, "invalidation retains owners",
              !resources.IsMapped(index_begin, index_span) &&
                  !resources.IsMapped(base + recycled_offset, index_page) &&
                  cache.IsRegionRegistered(index_begin, index_span) &&
                  BufferCacheTestAccess::PageOwner(cache, index_begin) ==
                      index_bridge &&
                  BufferCacheTestAccess::PageOwner(
                      cache, index_begin + index_page) == index_bridge &&
                  BufferCacheTestAccess::PageOwner(
                      cache, index_begin + 2 * index_page) == index_bridge &&
                  BufferCacheTestAccess::PageOwner(
                      cache, base + recycled_offset) == recycled &&
                  cache.IsRegionCpuModified(index_begin, index_span) &&
                  cache.IsRegionCpuModified(base + recycled_offset,
                                            index_page),
              "CPU invalidation retired a reusable Buffer owner");
      resources.MapMemory(index_begin, index_span);
      resources.MapMemory(base + recycled_offset, index_page);

      MarkGpuWrite(base + first_offset, sizeof(first_value));
      MarkGpuWrite(base + second_offset, sizeof(second_value));
      cache.FillBuffer(base + first_offset, sizeof(first_value), first_value,
                       false);
      cache.FillBuffer(base + second_offset, sizeof(second_value), second_value,
                       false);
      auto &download = BufferCacheTestAccess::DownloadBuffer(cache);
      auto *fixed_download = &download;
      const auto fixed_download_handle = download.Handle();
      auto [ring_tail, ring_tail_offset] =
          download.Map(download.Size() - sizeof(uint32_t), 4);
      Require(name, "fault-ring tail reservation",
              ring_tail != nullptr && ring_tail_offset == 0,
              "failed to position the reusable download ring near wrap");
      download.Commit();
      MarkGpuWrite(base + ring_fault_first_offset,
                   sizeof(ring_fault_first_value));
      MarkGpuWrite(base + ring_fault_second_offset,
                   sizeof(ring_fault_second_value));
      cache.FillBuffer(base + ring_fault_first_offset,
                       sizeof(ring_fault_first_value), ring_fault_first_value,
                       false);
      cache.FillBuffer(base + ring_fault_second_offset,
                       sizeof(ring_fault_second_value), ring_fault_second_value,
                       false);
      Require(name, "fault-ring wrapped batch",
              resources.HandleFault(PageFaultAccess::Read,
                                    base + ring_fault_first_offset),
              "fault readback could not wrap a live download-ring tick");
      // Both disjoint writes on the faulting page share one wrapped batch.
      uint64_t expected_packing_offset = 2 * 64;
      uint64_t expected_packing_alignment = 1;
      if (!download.IsCoherent()) {
        Require(name, "fault-ring atom policy",
                StreamBufferTestAccess::NormalizeReservation(
                    false,
                    m_runtime_context.physical_device_properties.limits
                        .nonCoherentAtomSize,
                    expected_packing_offset, expected_packing_alignment),
                "failed to calculate non-coherent fault-ring stride");
      }
      auto [packing_probe, packing_offset] = download.Map(1, 1, false);
      Require(name, "adjacent download reservation stride",
              packing_probe != nullptr &&
                  packing_offset == expected_packing_offset,
              "adjacent fault downloads did not reserve an atom-safe stride");
      download.Commit();
      uint32_t first_page_backing = 0;
      uint32_t second_page_backing = 0;
      uint32_t ring_fault_first_backing = 0;
      uint32_t ring_fault_second_backing = 0;
      Libs::LibKernel::Memory::TryReadBacking(base + first_offset,
          &first_page_backing, sizeof(first_page_backing));
      Libs::LibKernel::Memory::TryReadBacking(base + second_offset,
          &second_page_backing, sizeof(second_page_backing));
      std::memcpy(&ring_fault_first_backing, memory + ring_fault_first_offset,
                  sizeof(ring_fault_first_backing));
      std::memcpy(&ring_fault_second_backing, memory + ring_fault_second_offset,
                  sizeof(ring_fault_second_backing));
      Require(name, "fault-ring wrapped contents",
              first_page_backing == first_stale &&
                  second_page_backing == second_stale &&
                  cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) &&
                  cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)) &&
                  ring_fault_first_backing == ring_fault_first_value &&
                  ring_fault_second_backing == ring_fault_second_value &&
                  &BufferCacheTestAccess::DownloadBuffer(cache) ==
                      fixed_download &&
                  download.Handle() == fixed_download_handle &&
                  download.Size() == (64ull << 20),
              "wrapped fault batch lost its page contents or published an unrelated page");
      cache.ReadMemory(base + first_offset, sizeof(first_value));
      Libs::LibKernel::Memory::TryReadBacking(base + first_offset,
          &first_page_backing, sizeof(first_page_backing));
      Libs::LibKernel::Memory::TryReadBacking(base + second_offset,
          &second_page_backing, sizeof(second_page_backing));
      Require(name, "deferred neighboring page read",
              first_page_backing == first_value && second_page_backing == second_value &&
                  !cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) &&
                  !cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)),
              "the earlier page did not publish its dirty ranges on its own read");

      constexpr uint64_t page_size = TRACKER_PAGE_SIZE;
      constexpr uint64_t page_owner_offset = 0x240000;
      constexpr uint64_t page_owner_size = 0xc0000;
      constexpr uint64_t page_fault_offset = page_owner_offset + 0x100;
      constexpr uint64_t page_inside_offset =
          page_owner_offset + page_size - sizeof(uint32_t);
      constexpr uint64_t page_outside_offset =
          page_owner_offset + page_size;
      constexpr uint32_t page_stale = 0x13579bdfu;
      constexpr uint32_t page_value = 0x2468ace0u;
      constexpr std::array page_offsets{
          page_fault_offset, page_inside_offset, page_outside_offset};
      static_assert((base + page_owner_offset) % page_size == 0);
      for (const auto offset : page_offsets) {
        Libs::LibKernel::Memory::WriteBacking(base + offset, &page_stale,
                                              sizeof(page_stale));
      }
      (void)cache.FindBuffer(base + page_owner_offset, page_owner_size);
      for (const auto offset : page_offsets) {
        MarkGpuWrite(base + offset, sizeof(page_value));
        cache.FillBuffer(base + offset, sizeof(page_value), page_value,
                         false);
      }
      const auto page_publication_tick = scheduler.CurrentTick();
      std::binary_semaphore page_publication_entered{0};
      std::binary_semaphore release_page_publication{0};
      std::atomic<bool> page_readback_returned{false};
      scheduler.DeferPriorityOperation([&] {
        page_publication_entered.release();
        release_page_publication.acquire();
        Libs::LibKernel::Memory::WriteBacking(
            base + page_inside_offset, &page_stale, sizeof(page_stale));
      });
      std::jthread release_page_callback([&] {
        page_publication_entered.acquire();
        const auto deadline = std::chrono::steady_clock::now() +
                              std::chrono::milliseconds(100);
        while (!page_readback_returned.load() &&
               std::chrono::steady_clock::now() < deadline) {
          std::this_thread::yield();
        }
        release_page_publication.release();
      });
      cache.ReadMemory(base + page_fault_offset, sizeof(page_value));
      page_readback_returned = true;
      release_page_callback.join();
      scheduler.WaitPriorityOperations(page_publication_tick);
      uint32_t page_inside_backing = 0;
      uint32_t page_outside_backing = 0;
      Libs::LibKernel::Memory::TryReadBacking(base + page_inside_offset,
                                              &page_inside_backing,
                                              sizeof(page_inside_backing));
      Libs::LibKernel::Memory::TryReadBacking(base + page_outside_offset,
                                              &page_outside_backing,
                                              sizeof(page_outside_backing));
      Require(name, "page publication order and boundary",
              page_inside_backing == page_value &&
                  page_outside_backing == page_stale &&
                  !cache.HasGpuDirtyBytes(base + page_inside_offset,
                                          sizeof(page_value)) &&
                  cache.HasGpuDirtyBytes(base + page_outside_offset,
                                         sizeof(page_value)) &&
                  !cache.IsRegionGpuModified(base + page_inside_offset,
                                             sizeof(page_value)) &&
                  cache.IsRegionGpuModified(base + page_outside_offset,
                                            sizeof(page_value)),
              "readback did not publish after the older overlapping callback "
              "or retained ownership of the wrong page");
      cache.ReadMemory(base + page_outside_offset, sizeof(page_value));
      Libs::LibKernel::Memory::TryReadBacking(base + page_outside_offset,
          &page_outside_backing, sizeof(page_outside_backing));
      Require(name, "next page publication",
              page_outside_backing == page_value &&
                  !cache.HasGpuDirtyBytes(base + page_outside_offset,
                                          sizeof(page_value)) &&
                  !cache.IsRegionGpuModified(base + page_outside_offset,
                                             sizeof(page_value)),
              "the next page did not publish and release ownership on its own read");

      Libs::LibKernel::Memory::WriteBacking(base + first_offset, &first_stale,
                                            sizeof(first_stale));
      Libs::LibKernel::Memory::WriteBacking(base + second_offset, &second_stale,
                                            sizeof(second_stale));
      MarkGpuWrite(base + first_offset, sizeof(first_value));
      MarkGpuWrite(base + second_offset, sizeof(second_value));
      cache.FillBuffer(base + first_offset, sizeof(first_value), first_value,
                       false);
      cache.FillBuffer(base + second_offset, sizeof(second_value), second_value,
                       false);

      for (uint32_t tick = 0; tick < 160; tick++) {
        cache.RunGarbageCollector();
      }
      Require(name, "age before pressure",
              cache.IsRegionRegistered(base, allocation_size),
              "buffer was reclaimed without memory pressure");
      BufferCacheTestAccess::SetGarbageCollectionThresholds(
          cache, 0, std::numeric_limits<uint64_t>::max());
      const auto gc_submission_tick = scheduler.CurrentTick();
      cache.RunGarbageCollector();
      uint32_t first_before_completion = 0;
      uint32_t second_before_completion = 0;
      Libs::LibKernel::Memory::TryReadBacking(base + first_offset,
                                              &first_before_completion,
                                              sizeof(first_before_completion));
      Libs::LibKernel::Memory::TryReadBacking(base + second_offset,
                                              &second_before_completion,
                                              sizeof(second_before_completion));
      Require(name, "normal dirty retention",
              first_before_completion == first_stale &&
                  second_before_completion == second_stale &&
                  cache.IsRegionRegistered(base, allocation_size) &&
                  cache.IsRegionGpuModified(base + first_offset,
                                            sizeof(first_value)) &&
                  cache.IsRegionGpuModified(base + second_offset,
                                            sizeof(second_value)) &&
                  cache.HasGpuDirtyBytes(base + first_offset,
                                         sizeof(first_value)) &&
                  cache.HasGpuDirtyBytes(base + second_offset,
                                         sizeof(second_value)) &&
                  scheduler.CurrentTick() == gc_submission_tick,
              "normal GC retired or synchronized a dirty Buffer");
      BufferCacheTestAccess::SetGarbageCollectionThresholds(cache, 0, 0);
      const auto older_publication_tick = scheduler.CurrentTick();
      std::binary_semaphore older_publication_entered{0};
      std::binary_semaphore release_older_publication{0};
      std::atomic<bool> gc_returned{false};
      scheduler.DeferPriorityOperation([&] {
        older_publication_entered.release();
        release_older_publication.acquire();
        Libs::LibKernel::Memory::WriteBacking(
            base + first_offset, &first_stale, sizeof(first_stale));
      });
      std::jthread release_publication([&] {
        older_publication_entered.acquire();
        const auto deadline = std::chrono::steady_clock::now() +
                              std::chrono::milliseconds(100);
        while (!gc_returned.load() &&
               std::chrono::steady_clock::now() < deadline) {
          std::this_thread::yield();
        }
        release_older_publication.release();
      });
      cache.RunGarbageCollector();
      gc_returned = true;
      release_publication.join();
      scheduler.WaitPriorityOperations(older_publication_tick);

      uint32_t first_backing = 0;
      uint32_t second_backing = 0;
      uint32_t clean_backing = 0;
      std::memcpy(&first_backing, memory + first_offset, sizeof(first_backing));
      std::memcpy(&second_backing, memory + second_offset,
                  sizeof(second_backing));
      std::memcpy(&clean_backing, memory + clean_offset, sizeof(clean_backing));
      Require(name, "downloaded contents",
              first_backing == first_value && second_backing == second_value &&
                  clean_backing == clean_value &&
                  !cache.IsRegionRegistered(base, allocation_size) &&
                  !cache.IsRegionGpuModified(base + first_offset,
                                             sizeof(first_value)) &&
                  !cache.IsRegionGpuModified(base + second_offset,
                                             sizeof(second_value)) &&
                  !cache.HasGpuDirtyBytes(base + first_offset,
                                          sizeof(first_value)) &&
                  !cache.HasGpuDirtyBytes(base + second_offset,
                                          sizeof(second_value)) &&
                  scheduler.CurrentTick() > gc_submission_tick,
              "critical GC did not publish exact ranges before clearing "
              "Buffer ownership");

      constexpr uint64_t starvation_offset = 0x100000;
      constexpr uint64_t starvation_stride = 2 * BufferCache::CACHING_PAGESIZE;
      constexpr uint32_t starvation_count = 33;
      constexpr uint32_t starvation_value = 0x31415926u;
      BufferCacheTestAccess::SetGarbageCollectionThresholds(
          cache, std::numeric_limits<uint64_t>::max(),
          std::numeric_limits<uint64_t>::max());
      for (uint32_t index = 0; index < starvation_count; index++) {
        const auto address = base + starvation_offset + index * starvation_stride;
        (void)cache.ObtainBuffer(address, sizeof(starvation_value), true);
        cache.FillBuffer(address, sizeof(starvation_value), starvation_value,
                         false);
      }
      for (uint32_t tick = 0; tick < 160; tick++) {
        cache.RunGarbageCollector();
      }
      constexpr uint64_t starvation_clean_offset =
          starvation_offset + starvation_count * starvation_stride;
      const auto starvation_clean = cache.FindBuffer(
          base + starvation_clean_offset, sizeof(starvation_value));
      Require(name, "normal-GC clean candidate",
              static_cast<bool>(starvation_clean),
              "failed to create the clean starvation candidate");
      for (uint32_t tick = 0; tick <= 160; tick++) {
        cache.RunGarbageCollector();
      }
      BufferCacheTestAccess::SetGarbageCollectionThresholds(
          cache, 0, std::numeric_limits<uint64_t>::max());
      cache.RunGarbageCollector();
      Require(
          name, "normal-GC dirty bypass",
          cache.IsRegionRegistered(base + starvation_offset,
                                   sizeof(starvation_value)) &&
              cache.IsRegionRegistered(
                  base + starvation_offset +
                      (starvation_count - 1) * starvation_stride,
                  sizeof(starvation_value)) &&
              !cache.IsRegionRegistered(base + starvation_clean_offset,
                                        sizeof(starvation_value)),
          "old dirty owners hid an eligible clean Buffer from normal GC");
      BufferCacheTestAccess::SetGarbageCollectionThresholds(cache, 0, 0);
      const auto starvation_retired =
          BufferCacheTestAccess::PageOwner(cache, base + starvation_offset);
      cache.RunGarbageCollector();
      Require(name, "critical-GC starvation cleanup",
              !cache.IsRegionRegistered(base + starvation_offset,
                                        sizeof(starvation_value)) &&
                  !cache.IsRegionRegistered(
                      base + starvation_offset +
                          (starvation_count - 1) * starvation_stride,
                      sizeof(starvation_value)) &&
                  !BufferCacheTestAccess::IsBufferAllocated(
                      cache, starvation_retired),
              "critical GC did not immediately free the skipped dirty owners");

      constexpr uint64_t lookup_only_offset = 0x218000;
      constexpr uint64_t obtained_offset = 0x220000;
      constexpr uint64_t residency_size =
          2 * BufferCache::CACHING_PAGESIZE;
      BufferCacheTestAccess::SetGarbageCollectionThresholds(
          cache, std::numeric_limits<uint64_t>::max(),
          std::numeric_limits<uint64_t>::max());
      const auto lookup_only = cache.FindBuffer(
          base + lookup_only_offset, residency_size);
      const auto obtained =
          cache.FindBuffer(base + obtained_offset, residency_size);
      for (uint32_t tick = 0; tick <= 160; tick++) {
        cache.RunGarbageCollector();
      }
      Require(name, "lookup-only owner identity",
              cache.FindBuffer(base + lookup_only_offset, residency_size) ==
                  lookup_only,
              "lookup-only discovery replaced its valid Buffer owner");
      const auto [obtained_buffer, obtained_buffer_offset] = cache.ObtainBuffer(
          base + obtained_offset, residency_size, false, false, obtained);
      BufferCacheTestAccess::SetGarbageCollectionThresholds(
          cache, 0, std::numeric_limits<uint64_t>::max());
      cache.RunGarbageCollector();
      Require(name, "lookup versus acquisition residency",
              !cache.IsRegionRegistered(base + lookup_only_offset,
                                        residency_size) &&
                  !BufferCacheTestAccess::PageOwner(
                      cache, base + lookup_only_offset) &&
                  obtained_buffer == &cache.GetBuffer(obtained) &&
                  obtained_buffer_offset == 0 &&
                  cache.IsRegionRegistered(base + obtained_offset,
                                           residency_size) &&
                  BufferCacheTestAccess::PageOwner(
                      cache, base + obtained_offset) == obtained,
              "lookup refreshed residency or actual acquisition did not");
      BufferCacheTestAccess::SetGarbageCollectionThresholds(cache, 0, 0);

      auto unmap_allocation = cache.ObtainBuffer(
          base + unmap_offset, sizeof(unmap_value), true, false);
      Require(name, "direct-invalidation allocation",
              unmap_allocation.first != nullptr,
              "direct-invalidation buffer allocation failed");
      cache.FillBuffer(base + unmap_offset, sizeof(unmap_value), unmap_value,
                       false);
      resources.UnmapMemory(base + 0x4000, 0x4000);
      uint32_t unmap_backing = 0;
      std::memcpy(&unmap_backing, memory + unmap_offset, sizeof(unmap_backing));
      Require(name, "direct-invalidation contents",
              unmap_backing == unmap_value &&
                  !resources.IsMapped(base + 0x4000, 0x4000) &&
                  cache.IsRegionRegistered(base + 0x4000, 0x4000) &&
                  cache.IsRegionCpuModified(base + 0x4000, 0x4000) &&
                  !cache.IsRegionGpuModified(base + 0x4000, 0x4000) &&
                  !cache.HasGpuDirtyBytes(base + 0x4000, 0x4000),
              "dirty invalidation did not publish bytes and retain its owner");
      std::memcpy(memory + unmap_offset, &remap_value, sizeof(remap_value));
      resources.MapMemory(base + 0x4000, 0x4000);
      const auto remapped =
          cache.ObtainBuffer(base + unmap_offset, sizeof(remap_value), true);
      Require(name, "same-address remap allocation",
              remapped.first == unmap_allocation.first,
              "same-address remap did not synchronize the retained Buffer");
      Require(name, "same-address remap contents",
              ReadNativeValue(*remapped.first, remapped.second) == remap_value,
              "same-address remap reused stale native Buffer bytes");

      auto partial_unmap_allocation =
          cache.ObtainBuffer(base + 0x8000, 0x8000, true, false);
      Require(name, "partial-invalidation allocation",
              partial_unmap_allocation.first != nullptr,
              "cross-page cached buffer allocation failed");
      cache.FillBuffer(base + partial_unmap_offset, sizeof(partial_unmap_value),
                       partial_unmap_value, false);
      cache.FillBuffer(base + partial_unmap_survivor_offset,
                       sizeof(partial_unmap_survivor_value),
                       partial_unmap_survivor_value, false);
      resources.UnmapMemory(base + 0x8000, 0x4000);
      Require(
          name, "partial-invalidation keeps disjoint bytes dirty",
          resources.IsMapped(base + partial_unmap_survivor_offset,
                             sizeof(partial_unmap_survivor_value)) &&
              !cache.IsRegionCpuModified(
                  base + partial_unmap_survivor_offset,
                  sizeof(partial_unmap_survivor_value)) &&
              cache.IsRegionGpuModified(base + partial_unmap_survivor_offset,
                                        sizeof(partial_unmap_survivor_value)) &&
              cache.HasGpuDirtyBytes(base + partial_unmap_survivor_offset,
                                     sizeof(partial_unmap_survivor_value)) &&
              !cache.HasGpuDirtyBytes(base + partial_unmap_offset,
                                      sizeof(partial_unmap_value)),
          "partial invalidation read back an unrelated mapped page");
      Require(name, "partial-invalidation survivor read fault",
              resources.HandleFault(PageFaultAccess::Read,
                                    base + partial_unmap_survivor_offset),
              "CPU read did not publish the retained GPU-dirty page");
      uint32_t partial_unmap_survivor_backing = 0;
      std::memcpy(&partial_unmap_survivor_backing,
                  memory + partial_unmap_survivor_offset,
                  sizeof(partial_unmap_survivor_backing));
      Require(
          name, "partial-invalidation ownership",
          !resources.IsMapped(base + 0x8000, 0x4000) &&
              cache.IsRegionRegistered(base + 0x8000, 0x8000) &&
              cache.IsRegionCpuModified(base + 0x8000, 0x4000) &&
              !cache.IsRegionCpuModified(
                  base + partial_unmap_survivor_offset,
                  sizeof(partial_unmap_survivor_value)) &&
              !cache.HasGpuDirtyBytes(base + partial_unmap_offset,
                                      sizeof(partial_unmap_value)) &&
              !cache.HasGpuDirtyBytes(base + partial_unmap_survivor_offset,
                                      sizeof(partial_unmap_survivor_value)) &&
              partial_unmap_survivor_backing == partial_unmap_survivor_value,
          "partial invalidation or subsequent CPU read mishandled ownership or "
          "backing");
      resources.MapMemory(base + 0x8000, 0x4000);
      auto survivor =
          cache.ObtainBuffer(base + partial_unmap_survivor_offset,
                             sizeof(partial_unmap_survivor_value), false);
      Require(name, "partial-invalidation survivor", survivor.first != nullptr,
              "cached-buffer remainder was not retained");
      Require(name, "partial-invalidation survivor contents",
              ReadNativeValue(*survivor.first, survivor.second) ==
                  partial_unmap_survivor_value,
              "partial invalidation lost disjoint native bytes");

      for (const uint64_t large_size : {64ull << 20, 68ull << 20}) {
        constexpr uint64_t large_offset = 0x10000;
        const uint32_t large_value =
            large_size == (64ull << 20) ? 0x5aa55aa5u : 0xa55aa55au;
        constexpr uint32_t large_stale = 0x12345678u;
        std::memcpy(memory + large_offset, &large_stale, sizeof(large_stale));
        std::memcpy(memory + large_offset + large_size - sizeof(large_stale),
                    &large_stale, sizeof(large_stale));
        auto large_allocation =
            cache.ObtainBuffer(base + large_offset, large_size, true, false);
        Require(name, "large dirty allocation",
                large_allocation.first != nullptr,
                "failed to allocate the large dirty native buffer");
        cache.FillBuffer(base + large_offset, large_size, large_value, false);
        const auto large_submission_tick = scheduler.CurrentTick();
        for (uint32_t tick = 0; tick <= 160; tick++) {
          cache.RunGarbageCollector();
        }
        // The capacity-sized case can wrap the ring; the oversized case must
        // keep its separate staging allocation alive through publication.
        Require(name, "large synchronized retirement",
                !cache.IsRegionRegistered(base + large_offset, large_size) &&
                    scheduler.CurrentTick() > large_submission_tick &&
                    scheduler.CurrentTick() <= large_submission_tick + 2,
                "large dirty Buffer exceeded the GC synchronization budget");
        uint32_t large_before_completion = 0;
        Libs::LibKernel::Memory::TryReadBacking(base + large_offset,
                                                &large_before_completion,
                                                sizeof(large_before_completion));
        Require(name, "large backing publication",
                large_before_completion == large_value,
                "large Buffer GC retired ownership before publication");
        const auto large_image_source =
            cache.ObtainBufferForImage(base + large_offset, sizeof(large_value));
        Require(name, "fixed download after Buffer retirement",
                large_image_source.first != nullptr &&
                    &BufferCacheTestAccess::DownloadBuffer(cache) ==
                        fixed_download &&
                    fixed_download->Handle() == fixed_download_handle &&
                    fixed_download->Size() == (64ull << 20),
                "image acquisition replaced the shared Buffer download stream");
        std::vector<uint32_t> large_published(large_size / sizeof(uint32_t));
        Require(name, "large Buffer publication contents",
                Libs::LibKernel::Memory::TryReadBacking(
                    base + large_offset, large_published.data(), large_size) &&
                    std::ranges::all_of(large_published, [large_value](uint32_t value) {
                      return value == large_value;
                    }),
                "large Buffer GC did not publish its complete transfer");
      }

      constexpr uint64_t grouped_first_offset = 0x10000;
      constexpr uint64_t grouped_second_offset = 0x2300000;
      constexpr uint64_t grouped_owner_size = 33ull * 1024 * 1024;
      constexpr uint32_t grouped_first_value = 0x1122aabbu;
      constexpr uint32_t grouped_second_value = 0x3344ccddu;
      constexpr uint32_t grouped_stale = 0;
      static_assert(grouped_owner_size < (64ull << 20) &&
                    grouped_owner_size * 2 > (64ull << 20));
      Libs::LibKernel::Memory::WriteBacking(
          base + grouped_first_offset, &grouped_stale, sizeof(grouped_stale));
      Libs::LibKernel::Memory::WriteBacking(
          base + grouped_second_offset, &grouped_stale, sizeof(grouped_stale));
      auto grouped_first = cache.ObtainBuffer(base + grouped_first_offset,
                                              grouped_owner_size, true, false);
      auto grouped_second = cache.ObtainBuffer(base + grouped_second_offset,
                                               grouped_owner_size, true, false);
      Require(name, "per-owner GC allocation",
              grouped_first.first != nullptr &&
                  grouped_second.first != nullptr &&
                  grouped_first.first != grouped_second.first,
              "disjoint GC candidates merged into one Buffer owner");
      cache.FillBuffer(base + grouped_first_offset, grouped_owner_size,
                       grouped_first_value, false);
      cache.FillBuffer(base + grouped_second_offset, grouped_owner_size,
                       grouped_second_value, false);
      for (uint32_t tick = 0; tick <= 160; tick++) {
        cache.RunGarbageCollector();
      }
      Require(name, "per-owner fixed-ring retirement",
              !cache.IsRegionRegistered(base + grouped_first_offset,
                                        grouped_owner_size) &&
                  !cache.IsRegionRegistered(base + grouped_second_offset,
                                            grouped_owner_size) &&
                  &BufferCacheTestAccess::DownloadBuffer(cache) ==
                      fixed_download &&
                  fixed_download->Handle() == fixed_download_handle,
              "GC aggregated disjoint retirees or replaced the download ring");
      uint32_t grouped_first_backing = 0;
      uint32_t grouped_second_backing = 0;
      Libs::LibKernel::Memory::TryReadBacking(base + grouped_first_offset,
                                              &grouped_first_backing,
                                              sizeof(grouped_first_backing));
      Libs::LibKernel::Memory::TryReadBacking(base + grouped_second_offset,
                                              &grouped_second_backing,
                                              sizeof(grouped_second_backing));
      Require(name, "per-owner fixed-ring contents",
              grouped_first_backing == grouped_first_value &&
                  grouped_second_backing == grouped_second_value,
              "per-owner GC transfers lost data while wrapping the fixed ring");

      constexpr uint64_t sparse_offset = 0x2180000;
      constexpr uint64_t sparse_owner_stride = 2 * BufferCache::CACHING_PAGESIZE;
      constexpr std::array<uint64_t, 3> sparse_starts{1, 7, 13};
      constexpr std::array<uint64_t, 3> sparse_sizes{3, 2, 1};
      constexpr std::array<uint32_t, 2> sparse_values{0x41372b19u, 0x957f6953u};
      std::array<uint8_t, 16> sparse_clean{};
      sparse_clean.fill(0xcdu);
      const auto FillSparseOwner = [&](uint32_t owner_index) {
        const auto address = base + sparse_offset +
                             owner_index * sparse_owner_stride;
        Libs::LibKernel::Memory::WriteBacking(address, sparse_clean.data(),
                                              sparse_clean.size());
        Libs::Graphics::Buffer *owner = nullptr;
        for (size_t index = 0; index < sparse_starts.size(); ++index) {
          const auto allocation = cache.ObtainBuffer(
              address + sparse_starts[index], sparse_sizes[index], true, false);
          Require(name, "sparse byte-range owner",
                  allocation.first != nullptr &&
                      (owner == nullptr || owner == allocation.first),
                  "disjoint byte ranges did not resolve to one cached owner");
          owner = allocation.first;
        }
        // Deliberately change native clean neighbors too: only the exact dirty
        // bytes may be published to guest memory.
        owner->Fill(owner->Offset(address), sparse_clean.size(),
                    sparse_values[owner_index]);
      };
      const auto CheckSparseOwner = [&](uint32_t owner_index) {
        const auto address = base + sparse_offset +
                             owner_index * sparse_owner_stride;
        auto expected = sparse_clean;
        std::array<uint8_t, sizeof(uint32_t)> value_bytes{};
        std::memcpy(value_bytes.data(), &sparse_values[owner_index],
                    value_bytes.size());
        for (size_t index = 0; index < sparse_starts.size(); ++index) {
          for (uint64_t byte = sparse_starts[index];
               byte < sparse_starts[index] + sparse_sizes[index]; ++byte) {
            expected[byte] = value_bytes[byte % value_bytes.size()];
          }
        }
        std::array<uint8_t, 16> published{};
        Require(name, "sparse byte-range publication",
                Libs::LibKernel::Memory::TryReadBacking(
                    address, published.data(), published.size()) &&
                    published == expected &&
                    !cache.HasGpuDirtyBytes(address, published.size()) &&
                    !cache.IsRegionGpuModified(address, published.size()),
                "odd-byte readback lost dirty bytes or overwrote clean neighbors");
      };
      FillSparseOwner(0);
      bool sparse_callback_ran = false;
      bool sparse_callback_clean = false;
      scheduler.DeferOperation([&] {
        sparse_callback_clean =
            !cache.HasGpuDirtyBytes(base + sparse_offset, sparse_clean.size()) &&
            !cache.IsRegionGpuModified(base + sparse_offset, sparse_clean.size());
        sparse_callback_ran = true;
      });
      const auto sparse_read_tick = scheduler.CurrentTick();
      cache.ReadMemory(base + sparse_offset + sparse_starts[0], sparse_sizes[0]);
      Require(name, "sparse byte-range submission",
              scheduler.CurrentTick() == sparse_read_tick + 1,
              "sparse source ranges were downloaded in separate submissions");
      scheduler.PopPendingOperations();
      Require(name, "readback callback ownership consistency",
              sparse_callback_ran && sparse_callback_clean,
              "readback drained a deferred callback before clearing page ownership");
      CheckSparseOwner(0);

      FillSparseOwner(0);
      FillSparseOwner(1);
      Require(name, "sparse disjoint GC owners",
              BufferCacheTestAccess::PageOwner(cache, base + sparse_offset) !=
                  BufferCacheTestAccess::PageOwner(
                      cache, base + sparse_offset + sparse_owner_stride),
              "sparse GC fixtures merged into one source owner");
      const auto sparse_gc_tick = scheduler.CurrentTick();
      for (uint32_t tick = 0; tick <= 160; ++tick) {
        cache.RunGarbageCollector();
      }
      Require(name, "sparse multi-owner GC submission",
              scheduler.CurrentTick() == sparse_gc_tick + 1 &&
                  !cache.IsRegionRegistered(base + sparse_offset,
                                             sparse_owner_stride + 16),
              "sparse GC owners did not share one completion drain");
      CheckSparseOwner(0);
      CheckSparseOwner(1);

      constexpr uint64_t disjoint_owner_offset = 0x2140000;
      constexpr uint64_t disjoint_owner_size = 0x8000;
      constexpr uint64_t disjoint_dirty_offset = disjoint_owner_offset + 0x100;
      constexpr uint64_t disjoint_new_owner_offset =
          disjoint_owner_offset + 0x5000;
      constexpr uint32_t disjoint_value = 0x89abcdefu;
      constexpr uint32_t disjoint_stale = 0x76543210u;
      std::memcpy(memory + disjoint_dirty_offset, &disjoint_stale,
                  sizeof(disjoint_stale));
      auto wide_owner = cache.ObtainBuffer(base + disjoint_owner_offset,
                                           disjoint_owner_size, false);
      Require(name, "disjoint retirement owner", wide_owner.first != nullptr,
              "failed to create the clean cross-page retirement owner");
      (void)cache.ObtainBuffer(base + disjoint_dirty_offset,
                               sizeof(disjoint_value), true, false);
      cache.FillBuffer(base + disjoint_dirty_offset, sizeof(disjoint_value),
                       disjoint_value, false);
      for (uint32_t tick = 0; tick <= 160; tick++) {
        cache.RunGarbageCollector();
      }
      Require(name, "disjoint synchronized retirement",
              !cache.IsRegionRegistered(base + disjoint_owner_offset,
                                        disjoint_owner_size),
              "cross-page owner survived pressured collection");
      uint32_t disjoint_before_unmap = 0;
      Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset,
                                              &disjoint_before_unmap,
                                              sizeof(disjoint_before_unmap));
      Require(
          name, "disjoint retirement publication",
          disjoint_before_unmap == disjoint_value,
          "whole-owner Buffer retired before publishing its dirty bytes");
      resources.UnmapMemory(base + disjoint_owner_offset + 0x4000, 0x4000);
      uint32_t disjoint_after_unmap = 0;
      Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset,
                                              &disjoint_after_unmap,
                                              sizeof(disjoint_after_unmap));
      Require(name, "disjoint synchronized invalidation",
              disjoint_after_unmap == disjoint_value &&
                  !cache.HasGpuDirtyBytes(base + disjoint_dirty_offset,
                                          sizeof(disjoint_value)),
              "disjoint invalidation lost the completed whole-owner publication");
      resources.MapMemory(base + disjoint_owner_offset + 0x4000, 0x4000);
      auto disjoint_new_owner =
          cache.ObtainBuffer(base + disjoint_new_owner_offset,
                             sizeof(disjoint_value), true, false);
      Require(name, "disjoint post-publication reacquire",
              disjoint_new_owner.first != nullptr,
              "disjoint acquisition failed after whole-owner publication");
      uint32_t disjoint_backing = 0;
      Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset,
                                              &disjoint_backing,
                                              sizeof(disjoint_backing));
      Require(name, "disjoint retirement contents",
              disjoint_backing == disjoint_value,
              "old retirement callback lost dirty bytes or retained ownership "
              "after a disjoint reacquire");
      cache.ReadMemory(base + disjoint_new_owner_offset,
                       sizeof(disjoint_value));

      constexpr uint64_t reacquire_owner_offset = 0x2200000;
      constexpr uint64_t reacquire_owner_size = 0x8000;
      constexpr uint64_t reacquire_dirty_offset =
          reacquire_owner_offset + 0x100;
      constexpr uint64_t reacquire_disjoint_offset =
          reacquire_owner_offset + 0x5000;
      constexpr uint32_t reacquire_value = 0xc001d00du;
      constexpr uint32_t reacquire_stale = 0x0badf00du;
      std::memcpy(memory + reacquire_dirty_offset, &reacquire_stale,
                  sizeof(reacquire_stale));
      auto reacquire_owner = cache.ObtainBuffer(base + reacquire_owner_offset,
                                                reacquire_owner_size, false);
      Require(name, "reacquire retirement owner",
              reacquire_owner.first != nullptr,
              "failed to create a fresh cross-page retirement owner");
      (void)cache.ObtainBuffer(base + reacquire_dirty_offset,
                               sizeof(reacquire_value), true, false);
      cache.FillBuffer(base + reacquire_dirty_offset, sizeof(reacquire_value),
                       reacquire_value, false);
      for (uint32_t tick = 0; tick <= 160; tick++) {
        cache.RunGarbageCollector();
      }
      Require(name, "reacquire synchronized retirement",
              !cache.IsRegionRegistered(base + reacquire_owner_offset,
                                        reacquire_owner_size),
              "fresh cross-page owner survived pressured collection");
      uint32_t reacquire_before = 0;
      Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset,
                                              &reacquire_before,
                                              sizeof(reacquire_before));
      Require(name, "reacquire publication completed",
              reacquire_before == reacquire_value,
              "fresh whole-owner ownership retired before publication");
      auto reacquired =
          cache.ObtainBuffer(base + reacquire_disjoint_offset,
                             sizeof(reacquire_value), true, false);
      Require(name, "independent disjoint post-publication reacquire",
              reacquired.first != nullptr,
              "clean disjoint-half acquisition failed after publication");
      uint32_t reacquire_after = 0;
      Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset,
                                              &reacquire_after,
                                              sizeof(reacquire_after));
      Require(name, "independent disjoint reacquire contents",
              reacquire_after == reacquire_value &&
                  !cache.HasGpuDirtyBytes(base + reacquire_dirty_offset,
                                          sizeof(reacquire_value)),
              "disjoint-half acquisition failed to publish the retired "
              "owner's dirty prefix");
      cache.ReadMemory(base + reacquire_disjoint_offset,
                       sizeof(reacquire_value));

      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
    }
    context.ShutdownGpu();
    Require(name, "unmap direct backing",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "dirty-GC direct-memory mapping release failed");
    Require(name, "release direct backing",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "dirty-GC direct-memory allocation release failed");
    std::printf("[host]    %-32s ok\n", name);
  }

  void CheckComputeMetaClearClassification() {
    constexpr const char *name = "ComputeMetaClearClassification";
    constexpr uint64_t read_only_meta = 0x0000000204201f00ull;
    constexpr uint64_t read_write_meta = 0x0000000204202000ull;
    constexpr uint64_t write_only_meta = 0x0000000204202100ull;
    EnsureRuntimeContext();
    RenderContext context(m_runtime_context);
    auto &scheduler = context.GetCommandScheduler();
    HW::Context registers{};
    HW::UserConfig user_config{};
    HW::Shader shaders{};
    scheduler.Begin(registers, user_config, shaders);
    auto &texture_cache = context.GetTextureCache();
    TextureCacheTestAccess::RegisterHtileMeta(texture_cache, read_only_meta);
    TextureCacheTestAccess::RegisterHtileMeta(texture_cache, read_write_meta);
    TextureCacheTestAccess::RegisterHtileMeta(texture_cache, write_only_meta);

    // An empty dispatch must not inspect even an unreadable shader address.
    shaders.SetCsShader({.data_addr = 1});
    constexpr std::array<std::array<uint32_t, 3>, 3> empty_dispatches{{
        {0, 1, 1}, {1, 0, 1}, {1, 1, 0}}};
    for (const auto mode : {0x41u, 0x61u}) {
      for (const auto &groups : empty_dispatches) {
        context.GetRenderExecutor().DispatchDirect(
            0, scheduler.Current(), groups[0], groups[1], groups[2], mode, false);
      }
    }
    Require(name, "HTile fixture",
            texture_cache.IsMeta(read_only_meta) &&
                texture_cache.IsMeta(read_write_meta) &&
                texture_cache.IsMeta(write_only_meta) &&
                !texture_cache.IsMetaCleared(read_only_meta, 0) &&
                !texture_cache.IsMetaCleared(read_write_meta, 0) &&
                !texture_cache.IsMetaCleared(write_only_meta, 0),
            "test HTile entries were not registered in their initial state");

    const auto MakeInput = [](uint64_t address, bool read, bool written,
                              ShaderRecompiler::IR::CompiledShaderInfo &program,
                              ShaderRecompiler::IR::ResourceSnapshot &snapshot) {
      program.stage = ShaderType::Compute;
      ShaderRecompiler::IR::BufferResource resource{};
      resource.read = read;
      resource.written = written;
      resource.formatted = true;
      program.info.buffers.push_back(resource);

      ShaderBufferResource descriptor{};
      descriptor.UpdateAddress48(address);
      descriptor.fields[1] |= 16u << 16u;
      descriptor.fields[2] = 8;
      descriptor.fields[3] =
          DstSel(4, 5, 6, 7) |
          (static_cast<uint32_t>(Prospero::BufferFormat::k32_32_32_32UInt)
           << 12u);
      ShaderRecompiler::IR::DescriptorValue value{};
      value.dword_count = 4;
      std::copy_n(descriptor.fields, value.dword_count, value.dwords.begin());
      ShaderComputeInputInfo input{};
      input.stage.program = &program;
      snapshot.buffers.push_back(value);
      input.stage.resources = &snapshot;
      return input;
    };
    auto &executor = context.GetRenderExecutor();
    auto &command = scheduler.Current();
    ShaderRecompiler::IR::CompiledShaderInfo read_only_program{};
    ShaderRecompiler::IR::ResourceSnapshot read_only_snapshot;
    const auto read_only_input =
        MakeInput(read_only_meta, true, false, read_only_program,
                  read_only_snapshot);
    const bool read_only_consumed =
        RenderExecutorTestAccess::TryConsumeComputeMetaClear(
            executor, read_only_input, command);
    Require(name, "read-only dispatch preserved",
            !read_only_consumed &&
                !texture_cache.IsMetaCleared(read_only_meta, 0),
            "a metadata read-only dispatch changed logical clear state");

    ShaderRecompiler::IR::CompiledShaderInfo read_write_program{};
    ShaderRecompiler::IR::ResourceSnapshot read_write_snapshot;
    const auto read_write_input =
        MakeInput(read_write_meta, true, true, read_write_program,
                  read_write_snapshot);
    const bool read_write_consumed =
        RenderExecutorTestAccess::TryConsumeComputeMetaClear(
            executor, read_write_input, command);
    Require(name, "read/write dispatch preserved",
            !read_write_consumed &&
                !texture_cache.IsMetaCleared(read_write_meta, 0),
            "a metadata read/modify/write dispatch was replaced by a clear");

    ShaderRecompiler::IR::CompiledShaderInfo write_only_program{};
    ShaderRecompiler::IR::ResourceSnapshot write_only_snapshot;
    const auto write_only_input =
        MakeInput(write_only_meta, false, true, write_only_program,
                  write_only_snapshot);
    const bool write_only_consumed =
        RenderExecutorTestAccess::TryConsumeComputeMetaClear(
            executor, write_only_input, command);
    Require(name, "write-only fill consumed",
            write_only_consumed &&
                texture_cache.IsMetaCleared(write_only_meta, 0),
            "a metadata write-only fill was not consumed as a clear");
    scheduler.Finish();
    std::printf("[host]    %-32s ok\n", name);
  }

  void CheckBdaPageTableUploads() {
    constexpr const char *name = "BdaPageTableUploads";
    constexpr uintptr_t base = 0x0000000207b00000ull;
    constexpr uint64_t page = BufferCache::CACHING_PAGESIZE;
    constexpr std::array<uint64_t, 3> page_counts{2, 521, 525};
    constexpr uint64_t result_entries = page_counts[0] + page_counts[1] + page_counts[2];
    EnsureRuntimeContext();
    RenderContext context(m_runtime_context);
    auto &scheduler = context.GetCommandScheduler();
    HW::Context registers{};
    HW::UserConfig user_config{};
    HW::Shader shaders{};
    scheduler.Begin(registers, user_config, shaders);
    auto &cache = context.GetBufferCache();
    // A small real ring exercises chunking and waits for same-command-buffer reuse.
    BufferCacheTestAccess::SetUnusedStagingBufferSize(cache, 4096);
    auto &table = *cache.GetBdaPageTableBuffer();
    const auto table_offset = BufferCache::PageIndex(base) * sizeof(vk::DeviceAddress);
    auto readback = CreateHostBuffer(name, result_entries * sizeof(vk::DeviceAddress),
                                    vk::BufferUsageFlagBits::eTransferDst, {});
    std::vector<uint64_t> expected;
    expected.reserve(result_entries);
    for (const auto pages : page_counts) {
      // The destination has a preceding GPU write and, after the first iteration,
      // a preceding read. Registration must preserve both ordering requirements.
      table.Fill(table_offset, pages * sizeof(vk::DeviceAddress), 0xa5a5a5a5u);
      const auto previous_tick = scheduler.CurrentTick();
      const auto id = cache.FindBuffer(base, pages * page);
      const auto &buffer = cache.GetBuffer(id);
      const auto destination_offset = expected.size() * sizeof(vk::DeviceAddress);
      for (uint64_t index = 0; index < pages; ++index) {
        expected.push_back(buffer.BufferDeviceAddress() + index * page);
      }
      const vk::BufferCopy copy{table_offset, destination_offset,
                                pages * sizeof(vk::DeviceAddress)};
      scheduler.Current().Handle().copyBuffer(table.Handle(), readback.buffer, 1, &copy);
      Require(name, "chunked registration",
              buffer.CpuAddress() == base && buffer.Size() == pages * page &&
                  (pages == page_counts.front() || scheduler.CurrentTick() > previous_tick),
              "BDA registration failed to replace its owner or wait for staging-ring reuse");
    }
    vk::BufferMemoryBarrier barrier{};
    barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
    barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
    barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
    barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
    barrier.buffer = readback.buffer;
    barrier.size = readback.size;
    scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
        vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr);
    scheduler.Finish();
    const auto words = ReadBuffer(name, readback, result_entries * 2);
    for (size_t index = 0; index < expected.size(); ++index) {
      const auto actual = uint64_t{words[index * 2]} | (uint64_t{words[index * 2 + 1]} << 32);
      Require(name, "GPU page-table snapshot", actual == expected[index],
              "staged BDA updates were stale, overwritten before their read, or incompletely flushed");
    }
    DestroyBuffer(&readback);
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckUnifiedTextureCacheFlow() {
    constexpr const char *name = "UnifiedTextureCacheFlow";
    constexpr uintptr_t base = 0x0000000200600000ull;
    constexpr uint64_t allocation_size = 0x3000000;
    constexpr uint64_t allocation_alignment = 0x200000;
    EnsureRuntimeContext();
    RenderContext context(m_runtime_context);
    auto &scheduler = context.GetCommandScheduler();
    HW::Context registers{};
    HW::UserConfig user_config{};
    HW::Shader shaders{};
    scheduler.Begin(registers, user_config, shaders);
    context.InitializeGpu(nullptr);
    auto &gpu = context.GetGpu();
    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_alignment, 0, &direct_offset) == 0,
            "cache test direct-memory allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_alignment) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "fixed cache direct-memory mapping failed");
    auto *memory = static_cast<uint8_t *>(mapped);
    const uint32_t initial = 0x44332211u;
    std::memcpy(memory, &initial, sizeof(initial));

    {
      auto &resources = context;
      LibKernel::Memory::InstallGpuResources(&resources);
      auto &texture_cache = resources.GetTextureCache();
      const auto [narrow_download, narrow_download_offset] =
          TextureCacheTestAccess::MapDownload(texture_cache, 4, 4);
      const auto [wide_download, wide_download_offset] =
          TextureCacheTestAccess::MapDownload(texture_cache, 1, 16);
      Require(name, "image download alignment",
              narrow_download != nullptr && narrow_download_offset % 4 == 0 &&
                  wide_download != nullptr && wide_download_offset % 16 == 0,
              "wide/block image readback was not aligned to its texel block");
      resources.MapMemory(base, allocation_size);

      auto sampled =
          MakeLinearDesc(base, sizeof(initial), vk::Format::eR8G8B8A8Srgb,
                         Prospero::BufferFormat::k8_8_8_8Srgb,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);

      auto &command = scheduler.Current();
      auto first_desc = sampled;
      const auto first = texture_cache.FindImage(first_desc);
      auto repeated_desc = sampled;
      const auto repeated = texture_cache.FindImage(repeated_desc);
      auto compatible_desc = sampled;
      compatible_desc.info.pixel_format = vk::Format::eR8G8B8A8Uint;
      compatible_desc.info.guest_format = Prospero::BufferFormat::k8_8_8_8UInt;
      compatible_desc.view_info.format = compatible_desc.info.pixel_format;
      const auto compatible = texture_cache.FindImage(compatible_desc);
      Require(name, "normalized FindImage",
              first && repeated == first && compatible == first &&
                  texture_cache.GetImage(first).info.pixel_format ==
                      vk::Format::eR8G8B8A8Srgb,
              "registered compatible backing did not reuse one ImageId");

      const auto IsOnlyImage = [](const std::vector<ImageId> &ids,
                                  ImageId expected) {
        return ids.size() == 1 && ids.front() == expected;
      };
      const auto exact_byte_miss =
          TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, false);
      const auto touched_page_hit =
          TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, true);
      const auto next_page_miss = TextureCacheTestAccess::FindImages(
          texture_cache, base + 0x1000, 1, true);
      Require(name, "exact and 4-KiB page filtering",
              exact_byte_miss.empty() && IsOnlyImage(touched_page_hit, first) &&
                  next_page_miss.empty(),
              "coarse candidates did not preserve exact-byte and touched-page "
              "semantics");

      const auto MakeOwnershipInfo = [](uint64_t address, uint64_t size) {
        ImageInfo info{};
        info.data = {address, size};
        info.extent = {1, 1, 1};
        info.resources = {1, 1};
        info.samples = 1;
        return info;
      };
      const auto spanning_info = MakeOwnershipInfo(base + 0x1ff000, 0x2000);
      const auto spanning =
          TextureCacheTestAccess::InsertImage(texture_cache, spanning_info);
      const auto spanning_results = TextureCacheTestAccess::FindImages(
          texture_cache, spanning_info.data.address, spanning_info.data.size,
          false);
      Require(name, "one-MiB cross-page deduplication",
              spanning && IsOnlyImage(spanning_results, spanning) &&
                  TextureCacheTestAccess::PageOwnerCount(
                      texture_cache, spanning_info.data.address) == 1 &&
                  TextureCacheTestAccess::PageOwnerCount(
                      texture_cache, spanning_info.data.End() - 1) == 1,
              "an image spanning two coarse pages was missing or returned more "
              "than once");
      TextureCacheTestAccess::DeleteImage(texture_cache, spanning);
      Require(name, "cross-page owner cleanup",
              TextureCacheTestAccess::PageOwnerCount(
                  texture_cache, spanning_info.data.address) == 0 &&
                  TextureCacheTestAccess::PageOwnerCount(
                      texture_cache, spanning_info.data.End() - 1) == 0 &&
                  TextureCacheTestAccess::FindImages(
                      texture_cache, spanning_info.data.address,
                      spanning_info.data.size, false)
                      .empty(),
              "cross-page unregister left a stale coarse-page membership");

      const auto shared_info = MakeOwnershipInfo(base + 0x500100, 0x100);
      const auto shared_first =
          TextureCacheTestAccess::InsertImage(texture_cache, shared_info);
      const auto shared_second =
          TextureCacheTestAccess::InsertImage(texture_cache, shared_info);
      const auto shared_results = TextureCacheTestAccess::FindImages(
          texture_cache, shared_info.data.address, shared_info.data.size,
          false);
      Require(name, "shared coarse-page registration",
              shared_results.size() == 2 &&
                  std::find(shared_results.begin(), shared_results.end(),
                            shared_first) != shared_results.end() &&
                  std::find(shared_results.begin(), shared_results.end(),
                            shared_second) != shared_results.end(),
              "two registered owners were not retained in one coarse page");
      TextureCacheTestAccess::DeleteImage(texture_cache, shared_first);
      const auto shared_survivor = TextureCacheTestAccess::FindImages(
          texture_cache, shared_info.data.address, shared_info.data.size,
          false);
      Require(name, "shared coarse-page unregister",
              IsOnlyImage(shared_survivor, shared_second) &&
                  TextureCacheTestAccess::PageOwnerCount(
                      texture_cache, shared_info.data.address) == 1,
              "unregistering one owner removed its coarse-page neighbor");
      TextureCacheTestAccess::DeleteImage(texture_cache, shared_second);
      Require(name, "final shared coarse-page unregister",
              TextureCacheTestAccess::PageOwnerCount(
                  texture_cache, shared_info.data.address) == 0,
              "the final shared owner remained registered");

      constexpr uint64_t large_owner_size = 64ull * 1024 * 1024;
      const auto large_info =
          MakeOwnershipInfo(base + 0x800000, large_owner_size);
      const auto large_owner =
          TextureCacheTestAccess::InsertImage(texture_cache, large_info);
      Require(name, "production one-MiB registration granularity",
              TextureCacheTestAccess::OwnedPageCount(
                  texture_cache, large_info.data.address, large_info.data.size,
                  large_owner) == 64,
              "64 MiB image registration did not create exactly 64 coarse "
              "memberships");
      TextureCacheTestAccess::DeleteImage(texture_cache, large_owner);
      Require(name, "production one-MiB unregister granularity",
              TextureCacheTestAccess::OwnedPageCount(
                  texture_cache, large_info.data.address, large_info.data.size,
                  large_owner) == 0,
              "large image unregister left coarse memberships behind");

      const ImageId stale{first.index, first.generation + 1};
      TextureCacheTestAccess::AddPageOwner(texture_cache, base, stale);
      const auto stale_filtered = TextureCacheTestAccess::FindImages(
          texture_cache, base, sizeof(initial), false);
      Require(name, "stale page owner filtering",
              IsOnlyImage(stale_filtered, first) &&
                  TextureCacheTestAccess::RemovePageOwner(texture_cache, base,
                                                          stale),
              "a stale generation escaped the direct page-owner lookup");

      TextureCacheTestAccess::SetQueryEpoch(texture_cache, UINT32_MAX);
      const auto wrap_results = TextureCacheTestAccess::FindImages(
          texture_cache, base, sizeof(initial), false);
      Require(name, "page-owner query epoch wrap",
              IsOnlyImage(wrap_results, first) &&
                  TextureCacheTestAccess::QueryEpoch(texture_cache) == 1,
              "query deduplication failed while wrapping its epoch");

      auto &image = texture_cache.GetImage(first);
      constexpr uint32_t final_sampled_value = 0x88776655u;
      Require(name, "sampled write between discovery and acquisition",
              resources.HandleFault(PageFaultAccess::Write,
                                    sampled.info.data.address),
              "sampled image did not accept a CPU update after discovery");
      std::memcpy(memory, &final_sampled_value, sizeof(final_sampled_value));
      Require(name, "sampled final-acquisition precondition",
              image.IsCpuDirty(),
              "the between-phase CPU write did not dirty the discovered image");
      const auto first_view = texture_cache.FindTexture(first, first_desc);
      const auto repeated_view =
          texture_cache.FindTexture(repeated, repeated_desc);
      auto mapped_view_info = sampled.view_info;
      mapped_view_info.mapping.r = vk::ComponentSwizzle::eB;
      mapped_view_info.mapping.b = vk::ComponentSwizzle::eR;
      const auto mapped_view = image.FindView(mapped_view_info);
      Require(
          name, "dynamic views",
          first_view != nullptr && repeated_view == first_view &&
              mapped_view != nullptr && mapped_view != first_view &&
              image.views.size() == 2 &&
              !image.IsGpuModified() && !image.IsCpuDirty() &&
              !TextureCacheTestAccess::PendingDownload(texture_cache, first),
          "TextureCache::FindTexture did not preserve sampled view "
          "identity, final refresh, and cache-only ownership");

      constexpr uint64_t resource_replacement_offset = 0x700000;
      const std::array<uint32_t, 3> resource_replacement_data{
          0x01020304u, 0x11121314u, 0x21222324u};
      std::memcpy(memory + resource_replacement_offset,
                  resource_replacement_data.data(),
                  sizeof(resource_replacement_data));
      auto underspecified = sampled;
      underspecified.info.data = {base + resource_replacement_offset,
                                  sizeof(resource_replacement_data)};
      underspecified.info.pixel_format = vk::Format::eR32Uint;
      underspecified.info.guest_format = Prospero::BufferFormat::k32UInt;
      underspecified.info.extent = {2, 1, 1};
      underspecified.info.resources = {1, 1};
      underspecified.info.pitch = 2;
      underspecified.info.mip_layout[0] = {0, 8, 2, 1, 0, 8, 2};
      underspecified.info.mip_layout[1] = {8, 4, 1, 1, 8, 4, 1};
      underspecified.info.linear_slice_stride = 0;
      underspecified.view_info.format = underspecified.info.pixel_format;
      underspecified.view_info.level_count = 1;
      const auto underspecified_id = texture_cache.FindImage(underspecified);
      (void)texture_cache.FindTexture(underspecified_id, underspecified);
      texture_cache.MarkGpuWritten(underspecified_id);

      auto complete = underspecified;
      complete.info.resources = {2, 1};
      complete.view_info.level_count = 2;
      const auto complete_id = texture_cache.FindImage(complete);
      const auto complete_owner =
          TextureCacheTestAccess::Owner(texture_cache, complete_id);
      Require(name, "insufficient-resource replacement",
              complete_id && complete_id != underspecified_id &&
                  !TextureCacheTestAccess::Contains(texture_cache,
                                                    underspecified_id) &&
                  complete_owner != nullptr && complete_owner->registered &&
                  complete_owner->info.resources == complete.info.resources &&
                  !complete_owner->IsGpuModified() &&
                  complete_owner->IsDefinitelyCpuDirty(),
              "an exact backing with insufficient resources was expanded "
              "instead of being discarded and recreated");

      ImageInfo chain = sampled.info;
      chain.data = {0x2394c1000, 0x384000};
      chain.extent = {64, 64, 1};
      chain.resources = {7, 150};
      chain.pitch = 64;
      chain.tile_mode = Prospero::TileMode::kStandard4KB;
      chain.mip_layout[0] = {0x2000, 0x258000, 64, 64};
      chain.mip_layout[1] = {0x1000, 0x96000, 32, 32};
      for (uint32_t level = 2; level < chain.resources.levels; ++level) {
        chain.mip_layout[level] = {0, 0x96000, 32, 32};
      }
      ImageInfo subresource = chain;
      subresource.data = {0x2394c3000, 0x4000};
      subresource.resources = {1, 1};
      subresource.mip_layout = {};
      subresource.mip_layout[0] = {0, 0x4000, 64, 64};
      const auto mip = subresource.MipOf(chain);
      Require(name, "overlap resolution",
              mip == 0 && subresource.SliceOf(chain, mip) == 0 &&
                  ImageRangeOverlaps(chain.data, subresource.data) &&
                  ImagePageRangesOverlap(chain.data, subresource.data),
              "captured PS5 atlas mip/slice overlap did not resolve");

      auto exact_desc = compatible_desc;
      const auto exact = texture_cache.FindImage(exact_desc, true);
      Require(name, "exact-format coexistence",
              exact && exact != first &&
                  TextureCacheTestAccess::Contains(texture_cache, first) &&
                  texture_cache.GetImage(first).info.pixel_format ==
                      sampled.info.pixel_format &&
                  texture_cache.GetImage(exact).info.pixel_format ==
                      compatible_desc.info.pixel_format,
              "exact-format lookup replaced or synchronized its compatible "
              "cache record");
      auto compatible_after_exact_desc = compatible_desc;
      const auto compatible_after_exact =
          texture_cache.FindImage(compatible_after_exact_desc);
      Require(name, "last compatible format winner",
              compatible_after_exact == exact,
              "non-exact lookup did not retain the last compatible "
              "registration");

      auto null_desc = sampled;
      null_desc.info.data = {};
      auto null_repeat_desc = null_desc;
      const auto null_image = texture_cache.FindImage(null_desc);
      const auto null_repeat = texture_cache.FindImage(null_repeat_desc);
      Require(name, "null image",
              null_image && null_repeat == null_image && null_image != exact,
              "typed null-image lookup was not stable");

      constexpr uint64_t extent_alias_offset = 0x2400000;
      auto narrow_target = sampled;
      narrow_target.type = BindingType::RenderTarget;
      narrow_target.info.data = {base + extent_alias_offset, 0x10000};
      narrow_target.info.pixel_format = vk::Format::eR16G16B16A16Sfloat;
      narrow_target.info.guest_format =
          Prospero::BufferFormat::k16_16_16_16Float;
      narrow_target.info.extent = {1, 1, 1};
      narrow_target.info.pitch = 128;
      narrow_target.info.bytes_per_block = 8;
      narrow_target.info.tile_mode = Prospero::TileMode::kRenderTarget;
      narrow_target.info.UpdateSize();
      narrow_target.view_info.format = narrow_target.info.pixel_format;
      narrow_target.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
      const auto narrow_target_id = texture_cache.FindImage(narrow_target);
      const auto narrow_target_view =
          texture_cache.FindRenderTarget(narrow_target_id, narrow_target);

      auto wide_target = narrow_target;
      wide_target.info.extent.width = 9;
      wide_target.info.UpdateSize();
      const auto wide_target_id = texture_cache.FindImage(wide_target);
      const auto wide_target_view =
          texture_cache.FindRenderTarget(wide_target_id, wide_target);
      const auto wide_owner =
          TextureCacheTestAccess::Owner(texture_cache, wide_target_id);
      Require(name, "equal-allocation render-target growth",
              narrow_target_view != nullptr && wide_target_view != nullptr &&
                  wide_target_id != narrow_target_id &&
                  !TextureCacheTestAccess::Contains(texture_cache,
                                                    narrow_target_id) &&
                  wide_owner != nullptr &&
                  wide_owner->info.extent == vk::Extent3D{9, 1, 1} &&
                  wide_owner->backing.extent == vk::Extent3D{9, 1, 1},
              "a 9x1 render target reused its equal-size 1x1 native backing");

      auto narrow_again = narrow_target;
      const auto narrow_again_id = texture_cache.FindImage(narrow_again);
      Require(name, "larger render-target backing reuse",
              narrow_again_id == wide_target_id &&
                  texture_cache.GetImage(narrow_again_id).backing.extent ==
                      vk::Extent3D{9, 1, 1},
              "a smaller render area replaced its compatible larger backing");

      const auto HostReadBarrier = [&](vk::Buffer buffer, uint64_t size,
                                       vk::PipelineStageFlags source_stage,
                                       vk::AccessFlags source_access) {
        vk::BufferMemoryBarrier barrier{};
        barrier.sType = vk::StructureType::eBufferMemoryBarrier;
        barrier.srcAccessMask = source_access;
        barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
        barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        barrier.buffer = buffer;
        barrier.offset = 0;
        barrier.size = size;
        scheduler.Current().Handle().pipelineBarrier(
            source_stage, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
            &barrier, 0, nullptr);
      };
      const auto TransferReadBarrier = [&](vk::Buffer buffer, uint64_t size) {
        vk::BufferMemoryBarrier barrier{};
        barrier.sType = vk::StructureType::eBufferMemoryBarrier;
        barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
        barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead;
        barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        barrier.buffer = buffer;
        barrier.offset = 0;
        barrier.size = size;
        scheduler.Current().Handle().pipelineBarrier(
            vk::PipelineStageFlagBits::eTransfer,
            vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr, 1, &barrier,
            0, nullptr);
      };

      constexpr uint64_t mip_layout_offset = 0x27f0000;
      std::vector<u32> mip_guest(0x2000 / sizeof(u32));
      std::iota(mip_guest.begin(), mip_guest.end(), 0x51000000u);
      ImageId previous{};
      for (const uint32_t width : {32u, 64u, 32u}) {
        if (previous) {
          Require(name, "mip-layout CPU overwrite",
                  resources.HandleFault(PageFaultAccess::Write,
                                        base + mip_layout_offset),
                  "mip-layout guest overwrite did not invalidate the old texture");
        }
        std::memcpy(memory + mip_layout_offset, mip_guest.data(),
                    mip_guest.size() * sizeof(u32));
        TileSurfaceLayout surface{};
        Require(name, "mip-tail layout",
                TileGetTiledTextureLayout(
                    {.format = Prospero::BufferFormat::kBc3Srgb,
                     .tile_mode = Prospero::TileMode::kStandard4KB,
                     .width = width, .height = 64, .levels = 7}, surface) &&
                    surface.first_tail_level == (width == 32 ? 0u : 1u) &&
                    surface.total_size == (width == 32 ? 0x1000u : 0x2000u),
                "BC3 width change did not move mip zero into or out of the tail");
        ImageDesc desc{};
        desc.type = BindingType::Texture;
        desc.info.data = {base + mip_layout_offset, surface.total_size};
        desc.info.pixel_format = vk::Format::eBc3SrgbBlock;
        desc.info.guest_format = Prospero::BufferFormat::kBc3Srgb;
        desc.info.extent = {width, 64, 1};
        desc.info.resources = {7, 1};
        desc.info.pitch = 64;
        desc.info.bytes_per_block = 16;
        desc.info.tile_mode = Prospero::TileMode::kStandard4KB;
        desc.view_info.format = desc.info.pixel_format;
        desc.view_info.type = vk::ImageViewType::e2D;
        desc.view_info.level_count = 7;
        desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
        desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
        std::vector<vk::BufferImageCopy> copies;
        std::vector<u32> expected;
        for (uint32_t level = 0; level < 7; level++) {
          const auto &layout = surface.mips[level];
          desc.info.mip_layout[level] = {
              layout.offset, layout.size, layout.padded_width,
              layout.padded_height};
          vk::BufferImageCopy copy{};
          copy.bufferOffset = expected.size() * sizeof(u32);
          copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, level, 0, 1};
          copy.imageExtent = {std::max(width >> level, 1u),
                              std::max(64u >> level, 1u), 1};
          copies.push_back(copy);
          for (uint32_t y = 0; y < layout.height; y++) {
            for (uint32_t x = 0; x < layout.width; x++) {
              uint32_t offset = 0;
              Require(name, "mip-tail CPU address",
                      TileGetBlockOffset(
                          surface.texture.block, x + layout.tail_x,
                          y + layout.tail_y, 0, offset),
                      "BC3 mip-tail block address could not be computed");
              const auto word = (layout.offset + offset) / sizeof(u32);
              expected.insert(expected.end(), mip_guest.begin() + word,
                              mip_guest.begin() + word + 4);
            }
          }
        }
        desc.info.UpdateSize();
        const auto id = texture_cache.FindImage(desc);
        Require(name, "distinct mip-layout backing", id && id != previous,
                "different guest mip layouts reused the same native image");
        (void)texture_cache.FindTexture(id, desc);
        auto readback = CreateHostBuffer(
            name, expected.size() * sizeof(u32),
            vk::BufferUsageFlagBits::eTransferDst,
            std::vector<u32>(expected.size()));
        texture_cache.GetImage(id).Download(copies, readback.buffer, 0,
                                            readback.size);
        HostReadBarrier(readback.buffer, readback.size,
                        vk::PipelineStageFlagBits::eTransfer,
                        vk::AccessFlagBits::eTransferWrite);
        scheduler.Finish();
        Require(name, "mip-layout GPU contents",
                ReadBuffer(name, readback, expected.size()) == expected,
                "cached native copying contaminated the requested guest mip layout");
        DestroyBuffer(&readback);
        if (width == 64) {
          // Leave only the larger backing so shrinking must reject its layout.
          TextureCacheTestAccess::DeleteImage(texture_cache, previous);
        }
        previous = id;
      }

      // The game reuses a raw render target allocation for a larger BC5 mip
      // chain. Equal block extents do not make their guest tiling compatible.
      constexpr uint64_t bc5_alias_offset = 0x1800000;
      TileSurfaceLayout bc5_surface{};
      Require(name, "BC5 alias layout",
              TileGetTiledTextureLayout(
                  {.format = Prospero::BufferFormat::kBc5UNorm,
                   .tile_mode = Prospero::TileMode::kStandard64KB,
                   .width = 1024, .height = 1024, .levels = 11}, bc5_surface) &&
                  bc5_surface.total_size == 0x160000,
              "captured BC5 allocation no longer has its 0x160000-byte footprint");
      std::memset(memory + bc5_alias_offset, 0, bc5_surface.total_size);
      auto raw_alias = MakeLinearDesc(
          base + bc5_alias_offset, 0x100000, vk::Format::eR32G32B32A32Sfloat,
          Prospero::BufferFormat::k32_32_32_32Float, Prospero::ImageType::kColor2D,
          {256, 256, 1}, 1, 16, 1);
      raw_alias.type = BindingType::RenderTarget;
      raw_alias.info.tile_mode = Prospero::TileMode::kRenderTarget;
      raw_alias.info.UpdateSize();
      raw_alias.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
      const auto raw_alias_id = texture_cache.FindImage(raw_alias);
      (void)texture_cache.FindRenderTarget(raw_alias_id, raw_alias);
      vk::ClearValue raw_alias_clear{};
      std::fill_n(raw_alias_clear.color.float32.data(), 4, 1.0f);
      TextureCacheTestAccess::ClearImage(
          texture_cache, scheduler.Current(), raw_alias_id,
          {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, raw_alias_clear);
      Require(name, "BC5 alias CPU overwrite",
              resources.InvalidateMemory(base + bc5_alias_offset,
                                         bc5_surface.total_size),
              "new BC5 data could not invalidate the old render target");
      std::vector<u32> bc5_guest(bc5_surface.total_size / sizeof(u32));
      std::iota(bc5_guest.begin(), bc5_guest.end(), 0x41000000u);
      std::memcpy(memory + bc5_alias_offset, bc5_guest.data(),
                  bc5_surface.total_size);
      auto bc5_alias = MakeLinearDesc(
          base + bc5_alias_offset, bc5_surface.total_size, vk::Format::eBc5UnormBlock,
          Prospero::BufferFormat::kBc5UNorm, Prospero::ImageType::kColor2D,
          {1024, 1024, 1}, 1, 16, 1);
      bc5_alias.info.resources.levels = 11;
      bc5_alias.info.tile_mode = Prospero::TileMode::kStandard64KB;
      bc5_alias.view_info.level_count = 11;
      std::vector<vk::BufferImageCopy> bc5_probes;
      std::vector<u32> bc5_expected;
      for (u32 level = 0; level < 11; ++level) {
        const auto &layout = bc5_surface.mips[level];
        bc5_alias.info.mip_layout[level] = {
            layout.offset, layout.size, layout.padded_width, layout.padded_height};
        u32 offset = 0;
        Require(name, "BC5 alias block address",
                TileGetBlockOffset(bc5_surface.texture.block, layout.tail_x,
                                   layout.tail_y, 0, offset),
                "BC5 mip origin could not be addressed in guest memory");
        vk::BufferImageCopy copy{};
        copy.bufferOffset = bc5_expected.size() * sizeof(u32);
        copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, level, 0, 1};
        copy.imageExtent = {std::min(1024u >> level, 4u),
                            std::min(1024u >> level, 4u), 1};
        bc5_probes.push_back(copy);
        const auto word = (layout.offset + offset) / sizeof(u32);
        bc5_expected.insert(bc5_expected.end(), bc5_guest.begin() + word,
                            bc5_guest.begin() + word + 4);
      }
      bc5_alias.info.UpdateSize();
      const auto bc5_alias_id = texture_cache.FindImage(bc5_alias);
      Require(name, "BC5 alias owner separation",
              bc5_alias_id != raw_alias_id &&
                  TextureCacheTestAccess::Contains(texture_cache, raw_alias_id) &&
                  !texture_cache.GetImage(bc5_alias_id).usage.render_target &&
                  texture_cache.GetImage(bc5_alias_id).info.tile_mode ==
                      Prospero::TileMode::kStandard64KB,
              "BC5 image expanded the unrelated raw render target or inherited its usage");
      (void)texture_cache.FindTexture(bc5_alias_id, bc5_alias);
      auto bc5_readback = CreateHostBuffer(
          name, bc5_expected.size() * sizeof(u32),
          vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>(bc5_expected.size()));
      texture_cache.GetImage(bc5_alias_id).Download(
          bc5_probes, bc5_readback.buffer, 0, bc5_readback.size);
      HostReadBarrier(bc5_readback.buffer, bc5_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      scheduler.Finish();
      Require(name, "BC5 alias mip contents",
              ReadBuffer(name, bc5_readback, bc5_expected.size()) == bc5_expected,
              "raw render-target copying overwrote the BC5 guest mip layout");
      DestroyBuffer(&bc5_readback);

      {
        constexpr uint64_t tiled_array_offset = 0x2c00000;
        constexpr uint32_t tiled_levels = 5, tiled_layers = 6;
        TileSurfaceLayout surface{};
        Require(name, "tiled array alias layout",
                TileGetTiledTextureLayout(
                    {Prospero::BufferFormat::k32UInt, Prospero::TileMode::kStandard64KB,
                     TileSurfaceDimension::Dim2D, 256, 256, 1, tiled_levels, tiled_layers},
                    surface) && surface.total_size == 0x240000 &&
                    surface.block_slice_size == 0x60000 &&
                    surface.mips[0].offset + surface.mips[0].size * tiled_layers == 0x1a0000,
                "fixture lost the gap between an array mip span and its full backing");
        auto tiled_array = MakeLinearDesc(
            base + tiled_array_offset, surface.total_size, vk::Format::eR32Uint,
            Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
            {256, 256, 1}, tiled_layers, sizeof(uint32_t), 1);
        tiled_array.info.tile_mode = Prospero::TileMode::kStandard64KB;
        tiled_array.info.resources.levels = tiled_levels;
        tiled_array.view_info.level_count = tiled_levels;
        for (uint32_t level = 0; level < tiled_levels; ++level) {
          const auto &mip = surface.mips[level];
          tiled_array.info.mip_layout[level] = {
              mip.offset, mip.size * tiled_layers, mip.padded_width, mip.padded_height};
        }
        tiled_array.info.UpdateSize();
        std::memset(memory + tiled_array_offset, 0, surface.total_size);
        const auto tiled_array_id = texture_cache.FindImage(tiled_array);
        (void)texture_cache.FindTexture(tiled_array_id, tiled_array);
        constexpr uint32_t all_mips = 0x12345678u, last_mip = 0x89abcdefu;
        vk::ClearValue clear{};
        clear.color.uint32[0] = all_mips;
        TextureCacheTestAccess::ClearImage(
            texture_cache, scheduler.Current(), tiled_array_id,
            {vk::ImageAspectFlagBits::eColor, 0, tiled_levels, 0, tiled_layers}, clear);
        clear.color.uint32[0] = last_mip;
        TextureCacheTestAccess::ClearImage(
            texture_cache, scheduler.Current(), tiled_array_id,
            {vk::ImageAspectFlagBits::eColor, tiled_levels - 1, 1, tiled_layers - 1, 1}, clear);
        Libs::Graphics::Buffer full_array(
            m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
            tiled_array.info.data.address, AllFlags, surface.total_size);
        Libs::Graphics::Buffer partial_array(
            m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
            tiled_array.info.data.address, AllFlags, 0x1a0000);
        Require(name, "tiled array alias containment",
                BufferCacheTestAccess::SynchronizeBufferFromImage(
                    resources.GetBufferCache(), full_array,
                    tiled_array.info.data.address, surface.total_size) &&
                    !BufferCacheTestAccess::SynchronizeBufferFromImage(
                        resources.GetBufferCache(), partial_array,
                        tiled_array.info.data.address, 0x1a0000),
                "full array synchronization failed or accepted an incomplete slice chain");
        auto readback = CreateHostBuffer(name, 2 * sizeof(uint32_t),
                                        vk::BufferUsageFlagBits::eTransferDst, {0, 0});
        TransferReadBarrier(full_array.Handle(), surface.total_size);
        for (uint32_t probe = 0; probe < 2; ++probe) {
          const auto &mip = surface.mips[probe == 0 ? 0 : tiled_levels - 1];
          uint32_t within_block = 0;
          Require(name, "tiled array probe address",
                  TileGetBlockOffset(surface.texture.block, mip.tail_x, mip.tail_y,
                                     0, within_block),
                  "last-layer mip origin has no guest address");
          const vk::BufferCopy copy{
              (tiled_layers - 1) * surface.block_slice_size + mip.offset + within_block,
              probe * sizeof(uint32_t), sizeof(uint32_t)};
          command.Handle().copyBuffer(full_array.Handle(), readback.buffer, 1, &copy);
        }
        HostReadBarrier(readback.buffer, readback.size, vk::PipelineStageFlagBits::eTransfer,
                        vk::AccessFlagBits::eTransferWrite);
        scheduler.Finish();
        Require(name, "tiled array last-layer contents",
                ReadBuffer(name, readback, 2) == std::vector<u32>{all_mips, last_mip},
                "array synchronization lost GPU contents beyond the first mip span");
        DestroyBuffer(&readback);
      }

      // A formatted Buffer read must use the private
      // Exercise the cache-native image-copy path. Use a request larger
      // than the stream shortcut and poison guest backing
      // after upload so stale CPU staging cannot accidentally
      // satisfy the content check.
      constexpr uint64_t mip_prefix_offset = 0x2740000;
      constexpr auto mip_format = Prospero::BufferFormat::k32UInt;
      constexpr auto mip_tile = Prospero::TileMode::kLinear;
      constexpr uint32_t mip_width = 4097;
      constexpr uint32_t mip_height = 1;
      constexpr uint32_t mip_levels = 1;
      const uint32_t mip_pitch =
          TileGetTexturePitch(mip_format, mip_width, mip_tile);
      TileSizeAlign mip_total{};
      std::array<TileSizeOffset, 16> mip_sizes{};
      std::array<TilePaddedSize, 16> mip_padded{};
      TileGetTextureSize(mip_format, mip_width, mip_height, mip_levels,
                         mip_tile, &mip_total, mip_sizes.data(),
                         mip_padded.data());
      const uint64_t mip_prefix_size = mip_sizes[0].offset + mip_sizes[0].size;
      const uint64_t mip_guest_size = mip_total.size + 256;
      Require(name, "formatted mip fixture",
              mip_sizes[0].offset == 0 &&
                  mip_prefix_size > BufferCache::CACHING_PAGESIZE &&
                  mip_prefix_size < mip_guest_size &&
                  mip_guest_size % sizeof(uint32_t) == 0,
              "single-mip fixture did not expose a cache-sized fitting backing "
              "prefix");
      std::vector<uint32_t> mip_native(mip_guest_size / sizeof(uint32_t));
      std::iota(mip_native.begin(), mip_native.end(), 0x61000000u);
      std::memcpy(memory + mip_prefix_offset, mip_native.data(),
                  mip_guest_size);
      auto mip_desc = MakeLinearDesc(
          base + mip_prefix_offset, mip_guest_size, vk::Format::eR32Uint,
          mip_format, Prospero::ImageType::kColor2D, {mip_width, mip_height, 1},
          1, sizeof(uint32_t), 1);
      mip_desc.info.resources.levels = mip_levels;
      mip_desc.info.pitch = mip_pitch;
      mip_desc.view_info.level_count = mip_levels;
      for (uint32_t level = 0; level < mip_levels; level++) {
        mip_desc.info.mip_layout[level] = {
            mip_sizes[level].offset, mip_sizes[level].size,
            mip_padded[level].width, mip_padded[level].height};
      }
      mip_desc.info.UpdateSize();
      const auto mip_image = texture_cache.FindImage(mip_desc);
      (void)texture_cache.FindTexture(mip_image, mip_desc);
      texture_cache.MarkGpuWritten(mip_image);
      std::vector<uint32_t> mip_stale(mip_native.size(), 0xdeadbeefu);
      Libs::LibKernel::Memory::WriteBacking(base + mip_prefix_offset,
                                            mip_stale.data(), mip_guest_size);

      Libs::Graphics::Buffer mip_insufficient(
          m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
          mip_desc.info.data.address, AllFlags, mip_prefix_size - 1);
      Libs::Graphics::Buffer mip_prefix(
          m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
          mip_desc.info.data.address, AllFlags, mip_prefix_size);
      Require(name, "formatted mip containment",
              !BufferCacheTestAccess::SynchronizeBufferFromImage(
                  resources.GetBufferCache(), mip_insufficient,
                  mip_desc.info.data.address, mip_prefix_size - 1) &&
                  BufferCacheTestAccess::SynchronizeBufferFromImage(
                      resources.GetBufferCache(), mip_prefix,
                      mip_desc.info.data.address, mip_prefix_size) &&
                  texture_cache.GetImage(mip_image).IsGpuModified(),
              "image synchronization accepted a partial first "
              "mip, rejected a fitting mip "
              "prefix, or transferred ownership");

      auto &buffer_cache = resources.GetBufferCache();
      std::pair<Libs::Graphics::Buffer *, uint64_t> mip_formatted;
      bool formatted_owner_was_absent = false;
      gpu.SendCommandSync([&] {
        formatted_owner_was_absent = !buffer_cache.IsRegionRegistered(
            mip_desc.info.data.address, mip_desc.info.data.size);
        mip_formatted = buffer_cache.ObtainBuffer(
            mip_desc.info.data.address, mip_desc.info.data.size, false, true);
      });
      Require(name, "formatted cache fixture", formatted_owner_was_absent,
              "formatted image already had a cached Buffer owner");
      Require(name, "formatted Buffer path",
              mip_formatted.first != nullptr &&
                  texture_cache.GetImage(mip_image).IsGpuModified(),
              "formatted read bypassed the cached image-copy "
              "path or transferred ownership");
      auto mip_prefix_readback = CreateHostBuffer(
          name, mip_prefix_size, vk::BufferUsageFlagBits::eTransferDst,
          std::vector<u32>(mip_prefix_size / sizeof(uint32_t), 0));
      auto mip_formatted_readback = CreateHostBuffer(
          name, mip_guest_size, vk::BufferUsageFlagBits::eTransferDst,
          std::vector<u32>(mip_guest_size / sizeof(uint32_t), 0));
      TransferReadBarrier(mip_prefix.Handle(), mip_prefix_size);
      const vk::BufferCopy mip_prefix_copy{0, 0, mip_prefix_size};
      command.Handle().copyBuffer(
          mip_prefix.Handle(), mip_prefix_readback.buffer, 1, &mip_prefix_copy);
      TransferReadBarrier(mip_formatted.first->Handle(), mip_guest_size);
      const vk::BufferCopy mip_formatted_copy{mip_formatted.second, 0,
                                              mip_guest_size};
      command.Handle().copyBuffer(mip_formatted.first->Handle(),
                                  mip_formatted_readback.buffer, 1,
                                  &mip_formatted_copy);
      HostReadBarrier(mip_prefix_readback.buffer, mip_prefix_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      HostReadBarrier(mip_formatted_readback.buffer,
                      mip_formatted_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      const std::array<uint32_t, 2> volume_values{0x10203040u, 0x50607080u};
      TileSizeAlign volume_size{};
      TileGetTextureTotalSize(Prospero::BufferFormat::k32UInt, 1, 1, 2, 1,
                              Prospero::TileMode::kLinear, true, volume_size);
      const auto volume_pitch = TileGetTexturePitch(
          Prospero::BufferFormat::k32UInt, 1, Prospero::TileMode::kLinear);
      const auto volume_slice_words = volume_size.size / 2 / sizeof(uint32_t);
      std::vector<uint32_t> volume_guest(volume_size.size / sizeof(uint32_t));
      volume_guest[0] = volume_values[0];
      volume_guest[volume_slice_words] = volume_values[1];
      std::memcpy(memory + 0x1000, volume_guest.data(), volume_size.size);
      auto array_desc =
          MakeLinearDesc(base + 0x1000, volume_size.size,
                         vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1);
      array_desc.info.pitch = volume_pitch;
      array_desc.info.UpdateSize();
      const auto array_image = texture_cache.FindImage(array_desc);
      (void)texture_cache.FindTexture(array_image, array_desc);
      texture_cache.MarkGpuWritten(array_image);
      auto volume_desc = array_desc;
      volume_desc.info.type = Prospero::ImageType::kColor3D;
      volume_desc.info.extent = {1, 1, 2};
      volume_desc.info.resources.layers = 1;
      volume_desc.info.UpdateSize();
      volume_desc.view_info.type = vk::ImageViewType::e3D;
      volume_desc.view_info.layer_count = 1;
      const auto volume_image = texture_cache.FindImage(volume_desc);
      Require(name, "2D-array/3D overlap copy",
              volume_image && volume_image != array_image &&
                  texture_cache.GetImage(volume_image).backing.image_type ==
                      vk::ImageType::e3D &&
                  texture_cache.GetImage(volume_image).IsGpuModified(),
              "array-to-volume expansion lost slices or GPU "
              "ownership");

      constexpr uint64_t unique_volume_offset = 0x27b0000;
      std::memcpy(memory + unique_volume_offset, volume_guest.data(),
                  volume_size.size);
      auto unique_volume_desc = volume_desc;
      unique_volume_desc.info.data.address = base + unique_volume_offset;
      const auto unique_volume_image =
          texture_cache.FindImage(unique_volume_desc);
      (void)texture_cache.FindTexture(unique_volume_image, unique_volume_desc);
      texture_cache.MarkGpuWritten(unique_volume_image);
      Libs::Graphics::Buffer partial_volume(
          m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
          unique_volume_desc.info.data.address, AllFlags,
          unique_volume_desc.info.data.size - sizeof(uint32_t));
      Libs::Graphics::Buffer full_volume(
          m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
          unique_volume_desc.info.data.address, AllFlags,
          unique_volume_desc.info.data.size);
      Require(name, "formatted volume containment",
              !BufferCacheTestAccess::SynchronizeBufferFromImage(
                  resources.GetBufferCache(), partial_volume,
                  unique_volume_desc.info.data.address,
                  unique_volume_desc.info.data.size - sizeof(uint32_t)) &&
                  BufferCacheTestAccess::SynchronizeBufferFromImage(
                      resources.GetBufferCache(), full_volume,
                      unique_volume_desc.info.data.address,
                      unique_volume_desc.info.data.size) &&
                  texture_cache.GetImage(unique_volume_image).IsGpuModified(),
              "partial 3D synchronization was accepted, "
              "full-volume synchronization was "
              "rejected, or ownership changed");
      auto volume_readback = CreateHostBuffer(
          name, volume_size.size, vk::BufferUsageFlagBits::eTransferDst,
          std::vector<u32>(volume_guest.size(), 0));
      TransferReadBarrier(full_volume.Handle(), volume_size.size);
      const vk::BufferCopy volume_copy{0, 0, volume_size.size};
      command.Handle().copyBuffer(full_volume.Handle(), volume_readback.buffer,
                                  1, &volume_copy);
      HostReadBarrier(volume_readback.buffer, volume_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);

      constexpr uint64_t depth_containment_offset = 0x27c0000;
      constexpr std::array<float, 3> depth_containment_values{0.25f, 0.75f,
                                                              0.5f};
      std::memcpy(memory + depth_containment_offset,
                  depth_containment_values.data(),
                  sizeof(depth_containment_values));
      auto depth_containment = MakeLinearDesc(
          base + depth_containment_offset, sizeof(depth_containment_values),
          vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float,
          Prospero::ImageType::kColor2D, {2, 1, 1}, 1, sizeof(float), 1);
      depth_containment.type = BindingType::DepthTarget;
      depth_containment.info.resources.levels = 2;
      depth_containment.info.mip_layout[0] = {
          0, 2 * sizeof(float), 2, 1, 0, 2 * sizeof(float), 2};
      depth_containment.info.mip_layout[1] = {
          2 * sizeof(float), sizeof(float), 1, 1, 2 * sizeof(float), sizeof(float), 1};
      depth_containment.info.linear_slice_stride = 0;
      depth_containment.view_info.format = vk::Format::eD32Sfloat;
      depth_containment.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
      depth_containment.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      depth_containment.view_info.level_count = 2;
      const auto depth_containment_image =
          texture_cache.FindImage(depth_containment);
      (void)texture_cache.FindDepthTarget(depth_containment_image,
                                          depth_containment);
      texture_cache.MarkGpuWritten(depth_containment_image);
      Libs::Graphics::Buffer partial_depth(
          m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
          depth_containment.info.data.address, AllFlags,
          depth_containment.info.mip_layout[0].size);
      Require(
          name, "formatted depth containment",
          !BufferCacheTestAccess::SynchronizeBufferFromImage(
              resources.GetBufferCache(), partial_depth,
              depth_containment.info.data.address,
              depth_containment.info.mip_layout[0].size) &&
              texture_cache.GetImage(depth_containment_image).IsGpuModified(),
          "partial depth synchronization was accepted or "
          "transferred ownership");

      constexpr uint64_t raw_d16_offset = 0x27d0000;
      constexpr std::array<uint16_t, 2> raw_d16_values{0x2000u, 0xc000u};
      std::memcpy(memory + raw_d16_offset, raw_d16_values.data(),
                  sizeof(raw_d16_values));
      auto raw_d16 = MakeLinearDesc(
          base + raw_d16_offset, sizeof(raw_d16_values), vk::Format::eD16Unorm,
          Prospero::BufferFormat::k16UNorm, Prospero::ImageType::kColor2D,
          {2, 1, 1}, 1, sizeof(uint16_t), 1);
      raw_d16.type = BindingType::DepthTarget;
      raw_d16.view_info.format = vk::Format::eD16Unorm;
      raw_d16.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
      raw_d16.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto raw_d16_image = texture_cache.FindImage(raw_d16);
      (void)texture_cache.FindDepthTarget(raw_d16_image, raw_d16);

      auto raw_d16_uint = raw_d16;
      raw_d16_uint.type = BindingType::Texture;
      raw_d16_uint.info.pixel_format = vk::Format::eR16Uint;
      raw_d16_uint.info.guest_format = Prospero::BufferFormat::k16UInt;
      raw_d16_uint.view_info.format = vk::Format::eR16Uint;
      raw_d16_uint.view_info.aspect = vk::ImageAspectFlagBits::eColor;
      raw_d16_uint.view_info.usage = vk::ImageUsageFlagBits::eSampled;
      const auto raw_d16_uint_image = texture_cache.FindImage(raw_d16_uint);
      Require(
          name, "raw D16 uint texture backing",
          raw_d16_uint_image && raw_d16_uint_image != raw_d16_image &&
              !TextureCacheTestAccess::Contains(texture_cache, raw_d16_image) &&
              texture_cache.GetImage(raw_d16_uint_image).backing.format ==
                  vk::Format::eR16Uint,
          "D16 uint sampling did not recreate a true integer color backing");
      Require(name, "raw D16 uint texture view",
              texture_cache.FindTexture(raw_d16_uint_image, raw_d16_uint) !=
                  nullptr,
              "D16 uint sampling did not create an integer sampled view");

      constexpr uint64_t layered_raw_d16_offset = raw_d16_offset + 0x1000;
      constexpr std::array<uint16_t, 12> layered_raw_d16_values{
          0x0102u, 0x0304u, 0x1112u, 0x1314u, 0x2122u, 0x2324u,
          0x3132u, 0x3334u, 0x4142u, 0x4344u, 0x5152u, 0x5354u};
      std::memcpy(memory + layered_raw_d16_offset,
                  layered_raw_d16_values.data(),
                  sizeof(layered_raw_d16_values));
      auto layered_d16_color = MakeLinearDesc(
          base + layered_raw_d16_offset, sizeof(layered_raw_d16_values),
          vk::Format::eR16Unorm, Prospero::BufferFormat::k16UNorm,
          Prospero::ImageType::kColor2D, {2, 1, 1}, 6, sizeof(uint16_t), 1);
      const auto layered_d16_color_image =
          texture_cache.FindImage(layered_d16_color);
      (void)texture_cache.FindTexture(layered_d16_color_image,
                                      layered_d16_color);
      auto &layered_d16_color_native =
          texture_cache.GetImage(layered_d16_color_image);
      layered_d16_color_native.Transit(vk::ImageLayout::eTransferDstOptimal,
                                       vk::AccessFlagBits2::eTransferWrite, {},
                                       scheduler.Current().Handle());
      vk::ClearColorValue layered_raw_d16_clear_value{};
      layered_raw_d16_clear_value.float32[0] = 1.0f;
      const vk::ImageSubresourceRange layered_raw_d16_clear_range{
          vk::ImageAspectFlagBits::eColor, 0, 1, 0, 6};
      scheduler.Current().Handle().clearColorImage(
          layered_d16_color_native.backing.image,
          vk::ImageLayout::eTransferDstOptimal, &layered_raw_d16_clear_value, 1,
          &layered_raw_d16_clear_range);
      texture_cache.MarkGpuWritten(layered_d16_color_image);

      auto layered_raw_d16_depth = layered_d16_color;
      layered_raw_d16_depth.type = BindingType::DepthTarget;
      layered_raw_d16_depth.info.data.size = sizeof(uint16_t) * 2 * 5;
      layered_raw_d16_depth.info.pixel_format = vk::Format::eD16Unorm;
      layered_raw_d16_depth.info.guest_format =
          Prospero::BufferFormat::k16UNorm;
      layered_raw_d16_depth.info.resources.layers = 5;
      layered_raw_d16_depth.info.mip_layout[0].size =
          layered_raw_d16_depth.info.data.size;
      layered_raw_d16_depth.view_info.format = vk::Format::eD16Unorm;
      layered_raw_d16_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
      layered_raw_d16_depth.view_info.layer_count = 5;
      layered_raw_d16_depth.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto layered_raw_d16_depth_image =
          texture_cache.FindImage(layered_raw_d16_depth);
      const auto &layered_depth_owner =
          texture_cache.GetImage(layered_raw_d16_depth_image);
      Require(name, "layered raw D16 owner layout",
              layered_raw_d16_depth_image != layered_d16_color_image &&
                  !TextureCacheTestAccess::Contains(texture_cache,
                                                    layered_d16_color_image) &&
                  layered_depth_owner.info.resources.layers == 6 &&
                  layered_depth_owner.info.data.size ==
                      sizeof(layered_raw_d16_values) &&
                  layered_depth_owner.info.mip_layout[0].size ==
                      sizeof(layered_raw_d16_values),
              "a partial depth view detached the retained six-layer owner from "
              "its full guest range");
      (void)texture_cache.FindDepthTarget(layered_raw_d16_depth_image,
                                          layered_raw_d16_depth);

      auto reacquired_layered_raw_d16_uint = layered_d16_color;
      reacquired_layered_raw_d16_uint.info.pixel_format = vk::Format::eR16Uint;
      reacquired_layered_raw_d16_uint.info.guest_format =
          Prospero::BufferFormat::k16UInt;
      reacquired_layered_raw_d16_uint.view_info.format = vk::Format::eR16Uint;
      const auto reacquired_layered_raw_d16_uint_image =
          texture_cache.FindImage(reacquired_layered_raw_d16_uint);
      const auto reacquired_layered_raw_d16_uint_view =
          texture_cache.FindTexture(reacquired_layered_raw_d16_uint_image,
                                    reacquired_layered_raw_d16_uint);
      auto layered_raw_d16_readback = CreateHostBuffer(
          name, sizeof(layered_raw_d16_values),
          vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>(6, 0));
      auto &reacquired_layered_raw_d16_native =
          texture_cache.GetImage(reacquired_layered_raw_d16_uint_image);
      reacquired_layered_raw_d16_native.Transit(
          vk::ImageLayout::eTransferSrcOptimal,
          vk::AccessFlagBits2::eTransferRead, {}, scheduler.Current().Handle());
      vk::BufferImageCopy layered_raw_d16_copy{};
      layered_raw_d16_copy.bufferRowLength = 2;
      layered_raw_d16_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor,
                                               0, 0, 6};
      layered_raw_d16_copy.imageExtent = {2, 1, 1};
      scheduler.Current().Handle().copyImageToBuffer(
          reacquired_layered_raw_d16_native.backing.image,
          vk::ImageLayout::eTransferSrcOptimal, layered_raw_d16_readback.buffer,
          1, &layered_raw_d16_copy);
      HostReadBarrier(layered_raw_d16_readback.buffer,
                      layered_raw_d16_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      scheduler.Finish();
      const auto layered_raw_d16_words =
          ReadBuffer(name, layered_raw_d16_readback, 6);
      std::array<uint16_t, 12> layered_raw_d16_after{};
      std::memcpy(layered_raw_d16_after.data(), layered_raw_d16_words.data(),
                  sizeof(layered_raw_d16_after));
      std::array<uint16_t, 12> layered_raw_d16_expected{};
      layered_raw_d16_expected.fill(UINT16_MAX);
      Require(
          name, "layered raw D16 round trip",
          reacquired_layered_raw_d16_uint_image &&
              reacquired_layered_raw_d16_uint_image !=
                  layered_raw_d16_depth_image &&
              !TextureCacheTestAccess::Contains(texture_cache,
                                                layered_raw_d16_depth_image) &&
              reacquired_layered_raw_d16_native.backing.format ==
                  vk::Format::eR16Uint &&
              reacquired_layered_raw_d16_uint_view != nullptr &&
              layered_raw_d16_after == layered_raw_d16_expected,
          "the partial depth view lost a layer or failed to restore raw UINT "
          "sampling");
      DestroyBuffer(&layered_raw_d16_readback);

      // RE2 converts a prefix of a 32-layer HTile depth array to R32F storage.
      constexpr uint32_t htile_layers = 32;
      constexpr uint64_t htile_slice_size = 0x10000;
      auto htile_depth = MakeLinearDesc(
          base + 0x2800000, htile_layers * htile_slice_size,
          vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float,
          Prospero::ImageType::kColor2D, {128, 128, 1}, htile_layers, 4, 1);
      htile_depth.type = BindingType::DepthTarget;
      htile_depth.info.tile_mode = Prospero::TileMode::kDepth;
      htile_depth.info.UpdateSize();
      htile_depth.info.metadata.kind = ImageMetadataKind::Htile;
      htile_depth.info.metadata.range = {base + 0x2a00000, htile_layers * 0x8000};
      htile_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
      htile_depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto htile_depth_image = texture_cache.FindImage(htile_depth);
      vk::ClearValue htile_clear{};
      htile_clear.depthStencil.depth = 0.25f;
      TextureCacheTestAccess::ClearImage(
          texture_cache, scheduler.Current(), htile_depth_image,
          {vk::ImageAspectFlagBits::eDepth, 0, 1, 0, htile_layers}, htile_clear);

      auto htile_storage = htile_depth;
      htile_storage.type = BindingType::Storage;
      htile_storage.info.pixel_format = vk::Format::eR32Sfloat;
      htile_storage.info.metadata = {};
      htile_storage.view_info.format = vk::Format::eR32Sfloat;
      htile_storage.view_info.aspect = vk::ImageAspectFlagBits::eColor;
      htile_storage.view_info.usage = vk::ImageUsageFlagBits::eStorage;
      ImageId htile_storage_image{};
      for (uint32_t layers : {1u, 2u}) {
        htile_storage.info.data.size = layers * htile_slice_size;
        htile_storage.info.resources.layers = layers;
        htile_storage.info.mip_layout[0].size = htile_storage.info.data.size;
        htile_storage.view_info.layer_count = layers;
        const auto id = texture_cache.FindImage(htile_storage);
        const auto &owner = texture_cache.GetImage(id);
        Require(name, "HTile depth storage prefix layout",
                id != htile_depth_image &&
                    (!htile_storage_image || id == htile_storage_image) &&
                    owner.info.data == htile_depth.info.data &&
                    owner.info.resources == htile_depth.info.resources &&
                    owner.info.mip_layout == htile_depth.info.mip_layout &&
                    owner.backing.layers == htile_layers &&
                    !owner.info.HasMetadata(),
                "partial storage conversion separated array layers from their guest layout");
        Require(name, "HTile depth storage prefix view",
                texture_cache.FindTexture(id, htile_storage) != nullptr,
                "converted depth prefix has no storage view");
        htile_storage_image = id;
      }
      auto htile_readback = CreateHostBuffer(
          name, htile_layers * sizeof(float), vk::BufferUsageFlagBits::eTransferDst,
          std::vector<u32>(htile_layers));
      vk::BufferImageCopy htile_copy{};
      htile_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, htile_layers};
      htile_copy.imageExtent = {1, 1, 1};
      texture_cache.GetImage(htile_storage_image).Download(
          std::span<const vk::BufferImageCopy>(&htile_copy, 1), htile_readback.buffer,
          0, htile_readback.size);
      HostReadBarrier(htile_readback.buffer, htile_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      scheduler.Finish();
      Require(name, "HTile depth storage array contents",
              ReadBuffer(name, htile_readback, htile_layers) ==
                  std::vector<u32>(htile_layers, std::bit_cast<u32>(0.25f)),
              "partial storage conversion lost native depth data outside its view");
      DestroyBuffer(&htile_readback);

      auto layered_stencil_desc = MakeLinearDesc(
          base + 0x27d4000, 6 * 4 * sizeof(float),
          vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k32Float,
          Prospero::ImageType::kColor2D, {4, 1, 1}, 6, 4, 1);
      layered_stencil_desc.type = BindingType::DepthTarget;
      // Four live stencil bytes per slice, with allocation padding.
      constexpr uint32_t stencil_slice_size = 16;
      layered_stencil_desc.info.stencil = {base + 0x27d6000, 6 * stencil_slice_size};
      layered_stencil_desc.view_info.aspect =
          vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
      layered_stencil_desc.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto layered_stencil_id = texture_cache.FindImage(layered_stencil_desc);
      auto &layered_stencil_native = texture_cache.GetImage(layered_stencil_id);
      layered_stencil_native.Transit(vk::ImageLayout::eTransferDstOptimal,
          vk::AccessFlagBits2::eTransferWrite, {}, scheduler.Current().Handle());
      const vk::ClearDepthStencilValue layered_stencil_clear{0.75f, 0x91};
      const vk::ImageSubresourceRange layered_stencil_range{
          layered_stencil_desc.view_info.aspect, 0, 1, 0, 6};
      scheduler.Current().Handle().clearDepthStencilImage(
          layered_stencil_native.backing.image,
          vk::ImageLayout::eTransferDstOptimal, &layered_stencil_clear, 1,
          &layered_stencil_range);
      texture_cache.MarkGpuWritten(layered_stencil_id);
      std::array<uint32_t, 6> stencil_expected;
      stencil_expected.fill(0x91919191u);
      constexpr std::array<std::array<uint32_t, 3>, 4> stencil_cases{{
          {0, 1, 0x35353535u}, {0, 3, 0x57575757u},
          {2, 1, 0x79797979u}, {0, 1, 0xa5a5a5a5u}}};
      for (uint32_t pass = 0; pass < stencil_cases.size(); pass++) {
        const auto [first, layers, stencil_word] = stencil_cases[pass];
        auto partial_stencil_desc = layered_stencil_desc;
        partial_stencil_desc.info.data.address += first * 4 * sizeof(float);
        partial_stencil_desc.info.data.size = layers * 4 * sizeof(float);
        partial_stencil_desc.info.stencil.address += first * stencil_slice_size;
        partial_stencil_desc.info.stencil.size = layers * stencil_slice_size;
        partial_stencil_desc.info.resources.layers = layers;
        partial_stencil_desc.info.mip_layout[0].size =
            partial_stencil_desc.info.data.size;
        partial_stencil_desc.view_info.type = layers == 1
            ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
        partial_stencil_desc.view_info.layer_count = layers;
        const auto stencil = partial_stencil_desc.info.stencil;
        if (pass != 3) {
          (void)resources.GetBufferCache().ObtainBuffer(stencil.address, stencil.size, true);
          resources.GetBufferCache().FillBuffer(
              stencil.address, stencil.size, stencil_word, false);
          const auto partial_stencil_id = texture_cache.FindImage(partial_stencil_desc);
          Require(name, "partial stencil retains depth owner",
                  partial_stencil_id == layered_stencil_id &&
                      texture_cache.GetImage(partial_stencil_id).info.resources.layers == 6 &&
                      partial_stencil_desc.view_info.base_layer == first &&
                      texture_cache.FindDepthTarget(partial_stencil_id,
                                                     partial_stencil_desc) != nullptr,
                  "a partial stencil view replaced its six-layer depth owner");
        } else {
          // The old prefix association must survive the owner's shifted binding.
          Require(name, "partial stencil buffer clear",
                  texture_cache.ClearImageFromBuffer(scheduler.Current(),
                      stencil.address, stencil.size, stencil_word),
                  "the existing partial stencil association could not be cleared");
          const auto rebound = texture_cache.FindImage(partial_stencil_desc);
          Require(name, "rebind cleared stencil association",
                  rebound == layered_stencil_id &&
                      texture_cache.FindDepthTarget(rebound, partial_stencil_desc) != nullptr,
                  "the cleared stencil association lost its depth owner");
        }
        std::fill_n(stencil_expected.begin() + first, layers, stencil_word);
        auto readback = CreateHostBuffer(name, 30 * sizeof(uint32_t),
            vk::BufferUsageFlagBits::eTransferDst, {});
        std::vector<vk::BufferImageCopy> copies(2);
        copies[0].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0, 6};
        copies[1].bufferOffset = 24 * sizeof(float);
        copies[1].imageSubresource = {vk::ImageAspectFlagBits::eStencil, 0, 0, 6};
        for (auto &copy : copies) copy.imageExtent = {4, 1, 1};
        texture_cache.GetImage(layered_stencil_id).Download(
            copies, readback.buffer, 0, readback.size);
        HostReadBarrier(readback.buffer, readback.size,
                        vk::PipelineStageFlagBits::eTransfer,
                        vk::AccessFlagBits::eTransferWrite);
        scheduler.Finish();
        const auto words = ReadBuffer(name, readback, 30);
        for (uint32_t layer = 0; layer < 6; layer++) {
          for (uint32_t pixel = 0; pixel < 4; pixel++) {
            Require(name, "partial stencil preserves depth",
                    words[layer * 4 + pixel] == std::bit_cast<uint32_t>(0.75f),
                    "stencil upload changed the retained depth plane");
          }
          Require(name, "partial stencil preserves other layers",
                  words[24 + layer] == stencil_expected[layer],
                  "stencil upload used the retained depth allocation's layer count");
        }
        DestroyBuffer(&readback);
      }

      auto native_array_info = array_desc.info;
      native_array_info.data = {};
      auto native_volume_info = volume_desc.info;
      native_volume_info.data = {};
      Libs::Graphics::Image native_array(m_runtime_context, scheduler,
                                         native_array_info);
      Libs::Graphics::Image native_volume(m_runtime_context, scheduler,
                                          native_volume_info);
      native_volume.CopyImage(native_array);
      native_array.CopyImage(native_volume);
      Require(name, "Image-owned 2D-array/3D copy state",
              native_volume.backing.state.layout ==
                      vk::ImageLayout::eTransferSrcOptimal &&
                  native_volume.backing.state.access_mask ==
                      vk::AccessFlagBits2::eTransferRead &&
                  native_array.backing.state.layout ==
                      vk::ImageLayout::eGeneral &&
                  native_array.backing.state.access_mask ==
                      (vk::AccessFlagBits2::eShaderRead |
                       vk::AccessFlagBits2::eTransferRead),
              "Image::CopyImage did not retain pinned "
              "source/destination states");

      ImageInfo resolve_source_info{};
      resolve_source_info.pixel_format = vk::Format::eR8G8B8A8Unorm;
      resolve_source_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm;
      resolve_source_info.type = Prospero::ImageType::kColor2D;
      resolve_source_info.extent = {4, 4, 1};
      resolve_source_info.resources = {1, 1};
      resolve_source_info.pitch = 4;
      resolve_source_info.bytes_per_block = 4;
      resolve_source_info.samples = 2;
      resolve_source_info.tile_mode = Prospero::TileMode::kRenderTarget;
      ImageInfo resolve_destination_info = resolve_source_info;
      resolve_destination_info.type = Prospero::ImageType::kColor2D;
      resolve_destination_info.samples = 1;
      Libs::Graphics::Image resolve_source(m_runtime_context, scheduler,
                                           resolve_source_info);
      Libs::Graphics::Image resolve_destination(m_runtime_context, scheduler,
                                                resolve_destination_info);
      resolve_destination.Resolve(resolve_source, {}, {});
      Require(
          name, "Image-owned MSAA resolve",
          resolve_source.backing.state.layout ==
                  vk::ImageLayout::eTransferSrcOptimal &&
              resolve_destination.backing.state.layout ==
                  vk::ImageLayout::eTransferDstOptimal,
          "Image::Resolve did not issue the MSAA-to-single-sample transfer");
      Libs::Graphics::Image copy_destination(m_runtime_context, scheduler,
                                             resolve_destination_info);
      copy_destination.Resolve(resolve_destination, {}, {});
      Require(name, "Image-owned single-sample resolve copy",
              resolve_destination.backing.state.layout ==
                      vk::ImageLayout::eTransferSrcOptimal &&
                  copy_destination.backing.state.layout ==
                      vk::ImageLayout::eTransferDstOptimal,
              "Image::Resolve did not retain the single-sample copy path");

      const std::array<uint16_t, 2> multisample_source{0x4000u, 0xc000u};
      std::memcpy(memory + 0x2000, &multisample_source,
                  sizeof(multisample_source));
      auto color_desc = MakeLinearDesc(
          base + 0x2000, sizeof(multisample_source), vk::Format::eR16G16Unorm,
          Prospero::BufferFormat::k16_16UNorm, Prospero::ImageType::kColor2D,
          {1, 1, 1}, 1, 4, 1);
      const auto color_image = texture_cache.FindImage(color_desc);
      (void)texture_cache.FindTexture(color_image, color_desc);
      texture_cache.MarkGpuWritten(color_image);
      const std::array<uint16_t, 2> refreshed_multisample_source{0x2000u,
                                                                 0xe000u};
      Require(name, "unequal-sample source CPU write",
              resources.HandleFault(PageFaultAccess::Write,
                                    color_desc.info.data.address),
              "color-to-MS-depth source did not accept a CPU update");
      std::memcpy(memory + 0x2000, refreshed_multisample_source.data(),
                  sizeof(refreshed_multisample_source));
      constexpr uint64_t ms_stencil_offset = 0x80000;
      constexpr uint64_t ms_stencil_size = 0x10000;
      auto ms_stencil_owner = resources.GetBufferCache().ObtainBuffer(
          base + ms_stencil_offset, ms_stencil_size, false);
      Require(name, "MS stencil buffer allocation",
              ms_stencil_owner.first != nullptr,
              "failed to create an unequal-sample stencil source");
      resources.GetBufferCache().FillBuffer(
          base + ms_stencil_offset, ms_stencil_size, 0x41414141u, false);
      auto ms_depth_desc = color_desc;
      ms_depth_desc.type = BindingType::DepthTarget;
      ms_depth_desc.info.stencil = {base + ms_stencil_offset, ms_stencil_size};
      ms_depth_desc.info.pixel_format = vk::Format::eD24UnormS8Uint;
      ms_depth_desc.info.guest_format = Prospero::BufferFormat::k16UNorm;
      ms_depth_desc.info.bytes_per_block = 2;
      ms_depth_desc.info.samples = 2;
      ms_depth_desc.info.type = Prospero::ImageType::kColor2D;
      ms_depth_desc.view_info.format = vk::Format::eD24UnormS8Uint;
      ms_depth_desc.view_info.aspect =
          vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
      ms_depth_desc.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto ms_depth_image = texture_cache.FindImage(ms_depth_desc);
      constexpr uint64_t ms_htile_address = base + 0x2600000;
      constexpr uint64_t ms_htile_size = 0x10000;
      ms_depth_desc.info.metadata.range = {ms_htile_address, ms_htile_size};
      ms_depth_desc.info.metadata.kind = ImageMetadataKind::Htile;
      const auto ms_depth_with_htile = texture_cache.FindImage(ms_depth_desc);
      const auto ms_depth_view =
          texture_cache.FindDepthTarget(ms_depth_with_htile, ms_depth_desc);
      auto sampled_ms_depth = ms_depth_desc;
      sampled_ms_depth.type = BindingType::Texture;
      sampled_ms_depth.info.metadata = {};
      sampled_ms_depth.view_info.usage = vk::ImageUsageFlagBits::eSampled;
      const auto sampled_ms_depth_image =
          texture_cache.FindImage(sampled_ms_depth);
      const auto ms_stencil_alias = texture_cache.FindImageFromRange(
          base + ms_stencil_offset, ms_stencil_size, false);
      Require(
          name, "unequal-sample depth overlap",
          ms_depth_image && ms_depth_with_htile == ms_depth_image &&
              sampled_ms_depth_image == ms_depth_image &&
              ms_depth_image != color_image && ms_depth_view != nullptr &&
              texture_cache.GetImage(ms_depth_image).backing.samples == 2 &&
              texture_cache.GetImage(ms_depth_image).backing.state.layout ==
                  vk::ImageLayout::eDepthStencilAttachmentOptimal &&
              texture_cache.GetImage(ms_depth_image).IsGpuModified() &&
              texture_cache.GetImage(ms_depth_image).info.htile_clear_mask ==
                  0 &&
              texture_cache.GetImage(ms_depth_image).info.metadata.range ==
                  ms_depth_desc.info.metadata.range &&
              texture_cache.IsMeta(ms_htile_address) &&
              !texture_cache.IsMetaCleared(ms_htile_address, 0) &&
              !resources.GetBufferCache().HasGpuDirtyBytes(
                  base + ms_stencil_offset, ms_stencil_size) &&
              ms_stencil_alias &&
              texture_cache.GetImage(ms_stencil_alias).depth_id == ms_depth_image &&
              texture_cache.GetImage(ms_stencil_alias).backing.image == nullptr,
          "unequal-sample overlap did not run the color-to-MS-depth pass "
          "without manufacturing stencil ownership");
      auto &oversized_ms = texture_cache.GetImage(ms_depth_image);
      const auto ms_data_size = oversized_ms.info.data.size;
      oversized_ms.info.data.size = (32ull << 20) + 4;
      const bool oversized_ms_readback =
          !TextureCacheTestAccess::TryDownload(texture_cache, ms_depth_image);
      oversized_ms.info.data.size = ms_data_size;
      Require(name, "oversized multisample download rejection",
              oversized_ms_readback && oversized_ms.IsGpuModified() &&
                  !oversized_ms.IsBufferModified() &&
                  !resources.GetBufferCache().HasGpuDirtyBytes(
                      ms_depth_desc.info.data.address, ms_data_size),
              "an unsupported multisample source reserved download storage or "
              "published Buffer ownership");

      const std::array<uint16_t, 4> multisample_source_4x{0x0000u, 0x4000u,
                                                          0x8000u, 0xffffu};
      std::memcpy(memory + 0x7000, multisample_source_4x.data(),
                  sizeof(multisample_source_4x));
      auto color_desc_4x =
          MakeLinearDesc(base + 0x7000, sizeof(multisample_source_4x),
                         vk::Format::eR16G16B16A16Unorm,
                         Prospero::BufferFormat::k16_16_16_16UNorm,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1);
      const auto color_image_4x = texture_cache.FindImage(color_desc_4x);
      auto ms_depth_desc_4x = color_desc_4x;
      ms_depth_desc_4x.type = BindingType::DepthTarget;
      ms_depth_desc_4x.info.pixel_format = vk::Format::eD16Unorm;
      ms_depth_desc_4x.info.guest_format = Prospero::BufferFormat::k16UNorm;
      ms_depth_desc_4x.info.bytes_per_block = 2;
      ms_depth_desc_4x.info.samples = 4;
      ms_depth_desc_4x.info.type = Prospero::ImageType::kColor2D;
      ms_depth_desc_4x.view_info.format = vk::Format::eD16Unorm;
      ms_depth_desc_4x.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
      ms_depth_desc_4x.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto ms_depth_image_4x = texture_cache.FindImage(ms_depth_desc_4x);
      Require(name, "four-sample depth overlap",
              ms_depth_image_4x && ms_depth_image_4x != color_image_4x &&
                  texture_cache.GetImage(ms_depth_image_4x).backing.samples ==
                      4,
              "four-sample color packing did not produce a depth image");

      const auto ms_depth_backing =
          texture_cache.GetImage(ms_depth_image).backing.image;
      const auto first_stencil_association = texture_cache.FindImageFromRange(
          base + ms_stencil_offset, ms_stencil_size, false);
      Require(
          name, "stencil association shape",
          first_stencil_association &&
              first_stencil_association != ms_depth_image &&
              texture_cache.GetImage(first_stencil_association)
                      .info.pixel_format == vk::Format::eS8Uint &&
              texture_cache.GetImage(first_stencil_association).backing.image ==
                  nullptr &&
              texture_cache.GetImage(first_stencil_association).depth_id ==
                  ms_depth_image,
          "stencil address did not create a lightweight depth association");
      Require(
          name, "stencil association download rejection",
          !TextureCacheTestAccess::TryDownload(texture_cache,
                                               first_stencil_association),
          "a lightweight stencil association entered image download planning");

      auto exact_ms_depth_alias = ms_depth_desc;
      exact_ms_depth_alias.type = BindingType::Texture;
      exact_ms_depth_alias.info.stencil = {};
      exact_ms_depth_alias.info.pixel_format = vk::Format::eR8G8B8A8Unorm;
      exact_ms_depth_alias.info.guest_format =
          Prospero::BufferFormat::k8_8_8_8UNorm;
      exact_ms_depth_alias.info.bytes_per_block = 4;
      exact_ms_depth_alias.view_info.format = vk::Format::eR8G8B8A8Unorm;
      exact_ms_depth_alias.view_info.aspect = vk::ImageAspectFlagBits::eColor;
      exact_ms_depth_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled;
      const auto exact_ms_depth_image =
          texture_cache.FindImage(exact_ms_depth_alias, true);
      Require(
          name, "exact multisample depth/stencil coexistence",
          exact_ms_depth_image && exact_ms_depth_image != ms_depth_image &&
              TextureCacheTestAccess::Contains(texture_cache, ms_depth_image) &&
              texture_cache.GetImage(ms_depth_image).backing.image ==
                  ms_depth_backing &&
              texture_cache.GetImage(first_stencil_association).depth_id ==
                  ms_depth_image,
          "exact multisample depth/stencil lookup replaced the old image "
          "or stole its stencil association");

      constexpr uint64_t second_stencil_offset = 0x90000;
      auto stencil_storage_desc = MakeLinearDesc(
          base + second_stencil_offset, ms_stencil_size, vk::Format::eR8Uint,
          Prospero::BufferFormat::k8UInt, Prospero::ImageType::kColor2D,
          ms_depth_desc.info.extent, 1, 1, 1);
      stencil_storage_desc.type = BindingType::Storage;
      stencil_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
      const auto stencil_storage_image =
          texture_cache.FindImage(stencil_storage_desc);
      (void)texture_cache.FindTexture(stencil_storage_image,
                                      stencil_storage_desc);
      auto switched_ms_depth = ms_depth_desc;
      switched_ms_depth.info.stencil = {base + second_stencil_offset,
                                        ms_stencil_size};
      const auto switched_ms_depth_id =
          texture_cache.FindImage(switched_ms_depth);
      TextureCacheTestAccess::AssociateStencil(
          texture_cache, switched_ms_depth_id, switched_ms_depth.info.stencil);
      const auto second_stencil_association = texture_cache.FindImageFromRange(
          base + second_stencil_offset, ms_stencil_size, false);
      Require(name, "stencil association switch",
              switched_ms_depth_id == ms_depth_image &&
                  texture_cache.GetImage(switched_ms_depth_id).backing.image ==
                      ms_depth_backing &&
                  texture_cache.FindImageFromRange(base + ms_stencil_offset,
                                                   ms_stencil_size, false) ==
                      first_stencil_association &&
                  texture_cache.GetImage(first_stencil_association).depth_id ==
                      ms_depth_image &&
                  second_stencil_association &&
                  second_stencil_association == stencil_storage_image &&
                  texture_cache.GetImage(second_stencil_association).IsTracked() &&
                  texture_cache.GetImage(second_stencil_association).depth_id ==
                      ms_depth_image,
              "changing stencil address recreated depth or discarded an "
              "existing lightweight association");
      Require(name, "CPU write to tracked stencil association",
              resources.HandleFault(PageFaultAccess::Write,
                                    base + second_stencil_offset) &&
                  !texture_cache.GetImage(second_stencil_association).IsTracked() &&
                  texture_cache.GetImage(second_stencil_association).IsCpuDirty() &&
                  texture_cache.GetImage(second_stencil_association).depth_id ==
                      ms_depth_image,
              "CPU invalidation left a reused stencil image write-protected");
      memory[second_stencil_offset] = 0x5a;

      constexpr uint64_t added_stencil_depth_offset = 0x60000;
      constexpr uint64_t added_stencil_offset = 0x70000;
      constexpr uint64_t added_stencil_size = 0x10000;
      constexpr float added_stencil_depth_value = 0.75f;
      std::memcpy(memory + added_stencil_depth_offset,
                  &added_stencil_depth_value,
                  sizeof(added_stencil_depth_value));
      auto depth_only_source = MakeLinearDesc(
          base + added_stencil_depth_offset, 0x10000, vk::Format::eD32Sfloat,
          Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
          {1, 1, 1}, 1, 4, 1);
      depth_only_source.type = BindingType::DepthTarget;
      depth_only_source.view_info.format = vk::Format::eD32Sfloat;
      depth_only_source.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
      depth_only_source.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto depth_only_source_image =
          texture_cache.FindImage(depth_only_source);

      auto combined_destination = depth_only_source;
      combined_destination.info.stencil = {base + added_stencil_offset,
                                           added_stencil_size};
      combined_destination.info.pixel_format = vk::Format::eD32SfloatS8Uint;
      combined_destination.view_info.format = vk::Format::eD32SfloatS8Uint;
      combined_destination.view_info.aspect =
          vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
      const auto combined_destination_image =
          texture_cache.FindImage(combined_destination);
      TextureCacheTestAccess::AssociateStencil(
          texture_cache, combined_destination_image,
          combined_destination.info.stencil);
      const auto combined_stencil_association =
          texture_cache.FindImageFromRange(base + added_stencil_offset,
                                           added_stencil_size, false);
      Require(
          name, "depth-only to combined association",
          combined_destination_image != depth_only_source_image &&
              combined_stencil_association &&
              texture_cache.GetImage(combined_stencil_association).depth_id ==
                  combined_destination_image &&
              texture_cache.GetImage(combined_stencil_association)
                      .backing.image == nullptr,
          "stencil addition did not recreate depth and register its "
          "lightweight association");

      const uint32_t mirror_value = 0xa1b2c3d4u;
      std::memcpy(memory + 0x5000, &mirror_value, sizeof(mirror_value));
      auto mirror_desc =
          MakeLinearDesc(base + 0x5000, sizeof(mirror_value),
                         vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
      const auto mirror_image = texture_cache.FindImage(mirror_desc);
      (void)texture_cache.FindTexture(mirror_image, mirror_desc);
      texture_cache.MarkGpuWritten(mirror_image);
      constexpr uint32_t mirror_cpu_value = 0x0f1e2d3cu;
      Require(name, "CPU-dirty formatted mirror fault",
              resources.HandleFault(PageFaultAccess::Write,
                                    mirror_desc.info.data.address),
              "GPU image did not accept a CPU write before Buffer mirroring");
      std::memcpy(memory + 0x5000, &mirror_cpu_value, sizeof(mirror_cpu_value));
      auto mirror_binding = resources.GetBufferCache().ObtainBuffer(
          base + 0x5000, sizeof(mirror_value), false, true);
      Require(name, "CPU-dirty formatted mirror source",
              mirror_binding.first != nullptr &&
                  !texture_cache.GetImage(mirror_image).IsBufferModified(),
              "formatted mirror did not expose a readable Buffer source");
      auto mirror_cpu_readback = CreateHostBuffer(
          name, sizeof(mirror_cpu_value), vk::BufferUsageFlagBits::eTransferDst,
          std::vector<u32>{0});
      const vk::BufferCopy mirror_cpu_copy{mirror_binding.second, 0,
                                           sizeof(mirror_cpu_value)};
      command.Handle().copyBuffer(mirror_binding.first->Handle(),
                                  mirror_cpu_readback.buffer, 1,
                                  &mirror_cpu_copy);
      HostReadBarrier(mirror_cpu_readback.buffer, mirror_cpu_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      auto mirror_refresh_desc = mirror_desc;
      const auto mirror_refresh = texture_cache.FindImage(mirror_refresh_desc);
      (void)texture_cache.FindTexture(mirror_refresh, mirror_refresh_desc);
      Require(name, "buffer-to-image ownership",
              mirror_refresh == mirror_image &&
                  !texture_cache.GetImage(mirror_refresh).IsBufferModified(),
              "buffer-backed refresh incorrectly transferred dirty ownership "
              "to the image");
      texture_cache.MarkGpuWritten(mirror_refresh);

      // Unity copies BC mip regions and packed mip tails through integer images.
      // The CPU allocation remains white; later array sampling must see GPU writes.
      for (uint32_t format_index = 0; format_index < 3; ++format_index) {
        constexpr std::array formats{vk::Format::eBc4UnormBlock,
                                     vk::Format::eBc5UnormBlock,
                                     vk::Format::eBc7UnormBlock};
        constexpr std::array guest_formats{Prospero::BufferFormat::kBc4UNorm,
                                           Prospero::BufferFormat::kBc5UNorm,
                                           Prospero::BufferFormat::kBc7UNorm};
        constexpr uint32_t width = 128, levels = 8, layers = 3;
        const uint32_t block_bytes = format_index == 0 ? 8 : 16;
        const uint64_t address = base + 0x1900000 + format_index * 0x40000;
        auto array = MakeLinearDesc(address, 0, formats[format_index],
            guest_formats[format_index], Prospero::ImageType::kColor2D,
            {width, width, 1}, layers, block_bytes, 1);
        array.info.resources.levels = levels;
        array.info.tile_mode = Prospero::TileMode::kStandard4KB;
        array.info.UpdateSize();
        array.view_info.level_count = levels;
        std::memset(memory + address - base, 0xff, array.info.data.size);
        auto value_for = [&](uint32_t layer, uint64_t offset) {
          return 0x7c000000u | (format_index << 16) | (layer << 8) |
                 static_cast<uint32_t>(offset / 4096);
        };
        auto write_raw = [&](uint32_t layer, uint32_t level, uint32_t value) {
          const auto &mip = array.info.mip_layout[level];
          auto raw = MakeLinearDesc(
              address + layer * array.info.tiled_slice_stride + mip.offset, 0,
              block_bytes == 8 ? vk::Format::eR32G32Uint
                               : vk::Format::eR32G32B32A32Uint,
              block_bytes == 8 ? Prospero::BufferFormat::k32_32UInt
                               : Prospero::BufferFormat::k32_32_32_32UInt,
              Prospero::ImageType::kColor2D, {mip.pitch, mip.height, 1},
              1, block_bytes, 1);
          raw.type = BindingType::Storage;
          raw.info.tile_mode = array.info.tile_mode;
          raw.info.UpdateSize();
          raw.view_info.usage = vk::ImageUsageFlagBits::eStorage;
          const auto producer = texture_cache.FindImage(raw);
          (void)texture_cache.FindTexture(producer, raw);
          vk::ClearValue clear{};
          std::fill_n(clear.color.uint32.data(), 4, value);
          TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(),
              producer, {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, clear);
        };
        for (const uint32_t layer : {0u, 2u}) {
          uint64_t previous_offset = UINT64_MAX;
          for (uint32_t level = 0; level < levels; ++level) {
            const auto offset = array.info.mip_layout[level].offset;
            if (offset == previous_offset) continue;
            previous_offset = offset;
            write_raw(layer, level, value_for(layer, offset));
          }
        }
        std::vector<vk::BufferImageCopy> probes;
        std::vector<uint32_t> expected;
        for (uint32_t layer = 0; layer < layers; ++layer) {
          for (uint32_t level = 0; level < levels; ++level) {
            vk::BufferImageCopy copy{};
            copy.bufferOffset = expected.size() * sizeof(uint32_t);
            copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, level, layer, 1};
            copy.imageExtent = {std::min(width >> level, 4u),
                                std::min(width >> level, 4u), 1};
            probes.push_back(copy);
            expected.insert(expected.end(), block_bytes / sizeof(uint32_t),
                layer == 1 ? UINT32_MAX
                           : value_for(layer, array.info.mip_layout[level].offset));
          }
        }
        auto verify_array = [&] {
          const auto consumer = texture_cache.FindImage(array);
          (void)texture_cache.FindTexture(consumer, array);
          auto readback = CreateHostBuffer(name, expected.size() * sizeof(uint32_t),
              vk::BufferUsageFlagBits::eTransferDst, std::vector<uint32_t>(expected.size()));
          texture_cache.GetImage(consumer).Download(probes, readback.buffer, 0, readback.size);
          HostReadBarrier(readback.buffer, readback.size, vk::PipelineStageFlagBits::eTransfer,
                          vk::AccessFlagBits::eTransferWrite);
          scheduler.Finish();
          Require(name, "raw image producers populate BC array",
                  ReadBuffer(name, readback, expected.size()) == expected,
                  "BC array lost GPU mip/tail writes or overwrote an untouched layer");
          DestroyBuffer(&readback);
        };
        verify_array();
        constexpr uint32_t replacement = 0x13579bdfu;
        write_raw(2, array.info.first_tail_level, replacement);
        for (uint32_t level = array.info.first_tail_level; level < levels; ++level) {
          const auto word = (2 * levels + level) * block_bytes / sizeof(uint32_t);
          std::fill_n(expected.begin() + word, block_bytes / sizeof(uint32_t), replacement);
        }
        verify_array();
      }

      // A write through a separate buffer must not replace untouched image
      // texels with stale backing bytes. Exact-base writes still refresh it.
      constexpr uint64_t buffer_alias_offset = 0x1a00000;
      constexpr uint32_t buffer_alias_image_value = 0x13579bdfu;
      constexpr uint32_t buffer_alias_write_value = 0x2468ace0u;
      std::memset(memory + buffer_alias_offset, 0, 4 * sizeof(uint32_t));
      auto buffer_alias_desc = MakeLinearDesc(
          base + buffer_alias_offset, 4 * sizeof(uint32_t), vk::Format::eR32Uint,
          Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
          {4, 1, 1}, 1, 4, 1);
      const auto buffer_alias_image = texture_cache.FindImage(buffer_alias_desc);
      for (const uint32_t write_offset : {0u, 12u}) {
        vk::ClearValue painted{};
        painted.color.uint32 = std::array{buffer_alias_image_value, 0u, 0u, 0u};
        TextureCacheTestAccess::ClearImage(
            texture_cache, scheduler.Current(), buffer_alias_image,
            {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
        const auto write_address = buffer_alias_desc.info.data.address + write_offset;
        auto writable = resources.GetBufferCache().ObtainBuffer(
            write_address, sizeof(uint32_t), true, false);
        Require(name, "image alias buffer allocation", writable.first != nullptr,
                "failed to allocate the image alias write buffer");
        writable.first->Fill(writable.second, sizeof(uint32_t), buffer_alias_write_value);
        texture_cache.InvalidateMemoryFromGPU(write_address, sizeof(uint32_t));
        (void)texture_cache.FindTexture(buffer_alias_image, buffer_alias_desc);
        const bool exact_base = write_offset == 0;
        const uint32_t read_texels = exact_base ? 1 : 3;
        Require(name, exact_base ? "exact-base buffer write refresh"
                                 : "overlapping buffer preserves untouched texels",
                ReadCachedTexel(name, context, buffer_alias_image, {},
                                {read_texels, 1, 1}) ==
                    std::vector<u32>(read_texels, exact_base ? buffer_alias_write_value
                                                           : buffer_alias_image_value),
                exact_base ? "exact-base buffer write did not refresh the native image"
                           : "overlapping buffer write replaced untouched GPU image texels");
      }

      constexpr uint64_t exact_buffer_offset = 0x90000;
      constexpr uint32_t exact_buffer_value = 0x3f234567u;
      std::memcpy(memory + exact_buffer_offset, &exact_buffer_value,
                  sizeof(exact_buffer_value));
      auto exact_buffer_desc =
          MakeLinearDesc(base + exact_buffer_offset, sizeof(exact_buffer_value),
                         vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
      const auto exact_buffer_image =
          texture_cache.FindImage(exact_buffer_desc);
      (void)texture_cache.FindTexture(exact_buffer_image, exact_buffer_desc);
      texture_cache.MarkGpuWritten(exact_buffer_image);
      auto exact_buffer_binding = resources.GetBufferCache().ObtainBuffer(
          exact_buffer_desc.info.data.address, exact_buffer_desc.info.data.size,
          false, true);
      Require(
          name, "exact replacement Buffer synchronization",
          exact_buffer_binding.first != nullptr &&
              !texture_cache.GetImage(exact_buffer_image).IsBufferModified() &&
              texture_cache.GetImage(exact_buffer_image).IsGpuModified(),
          "exact replacement copy transferred cache ownership");
      auto exact_float_desc = exact_buffer_desc;
      exact_float_desc.info.pixel_format = vk::Format::eR32Sfloat;
      exact_float_desc.info.guest_format = Prospero::BufferFormat::k32Float;
      exact_float_desc.view_info.format = vk::Format::eR32Sfloat;
      const auto exact_float_image =
          texture_cache.FindImage(exact_float_desc, true);
      (void)texture_cache.FindTexture(exact_float_image, exact_float_desc);
      Require(
          name, "Buffer-superseded exact coexistence",
          exact_float_image != exact_buffer_image &&
              TextureCacheTestAccess::Contains(texture_cache,
                                               exact_buffer_image) &&
              texture_cache.GetImage(exact_buffer_image).IsGpuModified() &&
              !texture_cache.GetImage(exact_float_image).IsBufferModified() &&
              !texture_cache.GetImage(exact_float_image).IsGpuModified(),
          "exact-format lookup retired its old record or transferred Buffer "
          "ownership");
      auto exact_buffer_readback = CreateHostBuffer(
          name, sizeof(exact_buffer_value),
          vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>{0});
      auto &exact_float_native = texture_cache.GetImage(exact_float_image);
      exact_float_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
                                 vk::AccessFlagBits2::eTransferRead, {},
                                 command.Handle());
      vk::BufferImageCopy exact_float_copy{};
      exact_float_copy.imageSubresource.aspectMask =
          vk::ImageAspectFlagBits::eColor;
      exact_float_copy.imageSubresource.layerCount = 1;
      exact_float_copy.imageExtent = {1, 1, 1};
      command.Handle().copyImageToBuffer(exact_float_native.backing.image,
                                         vk::ImageLayout::eTransferSrcOptimal,
                                         exact_buffer_readback.buffer, 1,
                                         &exact_float_copy);
      HostReadBarrier(exact_buffer_readback.buffer, exact_buffer_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);

      const uint32_t clear_source = 0x01010101u;
      std::memcpy(memory + 0x6000, &clear_source, sizeof(clear_source));
      auto clear_desc = MakeLinearDesc(
          base + 0x6000, sizeof(clear_source), vk::Format::eR8G8B8A8Unorm,
          Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
          {1, 1, 1}, 1, 4, 1);
      const auto clear_image = texture_cache.FindImage(clear_desc);
      auto buffer_write = resources.GetBufferCache().ObtainBuffer(
          base + 0x6000, sizeof(clear_source), true, true);
      texture_cache.InvalidateMemoryFromGPU(base + 0x6000,
                                            sizeof(clear_source));
      Require(name, "buffer-write ownership",
              buffer_write.first != nullptr &&
                  texture_cache.GetImage(clear_image).IsBufferModified(),
              "formatted buffer write did not supersede the image");
      Require(name, "buffer-owned full image clear",
              texture_cache.ClearImageFromBuffer(
                  command, base + 0x6000, sizeof(clear_source), 0xaabbccddu) &&
                  !texture_cache.GetImage(clear_image).IsBufferModified() &&
                  texture_cache.GetImage(clear_image).IsGpuModified(),
              "full image clear retained dual buffer/image GPU ownership");
      texture_cache.MarkGpuWritten(clear_image);
      auto clear_read_desc = clear_desc;
      Require(name, "post-clear image reuse",
              texture_cache.FindImage(clear_read_desc) == clear_image,
              "cleared image could not be read or written after ownership "
              "transfer");

      auto stencil_clear_desc = MakeLinearDesc(
          base + 0x27e0000, 0x1000, vk::Format::eD32SfloatS8Uint,
          Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
          {13, 11, 1}, 2, 4, 1);
      stencil_clear_desc.info.stencil = {base + 0x27e2000, 0x1000};
      stencil_clear_desc.info.resources.levels = 2;
      stencil_clear_desc.info.mip_layout[0] = {0, 0x800, 13, 11, 0, 0x400, 13};
      stencil_clear_desc.info.mip_layout[1] = {0x800, 0x800, 6, 5, 0x800, 0x400, 6};
      stencil_clear_desc.info.linear_slice_stride = 0x400;
      stencil_clear_desc.view_info.aspect =
          vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
      const auto stencil_clear_id = texture_cache.FindImage(stencil_clear_desc);
      auto &stencil_clear_image = texture_cache.GetImage(stencil_clear_id);
      stencil_clear_image.Transit(vk::ImageLayout::eTransferDstOptimal,
          vk::AccessFlagBits2::eTransferWrite, {}, command.Handle());
      const vk::ClearDepthStencilValue initial_depth_stencil{0.75f, 0x91};
      const vk::ImageSubresourceRange all_depth_stencil{
          stencil_clear_desc.view_info.aspect, 0, 2, 0, 2};
      command.Handle().clearDepthStencilImage(stencil_clear_image.backing.image,
          vk::ImageLayout::eTransferDstOptimal, &initial_depth_stencil, 1,
          &all_depth_stencil);
      texture_cache.MarkGpuWritten(stencil_clear_id);
      vk::ClearValue stencil_value{};
      stencil_value.depthStencil = vk::ClearDepthStencilValue{0.f, 0x35};
      TextureCacheTestAccess::ClearImage(texture_cache, command, stencil_clear_id,
          {vk::ImageAspectFlagBits::eStencil, 1, 1, 1, 1}, stencil_value);

      // Read every mip/layer of both aspects to prove that the stencil clear is local.
      auto stencil_readback = CreateHostBuffer(name, 0x4000,
          vk::BufferUsageFlagBits::eTransferDst, {});
      std::vector<vk::BufferImageCopy> stencil_copies;
      for (uint32_t mip = 0; mip < 2; mip++) {
        for (uint32_t layer = 0; layer < 2; layer++) {
          for (const auto aspect : {vk::ImageAspectFlagBits::eDepth,
                                    vk::ImageAspectFlagBits::eStencil}) {
            vk::BufferImageCopy copy{};
            copy.bufferOffset = stencil_copies.size() * 0x800;
            copy.imageSubresource = {aspect, mip, layer, 1};
            copy.imageExtent = {13u >> mip, 11u >> mip, 1};
            stencil_copies.push_back(copy);
          }
        }
      }
      stencil_clear_image.Download(stencil_copies, stencil_readback.buffer, 0,
                                   stencil_readback.size);
      HostReadBarrier(stencil_readback.buffer, stencil_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      scheduler.Finish();
      const auto stencil_words = ReadBuffer(name, stencil_readback, 0x1000);
      const auto *stencil_bytes = reinterpret_cast<const uint8_t *>(stencil_words.data());
      for (const auto &copy : stencil_copies) {
        const auto &sub = copy.imageSubresource;
        const auto count = copy.imageExtent.width * copy.imageExtent.height;
        const auto *bytes = stencil_bytes + copy.bufferOffset;
        const bool depth = sub.aspectMask == vk::ImageAspectFlagBits::eDepth;
        uint32_t expected = 0x91u;
        if (depth) {
          expected = std::bit_cast<uint32_t>(0.75f);
        } else if (sub.mipLevel == 1 && sub.baseArrayLayer == 1) {
          expected = 0x35u;
        }
        for (uint32_t pixel = 0; pixel < count; pixel++) {
          uint32_t actual = bytes[pixel];
          if (depth) std::memcpy(&actual, bytes + pixel * 4, 4);
          Require(name, "stencil clear preserves depth and neighbors", actual == expected,
                  "selected stencil clear changed depth, another mip, or another layer");
        }
      }
      DestroyBuffer(&stencil_readback);

      constexpr uint64_t partial_image_offset = 0xa000;
      constexpr uint64_t partial_buffer_offset = 0xa010;
      constexpr uint64_t partial_clean_offset = 0xa020;
      constexpr uint32_t partial_image_value = 0x31415926u;
      constexpr uint32_t partial_buffer_value = 0x27182818u;
      constexpr uint32_t partial_clean_value = 0xabcdef01u;
      std::memcpy(memory + partial_clean_offset, &partial_clean_value,
                  sizeof(partial_clean_value));
      auto partial_write = resources.GetBufferCache().ObtainBuffer(
          base + partial_image_offset, sizeof(partial_image_value), true);
      Require(name, "partial-page buffer allocation",
              partial_write.first != nullptr,
              "partial-page GPU write did not create a native buffer");
      resources.GetBufferCache().FillBuffer(base + partial_image_offset,
                                            sizeof(partial_image_value),
                                            partial_image_value, false);
      resources.GetBufferCache().FillBuffer(base + partial_buffer_offset,
                                            sizeof(partial_buffer_value),
                                            partial_buffer_value, false);
      auto partial_desc = MakeLinearDesc(
          base + partial_image_offset, sizeof(partial_image_value),
          vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
          Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
      const auto partial_image = texture_cache.FindImage(partial_desc);
      Require(name, "partial-page image upload",
              !texture_cache.GetImage(partial_image).IsGpuModified(),
              "image upload incorrectly consumed buffer dirty ownership");
      auto partial_image_mirror = resources.GetBufferCache().ObtainBuffer(
          base + partial_image_offset, sizeof(partial_image_value), false,
          true);
      Require(name, "partial-page image mirror",
              partial_image_mirror.first != nullptr &&
                  !texture_cache.GetImage(partial_image).IsGpuModified(),
              "same-page image upload lost its buffer source");
      const auto partial_clean_source =
          resources.GetBufferCache().ObtainBufferForImage(
              base + partial_clean_offset, sizeof(partial_clean_value));
      Require(name, "partial-page clean source",
              partial_clean_source.first != nullptr,
              "clean same-page bytes inherited unrelated buffer ownership");
      Require(name, "partial-page remaining fault",
              resources.HandleFault(PageFaultAccess::Read,
                                    base + partial_buffer_offset),
              "same-page buffer ownership was lost after image transfer");
      uint32_t partial_buffer_backing = 0;
      std::memcpy(&partial_buffer_backing, memory + partial_buffer_offset,
                  sizeof(partial_buffer_backing));
      uint32_t partial_image_backing = 0;
      std::memcpy(&partial_image_backing, memory + partial_image_offset,
                  sizeof(partial_image_backing));
      Require(name, "partial-page readback values",
              partial_image_backing == partial_image_value &&
                  partial_buffer_backing == partial_buffer_value,
              "same-page image/buffer readback published incorrect bytes");
      constexpr uint32_t partial_cpu_refresh_value = 0x13579bdfu;
      Require(name, "partial-page CPU write fault",
              resources.HandleFault(PageFaultAccess::Write,
                                    base + partial_clean_offset),
              "same-page CPU write did not invalidate cached ownership");
      std::memcpy(memory + partial_clean_offset, &partial_cpu_refresh_value,
                  sizeof(partial_cpu_refresh_value));
      constexpr std::array<uint32_t, 2> partial_neighbors{0x2468ace0u,
                                                        0xfedcba98u};
      std::memcpy(memory + partial_clean_offset - sizeof(uint32_t),
                  &partial_neighbors[0], sizeof(uint32_t));
      std::memcpy(memory + partial_clean_offset + sizeof(uint32_t),
                  &partial_neighbors[1], sizeof(uint32_t));
      std::vector<u32> partial_cpu_refresh_expected(TRACKER_PAGE_SIZE / sizeof(u32));
      std::memcpy(partial_cpu_refresh_expected.data(), memory + partial_image_offset,
                  TRACKER_PAGE_SIZE);
      const auto partial_cpu_refresh_source =
          resources.GetBufferCache().ObtainBufferForImage(
              base + partial_clean_offset, sizeof(partial_cpu_refresh_value));
      Require(name, "partial-page CPU refresh source",
              partial_cpu_refresh_source.first != nullptr,
              "CPU-dirty same-page bytes did not resolve through the cached "
              "buffer");
      auto partial_cpu_refresh_readback =
          CreateHostBuffer(name, TRACKER_PAGE_SIZE,
                           vk::BufferUsageFlagBits::eTransferDst, {0});
      const vk::BufferCopy partial_cpu_refresh_copy{
          partial_cpu_refresh_source.first->Offset(base + partial_image_offset), 0,
          TRACKER_PAGE_SIZE};
      command.Handle().copyBuffer(partial_cpu_refresh_source.first->Handle(),
                                  partial_cpu_refresh_readback.buffer, 1,
                                  &partial_cpu_refresh_copy);
      vk::BufferMemoryBarrier partial_cpu_refresh_barrier{};
      partial_cpu_refresh_barrier.sType =
          vk::StructureType::eBufferMemoryBarrier;
      partial_cpu_refresh_barrier.srcAccessMask =
          vk::AccessFlagBits::eTransferWrite;
      partial_cpu_refresh_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
      partial_cpu_refresh_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      partial_cpu_refresh_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      partial_cpu_refresh_barrier.buffer = partial_cpu_refresh_readback.buffer;
      partial_cpu_refresh_barrier.size = partial_cpu_refresh_readback.size;
      command.Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
                                       vk::PipelineStageFlagBits::eHost, {}, 0,
                                       nullptr, 1, &partial_cpu_refresh_barrier,
                                       0, nullptr);

      const uint32_t fault_a = 0x01020304u;
      const uint32_t fault_b = 0x11121314u;
      std::memcpy(memory + 0x8000, &fault_a, sizeof(fault_a));
      std::memcpy(memory + 0x8010, &fault_b, sizeof(fault_b));
      auto fault_a_desc =
          MakeLinearDesc(base + 0x8000, sizeof(fault_a), vk::Format::eR32Uint,
                         Prospero::BufferFormat::k32UInt,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
      auto fault_b_desc = fault_a_desc;
      fault_b_desc.info.data.address = base + 0x8010;
      const auto fault_a_image = texture_cache.FindImage(fault_a_desc);
      const auto fault_b_image = texture_cache.FindImage(fault_b_desc);
      (void)texture_cache.FindTexture(fault_a_image, fault_a_desc);
      (void)texture_cache.FindTexture(fault_b_image, fault_b_desc);
      texture_cache.MarkGpuWritten(fault_a_image);
      texture_cache.MarkGpuWritten(fault_b_image);
      Require(name, "per-image watcher installation",
              texture_cache.GetImage(fault_a_image).IsTracked() &&
                  texture_cache.GetImage(fault_b_image).IsTracked(),
              "same-page images did not install independent write watchers");
      constexpr uint64_t padding_fault_offset = 0x8080;
      uint32_t write_only_read_a = 0;
      uint32_t write_only_read_b = 0;
      std::memcpy(&write_only_read_a, memory + 0x8000,
                  sizeof(write_only_read_a));
      std::memcpy(&write_only_read_b, memory + 0x8010,
                  sizeof(write_only_read_b));
      Require(name, "write-only image read policy",
              write_only_read_a == fault_a && write_only_read_b == fault_b &&
                  texture_cache.GetImage(fault_a_image).IsGpuModified() &&
                  texture_cache.GetImage(fault_b_image).IsGpuModified(),
              "TextureCache claimed a CPU read fault instead of preserving "
              "GPU image ownership");
      Require(name, "same-page image write invalidation",
              resources.HandleFault(PageFaultAccess::Write,
                                    base + padding_fault_offset) &&
                  texture_cache.GetImage(fault_a_image).IsGpuModified() &&
                  texture_cache.GetImage(fault_b_image).IsGpuModified() &&
                  !texture_cache.GetImage(fault_a_image).IsTracked() &&
                  !texture_cache.GetImage(fault_b_image).IsTracked() &&
                  texture_cache.GetImage(fault_a_image).IsMaybeCpuDirty() &&
                  texture_cache.GetImage(fault_b_image).IsMaybeCpuDirty() &&
                  !texture_cache.GetImage(fault_a_image).IsDefinitelyCpuDirty() &&
                  !texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty(),
              "a byte-disjoint CPU write discarded authoritative images");
      const auto retracked_a = texture_cache.FindImage(fault_a_desc);
      const auto retracked_b = texture_cache.FindImage(fault_b_desc);
      (void)texture_cache.FindTexture(retracked_a, fault_a_desc);
      (void)texture_cache.FindTexture(retracked_b, fault_b_desc);
      auto fault_a_mirror = resources.GetBufferCache().ObtainBuffer(
          base + 0x8000, sizeof(fault_a), false, true);
      Require(name, "same-page image re-track",
              retracked_a == fault_a_image && retracked_b == fault_b_image &&
                  texture_cache.GetImage(fault_a_image).IsTracked() &&
                  texture_cache.GetImage(fault_b_image).IsTracked() &&
                  !texture_cache.GetImage(fault_a_image).IsCpuDirty() &&
                  !texture_cache.GetImage(fault_b_image).IsCpuDirty() &&
                  fault_a_mirror.first != nullptr &&
                  texture_cache.GetImage(fault_a_image).IsTracked() &&
                  texture_cache.GetImage(fault_b_image).IsTracked() &&
                  texture_cache.GetImage(fault_a_image).IsGpuModified() &&
                  texture_cache.GetImage(fault_b_image).IsGpuModified(),
              "copying one same-page image lost an authoritative owner");
      Require(name, "same-page survivor write fault",
              resources.HandleFault(PageFaultAccess::Write, base + 0x8010) &&
                  texture_cache.GetImage(fault_b_image).IsGpuModified() &&
                  texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty(),
              "the surviving image was not protected after its alias retired");
      constexpr uint32_t fault_b_cpu = 0xa5a6a7a8u;
      std::memcpy(memory + 0x8010, &fault_b_cpu, sizeof(fault_b_cpu));
      const auto refreshed_b = texture_cache.FindImage(fault_b_desc);
      (void)texture_cache.FindTexture(refreshed_b, fault_b_desc);
      Require(
          name, "same-page survivor refresh",
          refreshed_b == fault_b_image &&
              !texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty() &&
              texture_cache.GetImage(fault_b_image).IsGpuModified(),
          "the surviving image could not reconcile its CPU write");

      // Further writes on the unprotected page must retain the first baseline.
      Require(name, "same-page repeated-write fault",
              resources.HandleFault(PageFaultAccess::Write,
                                    base + padding_fault_offset),
              "the padding write fault was not handled");
      constexpr uint32_t fault_a_cpu = 0x81828384u;
      std::memcpy(memory + 0x8000, &fault_a_cpu, sizeof(fault_a_cpu));
      texture_cache.InvalidateMemory(base + padding_fault_offset, 1);
      Require(name, "repeated same-page invalidation",
              texture_cache.GetImage(fault_a_image).IsMaybeCpuDirty() &&
                  !texture_cache.GetImage(fault_a_image).IsDefinitelyCpuDirty(),
              "a repeated padding notification lost maybe-dirty ownership");
      texture_cache.InvalidateMemory(base + 0x8010, sizeof(fault_b));
      Require(name, "definite write supersedes maybe-dirty state",
              texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty() &&
                  !texture_cache.GetImage(fault_b_image).IsMaybeCpuDirty(),
              "an overlapping CPU write retained the padding-only dirty state");
      texture_cache.UpdateImage(fault_a_image);
      texture_cache.UpdateImage(fault_b_image);

      auto fault_readback = CreateHostBuffer(
          name, 2 * sizeof(uint32_t), vk::BufferUsageFlagBits::eTransferDst, {0, 0});
      vk::BufferImageCopy fault_copy{};
      fault_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1};
      fault_copy.imageExtent = {1, 1, 1};
      for (const auto id : {fault_a_image, fault_b_image}) {
        texture_cache.GetImage(id).Download(
            {&fault_copy, 1}, fault_readback.buffer, fault_copy.bufferOffset,
            sizeof(uint32_t));
        fault_copy.bufferOffset += sizeof(uint32_t);
      }
      HostReadBarrier(fault_readback.buffer, fault_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      scheduler.Finish();
      Require(name, "same-page CPU refresh contents",
              ReadBuffer(name, fault_readback, 2) ==
                  std::vector<u32>{fault_a_cpu, fault_b_cpu},
              "repeated padding invalidation lost a CPU write or definite "
              "invalidation failed to refresh the native image");
      DestroyBuffer(&fault_readback);

      constexpr uint64_t publish_image_offset = 0x26000;
      constexpr uint64_t publish_buffer_offset = 0x26010;
      constexpr uint32_t publish_image_value = 0x31415926u;
      constexpr uint32_t publish_buffer_value = 0x27182818u;
      std::memcpy(memory + publish_image_offset, &publish_image_value,
                  sizeof(publish_image_value));
      auto publish_image_desc = MakeLinearDesc(
          base + publish_image_offset, sizeof(publish_image_value),
          vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
          Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
      const auto publish_image = texture_cache.FindImage(publish_image_desc);
      (void)texture_cache.FindTexture(publish_image, publish_image_desc);
      texture_cache.MarkGpuWritten(publish_image);
      resources.GetBufferCache().FillBuffer(base + publish_buffer_offset,
                                            sizeof(publish_buffer_value),
                                            publish_buffer_value, false);
      auto publish_replacement_desc = publish_image_desc;
      publish_replacement_desc.info.pixel_format = vk::Format::eR32Sfloat;
      publish_replacement_desc.info.guest_format =
          Prospero::BufferFormat::k32Float;
      publish_replacement_desc.view_info.format = vk::Format::eR32Sfloat;
      const auto publish_replacement =
          texture_cache.FindImage(publish_replacement_desc, true);
      Require(name, "exact-disjoint image replacement",
              publish_replacement && publish_replacement != publish_image,
              "exact-format lookup did not replace the image");
      Require(name, "exact-disjoint image publication",
              !resources.GetBufferCache().HasGpuDirtyBytes(
                  base + publish_buffer_offset, sizeof(publish_buffer_value)),
              "clean neighboring buffer unexpectedly became GPU-owned");
      uint32_t published_image_backing = 0;
      uint32_t published_buffer_backing = 0;
      std::memcpy(&published_image_backing, memory + publish_image_offset,
                  sizeof(published_image_backing));
      std::memcpy(&published_buffer_backing, memory + publish_buffer_offset,
                  sizeof(published_buffer_backing));
      Require(name, "exact-disjoint publication contents",
              published_image_backing == publish_image_value &&
                  published_buffer_backing == publish_buffer_value,
              "image publication or neighboring buffer readback changed bytes");

      for (const auto [offset, samples] :
           std::array<std::pair<uint64_t, uint32_t>, 2>{
               {{0x10000, 2}, {0x11000, 4}}}) {
        std::memset(memory + offset, 0, samples * sizeof(uint16_t));
        auto standalone_ms = MakeLinearDesc(
            base + offset, samples * sizeof(uint16_t), vk::Format::eD16Unorm,
            Prospero::BufferFormat::k16UNorm, Prospero::ImageType::kColor2D,
            {1, 1, 1}, 1, 2, samples);
        standalone_ms.type = BindingType::DepthTarget;
        standalone_ms.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
        standalone_ms.view_info.usage =
            vk::ImageUsageFlagBits::eDepthStencilAttachment;
        const auto image = texture_cache.FindImage(standalone_ms);
        (void)texture_cache.FindDepthTarget(image, standalone_ms);
        texture_cache.MarkGpuWritten(image);
        uint16_t standalone_cpu_read = 1;
        std::memcpy(&standalone_cpu_read, memory + offset,
                    sizeof(standalone_cpu_read));
        Require(
            name, "standalone MSAA ownership",
            texture_cache.GetImage(image).IsGpuModified() &&
                standalone_cpu_read == 0 &&
                texture_cache.GetImage(image).IsGpuModified() &&
                resources.HandleFault(PageFaultAccess::Write, base + offset) &&
                texture_cache.GetImage(image).IsGpuModified() &&
                texture_cache.GetImage(image).IsDefinitelyCpuDirty(),
            "fresh multisample target violated write-only ownership");
      }

      // A one-layer guest descriptor can alias layer 1 of a cached native array.
      constexpr uint64_t alias_dcc_address = base + 0x2120000;
      auto dcc_array = MakeLinearDesc(
          base + 0x2100000, 0x20000, vk::Format::eR16G16B16A16Sfloat,
          Prospero::BufferFormat::k16_16_16_16Float, Prospero::ImageType::kColor2D,
          {128, 64, 1}, 2, 8, 1);
      dcc_array.type = BindingType::RenderTarget;
      dcc_array.info.tile_mode = Prospero::TileMode::kRenderTarget;
      dcc_array.info.pitch = TileGetRenderTargetPitch(128, 8);
      dcc_array.info.UpdateSize();
      dcc_array.info.metadata.kind = ImageMetadataKind::Dcc;
      dcc_array.info.metadata.range = {alias_dcc_address, 0x2000};
      dcc_array.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
      WriteMetadata(context, alias_dcc_address, 0x2000, UINT32_MAX);
      WriteMetadata(context, alias_dcc_address + 0x2000, 0x1000, 0x5a5a5a5au);
      const auto dcc_array_id = texture_cache.FindImage(dcc_array);
      vk::ClearValue array_paint{};
      array_paint.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
      TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), dcc_array_id,
          {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 2}, array_paint);
      auto dcc_alias = dcc_array;
      dcc_alias.info.data = {dcc_array.info.data.address + 0x10000, 0x10000};
      dcc_alias.info.resources.layers = 1;
      dcc_alias.info.UpdateSize();
      dcc_alias.info.metadata.range = {alias_dcc_address + 0x1000, 0x1000};
      dcc_alias.view_info.type = vk::ImageViewType::e2D;
      dcc_alias.view_info.layer_count = 1;
      WriteMetadata(context, dcc_alias.info.metadata.range.address, 0x1000, 0x40404040u);
      Require(name, "DCC alias host layer",
              texture_cache.FindImage(dcc_alias) == dcc_array_id &&
                  dcc_alias.view_info.base_layer == 1 &&
                  ReadCachedTexel(name, context, dcc_array_id) ==
                      std::vector<u32>{0x00003c00u, 0x3c003c00u} &&
                  ReadCachedTexel(name, context, dcc_array_id, {}, {1, 1, 1}, 1) ==
                      std::vector<u32>{0, 0x3c000000u},
              "guest metadata layer 0 did not clear only remapped native layer 1");
      resources.GetBufferCache().ReadMemory(alias_dcc_address, 0x3000, false);
      std::array<uint8_t, 0x3000> alias_metadata{};
      Require(name, "DCC alias metadata bounds",
              LibKernel::Memory::TryReadBacking(alias_dcc_address, alias_metadata.data(),
                                               alias_metadata.size()) &&
                  std::all_of(alias_metadata.begin(), alias_metadata.begin() + 0x2000,
                              [](uint8_t value) { return value == 0xff; }) &&
                  std::all_of(alias_metadata.begin() + 0x2000, alias_metadata.end(),
                              [](uint8_t value) { return value == 0x5a; }),
              "remapped host layer changed metadata outside the guest alias allocation");

      RenderColorInfo retired_alias{};
      retired_alias.desc = dcc_alias;
      retired_alias.image_id = dcc_array_id;
      retired_alias.guest_mip_level = 0;
      retired_alias.guest_array_layer = 0;
      texture_cache.UnmapMemory(dcc_array.info.data.address, dcc_array.info.data.size);
      WriteMetadata(context, dcc_alias.info.metadata.range.address, 0x1000, 0x40404040u);
      RenderExecutorTestAccess::PrepareGraphicsBindings(
          context.GetRenderExecutor(), std::span<PreparedBindings *const>{},
          std::span{&retired_alias, 1u});
      RenderDepthInfo alias_depth{};
      const auto alias_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
          context.GetRenderExecutor(), scheduler.Current(), &retired_alias, 1, alias_depth);
      Require(name, "retired DCC alias native coordinates",
              retired_alias.image_id != dcc_array_id &&
                  retired_alias.desc.view_info.base_level == 0 &&
                  retired_alias.desc.view_info.base_layer == 0 &&
                  retired_alias.desc.info.resources.layers == 1 &&
                  alias_rendering.color_attachments[0].image_view != nullptr &&
                  ReadCachedTexel(name, context, retired_alias.image_id) ==
                      std::vector<u32>{0, 0x3c000000u},
              "rediscovery treated a retired host layer as the native DCC slice");
      resources.GetBufferCache().ReadMemory(alias_dcc_address, 0x3000);
      Require(name, "retired DCC alias metadata bounds",
              LibKernel::Memory::TryReadBacking(alias_dcc_address, alias_metadata.data(),
                                               alias_metadata.size()) &&
                  std::all_of(alias_metadata.begin(), alias_metadata.begin() + 0x2000,
                              [](uint8_t byte) { return byte == 0xff; }) &&
                  std::all_of(alias_metadata.begin() + 0x2000, alias_metadata.end(),
                              [](uint8_t byte) { return byte == 0x5a; }),
              "retired alias rediscovery overwrote metadata outside its native slice");
      RenderExecutorTestAccess::ResetBindings(context.GetRenderExecutor());

      constexpr uint64_t metadata_data_a = 0x12000;
      constexpr uint64_t metadata_data_b = 0x12100;
      constexpr uint64_t metadata_a = 0x13000;
      constexpr uint64_t metadata_b = 0x13100;
      auto MakeMetadataDepth = [&](uint64_t data, uint64_t metadata) {
        auto desc = MakeLinearDesc(
            base + data, sizeof(uint32_t), vk::Format::eD32Sfloat,
            Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
            {1, 1, 1}, 1, 4, 1);
        desc.type = BindingType::DepthTarget;
        desc.info.metadata.kind = ImageMetadataKind::Htile;
        desc.info.metadata.range = {base + metadata, 0x80};
        desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
        desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
        return desc;
      };
      auto metadata_depth_a = MakeMetadataDepth(metadata_data_a, metadata_a);
      auto metadata_depth_b = MakeMetadataDepth(metadata_data_b, metadata_b);
      metadata_depth_a.info.htile_clear_mask = 0;
      const auto metadata_depth_a_id =
          texture_cache.FindImage(metadata_depth_a);
      const auto metadata_depth_b_id =
          texture_cache.FindImage(metadata_depth_b);
      WriteMetadata(context, base + metadata_a, 0x80, 0);
      Require(
          name, "unbound metadata separation",
          !texture_cache.IsMeta(base + metadata_a),
          "unbound metadata bytes were exposed as registered HTile");
      const auto metadata_depth_a_view =
          texture_cache.FindDepthTarget(metadata_depth_a_id, metadata_depth_a);
      const auto metadata_depth_b_view =
          texture_cache.FindDepthTarget(metadata_depth_b_id, metadata_depth_b);
      Require(name, "shared-page metadata state",
              metadata_depth_a_view != nullptr &&
                  metadata_depth_b_view != nullptr &&
                  !texture_cache.IsMetaCleared(base + metadata_a, 0) &&
                  texture_cache.ClearMeta(base + metadata_a) &&
                  texture_cache.ClearMeta(base + metadata_b),
              "shared-page metadata did not retain logical clear state");
      auto metadata_alias = MakeLinearDesc(
          base + metadata_a, sizeof(uint32_t), vk::Format::eR32Sfloat,
          Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
          {1, 1, 1}, 1, 4, 1);
      metadata_alias.type = BindingType::RenderTarget;
      metadata_alias.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
      const auto metadata_alias_id =
          texture_cache.FindImage(metadata_alias, true);
      const auto metadata_alias_view =
          texture_cache.FindRenderTarget(metadata_alias_id, metadata_alias);
      Require(
          name, "exact alias metadata coexistence",
          metadata_alias_view != nullptr &&
              texture_cache.GetImage(metadata_depth_a_id).IsGpuModified() &&
              texture_cache.GetImage(metadata_depth_b_id).IsGpuModified() &&
              texture_cache.IsMeta(base + metadata_a) &&
              texture_cache.IsMetaCleared(base + metadata_a, 0) &&
              texture_cache.IsMeta(base + metadata_b) &&
              texture_cache.IsMetaCleared(base + metadata_b, 0),
          "image discovery retired aliased metadata or its live depth image");
      const bool touched_metadata = texture_cache.TouchMeta(base + metadata_a, 0, false);
      WriteMetadata(context, base + metadata_a, 0x80, 0);
      Require(
          name, "registered DCC/HTile separation",
          touched_metadata && !texture_cache.IsMetaCleared(base + metadata_a, 0),
          "a PS5 DCC fill changed registered HTile state");
      Require(
          name, "metadata first-touch state",
          texture_cache.TouchMeta(base + metadata_b, 0, true) &&
              texture_cache.IsMetaCleared(base + metadata_b, 0) &&
              texture_cache.TouchMeta(base + metadata_b, 0, false) &&
              !texture_cache.IsMetaCleared(base + metadata_b, 0),
          "per-slice metadata update incorrectly required prior GPU ownership");
      resources.GetBufferCache().FillBuffer(base + metadata_b, sizeof(uint32_t),
                                            0, false);
      Require(
          name, "metadata buffer fill state",
          texture_cache.IsMetaCleared(base + metadata_b, 0),
          "BufferCache fill did not publish an exact-address metadata clear");

      // R-Type Final 3 reuses an old color allocation inside a new stencil plane,
      // with the same metadata address changing from DCC to HTile.
      auto reused_depth = MakeMetadataDepth(0x12300, 0x28000);
      reused_depth.info.pixel_format = vk::Format::eD32SfloatS8Uint;
      reused_depth.info.stencil = {base + 0x12200, 0x80};
      reused_depth.info.htile_clear_mask = 0;
      reused_depth.view_info.format = vk::Format::eD32SfloatS8Uint;
      reused_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth |
                                      vk::ImageAspectFlagBits::eStencil;
      auto reused_color = MakeLinearDesc(
          base + 0x12240, sizeof(uint32_t), vk::Format::eR8G8B8A8Unorm,
          Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
          {1, 1, 1}, 1, 4, 1);
      reused_color.type = BindingType::RenderTarget;
      reused_color.info.metadata.kind = ImageMetadataKind::Dcc;
      reused_color.info.metadata.range = {base + 0x28000, 0x1000};
      reused_color.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
      const auto reused_color_id = texture_cache.FindImage(reused_color);
      (void)texture_cache.FindRenderTarget(reused_color_id, reused_color);
      // Repeated CPU stores must remain visible through ordinary buffer ownership.
      for (uint32_t repeat = 0; repeat < 2; repeat++) {
        vk::ClearValue painted{};
        painted.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
        TextureCacheTestAccess::ClearImage(
            texture_cache, scheduler.Current(), reused_color_id,
            {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
        std::memset(memory + 0x28000, 0, 0x1000);
        Require(name, "CPU metadata rediscovery",
                texture_cache.FindImage(reused_color) == reused_color_id,
                "metadata writes replaced the live expanded image");
        (void)texture_cache.FindRenderTarget(reused_color_id, reused_color);
        Require(name, "repeated CPU metadata clear",
                ReadCachedTexel(name, context, reused_color_id) == std::vector<u32>{0},
                "a same-value CPU metadata write lost its clear after rendering");
      }
      WriteMetadata(context, base + 0x28000, 0x1000, 0);
      const auto reused_depth_id = texture_cache.FindImage(reused_depth);
      (void)texture_cache.FindDepthTarget(reused_depth_id, reused_depth);
      Require(
          name, "DCC allocation reused as HTile",
          TextureCacheTestAccess::Contains(texture_cache, reused_color_id) &&
              !texture_cache.IsMetaCleared(base + 0x28000, 0) &&
              texture_cache.ClearMeta(base + 0x28000) &&
              texture_cache.TouchMeta(base + 0x28000, 0, false),
          "depth binding retained incompatible DCC clear state");
      texture_cache.UnmapMemory(reused_color.info.data.address,
                                reused_color.info.data.size);
      (void)texture_cache.FindDepthTarget(reused_depth_id, reused_depth);
      Require(
          name, "reused metadata owner retirement",
          !TextureCacheTestAccess::Contains(texture_cache, reused_color_id) &&
              texture_cache.IsMeta(base + 0x28000) &&
              !texture_cache.IsMetaCleared(base + 0x28000, 0) &&
              texture_cache.IsMetaCleared(base + 0x28000, 1),
          "retiring the old DCC image erased the live HTile slice state");
      texture_cache.UnmapMemory(reused_depth.info.data.address,
                                reused_depth.info.data.size);
      Require(name, "reused metadata final retirement",
              !texture_cache.IsMeta(base + 0x28000),
              "retiring the HTile image left its metadata registered");

      constexpr uint64_t partial_unmap_image_offset = 0x2700000;
      auto partial_unmap_image =
          MakeLinearDesc(base + partial_unmap_image_offset, 0x2000,
                         vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
                         Prospero::ImageType::kColor2D, {2048, 1, 1}, 1, 4, 1);
      const auto partial_unmap_image_id =
          texture_cache.FindImage(partial_unmap_image);
      const auto image_address = partial_unmap_image.info.data.address;
      Require(name, "registered image boundaries",
              texture_cache.IsRegionRegistered(image_address, 1) &&
                  texture_cache.IsRegionRegistered(image_address + 0x1fff, 1) &&
                  !texture_cache.IsRegionRegistered(image_address - 1, 1) &&
                  !texture_cache.IsRegionRegistered(image_address + 0x2000, 1) &&
                  texture_cache.IsRegionRegistered(image_address - 1, 0x2002),
              "image lookup mishandled a partial overlap or adjacent range");
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      const auto image_unmap_tick = scheduler.CurrentTick();
      resources.UnmapMemory(image_address + 0x1000, 0x1000);
      Require(name, "partial image unmap tracking",
              partial_unmap_image_id &&
                  scheduler.CurrentTick() == image_unmap_tick + 1 &&
                  !texture_cache.IsRegionRegistered(image_address, 0x2000) &&
                  !texture_cache.FindImageFromRange(image_address, 0x2000, false),
              "partial image unmap skipped its drain or retained the image");

      constexpr uint64_t unformatted_alias_offset = 0x2500000;
      auto unformatted_alias =
          MakeLinearDesc(base + unformatted_alias_offset, sizeof(uint32_t),
                         vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
      const auto unformatted_alias_image =
          texture_cache.FindImage(unformatted_alias);
      (void)texture_cache.FindTexture(unformatted_alias_image,
                                      unformatted_alias);
      texture_cache.MarkGpuWritten(unformatted_alias_image);
      auto unformatted_alias_buffer = resources.GetBufferCache().ObtainBuffer(
          unformatted_alias.info.data.address, unformatted_alias.info.data.size,
          false);
      Require(
          name, "unformatted buffer/image alias",
          unformatted_alias_buffer.first != nullptr &&
              texture_cache.GetImage(unformatted_alias_image).IsGpuModified() &&
              !texture_cache.GetImage(unformatted_alias_image)
                   .IsBufferModified(),
          "read-only unformatted buffer alias did not follow ordinary "
          "BufferCache acquisition");

      constexpr uint64_t layered_offset = 0x14000;
      constexpr uint64_t layered_guest_size = 0x800;
      const std::array<float, 10> layered_values{0.0f, 0.125f, 0.25f, 0.375f,
                                                 0.5f, 0.625f, 0.75f, 0.875f,
                                                 1.0f, 0.0625f};
      std::memset(memory + layered_offset, 0, layered_guest_size);
      const auto layered_layout = MakeTilingImage(
          Prospero::BufferFormat::k32Float, 2, 2, 2, 2,
          Prospero::TileMode::kLinear, layered_guest_size, false);
      const auto layered_upload_regions = layered_layout.BufferCopies();
      Require(name, "layered guest layout", layered_upload_regions.size() == 2,
              "linear array upload did not batch each mip");
      for (const auto &region : layered_upload_regions) {
        for (uint32_t layer = 0; layer < region.imageSubresource.layerCount;
             ++layer) {
          for (uint32_t y = 0; y < region.imageExtent.height; ++y) {
            for (uint32_t x = 0; x < region.imageExtent.width; ++x) {
              const uint32_t absolute_layer =
                  region.imageSubresource.baseArrayLayer + layer;
              const uint32_t logical = region.imageSubresource.mipLevel == 0
                                           ? absolute_layer * 4 + y * 2 + x
                                           : 8 + absolute_layer;
              const uint64_t byte_offset =
                  region.bufferOffset +
                  ((uint64_t{layer} * region.bufferImageHeight + y) *
                       region.bufferRowLength +
                   x) *
                      sizeof(float);
              Require(name, "layered guest layout bounds",
                      byte_offset + sizeof(float) <= layered_guest_size,
                      "layered guest texel lies outside its backing range");
              std::memcpy(memory + layered_offset + byte_offset,
                          &layered_values[logical], sizeof(float));
            }
          }
        }
      }
      auto layered_color = MakeLinearDesc(
          base + layered_offset, layered_guest_size, vk::Format::eR32Sfloat,
          Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
          {2, 2, 1}, 2, 4, 1);
      layered_color.info.resources.levels = 2;
      layered_color.info.mip_layout[0] = {0, 32, 2, 2};
      layered_color.info.mip_layout[1] = {32, 8, 1, 1};
      layered_color.view_info.level_count = 2;
      layered_color.info.UpdateSize();
      const auto layered_color_image = texture_cache.FindImage(layered_color);
      (void)texture_cache.FindTexture(layered_color_image, layered_color);
      texture_cache.MarkGpuWritten(layered_color_image);
      auto layered_depth = layered_color;
      layered_depth.type = BindingType::DepthTarget;
      layered_depth.info.resources = {1, 4};
      layered_depth.info.mip_layout = {};
      layered_depth.info.mip_layout[0] = {0, layered_guest_size, 2, 2};
      layered_depth.view_info.level_count = 1;
      layered_depth.view_info.layer_count = 4;
      layered_depth.info.pixel_format = vk::Format::eD32Sfloat;
      layered_depth.view_info.format = vk::Format::eD32Sfloat;
      layered_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
      layered_depth.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      layered_depth.info.UpdateSize();
      const auto layered_depth_image = texture_cache.FindImage(layered_depth);
      const auto &layered_native = texture_cache.GetImage(layered_depth_image);
      Require(
          name, "layered mipped depth alias",
          layered_depth_image != layered_color_image &&
              layered_native.backing.layers == 4 &&
              layered_native.backing.mip_levels == 1 &&
              layered_native.info.resources == layered_depth.info.resources &&
              layered_native.info.mip_layout == layered_depth.info.mip_layout &&
              layered_native.IsGpuModified(),
          "depth/color conversion mixed the source mip count with the "
          "requested layout");
      const volatile auto layered_guest_byte =
          *reinterpret_cast<const volatile uint8_t *>(memory + layered_offset);
      (void)layered_guest_byte;
      Require(name, "layered depth write-only read policy",
              texture_cache.GetImage(layered_depth_image).IsGpuModified(),
              "a CPU read invalidated layered depth GPU ownership");

      constexpr uint64_t replacement_offset = 0x18000;
      constexpr uint64_t replacement_size = 0x2000;
      constexpr uint32_t replacement_pixel = 0xcafebabeu;
      constexpr uint32_t replacement_tail = 0x0badf00du;
      std::memset(memory + replacement_offset, 0, replacement_size);
      std::memcpy(memory + replacement_offset, &replacement_pixel,
                  sizeof(replacement_pixel));
      std::memcpy(memory + replacement_offset + replacement_size -
                      sizeof(replacement_tail),
                  &replacement_tail, sizeof(replacement_tail));
      auto replacement_source =
          MakeLinearDesc(base + replacement_offset, sizeof(uint32_t),
                         vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
      const auto replacement_source_image =
          texture_cache.FindImage(replacement_source);
      (void)texture_cache.FindTexture(replacement_source_image,
                                      replacement_source);
      texture_cache.MarkGpuWritten(replacement_source_image);
      auto replacement_desc =
          MakeLinearDesc(base + replacement_offset, replacement_size,
                         vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1);
      const auto replacement = texture_cache.FindImage(replacement_desc);
      const bool replacement_changed = replacement != replacement_source_image;
      const bool replacement_gpu_before =
          texture_cache.GetImage(replacement).IsGpuModified();
      const bool replacement_fault = resources.HandleFault(
          PageFaultAccess::Write, base + replacement_offset + 0x1000);
      const bool replacement_gpu_after =
          texture_cache.GetImage(replacement).IsGpuModified();
      uint32_t replacement_pixel_after = 0;
      uint32_t replacement_tail_after = 0;
      std::memcpy(&replacement_pixel_after, memory + replacement_offset,
                  sizeof(replacement_pixel_after));
      std::memcpy(&replacement_tail_after,
                  memory + replacement_offset + replacement_size -
                      sizeof(replacement_tail_after),
                  sizeof(replacement_tail_after));
      Require(name, "replacement tracking",
              replacement_changed && replacement_gpu_before &&
                  replacement_fault && replacement_gpu_after &&
                  texture_cache.GetImage(replacement).IsDefinitelyCpuDirty() &&
                  replacement_pixel_after == replacement_pixel &&
                  replacement_tail_after == replacement_tail,
              "replacement write invalidation lost native authority or guest "
              "backing");

      const uint32_t compressed_value = 0x55667788u;
      std::memcpy(memory + 0xc000, &compressed_value, sizeof(compressed_value));
      auto compressed_desc = MakeLinearDesc(
          base + 0xc000, sizeof(compressed_value), vk::Format::eR8G8B8A8Srgb,
          Prospero::BufferFormat::k8_8_8_8Srgb, Prospero::ImageType::kColor2D,
          {1, 1, 1}, 1, 4, 1);
      compressed_desc.type = BindingType::RenderTarget;
      compressed_desc.info.metadata.kind = ImageMetadataKind::Dcc;
      compressed_desc.info.metadata.range = {base + 0xd000, 0};
      compressed_desc.info.metadata.compression =
          VideoOutCompression::Dcc256_256_0;
      compressed_desc.view_info.usage =
          vk::ImageUsageFlagBits::eColorAttachment;
      const auto compressed_image = texture_cache.FindImage(compressed_desc);
      (void)texture_cache.FindRenderTarget(compressed_image, compressed_desc);
      uint32_t compressed_cpu_read = 0;
      std::memcpy(&compressed_cpu_read, memory + 0xc000,
                  sizeof(compressed_cpu_read));
      Require(name, "compressed write-only read policy",
              compressed_cpu_read == compressed_value &&
                  texture_cache.GetImage(compressed_image).IsGpuModified(),
              "compressed image incorrectly claimed a CPU read fault");
      Require(name, "compressed download rejection",
              !TextureCacheTestAccess::TryDownload(texture_cache,
                                                   compressed_image) &&
                  texture_cache.GetImage(compressed_image).IsGpuModified() &&
                  !texture_cache.GetImage(compressed_image).IsBufferModified(),
              "a compressed image escaped the unified download guard");
      auto compressed_video_desc = compressed_desc;
      compressed_video_desc.type = BindingType::VideoOut;
      compressed_video_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
      const auto compressed_video =
          texture_cache.FindImage(compressed_video_desc);
      const auto &compressed_video_image =
          texture_cache.GetImage(compressed_video);
      Require(
          name, "cache-only video-out discovery",
          compressed_video == compressed_image &&
              !compressed_video_image.binding.is_bound &&
              !compressed_video_image.binding.is_target &&
              !compressed_video_image.binding.needs_rebind &&
              !compressed_video_image.usage.video_out,
          "FindImage claimed caller-owned video-out usage or binding state");

      constexpr uint64_t mixed_source_offset = 0x20000;
      constexpr uint64_t mixed_source_size = 0x1004;
      constexpr uint32_t mixed_source_width = 1025;
      constexpr uint32_t mixed_cpu_value = 0x1234abcdu;
      constexpr uint32_t mixed_gpu_value = 0x9876fedcu;
      auto mixed_owner = resources.GetBufferCache().ObtainBuffer(
          base + mixed_source_offset, mixed_source_size, true);
      Require(name, "mixed-page source allocation",
              mixed_owner.first != nullptr,
              "mixed CPU/GPU image source did not create a containing buffer");
      Require(name, "mixed-page CPU write fault",
              resources.HandleFault(PageFaultAccess::Write,
                                    base + mixed_source_offset),
              "mixed image source could not dirty its first page");
      std::memcpy(memory + mixed_source_offset, &mixed_cpu_value,
                  sizeof(mixed_cpu_value));
      resources.GetBufferCache().FillBuffer(base + mixed_source_offset + 0x1000,
                                            sizeof(mixed_gpu_value),
                                            mixed_gpu_value, false);
      auto mixed_source_desc = MakeLinearDesc(
          base + mixed_source_offset, mixed_source_size, vk::Format::eR32Uint,
          Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
          {mixed_source_width, 1, 1}, 1, 4, 1);
      const auto mixed_source_image =
          texture_cache.FindImage(mixed_source_desc);
      (void)texture_cache.FindTexture(mixed_source_image, mixed_source_desc);
      Require(name, "mixed-page image upload",
              !texture_cache.GetImage(mixed_source_image).IsGpuModified(),
              "clean Buffer source manufactured GPU image ownership");
      auto mixed_source_readback = CreateHostBuffer(
          name, mixed_source_size, vk::BufferUsageFlagBits::eTransferDst,
          std::vector<u32>(mixed_source_size / sizeof(u32), 0));
      auto &mixed_source_native = texture_cache.GetImage(mixed_source_image);
      mixed_source_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
                                  vk::AccessFlagBits2::eTransferRead, {},
                                  command.Handle());
      vk::BufferImageCopy mixed_source_copy{};
      mixed_source_copy.bufferRowLength = mixed_source_width;
      mixed_source_copy.imageSubresource.aspectMask =
          vk::ImageAspectFlagBits::eColor;
      mixed_source_copy.imageSubresource.layerCount = 1;
      mixed_source_copy.imageExtent = {mixed_source_width, 1, 1};
      command.Handle().copyImageToBuffer(mixed_source_native.backing.image,
                                         vk::ImageLayout::eTransferSrcOptimal,
                                         mixed_source_readback.buffer, 1,
                                         &mixed_source_copy);
      HostReadBarrier(mixed_source_readback.buffer, mixed_source_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);

      constexpr uint64_t bridged_gpu_offset = 0x101000;
      constexpr uint64_t bridged_cpu_offset = 0x105000;
      constexpr uint64_t bridged_texel_size = 16;
      constexpr uint64_t bridged_source_size =
          bridged_cpu_offset - bridged_gpu_offset + bridged_texel_size;
      constexpr uint32_t bridged_source_width =
          bridged_source_size / bridged_texel_size;
      constexpr uint32_t bridged_gpu_value = 0x10293847u;
      constexpr uint32_t bridged_cpu_value = 0x56473829u;
      constexpr uint32_t bridged_gpu_guest_value = 0xa5a5a5a5u;
      std::array<uint32_t, bridged_texel_size / sizeof(uint32_t)>
          bridged_cpu_values{};
      bridged_cpu_values.fill(bridged_cpu_value);
      std::memcpy(memory + bridged_cpu_offset, bridged_cpu_values.data(),
                  sizeof(bridged_cpu_values));
      auto bridged_cpu_owner = resources.GetBufferCache().ObtainBuffer(
          base + bridged_cpu_offset, bridged_texel_size, true);
      Require(name, "bridged CPU owner allocation",
              bridged_cpu_owner.first != nullptr,
              "bridged image source did not create its CPU-side cache owner");
      resources.GetBufferCache().FillBuffer(base + bridged_cpu_offset,
                                            bridged_texel_size, 0, false);
      Require(
          name, "bridged CPU write fault",
          resources.HandleFault(PageFaultAccess::Write,
                                base + bridged_cpu_offset),
          "bridged image source could not transfer its second owner to CPU");
      std::memcpy(memory + bridged_cpu_offset, bridged_cpu_values.data(),
                  sizeof(bridged_cpu_values));

      std::array<uint32_t, bridged_texel_size / sizeof(uint32_t)>
          bridged_gpu_guest_values{};
      bridged_gpu_guest_values.fill(bridged_gpu_guest_value);
      std::memcpy(memory + bridged_gpu_offset, bridged_gpu_guest_values.data(),
                  sizeof(bridged_gpu_guest_values));

      auto bridged_gpu_owner = resources.GetBufferCache().ObtainBuffer(
          base + bridged_gpu_offset, bridged_texel_size, true);
      Require(name, "bridged GPU owner allocation",
              bridged_gpu_owner.first != nullptr &&
                  bridged_gpu_owner.first != bridged_cpu_owner.first,
              "bridged image source did not retain two disjoint cache owners");
      resources.GetBufferCache().FillBuffer(base + bridged_gpu_offset,
                                            bridged_texel_size,
                                            bridged_gpu_value, false);
      Require(name, "bridged ownership split",
              resources.GetBufferCache().HasGpuDirtyBytes(
                  base + bridged_gpu_offset, bridged_texel_size) &&
                  !resources.GetBufferCache().HasGpuDirtyBytes(
                      base + bridged_cpu_offset, bridged_texel_size) &&
                  resources.GetBufferCache().IsRegionCpuModified(
                      base + bridged_cpu_offset, bridged_texel_size),
              "bridged image source did not retain distinct GPU and CPU "
              "ownership domains");

      auto bridged_source_desc = MakeLinearDesc(
          base + bridged_gpu_offset, bridged_source_size,
          vk::Format::eR32G32B32A32Uint,
          Prospero::BufferFormat::k32_32_32_32UInt,
          Prospero::ImageType::kColor2D, {bridged_source_width, 1, 1}, 1,
          bridged_texel_size, 1);
      const auto bridged_source_image =
          texture_cache.FindImage(bridged_source_desc);
      (void)texture_cache.FindTexture(bridged_source_image,
                                      bridged_source_desc);
      auto bridged_source_readback = CreateHostBuffer(
          name, bridged_source_size, vk::BufferUsageFlagBits::eTransferDst,
          std::vector<u32>(bridged_source_size / sizeof(u32), 0));
      auto &bridged_source_native =
          texture_cache.GetImage(bridged_source_image);
      bridged_source_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
                                    vk::AccessFlagBits2::eTransferRead, {},
                                    scheduler.Current().Handle());
      vk::BufferImageCopy bridged_source_copy{};
      bridged_source_copy.bufferRowLength = bridged_source_width;
      bridged_source_copy.imageSubresource.aspectMask =
          vk::ImageAspectFlagBits::eColor;
      bridged_source_copy.imageSubresource.layerCount = 1;
      bridged_source_copy.imageExtent = {bridged_source_width, 1, 1};
      scheduler.Current().Handle().copyImageToBuffer(
          bridged_source_native.backing.image,
          vk::ImageLayout::eTransferSrcOptimal, bridged_source_readback.buffer,
          1, &bridged_source_copy);
      HostReadBarrier(bridged_source_readback.buffer,
                      bridged_source_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);

      constexpr uint64_t byte_mirror_page_offset = 0x24000;
      constexpr uint64_t byte_mirror_offset = byte_mirror_page_offset + 1;
      const std::array<uint8_t, 4> byte_mirror_guest{0xacu, 0x5au, 0xbdu,
                                                     0xceu};
      std::memcpy(memory + byte_mirror_page_offset, byte_mirror_guest.data(),
                  byte_mirror_guest.size());
      auto byte_mirror_desc =
          MakeLinearDesc(base + byte_mirror_offset, 1, vk::Format::eR8Unorm,
                         Prospero::BufferFormat::k8UNorm,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 1, 1);
      const auto byte_mirror_image = texture_cache.FindImage(byte_mirror_desc);
      (void)texture_cache.FindTexture(byte_mirror_image, byte_mirror_desc);
      texture_cache.MarkGpuWritten(byte_mirror_image);
      auto byte_mirror = resources.GetBufferCache().ObtainBuffer(
          base + byte_mirror_offset, 1, false, true);
      Require(
          name, "byte image mirror",
          byte_mirror.first != nullptr &&
              !texture_cache.GetImage(byte_mirror_image).IsBufferModified() &&
              texture_cache.GetImage(byte_mirror_image).IsGpuModified(),
          "one-byte image copy transferred cache ownership");

      const std::array<uint16_t, 4> bgra16_guest{0x3c00u, 0x4000u, 0x4200u,
                                                 0x4400u};
      std::memcpy(memory + 0xb000, bgra16_guest.data(), sizeof(bgra16_guest));
      auto bgra16_desc = MakeLinearDesc(
          base + 0xb000, sizeof(bgra16_guest), vk::Format::eR16G16B16A16Sfloat,
          Prospero::BufferFormat::k16_16_16_16Float,
          Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1);
      bgra16_desc.type = BindingType::RenderTarget;
      bgra16_desc.info.bgra16 = true;
      bgra16_desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
      const auto bgra16_image = texture_cache.FindImage(bgra16_desc);
      (void)texture_cache.FindRenderTarget(bgra16_image, bgra16_desc);
      auto bgra16_readback =
          CreateHostBuffer(name, sizeof(bgra16_guest),
                           vk::BufferUsageFlagBits::eTransferDst, {0, 0});
      auto &bgra16_native = texture_cache.GetImage(bgra16_image);
      bgra16_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
                            vk::AccessFlagBits2::eTransferRead, {},
                            command.Handle());
      vk::BufferImageCopy bgra16_copy{};
      bgra16_copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
      bgra16_copy.imageSubresource.layerCount = 1;
      bgra16_copy.imageExtent = {1, 1, 1};
      command.Handle().copyImageToBuffer(
          bgra16_native.backing.image, vk::ImageLayout::eTransferSrcOptimal,
          bgra16_readback.buffer, 1, &bgra16_copy);
      HostReadBarrier(bgra16_readback.buffer, bgra16_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);

      auto layered_readback = CreateHostBuffer(
          name, 8 * sizeof(float), vk::BufferUsageFlagBits::eTransferDst,
          std::vector<u32>(8, 0));
      auto &layered_depth_native = texture_cache.GetImage(layered_depth_image);
      layered_depth_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
                                   vk::AccessFlagBits2::eTransferRead, {},
                                   command.Handle());
      vk::BufferImageCopy layered_copy{};
      layered_copy.imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0, 2};
      layered_copy.imageExtent = {2, 2, 1};
      command.Handle().copyImageToBuffer(
          layered_depth_native.backing.image,
          vk::ImageLayout::eTransferSrcOptimal, layered_readback.buffer,
          1, &layered_copy);
      HostReadBarrier(layered_readback.buffer, layered_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);

      vk::ShaderModuleCreateInfo module_info{};
      module_info.sType = vk::StructureType::eShaderModuleCreateInfo;
      module_info.codeSize = sizeof(GPU_TEST_MS_DEPTH_SPV);
      module_info.pCode = GPU_TEST_MS_DEPTH_SPV;
      vk::ShaderModule ms_depth_module = nullptr;
      RequireVk(
          name, "MS depth content",
          m_device.createShaderModule(&module_info, nullptr, &ms_depth_module),
          "vkCreateShaderModule");
      std::array<vk::DescriptorSetLayoutBinding, 2> observer_bindings{};
      observer_bindings[0].binding = 0;
      observer_bindings[0].descriptorType = vk::DescriptorType::eSampledImage;
      observer_bindings[0].descriptorCount = 1;
      observer_bindings[0].stageFlags = vk::ShaderStageFlagBits::eCompute;
      observer_bindings[1].binding = 1;
      observer_bindings[1].descriptorType = vk::DescriptorType::eStorageBuffer;
      observer_bindings[1].descriptorCount = 1;
      observer_bindings[1].stageFlags = vk::ShaderStageFlagBits::eCompute;
      vk::DescriptorSetLayoutCreateInfo observer_layout_info{};
      observer_layout_info.sType =
          vk::StructureType::eDescriptorSetLayoutCreateInfo;
      observer_layout_info.bindingCount =
          static_cast<uint32_t>(observer_bindings.size());
      observer_layout_info.pBindings = observer_bindings.data();
      vk::DescriptorSetLayout observer_set_layout = nullptr;
      RequireVk(name, "MS depth content",
                m_device.createDescriptorSetLayout(
                    &observer_layout_info, nullptr, &observer_set_layout),
                "vkCreateDescriptorSetLayout");
      const vk::PushConstantRange observer_push{
          vk::ShaderStageFlagBits::eCompute, 0, sizeof(uint32_t)};
      vk::PipelineLayoutCreateInfo observer_pipeline_layout_info{};
      observer_pipeline_layout_info.sType =
          vk::StructureType::ePipelineLayoutCreateInfo;
      observer_pipeline_layout_info.setLayoutCount = 1;
      observer_pipeline_layout_info.pSetLayouts = &observer_set_layout;
      observer_pipeline_layout_info.pushConstantRangeCount = 1;
      observer_pipeline_layout_info.pPushConstantRanges = &observer_push;
      vk::PipelineLayout observer_pipeline_layout = nullptr;
      RequireVk(name, "MS depth content",
                m_device.createPipelineLayout(&observer_pipeline_layout_info,
                                              nullptr,
                                              &observer_pipeline_layout),
                "vkCreatePipelineLayout");
      vk::PipelineShaderStageCreateInfo observer_stage{};
      observer_stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
      observer_stage.stage = vk::ShaderStageFlagBits::eCompute;
      observer_stage.module = ms_depth_module;
      observer_stage.pName = "main";
      vk::ComputePipelineCreateInfo observer_pipeline_info{};
      observer_pipeline_info.sType =
          vk::StructureType::eComputePipelineCreateInfo;
      observer_pipeline_info.stage = observer_stage;
      observer_pipeline_info.layout = observer_pipeline_layout;
      vk::Pipeline observer_depth_pipeline = nullptr;
      RequireVk(
          name, "MS depth content",
          m_device.createComputePipelines(nullptr, 1, &observer_pipeline_info,
                                          nullptr, &observer_depth_pipeline),
          "vkCreateComputePipelines");
      const std::array<vk::DescriptorPoolSize, 2> observer_pool_sizes{{
          {vk::DescriptorType::eSampledImage, 2},
          {vk::DescriptorType::eStorageBuffer, 2},
      }};
      vk::DescriptorPoolCreateInfo observer_pool_info{};
      observer_pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo;
      observer_pool_info.maxSets = 2;
      observer_pool_info.poolSizeCount =
          static_cast<uint32_t>(observer_pool_sizes.size());
      observer_pool_info.pPoolSizes = observer_pool_sizes.data();
      vk::DescriptorPool observer_pool = nullptr;
      RequireVk(name, "MS depth content",
                m_device.createDescriptorPool(&observer_pool_info, nullptr,
                                              &observer_pool),
                "vkCreateDescriptorPool");
      const std::array<vk::DescriptorSetLayout, 2> observer_layouts{
          observer_set_layout, observer_set_layout};
      std::array<vk::DescriptorSet, 2> observer_sets{};
      vk::DescriptorSetAllocateInfo observer_allocate{};
      observer_allocate.sType = vk::StructureType::eDescriptorSetAllocateInfo;
      observer_allocate.descriptorPool = observer_pool;
      observer_allocate.descriptorSetCount =
          static_cast<uint32_t>(observer_sets.size());
      observer_allocate.pSetLayouts = observer_layouts.data();
      RequireVk(name, "MS depth content",
                m_device.allocateDescriptorSets(&observer_allocate,
                                                observer_sets.data()),
                "vkAllocateDescriptorSets");
      std::array<Buffer, 2> ms_observer_outputs{
          CreateHostBuffer(name, 4 * sizeof(u32),
                           vk::BufferUsageFlagBits::eStorageBuffer,
                           std::vector<u32>(4, 0)),
          CreateHostBuffer(name, 4 * sizeof(u32),
                           vk::BufferUsageFlagBits::eStorageBuffer,
                           std::vector<u32>(4, 0))};
      const std::array<ImageId, 2> observed_images{ms_depth_image,
                                                   ms_depth_image_4x};
      const std::array<uint32_t, 2> observed_samples{2, 4};
      for (uint32_t index = 0; index < observed_images.size(); index++) {
        auto &observed = texture_cache.GetImage(observed_images[index]);
        observed.Transit(vk::ImageLayout::eShaderReadOnlyOptimal,
                         vk::AccessFlagBits2::eShaderRead, {},
                         command.Handle());
        ImageViewInfo view_info{};
        view_info.format = observed.backing.format;
        view_info.type = vk::ImageViewType::e2D;
        view_info.aspect = vk::ImageAspectFlagBits::eDepth;
        view_info.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR,
                             vk::ComponentSwizzle::eR,
                             vk::ComponentSwizzle::eR};
        view_info.usage = vk::ImageUsageFlagBits::eSampled;
        const auto view = observed.FindView(view_info);
        const vk::DescriptorImageInfo image_info{
            nullptr, view, vk::ImageLayout::eShaderReadOnlyOptimal};
        const vk::DescriptorBufferInfo buffer_info{
            ms_observer_outputs[index].buffer, 0,
            ms_observer_outputs[index].size};
        std::array<vk::WriteDescriptorSet, 2> writes{};
        writes[0].sType = vk::StructureType::eWriteDescriptorSet;
        writes[0].dstSet = observer_sets[index];
        writes[0].dstBinding = 0;
        writes[0].descriptorCount = 1;
        writes[0].descriptorType = vk::DescriptorType::eSampledImage;
        writes[0].pImageInfo = &image_info;
        writes[1].sType = vk::StructureType::eWriteDescriptorSet;
        writes[1].dstSet = observer_sets[index];
        writes[1].dstBinding = 1;
        writes[1].descriptorCount = 1;
        writes[1].descriptorType = vk::DescriptorType::eStorageBuffer;
        writes[1].pBufferInfo = &buffer_info;
        m_device.updateDescriptorSets(static_cast<uint32_t>(writes.size()),
                                      writes.data(), 0, nullptr);
        command.Handle().bindPipeline(vk::PipelineBindPoint::eCompute,
                                      observer_depth_pipeline);
        command.Handle().bindDescriptorSets(vk::PipelineBindPoint::eCompute,
                                            observer_pipeline_layout, 0, 1,
                                            &observer_sets[index], 0, nullptr);
        command.Handle().pushConstants(
            observer_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0,
            sizeof(observed_samples[index]), &observed_samples[index]);
        command.Handle().dispatch(1, 1, 1);
        HostReadBarrier(ms_observer_outputs[index].buffer,
                        ms_observer_outputs[index].size,
                        vk::PipelineStageFlagBits::eComputeShader,
                        vk::AccessFlagBits::eShaderWrite);
      }

      scheduler.Finish();

      const auto mip_prefix_words = ReadBuffer(
          name, mip_prefix_readback, mip_prefix_size / sizeof(uint32_t));
      const auto volume_words =
          ReadBuffer(name, volume_readback, volume_guest.size());
      Require(name, "formatted volume content",
              volume_words[0] == volume_values[0] &&
                  volume_words[volume_slice_words] == volume_values[1],
              "volume synchronization lost a padded depth slice");
      DestroyBuffer(&volume_readback);
      const auto mip_formatted_words = ReadBuffer(
          name, mip_formatted_readback, mip_guest_size / sizeof(uint32_t));
      const auto mip0_word = mip_sizes[0].offset / sizeof(uint32_t);
      Require(name, "formatted mip prefix content",
              mip0_word < mip_prefix_words.size() &&
                  mip0_word < mip_native.size() &&
                  mip_prefix_words[mip0_word] == mip_native[mip0_word],
              "fitting mip-prefix synchronization copied "
              "stale guest backing");
      Require(name, "formatted Buffer content",
              mip0_word < mip_formatted_words.size() &&
                  mip0_word < mip_native.size() &&
                  mip_formatted_words[mip0_word] == mip_native[mip0_word],
              "formatted Buffer read bypassed authoritative "
              "native image mip data");
      Require(name, "CPU-dirty formatted mirror content",
              ReadBuffer(name, mirror_cpu_readback, 1) ==
                  std::vector<u32>{mirror_cpu_value},
              "formatted Buffer mirror published stale "
              "native image bytes");
      Require(name, "Buffer-superseded exact content",
              ReadBuffer(name, exact_buffer_readback, 1) ==
                  std::vector<u32>{exact_buffer_value},
              "exact-format recreation initialized from "
              "stale guest bytes");
      Require(name, "partial-page CPU refresh content",
              ReadBuffer(name, partial_cpu_refresh_readback,
                         partial_cpu_refresh_expected.size()) ==
                  partial_cpu_refresh_expected,
              "cached buffer uploaded bytes outside the exact "
              "staged guest range");
      const auto mixed_source_words = ReadBuffer(
          name, mixed_source_readback, mixed_source_size / sizeof(u32));
      Require(name, "mixed-page image content",
              mixed_source_words.front() == mixed_cpu_value &&
                  mixed_source_words[0x1000 / sizeof(u32)] == mixed_gpu_value,
              "mixed CPU/GPU source upload lost one "
              "ownership domain");
      const auto bridged_source_words = ReadBuffer(
          name, bridged_source_readback, bridged_source_size / sizeof(u32));
      Require(name, "bridged-owner image content",
              bridged_source_words.front() == bridged_gpu_value &&
                  bridged_source_words.back() == bridged_cpu_value,
              "image upload bridging separate cache owners lost a GPU or CPU "
              "ownership domain");
      Require(name, "BGRA16 content",
              ReadBuffer(name, bgra16_readback, 2) ==
                  std::vector<u32>{0x40004200u, 0x44003c00u},
              "GPU BGRA16 swap did not publish RGBA half-word order");
      scheduler.Finish();
      const auto layered_words = ReadBuffer(name, layered_readback, 8);
      const std::array<float, 8> layered_expected{
          layered_values[0], layered_values[1], layered_values[2],
          layered_values[3], layered_values[4], layered_values[5],
          layered_values[6], layered_values[7]};
      bool layered_content = layered_words.size() == layered_expected.size();
      for (uint32_t index = 0;
           layered_content && index < layered_expected.size(); index++) {
        layered_content &=
            layered_words[index] == std::bit_cast<u32>(layered_expected[index]);
      }
      Require(name, "layered depth common-mip content", layered_content,
              "depth/color conversion changed a shared array-layer value");
      const std::array<std::array<float, 4>, 2> expected_ms_depth{{
          {0x2000 / 65535.0f, 0xe000 / 65535.0f, 0.0f, 0.0f},
          {0.0f, 0x4000 / 65535.0f, 0x8000 / 65535.0f, 1.0f},
      }};
      for (uint32_t image = 0; image < expected_ms_depth.size(); image++) {
        const auto words = ReadBuffer(name, ms_observer_outputs[image], 4);
        bool content = words.size() == 4;
        for (uint32_t sample = 0; content && sample < observed_samples[image];
             sample++) {
          const float actual = std::bit_cast<float>(words[sample]);
          content &= std::abs(actual - expected_ms_depth[image][sample]) <=
                     1.5f / 65535.0f;
        }
        Require(name,
                image == 0 ? "two-sample depth content"
                           : "four-sample depth content",
                content,
                "color-to-multisample-depth changed a packed sample value");
      }

      auto unrelated_stencil_alias = MakeLinearDesc(
          base + second_stencil_offset, ms_stencil_size, vk::Format::eR32Uint,
          Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
          {static_cast<uint32_t>(ms_stencil_size / sizeof(uint32_t)), 1, 1}, 1,
          4, 1);
      const auto unrelated_stencil_image =
          texture_cache.FindImage(unrelated_stencil_alias);
      Require(name, "stencil association filtering",
              unrelated_stencil_image &&
                  unrelated_stencil_image != ms_depth_image &&
                  !texture_cache.GetImage(unrelated_stencil_image).depth_id,
              "an unrelated color alias was hijacked by the stencil "
              "association record");

      texture_cache.UnmapMemory(base + second_stencil_offset, ms_stencil_size);
      Require(name, "stencil proxy unmap",
              !texture_cache.FindImageFromRange(base + second_stencil_offset,
                                                ms_stencil_size, false) &&
                  texture_cache.GetImage(ms_depth_image).backing.image ==
                      ms_depth_backing,
              "stencil unmap removed the depth backing or retained its proxy");
      const auto reassociated_depth =
          texture_cache.FindImage(switched_ms_depth);
      TextureCacheTestAccess::AssociateStencil(
          texture_cache, reassociated_depth, switched_ms_depth.info.stencil);
      const auto reassociated_proxy = texture_cache.FindImageFromRange(
          base + second_stencil_offset, ms_stencil_size, false);
      Require(name, "stencil proxy reassociation",
              reassociated_depth == ms_depth_image && reassociated_proxy &&
                  texture_cache.GetImage(reassociated_proxy).depth_id ==
                      ms_depth_image,
              "stencil proxy did not re-associate with the surviving depth");
      texture_cache.UnmapMemory(ms_depth_desc.info.data.address,
                                ms_depth_desc.info.data.size);
      Require(name, "depth unmap proxy retirement",
              !texture_cache.FindImageFromRange(base + ms_stencil_offset,
                                                ms_stencil_size, false) &&
                  !texture_cache.FindImageFromRange(base + second_stencil_offset,
                                                    ms_stencil_size, false),
              "unmapping depth retained one of its stencil proxies");

      auto reused_stencil_color = MakeLinearDesc(
          base + 0x2200000, 0x40000, vk::Format::eR8Unorm,
          Prospero::BufferFormat::k8UNorm, Prospero::ImageType::kColor2D,
          {480, 408, 1}, 1, 1, 1);
      const auto reused_stencil_color_id =
          texture_cache.FindImage(reused_stencil_color);
      auto reused_stencil_depth = MakeLinearDesc(
          base + 0x2240000, 0x10000, vk::Format::eD32SfloatS8Uint,
          Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
          {128, 128, 1}, 1, 4, 1);
      reused_stencil_depth.type = BindingType::DepthTarget;
      reused_stencil_depth.info.stencil = {reused_stencil_color.info.data.address,
                                          0x10000};
      reused_stencil_depth.view_info.aspect =
          vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
      reused_stencil_depth.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto reused_stencil_depth_id =
          texture_cache.FindImage(reused_stencil_depth);
      TextureCacheTestAccess::AssociateStencil(
          texture_cache, reused_stencil_depth_id,
          reused_stencil_depth.info.stencil);
      const auto reused_stencil_proxy = texture_cache.FindImageFromRange(
          reused_stencil_depth.info.stencil.address,
          reused_stencil_depth.info.stencil.size, false);
      Require(name, "stencil address reuse preserves color",
              reused_stencil_proxy &&
                  reused_stencil_proxy != reused_stencil_color_id &&
                  !texture_cache.GetImage(reused_stencil_color_id).depth_id &&
                  texture_cache.GetImage(reused_stencil_proxy).depth_id ==
                      reused_stencil_depth_id &&
                  texture_cache.GetImage(reused_stencil_proxy).info.data ==
                      reused_stencil_depth.info.stencil &&
                  texture_cache.GetImage(reused_stencil_proxy).info.extent ==
                      reused_stencil_depth.info.extent,
              "a smaller stencil plane took ownership of an existing R8 image");
      vk::ClearValue reused_stencil_color_clear{};
      reused_stencil_color_clear.color.float32 =
          std::array{1.0f, 0.0f, 0.0f, 0.0f};
      TextureCacheTestAccess::ClearImage(
          texture_cache, scheduler.Current(), reused_stencil_color_id,
          {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1},
          reused_stencil_color_clear);
      Require(name, "reused stencil plane clear",
              texture_cache.ClearImageFromBuffer(
                  scheduler.Current(), reused_stencil_depth.info.stencil.address,
                  reused_stencil_depth.info.stencil.size, 0x5a5a5a5au) &&
                  texture_cache.GetImage(reused_stencil_depth_id).IsGpuModified(),
              "the separate stencil association did not resolve to depth");
      Require(name, "color clear after stencil address reuse",
              ReadCachedTexel(name, context, reused_stencil_color_id, {},
                              {4, 1, 1}) == std::vector<u32>{0xffffffffu},
              "stencil address reuse changed the native R8 clear contents");
      auto successor_depth = reused_stencil_depth;
      successor_depth.info.data.address += successor_depth.info.data.size;
      const auto successor_depth_id = texture_cache.FindImage(successor_depth);
      TextureCacheTestAccess::AssociateStencil(
          texture_cache, successor_depth_id, successor_depth.info.stencil);
      texture_cache.UnmapMemory(reused_stencil_depth.info.data.address,
                                reused_stencil_depth.info.data.size);
      Require(name, "reparented stencil survives old depth retirement",
              !TextureCacheTestAccess::Contains(texture_cache,
                                                reused_stencil_depth_id) &&
                  TextureCacheTestAccess::Contains(texture_cache,
                                                   successor_depth_id) &&
                  texture_cache.FindImageFromRange(
                      successor_depth.info.stencil.address,
                      successor_depth.info.stencil.size, false) ==
                      reused_stencil_proxy &&
                  texture_cache.GetImage(reused_stencil_proxy).depth_id ==
                      successor_depth_id,
              "retiring the previous depth owner removed its reparented proxy");
      texture_cache.UnmapMemory(successor_depth.info.data.address,
                                successor_depth.info.data.size);
      Require(name, "reparented stencil retires with new depth",
              !TextureCacheTestAccess::Contains(texture_cache,
                                                reused_stencil_proxy),
              "retiring the new depth owner retained its reparented proxy");
      texture_cache.UnmapMemory(reused_stencil_color.info.data.address,
                                reused_stencil_color.info.data.size);

      constexpr uint64_t exact_image_offset = 0x334100;
      constexpr uint64_t dirty_sibling_offset = 0x334200;
      constexpr uint32_t dirty_sibling_value = 0xc001d00du;
      auto dirty_sibling = resources.GetBufferCache().ObtainBuffer(
          base + dirty_sibling_offset, sizeof(dirty_sibling_value), true);
      Require(name, "same-page dirty sibling allocation",
              dirty_sibling.first != nullptr,
              "failed to create the disjoint same-page Buffer owner");
      resources.GetBufferCache().FillBuffer(base + dirty_sibling_offset,
                                            sizeof(dirty_sibling_value),
                                            dirty_sibling_value, false);
      auto exact_image_desc =
          MakeLinearDesc(base + exact_image_offset, sizeof(uint32_t),
                         vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
      const auto exact_image = texture_cache.FindImage(exact_image_desc);
      (void)texture_cache.FindTexture(exact_image, exact_image_desc);
      texture_cache.MarkGpuWritten(exact_image);
      Require(name, "same-page exact alias ownership",
              resources.GetBufferCache().HasGpuDirtyBytes(
                  base + dirty_sibling_offset, sizeof(dirty_sibling_value)) &&
                  !resources.GetBufferCache().HasGpuDirtyBytes(
                      base + exact_image_offset, sizeof(uint32_t)) &&
                  texture_cache.GetImage(exact_image).IsGpuModified(),
              "image ownership discarded or conflicted with disjoint dirty "
              "Buffer bytes on the same tracker page");
      auto exact_buffer = resources.GetBufferCache().ObtainBuffer(
          base + exact_image_offset, sizeof(uint32_t), true);
      Require(name, "overlapping buffer ownership allocation",
              exact_buffer.first != nullptr,
              "failed to create the overlapping dirty Buffer owner");
      exact_buffer.first->Fill(exact_buffer.second, sizeof(uint32_t),
                               dirty_sibling_value);
      texture_cache.InvalidateMemoryFromGPU(base + exact_image_offset,
                                            sizeof(uint32_t));
      Libs::Graphics::Buffer exact_download(
          m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
          base + exact_image_offset, AllFlags, sizeof(uint32_t));
      Require(name, "overlapping buffer ownership rejection",
              !texture_cache.FindImageFromRange(base + exact_image_offset,
                                                sizeof(uint32_t)) &&
                  !TextureCacheTestAccess::TryDownload(texture_cache,
                                                       exact_image) &&
                  !BufferCacheTestAccess::SynchronizeBufferFromImage(
                      resources.GetBufferCache(), exact_download,
                      base + exact_image_offset, sizeof(uint32_t)),
              "dirty Buffer bytes did not block the stale image source");
      resources.GetBufferCache().ReadMemory(base + exact_image_offset,
                                            sizeof(uint32_t));
      resources.GetBufferCache().ReadMemory(base + dirty_sibling_offset,
                                            sizeof(dirty_sibling_value));

      constexpr std::array<uint64_t, 2> gc_image_offsets{0x330000, 0x332000};
      constexpr std::array<uint32_t, 2> gc_image_values{0x76543210u,
                                                        0x89abcdefu};
      constexpr std::array<uint32_t, 2> gc_stale_values{0x10293847u,
                                                        0x56473829u};
      for (size_t index = 0; index < gc_image_offsets.size(); index++) {
        std::memcpy(memory + gc_image_offsets[index], &gc_image_values[index],
                    sizeof(uint32_t));
      }
      auto gc_image_desc_a =
          MakeLinearDesc(base + gc_image_offsets[0], sizeof(uint32_t),
                         vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
      auto gc_image_desc_b = gc_image_desc_a;
      gc_image_desc_b.info.data.address = base + gc_image_offsets[1];
      auto clean_buffer_alias = resources.GetBufferCache().ObtainBuffer(
          gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size,
          true);
      Require(name, "clean pre-image buffer alias",
              clean_buffer_alias.first != nullptr,
              "failed to create the clean cached Buffer alias");
      resources.GetBufferCache().ReadMemory(gc_image_desc_a.info.data.address,
                                            gc_image_desc_a.info.data.size);
      const std::array gc_images{texture_cache.FindImage(gc_image_desc_a),
                                 texture_cache.FindImage(gc_image_desc_b)};
      Require(
          name, "non-GPU image range validity",
          !texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address,
                                            gc_image_desc_a.info.data.size),
          "FindImageFromRange accepted an image without current GPU "
          "contents");
      (void)texture_cache.FindTexture(gc_images[0], gc_image_desc_a);
      (void)texture_cache.FindTexture(gc_images[1], gc_image_desc_b);
      for (const auto image : gc_images) {
        texture_cache.MarkGpuWritten(image);
      }
      Require(name, "GPU image range validity",
              texture_cache.FindImageFromRange(
                  gc_image_desc_a.info.data.address,
                  gc_image_desc_a.info.data.size) == gc_images[0],
              "FindImageFromRange rejected a clean GPU-current image");
      auto &gc_native = texture_cache.GetImage(gc_images[0]);
      texture_cache.InvalidateMemory(gc_image_desc_a.info.data.address,
                                     gc_image_desc_a.info.data.size);
      Require(
          name, "CPU-dirty image range validity",
          !texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address,
                                            gc_image_desc_a.info.data.size) &&
              !TextureCacheTestAccess::TryDownload(texture_cache, gc_images[0]) &&
              gc_native.IsDefinitelyCpuDirty() && gc_native.IsGpuModified() &&
              !gc_native.IsTracked(),
          "CPU writes did not supersede native contents while retaining GPU history");
      texture_cache.MarkGpuWritten(gc_images[0]);
      Require(name, "GPU image reacquisition",
              gc_native.SafeToDownload(),
              "a new GPU write did not reclaim image authority");
      for (size_t index = 0; index < gc_image_offsets.size(); index++) {
        Libs::LibKernel::Memory::WriteBacking(base + gc_image_offsets[index],
                                              &gc_stale_values[index],
                                              sizeof(uint32_t));
      }
      TextureCacheTestAccess::ConfigureGarbageCollection(
          texture_cache, gc_images, 17, UINT64_MAX);
      texture_cache.RunGarbageCollector();
      Require(name, "downloadable image pre-pressure retention",
              std::ranges::all_of(gc_images,
                                  [&](ImageId image) {
                                    return TextureCacheTestAccess::Contains(
                                        texture_cache, image);
                                  }),
              "GC retired safely downloadable GPU images before pressure");
      const auto gc_batch_tick = scheduler.CurrentTick();
      TextureCacheTestAccess::ConfigureGarbageCollection(texture_cache,
                                                         gc_images, 81, 0);
      texture_cache.RunGarbageCollector();
      std::array<uint32_t, 2> gc_before_completion{};
      for (size_t index = 0; index < gc_image_offsets.size(); index++) {
        Libs::LibKernel::Memory::TryReadBacking(base + gc_image_offsets[index],
                                                &gc_before_completion[index],
                                                sizeof(uint32_t));
      }
      Require(name, "batched image pressure retirement",
              std::ranges::none_of(gc_images,
                                   [&](ImageId image) {
                                     return TextureCacheTestAccess::Contains(
                                         texture_cache, image);
                                   }) &&
                  scheduler.CurrentTick() == gc_batch_tick &&
                  gc_before_completion == gc_stale_values,
              "GC submitted per image or published a readback before GPU "
              "completion");
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      auto refreshed_buffer_alias = resources.GetBufferCache().ObtainBuffer(
          gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size,
          false);
      Require(name, "post-publication Buffer reacquire",
              refreshed_buffer_alias.first != nullptr,
              "Buffer lookup failed after the retired image publication");
      std::array<uint32_t, 2> gc_after_completion{};
      for (size_t index = 0; index < gc_image_offsets.size(); index++) {
        Libs::LibKernel::Memory::TryReadBacking(base + gc_image_offsets[index],
                                                &gc_after_completion[index],
                                                sizeof(uint32_t));
      }
      Require(name, "batched image readback publication",
              scheduler.CurrentTick() == gc_batch_tick + 1 &&
                  gc_after_completion == gc_image_values,
              "one submission did not publish both deferred image readbacks");
      Require(name, "refreshed post-image buffer alias",
              refreshed_buffer_alias.first != nullptr,
              "failed to reacquire the cached Buffer alias after image "
              "publication");
      auto alias_readback = CreateHostBuffer(
          name, sizeof(uint32_t), vk::BufferUsageFlagBits::eTransferDst, {0});
      const vk::BufferCopy alias_copy{refreshed_buffer_alias.second, 0,
                                      sizeof(uint32_t)};
      scheduler.Current().Handle().copyBuffer(
          refreshed_buffer_alias.first->Handle(), alias_readback.buffer, 1,
          &alias_copy);
      HostReadBarrier(alias_readback.buffer, alias_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      scheduler.Finish();
      Require(name, "post-image Buffer alias content",
              ReadBuffer(name, alias_readback, 1) ==
                  std::vector<u32>{gc_image_values[0]},
              "a clean cached Buffer alias survived image ownership with "
              "stale native bytes");
      DestroyBuffer(&alias_readback);

      constexpr uint64_t submit_readback_offset = 0x336000;
      constexpr uint32_t submit_readback_value = 0x13579bdfu;
      constexpr uint32_t submit_readback_stale = 0x2468ace0u;
      auto submit_readback_desc =
          MakeLinearDesc(base + submit_readback_offset, sizeof(uint32_t),
                         vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
                         Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
      submit_readback_desc.type = BindingType::Storage;
      submit_readback_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
      TextureCacheTestAccess::SetLinearReadback(texture_cache, true);
      const auto submit_readback_image =
          texture_cache.FindImage(submit_readback_desc);
      auto submit_buffer = resources.GetBufferCache().ObtainBuffer(
          submit_readback_desc.info.data.address,
          submit_readback_desc.info.data.size, true);
      Require(name, "linear submit dirty buffer allocation",
              submit_buffer.first != nullptr,
              "failed to create the older GPU Buffer contents");
      submit_buffer.first->Fill(submit_buffer.second, sizeof(uint32_t),
                                submit_readback_stale);
      texture_cache.InvalidateMemoryFromGPU(
          submit_readback_desc.info.data.address,
          submit_readback_desc.info.data.size);
      (void)texture_cache.FindTexture(submit_readback_image,
                                      submit_readback_desc);
      Require(name, "linear submit image clear",
              texture_cache.ClearImageFromBuffer(
                  command, submit_readback_desc.info.data.address,
                  submit_readback_desc.info.data.size, submit_readback_value),
              "failed to create GPU-current linear storage contents");
      Require(name, "older buffer permits current image readback",
              resources.GetBufferCache().HasGpuDirtyBytes(
                  submit_readback_desc.info.data.address,
                  submit_readback_desc.info.data.size) &&
                  texture_cache.FindImageFromRange(
                      submit_readback_desc.info.data.address,
                      submit_readback_desc.info.data.size) ==
                      submit_readback_image,
              "historical GPU Buffer dirtiness rejected newer image contents");
      TextureCacheTestAccess::TrackDownload(texture_cache,
                                            submit_readback_image);
      Libs::LibKernel::Memory::WriteBacking(
          submit_readback_desc.info.data.address, &submit_readback_stale,
          sizeof(submit_readback_stale));
      const auto submit_readback_tick = scheduler.CurrentTick();
      texture_cache.ProcessDownloadImages();
      TextureCacheTestAccess::SetLinearReadback(texture_cache, false);
      uint32_t submit_before_completion = 0;
      Libs::LibKernel::Memory::TryReadBacking(
          submit_readback_desc.info.data.address, &submit_before_completion,
          sizeof(submit_before_completion));
      Require(name, "linear submit deferred publication",
              submit_before_completion == submit_readback_stale &&
                  scheduler.CurrentTick() == submit_readback_tick,
              "submit-time linear readback published synchronously");
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      uint32_t submit_after_completion = 0;
      Libs::LibKernel::Memory::TryReadBacking(
          submit_readback_desc.info.data.address, &submit_after_completion,
          sizeof(submit_after_completion));
      Require(name, "linear submit readback reuse",
              submit_after_completion == submit_readback_value &&
                  TextureCacheTestAccess::Contains(texture_cache,
                                                   submit_readback_image) &&
                  texture_cache.GetImage(submit_readback_image).IsGpuModified(),
              "submit-time readback did not publish or retained no reusable "
              "GPU-current image");

      constexpr uint64_t linear_depth_offset = 0x33a000;
      constexpr uint64_t linear_stencil_offset = 0x33b000;
      constexpr uint32_t linear_depth_width = 3;
      constexpr uint32_t linear_depth_height = 2;
      constexpr uint32_t linear_depth_pitch = 4;
      constexpr uint32_t linear_depth_layers = 2;
      constexpr uint32_t linear_depth_words =
          linear_depth_pitch * linear_depth_height * linear_depth_layers;
      constexpr uint32_t linear_depth_clear = 0x3f400000u;
      std::array<uint32_t, linear_depth_words> linear_depth_guest{};
      for (uint32_t index = 0; index < linear_depth_words; index++) {
        linear_depth_guest[index] = 0x51000000u + index;
      }
      constexpr std::array<uint8_t, 16> linear_stencil_guest{
          0x91, 0x82, 0x73, 0x64, 0x55, 0x46, 0x37, 0x28,
          0x19, 0x2a, 0x3b, 0x4c, 0x5d, 0x6e, 0x7f, 0x80};
      std::memcpy(memory + linear_depth_offset, linear_depth_guest.data(),
                  sizeof(linear_depth_guest));
      std::memcpy(memory + linear_stencil_offset, linear_stencil_guest.data(),
                  sizeof(linear_stencil_guest));
      auto linear_depth_desc = MakeLinearDesc(
          base + linear_depth_offset, sizeof(linear_depth_guest),
          vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k32Float,
          Prospero::ImageType::kColor2D,
          {linear_depth_width, linear_depth_height, 1}, linear_depth_layers, 4,
          1);
      linear_depth_desc.type = BindingType::DepthTarget;
      linear_depth_desc.info.pitch = linear_depth_pitch;
      linear_depth_desc.info.stencil = {base + linear_stencil_offset,
                                        sizeof(linear_stencil_guest)};
      linear_depth_desc.info.mip_layout[0] = {0, sizeof(linear_depth_guest),
                                              linear_depth_pitch,
                                              linear_depth_height};
      linear_depth_desc.info.UpdateSize();
      linear_depth_desc.view_info.aspect =
          vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
      linear_depth_desc.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto linear_depth_image =
          texture_cache.FindImage(linear_depth_desc);
      Require(name, "linear depth native clear",
              texture_cache.ClearImageFromBuffer(
                  command, linear_depth_desc.info.data.address,
                  linear_depth_desc.info.data.size, linear_depth_clear),
              "failed to create GPU-current padded linear depth contents");
      TextureCacheTestAccess::SetLinearReadback(texture_cache, true);
      TextureCacheTestAccess::TrackDownload(texture_cache, linear_depth_image);
      const auto linear_depth_tick = scheduler.CurrentTick();
      texture_cache.ProcessDownloadImages();
      TextureCacheTestAccess::SetLinearReadback(texture_cache, false);
      std::array<uint32_t, linear_depth_words> linear_depth_before{};
      std::memcpy(linear_depth_before.data(), memory + linear_depth_offset,
                  sizeof(linear_depth_before));
      Require(name, "linear depth deferred publication",
              linear_depth_before == linear_depth_guest &&
                  scheduler.CurrentTick() == linear_depth_tick,
              "linear depth readback published before GPU completion");
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      std::array<uint32_t, linear_depth_words> linear_depth_after{};
      std::array<uint8_t, linear_stencil_guest.size()> linear_stencil_after{};
      std::memcpy(linear_depth_after.data(), memory + linear_depth_offset,
                  sizeof(linear_depth_after));
      std::memcpy(linear_stencil_after.data(), memory + linear_stencil_offset,
                  sizeof(linear_stencil_after));
      bool linear_depth_matches = true;
      uint32_t linear_depth_mismatch = UINT32_MAX;
      for (uint32_t layer = 0; layer < linear_depth_layers; layer++) {
        for (uint32_t y = 0; y < linear_depth_height; y++) {
          for (uint32_t x = 0; x < linear_depth_pitch; x++) {
            const auto index =
                layer * linear_depth_pitch * linear_depth_height +
                y * linear_depth_pitch + x;
            const auto expected = x < linear_depth_width
                                      ? linear_depth_clear
                                      : linear_depth_guest[index];
            if (linear_depth_after[index] != expected &&
                linear_depth_mismatch == UINT32_MAX) {
              linear_depth_mismatch = index;
            }
            linear_depth_matches &= linear_depth_after[index] == expected;
          }
        }
      }
      Require(name, "linear depth native contents",
              linear_depth_matches &&
                  linear_stencil_after == linear_stencil_guest &&
                  TextureCacheTestAccess::Contains(texture_cache,
                                                   linear_depth_image) &&
                  texture_cache.GetImage(linear_depth_image).IsGpuModified(),
              fmt::format(
                  "linear depth mismatch={} actual=0x{:08x} expected=0x{:08x} "
                  "stencil={} contains={} gpu={}",
                  linear_depth_mismatch,
                  linear_depth_mismatch == UINT32_MAX
                      ? 0
                      : linear_depth_after[linear_depth_mismatch],
                  linear_depth_mismatch == UINT32_MAX
                      ? 0
                      : (linear_depth_mismatch % linear_depth_pitch <
                                 linear_depth_width
                             ? linear_depth_clear
                             : linear_depth_guest[linear_depth_mismatch]),
                  linear_stencil_after == linear_stencil_guest,
                  TextureCacheTestAccess::Contains(texture_cache,
                                                   linear_depth_image),
                  texture_cache.GetImage(linear_depth_image).IsGpuModified())
                  .c_str());

      constexpr uint64_t tiled_depth_offset = 0x380000;
      constexpr uint32_t tiled_depth_width = 3;
      constexpr uint32_t tiled_depth_height = 2;
      constexpr uint32_t tiled_depth_pitch = 4;
      constexpr uint32_t tiled_depth_layers = 2;
      constexpr uint64_t tiled_depth_slice = 0x10000;
      constexpr uint64_t tiled_depth_size =
          tiled_depth_slice * tiled_depth_layers;
      constexpr uint32_t tiled_depth_clear = 0x3f200000u;
      constexpr uint32_t tiled_depth_stale = 0xdeadbeefu;
      std::vector<uint32_t> tiled_depth_guest(
          tiled_depth_size / sizeof(uint32_t), tiled_depth_stale);
      std::memcpy(memory + tiled_depth_offset, tiled_depth_guest.data(),
                  tiled_depth_size);
      auto tiled_depth_desc = MakeLinearDesc(
          base + tiled_depth_offset, tiled_depth_size, vk::Format::eD32Sfloat,
          Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
          {tiled_depth_width, tiled_depth_height, 1}, tiled_depth_layers, 4, 1);
      tiled_depth_desc.type = BindingType::DepthTarget;
      tiled_depth_desc.info.pitch = tiled_depth_pitch;
      tiled_depth_desc.info.tile_mode = Prospero::TileMode::kDepth;
      tiled_depth_desc.info.mip_layout[0] = {
          0, tiled_depth_size, tiled_depth_pitch, tiled_depth_height};
      tiled_depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
      tiled_depth_desc.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      tiled_depth_desc.info.UpdateSize();
      const auto tiled_depth_image = texture_cache.FindImage(tiled_depth_desc);
      Require(name, "tiled depth native clear",
              texture_cache.ClearImageFromBuffer(
                  command, tiled_depth_desc.info.data.address,
                  tiled_depth_desc.info.data.size, tiled_depth_clear),
              "failed to create GPU-current layered tiled depth contents");
      const auto [tiled_prefix, tiled_prefix_offset] =
          TextureCacheTestAccess::MapDownload(texture_cache, 64, 64);
      Require(name, "tiled depth download prefix",
              tiled_prefix != nullptr && tiled_prefix_offset != 0,
              "failed to force a nonzero reusable-download offset");
      const auto tiled_depth_tick = scheduler.CurrentTick();
      Require(
          name, "tiled depth download queue",
          TextureCacheTestAccess::TryDownload(texture_cache, tiled_depth_image),
          "layered tiled depth readback was rejected");
      std::vector<uint32_t> tiled_depth_before(tiled_depth_guest.size());
      std::memcpy(tiled_depth_before.data(), memory + tiled_depth_offset,
                  tiled_depth_size);
      Require(name, "tiled depth deferred publication",
              tiled_depth_before == tiled_depth_guest &&
                  scheduler.CurrentTick() == tiled_depth_tick,
              "tiled depth readback published before GPU completion");
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      std::vector<uint32_t> tiled_depth_after(tiled_depth_guest.size());
      std::memcpy(tiled_depth_after.data(), memory + tiled_depth_offset,
                  tiled_depth_size);
      const auto active_tiled_words =
          tiled_depth_width * tiled_depth_height * tiled_depth_layers;
      const auto clear_tiled_words =
          std::ranges::count(tiled_depth_after, tiled_depth_clear);
      const auto stale_tiled_words =
          std::ranges::count(tiled_depth_after, tiled_depth_stale);
      Require(
          name, "tiled depth backing preservation",
          clear_tiled_words == active_tiled_words &&
              stale_tiled_words ==
                  tiled_depth_after.size() - active_tiled_words,
          fmt::format("tiled depth readback changed clear={}/{} stale={}/{}",
                      clear_tiled_words, active_tiled_words, stale_tiled_words,
                      tiled_depth_after.size() - active_tiled_words)
              .c_str());

      TileBlockLayout depth_block{};
      Require(name, "tiled depth block layout",
              TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint32_t),
                                 depth_block),
              "failed to describe the tiled depth test surface");
      auto tiled_depth_info =
          MakeTilingImage(Prospero::BufferFormat::k32Float, tiled_depth_width,
                          tiled_depth_height, 1, tiled_depth_layers,
                          Prospero::TileMode::kDepth, tiled_depth_size, false);
      tiled_depth_info.linear_slice_stride = tiled_depth_slice;
      tiled_depth_info.mip_layout[0].linear_size = tiled_depth_slice;
      tiled_depth_info.mip_layout[0].linear_pitch = tiled_depth_pitch;
      auto tiled_depth_input =
          CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_depth_after);
      auto tiled_depth_linear =
          TextureCacheTestAccess::Tiler(texture_cache)
              .Detile(tiled_depth_input.buffer, 0, tiled_depth_info);
      auto tiled_depth_output =
          CreateHostBuffer(name, tiled_depth_size, AllFlags,
                           std::vector<uint32_t>(tiled_depth_guest.size(), 0));
      const vk::BufferCopy tiled_depth_copy{tiled_depth_linear.offset, 0,
                                            tiled_depth_size};
      scheduler.Current().Handle().copyBuffer(tiled_depth_linear.buffer,
                                              tiled_depth_output.buffer, 1,
                                              &tiled_depth_copy);
      HostReadBarrier(tiled_depth_output.buffer, tiled_depth_size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      scheduler.Finish();
      const auto tiled_depth_words =
          ReadBuffer(name, tiled_depth_output, tiled_depth_guest.size());
      bool tiled_depth_matches = true;
      for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) {
        for (uint32_t y = 0; y < tiled_depth_height; y++) {
          for (uint32_t x = 0; x < tiled_depth_width; x++) {
            const auto index = layer * (tiled_depth_slice / sizeof(uint32_t)) +
                               y * tiled_depth_pitch + x;
            tiled_depth_matches &=
                tiled_depth_words[index] == tiled_depth_clear;
          }
        }
      }
      Require(name, "tiled depth layered contents",
              tiled_depth_matches &&
                  TextureCacheTestAccess::Contains(texture_cache,
                                                   tiled_depth_image) &&
                  texture_cache.GetImage(tiled_depth_image).IsGpuModified(),
              "tiled depth readback lost a padded slice or retired its image");
      DestroyBuffer(&tiled_depth_output);
      DestroyBuffer(&tiled_depth_input);

      constexpr uint64_t tiled_d16_offset = 0x3a0000;
      constexpr uint64_t tiled_d16_stencil_offset = 0x3c0000;
      constexpr uint16_t tiled_d16_clear = 0x8000u;
      constexpr uint16_t tiled_d16_stale = 0xbeefu;
      std::vector<uint16_t> tiled_d16_guest(tiled_depth_size / sizeof(uint16_t),
                                            tiled_d16_stale);
      constexpr std::array<uint8_t, 16> tiled_d16_stencil{
          0x10, 0x21, 0x32, 0x43, 0x54, 0x65, 0x76, 0x87,
          0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f};
      std::memcpy(memory + tiled_d16_offset, tiled_d16_guest.data(),
                  tiled_depth_size);
      std::memcpy(memory + tiled_d16_stencil_offset, tiled_d16_stencil.data(),
                  tiled_d16_stencil.size());
      auto tiled_d16_desc = MakeLinearDesc(
          base + tiled_d16_offset, tiled_depth_size,
          vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k16UNorm,
          Prospero::ImageType::kColor2D,
          {tiled_depth_width, tiled_depth_height, 1}, tiled_depth_layers, 2, 1);
      tiled_d16_desc.type = BindingType::DepthTarget;
      tiled_d16_desc.info.pitch = tiled_depth_pitch;
      tiled_d16_desc.info.tile_mode = Prospero::TileMode::kDepth;
      tiled_d16_desc.info.stencil = {base + tiled_d16_stencil_offset,
                                     tiled_d16_stencil.size()};
      tiled_d16_desc.info.mip_layout[0] = {
          0, tiled_depth_size, tiled_depth_pitch, tiled_depth_height};
      tiled_d16_desc.view_info.aspect =
          vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
      tiled_d16_desc.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      tiled_d16_desc.info.UpdateSize();
      const auto tiled_d16_image = texture_cache.FindImage(tiled_d16_desc);
      Require(name, "tiled D16 native clear",
              texture_cache.ClearImageFromBuffer(
                  command, tiled_d16_desc.info.data.address,
                  tiled_d16_desc.info.data.size,
                  EncodeD16AsD32(tiled_d16_clear)),
              "failed to clear a layered tiled D16 fallback image");
      Require(
          name, "tiled D16 download queue",
          TextureCacheTestAccess::TryDownload(texture_cache, tiled_d16_image),
          "layered tiled D16 fallback readback was rejected");
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      std::vector<uint16_t> tiled_d16_after(tiled_d16_guest.size());
      std::array<uint8_t, tiled_d16_stencil.size()> tiled_d16_stencil_after{};
      std::memcpy(tiled_d16_after.data(), memory + tiled_d16_offset,
                  tiled_depth_size);
      std::memcpy(tiled_d16_stencil_after.data(),
                  memory + tiled_d16_stencil_offset,
                  tiled_d16_stencil_after.size());
      Require(name, "tiled D16 backing preservation",
              std::ranges::count(tiled_d16_after, tiled_d16_clear) ==
                      active_tiled_words &&
                  std::ranges::count(tiled_d16_after, tiled_d16_stale) ==
                      tiled_d16_after.size() - active_tiled_words &&
                  tiled_d16_stencil_after == tiled_d16_stencil,
              "tiled D16 fallback overwrote inactive depth or stencil backing");

      std::vector<uint32_t> tiled_d16_words(tiled_depth_size /
                                            sizeof(uint32_t));
      std::memcpy(tiled_d16_words.data(), tiled_d16_after.data(),
                  tiled_depth_size);
      Require(name, "tiled D16 block layout",
              TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint16_t),
                                 depth_block),
              "failed to describe the tiled D16 test surface");
      auto tiled_d16_info =
          MakeTilingImage(Prospero::BufferFormat::k16UNorm, tiled_depth_width,
                          tiled_depth_height, 1, tiled_depth_layers,
                          Prospero::TileMode::kDepth, tiled_depth_size, false);
      tiled_d16_info.linear_slice_stride = tiled_depth_slice;
      tiled_d16_info.mip_layout[0].linear_size = tiled_depth_slice;
      tiled_d16_info.mip_layout[0].linear_pitch = tiled_depth_pitch;
      auto tiled_d16_input =
          CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_d16_words);
      auto tiled_d16_linear =
          TextureCacheTestAccess::Tiler(texture_cache)
              .Detile(tiled_d16_input.buffer, 0, tiled_d16_info);
      auto tiled_d16_output =
          CreateHostBuffer(name, tiled_depth_size, AllFlags,
                           std::vector<uint32_t>(tiled_d16_words.size(), 0));
      const vk::BufferCopy tiled_d16_copy{tiled_d16_linear.offset, 0,
                                          tiled_depth_size};
      scheduler.Current().Handle().copyBuffer(
          tiled_d16_linear.buffer, tiled_d16_output.buffer, 1, &tiled_d16_copy);
      HostReadBarrier(tiled_d16_output.buffer, tiled_depth_size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      scheduler.Finish();
      const auto tiled_d16_linear_words =
          ReadBuffer(name, tiled_d16_output, tiled_d16_words.size());
      std::vector<uint16_t> tiled_d16_linear_values(tiled_d16_guest.size());
      std::memcpy(tiled_d16_linear_values.data(), tiled_d16_linear_words.data(),
                  tiled_depth_size);
      bool tiled_d16_matches = true;
      for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) {
        for (uint32_t y = 0; y < tiled_depth_height; y++) {
          for (uint32_t x = 0; x < tiled_depth_width; x++) {
            const auto index = layer * (tiled_depth_slice / sizeof(uint16_t)) +
                               y * tiled_depth_pitch + x;
            tiled_d16_matches &=
                tiled_d16_linear_values[index] == tiled_d16_clear;
          }
        }
      }
      Require(name, "tiled D16 layered contents",
              tiled_d16_matches &&
                  TextureCacheTestAccess::Contains(texture_cache,
                                                   tiled_d16_image) &&
                  texture_cache.GetImage(tiled_d16_image).IsGpuModified(),
              "tiled D16 fallback lost a converted layer or retired its image");
      DestroyBuffer(&tiled_d16_output);
      DestroyBuffer(&tiled_d16_input);

      constexpr uint64_t d16_fallback_offset = 0x33c000;
      constexpr uint64_t d16_fallback_stencil_offset = 0x33d000;
      constexpr std::array<uint16_t, 4> d16_fallback_values{0x0000u, 0x2468u,
                                                            0xabcdu, 0xffffu};
      std::memcpy(memory + d16_fallback_offset, d16_fallback_values.data(),
                  sizeof(d16_fallback_values));
      std::memset(memory + d16_fallback_stencil_offset, 0x6d, 4);
      auto d16_depth_desc = MakeLinearDesc(
          base + d16_fallback_offset, sizeof(d16_fallback_values),
          vk::Format::eD24UnormS8Uint, Prospero::BufferFormat::k16UNorm,
          Prospero::ImageType::kColor2D, {4, 1, 1}, 1, 2, 1);
      d16_depth_desc.type = BindingType::DepthTarget;
      d16_depth_desc.info.stencil = {base + d16_fallback_stencil_offset, 4};
      d16_depth_desc.view_info.aspect =
          vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
      d16_depth_desc.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto d16_depth_image = texture_cache.FindImage(d16_depth_desc);
      (void)texture_cache.FindDepthTarget(d16_depth_image, d16_depth_desc);
      texture_cache.MarkGpuWritten(d16_depth_image);
      auto d16_storage_desc = d16_depth_desc;
      d16_storage_desc.type = BindingType::Storage;
      d16_storage_desc.info.stencil = {};
      d16_storage_desc.info.pixel_format = vk::Format::eR16Unorm;
      d16_storage_desc.view_info.format = vk::Format::eR16Unorm;
      d16_storage_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
      d16_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
      const auto d16_storage_image =
          texture_cache.FindImage(d16_storage_desc, true);
      auto d16_storage_readback = CreateHostBuffer(
          name, sizeof(d16_fallback_values),
          vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>(2, 0));
      auto &d16_storage_native = texture_cache.GetImage(d16_storage_image);
      d16_storage_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
                                 vk::AccessFlagBits2::eTransferRead, {},
                                 scheduler.Current().Handle());
      vk::BufferImageCopy d16_storage_copy{};
      d16_storage_copy.bufferRowLength = 4;
      d16_storage_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0,
                                           0, 1};
      d16_storage_copy.imageExtent = {4, 1, 1};
      scheduler.Current().Handle().copyImageToBuffer(
          d16_storage_native.backing.image,
          vk::ImageLayout::eTransferSrcOptimal, d16_storage_readback.buffer, 1,
          &d16_storage_copy);
      HostReadBarrier(d16_storage_readback.buffer, d16_storage_readback.size,
                      vk::PipelineStageFlagBits::eTransfer,
                      vk::AccessFlagBits::eTransferWrite);
      scheduler.Finish();
      const auto d16_storage_words = ReadBuffer(name, d16_storage_readback, 2);
      std::array<uint16_t, 4> d16_storage_values{};
      std::memcpy(d16_storage_values.data(), d16_storage_words.data(),
                  sizeof(d16_storage_values));
      Require(
          name, "D16 fallback depth/storage copy",
          d16_storage_values == d16_fallback_values &&
              d16_storage_image != d16_depth_image &&
              texture_cache.GetImage(d16_storage_image).IsGpuModified(),
          "D16 fallback reinterpreted four-byte host depth as two-byte color");
      DestroyBuffer(&d16_storage_readback);

      if (!texture_cache.GetImage(combined_destination_image).IsGpuModified()) {
        texture_cache.MarkGpuWritten(combined_destination_image);
      }
      constexpr float stale_added_stencil_depth = 0.125f;
      Libs::LibKernel::Memory::WriteBacking(
          combined_destination.info.data.address, &stale_added_stencil_depth,
          sizeof(stale_added_stencil_depth));
      const auto depth_gc_tick = scheduler.CurrentTick();
      TextureCacheTestAccess::ConfigureGarbageCollection(
          texture_cache, std::array{combined_destination_image}, 81, 0);
      texture_cache.RunGarbageCollector();
      float depth_before_completion = 0.0f;
      Libs::LibKernel::Memory::TryReadBacking(
          combined_destination.info.data.address, &depth_before_completion,
          sizeof(depth_before_completion));
      const bool depth_image_retired = !TextureCacheTestAccess::Contains(
          texture_cache, combined_destination_image);
      const bool depth_proxy_retired = !texture_cache.FindImageFromRange(
          base + added_stencil_offset, added_stencil_size, false);
      Require(
          name, "depth/stencil deferred pressure retirement",
          depth_image_retired && depth_proxy_retired &&
              scheduler.CurrentTick() == depth_gc_tick &&
              depth_before_completion == stale_added_stencil_depth,
          fmt::format(
              "GC failed to retire/defer depth: image={} proxy={} tick={}/{} "
              "depth={}/{}",
              depth_image_retired, depth_proxy_retired, scheduler.CurrentTick(),
              depth_gc_tick, depth_before_completion, stale_added_stencil_depth)
              .c_str());
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      float depth_after_completion = 0.0f;
      Libs::LibKernel::Memory::TryReadBacking(
          combined_destination.info.data.address, &depth_after_completion,
          sizeof(depth_after_completion));
      Require(name, "depth/stencil depth-plane preservation",
              depth_after_completion == added_stencil_depth_value,
              "GC discarded the current depth plane of a depth/stencil "
              "image");

      constexpr size_t gc_depth_pair_count = 6;
      constexpr uint64_t gc_depth_pair_offset = 0x350000;
      constexpr uint64_t gc_depth_pair_stride = 0x1000;
      std::array<ImageId, gc_depth_pair_count> gc_depth_images{};
      std::array<ImageId, gc_depth_pair_count> gc_stencil_images{};
      std::array<ImageId, gc_depth_pair_count * 2> gc_depth_lru{};
      for (size_t index = 0; index < gc_depth_pair_count; index++) {
        auto depth = MakeLinearDesc(
            base + gc_depth_pair_offset + index * gc_depth_pair_stride,
            sizeof(float), vk::Format::eD32SfloatS8Uint,
            Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
            {1, 1, 1}, 1, 4, 1);
        depth.type = BindingType::DepthTarget;
        depth.info.stencil = {depth.info.data.address +
                                  gc_depth_pair_stride / 2,
                              sizeof(uint32_t)};
        depth.view_info.format = vk::Format::eD32SfloatS8Uint;
        depth.view_info.aspect =
            vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
        depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
        gc_depth_images[index] = texture_cache.FindImage(depth);
        TextureCacheTestAccess::AssociateStencil(
            texture_cache, gc_depth_images[index], depth.info.stencil);
        gc_stencil_images[index] = texture_cache.FindImageFromRange(
            depth.info.stencil.address, depth.info.stencil.size, false);
        gc_depth_lru[index * 2] = gc_depth_images[index];
        gc_depth_lru[index * 2 + 1] = gc_stencil_images[index];
      }
      Require(
          name, "depth/stencil GC fixtures",
          std::ranges::all_of(gc_depth_images,
                              [](ImageId image) { return bool{image}; }) &&
              std::ranges::all_of(gc_stencil_images,
                                  [](ImageId image) { return bool{image}; }),
          "failed to create six depth/stencil association pairs");
      TextureCacheTestAccess::ConfigureGarbageCollection(
          texture_cache, gc_depth_lru, 81, UINT64_MAX);
      texture_cache.RunGarbageCollector();
      bool gc_depth_budget = true;
      for (size_t index = 0; index < gc_depth_pair_count; index++) {
        const bool expected_live = index == gc_depth_pair_count - 1;
        gc_depth_budget &=
            TextureCacheTestAccess::Contains(
                texture_cache, gc_depth_images[index]) == expected_live &&
            TextureCacheTestAccess::Contains(
                texture_cache, gc_stencil_images[index]) == expected_live;
      }
      Require(name, "depth/stencil GC traversal budget", gc_depth_budget,
              "recursive association deletion stopped LRU traversal or "
              "exceeded the ten-entry deletion budget");

      constexpr uint64_t large_offset = 0x400000;
      constexpr uint32_t large_width = 4096;
      constexpr uint32_t large_height = 2047;
      constexpr uint64_t large_size =
          uint64_t{large_width} * large_height * sizeof(uint32_t);
      static_assert(large_size < (32ull << 20));
      std::memset(memory + large_offset, 0, static_cast<size_t>(large_size));
      auto large_desc = MakeLinearDesc(
          base + large_offset, large_size, vk::Format::eR8G8B8A8Unorm,
          Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
          {large_width, large_height, 1}, 1, 4, 1);
      const auto RunLargeReadback = [&](uint32_t clear_value) {
        const auto image = texture_cache.FindImage(large_desc);
        Require(name, "near-capacity image clear",
                texture_cache.ClearImageFromBuffer(
                    command, large_desc.info.data.address,
                    large_desc.info.data.size, clear_value),
                "failed to write the near-capacity readback source image");
        const std::array<uint64_t, 3> sample_offsets{
            0, large_size / 2, large_size - sizeof(uint32_t)};
        constexpr uint32_t stale = 0;
        for (const auto offset : sample_offsets) {
          Libs::LibKernel::Memory::WriteBacking(base + large_offset + offset,
                                                &stale, sizeof(stale));
        }
        TextureCacheTestAccess::ConfigureGarbageCollection(
            texture_cache, std::array{image}, 81, 0);
        const auto tick = scheduler.CurrentTick();
        texture_cache.RunGarbageCollector();
        const auto handle =
            BufferCacheTestAccess::DownloadBuffer(resources.GetBufferCache())
                .Handle();
        Require(
            name, "near-capacity deferred retirement",
            !TextureCacheTestAccess::Contains(texture_cache, image) &&
                scheduler.CurrentTick() == tick,
            "near-capacity readback was rejected or synchronously submitted");
        scheduler.Finish();
        scheduler.DrainPriorityOperations();
        bool content = true;
        for (const auto offset : sample_offsets) {
          uint32_t value = 0;
          Libs::LibKernel::Memory::TryReadBacking(base + large_offset + offset,
                                                  &value, sizeof(value));
          content &= value == clear_value;
        }
        Require(
            name, "near-capacity readback content", content,
            "near-capacity image readback did not publish its GPU contents");
        return handle;
      };
      const auto large_download = RunLargeReadback(0xa5a5a5a5u);
      const auto reused_large_download = RunLargeReadback(0x5a5a5a5au);
      Require(name, "near-capacity shared download reuse",
              large_download == reused_large_download,
              "successive near-capacity image transfers replaced the shared "
              "download buffer");

      constexpr uint64_t tile_alias_offset = 0x2000000;
      constexpr uint64_t tile_alias_size = 0x400000;
      constexpr uint32_t tile_alias_extent = 1024;
      std::memset(memory + tile_alias_offset, 0,
                  static_cast<size_t>(tile_alias_size));
      auto render_target_alias = MakeLinearDesc(
          base + tile_alias_offset, tile_alias_size, vk::Format::eR8G8B8A8Unorm,
          Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
          {tile_alias_extent, tile_alias_extent, 1}, 1, 4, 1);
      render_target_alias.type = BindingType::Storage;
      render_target_alias.info.tile_mode = Prospero::TileMode::kRenderTarget;
      render_target_alias.info.UpdateSize();
      render_target_alias.view_info.usage = vk::ImageUsageFlagBits::eStorage;
      const auto render_target_alias_image =
          texture_cache.FindImage(render_target_alias);

      auto standard_4kb_alias = render_target_alias;
      standard_4kb_alias.type = BindingType::Texture;
      standard_4kb_alias.info.tile_mode = Prospero::TileMode::kStandard4KB;
      standard_4kb_alias.info.UpdateSize();
      standard_4kb_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled;
      const auto standard_4kb_alias_image =
          texture_cache.FindImage(standard_4kb_alias);
      auto repeated_standard_4kb_alias = standard_4kb_alias;
      const auto repeated_standard_4kb_alias_image =
          texture_cache.FindImage(repeated_standard_4kb_alias);
      Require(
          name, "equal-size tile-mode alias",
          render_target_alias_image && standard_4kb_alias_image &&
              standard_4kb_alias_image != render_target_alias_image &&
              repeated_standard_4kb_alias_image == standard_4kb_alias_image &&
              texture_cache.GetImage(render_target_alias_image)
                      .info.tile_mode == Prospero::TileMode::kRenderTarget &&
              texture_cache.GetImage(standard_4kb_alias_image).info.tile_mode ==
                  Prospero::TileMode::kStandard4KB,
          "equal address/size lookup reused an incompatible "
          "tiled backing");

      for (auto &output : ms_observer_outputs) {
        DestroyBuffer(&output);
      }
      DestroyBuffer(&layered_readback);
      DestroyBuffer(&bgra16_readback);
      DestroyBuffer(&mixed_source_readback);
      DestroyBuffer(&bridged_source_readback);
      DestroyBuffer(&mip_formatted_readback);
      DestroyBuffer(&mip_prefix_readback);
      DestroyBuffer(&partial_cpu_refresh_readback);
      DestroyBuffer(&mirror_cpu_readback);
      DestroyBuffer(&exact_buffer_readback);
      m_device.destroyDescriptorPool(observer_pool, nullptr);
      m_device.destroyPipeline(observer_depth_pipeline, nullptr);
      m_device.destroyPipelineLayout(observer_pipeline_layout, nullptr);
      m_device.destroyDescriptorSetLayout(observer_set_layout, nullptr);
      m_device.destroyShaderModule(ms_depth_module, nullptr);

      constexpr uint64_t mapped_end = base + allocation_size;
      constexpr uint64_t final_word_address = mapped_end - sizeof(uint32_t);
      auto *final_word = reinterpret_cast<volatile uint32_t *>(final_word_address);
      auto *final_byte = reinterpret_cast<volatile uint8_t *>(mapped_end - 1);
      Require(name, "fault boundary containment",
              resources.IsMapped(final_word_address, sizeof(uint32_t)) &&
                  !resources.IsMapped(final_word_address, sizeof(uint64_t)) &&
                  !resources.HandleFault(PageFaultAccess::Write, mapped_end) &&
                  !resources.HandleFault(PageFaultAccess::Read, mapped_end),
              "fault handling changed the mapped range's exclusive end");
      Require(name, "fault boundary setup",
              resources.InvalidateMemory(final_word_address, sizeof(uint32_t)),
              "the final mapped word could not be initialized");
      *final_word = 0;
      auto final_word_desc = MakeLinearDesc(
          final_word_address, sizeof(uint32_t), vk::Format::eR32Uint,
          Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
          {1, 1, 1}, 1, 4, 1);
      const auto final_word_image = texture_cache.FindImage(final_word_desc);
      (void)texture_cache.FindTexture(final_word_image, final_word_desc);
      Require(name, "fault boundary image tracking",
              texture_cache.GetImage(final_word_image).IsTracked() &&
                  !texture_cache.GetImage(final_word_image).IsCpuDirty(),
              "the final mapped word's image was not clean before its CPU store");
      *final_word = 45;
      Require(name, "fault boundary CPU store",
              *final_word == 45 &&
                  texture_cache.GetImage(final_word_image).IsDefinitelyCpuDirty(),
              "a valid four-byte store at the mapping end did not invalidate its image");

      constexpr uint32_t final_gpu_value = 0xa1b2c3d4u;
      auto &boundary_buffer_cache = resources.GetBufferCache();
      (void)boundary_buffer_cache.ObtainBuffer(final_word_address, sizeof(uint32_t), true);
      boundary_buffer_cache.FillBuffer(final_word_address, sizeof(uint32_t),
                                       final_gpu_value, false);
      Require(name, "fault boundary GPU ownership",
              boundary_buffer_cache.HasGpuDirtyBytes(final_word_address, sizeof(uint32_t)) &&
                  !HostMemoryIsReadable(mapped_end - 1),
              "the final mapped byte was not protected by GPU ownership");
      const uint8_t final_gpu_byte = *final_byte;
      Require(name, "fault boundary GPU readback",
              final_gpu_byte == static_cast<uint8_t>(final_gpu_value >> 24u) &&
                  *final_word == final_gpu_value &&
                  !boundary_buffer_cache.HasGpuDirtyBytes(final_word_address, sizeof(uint32_t)),
              "a final-byte CPU read did not publish the GPU-owned final word");

      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
      LibKernel::Memory::InstallGpuResources(nullptr);
    }
    context.ShutdownGpu();
    Require(name, "unmap direct backing",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "cache direct-memory mapping release failed");
    Require(name, "release direct backing",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "cache direct-memory allocation release failed");
    std::printf("[host]    %-32s ok\n", name);
  }

  void CheckBgra16Readback() {
    constexpr const char *name = "Bgra16Readback";
    constexpr uintptr_t base = 0x0000000204000000ull;
    constexpr uint64_t allocation_size = 0x200000;
    constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float;
    constexpr auto tile = Prospero::TileMode::kRenderTarget;
    EnsureRuntimeContext();

    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_size, 0, &direct_offset) == 0,
            "BGRA16 direct-memory allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_size) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "BGRA16 fixed mapping failed");

    RenderContext context(m_runtime_context);
    auto &scheduler = context.GetCommandScheduler();
    HW::Context registers{};
    HW::UserConfig user_config{};
    HW::Shader shaders{};
    scheduler.Begin(registers, user_config, shaders);
    {
      auto &resources = context;
      resources.MapMemory(base, allocation_size);
      const uint32_t pitch = TileGetTexturePitch(format, 1, tile);
      TileSizeAlign total{};
      TileSizeOffset mip{};
      TilePaddedSize padded{};
      TileGetTextureSize(format, 1, 1, 1, tile, &total, &mip, &padded);
      Require(name, "tiled layout",
              total.size >= 8 && total.size <= allocation_size,
              "BGRA16 tiled layout exceeds its guest allocation");
      std::memset(mapped, 0x5a, total.size);
      constexpr std::array<uint16_t, 4> initial{0x3c00u, 0x4000u, 0x4200u, 0x4400u};
      std::memcpy(mapped, initial.data(), sizeof(initial));

      ImageDesc desc{};
      desc.type = BindingType::RenderTarget;
      desc.info.data = {base, total.size};
      desc.info.pixel_format = vk::Format::eR16G16B16A16Sfloat;
      desc.info.guest_format = format;
      desc.info.type = Prospero::ImageType::kColor2D;
      desc.info.extent = {1, 1, 1};
      desc.info.resources = {1, 1};
      desc.info.pitch = pitch;
      desc.info.bytes_per_block = 8;
      desc.info.samples = 1;
      desc.info.tile_mode = tile;
      desc.info.bgra16 = true;
      desc.info.UpdateSize();
      desc.view_info.format = desc.info.pixel_format;
      desc.view_info.type = vk::ImageViewType::e2D;
      desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
      desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;

      auto &cache = resources.GetTextureCache();
      const auto id = cache.FindImage(desc);
      (void)cache.FindRenderTarget(id, desc);
      Require(name, "tiled BGRA16 upload",
              ReadCachedTexel(name, context, id) ==
                  std::vector<u32>{0x40004200u, 0x44003c00u},
              "fused detiling lost the BGRA16 component transform");
      auto &image = cache.GetImage(id);
      image.Transit(vk::ImageLayout::eTransferDstOptimal,
                    vk::AccessFlagBits2::eTransferWrite, {},
                    scheduler.Current().Handle());
      vk::ClearColorValue clear{};
      clear.float32[0] = 1.0f;
      clear.float32[1] = 2.0f;
      clear.float32[2] = 3.0f;
      clear.float32[3] = 4.0f;
      const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eColor, 0,
                                            1, 0, 1};
      scheduler.Current().Handle().clearColorImage(
          image.backing.image, vk::ImageLayout::eTransferDstOptimal, clear,
          range);
      cache.MarkGpuWritten(id);
      Require(name, "guest readback queue",
              TextureCacheTestAccess::TryDownload(cache, id),
              "tiled BGRA16 guest readback was rejected");
      auto mirror = resources.GetBufferCache().ObtainBuffer(base, total.size,
                                                            false, true);
      Require(name, "mirror owner", mirror.first != nullptr,
              "tiled BGRA16 mirror has no BufferCache owner");
      auto mirror_readback = CreateHostBuffer(
          name, 8, vk::BufferUsageFlagBits::eTransferDst, {0, 0});
      const vk::BufferCopy copy{mirror.second, 0, 8};
      scheduler.Current().Handle().copyBuffer(mirror.first->Handle(),
                                              mirror_readback.buffer, 1, &copy);
      vk::BufferMemoryBarrier barrier{};
      barrier.sType = vk::StructureType::eBufferMemoryBarrier;
      barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
      barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
      barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barrier.buffer = mirror_readback.buffer;
      barrier.size = mirror_readback.size;
      scheduler.Current().Handle().pipelineBarrier(
          vk::PipelineStageFlagBits::eTransfer,
          vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
          nullptr);
      scheduler.Finish();
      scheduler.DrainPriorityOperations();

      std::vector<uint8_t> backing(total.size);
      Require(name, "guest backing",
              Libs::LibKernel::Memory::TryReadBacking(base, backing.data(),
                                                      backing.size()),
              "tiled BGRA16 guest backing is unreadable");
      const std::array<uint16_t, 4> expected{0x4200u, 0x4000u, 0x3c00u,
                                             0x4400u};
      std::array<uint16_t, 4> observed{};
      std::memcpy(observed.data(), backing.data(), sizeof(observed));
      Require(name, "guest component order",
              observed == expected &&
                  std::all_of(backing.begin() + sizeof(observed), backing.end(),
                              [](uint8_t value) { return value == 0x5a; }),
              "tiled BGRA16 guest readback changed component order or padding");
      Require(name, "mirror component order",
              ReadBuffer(name, mirror_readback, 2) ==
                  std::vector<u32>{0x40004200u, 0x44003c00u},
              "tiled BGRA16 Buffer mirror changed guest component order");
      DestroyBuffer(&mirror_readback);
      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
    }
    Require(name, "unmap",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "BGRA16 fixed mapping release failed");
    Require(name, "release",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "BGRA16 direct-memory release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckRenderExecutorColorDiscovery() {
    constexpr const char *name = "RenderExecutorColorDiscovery";
    constexpr uintptr_t base = 0x0000000203c00000ull;
    constexpr uint32_t width = 512;
    constexpr uint32_t height = 1;
    constexpr uint32_t bytes_per_element = 4;
    constexpr uint64_t allocation_alignment = 0x10000;
    EnsureRuntimeContext();

    const auto pitch =
        TileGetRenderTargetPitch(width, bytes_per_element, 0);
    TileSizeAlign layout{};
    Require(name, "tile layout",
            pitch != 0 && TileGetRenderTargetSize(
                              width, height, pitch, bytes_per_element, layout, 0),
            "1D render-target layout is unavailable");
    struct TargetCase {
      uint32_t width, height, dimension, pitch;
      uint64_t size;
      Prospero::TileMode tile;
      uint32_t levels = 1;
      bool float16 = false;
    };
    const std::array cases{
        TargetCase{width, height, 0, pitch, layout.size,
                   Prospero::TileMode::kRenderTarget},
        // PPSA28068: the 1440x720 agreement target must alias its sampled texture.
        TargetCase{1440, 720, 1, 1472, 0x40b000, Prospero::TileMode::kLinear},
        // PPSA01416: linear RGBA16F chains place the smallest mip first.
        TargetCase{128, 128, 1, 128, 0x2be00, Prospero::TileMode::kLinear, 7, true},
        // PPSA03541: the single texel still occupies a padded 128x64 block.
        TargetCase{1, 1, 1, 128, 0x10000, Prospero::TileMode::kRenderTarget, 1, true},
    };
    constexpr uint64_t allocation_size = 0x410000;

    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_alignment, 0, &direct_offset) == 0,
            "color direct-memory allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_alignment) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "color fixed mapping failed");
    std::memset(mapped, 0, allocation_size);

    for (const auto &target : cases) {
      const auto width = target.width;
      const auto height = target.height;
      const bool is_1d = target.dimension == 0;
      RenderContext context(m_runtime_context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      registers.SetColorBase(0, {.addr = base});
      registers.SetColorInfo(
          0, {.format = target.float16 ? Prospero::ChannelLayout::k16_16_16_16
                                      : Prospero::ChannelLayout::k8_8_8_8,
              .channel_type = target.float16 ? Prospero::ChannelType::kFloat
                                            : Prospero::ChannelType::kSrgb,
              .channel_order = Prospero::ChannelOrder::kStandard});
      registers.SetColorAttrib2(0, {.height = height - 1, .width = width - 1,
                                   .num_mip_levels = target.levels - 1});
      registers.SetColorAttrib3(0,
                                {.tile_mode = target.tile,
                                 .dimension = target.dimension,
                                 .metadata_pipe_aligned = true});
      registers.SetRenderTargetMask(0x0f);
      scheduler.Begin(registers, user_config, shaders);

      auto &resources = context;
      auto &texture_cache = resources.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      resources.MapMemory(base, allocation_size);

      RenderColorInfo color{};
      RenderExecutorTestAccess::ResolveRenderColorTarget(
          executor, scheduler.Current(), color, 0);
      const auto attachment =
          texture_cache.FindRenderTarget(color.image_id, color.desc);
      const auto &image = texture_cache.GetImage(color.image_id);
      RenderDepthInfo no_depth{};
      const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
          executor, scheduler.Current(), &color, 1, no_depth);
      scheduler.Current().BeginRendering(rendering);
      scheduler.Current().EndRendering();
      Require(name, "captured color target",
              color.image_id && attachment != nullptr &&
                  rendering.color_attachments[0].image_view != nullptr &&
                  color.desc.info.type == (is_1d ? Prospero::ImageType::kColor1D
                                                  : Prospero::ImageType::kColor2D) &&
                  color.desc.info.extent == vk::Extent3D{width, height, 1} &&
                  color.desc.info.resources == ImageSubresources{target.levels, 1} &&
                  color.desc.info.pitch == target.pitch &&
                  color.desc.info.data.size == target.size &&
                  color.desc.info.mip_layout[0].offset ==
                      (target.levels == 1 ? 0 : 0xbe00) &&
                  color.desc.info.mip_layout[0].size ==
                      (target.levels == 1 ? target.size : 0x20000) &&
                  color.desc.view_info.type == (is_1d ? vk::ImageViewType::e1D
                                                       : vk::ImageViewType::e2D) &&
                  image.backing.image_type == (is_1d ? vk::ImageType::e1D
                                                      : vk::ImageType::e2D) &&
                  rendering.width == width && rendering.height == height &&
                  rendering.num_layers == 1 &&
                  rendering.num_color_attachments == 1,
              "color attachment lost its dimensions or padded guest layout");

      if (width == 1 && height == 1) {
        Require(name, "single-texel tiled layout",
                color.desc.info.mip_layout[0].offset == 0 &&
                    color.desc.info.mip_layout[0].size == 0x10000 &&
                    color.desc.info.mip_layout[0].pitch == 128 &&
                    color.desc.info.mip_layout[0].height == 64,
                "the render target lost the expected padded block height");
      }

      if (target.levels == 7) {
        constexpr std::array offsets{0xbe00u, 0x3e00u, 0x1e00u, 0xe00u,
                                     0x600u, 0x200u, 0u};
        constexpr std::array pitches{128u, 64u, 32u, 32u, 32u, 32u, 32u};
        for (uint32_t mip = 0; mip < target.levels; ++mip) {
          const auto &layout = color.desc.info.mip_layout[mip];
          Require(name, "linear mip chain", layout.offset == offsets[mip] &&
                      layout.pitch == pitches[mip] &&
                      layout.size == pitches[mip] * (height >> mip) * 8,
                  "linear target mip offsets or padded pitches differ from the expected layout");
        }
      }

      {
        vk::ClearValue clear{};
        clear.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
        TextureCacheTestAccess::ClearImage(
            texture_cache, scheduler.Current(), color.image_id,
            {vk::ImageAspectFlagBits::eColor, 0, target.levels, 0, 1}, clear);
        RenderExecutorTestAccess::ResetBindings(executor);
        ShaderTextureResource descriptor{{
            static_cast<uint32_t>(base >> 8u),
            (static_cast<uint32_t>(target.float16
                                      ? Prospero::BufferFormat::k16_16_16_16Float
                                      : Prospero::BufferFormat::k8_8_8_8Srgb) << 20u) |
                (((width - 1u) & 3u) << 30u),
            ((width - 1u) >> 2u) | ((height - 1u) << 14u),
            DstSel(4, 5, 6, 7) | ((target.levels - 1) << 16u) |
                (static_cast<uint32_t>(target.tile) << 20u) |
                (static_cast<uint32_t>(is_1d ? Prospero::ImageType::kColor1D
                                             : Prospero::ImageType::kColor2D) << 28u),
            0, 0x00700000u | ((target.levels - 1) << 4u), 0, 0}};
        ShaderRecompiler::IR::DescriptorValue value{};
        value.dword_count = 8;
        std::copy_n(descriptor.fields, 8, value.dwords.begin());
        ShaderRecompiler::IR::ImageResource resource{};
        resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
        resource.numeric_class = Prospero::TextureNumericClass::Float;
        resource.dimension = is_1d ? ShaderRecompiler::Decoder::ImageDimension::Dim1D
                                   : ShaderRecompiler::Decoder::ImageDimension::Dim2D;
        resource.read = true;
        const auto sampled =
            RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
        Require(name, "target sampled alias",
                sampled.image_id == color.image_id &&
                    sampled.desc.info.pitch == target.pitch &&
                    sampled.desc.info.data.size == target.size &&
                    sampled.desc.info.mip_layout == color.desc.info.mip_layout &&
                    texture_cache.FindTexture(sampled.image_id, sampled.desc) != nullptr,
                "sampling the target changed its padded layout or created a second image");
        const auto expected = target.float16
                                  ? std::vector<u32>{0x00003c00u, 0x3c003c00u}
                                  : std::vector<u32>{0xffff00ffu};
        for (uint32_t mip = 0; mip < target.levels; ++mip) {
          Require(name, "target sampled contents",
                  ReadCachedTexel(name, context, sampled.image_id,
                                  {static_cast<int32_t>((width >> mip) - 1),
                                   static_cast<int32_t>((height >> mip) - 1), 0},
                                  {1, 1, 1}, 0, mip) == expected,
                  "the sampled target lost its GPU-written final texel");
        }
        if (width == 1 && height == 1) {
          for (const bool uint_view : {true, false}) {
            descriptor.fields[1] = static_cast<uint32_t>(
                uint_view ? Prospero::BufferFormat::k16_16_16_16UInt
                          : Prospero::BufferFormat::k16_16_16_16Float) << 20u;
            descriptor.fields[3] = 0x81b00facu; // Captured 1D view, tile27, RGBA swizzle.
            std::copy_n(descriptor.fields, 8, value.dwords.begin());
            resource.numeric_class = uint_view ? Prospero::TextureNumericClass::Uint
                                                : Prospero::TextureNumericClass::Float;
            resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim1D;
            const auto one_d =
                RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
            const auto one_d_view = texture_cache.FindTexture(one_d.image_id, one_d.desc);
            const auto repeated =
                RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
            Require(name, "1D sampled backing and reuse",
                    one_d.image_id != color.image_id && one_d_view != nullptr &&
                        one_d.desc.view_info.type == vk::ImageViewType::e1D &&
                        texture_cache.GetImage(one_d.image_id).backing.image_type ==
                            vk::ImageType::e1D &&
                        repeated.image_id == one_d.image_id &&
                        texture_cache.FindTexture(repeated.image_id, repeated.desc) == one_d_view &&
                        ReadCachedTexel(name, context, one_d.image_id) == expected,
                    "the 1D alias reused an incompatible backing or lost the rendered texel");

            RenderExecutorTestAccess::ResetBindings(executor);
            RenderExecutorTestAccess::ResolveRenderColorTarget(
                executor, scheduler.Current(), color, 0);
            const auto restored = RenderExecutorTestAccess::AcquireRenderTargets(
                executor, scheduler.Current(), &color, 1, no_depth);
            scheduler.Current().BeginRendering(restored);
            scheduler.Current().EndRendering();
            Require(name, "2D target restoration",
                    color.image_id != one_d.image_id &&
                        restored.color_attachments[0].image_view != nullptr &&
                        texture_cache.GetImage(color.image_id).backing.image_type ==
                            vk::ImageType::e2D &&
                        texture_cache.GetImage(color.image_id).info.pixel_format ==
                            vk::Format::eR16G16B16A16Sfloat &&
                        ReadCachedTexel(name, context, color.image_id) == expected,
                    "restoring the 2D target lost its view type or GPU-written texel");
            RenderExecutorTestAccess::ResetBindings(executor);
          }
        }
      }

      RenderExecutorTestAccess::ResetBindings(executor);
      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
    }

    Require(name, "unmap direct backing",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "color direct mapping release failed");
    Require(name, "release direct backing",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "color direct-memory allocation release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckRenderExecutorColor1DArrayDiscovery() {
    constexpr const char *name = "RenderExecutorColor1DArrayDiscovery";
    constexpr uintptr_t base = 0x0000000203a00000ull;
    constexpr uint32_t width = 512;
    constexpr uint32_t height = 1;
    constexpr uint32_t bytes_per_element = 4;
    constexpr uint32_t first_slice = 2;
    constexpr uint32_t last_slice = 5;
    constexpr uint32_t image_layers = last_slice + 1;
    constexpr uint32_t view_layers = last_slice - first_slice + 1;
    constexpr uint64_t allocation_alignment = 0x10000;
    EnsureRuntimeContext();

    const auto pitch = TileGetRenderTargetPitch(width, bytes_per_element, 0);
    TileSizeAlign layout{};
    Require(name, "tile layout",
            pitch == width &&
                TileGetRenderTargetSize(width, height, pitch,
                                        bytes_per_element, layout, 0) &&
                layout.size == 0x40000 &&
                layout.align == allocation_alignment,
            "1D-array render-target layout is unavailable");
    const auto allocation_size = layout.size * image_layers;

    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_alignment, 0, &direct_offset) == 0,
            "1D-array color direct-memory allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_alignment) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "1D-array color fixed mapping failed");
    std::memset(mapped, 0, allocation_size);

    {
      RenderContext context(m_runtime_context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      registers.SetColorBase(0, {.addr = base});
      registers.SetColorInfo(
          0, {.format = Prospero::ChannelLayout::k8_8_8_8,
              .channel_type = Prospero::ChannelType::kSrgb,
              .channel_order = Prospero::ChannelOrder::kStandard});
      registers.SetColorView(
          0, {.base_array_slice_index = first_slice,
              .last_array_slice_index = last_slice});
      registers.SetColorAttrib2(0, {.height = height - 1, .width = width - 1});
      registers.SetColorAttrib3(0,
                                {.tile_mode = Prospero::TileMode::kRenderTarget,
                                 .dimension = 0,
                                 .metadata_pipe_aligned = true});
      registers.SetRenderTargetMask(0x0f);
      scheduler.Begin(registers, user_config, shaders);

      auto &resources = context;
      auto &texture_cache = resources.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      resources.MapMemory(base, allocation_size);

      RenderColorInfo color{};
      RenderExecutorTestAccess::ResolveRenderColorTarget(
          executor, scheduler.Current(), color, 0);
      const auto attachment =
          texture_cache.FindRenderTarget(color.image_id, color.desc);
      const auto &image = texture_cache.GetImage(color.image_id);
      RenderDepthInfo no_depth{};
      const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
          executor, scheduler.Current(), &color, 1, no_depth);
      scheduler.Current().BeginRendering(rendering);
      scheduler.Current().EndRendering();
      Require(
          name, "captured 1D-array target",
          color.image_id && attachment != nullptr &&
              rendering.color_attachments[0].image_view == attachment &&
              color.guest_array_layer == first_slice &&
              color.desc.info.type == Prospero::ImageType::kColor1D &&
              color.desc.info.extent == vk::Extent3D{width, height, 1} &&
              color.desc.info.resources == ImageSubresources{1, image_layers} &&
              color.desc.info.pitch == pitch &&
              color.desc.info.data.size == allocation_size &&
              color.desc.info.mip_layout[0].offset == 0 &&
              color.desc.info.mip_layout[0].size == allocation_size &&
              color.desc.view_info.type == vk::ImageViewType::e1DArray &&
              color.desc.view_info.base_layer == first_slice &&
              color.desc.view_info.layer_count == view_layers &&
              image.backing.image_type == vk::ImageType::e1D &&
              image.backing.layers == image_layers &&
              rendering.width == width && rendering.height == height &&
              rendering.num_layers == view_layers &&
              rendering.num_color_attachments == 1,
          "dimension=0 slice range did not create a Vulkan 1D-array "
          "attachment view");

      RenderExecutorTestAccess::ResetBindings(executor);
      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
    }

    Require(name, "unmap direct backing",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "1D-array color direct mapping release failed");
    Require(name, "release direct backing",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "1D-array color direct-memory allocation release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckRenderExecutorColorVolumeDiscovery() {
    constexpr const char *name = "RenderExecutorColorVolumeDiscovery";
    constexpr uintptr_t base = 0x0000000203e00000ull;
    constexpr uint64_t slice_size = 0x10000;
    constexpr uint64_t color_size = 0x200000;
    constexpr uint64_t metadata_size = 0x20000;
    constexpr uint64_t allocation_size = 56 * slice_size;
    constexpr uint64_t allocation_alignment = 0x10000;
    EnsureRuntimeContext();

    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_alignment, 0, &direct_offset) == 0,
            "color-volume direct-memory allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_alignment) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "color-volume fixed mapping failed");
    std::memset(mapped, 0x5a, allocation_size);

    {
      RenderContext context(m_runtime_context);
      context.InitializeGpu(nullptr);
      LibKernel::Memory::InstallGpuResources(&context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      registers.SetColorBase(0, {.addr = base});
      registers.SetColorInfo(
          0, {.dcc_compression_enable = true,
              .format = Prospero::ChannelLayout::k10_10_10_2,
              .channel_type = Prospero::ChannelType::kUNorm,
              .channel_order = Prospero::ChannelOrder::kStandard});
      registers.SetColorAttrib2(0, {.height = 31, .width = 31});
      registers.SetColorAttrib3(0,
                                {.depth = 31,
                                 .tile_mode = Prospero::TileMode::kRenderTarget,
                                 .dimension = 2,
                                 .metadata_pipe_aligned = true});
      constexpr uint64_t dcc_address = base + color_size;
      registers.SetColorDccAddr(0, {.addr = dcc_address});
      registers.SetColorClearWord0(0, {.word0 = 0});
      registers.SetColorClearWord1(0, {.word1 = 0});
      registers.SetRenderTargetMask(0x0f);
      scheduler.Begin(registers, user_config, shaders);

      auto &resources = context;
      auto &texture_cache = resources.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      resources.MapMemory(base, allocation_size);
      WriteMetadata(context, dcc_address, metadata_size, 0);
      Require(name, "deferred DCC clear",
              !texture_cache.IsMeta(dcc_address),
              "a pre-registration DCC clear was not deferred");

      RenderColorInfo color{};
      RenderExecutorTestAccess::ResolveRenderColorTarget(
          executor, scheduler.Current(), color, 0);
      const auto attachment =
          texture_cache.FindRenderTarget(color.image_id, color.desc);
      const auto &image = texture_cache.GetImage(color.image_id);
      Require(
          name, "captured 3D target",
          color.image_id && attachment != nullptr &&
              color.desc.info.type == Prospero::ImageType::kColor3D &&
              color.desc.info.extent == vk::Extent3D{32, 32, 32} &&
              color.desc.info.resources == ImageSubresources{1, 1} &&
              color.desc.info.pitch == 128 &&
              color.desc.info.data.size == color_size &&
              color.desc.info.mip_layout[0].size == 0x10000 &&
              color.desc.info.metadata.kind == ImageMetadataKind::Dcc &&
              color.desc.info.metadata.range.address == dcc_address &&
              !texture_cache.IsMeta(dcc_address) && !texture_cache.ClearMeta(dcc_address) &&
              color.desc.view_info.type == vk::ImageViewType::e2D &&
              color.desc.view_info.layer_count == 1 &&
              image.backing.image_type == vk::ImageType::e3D &&
              static_cast<bool>(image.backing.flags &
                                vk::ImageCreateFlagBits::e2DArrayCompatible) &&
              image.usage.render_target && image.IsGpuModified(),
          "dimension=2/depth=31 did not create the expected 32x32x32 "
          "backing and 2D "
          "attachment slice");
      constexpr uint64_t metadata_slice_size = metadata_size / 32;
      std::vector<uint8_t> metadata_bytes(metadata_size);
      resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
      Require(name, "first volume slice expanded metadata",
              LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
                                               metadata_bytes.size()) &&
                  std::all_of(metadata_bytes.begin(),
                              metadata_bytes.begin() + metadata_slice_size,
                              [](uint8_t byte) { return byte == 0xff; }) &&
                  std::all_of(metadata_bytes.begin() + metadata_slice_size,
                              metadata_bytes.end(), [](uint8_t byte) { return byte == 0; }),
              "consuming volume slice 0 changed unrequested native metadata slices");

      RenderExecutorTestAccess::ResetBindings(executor);
      registers.SetColorView(
          0, {.base_array_slice_index = 7, .last_array_slice_index = 7});
      RenderColorInfo sliced_color{};
      RenderExecutorTestAccess::ResolveRenderColorTarget(
          executor, scheduler.Current(), sliced_color, 0);
      RenderDepthInfo no_depth{};
      const auto sliced_rendering =
          RenderExecutorTestAccess::AcquireRenderTargets(
              executor, scheduler.Current(), &sliced_color, 1, no_depth);
      Require(
          name, "3D slice transition",
          sliced_color.image_id == color.image_id &&
              sliced_rendering.color_attachments[0].image_view != nullptr &&
              sliced_color.desc.view_info.base_layer == 7 &&
              sliced_rendering.num_color_attachments == 1 &&
              sliced_rendering.num_layers == 1,
          "a nonzero 3D attachment slice was treated as a Vulkan array layer");
      resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
      Require(name, "mixed volume metadata readback",
              LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
                                               metadata_bytes.size()),
              "mixed volume metadata is unavailable");
      for (uint32_t slice = 0; slice < 32; slice++) {
        const uint8_t expected = slice == 0 || slice == 7 ? 0xff : 0;
        const auto first = metadata_bytes.begin() + slice * metadata_slice_size;
        Require(name, "independent volume metadata consumption",
                std::all_of(first, first + metadata_slice_size,
                            [expected](uint8_t byte) { return byte == expected; }),
                "mixed DCC metadata lost a pending slice or consumed an unrequested slice");
      }
      Require(name, "3D slice clear and preservation",
              ReadCachedTexel(name, context, color.image_id, {0, 0, 7}) ==
                  std::vector<u32>{0} &&
                  ReadCachedTexel(name, context, color.image_id, {0, 0, 31}) ==
                      std::vector<u32>{0x5a5a5a5a},
              "clearing one volume slice changed untouched slice contents");
      auto storage_desc = color.desc;
      storage_desc.type = BindingType::Storage;
      storage_desc.info.metadata = {};
      storage_desc.info.guest_format = Prospero::BufferFormat::k10_10_10_2UNorm;
      storage_desc.view_info.type = vk::ImageViewType::e3D;
      storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
      const auto storage_id = texture_cache.FindImage(storage_desc);
      const auto storage_view =
          texture_cache.FindTexture(storage_id, storage_desc);
      const auto &shared_image = texture_cache.GetImage(storage_id);
      Require(name, "storage alias reuse",
              storage_id == color.image_id && storage_view != nullptr &&
                  shared_image.IsGpuModified() &&
                  shared_image.usage.render_target,
              "the matching 3D storage binding did not reuse the live "
              "render-target image");

      Require(
          name, "volume readback queue",
          TextureCacheTestAccess::TryDownload(texture_cache, storage_id),
          "the 3D render target could not be queued for guest-layout readback");
      auto mirror = resources.GetBufferCache().ObtainBuffer(
          base, color_size, false, true);
      Require(name, "volume mirror", mirror.first != nullptr,
              "the 3D render-target readback has no BufferCache owner");
      auto slice_probe =
          CreateHostBuffer(name, 4, vk::BufferUsageFlagBits::eTransferDst, {0});
      const vk::BufferCopy copy{mirror.second + 31 * slice_size, 0, 4};
      scheduler.Current().Handle().copyBuffer(mirror.first->Handle(),
                                              slice_probe.buffer, 1, &copy);
      vk::BufferMemoryBarrier barrier{};
      barrier.sType = vk::StructureType::eBufferMemoryBarrier;
      barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
      barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
      barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barrier.buffer = slice_probe.buffer;
      barrier.size = slice_probe.size;
      scheduler.Current().Handle().pipelineBarrier(
          vk::PipelineStageFlagBits::eTransfer,
          vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
          nullptr);
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      Require(name, "volume Z transfer",
              ReadBuffer(name, slice_probe, 1) == std::vector<u32>{0x5a5a5a5a},
              "render-target upload/readback lost the final Z slice");
      DestroyBuffer(&slice_probe);

      vk::ClearValue painted{};
      painted.color.float32 = std::array{1.0f, 1.0f, 1.0f, 0.0f};
      TextureCacheTestAccess::ClearImage(
          texture_cache, scheduler.Current(), color.image_id,
          {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
      WriteMetadata(context, dcc_address, metadata_size, 0xffffffffu);
      color.image_id = texture_cache.FindImage(color.desc);
      (void)texture_cache.FindRenderTarget(color.image_id, color.desc);
      resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
      Require(name, "expanded volume metadata preserves rendering",
              ReadCachedTexel(name, context, color.image_id) ==
                  std::vector<u32>{0x3fffffffu} &&
                  LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
                                                   metadata_bytes.size()) &&
                  std::all_of(metadata_bytes.begin(), metadata_bytes.end(),
                              [](uint8_t byte) { return byte == 0xff; }),
              "uncompressed native keys reapplied an old volume clear");
      WriteMetadata(context, dcc_address, metadata_size, 0);
      color.image_id = texture_cache.FindImage(color.desc);
      (void)texture_cache.FindRenderTarget(color.image_id, color.desc);
      resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
      Require(name, "native volume re-clear",
              ReadCachedTexel(name, context, color.image_id) == std::vector<u32>{0} &&
                  LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
                                                   metadata_bytes.size()) &&
                  std::all_of(metadata_bytes.begin(),
                              metadata_bytes.begin() + metadata_slice_size,
                              [](uint8_t byte) { return byte == 0xff; }) &&
                  std::all_of(metadata_bytes.begin() + metadata_slice_size,
                              metadata_bytes.end(), [](uint8_t byte) { return byte == 0; }),
              "a native metadata overwrite did not clear only its requested volume slice");

      for (const uint32_t base_slice : {0u, 7u}) {
        RenderExecutorTestAccess::ResetBindings(executor);
        registers.SetColorView(0, {.base_array_slice_index = base_slice,
                                   .last_array_slice_index = 32});
        RenderColorInfo volume_color{};
        RenderExecutorTestAccess::ResolveRenderColorTarget(
            executor, scheduler.Current(), volume_color, 0);
        const auto volume_rendering =
            RenderExecutorTestAccess::AcquireRenderTargets(
                executor, scheduler.Current(), &volume_color, 1, no_depth);
        Require(name, "3D export bound exceeds depth",
                volume_color.image_id == color.image_id &&
                    volume_color.desc.info.extent == vk::Extent3D{32, 32, 32} &&
                    volume_color.desc.info.data.size == color_size &&
                    volume_color.desc.view_info.type == vk::ImageViewType::e2DArray &&
                    volume_color.desc.view_info.base_layer == base_slice &&
                    volume_color.desc.view_info.layer_count == 32 - base_slice &&
                    volume_rendering.num_layers == 32 - base_slice &&
                    registers.GetRenderTarget(0).view.last_array_slice_index == 32,
                "3D export bounds changed the backing or exceeded its attachment depth");
        scheduler.Current().BeginRendering(volume_rendering);
        scheduler.Current().EndRendering();
      }
      resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
      Require(name, "complete volume expansion",
              LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
                                               metadata_bytes.size()) &&
                  std::all_of(metadata_bytes.begin(), metadata_bytes.end(),
                              [](uint8_t byte) { return byte == 0xff; }),
              "full volume acquisition left an unconsumed native clear slice");

      TextureCacheTestAccess::ClearImage(
          texture_cache, scheduler.Current(), color.image_id,
          {vk::ImageAspectFlagBits::eColor, 0, 1, 31, 1}, painted);
      auto expanded = storage_desc;
      expanded.info.extent.depth = 56;
      expanded.info.data.size = allocation_size;
      expanded.info.UpdateSize();
      const auto expanded_id = texture_cache.FindImage(expanded);
      Require(name, "3D storage depth expansion",
              expanded_id != color.image_id &&
                  !TextureCacheTestAccess::Contains(texture_cache, color.image_id) &&
                  texture_cache.GetImage(expanded_id).backing.extent.depth == 56 &&
                  texture_cache.GetImage(expanded_id).IsGpuModified(),
              "a deeper storage binding reused the smaller native volume");
      Require(name, "3D expansion preserves native prefix and guest suffix",
              ReadCachedTexel(name, context, expanded_id, {0, 0, 31}) ==
                  std::vector<u32>{0x3fffffffu} &&
                  ReadCachedTexel(name, context, expanded_id, {0, 0, 55}) ==
                      std::vector<u32>{0x5a5a5a5a},
              "volume expansion lost GPU writes or copied beyond the old depth");

      RenderExecutorTestAccess::ResetBindings(executor);
      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
      context.ShutdownGpu();
      LibKernel::Memory::InstallGpuResources(nullptr);
    }

    Require(name, "unmap direct backing",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "color-volume direct mapping release failed");
    Require(name, "release direct backing",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "color-volume direct-memory allocation release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  static void WriteMetadata(RenderContext &context, uint64_t address,
                            uint64_t size, uint32_t value) {
    Require("DccMetadata", "guest write range", size % sizeof(value) == 0 &&
                context.InvalidateMemory(address, size),
            "metadata write is outside the mapped allocation");
    const std::vector<uint32_t> words(size / sizeof(value), value);
    LibKernel::Memory::WriteBacking(address, words.data(), size);
  }

  static TextureCache::ImageDesc MakeLinearDesc(
      uint64_t address, uint64_t size, vk::Format format,
      Prospero::BufferFormat guest_format, Prospero::ImageType type,
      vk::Extent3D extent, uint32_t layers, uint32_t bytes_per_block,
      uint32_t samples) {
    TextureCache::ImageDesc desc{};
    desc.type = TextureCache::BindingType::Texture;
    desc.info.data = {address, size};
    desc.info.pixel_format = format;
    desc.info.guest_format = guest_format;
    desc.info.type = type;
    desc.info.extent = extent;
    desc.info.resources = {1, layers};
    desc.info.pitch = extent.width;
    desc.info.bytes_per_block = bytes_per_block;
    desc.info.samples = samples;
    desc.info.tile_mode = Prospero::TileMode::kLinear;
    desc.info.mip_layout[0] = {0, size, extent.width, extent.height};
    desc.info.tiling.texel_width = desc.info.IsBlock() ? 4u : 1u;
    desc.info.tiling.texel_height = desc.info.IsBlock() ? 4u : 1u;
    desc.info.tiling.block.bytes_per_element = bytes_per_block;
    desc.info.linear_slice_stride =
        size / (type == Prospero::ImageType::kColor3D ? extent.depth : layers);
    desc.info.mip_layout[0].pitch =
        (extent.width + desc.info.tiling.texel_width - 1u) /
        desc.info.tiling.texel_width;
    desc.info.mip_layout[0].height =
        (extent.height + desc.info.tiling.texel_height - 1u) /
        desc.info.tiling.texel_height;
    desc.info.mip_layout[0].linear_pitch = desc.info.mip_layout[0].pitch;
    desc.info.mip_layout[0].linear_size = desc.info.linear_slice_stride;
    desc.view_info.format = format;
    desc.view_info.type = type == Prospero::ImageType::kColor3D
                              ? vk::ImageViewType::e3D
                          : layers > 1 ? vk::ImageViewType::e2DArray
                                       : vk::ImageViewType::e2D;
    desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
    desc.view_info.layer_count = layers;
    desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
    return desc;
  }

  std::vector<u32> ReadCachedTexel(const char *name, RenderContext &context,
                                 ImageId id, vk::Offset3D offset = {},
                                 vk::Extent3D extent = {1, 1, 1}, uint32_t layer = 0,
                                 uint32_t mip = 0) {
    auto &scheduler = context.GetCommandScheduler();
    auto &image = context.GetTextureCache().GetImage(id);
    const auto bytes = image.info.bytes_per_block * extent.width *
                       extent.height * extent.depth;
    auto probe = CreateHostBuffer(name, bytes,
                                 vk::BufferUsageFlagBits::eTransferDst, {});
    scheduler.Current().EndRendering();
    image.Transit(vk::ImageLayout::eTransferSrcOptimal,
                  vk::AccessFlagBits2::eTransferRead, {},
                  scheduler.Current().Handle());
    vk::BufferImageCopy copy{};
    copy.imageSubresource = {image.info.IsDepth() ? vk::ImageAspectFlagBits::eDepth
                                                : vk::ImageAspectFlagBits::eColor, mip, layer, 1};
    copy.imageOffset = offset;
    copy.imageExtent = extent;
    scheduler.Current().Handle().copyImageToBuffer(
        image.backing.image, vk::ImageLayout::eTransferSrcOptimal,
        probe.buffer, 1, &copy);
    vk::BufferMemoryBarrier barrier{};
    barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
    barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
    barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
    barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
    barrier.buffer = probe.buffer;
    barrier.size = probe.size;
    scheduler.Current().Handle().pipelineBarrier(
        vk::PipelineStageFlagBits::eTransfer,
        vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
        nullptr);
    scheduler.Finish();
    auto result = ReadBuffer(name, probe, bytes / 4);
    DestroyBuffer(&probe);
    return result;
  }

  void CheckScalarLeafCall() {
    constexpr const char* name = "ScalarLeafCall";
    constexpr uintptr_t base = 0x0000000204800000ull;
    constexpr uint64_t size = 0x10000;
    constexpr auto table = base + 0x100;
    constexpr auto first = base + 0x1000;
    constexpr auto second = base + 0x2000;
    constexpr auto output = base + 0x4000;
    std::vector<u32> code{
        EncodeSMovB32(20, InlineU32(0)),
        0xbeea03ffu, static_cast<u32>(base), 0xbeeb03ffu, static_cast<u32>(base >> 32u),
        0xf4080135u, 0xfa000000u, 0xbf8cc07fu, 0xf4240382u, 0xfa000020u,
        0xbf8cc07fu, 0xbf130e80u, 0xbf840004u, 0xf4240402u, 0xfa000028u,
        0xbf8cc07fu, 0xbe8e210eu, EncodeVop1(0x01, 1, 20),
        EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(1, 0, 0)};
    AppendEnd(&code);
    ShaderMapUserData(reinterpret_cast<uint64_t>(code.data()),
        {.type = Prospero::ShaderBinaryType::kCs,
         .code_size_bytes = static_cast<u32>(code.size() * sizeof(u32))});
    EnsureRuntimeContext();
    int64_t physical = -1;
    Require(name, "allocate", LibKernel::Memory::KernelAllocateDirectMemory(
        0, LibKernel::Memory::KernelGetDirectMemorySize(), size, size, 0, &physical) == 0,
        "call fixture allocation failed");
    void* mapped = reinterpret_cast<void*>(base);
    Require(name, "map", LibKernel::Memory::KernelMapDirectMemory(
        &mapped, size, 0x3, 0x10, physical, size) == 0 && mapped == reinterpret_cast<void*>(base),
        "call fixture mapping failed");
    std::memset(mapped, 0, size);
    const ShaderBufferResource table_desc{{static_cast<u32>(table),
        static_cast<u32>(table >> 32u) | (16u << 16u), 8, 0x5204u}};
    LibKernel::Memory::WriteBacking(base, table_desc.fields, sizeof(table_desc.fields));
    const std::array leaf{EncodeSMovB32(20, 255), 41u, EncodeSop1(0x21, 125, 14)};
    for (const auto address : {first, second})
      LibKernel::Memory::WriteBacking(address, leaf.data(), sizeof(leaf));

    RenderContext context(m_runtime_context);
    context.InitializeGpu(nullptr);
    LibKernel::Memory::InstallGpuResources(&context);
    context.GetGpu().SendCommandSync([&] {
      CommandProcessor processor(context, 0);
      processor.BufferInit();
      context.MapMemory(base, size);
      auto& cache = context.GetBufferCache();
      auto& shaders = processor.GetShCtx();
      shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(code.data()),
                           .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
                           .wave_size = 64, .user_sgpr = 4});
      ShaderBufferResource out{};
      out.UpdateAddress48(output);
      out.fields[2] = 4;
      out.fields[3] = DstSel(4, 5, 6, 7) |
          (static_cast<u32>(Prospero::BufferFormat::k32UInt) << 12u);
      for (u32 i = 0; i < 4; ++i)
        shaders.SetCsUserSgpr(i, out.fields[i], HW::UserSgprType::Unknown);
      uint64_t original_id = 0;
      struct Case { uint64_t address; u32 value; bool update; u32 field = 0; };
      for (const auto test : {Case{first, 41, false}, Case{first, 41, false},
                              Case{second, 41, false}, Case{second, 73, true},
                              Case{first, 41, false}, Case{first, 6, false, 0x1818},
                              Case{first, 0, false, 0x0a18}}) {
        Require(name, "refresh function pointer", context.InvalidateMemory(table + 0x20, 8),
                "function table escaped mapped memory");
        const uint64_t target = test.address | 3u;
        LibKernel::Memory::WriteBacking(table + 0x20, &target, sizeof(target));
        if (test.field != 0) {
          Require(name, "refresh hardware-field leaf", context.InvalidateMemory(first, sizeof(leaf)),
                  "hardware-field leaf escaped mapped memory");
          const std::array field_leaf{0xb96a0000u | test.field,
              EncodeSMovB32(20, 106), EncodeSop1(0x21, 125, 14)};
          LibKernel::Memory::WriteBacking(first, field_leaf.data(), sizeof(field_leaf));
        }
        if (test.update) {
          auto [buffer, offset] = cache.ObtainBuffer(test.address + 4, 4, true);
          buffer->Fill(offset, 4, test.value);
          Require(name, "GPU-written function code", cache.HasGpuDirtyBytes(test.address + 4, 4),
                  "code mutation did not remain GPU-owned before refresh");
        }
        ShaderComputeInputInfo input{};
        const auto program = context.GetPipelineCache().GetComputeProgram(
            shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
        if (original_id == 0) original_id = program.id;
        Require(name, "exact function identity",
                (program.id == original_id) == (test.value == 41),
                "helper bytes were stale or identical relocated/restored code recompiled");
        Require(name, "current code dependency",
                std::ranges::find(input.stage.resources->specialization_reads,
                    std::pair<uint64_t, uint64_t>{test.address, sizeof(leaf)}) !=
                    input.stage.resources->specialization_reads.end(),
                "function relocation retained an old code dependency");
        processor.DispatchDirect(1, 1, 1, 0x41u);
        cache.ReadMemory(output, sizeof(u32));
        u32 actual = 0;
        Require(name, "scalar leaf execution and return",
                LibKernel::Memory::TryReadBacking(output, &actual, sizeof(actual)) && actual == test.value,
                "caller did not observe the leaf's scalar result after return");
      }
      context.UnmapMemory(base, size);
      context.GetCommandScheduler().Finish();
    });
    LibKernel::Memory::InstallGpuResources(nullptr);
    context.ShutdownGpu();
    Require(name, "release", LibKernel::Memory::KernelCheckedReleaseDirectMemory(physical, size) == 0,
            "call fixture release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckNativeIndirectDispatch() {
    constexpr const char *name = "NativeIndirectDispatch";
    constexpr uintptr_t base = 0x0000000204600000ull;
    constexpr uint64_t allocation_size = 0x100000;
    constexpr uint64_t case_size = 0x10000;
    constexpr uint32_t sentinel = 0xa5a5a5a5u;
    struct DispatchCase {
      std::array<uint32_t, 3> dimensions;
      uint32_t mode;
      uint32_t expected_threads;
      bool transfer = false;
    };
    constexpr std::array cases{
        DispatchCase{{2, 1, 1}, 0x41u, 8},
        DispatchCase{{0, 1, 1}, 0x41u, 0},
        DispatchCase{{1, 0, 1}, 0x41u, 0},
        DispatchCase{{1, 1, 0}, 0x41u, 0},
        DispatchCase{{3, 1, 1}, 0x41u, 12, true},
        DispatchCase{{8, 1, 1}, 0x61u, 8},
    };

    // Separate DWORD descriptors preserve two owners until the indirect argument
    // range spans them. The consumer's output aliases the second owner.
    std::vector<u32> writer;
    for (u32 i = 0; i < 3; i++) {
      writer.push_back(EncodeVop1(0x01u, 1, 12u + i));
      writer.push_back(EncodeMubuf0(0x1cu, 0, false, false));
      writer.push_back(EncodeMubuf1(1, i, 0));
    }
    AppendEnd(&writer);
    std::vector<u32> consumer;
    AppendVop3(&consumer, 0x346u, 1, 4, InlineU32(2), Vgpr(0));
    consumer.push_back(EncodeVop2(0x25u, 2, InlineU32(1), 1));
    consumer.push_back(EncodeVop2(0x1au, 1, InlineU32(2), 1));
    consumer.push_back(EncodeMubuf0(0x1cu));
    consumer.push_back(EncodeMubuf1(2, 0, 1));
    AppendEnd(&consumer);
    std::vector<u32> counted_consumer{
        EncodeMubuf0(0x0cu, 0, false, false), EncodeMubuf1(3, 1, 0),
        EncodeSopp(0x0c, 0), EncodeVop1(0x02, 2, Vgpr(3))};
    AppendVop3(&counted_consumer, 0x346u, 1, 8, InlineU32(2), Vgpr(0));
    AppendVop3(&counted_consumer, 0x36du, 2, Vgpr(1), Vgpr(3), InlineU32(1));
    counted_consumer.push_back(EncodeMubuf0(0x1cu, 0, true, false));
    counted_consumer.push_back(EncodeMubuf1(2, 0, 1));
    AppendEnd(&counted_consumer);
    for (const auto *code : {&writer, &consumer, &counted_consumer}) {
      ShaderMapUserData(reinterpret_cast<uint64_t>(code->data()),
          {.type = Prospero::ShaderBinaryType::kCs,
           .code_size_bytes = static_cast<uint32_t>(code->size() * sizeof(u32))});
    }

    EnsureRuntimeContext();
    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, case_size, 0, &direct_offset) == 0,
            "indirect argument allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset, case_size) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "indirect argument mapping failed");
    std::memset(mapped, 0xa5, allocation_size);
    const auto argument_address = [&](size_t index) {
      return base + index * case_size +
          (cases[index].transfer ? 0x200u : BufferCache::CACHING_PAGESIZE - 4u);
    };
    for (size_t i = 0; i < cases.size(); i++) {
      std::memset(reinterpret_cast<void *>(argument_address(i)), 0, 12);
    }

    RenderContext context(m_runtime_context);
    context.InitializeGpu(nullptr);
    LibKernel::Memory::InstallGpuResources(&context);
    context.GetGpu().SendCommandSync([&] {
      CommandProcessor processor(context, 0);
      processor.BufferInit();
      auto &scheduler = context.GetCommandScheduler();
      auto &cache = context.GetBufferCache();
      auto &shaders = processor.GetShCtx();
      context.MapMemory(base, allocation_size);
      const auto set_buffer = [&](u32 sgpr, uint64_t address, u32 records, u32 stride = 0) {
        ShaderBufferResource descriptor{};
        descriptor.UpdateAddress48(address);
        descriptor.fields[1] |= stride << 16u;
        descriptor.fields[2] = records;
        descriptor.fields[3] = DstSel(4, 5, 6, 7) |
            (static_cast<u32>(Prospero::BufferFormat::k32UInt) << 12u);
        for (u32 i = 0; i < 4; i++) {
          shaders.SetCsUserSgpr(sgpr + i, descriptor.fields[i],
                               HW::UserSgprType::Unknown);
        }
      };
      for (size_t index = 0; index < cases.size(); index++) {
        const auto &test = cases[index];
        const auto args = argument_address(index);
        const auto output = base + index * case_size + BufferCache::CACHING_PAGESIZE + 0x100u;
        if (test.transfer) {
          for (u32 i = 0; i < 3; i++) {
            auto [buffer, offset] = cache.ObtainBuffer(args + i * 4u, 4, true);
            buffer->Fill(offset, 4, test.dimensions[i]);
          }
        } else {
          shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(writer.data()),
                               .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
                               .wave_size = 64, .user_sgpr = 15});
          for (u32 i = 0; i < 3; i++) {
            set_buffer(i * 4u, args + i * 4u, 4);
            shaders.SetCsUserSgpr(12u + i, test.dimensions[i], HW::UserSgprType::Unknown);
          }
          processor.DispatchDirect(1, 1, 1, 0x41u);
        }
        std::array<u32, 3> stale{};
        Require(name, "GPU-only argument write",
                LibKernel::Memory::TryReadBacking(args, stale.data(), sizeof(stale)) &&
                    stale == std::array<u32, 3>{},
                "argument publication changed the CPU backing before indirect dispatch");
        const auto first_owner = BufferCacheTestAccess::PageOwner(cache, args);
        const auto second_owner = BufferCacheTestAccess::PageOwner(cache, args + 8u);
        if (!test.transfer) {
          Require(name, "separate argument owners",
                  first_owner && second_owner && first_owner != second_owner,
                  "the indirect argument merge fixture already shared one owner");
        }

        shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(consumer.data()),
                             .num_thread_x = 4, .num_thread_y = 1, .num_thread_z = 1,
                             .wave_size = 64, .user_sgpr = 4, .tgid_x_en = true});
        set_buffer(0, output, 16u * sizeof(u32));
        const auto tick = scheduler.CurrentTick();
        if (test.transfer) {
          const std::array<u32, 3> packet{static_cast<u32>(args),
                                         static_cast<u32>(args >> 32u), test.mode};
          Require(name, "absolute indirect packet",
                  CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3,
                  "the absolute indirect packet was not consumed");
        } else {
          processor.SetDispatchIndirectArgsBaseAddress(base + index * case_size);
          const std::array<u32, 2> packet{
              static_cast<u32>(BufferCache::CACHING_PAGESIZE - 4u), test.mode};
          Require(name, "offset indirect packet",
                  CpOpDispatchIndirect(processor, 0xc0011600u, packet.data(), 0, 0) == 2,
                  "the offset indirect packet was not consumed");
        }
        if (test.mode == 0x41u) {
          Require(name, "asynchronous indirect dispatch", scheduler.CurrentTick() == tick,
                  "workgroup-count arguments caused a host submission or readback");
          if (!test.transfer) {
            const auto merged = BufferCacheTestAccess::PageOwner(cache, args);
            Require(name, "argument/output owner merge",
                    merged && merged != first_owner && merged != second_owner &&
                        merged == BufferCacheTestAccess::PageOwner(cache, output),
                    "argument discovery did not merge the consumer's buffer owner");
          }
        }
        cache.ReadMemory(output, 16u * sizeof(u32));
        std::array<u32, 16> actual{};
        Require(name, "output readback",
                LibKernel::Memory::TryReadBacking(output, actual.data(), sizeof(actual)),
                "consumer output could not be read back");
        for (u32 i = 0; i < actual.size(); i++) {
          const auto expected = i < test.expected_threads ? i + 1u : sentinel;
          Require(name, "indirect invocation coverage", actual[i] == expected,
                  "case " + std::to_string(index) + " word " + std::to_string(i) +
                      " expected " + Hex(expected) + ", got " + Hex(actual[i]));
        }
      }
      // The same GPU-written DWORD supplies indirect X and the output descriptor's
      // native NUM_RECORDS. Four lanes per group exercise the descriptor's OOB bound.
      constexpr auto image_address = base + cases.size() * case_size;
      constexpr auto count_args = image_address + 4u * sizeof(u32);
      constexpr auto count_output = image_address + 2u * BufferCache::CACHING_PAGESIZE;
      constexpr u32 image_value = 0x13579bdfu;
      auto &textures = context.GetTextureCache();
      auto image_desc = MakeLinearDesc(
          image_address, 8u * sizeof(u32), vk::Format::eR32Uint,
          Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
          {8, 1, 1}, 1, 4, 1);
      const auto image = textures.FindImage(image_desc);
      ShaderProgram previous_program;
      for (const auto count : {3u, 7u}) {
        cache.FillBuffer(count_output, 16u * sizeof(u32), sentinel, false);
        shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(writer.data()),
                             .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
                             .wave_size = 64, .user_sgpr = 15});
        for (u32 i = 0; i < 3; i++) {
          set_buffer(i * 4u, count_args + i * 4u, 4);
          shaders.SetCsUserSgpr(12u + i, i == 0 ? count : 1u,
                               HW::UserSgprType::Unknown);
        }
        processor.DispatchDirect(1, 1, 1, 0x41u);
        Require(name, "GPU-owned descriptor count", cache.HasGpuDirtyBytes(count_args, 4),
                "the descriptor count was already visible to the CPU");
        // Raw buffer access keeps its own contents even after a newer overlapping image write.
        vk::ClearValue painted{};
        painted.color.uint32 = std::array{image_value + count, 0u, 0u, 0u};
        TextureCacheTestAccess::ClearImage(
            textures, scheduler.Current(), image,
            {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
        u32 stale_count = 0;
        Require(name, "independent raw buffer and image contents",
                textures.GetImage(image).SafeToDownload() &&
                    cache.HasGpuDirtyBytes(count_args, sizeof(u32)) &&
                    LibKernel::Memory::TryReadBacking(count_args, &stale_count,
                                                     sizeof(stale_count)) &&
                    stale_count != count,
                "the alias fixture did not retain GPU buffer data and a newer independent image");

        shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(counted_consumer.data()),
                             .num_thread_x = 4, .num_thread_y = 1, .num_thread_z = 1,
                             .wave_size = 64, .user_sgpr = 8, .tgid_x_en = true});
        set_buffer(0, count_output, 0, 4);
        set_buffer(4, count_args, 1, 4);
        const std::array<u32, 3> packet{static_cast<u32>(count_args),
                                      static_cast<u32>(count_args >> 32u), 0x41u};
        const auto dirty_tick = scheduler.CurrentTick();
        Require(name, "GPU count indirect dispatch",
                CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3 &&
                    scheduler.CurrentTick() > dirty_tick &&
                    !cache.HasGpuDirtyBytes(count_args, 4),
                "the descriptor dependency did not synchronize its preceding GPU writer");
        const auto clean_tick = scheduler.CurrentTick();
        ShaderComputeInputInfo input{};
        const auto program = context.GetPipelineCache().GetComputeProgram(
            shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
        if (previous_program.id != 0) {
          Require(name, "count permutation reuse", program.id == previous_program.id,
                  "a changing GPU count compiled a new shader permutation");
        }
        previous_program = program;
        Require(name, "clean count repeat",
                CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3 &&
                    scheduler.CurrentTick() == clean_tick,
                "a clean descriptor dependency caused another GPU drain");
        cache.ReadMemory(count_output, 16u * sizeof(u32));
        std::array<u32, 16> actual{};
        Require(name, "counted output readback",
                LibKernel::Memory::TryReadBacking(count_output, actual.data(), sizeof(actual)),
                "the counted consumer output could not be read back");
        for (u32 i = 0; i < actual.size(); i++) {
          Require(name, "native descriptor count bound",
                  actual[i] == (i < count ? count + i + 1u : sentinel),
                  "GPU count " + std::to_string(count) + " wrote an incorrect output word " +
                      std::to_string(i));
        }
        Require(name, "raw count preserves independent image contents",
                ReadCachedTexel(name, context, image, {}, {8, 1, 1}) ==
                    std::vector<u32>(8, image_value + count),
                "raw buffer descriptor materialization overwrote the newer native image");
      }

      // PPSA24156 loads an image sharp from a 148-byte record selected by
      // WorkGroupID.z, then uses the same descriptor words for dimensions.
      constexpr auto parameters = base + 0x80000;
      constexpr auto table_address = parameters + 0x100;
      std::vector<u32> table_shader{
          EncodeSop2(0x26, 8, 4, 255), 148,
          EncodeSmem0(0x02, 16), EncodeSmem1(0, 125),
          EncodeSmem0(0x0b, 32, 8), EncodeSmem1(0, 8),
          EncodeSop2(0x1e, 8, 34, InlineU32(2)),
          EncodeSop2(0x20, 9, 33, InlineU32(30)),
          EncodeSop2(0x0e, 8, 8, 255), 0xfffcu,
          EncodeSop2(0x10, 8, 8, 9),
          EncodeSop2(0x27, 9, 34, 255), 0x000e000eu,
          EncodeSop2(0x02, 8, 8, 9),
          EncodeSop2(0x02, 8, 8, InlineU32(2)),
          EncodeVop1(0x01, 0, 4), EncodeVop1(0x01, 1, InlineU32(0)),
          EncodeVop1(0x01, 2, 8),
          EncodeMimg0(0x08, 1), EncodeMimg1(2, 0, 8)};
      AppendEnd(&table_shader);
      ShaderMapUserData(reinterpret_cast<uint64_t>(table_shader.data()),
          {.type = Prospero::ShaderBinaryType::kCs,
           .code_size_bytes = static_cast<u32>(table_shader.size() * sizeof(u32))});
      shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(table_shader.data()),
                           .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
                           .wave_size = 64, .user_sgpr = 2, .tgid_x_en = true,
                           .tgid_y_en = true, .tgid_z_en = true});
      shaders.SetCsUserSgpr(0, static_cast<u32>(parameters), HW::UserSgprType::Unknown);
      shaders.SetCsUserSgpr(1, static_cast<u32>(parameters >> 32u), HW::UserSgprType::Unknown);
      const ShaderBufferResource table_buffer{{static_cast<u32>(table_address),
          static_cast<u32>(table_address >> 32u) | (148u << 16u), 2, 0x5204u}};
      LibKernel::Memory::WriteBacking(parameters, table_buffer.fields, sizeof(table_buffer.fields));
      struct TableCase { u32 groups, width, height, image; };
      uint64_t table_program_id = 0;
      for (const auto test : {TableCase{1, 4, 2, 0}, TableCase{2, 4, 2, 0},
                              TableCase{2, 8, 3, 1}, TableCase{1, 4, 2, 0}}) {
        const auto address = base + 0x90000 + test.image * 0x10000;
        const ShaderTextureResource sharp{{static_cast<u32>(address >> 8u),
            (static_cast<u32>(Prospero::BufferFormat::k32UInt) << 20u) |
                (((test.width - 1u) & 3u) << 30u),
            ((test.width - 1u) >> 2u) | ((test.height - 1u) << 14u),
            DstSel(4, 5, 6, 7) |
                (static_cast<u32>(Prospero::TileMode::kLinear) << 20u) |
                (static_cast<u32>(Prospero::ImageType::kColor2D) << 28u),
            0, 0x00700000u, 0, 0}};
        std::array<u32, 74> table{};
        for (u32 group = 0; group < test.groups; group++) {
          std::copy_n(sharp.fields, 8, table.begin() + group * 37);
        }
        Require(name, "descriptor table refresh", context.InvalidateMemory(table_address, sizeof(table)),
                "the descriptor table escaped its mapped allocation");
        LibKernel::Memory::WriteBacking(table_address, table.data(), sizeof(table));
        ShaderComputeInputInfo input{};
        input.workgroup_counts[0] = input.workgroup_counts[1] = 1;
        input.workgroup_counts[2] = test.groups;
        const auto program = context.GetPipelineCache().GetComputeProgram(
            shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
        Require(name, "workgroup table refresh and coalescing",
                input.workgroup_counts[2] == test.groups &&
                    input.stage.program->info.images.size() == 1 &&
                    input.stage.program->info.images[0].indirect_resources.empty() &&
                    input.stage.resources->images.size() == 1 &&
                    std::equal(std::begin(sharp.fields), std::end(sharp.fields),
                               input.stage.resources->images[0].dwords.begin()) &&
                    !input.stage.program->info.buffers.empty(),
                "workgroup bounds, refreshed sharp, or live descriptor dimension reads were lost");
        Require(name, "workgroup table program reuse",
                table_program_id == 0 || program.id == table_program_id,
                "changing the workgroup count or identical sharp values recompiled the shader");
        table_program_id = program.id;
        const auto *snapshot = input.stage.resources;
        const auto *image_storage = snapshot->images.data();
        const auto flattened_capacity = snapshot->flattened_srt.capacity();
        const auto repeated = context.GetPipelineCache().GetComputeProgram(
            shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
        Require(name, "workgroup table cache-hit storage reuse",
                repeated.id == program.id && input.stage.resources == snapshot &&
                    snapshot->images.data() == image_storage &&
                    snapshot->flattened_srt.capacity() == flattened_capacity,
                "a repeated descriptor table hit replaced its snapshot or storage");
        auto binding = RenderExecutorTestAccess::ResolveTexture(
            context.GetRenderExecutor(), input.stage.program->info.images[0],
            input.stage.resources->images[0]);
        const auto target = textures.FindImage(binding.desc);
        TextureCacheTestAccess::ClearImage(textures, scheduler.Current(), target,
            {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
        processor.DispatchDirect(1, 1, test.groups, 0x41u);
        const auto pixels = ReadCachedTexel(name, context, target, {},
                                            {test.width, test.height, 1});
        for (u32 pixel = 0; pixel < pixels.size(); pixel++) {
          const auto expected = pixel < test.groups ? test.width + test.height : 0u;
          Require(name, "workgroup table GPU dimension reads", pixels[pixel] == expected,
                  "a retained descriptor load used stale dimensions or the wrong record stride");
        }
      }

      // Exchange values across two waves in the last 512 bytes of each
      // workgroup's 64-KiB LDS, on a host restricted to 48-KiB native shared memory.
      constexpr auto lds_output = base + 0xb0000;
      constexpr auto lds_args = base + 0xc0000;
      constexpr u32 lds_words = 8 * 128;
      std::vector<u32> lds_shader{
          EncodeSop2(0x1e, 8, 5, InlineU32(1)),
          EncodeSop2(0x1e, 9, 6, InlineU32(2)),
          EncodeSop2(0x02, 8, 8, 9), EncodeSop2(0x02, 8, 8, 4)};
      AppendVop3(&lds_shader, 0x346, 2, 8, InlineU32(7), Vgpr(0));
      lds_shader.push_back(EncodeVop2(0x1a, 5, InlineU32(2), 2));
      lds_shader.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2));
      lds_shader.push_back(EncodeVop2(0x1a, 1, InlineU32(2), 0));
      AppendVMovLiteral(&lds_shader, 7, 64 * 1024 - 128 * 4);
      lds_shader.insert(lds_shader.end(), {
          EncodeVop2(0x25, 1, Vgpr(7), 1),
          EncodeDs0(0x0d), EncodeDs1(0, 2, 1), EncodeSopp(0x0a, 0),
          EncodeVop2(0x1d, 3, InlineU32(64), 0),
          EncodeVop2(0x1a, 3, InlineU32(2), 3), EncodeVop2(0x25, 3, Vgpr(7), 3),
          EncodeDs0(0x36), EncodeDs1(4, 0, 3),
          EncodeMubuf0(0x1c), EncodeMubuf1(4, 0, 5)});
      AppendEnd(&lds_shader);
      ShaderMapUserData(reinterpret_cast<uint64_t>(lds_shader.data()),
          {.type = Prospero::ShaderBinaryType::kCs,
           .code_size_bytes = static_cast<u32>(lds_shader.size() * sizeof(u32))});
      auto &host_lds_limit = context.GetGraphics().physical_device_properties.limits.maxComputeSharedMemorySize;
      const auto saved_lds_limit = host_lds_limit;
      host_lds_limit = std::min(host_lds_limit, 48u * 1024u);
      for (u32 dispatch = 0; dispatch < 4; dispatch++) {
        cache.FillBuffer(lds_output, lds_words * sizeof(u32), sentinel, false);
        const bool indirect = dispatch == 1 || dispatch == 2;
        const bool empty = dispatch == 2;
        if (indirect) {
          const auto groups = empty ? std::array{4096u, 0u, 1u} : std::array{2u, 2u, 2u};
          shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(writer.data()),
                               .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
                               .wave_size = 64, .user_sgpr = 15});
          for (u32 axis = 0; axis < 3; axis++) {
            set_buffer(axis * 4, lds_args + axis * sizeof(u32), sizeof(u32));
            shaders.SetCsUserSgpr(12 + axis, groups[axis], HW::UserSgprType::Unknown);
          }
          processor.DispatchDirect(1, 1, 1, 0x41u);
          Require(name, "GPU-authored LDS dispatch arguments",
                  cache.HasGpuDirtyBytes(lds_args, 3 * sizeof(u32)),
                  "indirect LDS dimensions were already visible on the CPU");
        }
        shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(lds_shader.data()),
                             .num_thread_x = 128, .num_thread_y = 1, .num_thread_z = 1,
                             .wave_size = 64, .user_sgpr = 4, .tgid_x_en = true,
                             .tgid_y_en = true, .tgid_z_en = true, .lds_size = 128});
        set_buffer(0, lds_output, lds_words * sizeof(u32));
        if (indirect) {
          const std::array<u32, 3> packet{static_cast<u32>(lds_args),
                                         static_cast<u32>(lds_args >> 32u), 0x41u};
          Require(name, "indirect storage LDS dispatch",
                  CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3,
                  "the GPU-authored LDS dispatch was not consumed");
        } else {
          processor.DispatchDirect(2, 2, 2, 0x41u);
        }
        cache.ReadMemory(lds_output, lds_words * sizeof(u32));
        std::array<u32, lds_words> actual{};
        Require(name, "storage LDS readback",
                LibKernel::Memory::TryReadBacking(lds_output, actual.data(), sizeof(actual)),
                "LDS result buffer could not be read back");
        for (u32 word = 0; word < actual.size(); word++) {
          const auto expected = empty ? sentinel : (word ^ 64u) + 1u;
          Require(name, "storage LDS upper range, isolation and barrier", actual[word] == expected,
                  "64-KiB LDS lost upper bytes, workgroup isolation, or cross-wave writes");
        }
      }
      host_lds_limit = saved_lds_limit;
      context.UnmapMemory(base, allocation_size);
      scheduler.Finish();
    });
    LibKernel::Memory::InstallGpuResources(nullptr);
    context.ShutdownGpu();
    Require(name, "unmap direct backing",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "indirect argument mapping release failed");
    Require(name, "release direct backing",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "indirect argument allocation release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckRenderExecutorColorMetadataClear() {
    constexpr const char *name = "RenderExecutorColorMetadataClear";
    struct PackedCase {
      vk::Format format;
      uint32_t word;
      std::array<float, 4> channels;
    };
    const std::array packed_cases{
        PackedCase{vk::Format::eR16Sfloat, 0x7bff7bffu, {65504.0f}},
        PackedCase{vk::Format::eR16G16Sfloat, 0xbc003c00u, {1.0f, -1.0f}},
        PackedCase{vk::Format::eR16Sfloat, 0xffff0001u, {0x1p-24f}},
        PackedCase{vk::Format::eR16Sfloat, 0x00008001u, {-0x1p-24f}},
        PackedCase{vk::Format::eR16Sfloat, 0x000003ffu, {1023.0f * 0x1p-24f}},
        PackedCase{vk::Format::eR16Sfloat, 0x00000400u, {0x1p-14f}},
        PackedCase{vk::Format::eR16G16Sfloat, 0x80000000u, {0.0f, -0.0f}},
        PackedCase{vk::Format::eR16G16Sfloat, 0xfc007c00u,
                   {std::numeric_limits<float>::infinity(),
                    -std::numeric_limits<float>::infinity()}},
        PackedCase{vk::Format::eR16Unorm, 0x0000ffffu, {1.0f}},
        PackedCase{vk::Format::eR16G16Unorm, 0x0000ffffu, {1.0f, 0.0f}},
        PackedCase{vk::Format::eR16G16Unorm, 0x80000001u,
                   {1.0f / 65535.0f, 32768.0f / 65535.0f}},
    };
    for (const auto &test : packed_cases) {
      vk::ClearColorValue clear{};
      Require(name, "packed 16-bit clear",
              DecodePackedColorClear(test.format, test.word, clear) &&
                  std::bit_cast<std::array<uint32_t, 4>>(clear.float32) ==
                      std::bit_cast<std::array<uint32_t, 4>>(test.channels),
              "16-bit clear lost channel order, sign, range or unused-channel zeros");
    }
    vk::ClearColorValue clear{};
    Require(name, "half NaN clear",
            DecodePackedColorClear(vk::Format::eR16Sfloat, 0x7e00u, clear) &&
                std::isnan(clear.float32[0]),
            "half NaN did not remain NaN");
    for (const auto format : {vk::Format::eR16Sfloat, vk::Format::eR16Unorm}) {
      Require(name, "R16 DWORD fill",
              !DecodeColorDwordFill(format, 0xbc003c00u, clear) &&
                  DecodeColorDwordFill(format, 0x3c003c00u, clear),
              "alternating R16 pixels were treated as a uniform clear");
    }
    constexpr uintptr_t base = 0x0000000204100000ull;
    constexpr uint64_t allocation_size = 0x200000;
    constexpr uint64_t allocation_alignment = 0x10000;
    struct FillCase {
      uint32_t fill;
      std::array<uint32_t, 2> texel;
      bool reuse_unorm = false;
      bool cmask = false;
      bool reuse_mips = false;
      Prospero::ChannelLayout layout = Prospero::ChannelLayout::k16_16_16_16;
      Prospero::ChannelType type = Prospero::ChannelType::kFloat;
      uint32_t clear_word = 0;
    };
    constexpr std::array cases{
        FillCase{0x40404040u, {0, 0x3c000000u}},
        FillCase{0x80808080u, {0x3c003c00u, 0x00003c00u}},
        FillCase{0x40404040u, {0, 0x3c000000u}, true},
        FillCase{0, {0x804020ffu, 0}, false, true},
        FillCase{0x40404040u, {0, 0x3c000000u}, false, false, true},
        FillCase{.fill = 0x20202020u, .texel = {0x7bff7bffu},
                 .layout = Prospero::ChannelLayout::k16, .clear_word = 0x7bff7bffu},
        FillCase{.fill = 0x20202020u, .texel = {0xbc003c00u},
                 .layout = Prospero::ChannelLayout::k16_16, .clear_word = 0xbc003c00u},
        FillCase{.fill = 0x20202020u, .texel = {0x80008000u},
                 .layout = Prospero::ChannelLayout::k16,
                 .type = Prospero::ChannelType::kUNorm, .clear_word = 0x80008000u},
        FillCase{.fill = 0x20202020u, .texel = {0x0000ffffu},
                 .layout = Prospero::ChannelLayout::k16_16,
                 .type = Prospero::ChannelType::kUNorm, .clear_word = 0x0000ffffu},
    };
    EnsureRuntimeContext();
    // Astro's generic metadata fill, through S_ENDPGM; trailing debug data is omitted.
    static constexpr std::array<uint32_t, 69> native_fill{
        0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u,
        0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu,
        0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u,
        0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u,
        0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u,
        0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u,
        0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d860609u,
        0x7d8c02f9u, 0x06068c02u, 0x87ea6a0cu, 0x50000080u,
        0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u,
        0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u,
        0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au,
        0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u,
        0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au,
        0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u,
        0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u,
        0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u,
        0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u,
        0xbf810000u,
    };
    static constexpr auto raw_fill = [] {
      auto words = native_fill;
      for (auto &word : words) {
        if (word == 0xe0102000u) {
          word = 0xe0702000u; // BUFFER_STORE_DWORD in place of BUFFER_STORE_FORMAT_X.
        }
      }
      return words;
    }();
    for (const auto *shader : {&native_fill, &raw_fill}) {
      ShaderMapUserData(reinterpret_cast<uint64_t>(shader->data()),
          {.type = Prospero::ShaderBinaryType::kCs,
           .code_size_bytes = sizeof(native_fill)});
    }
    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_alignment, 0, &direct_offset) == 0,
            "fixed-clear direct-memory allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_alignment) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "fixed-clear direct mapping failed");
    std::memset(mapped, 0, allocation_size);

    for (const auto &fill_case : cases) {
      const auto layout = fill_case.cmask ? Prospero::ChannelLayout::k8_8_8_8 : fill_case.layout;
      const auto type = fill_case.cmask ? Prospero::ChannelType::kUNorm : fill_case.type;
      const auto bytes_per_pixel = TextureGetRenderTargetFormat(
          layout, type, Prospero::ChannelOrder::kStandard).bytes_per_element;
      const uint32_t probe_width = bytes_per_pixel == 2 ? 2 : 1;
      TileSizeAlign dcc_size{};
      (void)TileGetDccSize(512, 256, 1, bytes_per_pixel, 1,
                          Prospero::TileMode::kRenderTarget, dcc_size, 0);
      const uint64_t dcc_address = base + (fill_case.reuse_mips ? 0x160000 : 0x100000);
      const uint32_t selected_layer = fill_case.cmask ? 1 : 0;
      const uint64_t metadata_size = fill_case.cmask ? 0x2000 : dcc_size.size;
      const std::vector<u32> expected(
          fill_case.texel.begin(), fill_case.texel.begin() + std::max(bytes_per_pixel / 4u, 1u));
      RenderContext context(m_runtime_context);
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      context.InitializeGpu(nullptr);
      LibKernel::Memory::InstallGpuResources(&context);
      context.GetGpu().SendCommandSync([&] {
        auto &scheduler = context.GetCommandScheduler();
        registers.SetColorBase(0, {.addr = base});
        registers.SetColorInfo(
            0, {.cmask_fast_clear_enable = !fill_case.reuse_unorm,
                .dcc_compression_enable = !fill_case.cmask,
                .format = layout,
                .channel_type = type,
                .channel_order = Prospero::ChannelOrder::kStandard});
        registers.SetColorView(0, {.base_array_slice_index = selected_layer,
                                   .last_array_slice_index = selected_layer});
        registers.SetColorAttrib2(0, {.height = 255, .width = 511});
        registers.SetColorAttrib3(0,
                                  {.tile_mode = Prospero::TileMode::kRenderTarget,
                                   .dimension = 1,
                                   .metadata_pipe_aligned = true});
        registers.SetColorDccAddr(0, {.addr = dcc_address});
        registers.SetColorCmask(0, {.addr = dcc_address});
        registers.SetColorClearWord0(0, {.word0 = fill_case.cmask ? expected[0] : fill_case.clear_word});
        registers.SetColorClearWord1(0, {.word1 = 0});
        registers.SetRenderTargetMask(0x0f);
        scheduler.Begin(registers, user_config, shaders);

        auto &resources = context;
        auto &texture_cache = resources.GetTextureCache();
        auto &executor = context.GetRenderExecutor();
        resources.MapMemory(base, allocation_size);
        ImageId preexisting_id{};
        if (fill_case.reuse_unorm || fill_case.reuse_mips) {
          const auto float_info = registers.GetRenderTarget(0).info;
          const auto single_mip = registers.GetRenderTarget(0).attrib2;
          auto initial_info = float_info;
          initial_info.dcc_compression_enable = false;
          initial_info.cmask_fast_clear_enable = false;
          if (fill_case.reuse_unorm) {
            initial_info.channel_type = Prospero::ChannelType::kUNorm;
          }
          auto initial_mips = single_mip;
          initial_mips.num_mip_levels = fill_case.reuse_mips ? 1 : 0;
          registers.SetColorInfo(0, initial_info);
          registers.SetColorAttrib2(0, initial_mips);
          RenderColorInfo preexisting{};
          RenderExecutorTestAccess::ResolveRenderColorTarget(
              executor, scheduler.Current(), preexisting, 0);
          preexisting_id = preexisting.image_id;
          (void)texture_cache.FindRenderTarget(preexisting_id, preexisting.desc);
          if (fill_case.reuse_mips) {
            Require(name, "mip chain excludes metadata",
                    preexisting.desc.info.data.address + preexisting.desc.info.data.size <=
                        dcc_address,
                    "the mip preservation fixture overlaps its color metadata");
            vk::ClearValue clear{};
            clear.color.float32 = std::array{1.0f, 0.0f, 1.0f, 0.0f};
            TextureCacheTestAccess::ClearImage(
                texture_cache, scheduler.Current(), preexisting_id,
                {vk::ImageAspectFlagBits::eColor, 0, 2, 0, 1}, clear);
          }
          RenderExecutorTestAccess::ResetBindings(executor);
          registers.SetColorInfo(0, float_info);
          registers.SetColorAttrib2(0, single_mip);
        }
        const auto fill_metadata = [&](uint32_t count, bool raw = false) {
          const auto *shader = &native_fill;
          if (raw) {
            shader = &raw_fill;
          }
          shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(shader->data()),
                               .num_thread_x = 64, .num_thread_y = 1, .num_thread_z = 1,
                               .wave_size = 64, .user_sgpr = 10, .tgid_x_en = true});
          const std::array<uint32_t, 10> user_data{
              static_cast<uint32_t>(dcc_address),
              static_cast<uint32_t>(dcc_address >> 32u) | (4u << 16u),
              count, 0x00014004u, fill_case.fill, 0, 0x200, 0, count, 1};
          for (uint32_t i = 0; i < user_data.size(); i++) {
            shaders.SetCsUserSgpr(i, user_data[i], HW::UserSgprType::Unknown);
          }
          executor.DispatchDirect(0, scheduler.Current(), (count + 63) / 64, 1, 1, 0x41u, false);
        };
        const auto metadata_words = static_cast<uint32_t>(metadata_size / 4);
        fill_metadata(metadata_words);
        Require(name, "metadata before descriptor",
                !texture_cache.IsMeta(dcc_address),
                "the native fill guessed an unbound metadata allocation's type");

        RenderColorInfo color{};
        RenderExecutorTestAccess::ResolveRenderColorTarget(
            executor, scheduler.Current(), color, 0);
        RenderDepthInfo no_depth{};
        const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
            executor, scheduler.Current(), &color, 1, no_depth);
        if (fill_case.reuse_unorm) {
          Require(name, "FLOAT clear reuses UNORM image",
                  color.image_id == preexisting_id &&
                      texture_cache.GetImage(preexisting_id).backing.format ==
                          vk::Format::eR16G16B16A16Unorm,
                  "the aliased clear did not reuse its existing UNORM allocation");
        }
        Require(name, "color clear metadata discovery",
                color.image_id &&
                    color.desc.info.metadata.kind ==
                        (fill_case.cmask ? ImageMetadataKind::Cmask : ImageMetadataKind::Dcc) &&
                    color.desc.info.metadata.range.address == dcc_address &&
                    color.desc.info.metadata.range.size == metadata_size &&
                    rendering.num_color_attachments == 1 &&
                    !texture_cache.IsMeta(dcc_address),
                "color metadata did not retain its type and full native allocation");
        const auto read_texel = [&] {
          return ReadCachedTexel(name, context, color.image_id, {}, {probe_width, 1, 1}, selected_layer);
        };
        Require(name, "GPU color clear value",
                read_texel() == expected &&
                    ReadCachedTexel(name, context, color.image_id,
                                    {static_cast<int32_t>(512 - probe_width), 255, 0},
                                    {probe_width, 1, 1}, selected_layer) == expected,
                "native metadata did not materialize the expected color");
        if (fill_case.reuse_mips) {
          const auto &image = texture_cache.GetImage(color.image_id);
          Require(name, "single-mip clear reuses mip chain",
                  color.image_id == preexisting_id && image.info.resources.levels == 2 &&
                      color.desc.info.resources.levels == 1 &&
                      color.desc.view_info.base_level == 0 && color.desc.view_info.level_count == 1,
                  "the single-mip target did not retain its cached image and view scope");
          Require(name, "single-mip clear preserves higher mip",
                  ReadCachedTexel(name, context, color.image_id, {255, 127, 0},
                                  {1, 1, 1}, 0, 1) ==
                      std::vector<u32>{0x00003c00u, 0x00003c00u},
                  "clearing mip zero changed its higher mip");
        }
        const auto check_expanded_metadata = [&] {
          context.GetBufferCache().ReadMemory(dcc_address, metadata_size, false);
          std::vector<uint8_t> bytes(metadata_size);
          Require(name, "expanded metadata readback",
                  LibKernel::Memory::TryReadBacking(dcc_address, bytes.data(), bytes.size()),
                  "native color metadata could not be read back");
          const auto selected = bytes.begin() + (fill_case.cmask ? 0x1000 : 0);
          Require(name, "selected metadata slice expansion",
                  std::all_of(bytes.begin(), selected, [](uint8_t v) { return v == 0; }) &&
                      std::all_of(selected, bytes.end(), [](uint8_t v) { return v == 0xff; }),
                  "materialization retained a clear key or changed an unselected slice");
        };
        check_expanded_metadata();

        const auto bind = [&] {
          RenderExecutorTestAccess::ResetBindings(executor);
          color.image_id = texture_cache.FindImage(color.desc);
          (void)texture_cache.FindRenderTarget(color.image_id, color.desc);
        };
        const auto paint = [&] {
          vk::ClearValue clear{};
          clear.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
          TextureCacheTestAccess::ClearImage(
              texture_cache, scheduler.Current(), color.image_id,
              {vk::ImageAspectFlagBits::eColor, 0, 1, selected_layer, 1}, clear);
        };
        std::vector<u32> painted{0x00003c00u, 0x3c003c00u};
        if (fill_case.reuse_unorm) {
          painted = {0x0000ffffu, 0xffffffffu};
        } else if (fill_case.cmask) {
          painted = {0xffff00ffu};
        } else if (bytes_per_pixel <= 4) {
          const uint32_t one = type == Prospero::ChannelType::kFloat ? 0x3c00u : 0xffffu;
          painted = {bytes_per_pixel == 2 ? one | (one << 16u) : one};
        }
        const auto retains_painted_texel = [&] {
          const auto texels = ReadCachedTexel(name, context, color.image_id, {},
                                              {512, 256, 1}, selected_layer);
          for (size_t i = 0; i < texels.size(); i += painted.size()) {
            if (std::equal(painted.begin(), painted.end(), texels.begin() + i)) {
              return true;
            }
          }
          return false;
        };
        paint();
        bind();
        Require(name, "unchanged metadata preserves rendering",
                read_texel() == painted,
                "rebinding unchanged color metadata reapplied an old clear");
        if (fill_case.cmask) {
          WriteMetadata(context, dcc_address, metadata_size, 0x40404040u);
          bind();
          Require(name, "nonzero CMASK is not a DCC clear", read_texel() == painted,
                  "CMASK bytes were decoded as a DCC fixed-color key");
        }
        WriteMetadata(context, dcc_address, metadata_size, fill_case.fill);
        bind();
        Require(name, "CPU metadata re-clear", read_texel() == expected,
                "CPU metadata writes did not clear an already drawn target");
        paint();
        fill_metadata(metadata_words - 1);
        bind();
        Require(name, "partial same-value metadata fill",
                retains_painted_texel(),
                "a partial same-value write reused stale clear bytes after rendering");
        fill_metadata(metadata_words, true);
        bind();
        Require(name, "same-value raw native re-clear",
                read_texel() == expected,
                "a repeated metadata fill did not clear an already drawn target");
        paint();
        WriteMetadata(context, dcc_address, metadata_size, UINT32_MAX);
        fill_metadata(metadata_words - 1);
        bind();
        Require(name, "partial native metadata fill",
                retains_painted_texel(),
                "a nonuniform metadata allocation cleared the whole target");
        fill_metadata(metadata_words);
        bind();
        Require(name, "complete fill after partial fill",
                read_texel() == expected,
                "completing the metadata overwrite did not restore its clear");

        // Consuming a clear key updates metadata backing without invalidating
        // a pooled image whose first texel precedes the metadata range.
        constexpr uint32_t alias_value = 0x13579bdfu;
        constexpr uint32_t expanded_words = 0x1000 / sizeof(uint32_t);
        const auto selected_metadata = dcc_address + metadata_size - 0x1000;
        ImageDesc metadata_alias{};
        metadata_alias.info.data = {dcc_address - 0x1000, 0x4000};
        metadata_alias.info.pixel_format = vk::Format::eR32Uint;
        metadata_alias.info.guest_format = Prospero::BufferFormat::k32UInt;
        metadata_alias.info.extent = {4096, 1, 1};
        metadata_alias.info.pitch = 4096;
        metadata_alias.info.bytes_per_block = sizeof(uint32_t);
        metadata_alias.info.UpdateSize();
        metadata_alias.view_info.format = vk::Format::eR32Uint;
        const auto metadata_alias_id = texture_cache.FindImage(metadata_alias);
        vk::ClearValue alias_paint{};
        alias_paint.color.uint32[0] = alias_value;
        TextureCacheTestAccess::ClearImage(
            texture_cache, scheduler.Current(), metadata_alias_id,
            {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, alias_paint);
        paint();
        fill_metadata(metadata_words);
        bind();
        Require(name, "pooled metadata clear target", read_texel() == expected,
                "the overlapping native image prevented the metadata clear");

        auto metadata_source = resources.GetBufferCache().ObtainBuffer(
            selected_metadata, 0x1000, false);
        Require(name, "expanded metadata buffer source", metadata_source.first != nullptr,
                "expanded metadata did not produce a GPU-readable buffer");
        auto metadata_readback = CreateHostBuffer(
            name, 0x1000, vk::BufferUsageFlagBits::eTransferDst, {});
        vk::MemoryBarrier metadata_barrier{};
        metadata_barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite;
        metadata_barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead;
        scheduler.Current().Handle().pipelineBarrier(
            vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eTransfer,
            {}, 1, &metadata_barrier, 0, nullptr, 0, nullptr);
        const vk::BufferCopy metadata_copy{metadata_source.second, 0, 0x1000};
        scheduler.Current().Handle().copyBuffer(metadata_source.first->Handle(),
                                                metadata_readback.buffer, 1, &metadata_copy);
        metadata_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
        metadata_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
        scheduler.Current().Handle().pipelineBarrier(
            vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost,
            {}, 1, &metadata_barrier, 0, nullptr, 0, nullptr);
        scheduler.Finish();
        Require(name, "expanded metadata GPU reader",
                ReadBuffer(name, metadata_readback, expanded_words) ==
                    std::vector<u32>(expanded_words, UINT32_MAX),
                "a later GPU buffer read retained the consumed clear key");
        DestroyBuffer(&metadata_readback);
        (void)texture_cache.FindTexture(metadata_alias_id, metadata_alias);
        Require(name, "metadata consumption preserves pooled image",
                ReadCachedTexel(name, context, metadata_alias_id) ==
                    std::vector<u32>{alias_value},
                "internal metadata publication replaced an untouched GPU image texel");

        ImageDesc previous_depth{};
        previous_depth.type = BindingType::DepthTarget;
        previous_depth.info.data = {base + 0x180000, 0x10000};
        previous_depth.info.pixel_format = vk::Format::eD32Sfloat;
        previous_depth.info.guest_format = Prospero::BufferFormat::k32Float;
        previous_depth.info.type = Prospero::ImageType::kColor2D;
        previous_depth.info.extent = {128, 128, 1};
        previous_depth.info.resources = {1, 1};
        previous_depth.info.pitch = 128;
        previous_depth.info.bytes_per_block = 4;
        previous_depth.info.samples = 1;
        previous_depth.info.tile_mode = Prospero::TileMode::kLinear;
        previous_depth.info.UpdateSize();
        previous_depth.info.metadata.kind = ImageMetadataKind::Htile;
        previous_depth.info.metadata.range = {dcc_address, metadata_size};
        previous_depth.info.htile_clear_mask = 0;
        previous_depth.view_info.format = vk::Format::eD32Sfloat;
        previous_depth.view_info.type = vk::ImageViewType::e2D;
        previous_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
        previous_depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
        const auto previous_depth_id = texture_cache.FindImage(previous_depth);
        Require(name, "old HTile interpretation",
                texture_cache.FindDepthTarget(previous_depth_id, previous_depth) != nullptr &&
                    texture_cache.IsMeta(dcc_address),
                "the old depth target did not register its metadata allocation");
        paint();
        bind();
        Require(name, "HTile allocation reused as expanded color metadata",
                !texture_cache.IsMeta(dcc_address) && !texture_cache.ClearMeta(dcc_address) &&
                    read_texel() == painted,
                "color discovery retained the old HTile classification or erased rendering");
        fill_metadata(metadata_words);
        bind();
        Require(name, "native color clear after HTile reuse", read_texel() == expected,
                "the former HTile entry swallowed the native color metadata fill");
        check_expanded_metadata();
        RenderExecutorTestAccess::ResetBindings(executor);
        resources.UnmapMemory(base, allocation_size);
        scheduler.Finish();
      });
      context.ShutdownGpu();
      LibKernel::Memory::InstallGpuResources(nullptr);
    }

    Require(name, "unmap direct backing",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "fixed-clear direct mapping release failed");
    Require(name, "release direct backing",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "fixed-clear direct-memory allocation release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckSampledDccClear() {
    constexpr const char *name = "SampledDccClear";
    constexpr uintptr_t base = 0x0000000205000000ull;
    constexpr uint64_t color_size = 0x3fc0000;
    constexpr uint64_t metadata_size = 0x48000;
    constexpr uint64_t dcc_address = base + color_size;
    constexpr uint64_t allocation_size = 0x4020000;
    constexpr uint64_t alignment = 0x10000;
    EnsureRuntimeContext();
    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, alignment, 0, &direct_offset) == 0,
            "captured scene allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset, alignment) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "captured scene mapping failed");
    for (const auto initial_size : {metadata_size - 0x1000, metadata_size}) {
      std::memset(mapped, 0x5a, color_size);
      std::memset(reinterpret_cast<void *>(dcc_address), 0xff, metadata_size);
      RenderContext context(m_runtime_context);
      context.InitializeGpu(nullptr);
      LibKernel::Memory::InstallGpuResources(&context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      scheduler.Begin(registers, user_config, shaders);
      auto &resources = context;
      auto &cache = resources.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      resources.MapMemory(base, allocation_size);
      WriteMetadata(context, dcc_address, initial_size, 0x40404040u);
      Require(name, "deferred metadata fill",
              !cache.IsMeta(dcc_address),
              "unknown metadata was consumed before descriptor discovery");

      // PPSA19244's first sampled scene descriptor, with relocated allocations.
      ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
          0xc4700000u, 0x021bc3bfu, 0x91b00facu, 0, 0x00700000u,
          0x006b0000u | (static_cast<uint32_t>(dcc_address >> 8u) << 24u),
          static_cast<uint32_t>(dcc_address >> 16u)}};
      ShaderRecompiler::IR::DescriptorValue value{};
      value.dword_count = 8;
      std::copy_n(descriptor.fields, 8, value.dwords.begin());
      ShaderRecompiler::IR::ImageResource resource{};
      resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
      resource.numeric_class = Prospero::TextureNumericClass::Float;
      resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
      resource.read = true;
      auto binding = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
      Require(name, "sampled metadata discovery",
              binding.desc.info.extent == vk::Extent3D{3840, 2160, 1} &&
                  binding.desc.info.data.size == color_size &&
                  binding.desc.info.metadata.kind == ImageMetadataKind::Dcc &&
                  binding.desc.info.metadata.range.address == dcc_address &&
                  binding.desc.info.metadata.range.size == metadata_size,
              "captured sampled descriptor lost its DCC allocation");
      Require(name, "sampled view", cache.FindTexture(binding.image_id, binding.desc) != nullptr,
              "captured scene did not produce a sampled view");
      if (initial_size != metadata_size) {
        Require(name, "partial metadata coverage",
                ReadCachedTexel(name, context, binding.image_id) ==
                    std::vector<u32>{0x5a5a5a5au, 0x5a5a5a5au},
                "an incomplete metadata fill cleared the whole scene");
        WriteMetadata(context, dcc_address, metadata_size, 0x40404040u);
        binding.image_id = cache.FindImage(binding.desc);
        (void)cache.FindTexture(binding.image_id, binding.desc);
      }
      Require(name, "opaque black before first sample",
              ReadCachedTexel(name, context, binding.image_id, {3839, 2159, 0}) ==
                  std::vector<u32>{0, 0x3c000000u},
              "sampling read stale color bytes instead of the DCC opaque black clear");
      context.GetBufferCache().ReadMemory(dcc_address, metadata_size);
      std::vector<uint8_t> expanded_metadata(metadata_size);
      Require(name, "sampled clear expands native metadata",
              LibKernel::Memory::TryReadBacking(dcc_address, expanded_metadata.data(),
                                               expanded_metadata.size()) &&
                  std::all_of(expanded_metadata.begin(), expanded_metadata.end(),
                              [](uint8_t byte) { return byte == 0xff; }),
              "the sampled clear retained native clear keys after materialization");
      WriteMetadata(context, dcc_address, metadata_size, 0x80808080u);
      binding.image_id = cache.FindImage(binding.desc);
      (void)cache.FindTexture(binding.image_id, binding.desc);
      Require(name, "later sampled clear",
              ReadCachedTexel(name, context, binding.image_id) ==
                  std::vector<u32>{0x3c003c00u, 0x00003c00u},
              "an existing sampled image retained the previous DCC clear");
      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
      context.ShutdownGpu();
      LibKernel::Memory::InstallGpuResources(nullptr);
    }
    {
      // PPSA01593 samples a DCC R32 color surface through a comparison sampler.
      constexpr uint64_t comparison_dcc = base + 0x10000;
      std::memset(mapped, 0, 0x10000);
      std::memset(reinterpret_cast<void *>(comparison_dcc), 0xff, 0x1000);
      RenderContext context(m_runtime_context);
      context.InitializeGpu(nullptr);
      LibKernel::Memory::InstallGpuResources(&context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      scheduler.Begin(registers, user_config, shaders);
      context.MapMemory(base, allocation_size);
      auto &cache = context.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      const ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
          0xc1600000u, 0x000fc00fu, 0x91b00204u, 0, 0x00700000u,
          0x002b0000u | (static_cast<uint32_t>(comparison_dcc >> 8u) << 24u),
          static_cast<uint32_t>(comparison_dcc >> 16u)}};
      ShaderRecompiler::IR::DescriptorValue value{};
      value.dword_count = 8;
      std::copy_n(descriptor.fields, 8, value.dwords.begin());
      ShaderRecompiler::IR::ImageResource resource{};
      resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
      resource.numeric_class = Prospero::TextureNumericClass::Float;
      resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
      resource.written = true;
      const auto color = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
      (void)cache.FindTexture(color.image_id, color.desc);
      vk::ClearValue paint{};
      paint.color.float32 = std::array{0.375f, 0.f, 0.f, 0.f};
      TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), color.image_id,
          {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, paint);
      resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
      resource.written = false;
      resource.read = true;
      resource.depth_compare = true;
      auto comparison = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
      (void)cache.FindTexture(comparison.image_id, comparison.desc);
      const auto &depth = cache.GetImage(comparison.image_id);
      Require(name, "expanded DCC comparison alias",
              comparison.image_id != color.image_id &&
                  depth.info.pixel_format == vk::Format::eD32Sfloat &&
                  depth.info.guest_format == Prospero::BufferFormat::k32Float &&
                  depth.info.tile_mode == Prospero::TileMode::kRenderTarget &&
                  depth.info.extent == vk::Extent3D{64, 64, 1} &&
                  depth.info.metadata.kind == ImageMetadataKind::Dcc &&
                  depth.info.metadata.range == GuestRange{comparison_dcc, 0x1000} &&
                  ReadCachedTexel(name, context, comparison.image_id) ==
                      std::vector<u32>{std::bit_cast<u32>(0.375f)},
              "comparison promotion lost native color data or guest DCC layout");
      for (const u32 code : {0x40404040u, 0x80808080u}) {
        WriteMetadata(context, comparison_dcc, 0x1000, code);
        comparison = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
        (void)cache.FindTexture(comparison.image_id, comparison.desc);
        Require(name, "DCC clear on a comparison backing",
                ReadCachedTexel(name, context, comparison.image_id, {63, 63, 0}) ==
                    std::vector<u32>{code == 0x40404040u ? 0u : 0x3f800000u},
                "DCC clear used the native depth format or color aspect");
        std::array<uint8_t, 0x1000> metadata{};
        Require(name, "comparison clear expands metadata",
                LibKernel::Memory::TryReadBacking(comparison_dcc, metadata.data(),
                                                 metadata.size()) &&
                    std::ranges::all_of(metadata, [](uint8_t byte) { return byte == 0xff; }),
                "comparison clear retained the native DCC clear key");
      }
      context.UnmapMemory(base, allocation_size);
      scheduler.Finish();
      context.ShutdownGpu();
      LibKernel::Memory::InstallGpuResources(nullptr);
    }
    Require(name, "unmap",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "captured scene mapping release failed");
    Require(name, "release",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "captured scene allocation release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckPackedTextureComponents() {
    constexpr const char *name = "PackedTextureComponents";
    constexpr uintptr_t base = 0x0000000204200000ull;
    constexpr uint64_t allocation_size = 0x10000;
    struct FormatCase {
      Prospero::BufferFormat format;
      std::array<uint16_t, 4> words;
      std::array<std::array<uint32_t, 4>, 4> channels;
      std::array<uint32_t, 4> maxima;
    };
    // Packed format channels start at the least significant bit.
    // PPSA24515's blue, white, and masked black exposed the 5551 reversal.
    constexpr std::array cases{
        FormatCase{Prospero::BufferFormat::k5_5_5_1UNorm,
                   {0x7c00, 0x7fff, 0x8000, 0x8c41},
                   {{{0, 0, 31, 0}, {31, 31, 31, 0}, {0, 0, 0, 1}, {1, 2, 3, 1}}},
                   {31, 31, 31, 1}},
        FormatCase{Prospero::BufferFormat::k4_4_4_4UNorm,
                   {0x000f, 0x00f0, 0x0f00, 0x4321},
                   {{{15, 0, 0, 0}, {0, 15, 0, 0}, {0, 0, 15, 0}, {1, 2, 3, 4}}},
                   {15, 15, 15, 15}},
    };
    const auto argb = TextureGetComponentMapping(DstSel(5, 1, 7, 0),
                                                 Prospero::ColorMappingArgb);
    Require(name, "inverse mapping",
            argb == vk::ComponentMapping{vk::ComponentSwizzle::eB,
                vk::ComponentSwizzle::eOne, vk::ComponentSwizzle::eR,
                vk::ComponentSwizzle::eZero},
            "cyclic host mapping reordered output slots or descriptor constants");
    const auto target = TextureGetRenderTargetFormat(
        Prospero::ChannelLayout::k1_5_5_5, Prospero::ChannelType::kUNorm,
        Prospero::ChannelOrder::kStandard);
    Require(name, "1555 render target",
            target.format == vk::Format::eR5G5B5A1UnormPack16 &&
                target.export_mapping == Prospero::ColorMappingAbgr,
            "shared format resolution changed the existing 1555 render target");
    for (const auto native : {target, TextureGetRenderTargetFormat(
                                          Prospero::ChannelLayout::k8,
                                          Prospero::ChannelType::kSNorm,
                                          Prospero::ChannelOrder::kStandard)}) {
      const auto upload =
          MakeTilingImage(native.guest_format, 4, 1, 1, 1,
                          Prospero::TileMode::kLinear, 256, false);
      Require(
          name, "render-target guest upload layout",
          upload.pitch * native.bytes_per_element == 256 &&
              upload.mip_layout[0].linear_size == 256,
          "retaining the render-target format lost its physical byte width");
    }
    EnsureRuntimeContext();
    int64_t direct_offset = -1;
    Require(name, "allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_size, 0, &direct_offset) == 0,
            "packed texture allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
            "packed texture mapping failed");
    for (const auto &format : cases) {
      std::memset(mapped, 0, allocation_size);
      std::memcpy(mapped, format.words.data(), sizeof(format.words));
      RenderContext context(m_runtime_context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      scheduler.Begin(registers, user_config, shaders);
      auto &resources = context;
      auto &cache = resources.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      resources.MapMemory(base, allocation_size);
      for (const auto swizzle : {DstSel(4, 5, 6, 7), DstSel(7, 1, 4, 0)}) {
        ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
            (static_cast<uint32_t>(format.format) << 20u) | (3u << 30u), 0,
            swizzle | (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u),
            0, 0x00700000u, 0, 0}};
        TestCase test;
        test.name = name;
        test.has_user_data = true;
        test.image_descriptor_swizzle = swizzle;
        std::copy_n(descriptor.fields, 8, test.user_data.begin());
        test.user_data[50] = 16 * sizeof(uint32_t);
        test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_LOAD,
                        ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
        for (uint32_t pixel = 0; pixel < format.words.size(); ++pixel) {
          AppendVMovU32(&test.code, 20, pixel);
          AppendVMovU32(&test.code, 21, 0);
          test.code.push_back(EncodeMimg0(0x00, 0xf));
          test.code.push_back(EncodeMimg1(0, 20));
          for (uint32_t channel = 0; channel < 4; ++channel) {
            AppendStoreVgpr(&test.code, channel, pixel * 4 + channel);
          }
        }
        AppendEnd(&test.code);
        const auto compiled = CompileCase(test, SubgroupSize());
        ShaderRecompiler::IR::DescriptorValue value{};
        value.dword_count = 8;
        std::copy_n(descriptor.fields, 8, value.dwords.begin());
        const auto binding = RenderExecutorTestAccess::ResolveTexture(
            executor, compiled.program.info.images.at(0), value);
        Image sampled;
        sampled.view = cache.FindTexture(binding.image_id, binding.desc);
        sampled.layout = cache.GetImage(binding.image_id).backing.state.layout;
        scheduler.Finish();
        auto output = CreateStorageBuffer(name, {}, 16);
        Dispatch(test, compiled, output, nullptr, &sampled);
        const auto actual = ReadBuffer(name, output, 16);
        for (uint32_t pixel = 0; pixel < format.words.size(); ++pixel) {
          for (uint32_t channel = 0; channel < 4; ++channel) {
            const auto selector = GetDstSel(swizzle, channel);
            const auto maximum = selector < 4 ? 1u : format.maxima[selector - 4];
            const auto expected = selector < 4 ? selector : format.channels[pixel][selector - 4];
            const float observed = std::bit_cast<float>(actual[pixel * 4 + channel]);
            // Recover the source bits without testing the host's UNORM rounding precision.
            const bool matches = maximum == 1 ? observed == expected
                : observed >= 0.0f && observed <= 1.0f &&
                      std::round(observed * maximum) == expected;
            if (!matches) {
              std::ostringstream message;
              message << "format=" << static_cast<uint32_t>(format.format)
                      << " swizzle=" << swizzle << " word=" << format.words[pixel]
                      << " channel=" << channel << " expected=" << expected
                      << " actual=" << observed;
              Fail(name, "GPU components", message.str());
            }
          }
        }
        DestroyBuffer(&output);
      }
      RenderExecutorTestAccess::ResetBindings(executor);
      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
    }
    for (const auto tile : {Prospero::TileMode::kLinear, Prospero::TileMode::kRenderTarget}) {
      constexpr std::array<u32, 4> words{0x883005e1u, 0x7c0e81c0u,
                                         0x800f0000u, 0x98120280u};
      constexpr std::array<std::array<float, 4>, 4> channels{{
          {1.03125f, 2.03125f, 4.0625f, 1.f}, {0.5f, 0.75f, 1.5f, 1.f},
          {0.f, 1.f, 2.f, 1.f}, {32.f, 8.f, 16.f, 1.f}}};
      const auto target = TextureGetRenderTargetFormat(
          Prospero::ChannelLayout::k10_11_11, Prospero::ChannelType::kFloat,
          Prospero::ChannelOrder::kReversed);
      Require(name, "10_11_11 reversed render target",
              target.format == vk::Format::eB10G11R11UfloatPack32 &&
                  target.guest_format == Prospero::BufferFormat::k10_11_11Float &&
                  target.export_mapping == Prospero::ColorMappingRgba,
              "PPSA24156 reversed packed-float target lost its physical encoding");
      std::memset(mapped, 0, allocation_size);
      std::array<u32, 4> offsets{0, 4, 8, 12};
      if (tile == Prospero::TileMode::kRenderTarget) {
        TileBlockLayout block{};
        u32 block_xor = 0;
        Require(name, "10_11_11 tile layout",
                TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, 4, block) &&
                    TileGetBlockXor(block, 0, 0, 0, block_xor),
                "packed-float render-target layout is unavailable");
        for (u32 pixel = 0; pixel < words.size(); ++pixel) {
          Require(name, "10_11_11 tile offset",
                  TileGetBlockOffset(block, pixel, 0, 0, offsets[pixel]),
                  "packed-float texel offset is unavailable");
          offsets[pixel] ^= block_xor;
        }
      }
      for (u32 pixel = 0; pixel < words.size(); ++pixel) {
        std::memcpy(static_cast<uint8_t *>(mapped) + offsets[pixel], &words[pixel], sizeof(u32));
      }
      RenderContext context(m_runtime_context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      scheduler.Begin(registers, user_config, shaders);
      context.MapMemory(base, allocation_size);
      auto &cache = context.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      constexpr u32 swizzle = DstSel(4, 5, 6, 1);
      ShaderTextureResource descriptor{{static_cast<u32>(base >> 8u),
          (static_cast<u32>(Prospero::BufferFormat::k10_11_11Float) << 20u) |
              (3u << 30u), 0,
          swizzle | (static_cast<u32>(tile) << 20u) |
              (static_cast<u32>(Prospero::ImageType::kColor2D) << 28u),
          0, 0x00700000u, 0, 0}};
      TestCase test;
      test.name = name;
      test.has_user_data = true;
      test.image_descriptor_swizzle = swizzle;
      std::copy_n(descriptor.fields, 8, test.user_data.begin());
      test.user_data[50] = 16 * sizeof(u32);
      for (u32 pixel = 0; pixel < words.size(); ++pixel) {
        AppendVMovU32(&test.code, 20, pixel);
        AppendVMovU32(&test.code, 21, 0);
        test.code.push_back(EncodeMimg0(0x00, 0xf));
        test.code.push_back(EncodeMimg1(0, 20));
        for (u32 channel = 0; channel < 4; ++channel) {
          AppendStoreVgpr(&test.code, channel, pixel * 4 + channel);
          test.expected.push_back(std::bit_cast<u32>(channels[pixel][channel]));
        }
      }
      AppendEnd(&test.code);
      const auto compiled = CompileCase(test, SubgroupSize());
      ShaderRecompiler::IR::DescriptorValue value{};
      value.dword_count = 8;
      std::copy_n(descriptor.fields, 8, value.dwords.begin());
      const auto binding = RenderExecutorTestAccess::ResolveTexture(
          executor, compiled.program.info.images.at(0), value);
      Image sampled;
      sampled.view = cache.FindTexture(binding.image_id, binding.desc);
      sampled.layout = cache.GetImage(binding.image_id).backing.state.layout;
      scheduler.Finish();
      auto output = CreateStorageBuffer(name, {}, 16);
      Dispatch(test, compiled, output, nullptr, &sampled);
      Require(name, "10_11_11 uploaded components",
              ReadBuffer(name, output, 16) == test.expected,
              "packed-float upload or sample changed a component's width or order");
      DestroyBuffer(&output);

      const auto clear = [&](std::array<float, 4> channels) {
        auto &native = cache.GetImage(binding.image_id);
        native.Transit(vk::ImageLayout::eTransferDstOptimal,
                       vk::AccessFlagBits2::eTransferWrite, {},
                       scheduler.Current().Handle());
        const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
        scheduler.Current().Handle().clearColorImage(native.backing.image,
            vk::ImageLayout::eTransferDstOptimal, vk::ClearColorValue{channels}, range);
        cache.MarkGpuWritten(binding.image_id);
      };
      clear({8.f, 4.f, 2.f, 1.f});
      Require(name, "10_11_11 guest download",
              TextureCacheTestAccess::TryDownload(cache, binding.image_id),
              "packed-float native contents could not be downloaded");
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      std::array<u32, 4> guest{};
      for (u32 pixel = 0; pixel < words.size(); ++pixel) {
        Require(name, "10_11_11 guest texel",
                Libs::LibKernel::Memory::TryReadBacking(base + offsets[pixel],
                                                        &guest[pixel], sizeof(u32)),
                "packed-float guest texel is inaccessible");
      }
      Require(name, "10_11_11 exact guest words",
              guest == std::array<u32, 4>{0x90110200u, 0x90110200u, 0x90110200u, 0x90110200u},
              "native packed-float write lost the guest 10/11/11 bit layout");

      clear({16.f, 8.f, 4.f, 1.f});
      auto alias_desc = binding.desc;
      alias_desc.info.guest_format = Prospero::BufferFormat::k11_11_10Float;
      alias_desc.view_info.mapping = {};
      Require(name, "same-format transform compatibility",
              !alias_desc.info.IsCompatible(binding.desc.info),
              "equal Vulkan formats hid incompatible guest color transforms");
      const auto alias = cache.FindImage(alias_desc);
      (void)cache.FindTexture(alias, alias_desc);
      Require(name, "same-format transformed alias",
              alias != binding.image_id &&
                  ReadCachedTexel(name, context, alias, {}, {4, 1, 1}) ==
                      std::vector<u32>(4, 0x98120220u),
              "packed-float alias reused a transformed image or bypassed guest bytes");
      RenderExecutorTestAccess::ResetBindings(executor);
      context.UnmapMemory(base, allocation_size);
      scheduler.Finish();
    }
    Require(name, "release",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0 &&
                Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                    direct_offset, allocation_size) == 0,
            "packed texture backing release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckSamplerBorderColors() {
    constexpr const char *name = "SamplerBorderColors";
    auto &context = RuntimeRenderer();
    constexpr std::array<std::array<float, 2>, 5> coordinates{{
        {{0.5f, -1.f}}, {{-1.f, 0.5f}}, {{2.f, 0.5f}},
        {{0.5f, 2.f}}, {{0.5f, 0.5f}}}};
    std::array<vk::Sampler, 3> float_samplers{};
    for (const bool integer : {false, true}) {
      auto image = CreateImage2D(name, 2, 2,
          integer ? vk::Format::eR8G8B8A8Uint : vk::Format::eR8G8B8A8Unorm,
          vk::ImageUsageFlagBits::eSampled, std::vector<u32>(4, 0xffffffffu), 1,
          vk::ImageLayout::eShaderReadOnlyOptimal);
      for (const auto border : {Prospero::SamplerBorderColor::kOpaqueWhite,
                                Prospero::SamplerBorderColor::kOpaqueBlack,
                                Prospero::SamplerBorderColor::kTransBlack}) {
        ShaderSamplerResource sampler_descriptor{};
        const auto clamp = static_cast<u32>(Prospero::SamplerClampMode::kClampBorder);
        sampler_descriptor.fields[0] = clamp | (clamp << 3u);
        sampler_descriptor.fields[2] = 1u << 24u;
        sampler_descriptor.fields[3] = static_cast<u32>(border) << 30u;
        TestCase test;
        test.name = name;
        test.has_user_data = true;
        test.user_data = MakeSampledTextureData(integer ? Prospero::BufferFormat::k8_8_8_8UInt
                                                        : Prospero::BufferFormat::k8_8_8_8UNorm);
        test.user_data[2] = 1u | (1u << 14u);
        std::copy_n(sampler_descriptor.fields, 4, test.user_data.begin() + 8);
        test.user_data[50] = coordinates.size() * 4u * sizeof(u32);
        test.user_data[51] = 3u << 28u;
        test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
                        ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
        for (u32 sample = 0; sample < coordinates.size(); ++sample) {
          AppendVMovLiteral(&test.code, 20, std::bit_cast<u32>(coordinates[sample][0]));
          AppendVMovLiteral(&test.code, 21, std::bit_cast<u32>(coordinates[sample][1]));
          test.code.push_back(EncodeMimg0(0x20, 0xf));
          test.code.push_back(EncodeMimg1(0, 20, 0, 2));
          for (u32 channel = 0; channel < 4; ++channel) {
            AppendStoreVgpr(&test.code, channel, sample * 4u + channel);
            const bool inside = sample == coordinates.size() - 1u;
            const bool one = inside || border == Prospero::SamplerBorderColor::kOpaqueWhite ||
                (border == Prospero::SamplerBorderColor::kOpaqueBlack && channel == 3u);
            test.expected.push_back(one ? (integer ? (inside ? 255u : 1u) : 0x3f800000u) : 0u);
          }
        }
        AppendEnd(&test.code);
        const auto compiled = CompileCase(test, SubgroupSize());
        const auto &sampler_info = compiled.program.info.samplers[0];
        Require(name, "sampler specialization", sampler_info.integer_border == integer,
                "sampled image numeric type did not select its sampler border class");
        auto output = CreateStorageBuffer(name, {}, test.expected.size());
        const auto sampler = context.GetSamplerCache().GetSampler(
            sampler_descriptor, sampler_info.integer_border);
        Require(name, "sampler reuse",
                sampler == context.GetSamplerCache().GetSampler(sampler_descriptor, integer),
                "identical typed sampler descriptors did not reuse the cached sampler");
        if (integer) {
          Require(name, "typed border cache key", sampler != float_samplers[static_cast<u32>(border)],
                  "integer image reused the floating-point border sampler");
        } else {
          float_samplers[static_cast<u32>(border)] = sampler;
        }
        Dispatch(test, compiled, output, nullptr, &image, nullptr, nullptr, sampler);
        const auto actual = ReadBuffer(name, output, test.expected.size());
        for (u32 i = 0; i < actual.size(); ++i) {
          if (actual[i] != test.expected[i]) {
            Fail(name, "native border sample", "integer=" + std::to_string(integer) +
                " border=" + std::to_string(static_cast<u32>(border)) + " word=" + std::to_string(i) +
                " expected=" + std::to_string(test.expected[i]) + " actual=" + std::to_string(actual[i]));
          }
        }
        DestroyBuffer(&output);
      }
      DestroyImage(&image);
    }
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckComparisonDepthTexture() {
    constexpr const char *name = "ComparisonDepthTexture";
    constexpr uintptr_t base = 0x0000000204200000ull;
    constexpr uint64_t allocation_size = 0x20000;
    constexpr uint64_t ordinary_address = base + 0x10000;
    constexpr std::array<uint16_t, 6> depths{0, 13107, 26214, 39321, 52428, 65535};
    constexpr std::array<float, 5> references{0.1f, 0.3f, 0.5f, 0.7f, 0.9f};
    EnsureRuntimeContext();
    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_size, 0, &direct_offset) == 0,
            "comparison texture allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
            "comparison texture mapping failed");
    // Sea of Stars' 1D comparison texture and the existing six-face cube.
    const auto check_layout = [&](bool cube, uint32_t swizzle) {
      const uint32_t layers = cube ? depths.size() : 1;
      const auto depth_at = [&](uint32_t layer, bool reversed) {
        const auto index = cube ? layer : 2u;
        return depths[reversed ? depths.size() - 1u - index : index];
      };
      std::memset(mapped, 0, allocation_size);
      for (uint32_t face = 0; face < layers; face++) {
        const auto depth = depth_at(face, false);
        for (const auto address : {static_cast<uint64_t>(base), ordinary_address}) {
          std::memcpy(reinterpret_cast<void *>(address + face * 0x100),
                      &depth, sizeof(depth));
        }
      }

      RenderContext context(m_runtime_context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      scheduler.Begin(registers, user_config, shaders);
      auto &resources = context;
      auto &cache = resources.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      resources.MapMemory(base, allocation_size);
      ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
          0x00700000u, 0, (cube ? 0xb0100000u : 0x81800000u) | swizzle,
          layers - 1u, 0x00700000u, 0, 0}};
      ShaderSamplerResource sampler_descriptor{{1u << 12u, 0, 1u << 24u, 0}};
      TestCase test;
      test.name = name;
      test.has_user_data = true;
      test.image_descriptor_swizzle = descriptor.DstSelXYZW();
      std::copy_n(descriptor.fields, 8, test.user_data.begin());
      std::copy_n(sampler_descriptor.fields, 4, test.user_data.begin() + 8);
      test.user_data[50] = layers * references.size() * sizeof(uint32_t);
      test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
                      ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
      test.required_spirv = {"OpImageSampleDrefExplicitLod"};
      std::vector<uint32_t> promoted_expected;
      for (uint32_t face = 0; face < layers; face++) {
        for (const auto reference : references) {
          AppendVMovLiteral(&test.code, 20, std::bit_cast<uint32_t>(reference));
          AppendVMovLiteral(&test.code, 21,
                            std::bit_cast<uint32_t>(cube ? 1.5f : 0.5f));
          if (cube) {
            AppendVMovLiteral(&test.code, 22, std::bit_cast<uint32_t>(1.5f));
            AppendVMovLiteral(&test.code, 23,
                              std::bit_cast<uint32_t>(static_cast<float>(face)));
          }
          test.code.push_back(EncodeMimg0(0x2f, 1, 0, false, cube ? 3 : 0));
          test.code.push_back(EncodeMimg1(0, 20, 0, 2));
          AppendStoreVgpr(&test.code, 0, static_cast<uint32_t>(test.expected.size()));
          test.expected.push_back(
              reference < static_cast<float>(depth_at(face, false)) / 65535.f
                  ? 0x3f800000u : 0u);
          promoted_expected.push_back(
              reference < static_cast<float>(depth_at(face, true)) / 65535.f
                  ? 0x3f800000u : 0u);
        }
      }
      AppendEnd(&test.code);
      const auto compiled = CompileCase(test, SubgroupSize());
      Require(name, "comparison resource", compiled.program.info.images.size() == 1 &&
                  compiled.program.info.images[0].depth_compare &&
                  compiled.program.info.images[0].cube == cube &&
                  compiled.program.info.images[0].dimension ==
                      (cube ? ShaderRecompiler::Decoder::ImageDimension::Dim2DArray
                            : ShaderRecompiler::Decoder::ImageDimension::Dim1D),
              "native comparison did not retain its resource semantics");
      ShaderRecompiler::IR::DescriptorValue value{};
      value.dword_count = 8;
      std::copy_n(descriptor.fields, 8, value.dwords.begin());
      auto binding = RenderExecutorTestAccess::ResolveTexture(
          executor, compiled.program.info.images[0], value);
      const auto view = cache.FindTexture(binding.image_id, binding.desc);
      auto &image = cache.GetImage(binding.image_id);
      if (swizzle == DstSel(4, 4, 4, 1)) {
        const vk::ComponentMapping mapping{vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR,
                                            vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eOne};
        Require(name, "RRR1 depth view", std::ranges::any_of(image.views, [&](const auto &cached) {
                  return cached.view == view && cached.info.mapping == mapping;
                }), "comparison depth view lost its replicated depth or constant alpha");
      }
      Require(name, "fresh depth backing", view != nullptr &&
                  image.info.pixel_format == vk::Format::eD16Unorm &&
                  image.info.guest_format == Prospero::BufferFormat::k16UNorm &&
                  image.info.tile_mode == descriptor.TileMode() &&
                  image.info.type == (cube ? Prospero::ImageType::kColor2D
                                          : Prospero::ImageType::kColor1D) &&
                  image.info.data.size == (cube ? 0x600u : 0x10000u) &&
                  image.info.resources.layers == layers,
              "comparison texture lost its guest layout or depth representation");
      auto ordinary_resource = compiled.program.info.images[0];
      ordinary_resource.depth_compare = false;
      value.dwords[0] = static_cast<uint32_t>(ordinary_address >> 8u);
      auto ordinary = RenderExecutorTestAccess::ResolveTexture(executor, ordinary_resource, value);
      (void)cache.FindTexture(ordinary.image_id, ordinary.desc);
      auto &color = cache.GetImage(ordinary.image_id);
      Require(name, "ordinary R16", color.info.pixel_format == vk::Format::eR16Unorm,
              "ordinary R16 sampling unexpectedly acquired a depth format");
      scheduler.Finish();

      auto output = CreateStorageBuffer(name, {}, test.expected.size());
      const auto sampler = context.GetSamplerCache().GetSampler(
          sampler_descriptor, compiled.program.info.samplers[0].integer_border);
      Image sampled;
      sampled.view = view;
      sampled.layout = image.backing.state.layout;
      Dispatch(test, compiled, output, nullptr, &sampled, nullptr, nullptr, sampler);
      Require(name, "native comparisons",
              ReadBuffer(name, output, test.expected.size()) == test.expected,
              "comparison samples lost a depth value or guest layer stride");

      image.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
                    {}, scheduler.Current().Handle());
      color.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
                    {}, scheduler.Current().Handle());
      for (uint32_t face = 0; face < layers; face++) {
        const vk::ImageSubresourceRange depth_range{vk::ImageAspectFlagBits::eDepth, 0, 1, face, 1};
        const vk::ClearDepthStencilValue clear_depth{
            static_cast<float>(depth_at(face, true)) / 65535.f, 0};
        scheduler.Current().Handle().clearDepthStencilImage(image.backing.image,
            vk::ImageLayout::eTransferDstOptimal, &clear_depth, 1, &depth_range);
        const vk::ImageSubresourceRange color_range{vk::ImageAspectFlagBits::eColor, 0, 1, face, 1};
        const vk::ClearColorValue clear_color{std::array<float, 4>{
            static_cast<float>(depth_at(face, true)) / 65535.f, 0.f, 0.f, 0.f}};
        scheduler.Current().Handle().clearColorImage(color.backing.image,
            vk::ImageLayout::eTransferDstOptimal, &clear_color, 1, &color_range);
      }
      cache.MarkGpuWritten(binding.image_id);
      cache.MarkGpuWritten(ordinary.image_id);
      Require(name, "depth readback", TextureCacheTestAccess::TryDownload(cache, binding.image_id),
              "ordinary tiled depth readback did not use the shared transfer path");
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      for (uint32_t face = 0; face < layers; face++) {
        uint16_t actual = 0;
        std::memcpy(&actual, reinterpret_cast<const void *>(base + face * 0x100), sizeof(actual));
        Require(name, "depth guest roundtrip", actual == depth_at(face, true),
                "GPU depth clear was not written to the correct guest face");
      }

      auto promoted = RenderExecutorTestAccess::ResolveTexture(
          executor, compiled.program.info.images[0], value);
      sampled.view = cache.FindTexture(promoted.image_id, promoted.desc);
      auto &promoted_image = cache.GetImage(promoted.image_id);
      Require(name, "GPU color owner promotion", promoted.image_id != ordinary.image_id &&
                  promoted_image.info.pixel_format == vk::Format::eD16Unorm &&
                  promoted_image.IsGpuModified(),
              "comparison binding did not preserve the GPU-current color owner");
      sampled.layout = promoted_image.backing.state.layout;
      scheduler.Finish();
      Dispatch(test, compiled, output, nullptr, &sampled, nullptr, nullptr, sampler);
      Require(name, "promoted comparisons",
              ReadBuffer(name, output, promoted_expected.size()) == promoted_expected,
              "color-to-depth ownership conversion changed comparison results");
      DestroyBuffer(&output);
      RenderExecutorTestAccess::ResetBindings(executor);
      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
    };
    check_layout(false, DstSel(4, 4, 4, 4));
    check_layout(false, DstSel(4, 4, 4, 1));
    check_layout(true, DstSel(4, 0, 0, 1));
    Require(name, "unmap", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "comparison texture unmap failed");
    Require(name, "release", Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "comparison texture release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }


  void CheckCubeFaceStorageExpansion() {
    constexpr const char *name = "CubeFaceStorageExpansion";
    constexpr uintptr_t base = 0x0000000204700000ull;
    constexpr uint64_t allocation_size = 0x10000;
    constexpr std::array<uint32_t, 6> half_red{0x3c00, 0x4000, 0x4200,
                                             0x4400, 0x4500, 0x4600};
    EnsureRuntimeContext();
    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_size, 0, &direct_offset) == 0,
            "cube-face allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
            "cube-face mapping failed");
    std::memset(mapped, 0, allocation_size);
    {
      RenderContext context(m_runtime_context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      scheduler.Begin(registers, user_config, shaders);
      context.MapMemory(base, allocation_size);
      auto &cache = context.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      for (uint32_t face = 0; face < half_red.size(); ++face) {
        // PPSA25380 clears six single-face cube views at one guest address.
        ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
            0xc4700000u, 0x0001c001u, 0xb0000facu,
            face | (face << 16u), 0x00700000u, 0, 0}};
        TestCase test;
        test.name = name;
        test.has_user_data = true;
        std::copy_n(descriptor.fields, 8, test.user_data.begin());
        test.has_compute_info = true;
        test.compute_info.threads_num[0] = test.compute_info.threads_num[1] = 8;
        test.compute_info.threads_num[2] = 1;
        test.compute_info.thread_ids_num = 2;
        test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_STORE,
                        ShaderOpcode::S_ENDPGM};
        test.code = {EncodeVop1(0x01, 20, 256), EncodeVop1(0x01, 21, 257)};
        AppendVMovU32(&test.code, 22, 0);
        AppendVMovLiteral(&test.code, 0,
                          std::bit_cast<uint32_t>(static_cast<float>(face + 1)));
        AppendVMovLiteral(&test.code, 1, std::bit_cast<uint32_t>(0.5f));
        AppendVMovLiteral(&test.code, 2, std::bit_cast<uint32_t>(0.25f));
        AppendVMovLiteral(&test.code, 3, std::bit_cast<uint32_t>(1.0f));
        test.code.push_back(EncodeMimg0(0x08, 0xf, 0, false, 5));
        test.code.push_back(EncodeMimg1(0, 20));
        AppendEnd(&test.code);
        const auto compiled = CompileCase(test, SubgroupSize());
        const auto &resource = compiled.program.info.images.at(0);
        Require(name, "cube storage specialization", resource.cube &&
                    resource.written && resource.dimension ==
                        ShaderRecompiler::Decoder::ImageDimension::Dim2DArray,
                "single-face cube store lost its array coordinates");
        ShaderRecompiler::IR::DescriptorValue value{};
        value.dword_count = 8;
        std::copy_n(descriptor.fields, 8, value.dwords.begin());
        const auto binding = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
        Image storage;
        storage.view = cache.FindTexture(binding.image_id, binding.desc);
        auto &image = cache.GetImage(binding.image_id);
        Require(name, "face view and backing", storage.view != nullptr &&
                    image.info.pixel_format == vk::Format::eR16G16B16A16Sfloat &&
                    image.info.bytes_per_block == 8 &&
                    image.info.resources.layers == face + 1 &&
                    image.backing.layers == face + 1 &&
                    binding.desc.view_info.type == vk::ImageViewType::e2DArray &&
                    binding.desc.view_info.base_layer == face &&
                    binding.desc.view_info.layer_count == 1,
                "cube face did not expand the array or select its own layer");
        image.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits2::eShaderWrite,
                      {}, scheduler.Current().Handle());
        storage.layout = image.backing.state.layout;
        scheduler.Finish();
        Dispatch(test, compiled, {}, nullptr, nullptr, &storage);
        cache.MarkGpuWritten(binding.image_id);
        for (uint32_t prior = 0; prior <= face; ++prior) {
          const auto pixels = ReadCachedTexel(name, context, binding.image_id,
                                             {}, {8, 8, 1}, prior);
          for (uint32_t pixel = 0; pixel < 64; ++pixel) {
            Require(name, "stored and preserved faces",
                    pixels[pixel * 2] == (0x38000000u | half_red[prior]) &&
                        pixels[pixel * 2 + 1] == 0x3c003400u,
                    "face " + std::to_string(prior) + " pixel " +
                        std::to_string(pixel) + " changed after writing face " +
                        std::to_string(face));
          }
        }
      }
      RenderExecutorTestAccess::ResetBindings(executor);
      context.UnmapMemory(base, allocation_size);
      scheduler.Finish();
    }
    Require(name, "release",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0 &&
                Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                    direct_offset, allocation_size) == 0,
            "cube-face backing release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckRenderExecutorColorStandardTileDiscovery() {
    constexpr const char *name = "RenderExecutorColorStandardTile";
    constexpr uintptr_t base = 0x0000000203b00000ull;
    constexpr uint64_t allocation_size = 0x190000;
    constexpr uint64_t allocation_alignment = 0x10000;
    constexpr auto format = Prospero::BufferFormat::k32Float;
    struct TileCase {
      Prospero::TileMode tile;
      uint64_t target_offset;
      uint64_t alignment;
    };
    constexpr std::array cases{
        TileCase{Prospero::TileMode::kStandard4KB, 0x4000, 0x1000},
        TileCase{Prospero::TileMode::kStandard64KB, 0x10000, 0x10000},
    };
    EnsureRuntimeContext();

    const auto odd_pitch =
        TileGetTexturePitch(format, 257, Prospero::TileMode::kStandard4KB);
    TileSizeAlign odd_slice{};
    TileGetTextureSize(format, 257, 131, 1, Prospero::TileMode::kStandard4KB,
                       &odd_slice, nullptr, nullptr);
    Require(name, "tile layout",
            odd_pitch == 288 && odd_slice.size == 0x2d000 &&
                odd_slice.align == 0x1000,
            "Standard4KB pitch, footprint, or block alignment is incorrect");

    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_alignment, 0, &direct_offset) == 0,
            "standard-tiled color allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_alignment) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "standard-tiled color mapping failed");
    std::memset(mapped, 0, allocation_size);

    for (const auto &tile_case : cases) {
      const auto pitch = TileGetTexturePitch(format, 256, tile_case.tile);
      TileSizeAlign slice{};
      TileGetTextureSize(format, 256, 256, 1, tile_case.tile, &slice, nullptr,
                         nullptr);
      Require(
          name, "tile layout",
          pitch == 256 && slice.size == 0x40000 &&
              slice.align == tile_case.alignment,
          "standard tile pitch, footprint, or block alignment is incorrect");

      const auto target_address = base + tile_case.target_offset;
      const auto target_size = slice.size * 6ull;
      RenderContext context(m_runtime_context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      registers.SetColorBase(0, {.addr = target_address});
      registers.SetColorInfo(
          0, {.format = Prospero::ChannelLayout::k32,
              .channel_type = Prospero::ChannelType::kFloat,
              .channel_order = Prospero::ChannelOrder::kStandard});
      registers.SetColorView(
          0, {.base_array_slice_index = 5, .last_array_slice_index = 5});
      registers.SetColorAttrib2(0, {.height = 255, .width = 255});
      registers.SetColorAttrib3(0, {.tile_mode = tile_case.tile,
                                    .dimension = 1,
                                    .metadata_pipe_aligned = true});
      registers.SetRenderTargetMask(0x0f);
      scheduler.Begin(registers, user_config, shaders);

      auto &resources = context;
      auto &texture_cache = resources.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      resources.MapMemory(base, allocation_size);

      RenderColorInfo color{};
      RenderExecutorTestAccess::ResolveRenderColorTarget(
          executor, scheduler.Current(), color, 0);
      const auto attachment =
          texture_cache.FindRenderTarget(color.image_id, color.desc);
      const auto &image = texture_cache.GetImage(color.image_id);
      Require(name, "captured target",
              color.image_id && attachment != nullptr &&
                  color.guest_array_layer == 5 &&
                  color.desc.view_info.base_layer == 5 &&
                  color.desc.info.data.address == target_address &&
                  color.desc.info.data.size == target_size &&
                  color.desc.info.tile_mode == tile_case.tile &&
                  color.desc.info.extent == vk::Extent3D{256, 256, 1} &&
                  color.desc.info.resources == ImageSubresources{1, 6} &&
                  color.desc.info.pitch == pitch &&
                  color.desc.info.mip_layout[0].offset == 0 &&
                  color.desc.info.mip_layout[0].size == target_size &&
                  color.desc.view_info.type == vk::ImageViewType::e2D &&
                  color.desc.view_info.base_layer == 5 &&
                  color.desc.view_info.layer_count == 1 &&
                  (target_address & (tile_case.alignment - 1)) == 0 &&
                  image.usage.render_target && image.IsGpuModified(),
              "slice 5 standard-tiled render target was not resolved");

      RenderExecutorTestAccess::ResetBindings(executor);
      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
    }

    Require(name, "unmap direct backing",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "standard-tiled color mapping release failed");
    Require(name, "release direct backing",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "standard-tiled color allocation release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckRenderExecutorColorDepthTileDiscovery() {
    constexpr const char *name = "RenderExecutorColorDepthTile";
    constexpr uintptr_t base = 0x0000000203d00000ull;
    constexpr uint64_t allocation_size = 0x10000;
    constexpr uint64_t allocation_alignment = 0x10000;
    EnsureRuntimeContext();

    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_alignment, 0, &direct_offset) == 0,
            "depth-tiled color allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_alignment) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "depth-tiled color mapping failed");
    constexpr uint32_t width = 64;
    constexpr uint32_t height = 64;
    TileTextureBlockLayout tile_layout{};
    Require(name, "tile layout",
            TileGetTextureBlockLayout(Prospero::BufferFormat::k32Float,
                                      Prospero::TileMode::kDepth, false,
                                      tile_layout),
            "depth-tiled color block layout is unavailable");
    uint32_t block_xor = 0;
    Require(name, "block xor",
            TileGetBlockXor(tile_layout.block, 0, 0, 0, block_xor),
            "depth-tiled color block XOR is unavailable");
    std::vector<uint32_t> expected(allocation_size / sizeof(uint32_t));
    for (uint32_t y = 0; y < height; ++y) {
      for (uint32_t x = 0; x < width; ++x) {
        uint32_t offset = 0;
        Require(name, "texel offset",
                TileGetBlockOffset(tile_layout.block, x, y, 0, offset),
                "depth-tiled color texel offset is unavailable");
        const auto index = (offset ^ block_xor) / sizeof(uint32_t);
        Require(name, "texel address", index < expected.size(),
                "depth-tiled color texel address exceeds its backing");
        expected[index] = 0x3f000000u + y * width + x;
      }
    }
    std::memcpy(mapped, expected.data(), allocation_size);

    {
      RenderContext context(m_runtime_context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      registers.SetColorBase(0, {.addr = base});
      registers.SetColorInfo(
          0, {.format = Prospero::ChannelLayout::k32,
              .channel_type = Prospero::ChannelType::kFloat,
              .channel_order = Prospero::ChannelOrder::kStandard});
      registers.SetColorAttrib2(0, {.height = 63, .width = 63});
      registers.SetColorAttrib3(0, {.tile_mode = Prospero::TileMode::kDepth,
                                    .dimension = 1,
                                    .metadata_pipe_aligned = true});
      registers.SetRenderTargetMask(0x0f);
      scheduler.Begin(registers, user_config, shaders);

      auto &resources = context;
      auto &texture_cache = resources.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      resources.MapMemory(base, allocation_size);

      RenderColorInfo color{};
      RenderExecutorTestAccess::ResolveRenderColorTarget(
          executor, scheduler.Current(), color, 0);
      const auto attachment =
          texture_cache.FindRenderTarget(color.image_id, color.desc);
      const auto &image = texture_cache.GetImage(color.image_id);
      Require(name, "captured target",
              color.image_id && attachment != nullptr &&
                  color.desc.info.tile_mode == Prospero::TileMode::kDepth &&
                  color.desc.info.pitch == 128 &&
                  color.desc.info.data.size == allocation_size &&
                  color.desc.info.mip_layout[0].size == allocation_size &&
                  image.usage.render_target && image.IsGpuModified(),
              "supported depth-tiled color target was not resolved");
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      std::vector<uint8_t> cleared(allocation_size);
      Require(name, "clear guest backing",
              Libs::LibKernel::Memory::TryWriteBacking(base, cleared.data(),
                                                       allocation_size),
              "depth-tiled color backing could not be cleared");
      Require(
          name, "readback queue",
          TextureCacheTestAccess::TryDownload(texture_cache, color.image_id),
          "depth-tiled color target could not be queued for readback");

      RenderExecutorTestAccess::ResetBindings(executor);
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      std::vector<uint32_t> observed(expected.size());
      Require(name, "readback backing",
              Libs::LibKernel::Memory::TryReadBacking(base, observed.data(),
                                                      allocation_size),
              "depth-tiled color readback is unavailable");
      bool recovered = true;
      for (uint32_t y = 0; y < height; ++y) {
        for (uint32_t x = 0; x < width; ++x) {
          uint32_t offset = 0;
          const auto valid =
              TileGetBlockOffset(tile_layout.block, x, y, 0, offset);
          const auto index = (offset ^ block_xor) / sizeof(uint32_t);
          recovered &= valid && index < observed.size() &&
                       observed[index] == expected[index];
        }
      }
      Require(name, "round trip", recovered,
              "depth-tiled color upload/readback changed active texels");
      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
    }

    Require(name, "unmap direct backing",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "depth-tiled color mapping release failed");
    Require(name, "release direct backing",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "depth-tiled color allocation release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  void CheckRenderExecutorStencilBindingDiscovery() {
    constexpr const char *name = "RenderExecutorStencilBindingDiscovery";
    constexpr uintptr_t base = 0x0000000203600000ull;
    constexpr uint64_t allocation_size = 0x1000000;
    constexpr uint64_t allocation_alignment = 0x10000;
    constexpr uint64_t depth_address = base + 0x40000;
    constexpr uint64_t stencil_address = base + 0x70000;
    constexpr uint64_t mipped_storage_address = base + 0x100000;
    constexpr uint32_t tail_backing_value = 0x2468ace0u;
    constexpr uint32_t tail_gpu_value = 0x13579bdfu;
    EnsureRuntimeContext();

    int64_t direct_offset = -1;
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_alignment, 0, &direct_offset) == 0,
            "descriptor discovery direct-memory allocation failed");
    void *mapped = reinterpret_cast<void *>(base);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_alignment) == 0 &&
                mapped == reinterpret_cast<void *>(base),
            "descriptor discovery fixed mapping failed");
    std::memset(mapped, 0, allocation_size);
    std::fill_n(reinterpret_cast<uint32_t *>(mipped_storage_address),
                0x10000 / 4, tail_backing_value);

    {
      RenderContext context(m_runtime_context);
      auto &scheduler = context.GetCommandScheduler();
      HW::Context registers{};
      HW::UserConfig user_config{};
      HW::Shader shaders{};
      context.InitializeGpu(nullptr);
      scheduler.Begin(registers, user_config, shaders);
      auto &resources = context;
      auto &texture_cache = resources.GetTextureCache();
      auto &executor = context.GetRenderExecutor();
      resources.MapMemory(base, allocation_size);
      std::vector<PipelineCache::Pipeline> descriptor_pipelines;

      // The virtual-texture atlas is written as raw BC3 blocks, then sampled as BC3.
      constexpr uint64_t block_alias_address = base + 0xf0000;
      constexpr std::array<uint32_t, 4> block_alias_data{
          0x01234567u, 0x89abcdefu, 0xfedcba98u, 0x76543210u};
      const auto resolve_block_alias = [&](bool compressed) {
        const uint32_t side = compressed ? 256 : 64;
        const auto format = compressed ? Prospero::BufferFormat::kBc3UNorm
                                       : Prospero::BufferFormat::k32_32_32_32UInt;
        ShaderRecompiler::IR::DescriptorValue value{};
        value.dword_count = 8;
        value.dwords = {static_cast<uint32_t>(block_alias_address >> 8u),
                        (static_cast<uint32_t>(format) << 20u) | (3u << 30u),
                        ((side - 1u) >> 2u) | ((side - 1u) << 14u),
                        0x90900facu, 0, 0x00700000u, 0, 0};
        ShaderRecompiler::IR::ImageResource resource{};
        resource.resource_class = compressed
            ? ShaderRecompiler::IR::ImageResourceClass::Sampled
            : ShaderRecompiler::IR::ImageResourceClass::Storage;
        resource.numeric_class = compressed ? Prospero::TextureNumericClass::Float
                                            : Prospero::TextureNumericClass::Uint;
        resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
        resource.read = compressed;
        resource.written = !compressed;
        return RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
      };
      auto raw_blocks = resolve_block_alias(false);
      (void)texture_cache.FindTexture(raw_blocks.image_id, raw_blocks.desc);
      auto &raw_blocks_native = texture_cache.GetImage(raw_blocks.image_id);
      raw_blocks_native.Transit(vk::ImageLayout::eTransferDstOptimal,
                                vk::AccessFlagBits2::eTransferWrite, {},
                                scheduler.Current().Handle());
      vk::ClearColorValue block_clear{};
      std::copy(block_alias_data.begin(), block_alias_data.end(),
                block_clear.uint32.begin());
      const vk::ImageSubresourceRange block_range{
          vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
      scheduler.Current().Handle().clearColorImage(
          raw_blocks_native.backing.image, vk::ImageLayout::eTransferDstOptimal,
          &block_clear, 1, &block_range);
      texture_cache.MarkGpuWritten(raw_blocks.image_id);
      auto compressed_blocks = resolve_block_alias(true);
      (void)texture_cache.FindTexture(compressed_blocks.image_id,
                                      compressed_blocks.desc);
      auto raw_blocks_again = resolve_block_alias(false);
      Require(name, "compressed atlas storage reuse",
              compressed_blocks.image_id != raw_blocks.image_id &&
                  raw_blocks_again.image_id == compressed_blocks.image_id &&
                  texture_cache.FindTexture(raw_blocks_again.image_id,
                                            raw_blocks_again.desc) != nullptr,
              "BC3 sampling replaced the GPU atlas on its next raw-block write");
      Require(name, "compressed atlas download",
              TextureCacheTestAccess::TryDownload(texture_cache,
                                                  raw_blocks_again.image_id),
              "the retained BC3 atlas could not publish its native contents");
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      const auto *block_words = reinterpret_cast<const uint32_t *>(block_alias_address);
      Require(name, "compressed atlas GPU contents",
              std::equal(block_alias_data.begin(), block_alias_data.end(), block_words) &&
                  std::equal(block_alias_data.begin(), block_alias_data.end(),
                             block_words + 0x10000 / sizeof(uint32_t) - 4),
              "compressed atlas aliases reloaded stale CPU bytes over GPU-written blocks");

      const auto set_image_bindings = [](ShaderRecompiler::IR::Program &program) {
        for (auto &image : program.info.images) {
          const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(image);
          const auto index = static_cast<size_t>(kind);
          image.descriptor_index = program.bindings.descriptor_counts[index];
          program.bindings.descriptor_counts[index] += image.mip_count;
          program.bindings.descriptor_mask |= uint64_t{1} << index;
        }
      };
      const auto make_buffer_program =
          [](ShaderType stage, ShaderRecompiler::IR::BufferResource resource) {
            ShaderRecompiler::IR::CompiledShaderInfo program{};
            program.stage = stage;
            resource.descriptor_index = 0;
            program.info.buffers.push_back(resource);
            program.bindings.descriptor_counts[0] = 1;
            program.bindings.descriptor_mask = 1;
            program.bindings.push_data_start_dword = 0;
            return program;
          };

      {
        const auto buffer_program = make_buffer_program(ShaderType::Compute, {.read = true});

        constexpr uint64_t buffer_address = base + allocation_size - 0x5000;
        ShaderBufferResource buffer_descriptor{};
        buffer_descriptor.UpdateAddress48(buffer_address);
        buffer_descriptor.fields[2] = 0x40000000; // Only oversized descriptors are clamped.
        ShaderRecompiler::IR::ResourceSnapshot buffer_snapshot;
        ShaderStageRuntime buffer_runtime{&buffer_program, &buffer_snapshot};
        auto &value = buffer_snapshot.buffers.emplace_back();
        std::memcpy(value.dwords.data(), buffer_descriptor.fields,
                    sizeof(buffer_descriptor.fields));
        value.dword_count = 4;
        PreparedBindings buffer_bindings;
        executor.PrepareBindings(buffer_runtime, buffer_bindings);
        std::array<PreparedBindings *, 1> buffer_stages{&buffer_bindings};
        executor.FindBuffers(buffer_stages);
        const auto original_id = buffer_bindings.buffer_sources[0].id;

        auto &buffer_cache = resources.GetBufferCache();
        const auto merged_id =
            buffer_cache.FindBuffer(base + allocation_size - 0x10000, 0x10000);
        Require(name, "buffer discovery invalidation", merged_id != original_id,
                "overlapping buffer discovery did not replace its smaller owner");
        executor.RebindBuffers(buffer_bindings);
        const auto &binding = buffer_bindings.buffers[0];
        const auto &owner = buffer_cache.GetBuffer(merged_id);
        Require(name, "clamped buffer rebind",
                binding.buffer == owner.Handle() &&
                    binding.offset == owner.Offset(buffer_address) &&
                    binding.range == 0x5000,
                "descriptor rebind lost its clamped guest range or retained a stale host owner");
      }

      {
        const auto buffer_program = make_buffer_program(ShaderType::Compute, {.read = true, .written = true});
        constexpr uint64_t buffer_address = base + 0xe0002;
        ShaderBufferResource descriptor{};
        descriptor.UpdateAddress48(buffer_address);
        descriptor.fields[1] |= 6u << 16u;
        descriptor.fields[2] = 2;
        descriptor.fields[3] = 0x5204;
        ShaderRecompiler::IR::ResourceSnapshot snapshot;
        auto &value = snapshot.buffers.emplace_back();
        std::copy_n(descriptor.fields, 4, value.dwords.begin());
        value.dword_count = 4;
        ShaderStageRuntime runtime{&buffer_program, &snapshot};
        PreparedBindings bindings;
        executor.PrepareBindings(runtime, bindings);
        std::array<PreparedBindings *, 1> stages{&bindings};
        executor.FindBuffers(stages);
        executor.RebindBuffers(bindings);
        const auto &owner = resources.GetBufferCache().GetBuffer(bindings.buffer_sources[0].id);
        Require(name, "halfword buffer base and exact tail",
                bindings.buffers[0].buffer == owner.Handle() &&
                    bindings.buffers[0].offset + 2 == owner.Offset(buffer_address) &&
                    bindings.buffers[0].range == 14 &&
                    bindings.shader_data[buffer_program.bindings.memory_offset_dword] == 2,
                "native binding lost the byte adjustment or rounded the final halfword range");
      }

      {
        constexpr uint64_t address = base + allocation_size - 0x4000 + 12;
        constexpr uint64_t mapped_tail = base + allocation_size - address;
        auto writer = make_buffer_program(ShaderType::Vertex, {.written = true});
        ShaderRecompiler::IR::CompiledShaderInfo reader{};
        reader.stage = ShaderType::Pixel;
        ShaderRecompiler::IR::ResourceSnapshot writer_snapshot, reader_snapshot;
        ShaderBufferResource descriptor{};
        descriptor.UpdateAddress48(address);
        descriptor.fields[2] = UINT32_MAX;
        auto &value = writer_snapshot.buffers.emplace_back();
        std::copy_n(descriptor.fields, 4, value.dwords.begin());
        value.dword_count = 4;
        ShaderStageRuntime writer_runtime{&writer, &writer_snapshot};
        ShaderStageRuntime reader_runtime{&reader, &reader_snapshot};
        PreparedBindings writer_bindings, reader_bindings;
        const auto check_range = [&](uint64_t expected, const char *description) {
          executor.PrepareBindings(writer_runtime, writer_bindings);
          executor.PrepareBindings(reader_runtime, reader_bindings);
          std::array<PreparedBindings *, 2> stages{&reader_bindings, &writer_bindings};
          executor.FindBuffers(stages);
          executor.RebindBuffers(writer_bindings);
          const auto &source = writer_bindings.buffer_sources[0];
          const auto &binding = writer_bindings.buffers[0];
          const auto &owner = resources.GetBufferCache().GetBuffer(source.id);
          Require(name, description, source.address == address && source.size == expected &&
                      binding.buffer == owner.Handle() && binding.offset <= owner.Offset(address) &&
                      binding.range == expected + owner.Offset(address) - binding.offset,
                  "descriptor discovery or rebind changed the write boundary or base adjustment");
        };
        writer_snapshot.specialization_reads = {{address + 0x500, 4},
            {address - 0x40, 0x40}, {address + 0x80, 0}};
        reader_snapshot.specialization_reads = {{address + 0x300, 4}, {address + 0x100, 4}};
        check_range(0x100, "unbounded write stops before nearest cross-stage read");
        reader_snapshot.specialization_reads.clear();
        check_range(0x500, "unbounded write stops before same-stage read");
        writer_snapshot.specialization_reads = {{address - 0x40, 0x40},
                                               {address + mapped_tail + 0x100, 4}};
        check_range(mapped_tail, "earlier and unmapped reads preserve mapped tail");
        writer_snapshot.specialization_reads.clear();
        check_range(mapped_tail, "unbounded write without reads preserves mapped tail");
        reader_snapshot.specialization_reads = {{address + 0x100, 4}};
        value.dwords[2] = 0x300;
        check_range(0x300, "finite write preserves its declared range");
        value.dwords[2] = UINT32_MAX;
        writer.info.buffers[0].written = false;
        writer.info.buffers[0].read = true;
        check_range(mapped_tail, "read-only sentinel preserves mapped tail");
        auto &graphics = context.GetGraphics();
        auto &storage_limit = graphics.physical_device_properties.limits.maxStorageBufferRange;
        const auto original_limit = storage_limit;
        storage_limit = 0x2000;
        value.dwords[1] |= 16u << 16u;
        check_range(storage_limit - (graphics.StorageMinAlignment() - 1),
                    "read-only structured sentinel reserves native alignment prefix");
        Require(name, "read-only native range fits device", writer_bindings.buffers[0].range <= storage_limit,
                "read-only cap omitted the native descriptor alignment prefix");
        value.dwords[2] = 0x300; // A finite 12 KiB descriptor remains finite despite the mock limit.
        std::array<PreparedBindings *, 1> stages{&writer_bindings};
        executor.FindBuffers(stages);
        Require(name, "finite read exceeds mock cap unchanged", writer_bindings.buffer_sources[0].size == 0x3000,
                "read-only window changed a finite descriptor's declared range");
        storage_limit = original_limit;
        check_range(0x3000, "finite structured read preserves native range");
        value.dwords[2] = UINT32_MAX;
        check_range(mapped_tail, "read-only structured sentinel preserves smaller mapping");
      }

      for (const auto format : {Prospero::BufferFormat::k8UNorm,
                                 Prospero::BufferFormat::k16UNorm}) {
        const uint64_t alias_address = base + 0xe00000 +
            Prospero::NumBytesPerElement(format) * 0x10000;
        constexpr uint8_t texel = 0x7f;
        Libs::LibKernel::Memory::WriteBacking(alias_address, &texel, sizeof(texel));
        u32 expected = texel;
        if (format == Prospero::BufferFormat::k16UNorm) {
          auto depth_desc = MakeLinearDesc(alias_address, 256, vk::Format::eD16Unorm,
              format, Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 2, 1);
          depth_desc.info.pitch = 128;
          depth_desc.info.UpdateSize();
          depth_desc.type = BindingType::DepthTarget;
          depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
          depth_desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
          const auto depth_id = texture_cache.FindImage(depth_desc);
          auto &depth = texture_cache.GetImage(depth_id);
          depth.Transit(vk::ImageLayout::eTransferDstOptimal,
                        vk::AccessFlagBits2::eTransferWrite, {}, scheduler.Current().Handle());
          const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eDepth, 0, 1, 0, 1};
          scheduler.Current().Handle().clearDepthStencilImage(depth.backing.image,
              vk::ImageLayout::eTransferDstOptimal, vk::ClearDepthStencilValue{1.f, 0}, range);
          texture_cache.MarkGpuWritten(depth_id);
          expected = 0xffff;
        }
        ShaderRecompiler::IR::CompiledShaderInfo program{};
        program.stage = ShaderType::Compute;
        ShaderRecompiler::IR::ResourceSnapshot snapshot;
        // PPSA24156 binds one-texel R8/R16 allocations as 3D, then 2D.
        for (const auto type : {Prospero::ImageType::kColor3D,
                                Prospero::ImageType::kColor2D}) {
          ShaderRecompiler::IR::ImageResource resource{};
          resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
          resource.read = true;
          resource.numeric_class = Prospero::TextureNumericClass::Float;
          resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
          if (type == Prospero::ImageType::kColor3D) {
            resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
          }
          program.info.images.push_back(resource);
          auto &value = snapshot.images.emplace_back();
          value.dword_count = 8;
          value.dwords = {static_cast<u32>(alias_address >> 8u),
              static_cast<u32>(format) << 20u, 0,
              DstSel(4, 4, 4, 1) | (static_cast<u32>(type) << 28u),
              0, 0x00700000u, 0, 0};
        }
        ShaderStageRuntime runtime{&program, &snapshot};
        PreparedBindings bindings;
        executor.PrepareBindings(runtime, bindings);
        Require(name, "overlapping dimension discovery",
                !texture_cache.GetImage(bindings.images[0].image_id).registered,
                "the 2D descriptor did not replace its earlier 3D alias");
        executor.RebindImages(bindings);
        auto output = CreateHostBuffer(name, 8, vk::BufferUsageFlagBits::eTransferDst, {0, 0});
        constexpr std::array image_types{vk::ImageType::e3D, vk::ImageType::e2D};
        for (u32 i = 0; i < bindings.images.size(); ++i) {
          const auto &binding = bindings.images[i];
          auto &image = texture_cache.GetImage(binding.image_id);
          Require(name, "overlapping dimension views", binding.image_view != nullptr &&
                      image.backing.image_type == image_types[i],
                  "rediscovery lost a live view or the descriptor's image dimension");
          image.Transit(vk::ImageLayout::eTransferSrcOptimal,
                        vk::AccessFlagBits2::eTransferRead, {}, scheduler.Current().Handle());
          vk::BufferImageCopy copy{};
          copy.bufferOffset = i * sizeof(u32);
          copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1};
          copy.imageExtent = {1, 1, 1};
          scheduler.Current().Handle().copyImageToBuffer(image.backing.image,
              vk::ImageLayout::eTransferSrcOptimal, output.buffer, 1, &copy);
        }
        vk::MemoryBarrier barrier{};
        barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
        barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
        scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
            vk::PipelineStageFlagBits::eHost, {}, 1, &barrier, 0, nullptr, 0, nullptr);
        RenderExecutorTestAccess::ResetBindings(executor);
        scheduler.Finish();
        Require(name, "overlapping dimension GPU contents",
                ReadBuffer(name, output, 2) == std::vector<u32>{expected, expected},
                "an alias replacement retired image contents before queued GPU reads");
        DestroyBuffer(&output);
      }

      for (const bool volume_first : {false, true}) {
        // PPSA24156 shares a six-face R8 allocation with a smaller 3D placeholder.
        const auto address = base + 0xe30000 + volume_first * 0x10000;
        auto array_desc = MakeLinearDesc(address, 1536, vk::Format::eR8Unorm,
            Prospero::BufferFormat::k8UNorm, Prospero::ImageType::kColor2D,
            {1, 1, 1}, 6, 1, 1);
        array_desc.info.pitch = 256;
        array_desc.info.UpdateSize();
        auto volume_desc = array_desc;
        volume_desc.info.type = Prospero::ImageType::kColor3D;
        volume_desc.info.data.size = 256;
        volume_desc.info.resources.layers = 1;
        volume_desc.info.mip_layout[0].size = 256;
        volume_desc.view_info.type = vk::ImageViewType::e3D;
        volume_desc.view_info.layer_count = 1;
        const auto first_id = texture_cache.FindImage(volume_first ? volume_desc : array_desc);
        const auto second_id = texture_cache.FindImage(volume_first ? array_desc : volume_desc);
        const auto array_id = volume_first ? second_id : first_id;
        const auto volume_id = volume_first ? first_id : second_id;
        Require(name, "smaller volume native image", volume_id != array_id &&
                    texture_cache.GetImage(volume_id).backing.image_type == vk::ImageType::e3D,
                "a smaller 3D descriptor reused the six-layer 2D owner");
        (void)texture_cache.FindTexture(array_id, array_desc);
        (void)texture_cache.FindTexture(volume_id, volume_desc);
        Require(name, "retained dimensional aliases",
                texture_cache.FindImage(array_desc) == array_id &&
                    texture_cache.FindImage(volume_desc) == volume_id &&
                    texture_cache.GetImage(array_id).info.resources.layers == 6,
                "dimensional alias lookup discarded the array owner or lost its faces");
        auto subarray_desc = array_desc;
        subarray_desc.info.data.size = 1280;
        subarray_desc.info.resources.layers = 5;
        subarray_desc.info.mip_layout[0].size = 1280;
        subarray_desc.view_info.layer_count = 5;
        const auto subarray_id = texture_cache.FindImage(subarray_desc);
        Require(name, "preserved dimensional overlap selection", subarray_id == array_id &&
                    texture_cache.GetImage(array_id).info.resources.layers == 6,
                "a smaller volume alias displaced the selected array owner");
        (void)texture_cache.FindTexture(subarray_id, subarray_desc);
      }

      constexpr auto stencil_format = Prospero::BufferFormat::k8UInt;
      constexpr auto linear = Prospero::TileMode::kLinear;
      TileSizeAlign stencil_layout{};
      TileGetTextureTotalSize(stencil_format, 1, 1, 1, 1, linear, false,
                              stencil_layout);
      Require(name, "stencil footprint",
              stencil_layout.size != 0 && stencil_layout.align != 0 &&
                  (stencil_address & (stencil_layout.align - 1u)) == 0,
              "linear stencil descriptor produced an invalid footprint");

      ShaderRecompiler::IR::Program null_program{};
      null_program.stage = ShaderType::Compute;
      null_program.resource_tracking_complete = true;
      ShaderRecompiler::IR::ImageResource null_resource{};
      null_resource.resource_class =
          ShaderRecompiler::IR::ImageResourceClass::Sampled;
      null_resource.numeric_class = Prospero::TextureNumericClass::Uint;
      null_resource.dimension =
          ShaderRecompiler::Decoder::ImageDimension::Dim2D;
      null_resource.read = true;
      null_program.info.images.push_back(null_resource);
      auto null_volume = null_resource;
      null_volume.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
      null_program.info.images.push_back(null_volume);
      auto null_storage = null_resource;
      null_storage.resource_class =
          ShaderRecompiler::IR::ImageResourceClass::Storage;
      null_storage.read = false;
      null_storage.written = true;
      null_program.info.images.push_back(null_storage);
      ShaderRecompiler::IR::ResourceSnapshot null_snapshot{};
      ShaderRecompiler::IR::DescriptorValue null_descriptor{};
      null_descriptor.dword_count = 8;
      null_snapshot.images.assign(3, null_descriptor);
      ShaderRecompiler::IR::ResourceSpecialization null_specialization{};
      null_specialization.images.resize(3);
      for (auto &image : null_specialization.images) {
        image.numeric_class = Prospero::TextureNumericClass::Float;
        image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
      }
      ShaderRecompiler::IR::ApplyResourceSpecialization(
          null_program, null_specialization);
      set_image_bindings(null_program);
      ShaderRecompiler::IR::CompiledShaderInfo null_info{};
      null_info.stage = null_program.stage;
      null_info.info = std::move(null_program.info);
      null_info.bindings = std::move(null_program.bindings);
      ShaderStageRuntime null_runtime{&null_info, &null_snapshot};
      PreparedBindings null_bindings;
      executor.PrepareBindings(null_runtime, null_bindings);
      executor.RebindImages(null_bindings);
      Require(name, "null descriptor count",
              null_bindings.images.size() == 3,
              "null descriptor preparation lost an image binding");
      const auto null_image_id = null_bindings.images[0].image_id;
      const auto &null_image = texture_cache.GetImage(null_image_id);
      Require(name, "one null image per format",
              null_bindings.images[1].image_id == null_image_id &&
                  null_bindings.images[2].image_id == null_image_id &&
                  TextureCacheTestAccess::NullImageCount(texture_cache) == 1,
              "null view dimension or usage created another image allocation");
      Require(name, "null descriptors share final cache acquisition",
              null_image.info.data.Empty() && !null_image.binding.is_bound &&
                  !null_image.binding.is_target &&
                  !null_image.binding.needs_rebind &&
                  null_image.usage.storage &&
                  null_image.IsGpuModified() &&
                  !TextureCacheTestAccess::PendingDownload(texture_cache,
                                                           null_image_id),
              "the shared null image did not preserve texture/storage "
              "acquisition without guest readback or RenderExecutor ownership");
      descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
          executor, scheduler.Current(), null_bindings));
      Require(name, "null descriptor general layouts",
              std::ranges::all_of(null_bindings.images,
                                  [](const auto &binding) {
                                    const auto info = MakeImageInfo(binding);
                                    return binding.layout ==
                                               vk::ImageLayout::eGeneral &&
                                           info.imageView ==
                                               binding.image_view &&
                                           info.imageLayout == binding.layout;
                                  }) &&
                  null_image.backing.state.layout == vk::ImageLayout::eGeneral,
              "shared sampled/storage null descriptors did not retain "
              "the general layout through descriptor generation");

      constexpr uint64_t target_mip_size = 0x10000;
      const auto make_target_desc = [&](uint64_t address, uint64_t size,
                                        vk::Extent3D extent) {
        ImageDesc desc{};
        desc.type = BindingType::RenderTarget;
        desc.info.data = {address, size};
        desc.info.pixel_format = vk::Format::eR32Uint;
        desc.info.guest_format = Prospero::BufferFormat::k32UInt;
        desc.info.type = Prospero::ImageType::kColor2D;
        desc.info.extent = extent;
        desc.info.resources = {1, 1};
        desc.info.pitch = extent.width;
        desc.info.bytes_per_block = 4;
        desc.info.samples = 1;
        desc.info.tile_mode = linear;
        desc.info.mip_layout[0] = {
            0, size, extent.width, extent.height, 0, size, extent.width};
        desc.info.tiling.block.bytes_per_element = 4;
        desc.info.linear_slice_stride = size;
        desc.view_info.format = desc.info.pixel_format;
        desc.view_info.type = vk::ImageViewType::e2D;
        desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
        desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
        return desc;
      };

      TextureCacheTestAccess::SetLinearReadback(texture_cache, true);
      ShaderTextureResource storage{};
      const uint64_t storage_address = base + 0x60000;
      constexpr uint32_t storage_native_value = 0x13579bdfu;
      constexpr uint32_t storage_stale_value = 0x2468ace0u;
      std::memcpy(reinterpret_cast<uint8_t *>(mapped) + 0x60000,
                  &storage_native_value, sizeof(storage_native_value));
      const uint64_t encoded_storage_address = storage_address >> 8u;
      storage.fields[0] = static_cast<uint32_t>(encoded_storage_address);
      storage.fields[1] =
          static_cast<uint32_t>(encoded_storage_address >> 32u) |
          (static_cast<uint32_t>(stencil_format) << 20u);
      storage.fields[3] =
          DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
          (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
      storage.fields[5] = 0x00700000u;
      ShaderRecompiler::IR::Program storage_program{};
      storage_program.stage = ShaderType::Vertex;
      storage_program.resource_tracking_complete = true;
      ShaderRecompiler::IR::ImageResource storage_resource{};
      storage_resource.resource_class =
          ShaderRecompiler::IR::ImageResourceClass::Storage;
      storage_resource.numeric_class = Prospero::TextureNumericClass::Uint;
      storage_resource.dimension =
          ShaderRecompiler::Decoder::ImageDimension::Dim2D;
      storage_resource.written = true;
      storage_program.info.images.push_back(storage_resource);
      set_image_bindings(storage_program);
      ShaderRecompiler::IR::DescriptorValue storage_descriptor{};
      std::copy(std::begin(storage.fields), std::end(storage.fields),
                storage_descriptor.dwords.begin());
      storage_descriptor.dword_count = 8;
      auto mipped_storage = storage;
      const auto encoded_mipped_storage_address = mipped_storage_address >> 8u;
      mipped_storage.fields[0] =
          static_cast<uint32_t>(encoded_mipped_storage_address);
      mipped_storage.fields[1] =
          static_cast<uint32_t>(encoded_mipped_storage_address >> 32u) |
          (static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u) |
          (3u << 30u);
      mipped_storage.fields[2] = 15u | (63u << 14u);
      mipped_storage.fields[3] =
          DstSel(4, 5, 6, 7) | (1u << 12u) | (3u << 16u) |
          (static_cast<uint32_t>(Prospero::TileMode::kStandard64KB) << 20u) |
          (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
      mipped_storage.fields[5] = 0x00700000u | (3u << 4u);
      ShaderRecompiler::IR::DescriptorValue mipped_storage_descriptor{};
      std::copy(std::begin(mipped_storage.fields),
                std::end(mipped_storage.fields),
                mipped_storage_descriptor.dwords.begin());
      mipped_storage_descriptor.dword_count = 8;
      auto overwide_mipped_storage = mipped_storage;
      overwide_mipped_storage.fields[3] =
          (overwide_mipped_storage.fields[3] & ~(0xfu << 16u)) | (4u << 16u);
      ShaderRecompiler::IR::DescriptorValue
          overwide_mipped_storage_descriptor{};
      std::copy(std::begin(overwide_mipped_storage.fields),
                std::end(overwide_mipped_storage.fields),
                overwide_mipped_storage_descriptor.dwords.begin());
      overwide_mipped_storage_descriptor.dword_count = 8;
      // The captured IMAGE_STORE addresses only BASE_LEVEL=1. Including
      // LAST_LEVEL=4 would shift this four-mip render-target allocation.
      auto fixed_storage_resource = storage_resource;
      fixed_storage_resource.numeric_class = Prospero::TextureNumericClass::Float;
      ShaderRecompiler::IR::DescriptorValue fixed_storage_descriptor{};
      fixed_storage_descriptor.dwords = {
          static_cast<uint32_t>((base + 0x300000) >> 8u),
          0xc4700000u, 0x001fc7ffu, 0x91b41facu,
          0, 0x00700030u, 0, 0};
      fixed_storage_descriptor.dword_count = 8;
      const auto fixed_storage_binding = RenderExecutorTestAccess::ResolveTexture(
          executor, fixed_storage_resource, fixed_storage_descriptor);
      Require(name, "fixed storage preserves physical mip layout",
              fixed_storage_binding.desc.info.resources.levels == 4 &&
                  fixed_storage_binding.desc.info.data.size == 0xb80000 &&
                  fixed_storage_binding.desc.info.mip_layout[1].offset == 0x180000 &&
                  fixed_storage_binding.desc.view_info.base_level == 1 &&
                  fixed_storage_binding.desc.view_info.level_count == 1 &&
                  texture_cache.FindTexture(fixed_storage_binding.image_id,
                                             fixed_storage_binding.desc) != nullptr,
              "IMAGE_STORE expanded an inaccessible mip and shifted physical storage");
      auto mipped_storage_resource = storage_resource;
      mipped_storage_resource.mip_mode =
          ShaderRecompiler::IR::ImageMipMode::Dynamic;
      mipped_storage_resource.mip_count = 3;
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
      ExpectFatal("MipViewPhysicalLayoutChange", [&] {
        auto linear_view = overwide_mipped_storage_descriptor;
        linear_view.dwords[3] &= ~(0x1fu << 20u);
        (void)RenderExecutorTestAccess::ResolveTexture(executor, mipped_storage_resource,
                                                       linear_view);
      });
#endif
      auto sampled_overwide_resource = storage_resource;
      sampled_overwide_resource.resource_class =
          ShaderRecompiler::IR::ImageResourceClass::Sampled;
      sampled_overwide_resource.numeric_class =
          Prospero::TextureNumericClass::Uint;
      sampled_overwide_resource.read = true;
      sampled_overwide_resource.written = false;
      {
        // MLB The Show 21 compresses a standalone final block through mip 9.
        constexpr uint64_t address = base + 0x180000;
        ShaderRecompiler::IR::DescriptorValue descriptor{};
        descriptor.dwords = {static_cast<uint32_t>(address >> 8u),
                             0x03e00000u, 0, 0x9019902cu, 0, 0x00700000u, 0, 0};
        descriptor.dword_count = 8;
        TestCase test;
        test.name = "RebasedLastMipQueryStore";
        test.has_user_data = true;
        std::copy_n(descriptor.dwords.begin(), 8, test.user_data.begin());
        test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_GET_RESINFO,
                        ShaderOpcode::IMAGE_STORE, ShaderOpcode::S_ENDPGM};
        AppendVMovU32(&test.code, 24, 0);
        AppendVMovU32(&test.code, 25, 0);
        test.code.push_back(EncodeMimg0(0x0e, 0x3));
        test.code.push_back(EncodeMimg1(0, 24));
        test.code.push_back(EncodeMimg0(0x08, 0x3));
        test.code.push_back(EncodeMimg1(0, 24));
        AppendEnd(&test.code);
        const auto compiled = CompileCase(test, SubgroupSize());
        auto sampled_descriptor = descriptor;
        sampled_descriptor.dwords[1] |= 9u << 16u;
        const auto query = RenderExecutorTestAccess::ResolveTexture(
            executor, compiled.program.info.images.at(0), sampled_descriptor);
        const auto store = RenderExecutorTestAccess::ResolveTexture(
            executor, compiled.program.info.images.at(1), descriptor);
        Require(name, "rebased last mip",
                query.image_id == store.image_id &&
                    query.desc.info.data.size == 256 &&
                    query.desc.info.resources.levels == 1 &&
                    query.desc.view_info.base_level == 0 &&
                    query.desc.view_info.level_count == 1 &&
                    query.desc.view_info.min_lod == 0 &&
                    store.desc.view_info.base_level == 0,
                "equivalent selected mip changed backing or relative LOD");
        Image sampled;
        sampled.view = texture_cache.FindTexture(query.image_id, query.desc);
        sampled.layout = vk::ImageLayout::eGeneral;
        Image storage_view;
        storage_view.view = texture_cache.FindTexture(store.image_id, store.desc);
        storage_view.layout = vk::ImageLayout::eGeneral;
        texture_cache.GetImage(store.image_id).Transit(
            vk::ImageLayout::eGeneral,
            vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
            {}, scheduler.Current().Handle());
        scheduler.Finish();
        Dispatch(test, compiled, {}, nullptr, &sampled, nullptr, &storage_view);
        texture_cache.MarkGpuWritten(store.image_id);
        Require(name, "rebased mip query and store",
                ReadCachedTexel(name, context, store.image_id) ==
                    std::vector<uint32_t>{1, 1},
                "query/store did not access the standalone 1x1 mip");
      }
      auto plain_mipped_storage_binding =
          RenderExecutorTestAccess::ResolveTexture(executor, storage_resource,
                                                   mipped_storage_descriptor);
      vk::ClearValue tail_clear{};
      tail_clear.color.uint32[0] = tail_gpu_value;
      TextureCacheTestAccess::ClearImage(
          texture_cache, scheduler.Current(), plain_mipped_storage_binding.image_id,
          {vk::ImageAspectFlagBits::eColor, 0, 4, 0, 1}, tail_clear);
      auto mipped_storage_binding = RenderExecutorTestAccess::ResolveTexture(
          executor, mipped_storage_resource, mipped_storage_descriptor);
      auto overwide_mipped_storage_binding =
          RenderExecutorTestAccess::ResolveTexture(
              executor, storage_resource, overwide_mipped_storage_descriptor);
      auto sampled_overwide_resolved = RenderExecutorTestAccess::ResolveTexture(
          executor, sampled_overwide_resource,
          overwide_mipped_storage_descriptor);
      const auto tail0 = ReadCachedTexel(name, context, sampled_overwide_resolved.image_id);
      const auto tail3 = ReadCachedTexel(name, context, sampled_overwide_resolved.image_id,
                                        {}, {1, 1, 1}, 0, 3);
      const auto tail4 = ReadCachedTexel(name, context, sampled_overwide_resolved.image_id,
                                        {}, {1, 1, 1}, 0, 4);
      Require(name, "expanded tail preserves GPU and backing data",
              tail0 == std::vector<u32>{tail_gpu_value} &&
                  tail3 == std::vector<u32>{tail_gpu_value} &&
                  tail4 == std::vector<u32>{tail_backing_value},
              "expanding a shared mip tail lost GPU writes or the added mip's backing bytes");
      PreparedBindings mipped_prepared{};
      ShaderRecompiler::IR::CompiledShaderInfo mipped_program{};
      mipped_program.info.images.push_back(storage_resource);
      mipped_program.info.images.push_back(mipped_storage_resource);
      mipped_program.info.images.push_back(storage_resource);
      mipped_program.info.images.push_back(sampled_overwide_resource);
      ShaderRecompiler::IR::ResourceSnapshot mipped_snapshot{};
      mipped_snapshot.images = {mipped_storage_descriptor,
                                mipped_storage_descriptor,
                                overwide_mipped_storage_descriptor,
                                overwide_mipped_storage_descriptor};
      ShaderStageRuntime mipped_runtime{&mipped_program, &mipped_snapshot};
      mipped_prepared.runtime = &mipped_runtime;
      mipped_prepared.images.push_back(
          std::move(plain_mipped_storage_binding));
      mipped_prepared.images.push_back(
          std::move(mipped_storage_binding));
      mipped_prepared.images.push_back(
          std::move(overwide_mipped_storage_binding));
      mipped_prepared.images.push_back(
          std::move(sampled_overwide_resolved));
      executor.RebindImages(mipped_prepared);
      auto &plain_mipped_binding = mipped_prepared.images[0];
      auto &mipped_binding = mipped_prepared.images[1];
      auto &overwide_mipped_binding = mipped_prepared.images[2];
      auto &sampled_overwide_binding = mipped_prepared.images[3];
      plain_mipped_binding.layout = vk::ImageLayout::eGeneral;
      mipped_binding.layout = vk::ImageLayout::eGeneral;
      overwide_mipped_binding.layout = vk::ImageLayout::eGeneral;
      sampled_overwide_binding.layout = vk::ImageLayout::eGeneral;
      Require(name, "dynamic storage mip views",
              plain_mipped_binding.desc.view_info.base_level == 1 &&
                  plain_mipped_binding.desc.view_info.level_count == 1 &&
                  plain_mipped_binding.mip_views.empty() &&
                  plain_mipped_binding.image_view ==
                      texture_cache.FindTexture(plain_mipped_binding.image_id,
                                                plain_mipped_binding.desc) &&
                  mipped_binding.desc.info.resources.levels == 4 &&
                  mipped_binding.desc.view_info.base_level == 1 &&
                  mipped_binding.desc.view_info.level_count == 3 &&
                  mipped_binding.mip_views.size() == 3 &&
                  mipped_binding.mip_views[0] != nullptr &&
                  mipped_binding.mip_views[1] != nullptr &&
                  mipped_binding.mip_views[2] != nullptr &&
                  mipped_binding.mip_views[0] != mipped_binding.mip_views[1] &&
                  mipped_binding.mip_views[1] != mipped_binding.mip_views[2] &&
                  MakeImageInfo(mipped_binding, 0).imageView ==
                      mipped_binding.mip_views[0] &&
                  MakeImageInfo(mipped_binding, 1).imageView ==
                      mipped_binding.mip_views[1] &&
                  MakeImageInfo(mipped_binding, 2).imageView ==
                      mipped_binding.mip_views[2],
              "base-1 through last-3 storage view did not create three "
              "ordered one-level descriptors");
      Require(name, "over-wide fixed storage mip view",
              overwide_mipped_storage.LastLevel() == 4 &&
                  overwide_mipped_storage.MaxMip() == 3 &&
                  overwide_mipped_binding.image_id ==
                      plain_mipped_binding.image_id &&
                  overwide_mipped_binding.desc.info.resources.levels == 4 &&
                  overwide_mipped_binding.desc.view_info.base_level == 1 &&
                  overwide_mipped_binding.desc.view_info.level_count == 1 &&
                  overwide_mipped_binding.mip_views.empty() &&
                  overwide_mipped_binding.image_view ==
                      plain_mipped_binding.image_view,
              "fixed storage view included mips that IMAGE_STORE cannot address");
      Require(name, "over-wide sampled mip view",
              sampled_overwide_binding.image_id ==
                      plain_mipped_binding.image_id &&
                  sampled_overwide_binding.desc.info.resources.levels == 5 &&
                  sampled_overwide_binding.desc.view_info.base_level == 1 &&
                  sampled_overwide_binding.desc.view_info.level_count == 4 &&
                  sampled_overwide_binding.mip_views.empty() &&
                  sampled_overwide_binding.image_view != nullptr,
              "sampled view lost addressable mips in the allocated tail");
      const auto *image_storage = mipped_prepared.images.data();
      const auto *mip_storage = mipped_prepared.images[1].mip_views.data();
      executor.PrepareBindings(mipped_runtime, mipped_prepared);
      Require(name, "prepared mip scratch",
              mipped_prepared.images.data() == image_storage &&
                  mipped_prepared.images[1].mip_views.empty() &&
                  mipped_prepared.images[1].mip_views.capacity() >= 3,
              "repeated image preparation retained stale mip views or reallocated scratch");
      executor.RebindImages(mipped_prepared);
      Require(name, "prepared mip views rebound",
              mipped_prepared.images[1].mip_views.size() == 3 &&
                  mipped_prepared.images[1].mip_views.data() == mip_storage,
              "reused image scratch did not rebuild dynamic storage mip views");
      // Streaming T# clamps apply after S# max LOD, without changing the view
      // base.
      {
        constexpr uint64_t lod_address = base + 0x170000;
        auto lod_descriptor = mipped_storage_descriptor;
        lod_descriptor.dwords[0] = static_cast<uint32_t>(lod_address >> 8u);
        lod_descriptor.dwords[1] =
            static_cast<uint32_t>(lod_address >> 40u) |
            (static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u) |
            (3u << 30u);
        lod_descriptor.dwords[3] &= ~(0xfu << 12u);
        ShaderSamplerResource lod_sampler{
            {0, 0,
             static_cast<uint32_t>(Prospero::SamplerMipFilter::kLinear) << 26u,
             0}};
        TestCase lod_test;
        lod_test.name = "TextureMinLodAfterSamplerClamp";
        lod_test.has_user_data = true;
        lod_test.image_descriptor_swizzle = mipped_storage.DstSelXYZW();
        std::copy_n(lod_descriptor.dwords.begin(), 8,
                    lod_test.user_data.begin());
        std::copy_n(lod_sampler.fields, 4, lod_test.user_data.begin() + 8);
        lod_test.user_data[50] = sizeof(uint32_t);
        lod_test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
                            ShaderOpcode::BUFFER_STORE_DWORD,
                            ShaderOpcode::S_ENDPGM};
        lod_test.required_spirv = {"OpImageSampleExplicitLod"};
        AppendVMovLiteral(&lod_test.code, 20, std::bit_cast<uint32_t>(0.5f));
        AppendVMovLiteral(&lod_test.code, 21, std::bit_cast<uint32_t>(0.5f));
        lod_test.code.push_back(EncodeMimg0(0x27, 1));
        lod_test.code.push_back(EncodeMimg1(0, 20, 0, 2));
        AppendStoreVgpr(&lod_test.code, 0, 0);
        AppendEnd(&lod_test.code);
        const auto lod_program = CompileCase(lod_test, SubgroupSize());
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
        ExpectFatal("TextureMinLodBeyondView", [&] {
          auto invalid_lod = lod_descriptor;
          invalid_lod.dwords[1] |= 1024u << 8u;
          (void)RenderExecutorTestAccess::ResolveTexture(
              executor, lod_program.program.info.images[0], invalid_lod);
        });
#endif
        const auto lod_binding = RenderExecutorTestAccess::ResolveTexture(
            executor, lod_program.program.info.images[0], lod_descriptor);
        for (uint32_t mip = 0; mip < 4; ++mip) {
          vk::ClearValue clear{};
          clear.color.float32[0] = static_cast<float>(1u << mip);
          TextureCacheTestAccess::ClearImage(
              texture_cache, scheduler.Current(), lod_binding.image_id,
              {vk::ImageAspectFlagBits::eColor, mip, 1, 0, 1}, clear);
        }
        const auto sampler = context.GetSamplerCache().GetSampler(
            lod_sampler, lod_program.program.info.samplers[0].integer_border);
        auto output = CreateStorageBuffer(lod_test.name, {}, 1);
        struct LodCase {
          uint32_t base_level;
          uint32_t min_lod;
          float expected;
          float integer_min_lod_expected;
        };
        constexpr std::array lod_cases{
            LodCase{0, 0, 1.0f, 1.0f},   LodCase{0, 256, 2.0f, 2.0f},
            LodCase{0, 384, 3.0f, 2.0f}, LodCase{1, 384, 3.0f, 2.0f},
            LodCase{1, 512, 4.0f, 4.0f}, LodCase{1, 0, 2.0f, 2.0f},
            LodCase{0, 256, 2.0f, 2.0f}};
        std::array<vk::ImageView, lod_cases.size()> views{};
        for (size_t index = 0; index < lod_cases.size(); ++index) {
          const auto &test = lod_cases[index];
          lod_descriptor.dwords[1] =
              (lod_descriptor.dwords[1] & ~0xfff00u) | (test.min_lod << 8u);
          lod_descriptor.dwords[3] =
              (lod_descriptor.dwords[3] & ~(0xfu << 12u)) |
              (test.base_level << 12u);
          std::copy_n(lod_descriptor.dwords.begin(), 8,
                      lod_test.user_data.begin());
          const auto binding = RenderExecutorTestAccess::ResolveTexture(
              executor, lod_program.program.info.images[0], lod_descriptor);
          views[index] =
              texture_cache.FindTexture(binding.image_id, binding.desc);
          auto &image = texture_cache.GetImage(binding.image_id);
          image.Transit(vk::ImageLayout::eShaderReadOnlyOptimal,
                        vk::AccessFlagBits2::eShaderRead, {},
                        scheduler.Current().Handle());
          Image sampled;
          sampled.view = views[index];
          sampled.layout = image.backing.state.layout;
          scheduler.Finish();
          Dispatch(lod_test, lod_program, output, nullptr, &sampled, nullptr,
                   nullptr, sampler);
          const auto result = ReadBuffer(lod_test.name, output, 1)[0];
          // Vulkan permits flooring imageViewMinLod instead of retaining its
          // fraction.
          Require(lod_test.name, "sampled mip",
                  result == std::bit_cast<uint32_t>(test.expected) ||
                      result == std::bit_cast<uint32_t>(
                                    test.integer_min_lod_expected),
                  "texture clamp was lost, applied before sampler max LOD, or "
                  "rebased incorrectly");
        }
        Require(lod_test.name, "view identity",
                views[0] != views[1] && views[1] != views[2] &&
                    views[2] != views[3] && views[1] == views.back(),
                "different texture clamps aliased or identical clamped views "
                "were not reused");
        ShaderRecompiler::IR::CompiledShaderInfo gather_info{};
        gather_info.stage = ShaderType::Compute;
        gather_info.info = lod_program.program.info;
        gather_info.info.images[0].mip_mode =
            ShaderRecompiler::IR::ImageMipMode::Dynamic;
        gather_info.info.images[0].mip_count = 4;
        auto gather_snapshot = lod_program.resources;
        gather_snapshot.images[0] = lod_descriptor;
        ShaderStageRuntime gather_runtime{&gather_info, &gather_snapshot};
        PreparedBindings gather_bindings;
        executor.PrepareBindings(gather_runtime, gather_bindings);
        executor.RebindImages(gather_bindings);
        const auto &gather_binding = gather_bindings.images[0];
        Require(lod_test.name, "dynamic sampled mip views",
                gather_binding.desc.view_info.min_lod == 256u &&
                    gather_binding.mip_views.size() == 4,
                "dynamic gather lost the guest minimum or accessible mip range");
        for (uint32_t mip = 0; mip < 4; ++mip) {
          auto single_mip = gather_binding.desc;
          single_mip.view_info.base_level = mip;
          single_mip.view_info.level_count = 1;
          single_mip.view_info.min_lod = 0;
          Require(lod_test.name, "gather minimum applied once",
                  gather_binding.mip_views[mip] == texture_cache.FindTexture(
                      gather_binding.image_id, single_mip),
                  "the gather mip view reapplied the shader's streaming clamp");
        }
        DestroyBuffer(&output);
        RenderExecutorTestAccess::ResetBindings(executor);
      }

      auto srgb_storage = storage;
      constexpr auto srgb_format =
          static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8Srgb);
      constexpr uint64_t srgb_storage_address = base + 0xc0000;
      const auto encoded_srgb_storage_address = srgb_storage_address >> 8u;
      srgb_storage.fields[0] =
          static_cast<uint32_t>(encoded_srgb_storage_address);
      srgb_storage.fields[1] =
          static_cast<uint32_t>(encoded_srgb_storage_address >> 32u) |
          (srgb_format << 20u);
      ShaderRecompiler::IR::DescriptorValue srgb_storage_descriptor{};
      std::copy(std::begin(srgb_storage.fields), std::end(srgb_storage.fields),
                srgb_storage_descriptor.dwords.begin());
      srgb_storage_descriptor.dword_count = 8;
      auto srgb_storage_resource = storage_resource;
      srgb_storage_resource.resource_class =
          ShaderRecompiler::IR::ImageResourceClass::Storage;
      srgb_storage_resource.numeric_class =
          Prospero::TextureNumericClass::Float;
      const auto srgb_storage_binding =
          RenderExecutorTestAccess::ResolveTexture(
              executor, srgb_storage_resource, srgb_storage_descriptor);
      const auto srgb_storage_view = texture_cache.FindTexture(
          srgb_storage_binding.image_id, srgb_storage_binding.desc);
      Require(name, "sRGB storage view",
              srgb_storage_view != nullptr &&
                  srgb_storage_binding.desc.info.pixel_format ==
                      vk::Format::eR8G8B8A8Srgb &&
                  srgb_storage_binding.desc.view_info.format ==
                      vk::Format::eR8G8B8A8Unorm &&
                  texture_cache.GetImage(srgb_storage_binding.image_id)
                          .backing.format == vk::Format::eR8G8B8A8Srgb,
              "storage descriptor did not preserve its sRGB backing and "
              "select an UNORM Vulkan view");

      ShaderTextureResource sint_storage{{0x01514b00u, 0xc1500000u, 0x000bc00bu,
                                          0x91b00204u, 0x00000000u, 0x00700000u,
                                          0x102b0000u, 0x0001514au}};
      Require(name, "PPSA06888 R32 SINT descriptor",
              sint_storage.Base40() == 0x1514b0000ull &&
                  sint_storage.Width5() + 1u == 48 &&
                  sint_storage.Height5() + 1u == 48 &&
                  sint_storage.Depth() + 1u == 1 &&
                  sint_storage.Format() == Prospero::BufferFormat::k32SInt &&
                  sint_storage.Type() == Prospero::ImageType::kColor2D &&
                  sint_storage.TileMode() ==
                      Prospero::TileMode::kRenderTarget &&
                  sint_storage.DstSelXYZW() == DstSel(4, 0, 0, 1),
              "captured write-only signed storage descriptor was decoded "
              "incorrectly");
      const uint64_t mapped_sint_address = base + 0xe0000;
      const auto encoded_sint_address = mapped_sint_address >> 8u;
      sint_storage.fields[0] = static_cast<uint32_t>(encoded_sint_address);
      sint_storage.fields[1] =
          (sint_storage.fields[1] & ~0xffu) |
          static_cast<uint32_t>(encoded_sint_address >> 32u);
      constexpr uint64_t sint_metadata_address = base + 0x160000;
      const auto encoded_sint_metadata = sint_metadata_address >> 8u;
      sint_storage.fields[6] = (sint_storage.fields[6] & 0x00ffffffu) |
          (static_cast<uint32_t>(encoded_sint_metadata & 0xffu) << 24u);
      sint_storage.fields[7] = static_cast<uint32_t>(encoded_sint_metadata >> 8u);
      WriteMetadata(context, sint_metadata_address, 0x1000, UINT32_MAX);
      ShaderRecompiler::IR::DescriptorValue sint_storage_descriptor{};
      std::copy(std::begin(sint_storage.fields), std::end(sint_storage.fields),
                sint_storage_descriptor.dwords.begin());
      sint_storage_descriptor.dword_count = 8;
      auto sint_storage_resource = srgb_storage_resource;
      sint_storage_resource.resource_class =
          ShaderRecompiler::IR::ImageResourceClass::Storage;
      sint_storage_resource.numeric_class = Prospero::TextureNumericClass::Uint;
      const auto sint_storage_binding =
          RenderExecutorTestAccess::ResolveTexture(
              executor, sint_storage_resource, sint_storage_descriptor);
      const auto sint_storage_view = texture_cache.FindTexture(
          sint_storage_binding.image_id, sint_storage_binding.desc);
      Require(name, "PPSA06888 raw R32 SINT storage view",
              sint_storage_view != nullptr &&
                  sint_storage_binding.desc.info.metadata.range ==
                      GuestRange{sint_metadata_address, 0x1000} &&
                  sint_storage_binding.desc.info.data.size == 0x10000 &&
                  sint_storage_binding.desc.info.pixel_format ==
                      vk::Format::eR32Sint &&
                  sint_storage_binding.desc.view_info.format ==
                      vk::Format::eR32Uint &&
                  texture_cache.GetImage(sint_storage_binding.image_id)
                          .backing.format == vk::Format::eR32Sint,
              "write-only R32 SINT storage did not select a bit-compatible "
              "uint view");

      auto narrowed_storage = storage;
      constexpr uint64_t narrowed_storage_address = base + 0xd0000;
      const auto encoded_narrowed_address = narrowed_storage_address >> 8u;
      narrowed_storage.fields[0] =
          static_cast<uint32_t>(encoded_narrowed_address);
      narrowed_storage.fields[1] =
          static_cast<uint32_t>(encoded_narrowed_address >> 32u) |
          (static_cast<uint32_t>(stencil_format) << 20u);
      narrowed_storage.fields[3] =
          DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
          (static_cast<uint32_t>(Prospero::ImageType::kColor1DArray) << 28u);
      narrowed_storage.fields[4] = 1u | (1u << 16u);
      ShaderRecompiler::IR::DescriptorValue narrowed_storage_descriptor{};
      std::copy(std::begin(narrowed_storage.fields),
                std::end(narrowed_storage.fields),
                narrowed_storage_descriptor.dwords.begin());
      narrowed_storage_descriptor.dword_count = 8;
      auto narrowed_storage_resource = storage_resource;
      narrowed_storage_resource.dimension =
          ShaderRecompiler::Decoder::ImageDimension::Dim1D;
      const auto narrowed_storage_binding =
          RenderExecutorTestAccess::ResolveTexture(
              executor, narrowed_storage_resource, narrowed_storage_descriptor);
      const auto narrowed_storage_view = texture_cache.FindTexture(
          narrowed_storage_binding.image_id, narrowed_storage_binding.desc);
      const auto &narrowed_storage_image =
          texture_cache.GetImage(narrowed_storage_binding.image_id);
      Require(name, "narrowed 1D array storage view",
              narrowed_storage_view != nullptr &&
                  narrowed_storage_binding.desc.info.type ==
                      Prospero::ImageType::kColor1D &&
                  narrowed_storage_binding.desc.info.resources.layers == 2 &&
                  narrowed_storage_binding.desc.view_info.type ==
                      vk::ImageViewType::e1D &&
                  narrowed_storage_binding.desc.view_info.base_layer == 1 &&
                  narrowed_storage_binding.desc.view_info.layer_count == 1 &&
                  narrowed_storage_image.backing.image_type ==
                      vk::ImageType::e1D,
              "non-array 1D specialization did not select the descriptor "
              "base layer from its 1D-array backing");

      ShaderRecompiler::IR::ResourceSnapshot storage_snapshot{};
      storage_snapshot.images.push_back(storage_descriptor);
      ShaderRecompiler::IR::CompiledShaderInfo storage_info{};
      storage_info.stage = storage_program.stage;
      storage_info.info = std::move(storage_program.info);
      storage_info.bindings = std::move(storage_program.bindings);
      ShaderStageRuntime storage_runtime{&storage_info, &storage_snapshot};
      PreparedBindings storage_discovery;
      executor.PrepareBindings(storage_runtime, storage_discovery);
      const auto storage_id = storage_discovery.images[0].image_id;
      Require(name, "storage prefetch purity",
              storage_discovery.images[0].image_view == nullptr &&
                  texture_cache.GetImage(storage_id).binding.is_bound &&
                  !texture_cache.GetImage(storage_id).usage.storage &&
                  !TextureCacheTestAccess::PendingDownload(texture_cache,
                                                           storage_id),
              "the first descriptor pass performed final storage acquisition");
      RenderExecutorTestAccess::ResetBindings(executor);

      ShaderRecompiler::IR::Program sampled_program{};
      sampled_program.stage = ShaderType::Pixel;
      sampled_program.resource_tracking_complete = true;
      ShaderRecompiler::IR::ImageResource sampled_resource{};
      sampled_resource.resource_class =
          ShaderRecompiler::IR::ImageResourceClass::Sampled;
      sampled_resource.numeric_class = Prospero::TextureNumericClass::Uint;
      sampled_resource.dimension =
          ShaderRecompiler::Decoder::ImageDimension::Dim2D;
      sampled_resource.read = true;
      sampled_program.info.images.push_back(sampled_resource);
      set_image_bindings(sampled_program);
      ShaderRecompiler::IR::ResourceSnapshot sampled_snapshot{};
      sampled_snapshot.images.push_back(storage_descriptor);
      ShaderRecompiler::IR::CompiledShaderInfo sampled_info{};
      sampled_info.stage = sampled_program.stage;
      sampled_info.info = std::move(sampled_program.info);
      sampled_info.bindings = std::move(sampled_program.bindings);
      ShaderStageRuntime sampled_runtime{&sampled_info, &sampled_snapshot};

      constexpr uint64_t ordered_sampled_address = base + 0x10000;
      const uint32_t ordered_sampled_value = 0x89abcdefu;
      std::memcpy(reinterpret_cast<uint8_t *>(mapped) + 0x10000,
                  &ordered_sampled_value, sizeof(ordered_sampled_value));
      auto ordered_sampled = storage;
      const auto encoded_ordered_address = ordered_sampled_address >> 8u;
      ordered_sampled.fields[0] =
          static_cast<uint32_t>(encoded_ordered_address);
      ordered_sampled.fields[1] =
          static_cast<uint32_t>(encoded_ordered_address >> 32u) |
          (static_cast<uint32_t>(stencil_format) << 20u);
      ShaderRecompiler::IR::DescriptorValue ordered_descriptor{};
      std::copy(std::begin(ordered_sampled.fields),
                std::end(ordered_sampled.fields),
                ordered_descriptor.dwords.begin());
      ordered_descriptor.dword_count = 8;
      ShaderRecompiler::IR::ResourceSnapshot ordered_snapshot{};
      ordered_snapshot.images.push_back(ordered_descriptor);
      ShaderStageRuntime ordered_sampled_runtime{sampled_runtime.program,
                                                 &ordered_snapshot};
      auto ordered_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings(
          executor, storage_runtime, ordered_sampled_runtime, true);
      const auto ordered_sampled_id =
          ordered_bindings.pixel->images[0].image_id;
      Require(
          name, "VS-before-PS retained-owner order",
          ordered_bindings.vertex[0].images[0].image_id == storage_id &&
              ordered_sampled_id != storage_id &&
              RenderExecutorTestAccess::BoundImagesInOrder(executor, storage_id,
                                                           ordered_sampled_id),
          "production graphics binding did not retain vertex resources "
          "before pixel resources");
      RenderExecutorTestAccess::ResetBindings(executor);

      auto graphics_bindings =
          RenderExecutorTestAccess::PrepareGraphicsBindings(
              executor, storage_runtime, sampled_runtime, true);
      const auto &storage_binding =
          graphics_bindings.vertex[0].images[0];
      const auto &sampled_binding =
          graphics_bindings.pixel->images[0];
      Require(name, "storage final acquisition",
              storage_binding.image_view != nullptr &&
                  texture_cache.GetImage(storage_id).usage.storage &&
                  texture_cache.GetImage(storage_id).IsGpuModified() &&
                  TextureCacheTestAccess::PendingDownload(texture_cache,
                                                          storage_id),
              "the production graphics binding path did not establish GPU "
              "ownership before storage readback");
      Require(name, "VS-to-PS image acquisition order",
              storage_binding.image_id == storage_id &&
                  sampled_binding.image_id == storage_id &&
                  texture_cache.GetImage(storage_id).binding.force_general &&
                  texture_cache.GetImage(storage_id).binding.shader_write &&
                  texture_cache.GetImage(storage_id).IsGpuModified() &&
                  texture_cache.GetImage(storage_id).usage.storage,
              "the production graphics binding path did not complete vertex "
              "storage acquisition before pixel sampling");
      descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
          executor, scheduler.Current(), graphics_bindings.vertex[0],
          *graphics_bindings.pixel));
      Require(name, "early writable alias retention",
              texture_cache.GetImage(storage_id).backing.state.access_mask ==
                  (vk::AccessFlagBits2::eShaderRead |
                   vk::AccessFlagBits2::eShaderWrite),
              "a later sampled alias dropped an earlier storage write access");
      RenderExecutorTestAccess::ResetBindings(executor);

      ShaderRecompiler::IR::Program vertex_sampled_program{};
      vertex_sampled_program.stage = ShaderType::Vertex;
      vertex_sampled_program.resource_tracking_complete = true;
      vertex_sampled_program.info.images.push_back(sampled_resource);
      set_image_bindings(vertex_sampled_program);
      ShaderRecompiler::IR::CompiledShaderInfo vertex_sampled_info{};
      vertex_sampled_info.stage = vertex_sampled_program.stage;
      vertex_sampled_info.info = std::move(vertex_sampled_program.info);
      vertex_sampled_info.bindings =
          std::move(vertex_sampled_program.bindings);
      ShaderStageRuntime vertex_sampled_runtime{
          &vertex_sampled_info, sampled_runtime.resources};
      ShaderRecompiler::IR::Program pixel_storage_program{};
      pixel_storage_program.stage = ShaderType::Pixel;
      pixel_storage_program.resource_tracking_complete = true;
      pixel_storage_program.info.images.push_back(storage_resource);
      set_image_bindings(pixel_storage_program);
      ShaderRecompiler::IR::CompiledShaderInfo pixel_storage_info{};
      pixel_storage_info.stage = pixel_storage_program.stage;
      pixel_storage_info.info = std::move(pixel_storage_program.info);
      pixel_storage_info.bindings = std::move(pixel_storage_program.bindings);
      ShaderStageRuntime pixel_storage_runtime{&pixel_storage_info,
                                               storage_runtime.resources};
      auto writable_alias_bindings =
          RenderExecutorTestAccess::PrepareGraphicsBindings(
              executor, vertex_sampled_runtime, pixel_storage_runtime, true);
      Require(name, "late writable alias promotion",
              texture_cache.GetImage(storage_id).binding.force_general,
              "an already sampled image was not promoted when a later "
              "storage alias bound the same backing");
      descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
          executor, scheduler.Current(), writable_alias_bindings.vertex[0],
          *writable_alias_bindings.pixel));
      Require(
          name, "forced-general descriptor capture",
          writable_alias_bindings.vertex[0].images[0].layout ==
                  vk::ImageLayout::eGeneral &&
              writable_alias_bindings.pixel->images[0].layout ==
                  vk::ImageLayout::eGeneral &&
              MakeImageInfo(writable_alias_bindings.vertex[0].images[0])
                      .imageLayout == vk::ImageLayout::eGeneral &&
              MakeImageInfo(writable_alias_bindings.pixel->images[0])
                      .imageLayout == vk::ImageLayout::eGeneral &&
              texture_cache.GetImage(storage_id).backing.state.layout ==
                  vk::ImageLayout::eGeneral &&
              texture_cache.GetImage(storage_id).backing.state.access_mask ==
                  (vk::AccessFlagBits2::eShaderRead |
                   vk::AccessFlagBits2::eShaderWrite),
          "sampled/storage aliases did not retain the promoted general "
          "layout and writable access in both generated descriptors");
      RenderExecutorTestAccess::ResetBindings(executor);

      const std::array<uint32_t, 3> split_mip_data{0x01020304u, 0x05060708u,
                                                   0x090a0b0cu};
      std::memcpy(reinterpret_cast<uint8_t *>(mapped) + 0x50000,
                  split_mip_data.data(), sizeof(split_mip_data));
      ImageDesc split_desc{};
      split_desc.type = BindingType::Texture;
      split_desc.info.data = {base + 0x50000, sizeof(split_mip_data)};
      split_desc.info.pixel_format = vk::Format::eR32Uint;
      split_desc.info.guest_format = Prospero::BufferFormat::k32UInt;
      split_desc.info.type = Prospero::ImageType::kColor2D;
      split_desc.info.extent = {2, 1, 1};
      split_desc.info.resources = {2, 1};
      split_desc.info.pitch = 2;
      split_desc.info.bytes_per_block = 4;
      split_desc.info.samples = 1;
      split_desc.info.tile_mode = linear;
      split_desc.info.mip_layout[0] = {0, 8, 2, 1, 0, 8, 2};
      split_desc.info.mip_layout[1] = {8, 4, 1, 1, 8, 4, 1};
      split_desc.info.tiling.block.bytes_per_element = 4;
      split_desc.view_info.format = split_desc.info.pixel_format;
      split_desc.view_info.type = vk::ImageViewType::e2D;
      split_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
      split_desc.view_info.level_count = 2;
      split_desc.view_info.layer_count = 1;
      split_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
      const auto split_id = texture_cache.FindImage(split_desc);

      auto split_storage_desc = split_desc;
      split_storage_desc.type = BindingType::Storage;
      split_storage_desc.view_info.base_level = 0;
      split_storage_desc.view_info.level_count = 1;
      split_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
      auto split_sampled_desc = split_desc;
      split_sampled_desc.view_info.base_level = 1;
      split_sampled_desc.view_info.level_count = 1;
      split_sampled_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;

      ShaderRecompiler::IR::Program split_ir{};
      split_ir.stage = ShaderType::Vertex;
      split_ir.resource_tracking_complete = true;
      split_ir.info.images = {storage_resource, sampled_resource};
      set_image_bindings(split_ir);
      ShaderRecompiler::IR::CompiledShaderInfo split_program{};
      split_program.stage = split_ir.stage;
      split_program.info = std::move(split_ir.info);
      split_program.bindings = std::move(split_ir.bindings);
      PreparedBindings split_bindings{};
      ShaderRecompiler::IR::ResourceSnapshot split_snapshot;
      ShaderStageRuntime split_runtime{&split_program, &split_snapshot};
      split_bindings.runtime = &split_runtime;
      split_bindings.images.push_back(
          {split_id, texture_cache.FindTexture(split_id, split_storage_desc),
           split_storage_desc});
      split_bindings.images.push_back(
          {split_id, texture_cache.FindTexture(split_id, split_sampled_desc),
           split_sampled_desc});
      descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
          executor, scheduler.Current(), split_bindings));
      const auto &split_image = texture_cache.GetImage(split_id);
      Require(
          name, "per-binding subresource layouts",
          split_bindings.images[0].layout ==
                  vk::ImageLayout::eGeneral &&
              split_bindings.images[1].layout ==
                  vk::ImageLayout::eShaderReadOnlyOptimal &&
              MakeImageInfo(split_bindings.images[0]).imageLayout ==
                  vk::ImageLayout::eGeneral &&
              MakeImageInfo(split_bindings.images[1]).imageLayout ==
                  vk::ImageLayout::eShaderReadOnlyOptimal &&
              split_image.backing.subresource_states.size() == 2 &&
              split_image.backing.subresource_states[0].layout ==
                  vk::ImageLayout::eGeneral &&
              split_image.backing.subresource_states[1].layout ==
                  vk::ImageLayout::eShaderReadOnlyOptimal,
          "descriptor layouts were queried after a later mip transition "
          "instead of being captured at each binding");

      auto disjoint_depth_desc = split_desc;
      disjoint_depth_desc.type = BindingType::DepthTarget;
      disjoint_depth_desc.info.data.address = base + 0xe00000;
      disjoint_depth_desc.info.pixel_format = vk::Format::eD32Sfloat;
      disjoint_depth_desc.info.guest_format = Prospero::BufferFormat::k32Float;
      disjoint_depth_desc.view_info.format = vk::Format::eD32Sfloat;
      disjoint_depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
      disjoint_depth_desc.view_info.level_count = 1;
      disjoint_depth_desc.view_info.usage =
          vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto disjoint_depth_id = texture_cache.FindImage(disjoint_depth_desc);
      auto disjoint_sampled_desc = disjoint_depth_desc;
      disjoint_sampled_desc.type = BindingType::Texture;
      disjoint_sampled_desc.view_info.base_level = 1;
      disjoint_sampled_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
      const auto disjoint_sampled_view =
          texture_cache.FindTexture(disjoint_depth_id, disjoint_sampled_desc);
      RenderDepthInfo disjoint_depth{};
      disjoint_depth.desc = disjoint_depth_desc;
      disjoint_depth.image_id = disjoint_depth_id;
      disjoint_depth.depth_test_enable = true;
      disjoint_depth.depth_write_enable = true;
      disjoint_depth.depth_compare_op = vk::CompareOp::eAlways;
      PreparedBindings disjoint_binding{};
      disjoint_binding.runtime = &sampled_runtime;
      disjoint_binding.images.push_back(
          {disjoint_depth_id, disjoint_sampled_view, disjoint_sampled_desc});
      RenderExecutorTestAccess::BindRenderTarget(executor, disjoint_depth_id);
      vk::ImageAspectFlags disjoint_feedback;
      std::array<PreparedBindings *, 1> disjoint_stages{&disjoint_binding};
      RenderColorInfo no_disjoint_color{};
      const auto disjoint_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
          executor, scheduler.Current(), &no_disjoint_color, 0, disjoint_depth,
          disjoint_stages, &disjoint_feedback);
      descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
          executor, scheduler.Current(), disjoint_binding));
      const auto &disjoint_image = texture_cache.GetImage(disjoint_depth_id);
      Require(name, "disjoint depth sampling during depth writes",
              !disjoint_feedback &&
                  disjoint_rendering.depth_stencil_attachment.image_layout ==
                      vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT &&
                  disjoint_binding.images[0].layout ==
                      disjoint_rendering.depth_stencil_attachment.image_layout &&
                  disjoint_image.backing.state.layout ==
                      disjoint_rendering.depth_stencil_attachment.image_layout &&
                  disjoint_image.backing.subresource_states.empty(),
              "disjoint sampling used inconsistent image layouts or enabled feedback");
      RenderExecutorTestAccess::ResetBindings(executor);

      auto overlapping_sampled_desc = disjoint_sampled_desc;
      overlapping_sampled_desc.view_info.base_level = 0;
      const auto overlapping_view = texture_cache.FindTexture(
          disjoint_depth_id, overlapping_sampled_desc);
      ShaderRecompiler::IR::Program depth_views_ir{};
      depth_views_ir.stage = ShaderType::Pixel;
      depth_views_ir.resource_tracking_complete = true;
      depth_views_ir.info.images = {sampled_resource, sampled_resource};
      set_image_bindings(depth_views_ir);
      ShaderRecompiler::IR::CompiledShaderInfo depth_views_program{};
      depth_views_program.stage = depth_views_ir.stage;
      depth_views_program.info = std::move(depth_views_ir.info);
      depth_views_program.bindings = std::move(depth_views_ir.bindings);
      ShaderRecompiler::IR::ResourceSnapshot depth_views_snapshot;
      ShaderStageRuntime depth_views_runtime{&depth_views_program, &depth_views_snapshot};
      PreparedBindings depth_views_binding{};
      depth_views_binding.runtime = &depth_views_runtime;
      depth_views_binding.images.push_back(
          {disjoint_depth_id, overlapping_view, overlapping_sampled_desc});
      depth_views_binding.images.push_back(
          {disjoint_depth_id, disjoint_sampled_view, disjoint_sampled_desc});
      RenderExecutorTestAccess::BindRenderTarget(executor, disjoint_depth_id);
      std::array<PreparedBindings *, 1> depth_views_stages{&depth_views_binding};
      vk::ImageAspectFlags depth_views_feedback;
      const auto depth_views_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
          executor, scheduler.Current(), &no_disjoint_color, 0, disjoint_depth,
          depth_views_stages, &depth_views_feedback);
      descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
          executor, scheduler.Current(), depth_views_binding));
      Require(name, "overlapping and disjoint depth views",
              overlapping_view != nullptr &&
                  depth_views_feedback == vk::ImageAspectFlagBits::eDepth &&
                  depth_views_rendering.depth_stencil_attachment.image_layout ==
                      vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT &&
                  MakeImageInfo(depth_views_binding.images[0]).imageLayout ==
                      depth_views_rendering.depth_stencil_attachment.image_layout &&
                  MakeImageInfo(depth_views_binding.images[1]).imageLayout ==
                      depth_views_rendering.depth_stencil_attachment.image_layout,
              "two depth views chose different descriptor layouts or feedback aspects");

      Libs::LibKernel::Memory::WriteBacking(
          storage_address, &storage_stale_value, sizeof(storage_stale_value));
      texture_cache.ProcessDownloadImages();
      Require(
          name, "storage acquisition download consumption",
          !TextureCacheTestAccess::PendingDownload(texture_cache, storage_id),
          "submit-time processing retained an acquired storage request");
      RenderExecutorTestAccess::ResetBindings(executor);
      scheduler.Finish();
      scheduler.DrainPriorityOperations();
      uint32_t storage_downloaded_value = 0;
      Libs::LibKernel::Memory::TryReadBacking(storage_address,
                                              &storage_downloaded_value,
                                              sizeof(storage_downloaded_value));
      const uint32_t storage_expected_value =
          (storage_stale_value & 0xffffff00u) | (storage_native_value & 0xffu);
      Require(name, "storage acquisition readback window",
              storage_downloaded_value == storage_expected_value &&
                  texture_cache.GetImage(storage_id).IsGpuModified(),
              "readback enrollment was consumed before final storage "
              "ownership became visible");

      constexpr uint64_t phased_depth_address = base + 0x90000;
      constexpr uint64_t phased_stencil_address = base + 0xa0000;
      constexpr uint64_t phased_htile_address = base + 0xb0000;
      HW::DepthRenderTarget phased_depth_target{};
      phased_depth_target.z_info.format = Prospero::DepthFormat::kZ32F;
      phased_depth_target.z_info.texture_compatibility =
          Prospero::TextureCompatiblePlaneCompression::kEnable;
      phased_depth_target.z_info.htile_acceleration = true;
      phased_depth_target.z_info.z_compare_base = Prospero::ZCompareBase::kZMax;
      phased_depth_target.stencil_info.format = Prospero::StencilFormat::k8UInt;
      phased_depth_target.stencil_info.texture_compatibility =
          Prospero::TextureCompatibleStencil::kEnable;
      phased_depth_target.stencil_info.htile_stencil_disabled = false;
      phased_depth_target.z_read_base_addr = phased_depth_address;
      phased_depth_target.z_write_base_addr = phased_depth_address;
      phased_depth_target.stencil_read_base_addr = phased_stencil_address;
      phased_depth_target.stencil_write_base_addr = phased_stencil_address;
      phased_depth_target.htile_data_base_addr = phased_htile_address;
      phased_depth_target.size.valid = true;
      registers.SetDepthRenderTarget(phased_depth_target);
      HW::DepthControl phased_depth_control{};
      phased_depth_control.z_write_enable = true;
      registers.SetDepthControl(phased_depth_control);
      registers.SetRenderControl({});
      RenderDepthInfo disabled_depth{};
      RenderExecutorTestAccess::ResolveRenderDepthTarget(
          executor, scheduler.Current(), disabled_depth);
      Require(name, "disabled tests with stale depth write enable",
              !disabled_depth.image_id,
              "a dormant depth write bit bound a depth attachment");
      phased_depth_control.z_enable = true;
      registers.SetDepthControl(phased_depth_control);
      HW::RenderControl phased_render_control{};
      phased_render_control.resummarize_enable = true;
      registers.SetRenderControl(phased_render_control);

      RenderDepthInfo phased_depth{};
      RenderExecutorTestAccess::ResolveRenderDepthTarget(
          executor, scheduler.Current(), phased_depth);
      // PRT permits missing backing; a fully resident target keeps the same layout and image.
      phased_depth_target.z_info.partially_resident = true;
      phased_depth_target.stencil_info.partially_resident = true;
      registers.SetDepthRenderTarget(phased_depth_target);
      RenderDepthInfo resident_prt_depth{};
      RenderExecutorTestAccess::ResolveRenderDepthTarget(
          executor, scheduler.Current(), resident_prt_depth);
      Require(name, "fully resident PRT depth identity",
              resident_prt_depth.image_id == phased_depth.image_id &&
                  resident_prt_depth.desc.info.data == phased_depth.desc.info.data &&
                  resident_prt_depth.desc.info.stencil == phased_depth.desc.info.stencil,
              "PRT permission changed a fully backed depth/stencil attachment");
      auto non_texture_compatible_target = phased_depth_target;
      non_texture_compatible_target.z_info.texture_compatibility =
          Prospero::TextureCompatiblePlaneCompression::kDisable;
      non_texture_compatible_target.stencil_info.texture_compatibility =
          Prospero::TextureCompatibleStencil::kDisable;
      registers.SetDepthRenderTarget(non_texture_compatible_target);
      RenderDepthInfo non_texture_compatible_depth{};
      RenderExecutorTestAccess::ResolveRenderDepthTarget(
          executor, scheduler.Current(), non_texture_compatible_depth);
      Require(
          name, "depth texture compatibility identity",
          phased_depth.image_id && (phased_depth.desc.info.metadata.kind == ImageMetadataKind::Htile) &&
              non_texture_compatible_depth.image_id == phased_depth.image_id &&
              non_texture_compatible_depth.desc.info.data ==
                  phased_depth.desc.info.data &&
              non_texture_compatible_depth.desc.info.stencil ==
                  phased_depth.desc.info.stencil &&
              non_texture_compatible_depth.desc.info.metadata.range ==
                  phased_depth.desc.info.metadata.range &&
              phased_depth.desc.info.metadata.stencil_compressed &&
              !phased_depth.depth_clear_enable &&
              !texture_cache.IsMeta(phased_depth.desc.info.metadata.range.address) &&
              !texture_cache.GetImage(phased_depth.image_id).IsGpuModified(),
          "valid PS5 texture-compatibility policy changed logical depth image "
          "identity or discovery side effects");

      auto read_only_depth_target = phased_depth_target;
      read_only_depth_target.z_write_base_addr = 0;
      read_only_depth_target.stencil_write_base_addr = 0;
      read_only_depth_target.depth_view.depth_write_disable = true;
      read_only_depth_target.depth_view.stencil_write_disable = true;
      registers.SetDepthRenderTarget(read_only_depth_target);
      auto read_only_depth_control = phased_depth_control;
      read_only_depth_control.z_write_enable = false;
      read_only_depth_control.zfunc =
          static_cast<uint8_t>(vk::CompareOp::eAlways);
      read_only_depth_control.stencil_enable = true;
      read_only_depth_control.stencilfunc =
          static_cast<uint8_t>(vk::CompareOp::eAlways);
      registers.SetDepthControl(read_only_depth_control);
      HW::StencilControl read_only_stencil_control{};
      read_only_stencil_control.stencil_zpass =
          static_cast<uint8_t>(Prospero::StencilOp::kReplaceOp);
      registers.SetStencilControl(read_only_stencil_control);
      HW::StencilMask read_only_stencil_mask{};
      read_only_stencil_mask.stencil_testval = 0x12;
      read_only_stencil_mask.stencil_mask = 0xff;
      read_only_stencil_mask.stencil_writemask = 0xff;
      read_only_stencil_mask.stencil_opval = 0x34;
      registers.SetStencilMask(read_only_stencil_mask);
      RenderDepthInfo read_only_depth{};
      RenderExecutorTestAccess::ResolveRenderDepthTarget(
          executor, scheduler.Current(), read_only_depth);
      Require(
          name, "read-only depth/stencil target without write addresses",
          read_only_depth.image_id == phased_depth.image_id &&
              read_only_depth.desc.info.data.address == phased_depth_address &&
              read_only_depth.desc.info.stencil.address == phased_stencil_address &&
              !read_only_depth.depth_write_enable &&
              !read_only_depth.stencil_clear_enable &&
              read_only_depth.stencil_test_enable &&
              read_only_depth.stencil_front.writeMask == 0 &&
              read_only_depth.stencil_back.writeMask == 0 &&
              read_only_depth.stencil_front.passOp ==
                  vk::StencilOp::eKeep &&
              !read_only_depth.AttachmentWriteAspects() &&
              depth_attachment_layout(read_only_depth) ==
                  vk::ImageLayout::eDepthStencilReadOnlyOptimal,
          "a PS5 read-only depth/stencil target required write addresses or "
          "retained host writes");

      auto read_only_render_control = phased_render_control;
      read_only_render_control.stencil_clear_enable = true;
      registers.SetRenderControl(read_only_render_control);
      RenderDepthInfo read_only_clear_depth{};
      RenderExecutorTestAccess::ResolveRenderDepthTarget(
          executor, scheduler.Current(), read_only_clear_depth);
      Require(name, "read-only stencil clear suppression",
              read_only_clear_depth.image_id == phased_depth.image_id &&
                  !read_only_clear_depth.stencil_clear_enable &&
                  !read_only_clear_depth.AttachmentWriteAspects() &&
                  depth_attachment_layout(read_only_clear_depth) ==
                      vk::ImageLayout::eDepthStencilReadOnlyOptimal,
              "a PS5 target-level stencil write disable retained a host clear");

      registers.SetDepthRenderTarget(phased_depth_target);
      registers.SetDepthControl(phased_depth_control);
      registers.SetStencilControl({});
      registers.SetStencilMask({});
      registers.SetRenderControl(phased_render_control);
      RenderColorInfo no_color{};
      auto phased_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
          executor, scheduler.Current(), &no_color, 0, phased_depth);
      Require(
          name, "HTile final acquisition",
          phased_rendering.depth_stencil_attachment.image_view != nullptr &&
              phased_rendering.num_color_attachments == 0 &&
              phased_rendering.depth_stencil_attachment.has_depth &&
              phased_rendering.depth_stencil_attachment.depth_clear &&
              phased_depth.depth_load_clear_enable &&
              texture_cache.IsMeta(phased_depth.desc.info.metadata.range.address) &&
              !texture_cache.IsMetaCleared(
                  phased_depth.desc.info.metadata.range.address,
                  phased_depth.desc.view_info.base_layer) &&
              texture_cache.GetImage(phased_depth.image_id).IsGpuModified(),
          "final depth acquisition did not publish and consume HTile "
          "state at the final acquisition boundary");
      RenderExecutorTestAccess::ResetBindings(executor);

      constexpr uint64_t depth_only_address = base + 0x140000;
      constexpr uint64_t depth_only_htile_address = base + 0x150000;
      constexpr uint32_t captured_depth_only_z_info = 0x22900983u;
      constexpr uint32_t captured_depth_only_stencil_info = 0x00100980u;
      HW::DepthRenderTarget depth_only_target{};
      depth_only_target.z_info =
          HW::DepthZInfo::Decode(captured_depth_only_z_info);
      depth_only_target.stencil_info =
          HW::DepthStencilInfo::Decode(captured_depth_only_stencil_info);
      depth_only_target.z_read_base_addr = depth_only_address;
      depth_only_target.z_write_base_addr = depth_only_address;
      depth_only_target.htile_data_base_addr = depth_only_htile_address;
      depth_only_target.size = {63, 63, true};
      registers.SetDepthRenderTarget(depth_only_target);
      HW::DepthControl depth_only_control{};
      depth_only_control.stencil_enable = true;
      depth_only_control.z_enable = true;
      depth_only_control.z_write_enable = true;
      depth_only_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eAlways);
      registers.SetDepthControl(depth_only_control);
      HW::RenderControl depth_only_render_control{};
      depth_only_render_control.depth_clear_enable = true;
      depth_only_render_control.stencil_clear_enable = true;
      registers.SetRenderControl(depth_only_render_control);
      RenderDepthInfo depth_only{};
      RenderExecutorTestAccess::ResolveRenderDepthTarget(
          executor, scheduler.Current(), depth_only);
      Require(
          name, "depth-only target with stale stencil state",
          depth_only.image_id && depth_only.desc.view_info.format == vk::Format::eD32Sfloat &&
              (depth_only.desc.info.metadata.kind == ImageMetadataKind::Htile) &&
              depth_only.depth_test_enable && !depth_only.depth_write_enable &&
              depth_only.depth_clear_enable &&
              !depth_only.stencil_test_enable &&
              !depth_only.stencil_clear_enable &&
              depth_only.desc.info.stencil.Empty() &&
              depth_only.desc.info.metadata.range.address ==
                  depth_only_htile_address &&
              depth_only.desc.info.data.size != 0 &&
              depth_only_address + depth_only.desc.info.data.size <=
                  base + allocation_size &&
              depth_only.AttachmentWriteAspects() ==
                  vk::ImageAspectFlagBits::eDepth,
          "inactive stencil state changed a depth-only HTile attachment");
      RenderExecutorTestAccess::ResetBindings(executor);

      const auto sampled_width = depth_only.desc.info.extent.width - 1u;
      const auto sampled_height = depth_only.desc.info.extent.height - 1u;
      const auto sampled_address = depth_only_address >> 8u;
      ShaderTextureResource sampled_depth_descriptor{};
      sampled_depth_descriptor.fields[0] =
          static_cast<uint32_t>(sampled_address);
      sampled_depth_descriptor.fields[1] =
          static_cast<uint32_t>(sampled_address >> 32u) |
          (static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u) |
          ((sampled_width & 3u) << 30u);
      sampled_depth_descriptor.fields[2] =
          (sampled_width >> 2u) | (sampled_height << 14u);
      sampled_depth_descriptor.fields[3] =
          DstSel(4, 4, 4, 4) |
          (static_cast<uint32_t>(Prospero::TileMode::kDepth) << 20u) |
          (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
      sampled_depth_descriptor.fields[5] = 0x00700000u;
      ShaderRecompiler::IR::DescriptorValue sampled_depth_value{};
      std::copy(std::begin(sampled_depth_descriptor.fields),
                std::end(sampled_depth_descriptor.fields),
                sampled_depth_value.dwords.begin());
      sampled_depth_value.dword_count = 8;
      ShaderRecompiler::IR::ImageResource sampled_depth_resource{};
      sampled_depth_resource.resource_class =
          ShaderRecompiler::IR::ImageResourceClass::Sampled;
      sampled_depth_resource.numeric_class =
          Prospero::TextureNumericClass::Uint;
      sampled_depth_resource.dimension =
          ShaderRecompiler::Decoder::ImageDimension::Dim2D;
      sampled_depth_resource.read = true;
      const auto sampled_depth_backing =
          texture_cache.GetImage(depth_only.image_id).backing.image;
      auto sampled_depth_binding = RenderExecutorTestAccess::ResolveTexture(
          executor, sampled_depth_resource, sampled_depth_value);
      const auto sampled_depth_view = texture_cache.FindTexture(
          sampled_depth_binding.image_id, sampled_depth_binding.desc);
      const auto &sampled_depth_image =
          texture_cache.GetImage(sampled_depth_binding.image_id);
      const bool normalized_depth_view = std::ranges::any_of(
          sampled_depth_image.views, [&](const auto &cached) {
            return cached.view == sampled_depth_view &&
                   cached.info.format == vk::Format::eD32Sfloat &&
                   cached.info.aspect == vk::ImageAspectFlagBits::eDepth &&
                   cached.info.usage == vk::ImageUsageFlags{};
          });
      Require(
          name, "R32 uint depth-backed view",
          sampled_depth_binding.image_id == depth_only.image_id &&
              sampled_depth_binding.desc.info.guest_format ==
                  Prospero::BufferFormat::k32UInt &&
              sampled_depth_binding.desc.info.pixel_format ==
                  vk::Format::eR32Uint &&
              sampled_depth_binding.desc.view_info.format ==
                  vk::Format::eR32Uint &&
              sampled_depth_binding.desc.view_info.aspect ==
                  vk::ImageAspectFlagBits::eColor &&
              sampled_depth_binding.desc.view_info.usage ==
                  vk::ImageUsageFlagBits::eSampled &&
              sampled_depth_image.info.pixel_format == vk::Format::eD32Sfloat &&
              sampled_depth_image.backing.format == vk::Format::eD32Sfloat &&
              sampled_depth_image.backing.image == sampled_depth_backing &&
              sampled_depth_view != nullptr && normalized_depth_view,
          "uint T# did not retain its requested format while the concrete "
          "view followed the D32 backing");
      RenderExecutorTestAccess::ResetBindings(executor);

      // Attachment acquisition consumes pending HTile clears using DB_DEPTH_CLEAR.
      (void)texture_cache.FindDepthTarget(depth_only.image_id, depth_only.desc);
      auto bounds_target = depth_only_target;
      bounds_target.z_write_base_addr = 0;
      bounds_target.depth_view.depth_write_disable = true;
      registers.SetDepthRenderTarget(bounds_target);
      HW::DepthControl bounds_control{};
      bounds_control.depth_bounds_enable = true;
      registers.SetDepthControl(bounds_control);
      registers.SetRenderControl({});
      registers.SetDepthClearValue(0.375f);
      ShaderRecompiler::IR::ResourceSnapshot bounds_snapshot;
      bounds_snapshot.images.push_back(sampled_depth_value);
      ShaderStageRuntime bounds_vertex{&vertex_sampled_info, &bounds_snapshot};
      ShaderStageRuntime bounds_pixel{&sampled_info, &bounds_snapshot};
      const auto depth_texels = depth_only.desc.info.extent.width *
                                depth_only.desc.info.extent.height;
      const auto depth_bytes = depth_texels * sizeof(uint32_t);
      auto bounds_readback = CreateHostBuffer(name, depth_bytes * 2,
          vk::BufferUsageFlagBits::eTransferDst, {});
      for (uint32_t pass = 0; pass < 2; ++pass) {
        Require(name, "bounds prior depth contents",
                texture_cache.ClearImageFromBuffer(scheduler.Current(),
                    depth_only.desc.info.data.address, depth_only.desc.info.data.size,
                    std::bit_cast<uint32_t>(0.625f)),
                "failed to seed depth before the deferred HTile clear");
        if (pass == 0) {
          Require(name, "bounds deferred HTile clear",
                  texture_cache.ClearMeta(depth_only_htile_address),
                  "depth-only HTile was not registered");
        } else {
          bounds_target.z_write_base_addr = depth_only_address;
          bounds_target.depth_view.depth_write_disable = false;
          bounds_control.z_write_enable = true;
          registers.SetDepthRenderTarget(bounds_target);
          registers.SetDepthControl(bounds_control);
        }
        RenderDepthInfo bounds_depth{};
        RenderExecutorTestAccess::ResolveRenderDepthTarget(
            executor, scheduler.Current(), bounds_depth);
        auto bounds_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings(
            executor, bounds_vertex, bounds_pixel, true);
        std::array<PreparedBindings *, 2> bounds_stages{
            &bounds_bindings.vertex[0], &*bounds_bindings.pixel};
        vk::ImageAspectFlags bounds_feedback;
        const auto bounds_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
            executor, scheduler.Current(), &no_color, 0, bounds_depth,
            bounds_stages, &bounds_feedback);
        descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
            executor, scheduler.Current(), bounds_bindings.vertex[0], *bounds_bindings.pixel));
        const auto &vertex_depth = bounds_bindings.vertex[0].images[0];
        const auto &pixel_depth = bounds_bindings.pixel->images[0];
        const auto expected_layout = pass == 0
            ? vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT
            : vk::ImageLayout::eDepthReadOnlyOptimal;
        Require(name, "deferred clear with sampled depth bounds",
                bounds_depth.image_id == depth_only.image_id &&
                    bounds_depth.depth_bounds_test_enable &&
                    !bounds_depth.depth_write_enable &&
                    bounds_depth.depth_load_clear_enable == (pass == 0) &&
                    !texture_cache.IsMetaCleared(depth_only_htile_address, 0) &&
                    bounds_feedback == (pass == 0
                                            ? vk::ImageAspectFlagBits::eDepth
                                            : vk::ImageAspectFlags{}) &&
                    bounds_rendering.depth_stencil_attachment.image_layout == expected_layout &&
                    bounds_rendering.depth_stencil_attachment.depth_clear == (pass == 0) &&
                    vertex_depth.image_id == depth_only.image_id &&
                    pixel_depth.image_id == depth_only.image_id &&
                    MakeImageInfo(vertex_depth).imageLayout == expected_layout &&
                    MakeImageInfo(pixel_depth).imageLayout == expected_layout,
                "a deferred clear sampled by vertex and pixel stages missed depth feedback");
        scheduler.BeginRendering(bounds_rendering);
        scheduler.EndRendering();
        RenderExecutorTestAccess::ResetBindings(executor);
        const vk::BufferImageCopy copy{pass * depth_bytes, 0, 0,
            {vk::ImageAspectFlagBits::eDepth, 0, 0, 1}, {}, depth_only.desc.info.extent};
        texture_cache.GetImage(depth_only.image_id).Download(
            std::span{&copy, 1}, bounds_readback.buffer, 0, bounds_readback.size);
      }
      vk::MemoryBarrier2 bounds_host_barrier{};
      bounds_host_barrier.sType = vk::StructureType::eMemoryBarrier2;
      bounds_host_barrier.srcStageMask = vk::PipelineStageFlagBits2::eTransfer;
      bounds_host_barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
      bounds_host_barrier.dstStageMask = vk::PipelineStageFlagBits2::eHost;
      bounds_host_barrier.dstAccessMask = vk::AccessFlagBits2::eHostRead;
      vk::DependencyInfo bounds_dependency{};
      bounds_dependency.sType = vk::StructureType::eDependencyInfo;
      bounds_dependency.memoryBarrierCount = 1;
      bounds_dependency.pMemoryBarriers = &bounds_host_barrier;
      scheduler.Current().Handle().pipelineBarrier2(bounds_dependency);
      scheduler.Finish();
      const auto bounds_result = ReadBuffer(name, bounds_readback, depth_texels * 2);
      Require(name, "deferred HTile clear occurs once",
              std::ranges::all_of(std::span{bounds_result}.first(depth_texels),
                  [](uint32_t value) { return value == std::bit_cast<uint32_t>(0.375f); }) &&
              std::ranges::all_of(std::span{bounds_result}.subspan(depth_texels),
                  [](uint32_t value) { return value == std::bit_cast<uint32_t>(0.625f); }),
              "pending clear did not use DB_DEPTH_CLEAR or cleared the next acquisition again");
      DestroyBuffer(&bounds_readback);
      registers.SetDepthClearValue(0.0f);

      // DB override copies run with depth and stencil tests disabled.
      constexpr uint64_t copy_source_depth = base + 0x180000;
      constexpr uint64_t copy_source_stencil = base + 0x1a0000;
      constexpr uint64_t copy_destination_depth = base + 0x1c0000;
      constexpr uint64_t copy_destination_stencil = base + 0x1e0000;
      constexpr uint32_t copy_side = 8;
      constexpr uint32_t copy_layer_texels = copy_side * copy_side;
      constexpr uint32_t copy_depth_bytes =
          copy_layer_texels * 2 * sizeof(uint32_t);
      constexpr uint32_t copy_readback_stride =
          copy_depth_bytes + copy_layer_texels * 2;
      HW::DepthRenderTarget copy_target{};
      copy_target.z_info.format = Prospero::DepthFormat::kZ32F;
      copy_target.z_info.texture_compatibility =
          Prospero::TextureCompatiblePlaneCompression::kEnable;
      copy_target.stencil_info.format = Prospero::StencilFormat::k8UInt;
      copy_target.stencil_info.texture_compatibility =
          Prospero::TextureCompatibleStencil::kEnable;
      copy_target.stencil_info.htile_stencil_disabled = true;
      copy_target.depth_view.slice_max = 1;
      copy_target.size = {copy_side - 1, copy_side - 1, true};
      HW::DepthControl copy_seed_control{};
      copy_seed_control.z_enable = true;
      copy_seed_control.z_write_enable = true;
      copy_seed_control.stencil_enable = true;
      registers.SetDepthControl(copy_seed_control);
      registers.SetRenderControl({});
      const auto make_copy_target = [&](uint64_t depth, uint64_t stencil) {
        auto target = copy_target;
        target.z_read_base_addr = depth;
        target.z_write_base_addr = depth;
        target.stencil_read_base_addr = stencil;
        target.stencil_write_base_addr = stencil;
        registers.SetDepthRenderTarget(target);
        RenderDepthInfo resolved{};
        RenderExecutorTestAccess::ResolveRenderDepthTarget(
            executor, scheduler.Current(), resolved);
        (void)texture_cache.FindDepthTarget(resolved.image_id, resolved.desc);
        RenderExecutorTestAccess::ResetBindings(executor);
        return resolved;
      };
      const auto copy_source =
          make_copy_target(copy_source_depth, copy_source_stencil);
      const auto copy_destination =
          make_copy_target(copy_destination_depth, copy_destination_stencil);
      const auto clear_copy_target = [&](ImageId id, bool source) {
        auto &image = texture_cache.GetImage(id);
        image.Transit(vk::ImageLayout::eTransferDstOptimal,
                      vk::AccessFlagBits2::eTransferWrite, {},
                      scheduler.Current().Handle());
        for (uint32_t layer = 0; layer < 2; ++layer) {
          const vk::ClearDepthStencilValue clear{
              source ? (layer == 0 ? 0.75f : 0.25f)
                     : (layer == 0 ? 0.875f : 0.625f),
              source ? (layer == 0 ? 0xa3u : 0x5au)
                     : (layer == 0 ? 0x31u : 0xc4u)};
          const vk::ImageSubresourceRange range{
              vk::ImageAspectFlagBits::eDepth |
                  vk::ImageAspectFlagBits::eStencil,
              0, 1, layer, 1};
          scheduler.Current().Handle().clearDepthStencilImage(
              image.backing.image, vk::ImageLayout::eTransferDstOptimal, &clear,
              1, &range);
        }
        texture_cache.MarkGpuWritten(id);
      };
      struct CopyCase {
        const char *label;
        HW::DepthRenderOverride override;
        uint32_t color_mode;
        bool depth;
        bool stencil;
      };
      constexpr std::array copy_cases{
          CopyCase{"no override", {}, 0, false, false},
          CopyCase{
              "depth valid only", {true, false, false, false}, 0, false, false},
          CopyCase{
              "depth dirty only", {false, true, false, false}, 0, false, false},
          CopyCase{"stencil valid only",
                   {false, false, true, false},
                   0,
                   false,
                   false},
          CopyCase{"stencil dirty only",
                   {false, false, false, true},
                   0,
                   false,
                   false},
          CopyCase{
              "unpaired aspects", {true, false, false, true}, 0, false, false},
          CopyCase{"color enabled", {true, true, true, true}, 1, false, false},
          CopyCase{"depth only", {true, true, false, false}, 0, true, false},
          CopyCase{"stencil only", {false, false, true, true}, 0, false, true},
          CopyCase{
              "depth and stencil", {true, true, true, true}, 0, true, true}};
      auto copy_readback =
          CreateHostBuffer(name, copy_cases.size() * copy_readback_stride,
                           vk::BufferUsageFlagBits::eTransferDst, {});
      copy_target.z_read_base_addr = copy_source_depth;
      copy_target.z_write_base_addr = copy_destination_depth;
      copy_target.stencil_read_base_addr = copy_source_stencil;
      copy_target.stencil_write_base_addr = copy_destination_stencil;
      copy_target.depth_view.slice_start = 1;
      registers.SetDepthRenderTarget(copy_target);
      registers.SetDepthControl({});
      RenderDepthInfo copy_disabled_depth{};
      RenderExecutorTestAccess::ResolveRenderDepthTarget(
          executor, scheduler.Current(), copy_disabled_depth);
      Require(name, "copy without attachment tests",
              !copy_disabled_depth.image_id,
              "disabled depth/stencil tests acquired a regular attachment");
      for (size_t index = 0; index < copy_cases.size(); ++index) {
        const auto &test = copy_cases[index];
        clear_copy_target(copy_source.image_id, true);
        clear_copy_target(copy_destination.image_id, false);
        registers.SetDepthRenderOverride(test.override);
        auto color_control = registers.GetColorControl();
        color_control.mode = test.color_mode;
        registers.SetColorControl(color_control);
        Require(name, test.label,
                RenderExecutorTestAccess::DepthStencilCopy(
                    executor, scheduler.Current()) ==
                    (test.depth || test.stencil),
                "DB override copy did not honor paired flags and disabled "
                "color output");
        RenderExecutorTestAccess::ResetBindings(executor);
        const auto offset = index * copy_readback_stride;
        const std::array<vk::BufferImageCopy, 2> copies{
            {{offset,
              0,
              0,
              {vk::ImageAspectFlagBits::eDepth, 0, 0, 2},
              {},
              copy_destination.desc.info.extent},
             {offset + copy_depth_bytes,
              0,
              0,
              {vk::ImageAspectFlagBits::eStencil, 0, 0, 2},
              {},
              copy_destination.desc.info.extent}}};
        texture_cache.GetImage(copy_destination.image_id)
            .Download(copies, copy_readback.buffer, 0, copy_readback.size);
      }
      scheduler.Current().Handle().pipelineBarrier2(bounds_dependency);
      scheduler.Finish();
      const auto copy_result = ReadBuffer(
          name, copy_readback,
          copy_cases.size() * copy_readback_stride / sizeof(uint32_t));
      for (size_t index = 0; index < copy_cases.size(); ++index) {
        const auto &test = copy_cases[index];
        const auto *depth = copy_result.data() +
                            index * copy_readback_stride / sizeof(uint32_t);
        const auto *stencil =
            reinterpret_cast<const uint8_t *>(depth) + copy_depth_bytes;
        for (uint32_t layer = 0; layer < 2; ++layer) {
          const auto expected_depth =
              std::bit_cast<uint32_t>(layer == 0   ? 0.875f
                                      : test.depth ? 0.25f
                                                   : 0.625f);
          const auto expected_stencil = layer == 0     ? 0x31u
                                        : test.stencil ? 0x5au
                                                       : 0xc4u;
          for (uint32_t texel = 0; texel < copy_layer_texels; ++texel) {
            const auto position = layer * copy_layer_texels + texel;
            Require(name, test.label,
                    depth[position] == expected_depth &&
                        stencil[position] == expected_stencil,
                    "DB override copy changed an unselected layer/aspect or "
                    "missed copied texels");
          }
        }
      }
      DestroyBuffer(&copy_readback);

      ShaderTextureResource copy_sample_descriptor{};
      copy_sample_descriptor.fields[0] =
          static_cast<uint32_t>(copy_destination_depth >> 8u);
      copy_sample_descriptor.fields[1] =
          static_cast<uint32_t>(copy_destination_depth >> 40u) |
          (static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u) |
          (((copy_side - 1u) & 3u) << 30u);
      copy_sample_descriptor.fields[2] =
          ((copy_side - 1u) >> 2u) | ((copy_side - 1u) << 14u);
      copy_sample_descriptor.fields[3] =
          DstSel(4, 4, 4, 4) |
          (static_cast<uint32_t>(Prospero::TileMode::kDepth) << 20u) |
          (static_cast<uint32_t>(Prospero::ImageType::kColor2DArray) << 28u);
      copy_sample_descriptor.fields[4] = 1;
      copy_sample_descriptor.fields[5] = 0x00700000u;
      TestCase copy_sample_test;
      copy_sample_test.name = "DepthOverrideR32Sampling";
      copy_sample_test.has_user_data = true;
      copy_sample_test.image_descriptor_swizzle =
          copy_sample_descriptor.DstSelXYZW();
      std::copy_n(copy_sample_descriptor.fields, 8,
                  copy_sample_test.user_data.begin());
      copy_sample_test.user_data[50] = 2 * sizeof(uint32_t);
      copy_sample_test.opcodes = {
          ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
          ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
      copy_sample_test.required_spirv = {"OpImageSampleExplicitLod"};
      for (uint32_t layer = 0; layer < 2; ++layer) {
        AppendVMovLiteral(&copy_sample_test.code, 20,
                          std::bit_cast<uint32_t>(0.5f));
        AppendVMovLiteral(&copy_sample_test.code, 21,
                          std::bit_cast<uint32_t>(0.5f));
        AppendVMovLiteral(&copy_sample_test.code, 22,
                          std::bit_cast<uint32_t>(static_cast<float>(layer)));
        copy_sample_test.code.push_back(EncodeMimg0(0x20, 1, 0, false, 5));
        copy_sample_test.code.push_back(EncodeMimg1(0, 20, 0, 2));
        AppendStoreVgpr(&copy_sample_test.code, 0, layer);
        copy_sample_test.expected.push_back(
            std::bit_cast<uint32_t>(layer == 0 ? 0.875f : 0.25f));
      }
      AppendEnd(&copy_sample_test.code);
      const auto copy_sample_program =
          CompileCase(copy_sample_test, SubgroupSize());
      ShaderRecompiler::IR::DescriptorValue copy_sample_value{};
      copy_sample_value.dword_count = 8;
      std::copy_n(copy_sample_descriptor.fields, 8,
                  copy_sample_value.dwords.begin());
      const auto copy_sample_binding = RenderExecutorTestAccess::ResolveTexture(
          executor, copy_sample_program.program.info.images[0],
          copy_sample_value);
      const auto copy_sample_view = texture_cache.FindTexture(
          copy_sample_binding.image_id, copy_sample_binding.desc);
      Require(
          name, "copied depth sampled owner",
          copy_sample_binding.image_id == copy_destination.image_id &&
              copy_sample_binding.desc.info.pixel_format ==
                  vk::Format::eR32Sfloat,
          "R32 sampling did not reuse the GPU-written depth-copy destination");
      auto &copy_sample_image = texture_cache.GetImage(copy_destination.image_id);
      copy_sample_image.Transit(vk::ImageLayout::eShaderReadOnlyOptimal,
                                vk::AccessFlagBits2::eShaderRead, {},
                                scheduler.Current().Handle());
      Image copied_depth_sample;
      copied_depth_sample.view = copy_sample_view;
      copied_depth_sample.layout = copy_sample_image.backing.state.layout;
      scheduler.Finish();
      auto copy_sample_output = CreateStorageBuffer(name, {}, 2);
      const auto copy_sampler = CreateSampler(name);
      Dispatch(copy_sample_test, copy_sample_program, copy_sample_output,
               nullptr, &copied_depth_sample, nullptr, nullptr, copy_sampler);
      Require(
          name, "copied depth R32 samples",
          ReadBuffer(name, copy_sample_output, 2) == copy_sample_test.expected,
          "R32 post-effect sampling saw stale depth after the override copy");
      m_device.destroySampler(copy_sampler);
      DestroyBuffer(&copy_sample_output);
      RenderExecutorTestAccess::ResetBindings(executor);
      registers.SetDepthRenderOverride({});
      registers.SetColorControl({});

      auto shared_depth_descriptor = sampled_depth_descriptor;
      shared_depth_descriptor.fields[0] =
          static_cast<uint32_t>(phased_depth_address >> 8u);
      shared_depth_descriptor.fields[1] =
          static_cast<uint32_t>(phased_depth_address >> 40u) |
          (static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u);
      shared_depth_descriptor.fields[2] = 0;
      ShaderRecompiler::IR::DescriptorValue shared_depth_value{};
      std::copy(std::begin(shared_depth_descriptor.fields),
                std::end(shared_depth_descriptor.fields),
                shared_depth_value.dwords.begin());
      shared_depth_value.dword_count = 8;
      ShaderRecompiler::IR::ResourceSnapshot shared_depth_snapshot;
      shared_depth_snapshot.images.push_back(shared_depth_value);
      ShaderStageRuntime shared_depth_vertex{&vertex_sampled_info,
                                              &shared_depth_snapshot};
      ShaderStageRuntime shared_depth_pixel{&sampled_info,
                                             &shared_depth_snapshot};
      for (const bool stencil_write : {true, false}) {
        registers.SetDepthRenderTarget(phased_depth_target);
        registers.SetRenderControl({});
        registers.SetDepthControl(read_only_depth_control);
        HW::StencilControl shared_stencil_control{};
        shared_stencil_control.stencil_zpass =
            static_cast<uint8_t>(Prospero::StencilOp::kReplaceTest);
        registers.SetStencilControl(shared_stencil_control);
        HW::StencilMask shared_stencil_mask{};
        shared_stencil_mask.stencil_testval = 0x20;
        shared_stencil_mask.stencil_mask = 0xff;
        shared_stencil_mask.stencil_writemask = stencil_write ? 0xf4 : 0;
        registers.SetStencilMask(shared_stencil_mask);
        RenderDepthInfo shared_depth{};
        RenderExecutorTestAccess::ResolveRenderDepthTarget(
            executor, scheduler.Current(), shared_depth);
        auto shared_bindings =
            RenderExecutorTestAccess::PrepareGraphicsBindings(
                executor, shared_depth_vertex, shared_depth_pixel, true);
        auto& shared_image = texture_cache.GetImage(shared_depth.image_id);
        auto stencil_view_info = shared_depth.desc.view_info;
        stencil_view_info.aspect = vk::ImageAspectFlagBits::eStencil;
        stencil_view_info.usage = vk::ImageUsageFlagBits::eSampled;
        const auto stencil_view = shared_image.FindView(stencil_view_info);
        auto& pixel_image = shared_bindings.pixel->images[0];
        pixel_image.image_view = stencil_view;
        pixel_image.desc.view_info = stencil_view_info;
        std::array<PreparedBindings *, 2> shared_stages{
            &shared_bindings.vertex[0], &*shared_bindings.pixel};
        vk::ImageAspectFlags shared_feedback;
        const auto shared_rendering =
            RenderExecutorTestAccess::AcquireRenderTargets(
                executor, scheduler.Current(), &no_color, 0, shared_depth,
                shared_stages, &shared_feedback);
        descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
            executor, scheduler.Current(), shared_bindings.vertex[0],
            *shared_bindings.pixel));
        const auto expected_layout =
            stencil_write
                ? vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT
                : vk::ImageLayout::eDepthStencilReadOnlyOptimal;
        const auto expected_access =
            vk::AccessFlagBits2::eShaderRead |
            vk::AccessFlagBits2::eDepthStencilAttachmentRead |
            vk::AccessFlagBits2::eDepthStencilAttachmentWrite;
        const auto &vertex_image = shared_bindings.vertex[0].images[0];
        Require(name, "sampled depth and stencil attachment layout",
                shared_depth.image_id == phased_depth.image_id &&
                    stencil_view != nullptr &&
                    shared_feedback == (stencil_write
                                            ? vk::ImageAspectFlagBits::eStencil
                                            : vk::ImageAspectFlags{}) &&
                    vertex_image.image_id == shared_depth.image_id &&
                    pixel_image.image_id == shared_depth.image_id &&
                    pixel_image.image_view == stencil_view &&
                    MakeImageInfo(vertex_image).imageLayout == expected_layout &&
                    MakeImageInfo(pixel_image).imageLayout == expected_layout &&
                    shared_rendering.depth_stencil_attachment.image_layout ==
                        expected_layout &&
                    shared_image.backing.state.layout == expected_layout &&
                    shared_image.backing.state.access_mask == expected_access,
                "stencil sampling and writes lost the feedback aspect, shared "
                "layout, or attachment access");
        scheduler.BeginRendering(shared_rendering);
        scheduler.EndRendering();
        RenderExecutorTestAccess::ResetBindings(executor);
        Require(name, "attachment binding reset",
                !shared_image.binding.is_target &&
                    shared_image.binding.attachment_layout ==
                        vk::ImageLayout::eUndefined &&
                    !shared_image.binding.attachment_access,
                "attachment usage leaked into the next draw");
      }

      // PPSA03541 reuses a tile24 depth allocation through a tile27 RGB10A2 T#.
      auto pooled_resource = sampled_depth_resource;
      pooled_resource.numeric_class = Prospero::TextureNumericClass::Float;
      ImageId pooled_color_id{};
      for (const bool color : {false, true, false, true}) {
        auto value = shared_depth_value;
        value.dwords[1] = static_cast<uint32_t>(phased_depth_address >> 40u) |
            (static_cast<uint32_t>(color ? Prospero::BufferFormat::k10_10_10_2UNorm
                                        : Prospero::BufferFormat::k32Float) << 20u);
        value.dwords[3] = color ? 0x91b00facu : 0x91800204u;
        const auto binding = RenderExecutorTestAccess::ResolveTexture(
            executor, pooled_resource, value);
        const auto view = texture_cache.FindTexture(binding.image_id, binding.desc);
        if (color && !pooled_color_id) {
          pooled_color_id = binding.image_id;
        }
        Require(name, "pooled depth/color tile identity",
                view != nullptr &&
                    binding.image_id == (color ? pooled_color_id : phased_depth.image_id) &&
                    (!color || binding.image_id != phased_depth.image_id) &&
                    texture_cache.GetImage(binding.image_id).backing.format ==
                        (color ? vk::Format::eA2B10G10R10UnormPack32
                               : vk::Format::eD32SfloatS8Uint) &&
                    binding.desc.info.tile_mode ==
                        (color ? Prospero::TileMode::kRenderTarget
                               : Prospero::TileMode::kDepth),
                "a changed tile family reused depth or repeated sampling lost its backing");
        RenderExecutorTestAccess::ResetBindings(executor);
      }

      auto video_subresource =
          make_target_desc(base + 0x20000, target_mip_size, {1, 1, 1});
      video_subresource.type = BindingType::VideoOut;
      video_subresource.info.pixel_format = vk::Format::eR8G8B8A8Srgb;
      video_subresource.info.guest_format =
          Prospero::BufferFormat::k8_8_8_8Srgb;
      video_subresource.view_info.format = video_subresource.info.pixel_format;
      video_subresource.view_info.usage = vk::ImageUsageFlagBits::eSampled;
      const auto video_subresource_id =
          texture_cache.FindImage(video_subresource);
      const auto video_subresource_owner =
          TextureCacheTestAccess::Owner(texture_cache, video_subresource_id);
      auto video_parent =
          make_target_desc(base + 0x20000, target_mip_size * 2, {2, 2, 1});
      video_parent.type = BindingType::VideoOut;
      video_parent.info.pixel_format = vk::Format::eR8G8B8A8Srgb;
      video_parent.info.guest_format = Prospero::BufferFormat::k8_8_8_8Srgb;
      video_parent.info.resources.levels = 2;
      video_parent.info.mip_layout[0] = {0, target_mip_size, 2, 2,
                                         0, target_mip_size, 2};
      video_parent.info.mip_layout[1] = {target_mip_size, target_mip_size, 1, 1,
                                         target_mip_size, target_mip_size, 1};
      video_parent.view_info.format = video_parent.info.pixel_format;
      video_parent.view_info.usage = vk::ImageUsageFlagBits::eSampled;
      const auto video_parent_id = texture_cache.FindImage(video_parent);
      const auto video_parent_owner =
          TextureCacheTestAccess::Owner(texture_cache, video_parent_id);
      Require(name, "video-out replacement remains cache-only",
              video_subresource_owner != nullptr &&
                  video_parent_owner != nullptr &&
                  video_parent_id != video_subresource_id &&
                  !video_subresource_owner->binding.is_bound &&
                  !video_subresource_owner->binding.is_target &&
                  !video_subresource_owner->binding.needs_rebind &&
                  !video_subresource_owner->usage.video_out &&
                  !video_parent_owner->binding.is_bound &&
                  !video_parent_owner->binding.is_target &&
                  !video_parent_owner->binding.needs_rebind &&
                  !video_parent_owner->usage.video_out,
              "FindImage leaked caller-owned state across a video-out "
              "replacement");

      constexpr uint64_t array_target_address = base + 0xd0000;
      auto array_target = make_target_desc(array_target_address,
                                           target_mip_size, {128, 128, 1});
      const auto array_target_id = texture_cache.FindImage(array_target);
      const auto array_target_owner =
          TextureCacheTestAccess::Owner(texture_cache, array_target_id);
      RenderExecutorTestAccess::BindRenderTarget(executor, array_target_id);
      const auto array_target_view =
          texture_cache.FindRenderTarget(array_target_id, array_target);

      ShaderTextureResource array_texture{};
      const auto encoded_array_address = array_target_address >> 8u;
      constexpr auto array_format =
          static_cast<uint32_t>(Prospero::BufferFormat::k32UInt);
      array_texture.fields[0] = static_cast<uint32_t>(encoded_array_address);
      array_texture.fields[1] =
          static_cast<uint32_t>(encoded_array_address >> 32u) |
          (array_format << 20u) | (3u << 30u);
      array_texture.fields[2] = (127u >> 2u) | (127u << 14u);
      array_texture.fields[3] =
          DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
          (static_cast<uint32_t>(Prospero::ImageType::kColor2DArray) << 28u);
      array_texture.fields[4] = 1;

      ShaderRecompiler::IR::CompiledShaderInfo array_program{};
      array_program.stage = ShaderType::Compute;
      ShaderRecompiler::IR::ImageResource array_resource{};
      array_resource.resource_class =
          ShaderRecompiler::IR::ImageResourceClass::Sampled;
      array_resource.numeric_class = Prospero::TextureNumericClass::Uint;
      array_resource.dimension =
          ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
      array_resource.read = true;
      array_program.info.images.push_back(array_resource);
      ShaderRecompiler::IR::DescriptorValue array_descriptor{};
      std::copy(std::begin(array_texture.fields),
                std::end(array_texture.fields),
                array_descriptor.dwords.begin());
      array_descriptor.dword_count = 8;
      ShaderRecompiler::IR::ResourceSnapshot array_snapshot{};
      array_snapshot.images.push_back(array_descriptor);
      ShaderStageRuntime array_runtime{&array_program, &array_snapshot};

      PreparedBindings array_binding;
      executor.PrepareBindings(array_runtime, array_binding);
      executor.RebindImages(array_binding);
      const auto expanded_array_id = array_binding.images[0].image_id;
      const auto &expanded_array = texture_cache.GetImage(expanded_array_id);
      Require(
          name, "2D target to array backing expansion",
          array_target_view != nullptr && array_target_owner != nullptr &&
              array_target_owner->binding.needs_rebind &&
              expanded_array_id != array_target_id &&
              !TextureCacheTestAccess::Contains(texture_cache,
                                                array_target_id) &&
              array_binding.images[0].image_view != nullptr &&
              array_binding.images[0].desc.info.type ==
                  Prospero::ImageType::kColor2D &&
              array_binding.images[0].desc.info.resources ==
                  ImageSubresources{1, 2} &&
              array_binding.images[0].desc.view_info.type ==
                  vk::ImageViewType::e2DArray &&
              array_binding.images[0].desc.view_info.layer_count ==
                  2 &&
              expanded_array.backing.layers == 2 &&
              expanded_array.IsGpuModified(),
          "raw array view did not expand and reuse the Color2D backing");

      RenderColorInfo rebound_array_target{};
      rebound_array_target.desc = array_target;
      rebound_array_target.image_id = array_target_id;
      RenderDepthInfo no_array_depth{};
      const auto acquired_array_view = array_binding.images[0].image_view;
      RenderExecutorTestAccess::PrepareGraphicsBindings(
          executor, std::span<PreparedBindings *const>{},
          std::span{&rebound_array_target, 1u});
      auto array_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
          executor, scheduler.Current(), &rebound_array_target, 1,
          no_array_depth);
      Require(name, "expanded array target rebind",
              rebound_array_target.image_id == expanded_array_id &&
                  array_binding.images[0].image_id == expanded_array_id &&
                  array_binding.images[0].image_view == acquired_array_view &&
                  array_rendering.color_attachments[0].image_view != nullptr &&
                  array_rendering.num_color_attachments == 1 &&
                  array_rendering.width == 128 &&
                  array_rendering.height == 128 &&
                  array_rendering.num_layers == 1 &&
                  texture_cache.GetImage(expanded_array_id).binding.is_bound &&
                  texture_cache.GetImage(expanded_array_id).binding.is_target,
              "the retained single-layer target did not rebind to the expanded "
              "backing");
      RenderExecutorTestAccess::ResetBindings(executor);

      Require(name, "raw image synchronization default",
              !Config::SyncRawImageBuffersEnabled(),
              "raw image synchronization should be disabled by default");
      for (const auto [sync_raw, formatted] :
           {std::pair{false, false}, {true, false}, {false, true}, {true, true}}) {
        Config::Load({.sync_raw_image_buffers = sync_raw});
        const bool sync_image = sync_raw || formatted;
        const uint32_t before = 0x13579bd0u | (uint32_t(sync_raw) << 1u) | uint32_t(formatted);
        constexpr uint32_t guard = 0x2468ace0u;
        constexpr uint32_t after = 0xa1b2c3d4u;
        vk::ClearValue clear{};
        clear.color.uint32[0] = before;
        TextureCacheTestAccess::ClearImage(
            texture_cache, scheduler.Current(), expanded_array_id,
            {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 2}, clear);
        clear.color.uint32[0] = guard;
        TextureCacheTestAccess::ClearImage(
            texture_cache, scheduler.Current(), expanded_array_id,
            {vk::ImageAspectFlagBits::eColor, 0, 1, 1, 1}, clear);

        const auto buffer_program = make_buffer_program(
            ShaderType::Vertex, {.read = true, .formatted = formatted});

        constexpr uint64_t buffer_size = 2 * target_mip_size;
        static_assert(buffer_size > BufferCache::CACHING_PAGESIZE);
        ShaderBufferResource descriptor{};
        descriptor.UpdateAddress48(array_target_address);
        descriptor.fields[1] |= 4u << 16u;
        descriptor.fields[2] = buffer_size / sizeof(uint32_t);
        descriptor.fields[3] = 0x00014004u;
        ShaderRecompiler::IR::ResourceSnapshot buffer_snapshot;
        ShaderStageRuntime buffer_runtime{&buffer_program, &buffer_snapshot};
        auto &value = buffer_snapshot.buffers.emplace_back();
        std::memcpy(value.dwords.data(), descriptor.fields, sizeof(descriptor.fields));
        value.dword_count = 4;
        PreparedBindings buffer_bindings;
        executor.PrepareBindings(buffer_runtime, buffer_bindings);
        std::array stages{&buffer_bindings};
        RenderColorInfo target{};
        target.desc = array_target;
        target.image_id = array_target_id;
        RenderExecutorTestAccess::PrepareGraphicsBindings(
            executor, stages, std::span{&target, 1u});
        const auto &buffer = buffer_bindings.buffers[0];
        Require(name, "rediscovered target buffer read",
                target.image_id == expanded_array_id &&
                    buffer.range == buffer_size &&
                    buffer_bindings.images.empty() &&
                    texture_cache.GetImage(target.image_id).backing.state.layout ==
                        (sync_image ? vk::ImageLayout::eTransferSrcOptimal
                                    : vk::ImageLayout::eTransferDstOptimal),
                "image-to-buffer synchronization ignored its setting or formatted reads");
        auto readback = CreateHostBuffer(name, 2 * sizeof(uint32_t),
                                        vk::BufferUsageFlagBits::eTransferDst, {0, 0});
        const std::array copies{
            vk::BufferCopy{buffer.offset, 0, sizeof(uint32_t)},
            vk::BufferCopy{buffer.offset + target_mip_size, sizeof(uint32_t),
                           sizeof(uint32_t)}};
        scheduler.Current().Handle().copyBuffer(buffer.buffer, readback.buffer,
                                                copies.size(), copies.data());
        vk::BufferMemoryBarrier readback_barrier{};
        readback_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
        readback_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
        readback_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        readback_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        readback_barrier.buffer = readback.buffer;
        readback_barrier.size = readback.size;
        scheduler.Current().Handle().pipelineBarrier(
            vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost,
            {}, 0, nullptr, 1, &readback_barrier, 0, nullptr);

        const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
            executor, scheduler.Current(), &target, 1, no_array_depth);
        Require(name, "attachment layout after buffer read",
                rendering.color_attachments[0].image_layout ==
                    vk::ImageLayout::eColorAttachmentOptimal &&
                    texture_cache.GetImage(target.image_id).backing.state.layout ==
                        vk::ImageLayout::eColorAttachmentOptimal,
                "the buffer copy left the render attachment in its transfer layout");
        vk::ClearAttachment clear_attachment{};
        clear_attachment.aspectMask = vk::ImageAspectFlagBits::eColor;
        clear_attachment.colorAttachment = 0;
        clear_attachment.clearValue.color.uint32[0] = after;
        const vk::ClearRect clear_rect{{{0, 0}, {128, 128}}, 0, 1};
        scheduler.BeginRendering(rendering);
        scheduler.Current().Handle().clearAttachments(1, &clear_attachment, 1, &clear_rect);
        scheduler.EndRendering();
        scheduler.Finish();
        Require(name, "buffer alias observes prior GPU contents",
                ReadBuffer(name, readback, 2) ==
                    (sync_image ? std::vector<u32>{before, guard} : std::vector<u32>{0, 0}),
                "buffer contents did not match the selected image synchronization mode");
        DestroyBuffer(&readback);
        Require(name, "rendering after buffer acquisition",
                ReadCachedTexel(name, context, target.image_id) == std::vector<u32>{after} &&
                    ReadCachedTexel(name, context, target.image_id, {}, {1, 1, 1}, 1) ==
                        std::vector<u32>{guard},
                "attachment rendering lost its new color or changed the untouched array layer");
        RenderExecutorTestAccess::ResetBindings(executor);
      }
      Config::Load({});

      auto colliding_msaa_texture = array_texture;
      colliding_msaa_texture.fields[3] =
          DstSel(4, 5, 6, 7) | (1u << 16u) |
          (static_cast<uint32_t>(Prospero::TileMode::kRenderTarget) << 20u) |
          (static_cast<uint32_t>(Prospero::ImageType::kColor2DMsaa) << 28u);
      colliding_msaa_texture.fields[4] = 0;
      colliding_msaa_texture.fields[5] = (7u << 20u) | (1u << 4u);
      ShaderRecompiler::IR::DescriptorValue colliding_msaa_descriptor{};
      std::copy(std::begin(colliding_msaa_texture.fields),
                std::end(colliding_msaa_texture.fields),
                colliding_msaa_descriptor.dwords.begin());
      colliding_msaa_descriptor.dword_count = 8;
      ShaderRecompiler::IR::CompiledShaderInfo colliding_msaa_program{};
      colliding_msaa_program.stage = ShaderType::Compute;
      auto colliding_msaa_resource = array_resource;
      colliding_msaa_resource.dimension =
          ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa;
      colliding_msaa_program.info.images.push_back(colliding_msaa_resource);
      ShaderRecompiler::IR::ResourceSnapshot colliding_msaa_snapshot{};
      colliding_msaa_snapshot.images.push_back(colliding_msaa_descriptor);
      ShaderStageRuntime colliding_msaa_runtime{
          &colliding_msaa_program, &colliding_msaa_snapshot};
      PreparedBindings colliding_msaa_binding;
      executor.PrepareBindings(colliding_msaa_runtime, colliding_msaa_binding);
      executor.RebindImages(colliding_msaa_binding);
      const auto &resolved_colliding_msaa =
          colliding_msaa_binding.images[0];
      Require(
          name, "equal-footprint sample-count identity",
          resolved_colliding_msaa.image_id != expanded_array_id &&
              resolved_colliding_msaa.image_view != nullptr &&
              resolved_colliding_msaa.desc.info.type ==
                  Prospero::ImageType::kColor2D &&
              resolved_colliding_msaa.desc.info.resources ==
                  ImageSubresources{1, 1} &&
              resolved_colliding_msaa.desc.info.samples == 2 &&
              resolved_colliding_msaa.desc.info.data.size == 0x20000 &&
              texture_cache.GetImage(resolved_colliding_msaa.image_id)
                      .backing.samples == 2 &&
              texture_cache.GetImage(expanded_array_id).backing.samples == 1 &&
              texture_cache.GetImage(expanded_array_id).backing.layers == 2,
          "equal-size 1x array and 2x image shared an incompatible backing");
      RenderExecutorTestAccess::ResetBindings(executor);

      constexpr uint64_t msaa_target_address = base + 0x100000;
      constexpr uint32_t msaa_samples = 4;
      auto msaa_target =
          make_target_desc(msaa_target_address, 0x40000, {128, 128, 1});
      msaa_target.info.samples = msaa_samples;
      msaa_target.info.tile_mode = Prospero::TileMode::kRenderTarget;
      msaa_target.info.mip_layout[0].size = msaa_target.info.data.size;
      const auto msaa_target_id = texture_cache.FindImage(msaa_target);
      RenderExecutorTestAccess::BindRenderTarget(executor, msaa_target_id);
      const auto msaa_target_view =
          texture_cache.FindRenderTarget(msaa_target_id, msaa_target);

      auto msaa_texture = array_texture;
      const auto encoded_msaa_address = msaa_target_address >> 8u;
      msaa_texture.fields[0] = static_cast<uint32_t>(encoded_msaa_address);
      msaa_texture.fields[1] =
          static_cast<uint32_t>(encoded_msaa_address >> 32u) |
          (array_format << 20u) | (3u << 30u);
      msaa_texture.fields[3] =
          DstSel(4, 5, 6, 7) | (2u << 16u) |
          (static_cast<uint32_t>(Prospero::TileMode::kRenderTarget) << 20u) |
          (static_cast<uint32_t>(Prospero::ImageType::kColor2DMsaa) << 28u);
      msaa_texture.fields[4] = 0;
      msaa_texture.fields[5] = (7u << 20u) | (2u << 4u);
      ShaderRecompiler::IR::DescriptorValue msaa_descriptor{};
      std::copy(std::begin(msaa_texture.fields), std::end(msaa_texture.fields),
                msaa_descriptor.dwords.begin());
      msaa_descriptor.dword_count = 8;
      ShaderRecompiler::IR::CompiledShaderInfo msaa_program{};
      msaa_program.stage = ShaderType::Compute;
      auto msaa_resource = array_resource;
      msaa_resource.dimension =
          ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa;
      msaa_program.info.images.push_back(msaa_resource);
      ShaderRecompiler::IR::ResourceSnapshot msaa_snapshot{};
      msaa_snapshot.images.push_back(msaa_descriptor);
      ShaderStageRuntime msaa_runtime{&msaa_program, &msaa_snapshot};
      PreparedBindings msaa_binding;
      executor.PrepareBindings(msaa_runtime, msaa_binding);
      executor.RebindImages(msaa_binding);
      const auto &resolved_msaa = msaa_binding.images[0];
      Require(
          name, "MSAA descriptor backing reuse",
          msaa_target_view != nullptr &&
              resolved_msaa.image_id == msaa_target_id &&
              resolved_msaa.image_view != nullptr &&
              resolved_msaa.desc.info.type == Prospero::ImageType::kColor2D &&
              resolved_msaa.desc.info.resources == ImageSubresources{1, 1} &&
              resolved_msaa.desc.info.samples == msaa_samples &&
              resolved_msaa.desc.info.data.size == 0x40000 &&
              resolved_msaa.desc.view_info.level_count == 1 &&
              texture_cache.GetImage(msaa_target_id).backing.samples ==
                  msaa_samples,
          "MSAA view fields did not resolve to the matching multisample "
          "backing");
      RenderExecutorTestAccess::ResetBindings(executor);

      auto msaa_array_texture = msaa_texture;
      msaa_array_texture.fields[3] =
          (msaa_array_texture.fields[3] & ~(0xfu << 28u)) |
          (static_cast<uint32_t>(Prospero::ImageType::kColor2DMsaaArray)
           << 28u);
      msaa_array_texture.fields[4] = 1;
      ShaderRecompiler::IR::DescriptorValue msaa_array_descriptor{};
      std::copy(std::begin(msaa_array_texture.fields),
                std::end(msaa_array_texture.fields),
                msaa_array_descriptor.dwords.begin());
      msaa_array_descriptor.dword_count = 8;
      ShaderRecompiler::IR::CompiledShaderInfo msaa_array_program{};
      msaa_array_program.stage = ShaderType::Compute;
      auto msaa_array_resource = array_resource;
      msaa_array_resource.dimension =
          ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray;
      msaa_array_program.info.images.push_back(msaa_array_resource);
      ShaderRecompiler::IR::ResourceSnapshot msaa_array_snapshot{};
      msaa_array_snapshot.images.push_back(msaa_array_descriptor);
      ShaderStageRuntime msaa_array_runtime{&msaa_array_program, &msaa_array_snapshot};
      PreparedBindings msaa_array_binding;
      executor.PrepareBindings(msaa_array_runtime, msaa_array_binding);
      executor.RebindImages(msaa_array_binding);
      const auto &resolved_msaa_array = msaa_array_binding.images[0];
      Require(name, "MSAA array backing expansion",
              resolved_msaa_array.image_id != msaa_target_id &&
                  resolved_msaa_array.image_view != nullptr &&
                  resolved_msaa_array.desc.info.type ==
                      Prospero::ImageType::kColor2D &&
                  resolved_msaa_array.desc.info.resources ==
                      ImageSubresources{1, 2} &&
                  resolved_msaa_array.desc.info.samples == msaa_samples &&
                  resolved_msaa_array.desc.info.data.size == 0x80000 &&
                  resolved_msaa_array.desc.view_info.type ==
                      vk::ImageViewType::e2DArray &&
                  resolved_msaa_array.desc.view_info.layer_count == 2 &&
                  texture_cache.GetImage(resolved_msaa_array.image_id)
                          .backing.layers == 2 &&
                  texture_cache.GetImage(resolved_msaa_array.image_id)
                          .backing.samples == msaa_samples,
              "MSAA array view did not expand the matching Color2D backing");
      RenderExecutorTestAccess::ResetBindings(executor);

      ImageDesc depth{};
      depth.type = BindingType::DepthTarget;
      depth.info.data = {depth_address, 0x10000};
      depth.info.stencil = {stencil_address, stencil_layout.size};
      depth.info.pixel_format = vk::Format::eD32SfloatS8Uint;
      depth.info.guest_format = Prospero::BufferFormat::k32Float;
      depth.info.type = Prospero::ImageType::kColor2D;
      depth.info.extent = {1, 1, 1};
      depth.info.resources = {1, 1};
      depth.info.pitch = 1;
      depth.info.bytes_per_block = 4;
      depth.info.samples = 1;
      depth.info.tile_mode = linear;
      depth.info.UpdateSize();
      depth.view_info.format = depth.info.pixel_format;
      depth.view_info.type = vk::ImageViewType::e2D;
      depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
      depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
      const auto depth_id = texture_cache.FindImage(depth);
      Require(name, "cache-only depth discovery",
              !texture_cache.FindImageFromRange(stencil_address,
                                                stencil_layout.size, false) &&
                  !texture_cache.GetImage(depth_id).binding.is_bound &&
                  !texture_cache.GetImage(depth_id).binding.is_target,
              "FindImage created depth acquisition state or a stencil proxy");
      RenderExecutorTestAccess::BindRenderTarget(executor, depth_id);
      Require(name, "depth prefetch purity",
              texture_cache.GetImage(depth_id).binding.is_target &&
                  !texture_cache.FindImageFromRange(stencil_address,
                                                    stencil_layout.size, false),
              "depth prefetch performed final target acquisition");

      auto target_parent = make_target_desc(base, 1536, {4, 4, 1});
      target_parent.type = BindingType::RenderTarget;
      target_parent.info.resources.levels = 2;
      // PS5 linear rows align to 256 bytes; smaller mips precede larger ones.
      target_parent.info.pitch = 64;
      target_parent.info.UpdateSize();
      target_parent.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
      auto target_base_subresource =
          make_target_desc(base + 512, 1024, {4, 4, 1});
      target_base_subresource.info.pitch = 64;
      target_base_subresource.info.UpdateSize();
      const auto target_base_subresource_id =
          texture_cache.FindImage(target_base_subresource);
      auto target_subresource = make_target_desc(base, 512, {2, 2, 1});
      target_subresource.info.pitch = 64;
      target_subresource.info.UpdateSize();
      const auto target_subresource_id =
          texture_cache.FindImage(target_subresource);
      const auto *target_subresource_owner =
          TextureCacheTestAccess::Owner(texture_cache, target_subresource_id);
      const auto &discovered_target =
          texture_cache.GetImage(target_subresource_id);
      Require(name, "cache-only target discovery",
              !discovered_target.binding.is_bound &&
                  !discovered_target.binding.is_target &&
                  !discovered_target.binding.needs_rebind &&
                  !discovered_target.usage.render_target &&
                  !TextureCacheTestAccess::PendingDownload(
                      texture_cache, target_subresource_id),
              "FindImage claimed RenderExecutor-owned render-target state");
      vk::ClearValue target_clear{};
      target_clear.color.uint32[0] = 0x12345678;
      TextureCacheTestAccess::ClearImage(
          texture_cache, scheduler.Current(), target_base_subresource_id,
          {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, target_clear);
      target_clear.color.uint32[0] = 0x89abcdef;
      TextureCacheTestAccess::ClearImage(
          texture_cache, scheduler.Current(), target_subresource_id,
          {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, target_clear);
      RenderExecutorTestAccess::BindRenderTarget(executor,
                                                 target_subresource_id);
      Require(name, "target prefetch purity",
              texture_cache.GetImage(target_subresource_id).binding.is_target &&
                  !texture_cache.GetImage(target_subresource_id)
                       .usage.render_target &&
                  !TextureCacheTestAccess::PendingDownload(
                      texture_cache, target_subresource_id),
              "target prefetch performed final target acquisition");
      const auto target_parent_id = texture_cache.FindImage(target_parent);
      Require(name, "active target overlap",
              target_parent_id &&
                  target_parent_id != target_base_subresource_id &&
                  target_parent_id != target_subresource_id &&
                  target_subresource_owner != nullptr &&
                  target_subresource_owner->binding.needs_rebind &&
                  texture_cache.GetImage(target_parent_id).binding.is_target &&
                  !TextureCacheTestAccess::PendingDownload(texture_cache,
                                                           target_parent_id),
              "target overlap did not transfer target state to the merged "
              "owner");
      auto target_readback = CreateHostBuffer(
          name, 2 * sizeof(u32), vk::BufferUsageFlagBits::eTransferDst,
          {0xaaaaaaaa, 0xaaaaaaaa});
      const std::array target_probes{
          vk::BufferImageCopy{0,
                              0,
                              0,
                              {vk::ImageAspectFlagBits::eColor, 0, 0, 1},
                              {},
                              {1, 1, 1}},
          vk::BufferImageCopy{sizeof(u32),
                              0,
                              0,
                              {vk::ImageAspectFlagBits::eColor, 1, 0, 1},
                              {},
                              {1, 1, 1}}};
      texture_cache.GetImage(target_parent_id)
          .Download(target_probes, target_readback.buffer, 0,
                    target_readback.size);
      const vk::MemoryBarrier2 target_host_barrier{
          .srcStageMask = vk::PipelineStageFlagBits2::eCopy,
          .srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
          .dstStageMask = vk::PipelineStageFlagBits2::eHost,
          .dstAccessMask = vk::AccessFlagBits2::eHostRead};
      vk::DependencyInfo target_dependency{};
      target_dependency.memoryBarrierCount = 1;
      target_dependency.pMemoryBarriers = &target_host_barrier;
      scheduler.Current().Handle().pipelineBarrier2(target_dependency);
      RenderColorInfo rebound_color{};
      rebound_color.desc = target_subresource;
      rebound_color.image_id = target_subresource_id;
      RenderDepthInfo rebound_depth{};
      rebound_depth.desc = depth;
      rebound_depth.image_id = depth_id;
      scheduler.Finish();
      Require(name, "promoted target mip contents",
              ReadBuffer(name, target_readback, 2) ==
                  std::vector<u32>{0x12345678, 0x89abcdef},
              "promotion discarded GPU contents from a child mip");
      DestroyBuffer(&target_readback);
      Require(name, "deferred target slot erasure",
              TextureCacheTestAccess::Owner(texture_cache,
                                            target_subresource_id) == nullptr,
              "the displaced target slot was not erased after GPU completion");
      RenderExecutorTestAccess::PrepareGraphicsBindings(
          executor, std::span<PreparedBindings *const>{}, std::span{&rebound_color, 1u});
      auto rebound_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
          executor, scheduler.Current(), &rebound_color, 1, rebound_depth);
      Require(name, "guest target extent after host mip remap",
              rebound_color.desc.view_info.base_level == 1 &&
                  rebound_color.guest_mip_level == 0 &&
                  rebound_color.Extent() == vk::Extent2D{2, 2},
              "remapping a guest target into its parent mip shrank the guest "
              "render area twice");
      const auto stencil_proxy_id = texture_cache.FindImageFromRange(
          stencil_address, stencil_layout.size, false);
      Require(
          name, "production target rebind",
          rebound_color.image_id == target_parent_id &&
              rebound_rendering.color_attachments[0].image_view != nullptr &&
              rebound_rendering.depth_stencil_attachment.image_view != nullptr &&
              rebound_rendering.num_color_attachments == 1 &&
              rebound_rendering.width == 1 && rebound_rendering.height == 1 &&
              texture_cache.GetImage(target_parent_id).binding.is_target &&
              texture_cache.GetImage(target_parent_id).usage.render_target &&
              texture_cache.GetImage(target_parent_id).IsGpuModified() &&
              texture_cache.GetImage(depth_id).IsGpuModified() &&
              TextureCacheTestAccess::PendingDownload(texture_cache,
                                                      target_parent_id) &&
              stencil_proxy_id &&
              texture_cache.GetImage(stencil_proxy_id).depth_id == depth_id,
          "the final target pass did not acquire color before depth and "
          "build the dynamic render state");
      scheduler.BeginRendering(rebound_rendering);
      scheduler.EndRendering();
      RenderExecutorTestAccess::ResetBindings(executor);
      Require(name, "draw-scoped target reset",
              TextureCacheTestAccess::Owner(texture_cache,
                                            target_subresource_id) == nullptr &&
                  !texture_cache.GetImage(target_parent_id).binding.is_target &&
                  !texture_cache.GetImage(depth_id).binding.is_target,
              "RenderExecutor retained target binding state after reset");

      constexpr uint64_t ordered_color_address = base + 0x80000;
      auto ordered_color_desc =
          make_target_desc(ordered_color_address, target_mip_size, {1, 1, 1});
      const auto stale_ordered_color =
          texture_cache.FindImage(ordered_color_desc);
      RenderExecutorTestAccess::BindRenderTarget(executor, stale_ordered_color);
      resources.UnmapMemory(ordered_color_address, target_mip_size);
      resources.MapMemory(ordered_color_address, target_mip_size);

      auto ordered_depth_desc = depth;
      ordered_depth_desc.info.stencil = {ordered_color_address,
                                         target_mip_size};
      const auto ordered_depth_id = texture_cache.FindImage(ordered_depth_desc);
      RenderExecutorTestAccess::BindRenderTarget(executor, ordered_depth_id);
      RenderColorInfo ordered_color{};
      ordered_color.desc = ordered_color_desc;
      ordered_color.image_id = stale_ordered_color;
      RenderDepthInfo ordered_depth{};
      ordered_depth.desc = ordered_depth_desc;
      ordered_depth.image_id = ordered_depth_id;
      RenderExecutorTestAccess::PrepareGraphicsBindings(
          executor, std::span<PreparedBindings *const>{}, std::span{&ordered_color, 1u});
      auto ordered_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
          executor, scheduler.Current(), &ordered_color, 1, ordered_depth);
      Require(name, "color-before-depth acquisition order",
              ordered_color.image_id != stale_ordered_color &&
                  ordered_rendering.color_attachments[0].image_view != nullptr &&
                  ordered_rendering.depth_stencil_attachment.image_view != nullptr &&
                  texture_cache.GetImage(ordered_color.image_id).depth_id ==
                      ordered_depth.image_id,
              "depth acquisition ran before the stale color target was "
              "recreated and associated");
      RenderExecutorTestAccess::ResetBindings(executor);
      TextureCacheTestAccess::SetLinearReadback(texture_cache, false);

      ShaderTextureResource stencil{};
      const uint64_t encoded_address = stencil_address >> 8u;
      stencil.fields[0] = static_cast<uint32_t>(encoded_address);
      stencil.fields[1] = static_cast<uint32_t>(encoded_address >> 32u) |
                          (static_cast<uint32_t>(stencil_format) << 20u);
      stencil.fields[3] =
          DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
          (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);

      ShaderRecompiler::IR::CompiledShaderInfo program{};
      program.stage = ShaderType::Compute;
      ShaderRecompiler::IR::ImageResource stencil_resource{};
      stencil_resource.resource_class =
          ShaderRecompiler::IR::ImageResourceClass::Sampled;
      stencil_resource.numeric_class = Prospero::TextureNumericClass::Uint;
      stencil_resource.dimension =
          ShaderRecompiler::Decoder::ImageDimension::Dim2D;
      stencil_resource.read = true;
      program.info.images.push_back(stencil_resource);

      ShaderRecompiler::IR::ResourceSnapshot snapshot{};
      ShaderRecompiler::IR::DescriptorValue descriptor{};
      std::copy(std::begin(stencil.fields), std::end(stencil.fields),
                descriptor.dwords.begin());
      descriptor.dword_count = 8;
      snapshot.images.push_back(descriptor);
      ShaderStageRuntime runtime{&program, &snapshot};

      PreparedBindings prepared;
      context.GetRenderExecutor().PrepareBindings(runtime, prepared);
      const auto sampled_stencil_id = prepared.images[0].image_id;
      Require(name, "first stencil discovery",
              prepared.images.size() == 1 &&
                  sampled_stencil_id == depth_id &&
                  prepared.images[0].image_view == nullptr &&
                  texture_cache.GetImage(sampled_stencil_id).binding.is_bound &&
                  !texture_cache.GetImage(stencil_proxy_id).binding.is_bound &&
                  !texture_cache.GetImage(stencil_proxy_id).binding.is_target,
              "the first sampled-stencil lookup did not use the canonical "
              "depth owner without acquiring the stencil proxy");

      context.GetRenderExecutor().RebindImages(prepared);
      Require(name, "first stencil acquisition",
              prepared.images[0].image_id == sampled_stencil_id &&
                  prepared.images[0].image_view != nullptr,
              "the first sampled-stencil image was not finally acquired");

      RenderExecutorTestAccess::ResetBindings(executor);
      RenderDepthInfo reassociated_depth = rebound_depth;
      RenderColorInfo no_reassociated_color{};
      (void)RenderExecutorTestAccess::AcquireRenderTargets(
          executor, scheduler.Current(), &no_reassociated_color, 0,
          reassociated_depth);
      Require(name, "existing stencil association selection",
              texture_cache.GetImage(stencil_proxy_id).depth_id ==
                  reassociated_depth.image_id,
              "final depth acquisition did not associate the existing image "
              "at the stencil guest address");
      RenderExecutorTestAccess::ResetBindings(executor);

      PreparedBindings redirected;
      context.GetRenderExecutor().PrepareBindings(runtime, redirected);
      Require(name, "redirected owner discovery",
              redirected.images.size() == 1 &&
                  redirected.images[0].image_id == depth_id &&
                  redirected.images[0].image_view == nullptr &&
                  texture_cache.GetImage(depth_id).binding.is_bound &&
                  !texture_cache.GetImage(stencil_proxy_id).binding.is_bound,
              "the established stencil association did not redirect the next "
              "discovery to the depth owner");
      context.GetRenderExecutor().RebindImages(redirected);
      Require(name, "second-pass stencil acquisition",
              redirected.images[0].image_id == depth_id &&
                  redirected.images[0].image_view != nullptr,
              "RebindImages did not acquire the associated depth owner");
      RenderExecutorTestAccess::ResetBindings(executor);

      // Execute the captured IMAGE_STORE clear through production fill recognition.
      const std::array<uint32_t, 12> stencil_shader{
          0xd7460000u, 0x0401060cu, 0xf4201a84u, 0xfa000000u,
          0xbf8cc07fu, 0x7e06026au, 0xd7460001u, 0x0405060du,
          0x7e04020eu, 0xf0200128u, 0x00000300u, 0xbf810000u};
      uint32_t stencil_byte = 0x35;
      const auto value_address = reinterpret_cast<uint64_t>(&stencil_byte);
      auto storage_stencil = stencil;
      storage_stencil.fields[3] = (storage_stencil.fields[3] & 0x0fffffffu) |
          (static_cast<uint32_t>(Prospero::ImageType::kColor2DArray) << 28u);
      storage_stencil.fields[5] = 0x00700000u;
      std::array<uint32_t, 12> stencil_userdata{};
      std::copy_n(storage_stencil.fields, 8, stencil_userdata.begin());
      stencil_userdata[8] = static_cast<uint32_t>(value_address);
      stencil_userdata[9] = static_cast<uint32_t>(value_address >> 32) | (4u << 16);
      stencil_userdata[10] = 1;
      stencil_userdata[11] = 0x14204u;
      ShaderComputeInputInfo stencil_compute{};
      stencil_compute.threads_num[0] = stencil_compute.threads_num[1] = 8;
      stencil_compute.threads_num[2] = 1;
      stencil_compute.workgroup_register = 12;
      stencil_compute.thread_ids_num = 2;
      std::fill_n(stencil_compute.group_id, 3, true);
      ShaderRecompiler::CompileOptions stencil_options;
      stencil_options.stage = ShaderType::Compute;
      stencil_options.user_data = stencil_userdata;
      stencil_options.input_info.compute = &stencil_compute;
      auto stencil_translated = ShaderRecompiler::TranslateProgram(stencil_shader, stencil_options);
      auto stencil_plan = ShaderRecompiler::IR::ExtractResourcePlan(stencil_translated.program);
      ShaderRecompiler::IR::ResourceSpecialization stencil_specialization;
      ShaderRecompiler::IR::ResourceSnapshot stencil_snapshot;
      stencil_compute.stage.resources = &stencil_snapshot;
      Require(name, "stencil fill materialization",
          ShaderRecompiler::IR::MaterializeResources(stencil_plan,
              {.user_data = stencil_userdata, .userdata = &stencil_byte,
               .read_specialization_memory = +[](void *data, uint64_t address, std::span<uint32_t> words) {
                 if (address != reinterpret_cast<uint64_t>(data) || words.size() != 1u) return false;
                 words[0] = *static_cast<uint32_t *>(data);
                 return true;
               }}, stencil_snapshot, stencil_specialization),
          "captured stencil shader could not resolve its clear byte");
      ShaderRecompiler::IR::ApplyResourceSpecialization(stencil_translated.program,
                                                       stencil_specialization);
      ShaderRecompiler::IR::CompiledShaderInfo stencil_program{};
      stencil_program.stage = ShaderType::Compute;
      stencil_program.info = std::move(stencil_translated.program.info);
      stencil_compute.stage.program = &stencil_program;
      vk::ClearValue stencil_initial{};
      stencil_initial.depthStencil = vk::ClearDepthStencilValue{0.625f, 0x91};
      TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), depth_id,
          {vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1},
          stencil_initial);
      Require(name, "stencil fill coverage",
          !RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
              scheduler.Current(), 0, 1, 1, 0x41u) &&
          !RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
              scheduler.Current(), 2, 1, 1, 0x41u),
          "incomplete or excessive stencil dispatch was consumed");
      auto &stencil_source = stencil_snapshot.buffers[0];
      const auto clean_source = stencil_source;
      stencil_source.dwords[0] = static_cast<uint32_t>(stencil_address);
      stencil_source.dwords[1] = static_cast<uint32_t>(stencil_address >> 32) | (4u << 16);
      Require(name, "stencil fill source alias",
          !RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
              scheduler.Current(), 1, 1, 1, 0x41u),
          "an aliased scalar load was consumed as a uniform image clear");
      stencil_source = clean_source;
      Require(name, "stencil IMAGE_STORE clear",
          RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
              scheduler.Current(), 1, 1, 1, 0x41u),
          "captured clear attempted an unsupported depth/stencil storage binding");
      auto stencil_readback = CreateHostBuffer(name, 8,
          vk::BufferUsageFlagBits::eTransferDst, {});
      const std::array<vk::BufferImageCopy, 2> stencil_copies{{
          {0, 0, 0, {vk::ImageAspectFlagBits::eDepth, 0, 0, 1}, {}, {1, 1, 1}},
          {4, 0, 0, {vk::ImageAspectFlagBits::eStencil, 0, 0, 1}, {}, {1, 1, 1}}}};
      texture_cache.GetImage(depth_id).Download(stencil_copies, stencil_readback.buffer, 0, 8);
      vk::MemoryBarrier2 stencil_host_barrier{};
      stencil_host_barrier.sType = vk::StructureType::eMemoryBarrier2;
      stencil_host_barrier.srcStageMask = vk::PipelineStageFlagBits2::eTransfer;
      stencil_host_barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
      stencil_host_barrier.dstStageMask = vk::PipelineStageFlagBits2::eHost;
      stencil_host_barrier.dstAccessMask = vk::AccessFlagBits2::eHostRead;
      vk::DependencyInfo stencil_dependency{};
      stencil_dependency.sType = vk::StructureType::eDependencyInfo;
      stencil_dependency.memoryBarrierCount = 1;
      stencil_dependency.pMemoryBarriers = &stencil_host_barrier;
      scheduler.Current().Handle().pipelineBarrier2(stencil_dependency);
      scheduler.Finish();
      const auto stencil_result = ReadBuffer(name, stencil_readback, 2);
      Require(name, "stencil clear retains depth",
          stencil_result[0] == std::bit_cast<uint32_t>(0.625f) &&
              (stencil_result[1] & 0xffu) == 0x35 &&
              !texture_cache.GetImage(depth_id).usage.storage,
          "stencil clear altered depth, lost its scalar value, or used a storage view");
      DestroyBuffer(&stencil_readback);

      // Buffer writes to the independent tiled stencil plane must reach the
      // combined native attachment, including values that are not a clear.
      {
        LibKernel::Memory::InstallGpuResources(&resources);
        constexpr uint64_t plane_size = 0x10000;
        constexpr uint64_t plane_address = base + 0x910000;
        constexpr uint32_t texels = 6;
        auto plane_depth = depth;
        plane_depth.info.data = {base + 0x900000, plane_size};
        plane_depth.info.stencil = {plane_address, plane_size};
        plane_depth.info.extent = {3, 2, 1};
        plane_depth.info.pitch = TileGetDepthPitch(3, sizeof(uint32_t));
        plane_depth.info.tile_mode = Prospero::TileMode::kDepth;
        plane_depth.info.UpdateSize();
        const auto plane_depth_id = texture_cache.FindImage(plane_depth);
        auto sampled_plane = plane_depth;
        sampled_plane.type = BindingType::Texture;
        sampled_plane.info.data = plane_depth.info.stencil;
        sampled_plane.info.stencil = {};
        sampled_plane.info.metadata = {};
        sampled_plane.info.pixel_format = vk::Format::eR8Uint;
        sampled_plane.info.guest_format = Prospero::BufferFormat::k8UInt;
        sampled_plane.info.bytes_per_block = 1;
        sampled_plane.info.pitch = TileGetDepthPitch(3, 1);
        sampled_plane.info.UpdateSize();
        sampled_plane.view_info.format = vk::Format::eR8Uint;
        sampled_plane.view_info.aspect = vk::ImageAspectFlagBits::eColor;
        sampled_plane.view_info.usage = vk::ImageUsageFlagBits::eSampled;
        const auto sampled_plane_id = texture_cache.FindImage(sampled_plane);
        (void)texture_cache.FindTexture(sampled_plane_id, sampled_plane);
        (void)texture_cache.FindDepthTarget(plane_depth_id, plane_depth);
        Require(name, "sampled stencil association",
                texture_cache.GetImage(sampled_plane_id).depth_id == plane_depth_id,
                "depth acquisition did not associate the sampled stencil plane");
        TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), plane_depth_id,
            {vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1},
            stencil_initial);

        TileBlockLayout stencil_block{};
        Require(name, "independent stencil tile pitch",
            TileGetBlockLayout(TileBlockFamily::Depth64KB, 1, stencil_block) &&
                plane_depth.info.pitch == 128 && TileGetDepthPitch(3, 1) == 256,
            "stencil must use its own one-byte tile layout");
        std::vector<uint32_t> plane_words(plane_size / sizeof(uint32_t), 0);
        auto *plane_bytes = reinterpret_cast<uint8_t *>(plane_words.data());
        std::array<uint32_t, texels> plane_offsets{};
        for (uint32_t i = 0; i < texels; ++i) {
          Require(name, "stencil tile offset",
              TileGetBlockOffset(stencil_block, i % 3, i / 3, 0, plane_offsets[i]),
              "stencil texel offset is unavailable");
          plane_bytes[plane_offsets[i]] = static_cast<uint8_t>(0x21 + i * 7);
        }
        auto plane_upload = CreateHostBuffer(name, plane_size,
            vk::BufferUsageFlagBits::eTransferSrc, plane_words);
        const auto plane_program = make_buffer_program(
            ShaderType::Compute, {.written = true, .formatted = true});
        ShaderBufferResource plane_descriptor{};
        plane_descriptor.UpdateAddress48(plane_address);
        plane_descriptor.fields[1] |= 4u << 16u;
        plane_descriptor.fields[2] = plane_size / sizeof(uint32_t);
        plane_descriptor.fields[3] = 0x00014204u;
        ShaderRecompiler::IR::ResourceSnapshot plane_snapshot;
        auto &plane_value = plane_snapshot.buffers.emplace_back();
        std::memcpy(plane_value.dwords.data(), plane_descriptor.fields,
                    sizeof(plane_descriptor.fields));
        plane_value.dword_count = 4;
        const ShaderStageRuntime plane_runtime{&plane_program, &plane_snapshot};
        PreparedBindings plane_bindings;
        executor.PrepareBindings(plane_runtime, plane_bindings);
        vk::MemoryBarrier2 plane_barrier{};
        plane_barrier.srcStageMask = plane_barrier.dstStageMask =
            vk::PipelineStageFlagBits2::eAllCommands;
        plane_barrier.srcAccessMask = plane_barrier.dstAccessMask =
            vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite;
        vk::DependencyInfo plane_dependency{};
        plane_dependency.memoryBarrierCount = 1;
        plane_dependency.pMemoryBarriers = &plane_barrier;
        const auto write_plane = [&] {
          std::array<PreparedBindings *, 1> stages{&plane_bindings};
          executor.FindBuffers(stages);
          executor.RebindBuffers(plane_bindings);
          const auto &plane_buffer = plane_bindings.buffers[0];
          const vk::BufferCopy plane_copy{0, plane_buffer.offset, plane_size};
          scheduler.Current().Handle().pipelineBarrier2(plane_dependency);
          scheduler.Current().Handle().copyBuffer(plane_upload.buffer, plane_buffer.buffer,
                                                  1, &plane_copy);
          scheduler.Current().Handle().pipelineBarrier2(plane_dependency);
          RenderExecutorTestAccess::ResetBindings(executor);
        };
        write_plane();

        auto plane_readback = CreateHostBuffer(name, texels * 4 + 8,
            vk::BufferUsageFlagBits::eTransferDst, {});
        const std::array<vk::BufferImageCopy, 2> plane_copies{{
            {0, 0, 0, {vk::ImageAspectFlagBits::eDepth, 0, 0, 1}, {}, {3, 2, 1}},
            {texels * 4, 0, 0, {vk::ImageAspectFlagBits::eStencil, 0, 0, 1}, {}, {3, 2, 1}}}};
        const auto check_plane = [&](const char *phase, bool native_clear) {
          (void)texture_cache.FindDepthTarget(plane_depth_id, plane_depth);
          texture_cache.GetImage(plane_depth_id).Download(
              plane_copies, plane_readback.buffer, 0, plane_readback.size);
          scheduler.Current().Handle().pipelineBarrier2(stencil_dependency);
          scheduler.Finish();
          const auto values = ReadBuffer(name, plane_readback, texels + 2);
          const auto *bytes = reinterpret_cast<const uint8_t *>(values.data() + texels);
          for (uint32_t i = 0; i < texels; ++i) {
            Require(name, phase,
                values[i] == std::bit_cast<uint32_t>(0.625f) &&
                    bytes[i] == (native_clear ? 0x67 : plane_bytes[plane_offsets[i]]),
                "stencil acquisition lost bytes or overwrote the independent depth plane");
          }
        };
        check_plane("GPU stencil buffer write", false);
        write_plane();
        vk::ClearValue stencil_again{};
        stencil_again.depthStencil.stencil = 0x67;
        TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), plane_depth_id,
            {vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1}, stencil_again);
        check_plane("consumed stencil dirtiness", true);
        check_plane("repeated clean stencil acquisition", true);
        for (uint32_t i = 0; i < texels; ++i) {
          plane_bytes[plane_offsets[i]] += 0x10;
        }
        std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
        check_plane("CPU stencil plane write", false);
        std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
        TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), plane_depth_id,
            {vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1}, stencil_again);
        for (uint32_t i = 0; i < texels; ++i) {
          plane_bytes[plane_offsets[i]] += 0x10;
        }
        std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
        check_plane("CPU stencil write after native clear", false);
        for (uint32_t i = 0; i < texels; ++i) {
          plane_bytes[plane_offsets[i]] += 0x10;
        }
        std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
        Require(name, "sampled stencil view",
                texture_cache.FindTexture(plane_depth_id, sampled_plane) != nullptr,
                "an associated stencil plane could not be sampled");
        texture_cache.GetImage(plane_depth_id).Download(
            plane_copies, plane_readback.buffer, 0, plane_readback.size);
        scheduler.Current().Handle().pipelineBarrier2(stencil_dependency);
        scheduler.Finish();
        const auto sampled_values = ReadBuffer(name, plane_readback, texels + 2);
        const auto *sampled_bytes =
            reinterpret_cast<const uint8_t *>(sampled_values.data() + texels);
        for (uint32_t i = 0; i < texels; ++i) {
          Require(name, "sampled stencil plane refresh",
                  sampled_bytes[i] == plane_bytes[plane_offsets[i]],
                  "sampling without another depth acquisition used stale stencil bytes");
        }
        DestroyBuffer(&plane_readback);
        DestroyBuffer(&plane_upload);
        LibKernel::Memory::InstallGpuResources(nullptr);
      }
      resources.UnmapMemory(base, allocation_size);
      scheduler.Finish();
      RenderExecutorTestAccess::DestroyDescriptorPipelines(
          executor, descriptor_pipelines);
      context.ShutdownGpu();
    }

    Require(name, "unmap direct backing",
            Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
            "descriptor discovery direct mapping release failed");
    Require(name, "release direct backing",
            Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                direct_offset, allocation_size) == 0,
            "descriptor discovery direct-memory allocation release failed");
    std::printf("[host]    %-32s ok\n", name);
  }

  Buffer CreateStorageBuffer(const char *shader_name,
                             const std::vector<u32> &initial,
                             size_t dword_count, bool device_address = false) {
    Buffer ret;
    ret.size = static_cast<vk::DeviceSize>(std::max<size_t>(dword_count, 1u) *
                                           sizeof(u32));

    vk::BufferCreateInfo buffer_info{};
    buffer_info.sType = vk::StructureType::eBufferCreateInfo;
    buffer_info.size = ret.size;
    buffer_info.usage = vk::BufferUsageFlagBits::eStorageBuffer;
    if (device_address) {
      buffer_info.usage |= vk::BufferUsageFlagBits::eShaderDeviceAddress;
    }
    buffer_info.sharingMode = vk::SharingMode::eExclusive;
    RequireVk(shader_name, "dispatch",
              m_device.createBuffer(&buffer_info, nullptr, &ret.buffer),
              "vkCreateBuffer");

    vk::MemoryRequirements req{};
    m_device.getBufferMemoryRequirements(ret.buffer, &req);
    u32 memory_type = 0;
    ret.coherent = FindMemoryType(req.memoryTypeBits,
                                  vk::MemoryPropertyFlagBits::eHostVisible |
                                      vk::MemoryPropertyFlagBits::eHostCoherent,
                                  &memory_type);
    if (!ret.coherent) {
      Require(shader_name, "dispatch",
              FindMemoryType(req.memoryTypeBits,
                             vk::MemoryPropertyFlagBits::eHostVisible,
                             &memory_type),
              "no host-visible memory type for storage buffer");
    }

    vk::MemoryAllocateInfo alloc{};
    alloc.sType = vk::StructureType::eMemoryAllocateInfo;
    alloc.allocationSize = req.size;
    alloc.memoryTypeIndex = memory_type;
    vk::MemoryAllocateFlagsInfo flags{};
    if (device_address) {
      flags.sType = vk::StructureType::eMemoryAllocateFlagsInfo;
      flags.flags = vk::MemoryAllocateFlagBits::eDeviceAddress;
      alloc.pNext = &flags;
    }
    RequireVk(shader_name, "dispatch",
              m_device.allocateMemory(&alloc, nullptr, &ret.memory),
              "vkAllocateMemory");
    RequireVk(shader_name, "dispatch",
              m_device.bindBufferMemory(ret.buffer, ret.memory, 0),
              "vkBindBufferMemory");
    if (device_address) {
      vk::BufferDeviceAddressInfo address_info{};
      address_info.sType = vk::StructureType::eBufferDeviceAddressInfo;
      address_info.buffer = ret.buffer;
      ret.device_address = m_device.getBufferAddress(address_info);
      Require(shader_name, "dispatch", ret.device_address != 0,
              "storage buffer has no device address");
    }

    std::vector<u32> contents(dword_count, 0);
    for (size_t i = 0; i < initial.size() && i < contents.size(); i++) {
      contents[i] = initial[i];
    }
    WriteBuffer(shader_name, ret, contents);
    return ret;
  }

  void DestroyBuffer(Buffer *buffer) {
    if (buffer == nullptr) {
      return;
    }
    if (buffer->buffer != nullptr) {
      m_device.destroyBuffer(buffer->buffer, nullptr);
      buffer->buffer = nullptr;
    }
    if (buffer->memory != nullptr) {
      m_device.freeMemory(buffer->memory, nullptr);
      buffer->memory = nullptr;
    }
  }

  Image CreateImage2D(const char *shader_name, u32 width, u32 height,
                      vk::Format format, vk::ImageUsageFlags usage,
                      const std::vector<u32> &initial, u32 dwords_per_pixel,
                      vk::ImageLayout final_layout,
                      vk::Format view_format = vk::Format::eUndefined) {
    std::vector<std::vector<u32>> mips;
    if (!initial.empty()) {
      mips.push_back(initial);
    }
    return CreateImageMips(shader_name, width, height, format, usage, mips,
                           dwords_per_pixel, final_layout, vk::ImageType::e2D,
                           vk::ImageViewType::e2D, 1, 0, 0,
                           vk::SampleCountFlagBits::e1, view_format);
  }

  static u32 MipExtent(u32 value, u32 level) {
    for (u32 i = 0; i < level && value > 1u; i++) {
      value >>= 1u;
    }
    return std::max(value, 1u);
  }

  static size_t ImageMipDwordCount(u32 width, u32 height, u32 dwords_per_pixel,
                                   u32 level, u32 layers = 1) {
    return static_cast<size_t>(MipExtent(width, level)) *
           static_cast<size_t>(MipExtent(height, level)) * layers *
           dwords_per_pixel;
  }

  Image CreateImageMips(const char *shader_name, u32 width, u32 height,
                        vk::Format format, vk::ImageUsageFlags usage,
                        const std::vector<std::vector<u32>> &initial_mips,
                        u32 dwords_per_pixel, vk::ImageLayout final_layout,
                        vk::ImageType image_type, vk::ImageViewType view_type,
                        u32 layers, u32 view_base_layer = 0,
                        u32 view_layers = 0,
                        vk::SampleCountFlagBits samples = vk::SampleCountFlagBits::e1,
                        vk::Format view_format = vk::Format::eUndefined,
                        u32 view_base_mip = 0) {
    Image ret;
    ret.format = format;
    ret.width = width;
    ret.height = height;
    ret.layers = layers;
    ret.mip_levels = std::max<u32>(static_cast<u32>(initial_mips.size()), 1u);
    ret.dwords_per_pixel = dwords_per_pixel;

    vk::ImageCreateInfo image_info{};
    image_info.sType = vk::StructureType::eImageCreateInfo;
    image_info.imageType = image_type;
    image_info.format = format;
    image_info.extent.width = width;
    image_info.extent.height = height;
    image_info.extent.depth = 1;
    image_info.mipLevels = ret.mip_levels;
    image_info.arrayLayers = layers;
    image_info.samples = samples;
    image_info.tiling = vk::ImageTiling::eOptimal;
    image_info.usage = usage | vk::ImageUsageFlagBits::eTransferDst |
                       vk::ImageUsageFlagBits::eTransferSrc;
    image_info.sharingMode = vk::SharingMode::eExclusive;
    image_info.initialLayout = vk::ImageLayout::eUndefined;
    if (view_format != vk::Format::eUndefined && view_format != format) {
      image_info.flags |= vk::ImageCreateFlagBits::eMutableFormat;
    }
    RequireVk(shader_name, "dispatch",
              m_device.createImage(&image_info, nullptr, &ret.image),
              "vkCreateImage");

    vk::MemoryRequirements req{};
    m_device.getImageMemoryRequirements(ret.image, &req);
    u32 memory_type = 0;
    if (!FindMemoryType(req.memoryTypeBits,
                        vk::MemoryPropertyFlagBits::eDeviceLocal,
                        &memory_type)) {
      Require(shader_name, "dispatch",
              FindMemoryType(req.memoryTypeBits, {}, &memory_type),
              "no memory type for image");
    }

    vk::MemoryAllocateInfo alloc{};
    alloc.sType = vk::StructureType::eMemoryAllocateInfo;
    alloc.allocationSize = req.size;
    alloc.memoryTypeIndex = memory_type;
    RequireVk(shader_name, "dispatch",
              m_device.allocateMemory(&alloc, nullptr, &ret.memory),
              "vk::Device::allocateMemory(image)");
    RequireVk(shader_name, "dispatch",
              m_device.bindImageMemory(ret.image, ret.memory, 0),
              "vkBindImageMemory");

    vk::ImageViewCreateInfo view_info{};
    view_info.sType = vk::StructureType::eImageViewCreateInfo;
    view_info.image = ret.image;
    view_info.viewType = view_type;
    view_info.format = view_format == vk::Format::eUndefined ? format : view_format;
    view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor;
    Require(shader_name, "dispatch", view_base_mip < ret.mip_levels,
            "image view base mip is out of bounds");
    view_info.subresourceRange.baseMipLevel = view_base_mip;
    view_info.subresourceRange.levelCount = ret.mip_levels - view_base_mip;
    Require(shader_name, "dispatch", view_base_layer < layers,
            "image view base layer is out of bounds");
    if (view_layers == 0) {
      view_layers = layers - view_base_layer;
    }
    Require(shader_name, "dispatch", view_layers <= layers - view_base_layer,
            "image view layer count is out of bounds");
    view_info.subresourceRange.baseArrayLayer = view_base_layer;
    view_info.subresourceRange.layerCount = view_layers;
    RequireVk(shader_name, "dispatch",
              m_device.createImageView(&view_info, nullptr, &ret.view),
              "vkCreateImageView");

    if (!initial_mips.empty()) {
      size_t total_dwords = 0;
      for (u32 level = 0; level < ret.mip_levels; level++) {
        total_dwords +=
            ImageMipDwordCount(width, height, dwords_per_pixel, level, layers);
      }
      std::vector<u32> contents(total_dwords, 0);
      size_t offset = 0;
      for (u32 level = 0; level < ret.mip_levels; level++) {
        const auto level_dwords =
            ImageMipDwordCount(width, height, dwords_per_pixel, level, layers);
        const auto &src = initial_mips[level];
        for (size_t i = 0; i < src.size() && i < level_dwords; i++) {
          contents[offset + i] = src[i];
        }
        offset += level_dwords;
      }
      auto staging =
          CreateHostBuffer(shader_name, contents.size() * sizeof(u32),
                           vk::BufferUsageFlagBits::eTransferSrc, contents);
      UploadImageMips(shader_name, &ret, staging.buffer, final_layout);
      DestroyBuffer(&staging);
    } else {
      TransitionImage(shader_name, &ret, final_layout,
                      vk::PipelineStageFlagBits::eTopOfPipe,
                      vk::PipelineStageFlagBits::eComputeShader, {},
                      AccessForLayout(final_layout));
    }
    return ret;
  }

  void DestroyImage(Image *image) {
    if (image == nullptr) {
      return;
    }
    if (image->view != nullptr) {
      m_device.destroyImageView(image->view, nullptr);
      image->view = nullptr;
    }
    if (image->image != nullptr) {
      m_device.destroyImage(image->image, nullptr);
      image->image = nullptr;
    }
    if (image->memory != nullptr) {
      m_device.freeMemory(image->memory, nullptr);
      image->memory = nullptr;
    }
  }

  std::vector<u32> ReadImage(const char *shader_name, Image *image) {
    const auto dword_count = static_cast<size_t>(image->width) *
                             static_cast<size_t>(image->height) *
                             image->layers * image->dwords_per_pixel;
    auto staging = CreateHostBuffer(shader_name, dword_count * sizeof(u32),
                                   vk::BufferUsageFlagBits::eTransferDst, {});

    vk::CommandBuffer cmd = BeginCommands(shader_name, "readback");
    AddImageBarrier(
        cmd, image->image, image->layout, vk::ImageLayout::eTransferSrcOptimal,
        vk::PipelineStageFlagBits::eAllCommands,
        vk::PipelineStageFlagBits::eTransfer,
        vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
        vk::AccessFlagBits::eTransferRead, image->mip_levels, image->layers);
    vk::BufferImageCopy copy{};
    copy.bufferOffset = 0;
    copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
    copy.imageSubresource.mipLevel = 0;
    copy.imageSubresource.baseArrayLayer = 0;
    copy.imageSubresource.layerCount = image->layers;
    copy.imageExtent.width = image->width;
    copy.imageExtent.height = image->height;
    copy.imageExtent.depth = 1;
    cmd.copyImageToBuffer(image->image, vk::ImageLayout::eTransferSrcOptimal,
                          staging.buffer, 1, &copy);
    EndSubmitAndFree(shader_name, "readback", cmd);
    image->layout = vk::ImageLayout::eTransferSrcOptimal;

    auto ret = ReadBuffer(shader_name, staging, dword_count);
    DestroyBuffer(&staging);
    return ret;
  }

  vk::Sampler CreateSampler(const char *shader_name, u32 mip_levels = 1,
                            vk::Filter filter = vk::Filter::eNearest) {
    vk::SamplerCreateInfo sampler_info{};
    sampler_info.sType = vk::StructureType::eSamplerCreateInfo;
    sampler_info.magFilter = filter;
    sampler_info.minFilter = filter;
    sampler_info.mipmapMode = vk::SamplerMipmapMode::eNearest;
    sampler_info.addressModeU = vk::SamplerAddressMode::eClampToEdge;
    sampler_info.addressModeV = vk::SamplerAddressMode::eClampToEdge;
    sampler_info.addressModeW = vk::SamplerAddressMode::eClampToEdge;
    sampler_info.minLod = 0.0f;
    sampler_info.maxLod = static_cast<float>(mip_levels - 1u);
    vk::Sampler sampler = nullptr;
    RequireVk(shader_name, "dispatch",
              m_device.createSampler(&sampler_info, nullptr, &sampler),
              "vkCreateSampler");
    return sampler;
  }

  std::vector<u32> ReadBuffer(const char *shader_name, const Buffer &buffer,
                              size_t dword_count) {
    if (!buffer.coherent) {
      vk::MappedMemoryRange range{};
      range.sType = vk::StructureType::eMappedMemoryRange;
      range.memory = buffer.memory;
      range.offset = 0;
      range.size = VK_WHOLE_SIZE;
      RequireVk(shader_name, "readback",
                m_device.invalidateMappedMemoryRanges(1, &range),
                "vkInvalidateMappedMemoryRanges");
    }

    void *data = nullptr;
    RequireVk(shader_name, "readback",
              m_device.mapMemory(buffer.memory, 0, buffer.size, {}, &data),
              "vkMapMemory");
    std::vector<u32> ret(dword_count, 0);
    std::memcpy(ret.data(), data, dword_count * sizeof(u32));
    m_device.unmapMemory(buffer.memory);
    return ret;
  }

  void Dispatch(const TestCase &test, const CompiledShader &compiled,
                const Buffer &buffer, const Buffer *gds_buffer = nullptr,
                const Image *sampled_image = nullptr,
                const Image *storage_image = nullptr,
                const Image *storage_image_uint = nullptr,
                vk::Sampler sampler = nullptr,
                std::span<const Image> sampled_resources = {},
                std::span<const vk::Sampler> sampler_resources = {}) {
    using Kind = ShaderRecompiler::IR::DescriptorBindingKind;
    const auto &layout = compiled.program.bindings;
    auto shader_data = compiled.packed_user_data;
    if (layout.dispatch_thread_dword != ShaderRecompiler::IR::PushData::NoStart) {
      Require(test.name, "dispatch dimensions", test.has_compute_info,
              "runtime thread extents are required by the compiled shader");
      std::copy_n(test.compute_info.dispatch_threads_num, 3,
                  shader_data.begin() + layout.dispatch_thread_dword);
    }
    auto Count = [&](Kind kind) {
      return layout.descriptor_counts[static_cast<size_t>(kind)];
    };
    auto Native = [](Kind kind) {
      return ShaderRecompiler::IR::NativeBinding(ShaderType::Compute, kind);
    };
    for (const auto &image : compiled.program.info.images) {
      bool supported =
          image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
          image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
      if (image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) {
        supported = supported ||
                    image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1D ||
                    image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1DArray;
      }
      Require(test.name, "dispatch", supported,
              "unsupported image dimension needs a matching Vulkan test view");
    }

    vk::ShaderModuleCreateInfo module_info{};
    module_info.sType = vk::StructureType::eShaderModuleCreateInfo;
    module_info.codeSize = compiled.spirv.size() * sizeof(u32);
    module_info.pCode = compiled.spirv.data();
    vk::ShaderModule module = nullptr;
    RequireVk(test.name, "SPIR-V module",
              m_device.createShaderModule(&module_info, nullptr, &module),
              "vkCreateShaderModule");

    std::vector<vk::DescriptorSetLayoutBinding> layout_bindings;
    auto add_layout_binding =
        [&layout_bindings](u32 binding, vk::DescriptorType type, u32 count) {
          if (count == 0) {
            return;
          }
          vk::DescriptorSetLayoutBinding item{};
          item.binding = binding;
          item.descriptorType = type;
          item.descriptorCount = count;
          item.stageFlags = vk::ShaderStageFlagBits::eCompute;
          layout_bindings.push_back(item);
        };
    for (auto mask = layout.descriptor_mask; mask != 0; mask &= mask - 1) {
      const auto index = std::countr_zero(mask);
      const auto kind = static_cast<Kind>(index);
      add_layout_binding(Native(kind), NativeDescriptorType(kind), layout.descriptor_counts[index]);
    }

    vk::DescriptorSetLayoutCreateInfo layout_info{};
    layout_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
    layout_info.bindingCount = static_cast<u32>(layout_bindings.size());
    layout_info.pBindings =
        layout_bindings.empty() ? nullptr : layout_bindings.data();
    vk::DescriptorSetLayout descriptor_layout = nullptr;
    RequireVk(test.name, "dispatch",
              m_device.createDescriptorSetLayout(&layout_info, nullptr,
                                                 &descriptor_layout),
              "vkCreateDescriptorSetLayout");

    vk::PipelineLayoutCreateInfo pipeline_layout_info{};
    pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
    pipeline_layout_info.setLayoutCount = 1;
    pipeline_layout_info.pSetLayouts = &descriptor_layout;
    vk::PushConstantRange push_range{};
    if (layout.UsesPushData()) {
      push_range.stageFlags = vk::ShaderStageFlagBits::eCompute;
      push_range.offset = 0;
      push_range.size = ShaderRecompiler::IR::NativePushConstantSize;
      pipeline_layout_info.pushConstantRangeCount = 1;
      pipeline_layout_info.pPushConstantRanges = &push_range;
    }
    vk::PipelineLayout pipeline_layout = nullptr;
    RequireVk(test.name, "dispatch",
              m_device.createPipelineLayout(&pipeline_layout_info, nullptr,
                                            &pipeline_layout),
              "vkCreatePipelineLayout");

    vk::PipelineShaderStageCreateInfo stage{};
    stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
    stage.stage = vk::ShaderStageFlagBits::eCompute;
    stage.module = module;
    stage.pName = "main";

    vk::ComputePipelineCreateInfo pipeline_info{};
    pipeline_info.sType = vk::StructureType::eComputePipelineCreateInfo;
    pipeline_info.stage = stage;
    pipeline_info.layout = pipeline_layout;
    vk::Pipeline pipeline = nullptr;
    RequireVk(test.name, "dispatch",
              m_device.createComputePipelines(nullptr, 1, &pipeline_info,
                                              nullptr, &pipeline),
              "vkCreateComputePipelines");

    std::vector<vk::DescriptorPoolSize> pool_sizes;
    auto add_pool_size = [&pool_sizes](vk::DescriptorType type, u32 count) {
      if (count == 0) {
        return;
      }
      for (auto &size : pool_sizes) {
        if (size.type == type) {
          size.descriptorCount += count;
          return;
        }
      }
      pool_sizes.push_back({type, count});
    };
    for (auto mask = layout.descriptor_mask; mask != 0; mask &= mask - 1) {
      const auto index = std::countr_zero(mask);
      add_pool_size(NativeDescriptorType(static_cast<Kind>(index)), layout.descriptor_counts[index]);
    }
    vk::DescriptorPoolCreateInfo pool_info{};
    pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo;
    pool_info.maxSets = 1;
    pool_info.poolSizeCount = static_cast<u32>(pool_sizes.size());
    pool_info.pPoolSizes = pool_sizes.empty() ? nullptr : pool_sizes.data();
    vk::DescriptorPool descriptor_pool = nullptr;
    RequireVk(
        test.name, "dispatch",
        m_device.createDescriptorPool(&pool_info, nullptr, &descriptor_pool),
        "vkCreateDescriptorPool");

    vk::DescriptorSetAllocateInfo set_info{};
    set_info.sType = vk::StructureType::eDescriptorSetAllocateInfo;
    set_info.descriptorPool = descriptor_pool;
    set_info.descriptorSetCount = 1;
    set_info.pSetLayouts = &descriptor_layout;
    vk::DescriptorSet descriptor_set = nullptr;
    RequireVk(test.name, "dispatch",
              m_device.allocateDescriptorSets(&set_info, &descriptor_set),
              "vkAllocateDescriptorSets");

    std::vector<vk::WriteDescriptorSet> writes;
    std::vector<vk::DescriptorBufferInfo> buffer_infos;
    std::array<std::vector<vk::DescriptorImageInfo>, ShaderRecompiler::IR::ImageBindingCount> image_infos;
    std::vector<vk::ImageView> sampled_mip_views;
    std::vector<vk::DescriptorImageInfo> sampler_infos;
    Buffer flattened_buffer;
    Buffer user_data_buffer;
    Buffer shared_buffer;
    vk::DescriptorBufferInfo flattened_info{};
    vk::DescriptorBufferInfo user_data_info{};
    vk::DescriptorBufferInfo shared_info{};
    vk::DescriptorBufferInfo gds_info{};
    vk::DescriptorBufferInfo bda_pagetable_info{};
    vk::DescriptorBufferInfo fault_buffer_info{};

    if (Count(Kind::SharedMemory) != 0) {
      const auto bytes = uint64_t{test.compute_info.lds_size_dwords} * sizeof(u32) *
          test.dispatch_x * test.dispatch_y * test.dispatch_z;
      shared_buffer = CreateHostBuffer(test.name, bytes, vk::BufferUsageFlagBits::eStorageBuffer, {});
      shared_info = {shared_buffer.buffer, 0, shared_buffer.size};
      vk::WriteDescriptorSet write{};
      write.dstSet = descriptor_set;
      write.dstBinding = Native(Kind::SharedMemory);
      write.descriptorCount = 1;
      write.descriptorType = vk::DescriptorType::eStorageBuffer;
      write.pBufferInfo = &shared_info;
      writes.push_back(write);
    }

    const bool uses_bda = Count(Kind::BdaPagetable) != 0;
    Require(test.name, "dispatch",
            uses_bda == (Count(Kind::FaultBuffer) != 0),
            "BDA page table and fault buffer must be bound together");
    if (uses_bda) {
      EnsureBdaBuffers(test.name);
      bda_pagetable_info = {m_bda_pagetable_buffer.buffer, 0,
                            m_bda_pagetable_buffer.size};
      fault_buffer_info = {m_fault_buffer.buffer, 0, m_fault_buffer.size};
      const std::array special_buffers{
          std::pair{Kind::BdaPagetable, &bda_pagetable_info},
          std::pair{Kind::FaultBuffer, &fault_buffer_info},
      };
      for (const auto &[kind, info] : special_buffers) {
        vk::WriteDescriptorSet write{};
        write.sType = vk::StructureType::eWriteDescriptorSet;
        write.dstSet = descriptor_set;
        write.dstBinding = Native(kind);
        write.descriptorCount = 1;
        write.descriptorType = vk::DescriptorType::eStorageBuffer;
        write.pBufferInfo = info;
        writes.push_back(write);
      }
    }

    if (Count(Kind::Buffers) != 0) {
      buffer_infos.resize(Count(Kind::Buffers));
      for (u32 resource = 0; resource < compiled.program.info.buffers.size(); ++resource) {
        const auto index = compiled.program.info.buffers[resource].descriptor_index;
        if (index == UINT32_MAX) continue;
        auto &info = buffer_infos[index];
        info.buffer = buffer.buffer;
        info.offset = 0;
        info.range = buffer.size;
        if (resource < test.storage_buffer_ranges.size()) {
          const auto [offset, size] = test.storage_buffer_ranges[resource];
          info.offset = offset;
          info.range = size;
          Require(test.name, "dispatch", uint64_t{offset} + size <= buffer.size,
                  "storage buffer view exceeds its backing buffer");
        } else if (test.storage_buffer_range_bytes != 0) {
          const auto offset = resource < test.storage_buffer_offsets.size()
                                  ? test.storage_buffer_offsets[resource]
                                  : 0u;
          info.range = static_cast<vk::DeviceSize>(
              test.storage_buffer_range_bytes + offset);
          Require(test.name, "dispatch", info.range <= buffer.size,
                  "storage buffer descriptor range exceeds backing buffer");
        }
      }
      vk::WriteDescriptorSet write{};
      write.sType = vk::StructureType::eWriteDescriptorSet;
      write.dstSet = descriptor_set;
      write.dstBinding = Native(Kind::Buffers);
      write.descriptorCount = static_cast<u32>(buffer_infos.size());
      write.descriptorType = vk::DescriptorType::eStorageBuffer;
      write.pBufferInfo = buffer_infos.data();
      writes.push_back(write);
    }
    if (Count(Kind::FlattenedSrt) != 0) {
      flattened_buffer =
          CreateStorageBuffer(test.name, compiled.resources.flattened_srt,
                              compiled.resources.flattened_srt.size());
      flattened_info = {flattened_buffer.buffer, 0, flattened_buffer.size};
      vk::WriteDescriptorSet write{};
      write.sType = vk::StructureType::eWriteDescriptorSet;
      write.dstSet = descriptor_set;
      write.dstBinding = Native(Kind::FlattenedSrt);
      write.descriptorCount = 1;
      write.descriptorType = vk::DescriptorType::eStorageBuffer;
      write.pBufferInfo = &flattened_info;
      writes.push_back(write);
    }
    if (Count(Kind::ShaderData) != 0) {
      user_data_buffer =
          CreateStorageBuffer(test.name, shader_data, shader_data.size());
      user_data_info = {user_data_buffer.buffer, 0, user_data_buffer.size};
      vk::WriteDescriptorSet write{};
      write.sType = vk::StructureType::eWriteDescriptorSet;
      write.dstSet = descriptor_set;
      write.dstBinding = Native(Kind::ShaderData);
      write.descriptorCount = 1;
      write.descriptorType = vk::DescriptorType::eStorageBuffer;
      write.pBufferInfo = &user_data_info;
      writes.push_back(write);
    }
    if (Count(Kind::Gds) != 0) {
      Require(test.name, "dispatch", gds_buffer != nullptr,
              "GDS descriptor requested but no GDS buffer was provided");
      gds_info = {gds_buffer->buffer, 0, gds_buffer->size};
      vk::WriteDescriptorSet write{};
      write.sType = vk::StructureType::eWriteDescriptorSet;
      write.dstSet = descriptor_set;
      write.dstBinding = Native(Kind::Gds);
      write.descriptorCount = 1;
      write.descriptorType = vk::DescriptorType::eStorageBuffer;
      write.pBufferInfo = &gds_info;
      writes.push_back(write);
    }
    Require(test.name, "dispatch", sampled_resources.empty() ||
                sampled_resources.size() == compiled.program.info.images.size(),
            "sampled resources must match the logical images");
    for (u32 resource = 0; resource < compiled.program.info.images.size(); ++resource) {
      const auto &image = compiled.program.info.images[resource];
      const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(image);
      auto &infos = image_infos[ShaderRecompiler::IR::ImageBindingIndex(kind)];
      infos.resize(Count(kind));
      const bool sampled = image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
      const auto *source_image = sampled
          ? (sampled_resources.empty() ? sampled_image : &sampled_resources[resource])
          : (image.numeric_class == Prospero::TextureNumericClass::Float
                 ? storage_image : storage_image_uint);
      Require(test.name, "dispatch", source_image != nullptr,
              "image descriptor requested but no matching image was provided");
      for (u32 mip_index = 0; mip_index < image.mip_count; ++mip_index) {
        auto &info = infos[image.descriptor_index + mip_index];
        info.imageView = source_image->view;
        info.imageLayout = source_image->layout;
        if (sampled && image.mip_mode == ShaderRecompiler::IR::ImageMipMode::Dynamic) {
          const auto mip = test.sampled_image_view_base_mip + mip_index;
          Require(test.name, "dispatch", mip < source_image->mip_levels,
                  "sampled mip descriptor exceeds the supplied image");
          vk::ImageViewCreateInfo view{};
          view.image = source_image->image;
          view.viewType = test.sampled_image_view_type;
          view.format = source_image->format;
          view.subresourceRange = {
              vk::ImageAspectFlagBits::eColor, mip, 1,
              test.sampled_image_view_base_layer,
              test.sampled_image_view_layers != 0
                  ? test.sampled_image_view_layers
                  : source_image->layers - test.sampled_image_view_base_layer};
          RequireVk(test.name, "dispatch",
                    m_device.createImageView(&view, nullptr, &info.imageView),
                    "vkCreateImageView(sampled mip)");
          sampled_mip_views.push_back(info.imageView);
        }
      }
    }
    for (u32 group = 0; group < image_infos.size(); ++group) {
      const auto &infos = image_infos[group];
      if (infos.empty()) continue;
      const auto kind = static_cast<Kind>(ShaderRecompiler::IR::FirstImageBinding + group);
      vk::WriteDescriptorSet write{};
      write.dstSet = descriptor_set;
      write.dstBinding = Native(kind);
      write.descriptorCount = static_cast<u32>(infos.size());
      write.descriptorType = NativeDescriptorType(kind);
      write.pImageInfo = infos.data();
      writes.push_back(write);
    }
    if (Count(Kind::Samplers) != 0) {
      Require(test.name, "dispatch", sampler_resources.empty()
                  ? sampler != nullptr : sampler_resources.size() == Count(Kind::Samplers),
              "sampler resources must match the native sampler array");
      sampler_infos.resize(Count(Kind::Samplers));
      for (u32 index = 0; index < sampler_infos.size(); ++index) {
        sampler_infos[index].sampler = sampler_resources.empty() ? sampler : sampler_resources[index];
      }
      vk::WriteDescriptorSet write{};
      write.dstSet = descriptor_set;
      write.dstBinding = Native(Kind::Samplers);
      write.descriptorCount = static_cast<u32>(sampler_infos.size());
      write.descriptorType = vk::DescriptorType::eSampler;
      write.pImageInfo = sampler_infos.data();
      writes.push_back(write);
    }
    if (!writes.empty()) {
      m_device.updateDescriptorSets(static_cast<u32>(writes.size()),
                                    writes.data(), 0, nullptr);
    }

    vk::CommandBuffer cmd = BeginCommands(test.name, "dispatch");
    if (uses_bda) {
      Require(test.name, "dispatch", buffer.device_address != 0,
              "BDA test backing has no device address");
      Require(test.name, "dispatch", !test.bda_mappings.empty(),
              "BDA test has no explicit guest mapping");
      cmd.fillBuffer(m_bda_pagetable_buffer.buffer, 0,
                     m_bda_pagetable_buffer.size, 0);
      cmd.fillBuffer(m_fault_buffer.buffer, 0, m_fault_buffer.size, 0);
      std::array<vk::BufferMemoryBarrier, 2> barriers{};
      barriers[0].sType = vk::StructureType::eBufferMemoryBarrier;
      barriers[0].srcAccessMask = vk::AccessFlagBits::eTransferWrite;
      barriers[0].dstAccessMask = vk::AccessFlagBits::eTransferWrite;
      barriers[0].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barriers[0].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barriers[0].buffer = m_bda_pagetable_buffer.buffer;
      barriers[0].offset = 0;
      barriers[0].size = m_bda_pagetable_buffer.size;
      barriers[1] = barriers[0];
      barriers[1].dstAccessMask = vk::AccessFlagBits::eShaderRead |
                                  vk::AccessFlagBits::eShaderWrite;
      barriers[1].buffer = m_fault_buffer.buffer;
      barriers[1].size = m_fault_buffer.size;
      cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
                          vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr,
                          1, barriers.data(), 0, nullptr);
      for (const auto &[guest_base, backing] : test.bda_mappings) {
        const auto page_offset = guest_base &
                                 (BufferCache::CACHING_PAGESIZE - 1);
        Require(test.name, "dispatch",
                backing >= page_offset && backing < buffer.size,
                "BDA test mapping begins outside its backing buffer");
        const auto bytes = page_offset + buffer.size - backing;
        const auto pages =
            (bytes + BufferCache::CACHING_PAGESIZE - 1) /
            BufferCache::CACHING_PAGESIZE;
        auto address = buffer.device_address + backing - page_offset;
        auto page = BufferCache::PageIndex(guest_base);
        for (uint64_t mapped = 0; mapped < pages; mapped++) {
          cmd.updateBuffer(m_bda_pagetable_buffer.buffer,
                           (page + mapped) * sizeof(vk::DeviceAddress),
                           sizeof(address), &address);
          address += BufferCache::CACHING_PAGESIZE;
        }
      }
      barriers[0].dstAccessMask = vk::AccessFlagBits::eShaderRead;
      cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
                          vk::PipelineStageFlagBits::eComputeShader, {}, 0,
                          nullptr, static_cast<u32>(barriers.size()),
                          barriers.data(), 0, nullptr);
    }
    cmd.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline);
    cmd.bindDescriptorSets(vk::PipelineBindPoint::eCompute, pipeline_layout, 0,
                           1, &descriptor_set, 0, nullptr);
    if (layout.UsesPushData()) {
      ShaderRecompiler::IR::PushData push_data;
      std::copy(shader_data.begin(), shader_data.end(),
                push_data.dwords.begin() + layout.push_data_start_dword);
      cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0,
                        sizeof(push_data), push_data.dwords.data());
    }
    cmd.dispatch(test.dispatch_x, test.dispatch_y, test.dispatch_z);

    if (Count(Kind::Buffers) != 0) {
      vk::BufferMemoryBarrier barrier{};
      barrier.sType = vk::StructureType::eBufferMemoryBarrier;
      barrier.srcAccessMask =
          vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite;
      barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
      barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barrier.buffer = buffer.buffer;
      barrier.offset = 0;
      barrier.size = buffer.size;
      cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader,
                          vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
                          &barrier, 0, nullptr);
    }
    if (gds_buffer != nullptr) {
      vk::BufferMemoryBarrier barrier{};
      barrier.sType = vk::StructureType::eBufferMemoryBarrier;
      barrier.srcAccessMask =
          vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite;
      barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
      barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barrier.buffer = gds_buffer->buffer;
      barrier.offset = 0;
      barrier.size = gds_buffer->size;
      cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader,
                          vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
                          &barrier, 0, nullptr);
    }
    Buffer fault_result;
    if (test.fault_free_address) {
      Require(test.name, "fault check", uses_bda, "fault check requires BDA");
      fault_result = CreateHostBuffer(test.name, sizeof(u32),
                                     vk::BufferUsageFlagBits::eTransferDst, {});
      const vk::MemoryBarrier ready{
          .srcAccessMask = vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite,
          .dstAccessMask = vk::AccessFlagBits::eTransferRead};
      cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer,
                          vk::PipelineStageFlagBits::eTransfer, {}, 1, &ready, 0, nullptr, 0, nullptr);
      const vk::BufferCopy region{(BufferCache::PageIndex(*test.fault_free_address) >> 5u) * sizeof(u32),
                                   0, sizeof(u32)};
      cmd.copyBuffer(m_fault_buffer.buffer, fault_result.buffer, 1, &region);
      const vk::MemoryBarrier readable{.srcAccessMask = vk::AccessFlagBits::eTransferWrite,
                                        .dstAccessMask = vk::AccessFlagBits::eHostRead};
      cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost,
                          {}, 1, &readable, 0, nullptr, 0, nullptr);
    }
    EndSubmitAndFree(test.name, "dispatch", cmd);
    if (test.fault_free_address) {
      Require(test.name, "fault check", ReadBuffer(test.name, fault_result, 1)[0] == 0u,
              "suppressed formatted access touched an unmapped page");
      DestroyBuffer(&fault_result);
    }
    for (const auto view : sampled_mip_views) {
      m_device.destroyImageView(view, nullptr);
    }
    if (flattened_buffer.buffer != nullptr) {
      DestroyBuffer(&flattened_buffer);
    }
    if (user_data_buffer.buffer != nullptr) {
      DestroyBuffer(&user_data_buffer);
    }
    DestroyBuffer(&shared_buffer);
    m_device.destroyDescriptorPool(descriptor_pool, nullptr);
    m_device.destroyPipeline(pipeline, nullptr);
    m_device.destroyPipelineLayout(pipeline_layout, nullptr);
    m_device.destroyDescriptorSetLayout(descriptor_layout, nullptr);
    m_device.destroyShaderModule(module, nullptr);
  }

  void CheckRasterization(
      bool depth_feedback,
      Prospero::BufferFormat color_format = Prospero::BufferFormat::k32_32_32_32Float) {
    const char *name = depth_feedback ? "DepthAttachmentFeedback" : "PolygonModeRasterization";
    uint32_t packed_color = 0;
    std::array<float, 4> packed_expected{};
    switch (color_format) {
      case Prospero::BufferFormat::k11_11_10Float:
        name = "PackedFloatVertexColor";
        packed_color = 0x801e0380u;
        packed_expected = {0.5f, 1.0f, 2.0f, 1.0f};
        break;
      case Prospero::BufferFormat::k10_10_10_2UScaled:
        name = "PackedUscaledVertexColor";
        packed_color = 513u | (257u << 10u) | (1023u << 20u) | (3u << 30u);
        packed_expected = {513.0f, 257.0f, 1023.0f, 3.0f};
        break;
      default: break;
    }
    const bool packed_vertex_color = color_format != Prospero::BufferFormat::k32_32_32_32Float;
    const uint32_t extent = depth_feedback ? 8 : 32;
    constexpr uintptr_t depth_address = 0x0000000204400000ull;
    constexpr uint64_t allocation_size = 0x40000;
    constexpr uint64_t rect_address = depth_address + 0x8000;
    constexpr uint64_t index_address = rect_address + 0x1000;
    EnsureRuntimeContext();
    RenderContext context(m_runtime_context);
    auto &scheduler = context.GetCommandScheduler();
    HW::Context registers{};
    HW::UserConfig user_config{};
    HW::Shader shaders{};
    registers.SetRenderTargetMask(0xf);
    registers.SetShaderMask(0xf);
    scheduler.Begin(registers, user_config, shaders);
    auto &resources = context;
    auto &cache = context.GetTextureCache();
    auto &executor = context.GetRenderExecutor();
    int64_t direct_offset = -1;
    RenderDepthInfo depth{};
    Require(name, "direct allocation",
            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
                allocation_size, allocation_size, 0, &direct_offset) == 0,
            "depth allocation failed");
    void *mapped = reinterpret_cast<void *>(depth_address);
    Require(name, "direct mapping",
            Libs::LibKernel::Memory::KernelMapDirectMemory(
                &mapped, allocation_size, 0x3, 0x10, direct_offset,
                allocation_size) == 0 &&
                mapped == reinterpret_cast<void *>(depth_address),
            "depth mapping failed");
    std::memset(mapped, 0, allocation_size);
    const auto initial_depth = [](uint32_t x, uint32_t y) {
      return 0.125f + static_cast<float>(x + 2 * y) / 128.0f;
    };
    for (uint32_t y = 0; y < extent; y++) {
      for (uint32_t x = 0; x < extent; x++) {
        static_cast<float *>(mapped)[y * 64 + x] = initial_depth(x, y);
      }
    }
    if (!depth_feedback && !packed_vertex_color) {
      // Rectangle input contains three position/UV records.
      constexpr std::array<float, 24> rect_vertices{
          -0.75f, -0.75f, 0, 0, 0, 0, 0, 0,
           0.75f, -0.75f, 0, 0, 1, 0, 0, 0,
          -0.75f,  0.75f, 0, 0, 0, 1, 0, 0};
      std::memcpy(reinterpret_cast<void *>(rect_address), rect_vertices.data(),
                  sizeof(rect_vertices));
      constexpr std::array<uint32_t, 3> indices{0, 1, 2};
      std::memcpy(reinterpret_cast<void *>(index_address), indices.data(), sizeof(indices));
    }
    resources.MapMemory(depth_address, allocation_size);
    if (depth_feedback) {
      depth.desc.type = BindingType::DepthTarget;
      depth.desc.info.data = {depth_address, 256 * extent};
      depth.desc.info.pixel_format = vk::Format::eD32Sfloat;
      depth.desc.info.guest_format = Prospero::BufferFormat::k32Float;
      depth.desc.info.extent = {extent, extent, 1};
      depth.desc.info.pitch = 64;
      depth.desc.info.bytes_per_block = 4;
      depth.desc.info.tile_mode = Prospero::TileMode::kLinear;
      depth.desc.info.UpdateSize();
      depth.desc.view_info.format = depth.desc.info.pixel_format;
      depth.desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
      depth.desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
      depth.depth_test_enable = depth.depth_write_enable = true;
      depth.depth_compare_op = vk::CompareOp::eAlways;
      depth.image_id = cache.FindImage(depth.desc);
    }

    GraphicsCase test;
    test.name = name;
    test.pixel_depth_export = depth_feedback;
    if (depth_feedback) {
      const ShaderTextureResource descriptor{{
          static_cast<uint32_t>(depth_address >> 8u),
          (static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u) |
              (((extent - 1u) & 3u) << 30u),
          ((extent - 1u) >> 2u) | ((extent - 1u) << 14u),
          DstSel(4, 4, 4, 4) |
              (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u),
          0, 0, 0, 0}};
      test.has_user_data = true;
      std::copy_n(descriptor.fields, 8, test.user_data.begin());
      // Each fragment samples its own texel through interpolated screen UVs.
      for (uint32_t component = 0; component < 2; component++) {
        test.fragment_code.push_back(EncodeVintrp(0, 20 + component, 0, component, 0));
        test.fragment_code.push_back(EncodeVintrp(1, 20 + component, 0, component, 1));
      }
      test.fragment_code.push_back(EncodeMimg0(0x27, 1));
      test.fragment_code.push_back(EncodeMimg1(0, 20, 0, 2));
      AppendVMovLiteral(&test.fragment_code, 1, 0x3e000000u);
      test.fragment_code.push_back(EncodeVop2(0x03, 1, Vgpr(0), 1));
      test.fragment_code.push_back(EncodeExp0(0x08, 0x1, false));
      test.fragment_code.push_back(EncodeExp1(1, 1, 1, 1));
    } else {
      test.pixel_interpolator_settings = {0x400u};
      for (uint32_t component = 0; component < (packed_vertex_color ? 4u : 1u); component++) {
        test.fragment_code.push_back(EncodeVintrp(0x02, component, 0, component, 2));
      }
    }
    test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
    test.fragment_code.push_back(packed_vertex_color ? EncodeExp1(0, 1, 2, 3)
                                                    : EncodeExp1(0, 0, 0, 0));
    AppendEnd(&test.fragment_code);
    auto fragment = CompileFragmentCase(test);
    const auto vertex_spirv = TestSpv::MakePassthroughVertexSpirv(false);
    ShaderProgram vertex_shader{1, CreateShaderModule(name, vertex_spirv)};
    ShaderProgram pixel_shader{2, CreateShaderModule(name, fragment.spirv)};
    ShaderRecompiler::IR::CompiledShaderInfo vertex_program{};
    vertex_program.stage = ShaderType::Vertex;
    vertex_program.info.vertex_fetch_components[0] = 2;
    vertex_program.info.vertex_fetch_components[1] = packed_vertex_color ? 3 : 4;
    ShaderRecompiler::IR::CompiledShaderInfo pixel_program{};
    pixel_program.stage = ShaderType::Pixel;
    pixel_program.info = fragment.program.info;
    pixel_program.bindings = fragment.program.bindings;
    ShaderVertexInputInfo vertex{};
    ShaderRecompiler::IR::ResourceSnapshot vertex_snapshot;
    vertex.stage.program = &vertex_program;
    vertex.stage.resources = &vertex_snapshot;
    vertex.resources_num = 2;
    vertex.buffers_num = 1;
    vertex.buffers[0].stride = 6 * sizeof(float);
    vertex.resources[1].UpdateAddress48(2 * sizeof(float));
    for (uint32_t i = 0; i < 2; i++) {
      const auto format = i == 0 ? Prospero::BufferFormat::k32_32Float : color_format;
      vertex.resources[i].fields[3] = DstSel(4, 5, 6, 7) |
                                      (static_cast<uint32_t>(format) << 12u);
      vertex.resources_dst[i].registers_num =
          i == 0 ? 2 : vertex_program.info.vertex_fetch_components[1];
    }
    ShaderPixelInputInfo pixel{};
    pixel.stage.program = &pixel_program;
    pixel.stage.resources = &fragment.resources;

    RenderColorInfo color{};
    color.desc.type = BindingType::RenderTarget;
    color.desc.info.data = {depth_address + 0x10000, extent * extent * 16};
    color.desc.info.mip_layout[0] = {
        0, color.desc.info.data.size, extent, extent, 0, color.desc.info.data.size, extent};
    color.desc.info.tiling.block.bytes_per_element = 16;
    color.desc.info.linear_slice_stride = color.desc.info.data.size;
    color.desc.info.pixel_format = vk::Format::eR32G32B32A32Sfloat;
    color.desc.info.guest_format = Prospero::BufferFormat::k32_32_32_32Float;
    color.desc.info.extent = {extent, extent, 1};
    color.desc.info.pitch = extent;
    color.desc.info.bytes_per_block = 16;
    color.desc.view_info.format = color.desc.info.pixel_format;
    color.desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
    color.image_id = cache.FindImage(color.desc);
    std::array<float, 18> vertices{
        -0.75f, -0.75f, 1, 1, 1, 1, 0.75f, -0.75f, 0.5f, 1, 1, 1,
        0.0f, 0.75f, 0.25f, 1, 1, 1};
    if (depth_feedback) {
      vertices = {-1, -1, 0, 0, 0, 1, 3, -1, 2, 0, 0, 1, -1, 3, 0, 2, 0, 1};
    }
    std::vector<u32> vertex_words(vertices.size());
    std::memcpy(vertex_words.data(), vertices.data(), sizeof(vertices));
    if (packed_vertex_color) {
      for (uint32_t i = 0; i < 3; i++) {
        vertex_words[i * 6 + 2] = packed_color;
      }
    }
    auto buffer = CreateHostBuffer(name, sizeof(vertices), vk::BufferUsageFlagBits::eVertexBuffer,
                                   vertex_words);
    const auto pipeline = [&](bool enabled, uint8_t front, uint8_t back,
                              bool provoking_last = false,
                              bool clockwise = false,
                              vk::PrimitiveTopology topology = vk::PrimitiveTopology::eTriangleList)
        -> PipelineCache::Pipeline & {
      HW::ModeControl mode{};
      mode.face = clockwise;
      mode.poly_mode = enabled;
      mode.polymode_front_ptype = front;
      mode.polymode_back_ptype = back;
      mode.provoking_vtx_last = provoking_last;
      registers.SetModeControl(mode);
      return context.GetPipelineCache().GetGraphicsPipeline(
          std::span{&color, 1u}, depth, std::span{&vertex, 1u}, scheduler.Current(), &pixel,
          topology, false,
          PipelineCache::GraphicsPrograms{{vertex_shader}, pixel_shader});
    };
    auto &filled = pipeline(true, 2, 2);
    const auto draw = [&](const PipelineCache::Pipeline &selected, uint32_t vertex_count = 3,
                          std::array<u32, 4> clear_value = {}) {
      RenderExecutorTestAccess::BindRenderTarget(executor, color.image_id);
      if (depth.image_id) {
        RenderExecutorTestAccess::BindRenderTarget(executor, depth.image_id);
      }
      auto bindings = RenderExecutorTestAccess::PrepareGraphicsBindings(
          executor, vertex.stage, pixel.stage, true);
      Require(name, "sampled attachment identity",
              !depth_feedback || bindings.pixel->images[0].image_id == depth.image_id,
              "the fragment must sample the depth attachment's native image");
      auto &command = scheduler.Current();
      vk::ImageAspectFlags feedback_aspects;
      std::array<PreparedBindings *, 2> stages{
          &bindings.vertex[0], &*bindings.pixel};
      auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
          executor, command, &color, 1, depth, stages, &feedback_aspects);
      const bool feedback_enabled = rendering.depth_stencil_attachment.image_layout ==
                                    vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT;
      Require(name, "per-draw depth feedback",
              feedback_enabled == (depth_feedback && depth.depth_write_enable) &&
                  feedback_aspects == (feedback_enabled
                                          ? vk::ImageAspectFlagBits::eDepth
                                          : vk::ImageAspectFlags{}),
              "feedback was not selected only for an overlapping fragment depth read/write");
      RenderExecutorTestAccess::CommitBindings(
          executor, command, selected, bindings.vertex[0], *bindings.pixel);
      rendering.color_attachments[0].is_clear = true;
      rendering.color_attachments[0].clear_value = clear_value;
      command.BeginRendering(rendering);
      auto cmd = command.Handle();
      cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, selected.pipeline);
      const vk::Viewport viewport{0, 0, static_cast<float>(extent),
                                  static_cast<float>(extent), 0, 1};
      const vk::Rect2D scissor{{0, 0}, {extent, extent}};
      cmd.setViewportWithCount(1, &viewport);
      cmd.setScissorWithCount(1, &scissor);
      cmd.setLineWidth(1);
      cmd.setDepthTestEnable(depth.depth_test_enable);
      cmd.setDepthWriteEnable(depth.depth_write_enable);
      cmd.setDepthCompareOp(depth.depth_compare_op);
#if !defined(__APPLE__)
      cmd.setDepthBoundsTestEnable(depth.depth_bounds_test_enable);
      if (depth.depth_bounds_test_enable) {
        cmd.setDepthBounds(registers.GetDepthBoundsMin(), registers.GetDepthBoundsMax());
      }
#endif
      cmd.setDepthBiasEnable(false);
      cmd.setStencilTestEnable(depth.stencil_test_enable);
      if (depth.stencil_test_enable) {
        const auto set_stencil = [&](vk::StencilFaceFlagBits face,
                                     const vk::StencilOpState &state) {
          cmd.setStencilOp(face, state.failOp, state.passOp, state.depthFailOp,
                           state.compareOp);
          cmd.setStencilCompareMask(face, state.compareMask);
          cmd.setStencilWriteMask(face, state.writeMask);
          cmd.setStencilReference(face, state.reference);
        };
        set_stencil(vk::StencilFaceFlagBits::eFront, depth.stencil_front);
        set_stencil(vk::StencilFaceFlagBits::eBack, depth.stencil_back);
      }
      const vk::Bool32 write = true;
      cmd.setColorWriteEnableEXT(1, &write);
      cmd.setAttachmentFeedbackLoopEnableEXT(feedback_aspects);
      const vk::DeviceSize offset = 0;
      cmd.bindVertexBuffers(0, 1, &buffer.buffer, &offset);
      cmd.draw(vertex_count, 1, 0, 0);
      command.EndRendering();
      RenderExecutorTestAccess::ResetBindings(executor);
    };
    const auto read_color = [&] {
      return ReadCachedTexel(name, context, color.image_id, {}, {extent, extent, 1});
    };
    draw(filled);
    const auto solid_pixels = read_color();
    if (packed_vertex_color) {
      const auto center = 4 * ((extent / 2) * extent + extent / 2);
      for (uint32_t component = 0; component < packed_expected.size(); component++) {
        Require(name, "packed vertex fetch and coverage",
                std::abs(std::bit_cast<float>(solid_pixels[center + component]) -
                         packed_expected[component]) < 0.0001f && solid_pixels[component] == 0,
                "packed vertex color changed channels, clamped HDR, or filled outside the triangle");
      }
    } else if (depth_feedback) {
      // Consecutive draws in the same submission must see each earlier depth write.
      draw(filled);
      draw(filled);
      depth.depth_write_enable = false;
      draw(filled); // Reset feedback in the same command buffer and preserve the depth writes.
      const auto repeated_pixels = read_color();
      const auto stored_depth = ReadCachedTexel(
          name, context, depth.image_id, {}, {extent, extent, 1});
      for (uint32_t y = 0; y < extent; y++) {
        for (uint32_t x = 0; x < extent; x++) {
          const auto index = y * extent + x;
          Require(name, "sampled and stored depth",
                  solid_pixels[index * 4] == std::bit_cast<uint32_t>(initial_depth(x, y)) &&
                      repeated_pixels[index * 4] ==
                          std::bit_cast<uint32_t>(initial_depth(x, y) + 0.375f) &&
                      stored_depth[index] == repeated_pixels[index * 4],
                  "depth feedback lost a prior write or the read-only draw changed depth");
        }
      }
      depth.depth_test_enable = false;
      for (uint32_t pass = 0; pass < 3; pass++) {
        depth.depth_bounds_test_enable = pass != 2;
        registers.SetDepthBoundsMin(pass == 0 ? 0.5f : 0.625f);
        registers.SetDepthBoundsMax(pass == 0 ? 0.5625f : 0.6875f);
        Require(name, "dynamic depth bounds pipeline reuse",
                pipeline(true, 2, 2).pipeline == filled.pipeline,
                "changing depth bounds or their enable state created another pipeline");
        draw(filled);
        const auto bounded_pixels = read_color();
        for (size_t index = 0; index < stored_depth.size(); index++) {
          const auto z = std::bit_cast<float>(stored_depth[index]);
          const bool passes = !depth.depth_bounds_test_enable ||
                              (z >= registers.GetDepthBoundsMin() &&
                               z <= registers.GetDepthBoundsMax());
          Require(name, "dynamic stored-depth bounds",
                  bounded_pixels[index * 4] == (passes ? stored_depth[index] : 0u),
                  "depth bounds did not filter stored depth independently of the depth test");
        }
      }
    } else {
      auto &wireframe = pipeline(true, 1, 1);
      Require(name, "pipeline cache", filled.pipeline != wireframe.pipeline &&
                  pipeline(true, 2, 2).pipeline == filled.pipeline &&
                  pipeline(false, 1, 0).pipeline == filled.pipeline,
              "polygon mode was lost from the cache key or dormant face modes were applied");
      draw(wireframe);
      const auto line_pixels = read_color();
      const auto interior = 4 * (16 * extent + 16);
      Require(name, "GPU coverage", solid_pixels[interior] == 0x3f800000u &&
                  line_pixels[interior] == 0 &&
                  std::ranges::any_of(line_pixels, [](u32 value) { return value == 0x3f800000u; }),
              "wireframe did not preserve triangle edges while leaving its interior empty");
      auto &last_vertex = pipeline(true, 2, 2, true);
      Require(name, "provoking vertex pipeline cache",
              last_vertex.pipeline != filled.pipeline &&
                  pipeline(true, 2, 2, true).pipeline == last_vertex.pipeline &&
                  pipeline(true, 2, 2).pipeline == filled.pipeline,
              "first and last provoking vertices did not keep distinct cached pipelines");
      draw(last_vertex);
      const auto last_pixels = read_color();
      for (size_t i = 0; i < solid_pixels.size(); i++) {
        Require(name, "flat provoking vertex output",
                last_pixels[i] == (solid_pixels[i] == 0 ? 0 : 0x3e800000u),
                "last-vertex flat shading changed coverage or did not use vertex two");
      }

      const auto blend_pipeline = [&](bool enabled, bool bypass,
                                      bool alternate = false) -> PipelineCache::Pipeline & {
        auto blend = registers.GetBlendControl(0);
        blend.enable = enabled;
        blend.color_srcblend = static_cast<uint8_t>(alternate ? Prospero::BlendFactor::kOne
                                                            : Prospero::BlendFactor::kZero);
        blend.alpha_srcblend = blend.color_srcblend;
        blend.separate_alpha_blend = alternate;
        registers.SetBlendControl(0, blend);
        auto target = registers.GetRenderTarget(0).info;
        target.blend_bypass = bypass;
        registers.SetColorInfo(0, target);
        return pipeline(true, 2, 2);
      };
      auto &disabled_blend = blend_pipeline(false, false);
      Require(name, "effective blend pipeline cache",
              blend_pipeline(false, true).pipeline == disabled_blend.pipeline &&
                  blend_pipeline(true, true).pipeline == disabled_blend.pipeline &&
                  blend_pipeline(false, false, true).pipeline == disabled_blend.pipeline &&
                  blend_pipeline(true, true, true).pipeline == disabled_blend.pipeline,
              "disabled and bypassed blending did not share the same host pipeline");
      auto &enabled_blend = blend_pipeline(true, false);
      Require(name, "enabled blend pipeline cache",
              enabled_blend.pipeline != disabled_blend.pipeline &&
                  blend_pipeline(true, false).pipeline == enabled_blend.pipeline,
              "active blending was lost from the pipeline key or missed its cached pipeline");
      auto dormant_alpha = registers.GetBlendControl(0);
      dormant_alpha.alpha_srcblend = static_cast<uint8_t>(Prospero::BlendFactor::kOne);
      registers.SetBlendControl(0, dormant_alpha);
      Require(name, "dormant alpha blend pipeline cache",
              pipeline(true, 2, 2).pipeline == enabled_blend.pipeline,
              "inactive separate-alpha state recompiled the pipeline");
      draw(enabled_blend);
      const auto blended_pixels = read_color();
      Require(name, "active blend output",
              std::ranges::all_of(blended_pixels, [](u32 value) { return value == 0; }),
              "zero-factor blending did not clear the fragment color");
      draw(blend_pipeline(true, true));
      Require(name, "bypassed blend output", read_color() == solid_pixels,
              "blend bypass did not restore the unblended fragment color and coverage");

      HW::DepthRenderTarget stencil_target{};
      stencil_target.z_info.format = Prospero::DepthFormat::kZ32F;
      stencil_target.stencil_info.format = Prospero::StencilFormat::k8UInt;
      stencil_target.stencil_info.htile_stencil_disabled = true;
      stencil_target.z_read_base_addr = stencil_target.z_write_base_addr = depth_address;
      stencil_target.stencil_read_base_addr = stencil_target.stencil_write_base_addr =
          depth_address + 0x20000;
      stencil_target.size = {static_cast<uint16_t>(extent - 1),
                             static_cast<uint16_t>(extent - 1), true};
      registers.SetDepthRenderTarget(stencil_target);
      HW::DepthControl stencil_control{};
      // Keep the same depth/stencil attachment active when stencil testing is disabled.
      stencil_control.z_enable = true;
      stencil_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eAlways);
      stencil_control.stencil_enable = stencil_control.backface_enable = true;
      stencil_control.stencilfunc = stencil_control.stencilfunc_bf =
          static_cast<uint8_t>(vk::CompareOp::eAlways);
      registers.SetDepthControl(stencil_control);
      // PPSA01325 sets bit 7 on front faces and clears it on back faces.
      registers.SetStencilControl({3, 3, 3, 4, 4, 4});
      registers.SetStencilMask({0x80, 0, 0x80, 1, 0x80, 0, 0x80, 1});
      RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
      auto stencil_readback = CreateHostBuffer(name, extent * extent,
          vk::BufferUsageFlagBits::eTransferDst, {});
      const auto read_stencil = [&] {
        const vk::BufferImageCopy copy{0, 0, 0,
            {vk::ImageAspectFlagBits::eStencil, 0, 0, 1}, {}, {extent, extent, 1}};
        cache.GetImage(depth.image_id).Download(std::span{&copy, 1},
            stencil_readback.buffer, 0, stencil_readback.size);
        const vk::MemoryBarrier2 barrier{
            .srcStageMask = vk::PipelineStageFlagBits2::eCopy,
            .srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
            .dstStageMask = vk::PipelineStageFlagBits2::eHost,
            .dstAccessMask = vk::AccessFlagBits2::eHostRead};
        vk::DependencyInfo dependency{};
        dependency.memoryBarrierCount = 1;
        dependency.pMemoryBarriers = &barrier;
        scheduler.Current().Handle().pipelineBarrier2(dependency);
        scheduler.Finish();
        return ReadBuffer(name, stencil_readback, extent * extent / 4);
      };
      const std::array stencil_pipelines{
          pipeline(true, 2, 2, false, true).pipeline,
          pipeline(true, 2, 2, false, false).pipeline};
      const auto check_stencil = [&](const char *label, bool back_face,
                                     uint8_t initial, uint8_t written) {
        vk::ClearValue clear{};
        clear.depthStencil = vk::ClearDepthStencilValue{0.625f, initial};
        TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), depth.image_id,
            {vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil,
             0, 1, 0, 1}, clear);
        const auto &selected = pipeline(true, 2, 2, false, !back_face);
        Require(name, "dynamic stencil pipeline reuse",
                selected.pipeline == stencil_pipelines[back_face],
                "stencil enable, operations or references changed the graphics pipeline");
        draw(selected);
        const auto result = read_stencil();
        const auto *stencil_bytes = reinterpret_cast<const uint8_t *>(result.data());
        for (uint32_t texel = 0; texel < extent * extent; ++texel) {
          const uint8_t expected = solid_pixels[texel * 4] != 0 ? written : initial;
          Require(name, label,
                  stencil_bytes[texel] == expected,
                  "captured stencil operations changed masked bits or triangle coverage");
        }
      };
      check_stencil("front-face ReplaceTest", false, 0x35, 0xb5);
      check_stencil("back-face masked ReplaceOp", true, 0xb5, 0x35);

      stencil_control.stencil_enable = false;
      registers.SetDepthControl(stencil_control);
      RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
      check_stencil("disabled front stencil", false, 0x35, 0x35);
      check_stencil("disabled back stencil", true, 0xb5, 0xb5);
      stencil_control.stencil_enable = true;
      registers.SetDepthControl(stencil_control);
      RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
      check_stencil("re-enabled front stencil", false, 0x35, 0xb5);
      check_stencil("re-enabled back stencil", true, 0xb5, 0x35);

      // Frame 6115 uses separate test/op replacement values with Always comparisons.
      stencil_control.z_enable = stencil_control.z_write_enable = true;
      stencil_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eLessOrEqual);
      registers.SetDepthControl(stencil_control);
      registers.SetStencilControl({0, 3, 9, 0, 4, 8});
      registers.SetStencilMask({8, 0xff, 0xff, 1, 8, 0xff, 0xff, 1});
      RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
      Require(name, "independent stencil references",
              depth.stencil_front.reference == 8 &&
                  depth.stencil_back.reference == 1,
              "always-pass faces must preserve their distinct replacement values");
      check_stencil("front test-value replacement", false, 0x35, 8);
      check_stencil("back operation-value replacement", true, 0x35, 1);
      stencil_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eNever);
      registers.SetDepthControl(stencil_control);
      RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
      check_stencil("front depth-fail subtract wrap", false, 0, 0xff);
      check_stencil("back depth-fail add wrap", true, 0xff, 0);

      // A compare on bit 3 and replacement on bit 0 share one Vulkan reference
      // without changing either native operation or the untouched stencil bits.
      stencil_control.z_enable = stencil_control.z_write_enable = false;
      stencil_control.stencilfunc = stencil_control.stencilfunc_bf =
          static_cast<uint8_t>(vk::CompareOp::eEqual);
      registers.SetDepthControl(stencil_control);
      registers.SetStencilControl({0, 4, 0, 0, 4, 0});
      registers.SetStencilMask({8, 8, 1, 1, 8, 8, 1, 1});
      RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
      Require(name, "masked compare and replace reference",
              depth.stencil_front.reference == 9 &&
                  depth.stencil_back.reference == 9,
              "disjoint comparison and replacement bits must combine into one reference");
      check_stencil("masked compare passes", false, 0xb8, 0xb9);
      check_stencil("masked compare fails", false, 0xb0, 0xb0);

      // SPI_PS_IN_CONTROL must select the native wave width through the actual
      // program cache. In wave32 a low-word compare preserves scalar VCC_HI.
      static const auto native_vertex = [] {
        std::vector<u32> code;
        AppendVMovLiteral(&code, 1, 0xbf800000u);
        AppendVMovLiteral(&code, 2, 0x40400000u);
        code.push_back(EncodeVopc(0xc2, InlineU32(1), 5));
        code.push_back(EncodeVop2(0x01, 3, Vgpr(1), 2));
        code.push_back(EncodeVopc(0xc2, InlineU32(2), 5));
        code.push_back(EncodeVop2(0x01, 4, Vgpr(1), 2));
        AppendVMovU32(&code, 0, 0);
        AppendVMovLiteral(&code, 6, 0x3f800000u);
        code.push_back(EncodeExp0(0x0c, 0xf));
        code.push_back(EncodeExp1(3, 4, 0, 6));
        for (u32 parameter = 0; parameter < 8; parameter++) {
          code.push_back(EncodeExp0(0x20 + parameter, 0xf));
          code.push_back(EncodeExp1(0, 0, 0, 0));
        }
        AppendEnd(&code);
        return code;
      }();
      static const auto native_pixel = [] {
        std::vector<u32> code;
        AppendVMovU32(&code, 0, 0);
        AppendSMovLiteral(&code, 107, 0x3e800000u);
        code.push_back(EncodeVopc(0xc2, InlineU32(1), 0));
        code.push_back(EncodeVop1(0x01, 1, 107));
        AppendVMovLiteral(&code, 2, 0x3f800000u);
        code.push_back(EncodeExp0(0x00, 0xf));
        code.push_back(EncodeExp1(1, 1, 1, 2));
        AppendEnd(&code);
        return code;
      }();
      static ShaderUserData native_user_data{};
      const auto vertex_address = reinterpret_cast<uint64_t>(native_vertex.data());
      const auto pixel_address = reinterpret_cast<uint64_t>(native_pixel.data());
      ShaderMapUserData(vertex_address,
          {.type = Prospero::ShaderBinaryType::kGs,
           .user_data = &native_user_data,
           .code_size_bytes = static_cast<uint32_t>(native_vertex.size() * sizeof(u32))});
      ShaderMapUserData(pixel_address,
          {.type = Prospero::ShaderBinaryType::kPs,
           .code_size_bytes = static_cast<uint32_t>(native_pixel.size() * sizeof(u32))});
      HW::VertexShaderInfo native_vertex_regs{};
      native_vertex_regs.es_regs.data_addr = vertex_address;
      HW::PixelShaderInfo native_pixel_regs{};
      native_pixel_regs.ps_regs.data_addr = pixel_address;
      const auto owned_vertex_shader = vertex_shader;
      const auto owned_pixel_shader = pixel_shader;
      depth.stencil_test_enable = false;
      registers.SetViewportTransformControl(0x300);
      user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriList);
      std::array<Prospero::ColorComponentMapping, 8> export_mapping{};
      std::array<ShaderVertexInputInfo, 3> native_vertex_info{};
      std::array<uint64_t, 2> wave_ids{};
      std::array<std::vector<u32>, 2> wave_keys;
      struct PixelWaveCase { uint32_t control, width, color; };
      for (const auto wave : {PixelWaveCase{0x0008, 64, 0},
                              PixelWaveCase{0x8008, 32, 0x3e800000},
                              PixelWaveCase{0x0008, 64, 0}}) {
        registers.SetPsInControl(wave.control);
        const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
            native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
            registers, user_config, export_mapping, true, native_vertex_info, pixel);
        Require(name, "pixel wave metadata propagation",
                pixel.input_num == 8 && pixel.stage.program->wave_size == wave.width,
                "SPI_PS_IN_CONTROL width did not reach the compiled pixel program");
        const auto index = static_cast<uint32_t>(wave.width == 32);
        std::vector<u32> key;
        BuildStageStaticKey(pixel, key);
        if (wave_ids[index] != 0) {
          Require(name, "pixel wave cache reuse",
                  programs.pixel.id == wave_ids[index] && key == wave_keys[index],
                  "restoring wave64 did not reuse its original program");
        }
        wave_ids[index] = programs.pixel.id;
        wave_keys[index] = std::move(key);
        vertex_shader = programs.vertex[0];
        pixel_shader = programs.pixel;
        vertex = native_vertex_info[0];
        draw(pipeline(true, 2, 2));
        const auto wave_pixels = read_color();
        for (size_t component = 0; component < wave_pixels.size(); component++) {
          const auto expected = component % 4 == 3 ? 0x3f800000u : wave.color;
          Require(name, "pixel wave VCC_HI readback", wave_pixels[component] == expected,
                  "pixel comparison clobbered VCC_HI in wave32 or retained it in wave64");
        }
      }
      Require(name, "pixel wave cache distinction",
              wave_ids[0] != wave_ids[1] && wave_keys[0] != wave_keys[1],
              "wave32 and wave64 pixel programs shared a cache key");

      static const auto waterfall_vertex = [&] {
        std::vector<u32> code;
        AppendVMovU32(&code, 9, 0);
        // d3e1ae284c4dea3b reconstructs EXEC twice from packed NGG s3.
        code.insert(code.end(), {0xbefe04c1u, 0x906a8803u, 0x81ea6a80u,
                                 0x90fe6ac1u, 0x81ea0380u, 0x90fe6ac1u,
                                 EncodeSop1(0x04, 10, 126)});
        const auto loop = code.size();
        code.push_back(EncodeSop1(0x14, 4, 10));
        AppendVop3(&code, 0x360, 6, Vgpr(5), 4);
        code.push_back(EncodeVopc(0xc2, 6, 5));
        code.push_back(EncodeSop1(0x24, 18, 106));
        AppendVMovLiteral(&code, 9, 0x3f800000u);
        code.push_back(EncodeSop2(0x15, 10, 10, 106));
        code.push_back(EncodeSop1(0x04, 126, 18));
        code.push_back(EncodeSopp(0x05, static_cast<int16_t>(loop - code.size() - 1u)));
        const auto base = code.size();
        code.insert(code.end(), native_vertex.begin(), native_vertex.end());
        const auto position = std::find(code.begin() + base, code.end(), EncodeExp0(0x0c, 0xf));
        // Unprocessed vertices keep W=0, making waterfall progress observable.
        *(position + 1) = EncodeExp1(3, 4, 0, 9);
        return code;
      }();
      native_vertex_regs.es_regs.data_addr = reinterpret_cast<uint64_t>(waterfall_vertex.data());
      ShaderMapUserData(native_vertex_regs.es_regs.data_addr,
          {.type = Prospero::ShaderBinaryType::kGs, .user_data = &native_user_data,
           .code_size_bytes = static_cast<u32>(waterfall_vertex.size() * sizeof(u32))});
      registers.SetPsInControl(0x8008);
      const auto saved_stages = registers.GetShaderStages();
      for (const u32 wave : {32u, 64u}) {
        registers.SetShaderStages((saved_stages & ~0x00400000u) |
                                   (wave == 32u ? 0x00400000u : 0u));
        const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
            native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
            registers, user_config, export_mapping, true, native_vertex_info, pixel);
        vertex_shader = programs.vertex[0];
        pixel_shader = programs.pixel;
        vertex = native_vertex_info[0];
        Require(name, "native NGG waterfall wave",
                vertex.stage.program->stage == ShaderType::Vertex &&
                    vertex.stage.program->wave_size == wave,
                "NGG waterfall bypassed the native vertex path or lost its guest wave width");
        for (const u32 count : {3u, 6u, 33u}) {
          draw(pipeline(true, 2, 2), count);
          const auto pixels = read_color();
          for (size_t component = 0; component < pixels.size(); ++component)
            Require(name, "NGG partial-wave waterfall output",
                    pixels[component] == (component % 4 == 3 ? 0x3f800000u : 0x3e800000u),
                    "native vertex waterfall did not terminate with every input vertex processed");
        }
      }
      registers.SetShaderStages(saved_stages);
      native_vertex_regs.es_regs.data_addr = vertex_address;

      // Reversed RGBA targets must use logical Sa for RGB, with a separate
      // zero or unit source factor for separate alpha attenuation or accumulation.
      static const auto blend_pixel = [] {
        std::vector<u32> code;
        constexpr std::array<float, 4> source{0.8f, 0.6f, 0.4f, 0.5f};
        for (u32 component = 0; component < source.size(); component++) {
          AppendVMovLiteral(&code, component, std::bit_cast<u32>(source[component]));
        }
        code.push_back(EncodeExp0(0, 0xf));
        code.push_back(EncodeExp1(0, 1, 2, 3));
        AppendEnd(&code);
        return code;
      }();
      native_pixel_regs.ps_regs.data_addr = reinterpret_cast<uint64_t>(blend_pixel.data());
      ShaderMapUserData(native_pixel_regs.ps_regs.data_addr,
          {.type = Prospero::ShaderBinaryType::kPs,
           .code_size_bytes = static_cast<uint32_t>(blend_pixel.size() * sizeof(u32))});
      const auto saved_blend = registers.GetBlendControl(0);
      const auto saved_target_info = registers.GetRenderTarget(0).info;
      const auto saved_output_mode = registers.GetShaderRegisters().target_output_mode[0];
      registers.SetTargetOutputMode(0, 4);
      auto blend_target_info = saved_target_info;
      blend_target_info.blend_bypass = false;
      registers.SetColorInfo(0, blend_target_info);
      export_mapping[0] = color.export_mapping = Prospero::ColorMappingAbgr;
      HW::BlendControl alpha_blend{};
      alpha_blend.enable = true;
      alpha_blend.color_srcblend = static_cast<uint8_t>(Prospero::BlendFactor::kSrcAlpha);
      alpha_blend.alpha_srcblend = static_cast<uint8_t>(Prospero::BlendFactor::kOne);
      alpha_blend.color_destblend = alpha_blend.alpha_destblend =
          static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusSrcAlpha);
      struct AlphaBlendCase {
        bool separate;
        Prospero::BlendFactor alpha_source;
        ShaderAlphaBlendSource mode;
        float alpha;
      };
      uint64_t broadcast_program = 0;
      vk::Pipeline broadcast_pipeline{};
      for (const auto test : {
          AlphaBlendCase{false, Prospero::BlendFactor::kOne, ShaderAlphaBlendSource::SourceAlpha, 0.4f},
          AlphaBlendCase{true, Prospero::BlendFactor::kSrcAlpha, ShaderAlphaBlendSource::SourceAlpha, 0.4f},
          AlphaBlendCase{true, Prospero::BlendFactor::kOne, ShaderAlphaBlendSource::SourceAlphaOne, 0.65f},
          AlphaBlendCase{true, Prospero::BlendFactor::kZero, ShaderAlphaBlendSource::SourceAlphaZero, 0.15f}}) {
        alpha_blend.separate_alpha_blend = test.separate;
        alpha_blend.alpha_srcblend = static_cast<uint8_t>(test.alpha_source);
        registers.SetBlendControl(0, alpha_blend);
        const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
            native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
            registers, user_config, export_mapping, true, native_vertex_info, pixel);
        Require(name, "logical alpha blend program", pixel.alpha_blend_source == test.mode &&
                    pixel.dual_source_blending,
                "the separate alpha equation did not reach the compiled pixel program");
        vertex_shader = programs.vertex[0];
        pixel_shader = programs.pixel;
        vertex = native_vertex_info[0];
        auto &selected = pipeline(true, 2, 2);
        if (test.mode == ShaderAlphaBlendSource::SourceAlpha) {
          Require(name, "equal blend equation cache reuse",
                  broadcast_program == 0 || (programs.pixel.id == broadcast_program &&
                                              selected.pipeline == broadcast_pipeline),
                  "equivalent separate alpha state created another shader or pipeline");
          broadcast_program = programs.pixel.id;
          broadcast_pipeline = selected.pipeline;
        }
        constexpr std::array<float, 4> destination{0.3f, 0.3f, 0.1f, 0.2f};
        draw(selected, 3, std::bit_cast<std::array<u32, 4>>(destination));
        const auto blend_pixels = read_color();
        const std::array<float, 4> expected{test.alpha, 0.35f, 0.35f, 0.5f};
        for (size_t component = 0; component < blend_pixels.size(); component++) {
          Require(name, "logical alpha blend readback",
                  std::abs(std::bit_cast<float>(blend_pixels[component]) -
                           expected[component % 4]) < 0.00001f,
                  "reversed blending changed logical RGB or the separate alpha equation");
        }
      }
      native_pixel_regs.ps_regs.data_addr = pixel_address;
      export_mapping[0] = color.export_mapping = {};
      registers.SetBlendControl(0, saved_blend);
      registers.SetColorInfo(0, saved_target_info);
      registers.SetTargetOutputMode(0, saved_output_mode);

      // Captured Playroom strips contain the fullscreen triangle followed by an
      // out-of-bounds fetch exporting (0,0,0,0). Additive color reveals any extra triangle.
      static constexpr std::array<std::array<u32, 4>, 4> fourth_positions{{
          {0, 0, 0, 0}, {0x80000000u, 0, 0x80000000u, 0x80000000u},
          {0, 0, 0, 0x3f800000u}, {0xbf800000u, 0xbf800000u, 0, 0}}};
      static const auto position_shaders = [&] {
        std::array<std::vector<u32>, fourth_positions.size()> result;
        for (size_t i = 0; i < result.size(); i++) {
          auto &code = result[i];
          const auto position_export = std::ranges::find(native_vertex, EncodeExp0(0x0c, 0xf));
          code.assign(native_vertex.begin(), position_export);
          code.push_back(EncodeVopc(0xc2, InlineU32(3), 5));
          constexpr std::array<u32, 4> position_registers{3, 4, 0, 6};
          for (size_t component = 0; component < position_registers.size(); component++) {
            AppendVMovLiteral(&code, 7, fourth_positions[i][component]);
            const auto reg = position_registers[component];
            code.push_back(EncodeVop2(0x01, reg, Vgpr(reg), 7));
          }
          code.insert(code.end(), position_export, native_vertex.end());
        }
        return result;
      }();
      auto additive_blend = registers.GetBlendControl(0);
      additive_blend.enable = true;
      additive_blend.color_srcblend = additive_blend.color_destblend =
          additive_blend.alpha_srcblend = additive_blend.alpha_destblend =
              static_cast<uint8_t>(Prospero::BlendFactor::kOne);
      registers.SetBlendControl(0, additive_blend);
      auto target_info = registers.GetRenderTarget(0).info;
      target_info.blend_bypass = false;
      registers.SetColorInfo(0, target_info);
      registers.SetPsInControl(0x8008);
      user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriStrip);
      for (size_t i = 0; i < position_shaders.size(); i++) {
        const auto &code = position_shaders[i];
        native_vertex_regs.es_regs.data_addr = reinterpret_cast<uint64_t>(code.data());
        ShaderMapUserData(native_vertex_regs.es_regs.data_addr,
            {.type = Prospero::ShaderBinaryType::kGs,
             .user_data = &native_user_data,
             .code_size_bytes = static_cast<uint32_t>(code.size() * sizeof(u32))});
        const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
            native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
            registers, user_config, export_mapping, true, native_vertex_info, pixel);
        vertex_shader = programs.vertex[0];
        pixel_shader = programs.pixel;
        vertex = native_vertex_info[0];
        draw(pipeline(true, 2, 2, false, false, vk::PrimitiveTopology::eTriangleStrip), 4);
        const auto pixels = read_color();
        bool second_triangle = false;
        for (size_t component = 0; component < pixels.size(); component += 4) {
          Require(name, "fullscreen triangle preserved",
                  pixels[component] == 0x3e800000u || pixels[component] == 0x3f000000u,
                  "clipping-error culling removed valid fullscreen coverage");
          second_triangle |= pixels[component] == 0x3f000000u;
        }
        Require(name, "zero homogeneous position culling", second_triangle == (i >= 2),
                "zero positions drew an extra triangle, or a valid position was culled");
      }

      // A NULL-export stencil pass must ignore smaller stale color targets,
      // materialize the whole pending HTile clear, and still execute pixel discard.
      static const auto null_pixels = [] {
        std::array<std::vector<u32>, 2> result;
        for (uint32_t discard = 0; discard < result.size(); discard++) {
          auto &code = result[discard];
          if (discard != 0) {
            code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
          }
          code.push_back(EncodeExp0(0x09, 0, true, false, true));
          code.push_back(EncodeExp1(0, 0, 0, 0));
          AppendEnd(&code);
        }
        return result;
      }();
      registers.Reset();
      registers.SetViewportTransformControl(0x300);
      registers.SetViewportScaleOffset(0, extent / 2, extent / 2,
                                       extent / 2, extent / 2, 1, 0);
      registers.SetViewportZMax(0, 1);
      registers.SetScreenScissor(0, 0, extent, extent);
      registers.SetWindowScissor(0, 0, extent, extent, false);
      registers.SetGenericScissor(0, 0, extent, extent, false);
      registers.SetViewportScissor(0, 0, 0, extent, extent, false);
      registers.SetRenderTargetMask(0xf00f);
      registers.SetShaderMask(0xf00f);
      registers.SetPsInControl(0x8000);
      registers.SetDepthShaderControl({.shader_kill_enable = true});
      for (const auto slot : {0u, 3u}) {
        registers.SetColorBase(slot,
            {.addr = depth_address + (slot == 0 ? 0x38000 : 0x10000)});
        registers.SetColorInfo(slot,
            {.format = Prospero::ChannelLayout::k32_32_32_32,
             .channel_type = Prospero::ChannelType::kFloat,
             .channel_order = Prospero::ChannelOrder::kStandard});
        const auto side = slot == 0 ? extent / 4 : extent;
        registers.SetColorAttrib2(slot, {.height = side - 1, .width = side - 1});
        registers.SetColorAttrib3(slot,
            {.tile_mode = Prospero::TileMode::kLinear, .dimension = 1});
      }
      registers.SetTargetOutputMode(0, 4);
      registers.SetTargetOutputMode(1, 4);
      stencil_target.z_info.htile_acceleration = true;
      stencil_target.htile_data_base_addr = depth_address + 0x30000;
      registers.SetDepthRenderTarget(stencil_target);
      registers.SetDepthClearValue(1);
      stencil_control = {};
      stencil_control.stencil_enable = true;
      stencil_control.stencilfunc = static_cast<uint8_t>(vk::CompareOp::eAlways);
      registers.SetDepthControl(stencil_control);
      registers.SetStencilControl({3, 3, 3, 3, 3, 3});
      registers.SetStencilMask({0x80, 0, 0x80, 1, 0x80, 0, 0x80, 1});
      user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriList);
      shaders.SetEsShaderBase(vertex_address);
      for (uint32_t discard = 0; discard < null_pixels.size(); discard++) {
        const auto &code = null_pixels[discard];
        const auto address = reinterpret_cast<uint64_t>(code.data());
        ShaderMapUserData(address,
            {.type = Prospero::ShaderBinaryType::kPs,
             .code_size_bytes = static_cast<uint32_t>(code.size() * sizeof(u32))});
        shaders.SetPsShaderBase(address);
        RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
        (void)cache.FindDepthTarget(depth.image_id, depth.desc);
        vk::ClearValue zero{};
        TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), depth.image_id,
            {vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil,
             0, 1, 0, 1}, zero);
        Require(name, "pending stencil-pass HTile clear",
                cache.ClearMeta(stencil_target.htile_data_base_addr),
                "the stencil pass did not retain its HTile metadata");
        RenderExecutorTestAccess::DrawAuto(
            executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
        const auto depth_pixels = ReadCachedTexel(name, context, depth.image_id,
                                                  {}, {extent, extent, 1});
        Require(name, "NULL-export full depth clear",
                std::ranges::all_of(depth_pixels, [](u32 v) { return v == 0x3f800000u; }),
                "an unexported color target clipped the pending depth clear");
        const auto stencil = read_stencil();
        const auto expected = discard == 0 ? 0x80808080u : 0u;
        Require(name, "NULL-export stencil coverage and discard",
                std::ranges::all_of(stencil, [=](u32 v) { return v == expected; }),
                "a stale color target clipped stencil coverage or the NULL-export shader was skipped");
      }

      // A depth-only draw retains the previous PS address and write masks, but
      // SPI_SHADER_COL_FORMAT disables its exports. Compiling this stale PS
      // with the new zero-input metadata would fail before the depth draw.
      static const auto stale_pixel = [] {
        std::vector<u32> code{EncodeVintrp(0, 0, 0, 0, 0),
                              EncodeVintrp(1, 0, 0, 0, 1),
                              EncodeExp0(0, 0xf), EncodeExp1(0, 0, 0, 0)};
        AppendEnd(&code);
        return code;
      }();
      const auto stale_address = reinterpret_cast<uint64_t>(stale_pixel.data());
      ShaderMapUserData(stale_address,
          {.type = Prospero::ShaderBinaryType::kPs,
           .code_size_bytes = static_cast<u32>(stale_pixel.size() * sizeof(u32))});
      shaders.SetPsShaderBase(stale_address);
      registers.SetPsInControl(0);
      registers.SetDepthShaderControl({});
      registers.SetDepthControl({.z_enable = true, .z_write_enable = true,
                                .zfunc = static_cast<uint8_t>(vk::CompareOp::eAlways)});
      for (const auto slot : {0u, 1u}) {
        registers.SetTargetOutputMode(slot, 0);
      }
      RenderExecutorTestAccess::DrawAuto(
          executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
      const auto depth_only_pixels = ReadCachedTexel(
          name, context, depth.image_id, {}, {extent, extent, 1});
      Require(name, "disabled stale PS preserves fixed-function depth",
              std::ranges::all_of(depth_only_pixels, [](u32 v) { return v == 0; }),
              "disabled color exports invoked the stale PS or clipped fixed-function depth");
      registers.SetPsInControl(0x8000);
      registers.SetTargetOutputMode(0, 4);

      // Export ordinals follow CB_SHADER_MASK. A disabled CB_TARGET_MASK slot
      // still consumes an ordinal, and stale MRT0 must not clip physical MRT3.
      static const auto sparse_pixels_code = [] {
        std::array<std::vector<u32>, 2> result{native_pixel, native_pixel};
        auto &code = result[1];
        const auto exp = std::ranges::find(code, EncodeExp0(0, 0xf));
        *exp = EncodeExp0(0, 0xf, false);
        *(exp + 1) = EncodeExp1(0, 0, 0, 0);
        code.insert(exp + 2, {EncodeExp0(1, 0xf), EncodeExp1(1, 1, 1, 2)});
        return result;
      }();
      registers.SetRenderTargetMask(0xf000);
      registers.SetDepthControl({});
      registers.SetDepthShaderControl({});
      RenderColorInfo sparse_color{};
      for (uint32_t index = 0; index < sparse_pixels_code.size(); index++) {
        const auto &code = sparse_pixels_code[index];
        const auto address = reinterpret_cast<uint64_t>(code.data());
        ShaderMapUserData(address,
            {.type = Prospero::ShaderBinaryType::kPs,
             .code_size_bytes = static_cast<uint32_t>(code.size() * sizeof(u32))});
        shaders.SetPsShaderBase(address);
        registers.SetShaderMask(index == 0 ? 0xf000 : 0xf00f);
        registers.SetTargetOutputMode(1, index == 0 ? 0 : 4);
        RenderExecutorTestAccess::ResolveRenderColorTarget(
            executor, scheduler.Current(), sparse_color, 3);
        TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), sparse_color.image_id,
            {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
        RenderExecutorTestAccess::DrawAuto(
            executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
        const auto pixels = ReadCachedTexel(name, context, sparse_color.image_id,
                                            {}, {extent, extent, 1});
        for (size_t component = 0; component < pixels.size(); component++) {
          const auto expected = component % 4 == 3 ? 0x3f800000u : 0x3e800000u;
          Require(name, "compact export to sparse MRT3", pixels[component] == expected,
                  "shader output routing used the write mask or clipped MRT3 with stale MRT0");
        }
      }

      static const auto snorm_pixel = [] {
        std::vector<u32> code;
        AppendVMovLiteral(&code, 0, 0xc0008000u);
        AppendVMovLiteral(&code, 1, 0x7fff4000u);
        code.insert(code.end(), {EncodeExp0(0, 0xf, true, true), EncodeExp1(0, 1, 0, 0)});
        AppendEnd(&code);
        return code;
      }();
      const auto snorm_address = reinterpret_cast<uint64_t>(snorm_pixel.data());
      ShaderMapUserData(snorm_address,
          {.type = Prospero::ShaderBinaryType::kPs,
           .code_size_bytes = static_cast<uint32_t>(snorm_pixel.size() * sizeof(u32))});
      shaders.SetPsShaderBase(snorm_address);
      registers.SetShaderMask(0xf000);
      registers.SetTargetOutputMode(0, 6);
      registers.SetTargetOutputMode(1, 0);
      RenderExecutorTestAccess::DrawAuto(
          executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
      const auto snorm_pixels = ReadCachedTexel(name, context, sparse_color.image_id,
                                              {}, {extent, extent, 1});
      constexpr std::array<float, 4> snorm_expected{-1, -16384.0f / 32767,
                                                    16384.0f / 32767, 1};
      for (size_t component = 0; component < snorm_pixels.size(); component++) {
        Require(name, "compressed SNORM16 color export",
                std::abs(std::bit_cast<float>(snorm_pixels[component]) -
                         snorm_expected[component % 4]) < 0.00001f,
                "SNORM16 export lost its sign, normalization or negative endpoint clamp");
      }
      registers.SetTargetOutputMode(0, 4);

      // Indirect arguments also reach s8/s9. Native VS prologs must add them
      // exactly once, for both indexed and automatically generated indices.
      static const auto indirect_vertex = [&] {
        auto code = native_vertex;
        for (auto &word : code) {
          if (word == EncodeVopc(0xc2, InlineU32(1), 5))
            word = EncodeVopc(0xc2, InlineU32(8), 5);
          else if (word == EncodeVopc(0xc2, InlineU32(2), 5))
            word = EncodeVopc(0xc2, InlineU32(9), 5);
        }
        code.insert(code.begin(), {EncodeVop2(0x25, 5, 8, 5),
                                   EncodeVop2(0x25, 8, 9, 8)});
        const auto position = std::ranges::find(code, EncodeExp0(0x0c, 0xf));
        code.insert(position, {EncodeVopc(0xc2, InlineU32(5), 8),
                               EncodeVop2(0x01, 3, Vgpr(1), 3),
                               EncodeVop2(0x01, 4, Vgpr(1), 4)});
        return code;
      }();
      const auto indirect_vs = reinterpret_cast<uint64_t>(indirect_vertex.data());
      ShaderMapUserData(indirect_vs,
          {.type = Prospero::ShaderBinaryType::kGs, .user_data = &native_user_data,
           .code_size_bytes = static_cast<uint32_t>(indirect_vertex.size() * sizeof(u32))});
      shaders.SetEsShaderBase(indirect_vs);
      shaders.SetPsShaderBase(pixel_address);
      shaders.SetGsShaderResource2({.user_sgpr = 2});
      shaders.SetGsUserSgpr(0, 7, HW::UserSgprType::Unknown);
      shaders.SetGsUserSgpr(1, 5, HW::UserSgprType::Unknown);
      user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriList);
      for (const bool indexed : {false, true}) {
        TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), sparse_color.image_id,
            {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
        if (indexed) {
          RenderExecutorTestAccess::DrawIndex(executor, scheduler.Current(),
              {.index_count = 3, .index_addr = reinterpret_cast<const void *>(index_address),
               .instance_count = 1, .index_type_and_size = 1, .base_vertex = 7, .first_instance = 5,
               .offset_source = DrawOffsetSource::IndirectArgs});
        } else {
          RenderExecutorTestAccess::DrawAuto(executor, scheduler.Current(),
              {.vertex_count = 3, .instance_count = 1, .first_vertex = 7, .first_instance = 5,
               .offset_source = DrawOffsetSource::IndirectArgs});
        }
        const auto pixels = ReadCachedTexel(name, context, sparse_color.image_id,
                                            {}, {extent, extent, 1});
        for (size_t component = 0; component < pixels.size(); component++) {
          Require("NativeIndirectDrawOffsets", indexed ? "indexed" : "auto",
                  pixels[component] == (component % 4 == 3 ? 0x3f800000u : 0x3e800000u),
                  "native vertex or instance offset was applied twice");
        }
      }

      // A fourth VS invocation reads beyond the descriptor instead of reconstructing the corner.
      const ShaderBufferResource rect_buffer{{
          static_cast<u32>(rect_address),
          static_cast<u32>(rect_address >> 32u) | (32u << 16u), 3,
          DstSel(4, 5, 6, 7) |
              (static_cast<u32>(Prospero::BufferFormat::k32Float) << 12u)}};
      static const auto rect_vertex_code = [] {
        std::vector<u32> code{
            EncodeMubuf0(0x0d, 0, true, false), EncodeMubuf1(0, 2, 5),
            EncodeMubuf0(0x0d, 16, true, false), EncodeMubuf1(2, 2, 5)};
        AppendVMovU32(&code, 4, 0);
        AppendVMovLiteral(&code, 6, 0x3f800000u);
        code.insert(code.end(), {EncodeExp0(0x0c, 0xf), EncodeExp1(0, 1, 4, 6),
                                 EncodeExp0(0x20, 0xf), EncodeExp1(2, 3, 4, 6)});
        AppendEnd(&code);
        return code;
      }();
      static const auto rect_pixel_code = [] {
        std::vector<u32> code;
        for (u32 component = 0; component < 2; component++) {
          code.push_back(EncodeVintrp(0, 20 + component, 0, component, 0));
          code.push_back(EncodeVintrp(1, 20 + component, 0, component, 1));
        }
        AppendVMovU32(&code, 22, 0);
        AppendVMovLiteral(&code, 23, 0x3f800000u);
        code.insert(code.end(), {EncodeExp0(0, 0xf), EncodeExp1(20, 21, 22, 23)});
        AppendEnd(&code);
        return code;
      }();
      const auto rect_vs = reinterpret_cast<uint64_t>(rect_vertex_code.data());
      const auto rect_ps = reinterpret_cast<uint64_t>(rect_pixel_code.data());
      ShaderMapUserData(rect_vs,
          {.type = Prospero::ShaderBinaryType::kGs, .user_data = &native_user_data,
           .code_size_bytes = static_cast<u32>(rect_vertex_code.size() * sizeof(u32))});
      ShaderMapUserData(rect_ps,
          {.type = Prospero::ShaderBinaryType::kPs,
           .code_size_bytes = static_cast<u32>(rect_pixel_code.size() * sizeof(u32))});
      shaders.SetEsShaderBase(rect_vs);
      shaders.SetPsShaderBase(rect_ps);
      shaders.SetGsShaderResource2({.user_sgpr = 4});
      for (u32 i = 0; i < 4; i++) {
        shaders.SetGsUserSgpr(i, rect_buffer.fields[i], HW::UserSgprType::Vsharp);
      }
      registers.SetPsInControl(0x8001);
      registers.SetPsInputEna(2);
      registers.SetPsInputAddr(2);
      registers.SetPsInputSettings(0, 0);
      for (const auto primitive : {Prospero::PrimitiveType::kRectList,
                                    Prospero::PrimitiveType::kRectListLegacy}) {
        const char *rect_name = primitive == Prospero::PrimitiveType::kRectList
                                    ? "RectListThreeRecords" : "LegacyRectListThreeRecords";
        user_config.SetPrimitiveType(primitive);
        registers.SetModeControl({});
        TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), sparse_color.image_id,
            {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
        RenderExecutorTestAccess::DrawAuto(
            executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
        const auto pixels = ReadCachedTexel(name, context, sparse_color.image_id,
                                            {}, {extent, extent, 1});
        for (u32 y = 0; y < extent; y++) {
          for (u32 x = 0; x < extent; x++) {
            const bool inside = x >= 4 && x < 28 && y >= 4 && y < 28;
            const auto offset = 4 * (y * extent + x);
            const std::array<float, 4> expected{
                inside ? (x + 0.5f - 4) / 24 : 0,
                inside ? (y + 0.5f - 4) / 24 : 0, 0, inside ? 1.0f : 0};
            for (u32 component = 0; component < 4; component++) {
              Require(rect_name, "three-vertex rectangle coverage and UV",
                  std::abs(std::bit_cast<float>(pixels[offset + component]) -
                           expected[component]) < 0.0001f,
                  "rectangle reconstruction lost its fourth corner or interpolated UVs");
            }
          }
        }
      }
      DestroyBuffer(&stencil_readback);

      vertex_shader = owned_vertex_shader;
      pixel_shader = owned_pixel_shader;
    }
    resources.UnmapMemory(depth_address, allocation_size);
    scheduler.Finish();
    DestroyBuffer(&buffer);
    m_device.destroyShaderModule(pixel_shader.module);
    m_device.destroyShaderModule(vertex_shader.module);
    Require(name, "release depth memory",
              Libs::LibKernel::Memory::KernelMunmap(depth_address, allocation_size) == 0 &&
                  Libs::LibKernel::Memory::KernelReleaseDirectMemory(
                      direct_offset, allocation_size) == 0,
              "depth mapping or allocation release failed");
    std::printf("[gpu]     %-32s ok\n", name);
  }

  std::vector<u32> RenderFragment(const GraphicsCase &test,
                                  const CompiledShader &fragment) {
    const auto vertex_spirv =
        TestSpv::MakePassthroughVertexSpirv(test.layers > 1, test.vertex_clip_w);
    ValidateSpirv(test.name, vertex_spirv);

    Image target =
        CreateImageMips(test.name, 1, 1, vk::Format::eR32G32B32A32Sfloat,
                      vk::ImageUsageFlagBits::eColorAttachment, {}, 4,
                      vk::ImageLayout::eGeneral, vk::ImageType::e2D,
                      test.layers > 1 ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D,
                      test.layers, 0, 0, test.samples);
    Image resolved;
    if (test.samples != vk::SampleCountFlagBits::e1) {
      resolved = CreateImageMips(
          test.name, 1, 1, target.format, vk::ImageUsageFlagBits::eColorAttachment,
          {}, 4, vk::ImageLayout::eGeneral, vk::ImageType::e2D,
          test.layers > 1 ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D,
          test.layers);
    }
    const std::vector<u32> default_vertices = {
        0xbf800000u, 0xbf800000u, 0x3e800000u, 0x3f000000u, 0x3f400000u,
        0x3f800000u, 0x40400000u, 0xbf800000u, 0x3e800000u, 0x3f000000u,
        0x3f400000u, 0x3f800000u, 0xbf800000u, 0x40400000u, 0x3e800000u,
        0x3f000000u, 0x3f400000u, 0x3f800000u,
    };
    const auto &vertices =
        test.vertices.empty() ? default_vertices : test.vertices;
    Require(test.name, "graphics", vertices.size() == 18u,
            "graphics vertex buffer must contain three pos2/color4 vertices");
    auto vertex_buffer =
        CreateHostBuffer(test.name, vertices.size() * sizeof(u32),
                         vk::BufferUsageFlagBits::eVertexBuffer, vertices);

    vk::ShaderModule vertex_module = CreateShaderModule(test.name, vertex_spirv);
    vk::ShaderModule fragment_module = CreateShaderModule(test.name, fragment.spirv);

    const auto &fragment_bind = fragment.program.bindings;
    vk::PushConstantRange push_constant_range{};
    if (fragment_bind.UsesPushData()) {
      Require(test.name, "graphics",
              test.push_constants.size() * sizeof(u32) ==
                  fragment_bind.ShaderDataDwords() * sizeof(u32),
              "fragment push constant data size does not match reflection");
      push_constant_range.stageFlags = vk::ShaderStageFlagBits::eFragment;
      push_constant_range.offset = 0;
      push_constant_range.size = ShaderRecompiler::IR::NativePushConstantSize;
    }

    vk::PipelineLayoutCreateInfo pipeline_layout_info{};
    pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
    pipeline_layout_info.pushConstantRangeCount =
        push_constant_range.size != 0 ? 1u : 0u;
    pipeline_layout_info.pPushConstantRanges =
        push_constant_range.size != 0 ? &push_constant_range : nullptr;
    vk::PipelineLayout pipeline_layout = nullptr;
    RequireVk(test.name, "graphics",
              m_device.createPipelineLayout(&pipeline_layout_info, nullptr,
                                            &pipeline_layout),
              "vkCreatePipelineLayout");

    vk::PipelineShaderStageCreateInfo stages[2] = {};
    stages[0].sType = vk::StructureType::ePipelineShaderStageCreateInfo;
    stages[0].stage = vk::ShaderStageFlagBits::eVertex;
    stages[0].module = vertex_module;
    stages[0].pName = "main";
    stages[1].sType = vk::StructureType::ePipelineShaderStageCreateInfo;
    stages[1].stage = vk::ShaderStageFlagBits::eFragment;
    stages[1].module = fragment_module;
    stages[1].pName = "main";

    vk::VertexInputBindingDescription vertex_binding{};
    vertex_binding.binding = 0;
    vertex_binding.stride = 6u * sizeof(float);
    vertex_binding.inputRate = vk::VertexInputRate::eVertex;
    vk::VertexInputAttributeDescription attributes[2] = {};
    attributes[0].location = 0;
    attributes[0].binding = 0;
    attributes[0].format = vk::Format::eR32G32Sfloat;
    attributes[0].offset = 0;
    attributes[1].location = 1;
    attributes[1].binding = 0;
    attributes[1].format = vk::Format::eR32G32B32A32Sfloat;
    attributes[1].offset = 2u * sizeof(float);
    vk::PipelineVertexInputStateCreateInfo vertex_input{};
    vertex_input.sType = vk::StructureType::ePipelineVertexInputStateCreateInfo;
    vertex_input.vertexBindingDescriptionCount = 1;
    vertex_input.pVertexBindingDescriptions = &vertex_binding;
    vertex_input.vertexAttributeDescriptionCount = 2;
    vertex_input.pVertexAttributeDescriptions = attributes;

    vk::PipelineInputAssemblyStateCreateInfo input_assembly{};
    input_assembly.sType =
        vk::StructureType::ePipelineInputAssemblyStateCreateInfo;
    input_assembly.topology = vk::PrimitiveTopology::eTriangleList;

    vk::Viewport viewport{};
    viewport.x = 0.0f;
    viewport.y = 0.0f;
    viewport.width = 1.0f;
    viewport.height = 1.0f;
    viewport.minDepth = 0.0f;
    viewport.maxDepth = 1.0f;
    vk::Rect2D scissor{};
    scissor.extent.width = 1;
    scissor.extent.height = 1;
    vk::PipelineViewportStateCreateInfo viewport_state{};
    viewport_state.sType = vk::StructureType::ePipelineViewportStateCreateInfo;
    viewport_state.viewportCount = 1;
    viewport_state.pViewports = &viewport;
    viewport_state.scissorCount = 1;
    viewport_state.pScissors = &scissor;

    vk::PipelineRasterizationStateCreateInfo raster{};
    raster.sType = vk::StructureType::ePipelineRasterizationStateCreateInfo;
    raster.polygonMode = vk::PolygonMode::eFill;
    raster.cullMode = vk::CullModeFlagBits::eNone;
    raster.frontFace = vk::FrontFace::eCounterClockwise;
    raster.lineWidth = 1.0f;

    vk::PipelineRasterizationProvokingVertexStateCreateInfoEXT provoking_vertex{};
    if (test.provoking_vtx_last) {
      provoking_vertex.provokingVertexMode = vk::ProvokingVertexModeEXT::eLastVertex;
      raster.pNext = &provoking_vertex;
    }

    vk::PipelineMultisampleStateCreateInfo multisample{};
    multisample.sType = vk::StructureType::ePipelineMultisampleStateCreateInfo;
    multisample.rasterizationSamples = test.samples;

    vk::PipelineColorBlendAttachmentState color_attachment{};
    color_attachment.colorWriteMask =
        vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG |
        vk::ColorComponentFlagBits::eB | vk::ColorComponentFlagBits::eA;
    vk::PipelineColorBlendStateCreateInfo color_blend{};
    color_blend.sType = vk::StructureType::ePipelineColorBlendStateCreateInfo;
    color_blend.attachmentCount = 1;
    color_blend.pAttachments = &color_attachment;

    vk::GraphicsPipelineCreateInfo pipeline_info{};
    const vk::Format color_format = vk::Format::eR32G32B32A32Sfloat;
    vk::PipelineRenderingCreateInfo rendering_pipeline{};
    rendering_pipeline.sType = vk::StructureType::ePipelineRenderingCreateInfo;
    rendering_pipeline.colorAttachmentCount = 1;
    rendering_pipeline.pColorAttachmentFormats = &color_format;
    pipeline_info.sType = vk::StructureType::eGraphicsPipelineCreateInfo;
    pipeline_info.pNext = &rendering_pipeline;
    pipeline_info.stageCount = 2;
    pipeline_info.pStages = stages;
    pipeline_info.pVertexInputState = &vertex_input;
    pipeline_info.pInputAssemblyState = &input_assembly;
    pipeline_info.pViewportState = &viewport_state;
    pipeline_info.pRasterizationState = &raster;
    pipeline_info.pMultisampleState = &multisample;
    pipeline_info.pColorBlendState = &color_blend;
    pipeline_info.layout = pipeline_layout;
    vk::Pipeline pipeline = nullptr;
    RequireVk(test.name, "graphics",
              m_device.createGraphicsPipelines(nullptr, 1, &pipeline_info,
                                               nullptr, &pipeline),
              "vkCreateGraphicsPipelines");

    vk::CommandBuffer cmd = BeginCommands(test.name, "graphics");
    vk::RenderingAttachmentInfo color{};
    color.sType = vk::StructureType::eRenderingAttachmentInfo;
    color.imageView = target.view;
    color.imageLayout = vk::ImageLayout::eGeneral;
    color.loadOp = vk::AttachmentLoadOp::eClear;
    color.storeOp = vk::AttachmentStoreOp::eStore;
    if (resolved.image != nullptr) {
      color.resolveMode = vk::ResolveModeFlagBits::eAverage;
      color.resolveImageView = resolved.view;
      color.resolveImageLayout = vk::ImageLayout::eGeneral;
    }
    vk::RenderingInfo rendering{};
    rendering.sType = vk::StructureType::eRenderingInfo;
    rendering.renderArea.extent = {1, 1};
    rendering.layerCount = test.layers;
    rendering.colorAttachmentCount = 1;
    rendering.pColorAttachments = &color;
    cmd.beginRendering(rendering);
    cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline);
    if (push_constant_range.size != 0) {
      ShaderRecompiler::IR::PushData push_data;
      std::copy(test.push_constants.begin(), test.push_constants.end(),
                push_data.dwords.begin() + fragment_bind.push_data_start_dword);
      cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eFragment, 0,
                        sizeof(push_data), push_data.dwords.data());
    }
    vk::DeviceSize offset = 0;
    cmd.bindVertexBuffers(0, 1, &vertex_buffer.buffer, &offset);
    cmd.draw(3, test.layers, 0, 0);
    cmd.endRendering();
    EndSubmitAndFree(test.name, "graphics", cmd);
    target.layout = vk::ImageLayout::eGeneral;

    auto pixel = ReadImage(test.name, resolved.image != nullptr ? &resolved : &target);
    pixel.resize(4 * test.layers);

    m_device.destroyPipeline(pipeline, nullptr);
    m_device.destroyPipelineLayout(pipeline_layout, nullptr);
    m_device.destroyShaderModule(fragment_module, nullptr);
    m_device.destroyShaderModule(vertex_module, nullptr);
    DestroyBuffer(&vertex_buffer);
    DestroyImage(&resolved);
    DestroyImage(&target);
    return pixel;
  }

  void CheckGpuTilerCpuParity() {
    constexpr const char *name = "GpuTilerCpuParity";
    EnsureRuntimeContext();

    CommandScheduler scheduler(Renderer(), m_runtime_context);
    HW::Context registers{};
    HW::UserConfig user_config{};
    HW::Shader shaders{};
    scheduler.Begin(registers, user_config, shaders);
    StreamBuffer parameters(m_runtime_context, scheduler, MemoryUsage::Stream,
                            1u << 20);
    TileManager tile_manager(m_runtime_context, scheduler, parameters);
    constexpr uint64_t conversion_limit = 128ull << 20;
    Require(name, "D16 conversion chunk boundaries",
            TileManagerTestAccess::ConversionRows(0, 32ull << 10, 32ull << 10,
                                                  4096, 256, conversion_limit,
                                                  65535) == 4096 &&
                TileManagerTestAccess::ConversionRows(
                    0, 32ull << 10, 32ull << 10, 4097, 256, conversion_limit,
                    65535) == 4096 &&
                TileManagerTestAccess::ConversionRows(
                    257, 32ull << 10, 32ull << 10, 4097, 256, conversion_limit,
                    65535) == 4095,
            "D16 conversion planner exceeds descriptor or dispatch limits");

    const auto align_dword = [](uint64_t size) {
      return (size + sizeof(u32) - 1u) & ~(uint64_t{sizeof(u32) - 1u});
    };
    const auto to_dwords = [&](const std::vector<uint8_t> &bytes,
                               uint64_t padded_size) {
      std::vector<u32> words(static_cast<size_t>(padded_size / sizeof(u32)), 0);
      std::memcpy(words.data(), bytes.data(), bytes.size());
      return words;
    };
    const auto read_bytes = [&](const Buffer &buffer, uint64_t size) {
      const auto words = ReadBuffer(
          name, buffer, static_cast<size_t>(align_dword(size) / sizeof(u32)));
      std::vector<uint8_t> bytes(static_cast<size_t>(size));
      std::memcpy(bytes.data(), words.data(), bytes.size());
      return bytes;
    };
    const auto host_barrier = [&](vk::Buffer buffer, uint64_t size,
                                  vk::PipelineStageFlags source_stage,
                                  vk::AccessFlags source_access) {
      vk::BufferMemoryBarrier barrier{};
      barrier.sType = vk::StructureType::eBufferMemoryBarrier;
      barrier.srcAccessMask = source_access;
      barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
      barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
      barrier.buffer = buffer;
      barrier.offset = 0;
      barrier.size = size;
      scheduler.Current().Handle().pipelineBarrier(
          source_stage, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
          &barrier, 0, nullptr);
    };
    auto scratch_input = CreateHostBuffer(name, 32, AllFlags,
        {0x00020001u, 0x00040003u, 0x00060005u, 0x00080007u,
         0x000a0009u, 0x000c000bu, 0x000e000du, 0x0010000fu});
    auto scratch_output = CreateHostBuffer(name, 56, AllFlags, {});
    const auto copy_scratch = [&](TileManager::Result result, uint64_t offset) {
      const vk::BufferCopy copy{result.offset, offset, result.size};
      scheduler.Current().Handle().copyBuffer(result.buffer, scratch_output.buffer, 1, &copy);
    };
    // Keep all four conversions in flight to check growth, overwrite ordering,
    // the requested span on reuse, and two simultaneously live workspaces.
    const auto first = tile_manager.TransformColor(
        {scratch_input.buffer, 0, 8}, ColorTransform::SwapBgra16, true);
    copy_scratch(first, 0);
    const auto grown = tile_manager.TransformColor(
        {scratch_input.buffer, 0, 32}, ColorTransform::SwapBgra16, true);
    copy_scratch(grown, 8);
    const auto reused = tile_manager.TransformColor(
        {scratch_input.buffer, 8, 8}, ColorTransform::SwapBgra16, true);
    copy_scratch(reused, 40);
    const auto paired = tile_manager.TransformColor(reused, ColorTransform::SwapBgra16, true);
    copy_scratch(paired, 48);
    Require(name, "scratch workspace reuse",
            first.buffer != grown.buffer && grown.buffer == reused.buffer &&
                reused.buffer != paired.buffer && reused.size == 8 && paired.size == 8,
            "scratch growth, input exclusion, or requested span was lost");
    host_barrier(scratch_output.buffer, scratch_output.size,
                    vk::PipelineStageFlagBits::eTransfer, vk::AccessFlagBits::eTransferWrite);
    scheduler.Finish();
    Require(name, "scratch workspace GPU contents",
            ReadBuffer(name, scratch_output, 14) == std::vector<u32>{
                0x00020003u, 0x00040001u,
                0x00020003u, 0x00040001u, 0x00060007u, 0x00080005u,
                0x000a000bu, 0x000c0009u, 0x000e000fu, 0x0010000du,
                0x00060007u, 0x00080005u, 0x00060005u, 0x00080007u},
            "scratch reuse overwrote an earlier result or its live conversion input");
    DestroyBuffer(&scratch_output);
    DestroyBuffer(&scratch_input);
    const auto gpu_detile =
        [&](const std::vector<uint8_t> &tiled, std::vector<uint8_t> *linear,
            uint64_t tiled_capacity, uint64_t linear_capacity,
            const ImageInfo &info, TileManager *manager = nullptr,
            uint64_t source_offset = 0) {
          const uint64_t padded_tiled = align_dword(tiled_capacity);
          const uint64_t padded_linear = align_dword(linear_capacity);
          std::vector<uint8_t> source(source_offset + padded_tiled, 0x7b);
          std::copy(tiled.begin(), tiled.end(), source.begin() + source_offset);
          auto input = CreateHostBuffer(name, source.size(), AllFlags,
                                        to_dwords(source, source.size()));
          auto output = CreateHostBuffer(
              name, padded_linear, AllFlags,
              std::vector<u32>(static_cast<size_t>(padded_linear / sizeof(u32)),
                               0));
          const auto result = (manager ? *manager : tile_manager)
                                  .Detile(input.buffer, source_offset, info);
          Require(name, "detile capacity", result.size <= padded_linear,
                  "canonical transfer size exceeded the fixture output");
          const vk::BufferCopy copy{result.offset, 0, align_dword(result.size)};
          scheduler.Current().Handle().copyBuffer(result.buffer, output.buffer,
                                                  1, &copy);
          host_barrier(output.buffer, padded_linear,
                       vk::PipelineStageFlagBits::eTransfer,
                       vk::AccessFlagBits::eTransferWrite);
          scheduler.Finish();
          *linear = read_bytes(output, linear_capacity);
          DestroyBuffer(&output);
          DestroyBuffer(&input);
        };
    const auto gpu_tile = [&](const std::vector<uint8_t> &linear,
                              std::vector<uint8_t> *tiled,
                              uint64_t tiled_capacity, uint64_t linear_capacity,
                              const ImageInfo &info,
                              TileManager *manager = nullptr,
                              uint32_t levels = 0, uint64_t source_offset = 0,
                              uint64_t target_offset = 0) {
      const uint64_t padded_tiled = align_dword(tiled_capacity);
      const uint64_t padded_linear = align_dword(linear_capacity);
      std::vector<uint8_t> source(source_offset + padded_linear, 0x7b);
      std::copy(linear.begin(), linear.end(), source.begin() + source_offset);
      auto input = CreateHostBuffer(name, source.size(), AllFlags,
                                    to_dwords(source, source.size()));
      auto output = CreateHostBuffer(
          name, target_offset + padded_tiled, AllFlags,
          std::vector<u32>(
              static_cast<size_t>((target_offset + padded_tiled) / sizeof(u32)),
              0xabababab));
      (manager ? *manager : tile_manager)
          .Tile(input.buffer, source_offset, padded_linear, output.buffer,
                target_offset, padded_tiled, info, levels);
      host_barrier(output.buffer, target_offset + padded_tiled,
                   vk::PipelineStageFlagBits::eComputeShader,
                   vk::AccessFlagBits::eShaderWrite);
      scheduler.Finish();
      const auto observed = read_bytes(output, target_offset + tiled_capacity);
      Require(name, "nonzero transfer target base",
              std::all_of(observed.begin(), observed.begin() + target_offset,
                          [](uint8_t value) { return value == 0xab; }),
              "tiling changed bytes preceding its target offset");
      tiled->assign(observed.begin() + target_offset, observed.end());
      DestroyBuffer(&output);
      DestroyBuffer(&input);
    };

    for (const bool d32 : std::array{true, false}) {
      constexpr uint32_t width = 5;
      constexpr uint32_t height = 3;
      constexpr uint32_t pitch = 8;
      constexpr uint32_t layers = 2;
      constexpr uint64_t d16_row = pitch * sizeof(uint16_t);
      constexpr uint64_t d32_row = pitch * sizeof(uint32_t);
      constexpr uint64_t d16_slice = d16_row * height;
      constexpr uint64_t d32_slice = d32_row * height;
      constexpr uint64_t d16_size = d16_slice * layers;
      constexpr uint64_t d32_size = d32_slice * layers;
      StreamBuffer conversion_upload(m_runtime_context, scheduler,
                                     MemoryUsage::Upload, 1u << 20);
      auto [prefix, prefix_offset] = conversion_upload.Map(4096, 256);
      Require(name, "D16 conversion prefix",
              prefix != nullptr && prefix_offset == 0,
              "failed to advance the D16 conversion upload ring");
      std::memset(prefix, 0x7b, 4096);
      conversion_upload.Commit();
      auto [source_bytes, source_offset] = conversion_upload.Map(d16_size, 4);
      Require(name, "D16 conversion source",
              source_bytes != nullptr && source_offset != 0,
              "failed to reserve a nonzero D16 conversion source");
      std::memset(source_bytes, 0x3c, d16_size);
      for (uint32_t layer = 0; layer < layers; layer++) {
        for (uint32_t y = 0; y < height; y++) {
          auto *row = reinterpret_cast<uint16_t *>(
              source_bytes + layer * d16_slice + y * d16_row);
          for (uint32_t x = 0; x < width; x++) {
            row[x] = static_cast<uint16_t>(
                0x1111u * (1u + x + y * width + layer * width * height));
          }
        }
      }
      conversion_upload.Commit();
      auto promoted = CreateHostBuffer(
          name, d32_size, AllFlags,
          std::vector<u32>(static_cast<size_t>(d32_size / sizeof(u32)), 0));
      auto round_trip = CreateHostBuffer(
          name, d16_size, AllFlags,
          std::vector<u32>(static_cast<size_t>(d16_size / sizeof(u32)),
                           0xa5a5a5a5u));
      const TileManager::D16Layout promote_layout{
          .width = width,
          .height = height,
          .layers = layers,
          .source_row_stride = d16_row,
          .target_row_stride = d32_row,
          .source_slice_stride = d16_slice,
          .target_slice_stride = d32_slice,
      };
      tile_manager.ConvertD16({conversion_upload.Handle(), source_offset,
                               conversion_upload.Size() - source_offset},
                              {promoted.buffer, 0, d32_size},
                              TileManager::D16Direction::Promote, d32,
                              promote_layout);
      tile_manager.ConvertD16({promoted.buffer, 0, d32_size},
                              {round_trip.buffer, 0, d16_size},
                              TileManager::D16Direction::Demote, d32,
                              {.width = width,
                               .height = height,
                               .layers = layers,
                               .source_row_stride = d32_row,
                               .target_row_stride = d16_row,
                               .source_slice_stride = d32_slice,
                               .target_slice_stride = d16_slice});
      host_barrier(round_trip.buffer, d16_size,
                   vk::PipelineStageFlagBits::eComputeShader,
                   vk::AccessFlagBits::eShaderWrite);
      scheduler.Finish();
      const auto result = read_bytes(round_trip, d16_size);
      bool conversion_matches = true;
      for (uint32_t layer = 0; layer < layers; layer++) {
        for (uint32_t y = 0; y < height; y++) {
          for (uint32_t x = 0; x < pitch; x++) {
            const auto offset =
                layer * d16_slice + y * d16_row + x * sizeof(uint16_t);
            conversion_matches &=
                x < width
                    ? std::memcmp(result.data() + offset, source_bytes + offset,
                                  sizeof(uint16_t)) == 0
                    : result[offset] == 0xa5 && result[offset + 1] == 0xa5;
          }
        }
      }
      Require(name,
              d32 ? "D32 conversion round trip" : "D24 conversion round trip",
              conversion_matches,
              "D16 conversion lost rows, layers, active values, or padding");
      DestroyBuffer(&round_trip);
      DestroyBuffer(&promoted);
    }

    auto fill = [](std::vector<uint8_t> *bytes, u32 salt) {
      for (size_t i = 0; i < bytes->size(); i++) {
        uint64_t value =
            i + static_cast<uint64_t>(salt) * 0x9e3779b97f4a7c15ull;
        value = (value ^ (value >> 30u)) * 0xbf58476d1ce4e5b9ull;
        value = (value ^ (value >> 27u)) * 0x94d049bb133111ebull;
        (*bytes)[i] = static_cast<uint8_t>((value ^ (value >> 31u)) >> 56u);
      }
    };
    auto compare = [&](const char *stage, const std::vector<uint8_t> &expected,
                       const std::vector<uint8_t> &actual) {
      if (expected != actual) {
        const auto mismatch = static_cast<size_t>(
            std::mismatch(expected.begin(), expected.end(), actual.begin())
                .first -
            expected.begin());
        std::ostringstream out;
        out << "first mismatch at " << mismatch << " of " << expected.size();
        Fail(name, stage, out.str());
      }
    };
    auto convert_reference = [&](bool to_tiled, std::vector<uint8_t> *dst,
                                 const std::vector<uint8_t> &src,
                                 const ImageInfo &info, uint32_t level = 0) {
      const auto &block = info.tiling.block;
      const auto &mip = info.mip_layout[level];
      const auto active = info.MipExtent(level);
      const uint32_t width = (active.width + info.tiling.texel_width - 1u) /
                             info.tiling.texel_width;
      const uint32_t height = (active.height + info.tiling.texel_height - 1u) /
                              info.tiling.texel_height;
      const uint64_t columns =
          (mip.pitch + block.block_width - 1u) / block.block_width;
      const uint64_t slice = info.linear_slice_stride != 0
                                 ? info.linear_slice_stride
                                 : mip.linear_size;
      const bool tail = level >= info.first_tail_level;
      for (u32 z = 0; z < active.depth; ++z) {
        for (u32 y = 0; y < height; ++y) {
          for (u32 x = 0; x < width; ++x) {
            const u32 bx = tail ? 0 : x / block.block_width;
            const u32 by = tail ? 0 : y / block.block_height;
            const u32 bz = z / block.block_depth;
            const u32 lx = tail ? x + mip.tail_x : x % block.block_width;
            const u32 ly = tail ? y + mip.tail_y : y % block.block_height;
            const u32 lz = z % block.block_depth;
            u32 local = 0, block_xor = 0;
            Require(name, "reference offset",
                    TileGetBlockOffset(block, lx, ly, lz, local) &&
                        TileGetBlockXor(block, bx, by, mip.surface_z + bz,
                                        block_xor),
                    "CPU reference address lookup failed");
            const uint64_t tiled =
                mip.offset + uint64_t{bz} * info.tiled_slice_stride +
                (by * columns + bx) * block.block_size + (local ^ block_xor);
            const uint64_t linear =
                mip.linear_offset + uint64_t{z} * slice +
                (uint64_t{y} * mip.linear_pitch + x) * block.bytes_per_element;
            const uint64_t dst_offset = to_tiled ? tiled : linear;
            const uint64_t src_offset = to_tiled ? linear : tiled;
            Require(name, "reference range",
                    dst_offset + block.bytes_per_element <= dst->size() &&
                        src_offset + block.bytes_per_element <= src.size(),
                    "CPU reference address escaped storage");
            if (info.GetColorTransform() == ColorTransform::Reverse10_11_11) {
              uint32_t packed = 0;
              std::memcpy(&packed, src.data() + src_offset, sizeof(packed));
              const std::array<uint32_t, 3> widths =
                  to_tiled ? std::array<uint32_t, 3>{11, 11, 10}
                           : std::array<uint32_t, 3>{10, 11, 11};
              std::array<uint32_t, 3> channels{};
              for (uint32_t channel = 0; channel < channels.size(); ++channel) {
                channels[channel] = packed & ((1u << widths[channel]) - 1u);
                packed >>= widths[channel];
              }
              uint32_t shift = 0;
              for (int channel = 2; channel >= 0; --channel) {
                packed |= channels[channel] << shift;
                shift += widths[channel];
              }
              std::memcpy(dst->data() + dst_offset, &packed, sizeof(packed));
            } else if (info.GetColorTransform() == ColorTransform::SwapBgra16) {
              std::array<uint16_t, 4> channels{};
              std::memcpy(channels.data(), src.data() + src_offset,
                          sizeof(channels));
              std::swap(channels[0], channels[2]);
              std::memcpy(dst->data() + dst_offset, channels.data(),
                          sizeof(channels));
            } else {
              std::memcpy(dst->data() + dst_offset, src.data() + src_offset,
                          block.bytes_per_element);
            }
          }
        }
      }
    };
    const auto block_image = [](const TileBlockLayout &block, u32 width,
                                u32 height, u32 depth, u32 pitch,
                                uint64_t guest_size) {
      ImageInfo info{};
      info.extent = {width, height, block.block_depth > 1 ? depth : 1u};
      info.type = block.block_depth > 1 ? Prospero::ImageType::kColor3D
                                        : Prospero::ImageType::kColor2D;
      info.resources = {1, block.block_depth > 1 ? 1u : depth};
      info.tile_mode = Prospero::TileMode::kStandard4KB;
      info.bytes_per_block = block.bytes_per_element;
      info.tiling = {block, 1, 1};
      info.data.size = guest_size;
      info.first_tail_level = 16;
      const uint64_t columns =
          (pitch + block.block_width - 1u) / block.block_width;
      const uint64_t rows =
          (height + block.block_height - 1u) / block.block_height;
      info.tiled_slice_stride = columns * rows * block.block_size;
      info.mip_layout[0] = {
          0, info.IsVolume() ? info.tiled_slice_stride : guest_size, pitch,
          static_cast<uint32_t>(rows * block.block_height)};
      info.mip_layout[0].linear_pitch = pitch;
      info.mip_layout[0].linear_size =
          uint64_t{pitch} * height * block.bytes_per_element;
      return info;
    };
    struct FamilyCase {
      TileBlockFamily family;
      u32 max_bpe;
    };
    constexpr FamilyCase families[] = {
        {TileBlockFamily::Standard256B, 16},
        {TileBlockFamily::Standard4KB, 16},
        {TileBlockFamily::Standard4KB3D, 16},
        {TileBlockFamily::Standard64KB, 16},
        {TileBlockFamily::Standard64KB3D, 16},
        {TileBlockFamily::Prt64KB, 16},
        {TileBlockFamily::Prt64KB3D, 16},
        {TileBlockFamily::RenderTarget64KB, 16},
        {TileBlockFamily::Depth64KB, 8},
    };

    {
      TileBlockLayout standard{}, prt{}, standard_3d{}, prt_3d{}, color{},
          depth{};
      u32 standard_offset = 0, prt_offset = 0, standard_3d_offset = 0,
          prt_3d_offset = 0, color_z = 0, depth_z = 0;
      bool fixed_addresses =
          TileGetBlockLayout(TileBlockFamily::Standard64KB, 4, standard) &&
          TileGetBlockLayout(TileBlockFamily::Prt64KB, 4, prt) &&
          TileGetBlockLayout(TileBlockFamily::Standard64KB3D, 4, standard_3d) &&
          TileGetBlockLayout(TileBlockFamily::Prt64KB3D, 4, prt_3d) &&
          TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, 4, color) &&
          TileGetBlockLayout(TileBlockFamily::Depth64KB, 4, depth) &&
          TileGetBlockOffset(standard, 64, 0, 0, standard_offset) &&
          TileGetBlockOffset(prt, 64, 0, 0, prt_offset) &&
          TileGetBlockOffset(standard_3d, 16, 0, 0, standard_3d_offset) &&
          TileGetBlockOffset(prt_3d, 16, 0, 0, prt_3d_offset) &&
          TileGetBlockXor(color, 0, 0, 1, color_z) &&
          TileGetBlockXor(depth, 0, 0, 15, depth_z);
      Require(name, "fixed address vectors",
              fixed_addresses && standard_offset == 0x8000 &&
                  prt_offset == 0x8100 && standard_3d_offset == 0x8000 &&
                  prt_3d_offset == 0x8400 && standard_3d.block_width == 32 &&
                  standard_3d.block_height == 32 &&
                  standard_3d.block_depth == 16 && color_z == 0x800 &&
                  depth_z == 0xf00,
              "a fixed block address changed");

      constexpr auto format = Prospero::BufferFormat::k32Float;
      constexpr u32 levels = 7;
      TileSizeAlign total{};
      TileSizeOffset mip[levels]{};
      TilePaddedSize padded[levels]{};
      TileGetTextureSize(format, 65, 33, levels,
                         Prospero::TileMode::kStandard256B, &total, mip,
                         padded);
      constexpr u32 offsets[levels] = {0x1a00, 0xb00, 0x500, 0x300,
                                       0x200,  0x100, 0};
      constexpr u32 sizes[levels] = {0x2d00, 0xf00, 0x600, 0x200,
                                     0x100,  0x100, 0x100};
      bool layout_matches = total.size == 0x4700 && total.align == 0x100 &&
                            padded[0].width == 72 && padded[0].height == 40;
      for (u32 level = 0; level < levels; ++level) {
        layout_matches &= mip[level].offset == offsets[level] &&
                          mip[level].size == sizes[level];
      }
      Require(name, "fixed mip vector", layout_matches,
              "the reverse-packed small-block mip layout changed");
    }

    u32 case_index = 0;
    auto check_round_trip = [&](const char *stage, uint64_t tiled_size,
                                const ImageInfo &info,
                                TileManager *manager = nullptr) {
      const uint64_t linear_size = info.LinearSize();
      std::vector<uint8_t> tiled(tiled_size);
      std::vector<uint8_t> cpu(linear_size, 0);
      std::vector<uint8_t> gpu(linear_size, 0xab);
      fill(&tiled, ++case_index);
      for (uint32_t level = 0; level < info.resources.levels; ++level) {
        convert_reference(false, &cpu, tiled, info, level);
      }
      gpu_detile(tiled, &gpu, tiled_size, linear_size, info, manager);
      compare((std::string(stage) + " detile bytes").c_str(), cpu, gpu);
      std::vector<uint8_t> linear(linear_size);
      std::vector<uint8_t> cpu_tiled(tiled_size, 0xab);
      std::vector<uint8_t> gpu_tiled(tiled_size, 0xab);
      fill(&linear, 0x280u + case_index);
      for (uint32_t level = 0; level < info.resources.levels; ++level) {
        convert_reference(true, &cpu_tiled, linear, info, level);
      }
      gpu_tile(linear, &gpu_tiled, tiled_size, linear_size, info, manager);
      compare((std::string(stage) + " tile bytes").c_str(), cpu_tiled,
              gpu_tiled);
    };

    {
      constexpr auto format = Prospero::BufferFormat::k8UNorm;
      constexpr auto tile = Prospero::TileMode::kStandard256B;
      constexpr u32 layers = 261;
      const auto info =
          MakeTilingImage(format, 1, 1, 1, layers, tile, 0, false);
      const auto regions = info.BufferCopies();
      Require(name, "large array mip record",
              regions.size() == 1 &&
                  regions[0].imageSubresource.layerCount == layers &&
                  info.MipExtent(0).depth == layers &&
                  info.tiled_slice_stride == 256 &&
                  info.data.size == uint64_t{layers} * 256,
              "array layers expanded the mip table or lost their guest stride");
      check_round_trip("large array", info.data.size, info);
    }
    {
      // Adjacent subword mips share output dwords and a guest tail block.
      TileBlockLayout block{};
      Require(name, "packed mip block",
              TileGetBlockLayout(TileBlockFamily::Standard4KB, 1, block),
              "packed mip block is unavailable");
      auto info = block_image(block, 1, 1, 1, 1, 4096);
      info.resources.levels = 16;
      info.first_tail_level = 0;
      for (u32 mip = 0; mip < info.resources.levels; ++mip) {
        info.mip_layout[mip] = {0, 4096, block.block_width, block.block_height};
        info.mip_layout[mip].linear_offset = mip;
        info.mip_layout[mip].linear_size = 1;
        info.mip_layout[mip].linear_pitch = 1;
        info.mip_layout[mip].tail_x = mip;
      }
      check_round_trip("16 packed subword mip records", 4096, info);
      const auto atom =
          std::max<uint64_t>(m_runtime_context.GetPhysicalDeviceProperties()
                                 .limits.nonCoherentAtomSize,
                             1);
      for (const auto [mips, bytes] :
           std::array{std::pair{1u, 64u}, std::pair{2u, 112u}}) {
        StreamBuffer compact_parameters(m_runtime_context, scheduler,
                                        MemoryUsage::Stream,
                                        Common::AlignUp<uint64_t>(bytes, atom));
        TileManager compact_tiler(m_runtime_context, scheduler,
                                  compact_parameters);
        auto compact_info = info;
        compact_info.resources.levels = mips;
        check_round_trip("compact parameter buffer", 4096, compact_info,
                         &compact_tiler);
      }
    }
    {
      constexpr auto format = Prospero::BufferFormat::k32Float;
      constexpr auto tile = Prospero::TileMode::kStandard256B;
      constexpr u32 levels = 7, layers = 3;
      constexpr uint64_t guest_stride = 0x4800,
                         guest_size = guest_stride * layers;
      constexpr uint64_t offsets[levels] = {0x1a00, 0xb00, 0x500, 0x300,
                                            0x200,  0x100, 0};
      const auto info = MakeTilingImage(format, 65, 33, levels, layers, tile,
                                        guest_size, false);
      const auto regions = info.BufferCopies();
      bool valid =
          info.tiled_slice_stride == guest_stride && regions.size() == levels;
      for (u32 mip = 0; mip < levels && valid; ++mip) {
        valid &= info.mip_layout[mip].offset == offsets[mip] &&
                 info.MipExtent(mip).depth == layers &&
                 regions[mip].imageSubresource.layerCount == layers;
      }
      Require(name, "reverse mip array addresses", valid,
              "mip order or padded guest layer stride changed");
      check_round_trip("reverse mip array", guest_size, info);
      std::vector<uint8_t> tiled(guest_size),
          expected_linear(info.LinearSize(), 0), observed_linear;
      fill(&tiled, ++case_index);
      for (u32 mip = 0; mip < levels; ++mip)
        convert_reference(false, &expected_linear, tiled, info, mip);
      gpu_detile(tiled, &observed_linear, guest_size, info.LinearSize(), info,
                 nullptr, 260);
      compare("detile nonzero source base", expected_linear, observed_linear);
      const auto saved_mips = info.mip_layout;
      const auto prefix = info.BufferCopies(256, 2);
      Require(name, "partial mip array addresses",
              prefix.size() == 2 && prefix[0].bufferOffset == 256 &&
                  prefix[1].bufferOffset ==
                      info.mip_layout[1].linear_offset + 256 &&
                  info.mip_layout[0].offset == 0x1a00 &&
                  info.mip_layout[1].offset == 0xb00,
              "partial mip transfer recomputed the full-chain guest stride");
      std::vector<uint8_t> linear(info.LinearSize());
      fill(&linear, ++case_index);
      for (const u32 count : {2u, levels, 1u, levels}) {
        std::vector<uint8_t> expected(guest_size, 0xab),
            observed(guest_size, 0xab);
        for (u32 mip = 0; mip < count; ++mip)
          convert_reference(true, &expected, linear, info, mip);
        gpu_tile(linear, &observed, guest_size, info.LinearSize(), info,
                 nullptr, count, count == levels ? 516 : 260,
                 count == levels ? 520 : 264);
        compare("repeated partial/full transfer", expected, observed);
        Require(
            name, "immutable transfer metadata",
            info.mip_layout == saved_mips && info.resources.levels == levels &&
                info.tiled_slice_stride == guest_stride,
            "partial or repeated transfer changed canonical image metadata");
      }
    }
    for (const auto family : families) {
      for (u32 bpe = 1; bpe <= family.max_bpe; bpe <<= 1u) {
        TileBlockLayout block{};
        Require(name, "block layout",
                TileGetBlockLayout(family.family, bpe, block),
                "admitted family/BPE has no block layout");
        const bool volume = block.block_depth > 1;
        const u32 width =
            block.block_width * 3u + std::min(block.block_width, 3u);
        const u32 height =
            block.block_height * 3u + std::min(block.block_height, 3u);
        const u32 depth = volume ? block.block_depth + 1u : 1u;
        const u32 pitch = block.block_width * 4u;
        const uint64_t block_columns =
            (pitch + block.block_width - 1u) / block.block_width;
        const uint64_t block_rows =
            (height + block.block_height - 1u) / block.block_height;
        const uint64_t block_slices =
            (depth + block.block_depth - 1u) / block.block_depth;
        const uint64_t storage_size =
            block_columns * block_rows * block_slices * block.block_size;
        std::vector<uint8_t> tiled(storage_size);
        std::vector<uint8_t> cpu(storage_size, 0);
        std::vector<uint8_t> gpu(storage_size, 0xab);
        fill(&tiled, ++case_index);

        auto info =
            block_image(block, width, height, depth, pitch, storage_size);
        info.mip_layout[0].surface_z =
            family.family == TileBlockFamily::RenderTarget64KB ||
                    family.family == TileBlockFamily::Depth64KB
                ? 3
                : 0;
        convert_reference(false, &cpu, tiled, info);
        gpu_detile(tiled, &gpu, storage_size, storage_size, info);
        const auto family_label = [&](const char *operation) {
          std::ostringstream out;
          out << operation << " family=" << static_cast<u32>(family.family)
              << " bpe=" << bpe;
          return out.str();
        };
        compare(family_label("detile bytes").c_str(), cpu, gpu);

        {
          std::vector<uint8_t> linear(storage_size);
          std::vector<uint8_t> cpu_tiled(storage_size, 0xab);
          std::vector<uint8_t> gpu_tiled(storage_size, 0xab);
          fill(&linear, 0x80u + case_index);
          convert_reference(true, &cpu_tiled, linear, info);
          gpu_tile(linear, &gpu_tiled, storage_size, storage_size, info);
          compare(family_label("tile bytes").c_str(), cpu_tiled, gpu_tiled);
        }
      }
    }

    // Exercise the real texture-layout/info-building seam for every format
    // admitted by each texture tile mode.  Formats sharing a byte/block width
    // intentionally share a shader, but this loop still validates their
    // texel-to-element conversion (notably every BCn format).
    struct TextureMode {
      Prospero::TileMode tile;
      TileBlockFamily family;
    };
    constexpr TextureMode texture_modes[] = {
        {Prospero::TileMode::kStandard256B, TileBlockFamily::Standard256B},
        {Prospero::TileMode::kStandard4KB, TileBlockFamily::Standard4KB},
        {Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB},
        {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB},
        {Prospero::TileMode::kRenderTarget, TileBlockFamily::RenderTarget64KB},
        {Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB},
    };
    struct TextureTileFormatCase {
      Prospero::BufferFormat format;
      u32 bytes_per_element;
      bool depth_supported;
    };
    constexpr TextureTileFormatCase texture_tile_formats[] = {
        {Prospero::BufferFormat::k8Srgb, 1, true},
        {Prospero::BufferFormat::k8_8Srgb, 2, true},
        {Prospero::BufferFormat::k9_9_9_5Float, 4, true},
        {Prospero::BufferFormat::kBc1Srgb, 8, false},
        {Prospero::BufferFormat::kBc7Srgb, 16, false},
        {Prospero::BufferFormat::k32_32_32_32Float, 16, false},
    };
    for (const auto &test : texture_tile_formats) {
      for (const auto tile :
           {Prospero::TileMode::kDepth, Prospero::TileMode::kRenderTarget}) {
        TileTextureBlockLayout texture{};
        const bool supported =
            TileGetTextureBlockLayout(test.format, tile, false, texture);
        const bool expected = tile == Prospero::TileMode::kRenderTarget ||
                              test.depth_supported;
        Require(name, "texture tile format policy",
                supported == expected &&
                    (!supported || texture.block.bytes_per_element ==
                                       test.bytes_per_element),
                "texture tile policy used attachment format restrictions");
      }
    }
    {
      TileSurfaceLayout invalid{};
      const TileSurfaceDescription description{
          Prospero::BufferFormat::k32Float,
          Prospero::TileMode::kStandard256B,
          TileSurfaceDimension::Dim3D,
          16,
          16,
          16,
          1,
          1,
      };
      Require(name, "3D tile policy",
              !TileGetTiledTextureLayout(description, invalid),
              "Standard256B volume silently fell back to a 2D array layout");
    }
    Require(
        name, "FMASK policy",
        Prospero::IsFmaskTextureFormat(Prospero::BufferFormat::kFmask8_S4_F4),
        "FMASK sample metadata lost its explicit tiler exclusion "
        "classification");
    u32 format_cases = 0;
    for (u32 raw_format = 1;
         raw_format <= static_cast<uint32_t>(Prospero::BufferFormat::kBc7Srgb);
         ++raw_format) {
      const auto format = static_cast<Prospero::BufferFormat>(raw_format);
      // FMASK contains sample-to-fragment metadata, not texels for the GPU tiler.
      if (Prospero::IsFmaskTextureFormat(format)) {
        continue;
      }
      for (const auto &mode : texture_modes) {
        TileTextureBlockLayout texture{};
        if (!TileGetTextureBlockLayout(format, mode.tile, false, texture)) {
          continue;
        }
        const u32 bpe = texture.block.bytes_per_element;
        Require(name, "format block", texture.block.family == mode.family,
                "CPU-supported format selected the wrong block family");

        constexpr u32 width = 67;
        constexpr u32 height = 51;
        TileSizeAlign total{};
        TileGetTextureSize(format, width, height, 1, mode.tile, &total, nullptr,
                           nullptr);
        Require(name, "format size", total.size != 0,
                "supported format has an empty layout");

        const auto layout = MakeTilingImage(format, width, height, 1, 1,
                                            mode.tile, total.size, false);
        const auto regions = layout.BufferCopies();
        Require(name, "format family",
                layout.resources.levels == 1 &&
                    layout.tiling.block.family == mode.family &&
                    layout.tiling.block.bytes_per_element == bpe,
                "texture info selected the wrong shader family");

        std::vector<uint8_t> tiled(total.size);
        std::vector<uint8_t> cpu(total.size, 0);
        std::vector<uint8_t> gpu(total.size, 0xab);
        fill(&tiled, ++case_index);
        convert_reference(false, &cpu, tiled, layout);
        gpu_detile(tiled, &gpu, total.size, total.size, layout);
        compare("format bytes", cpu, gpu);
        ++format_cases;
      }
    }
    Require(name, "format coverage", format_cases != 0,
            "no CPU-supported texture formats were tested");

    {
      constexpr auto format = Prospero::BufferFormat::kBc1Srgb;
      constexpr auto tile = Prospero::TileMode::kRenderTarget;
      constexpr u32 layers = 8;
      const auto captured =
          MakeTilingImage(format, 2048, 2048, 1, layers, tile, 0, false);
      Require(name, "PPSA19577 BC1 array footprint",
              captured.data.size == 0x1000000 &&
                  captured.tiled_slice_stride == 0x200000 &&
                  captured.first_tail_level == 1 &&
                  captured.tiling.block.block_width == 128 &&
                  captured.tiling.block.block_height == 64 &&
                  captured.mip_layout[0].pitch == 512 &&
                  captured.mip_layout[0].height == 512,
              "R64 BC1 texture lost its compressed blocks or array stride");
      const auto small_array =
          MakeTilingImage(format, 67, 51, 1, layers, tile, 0, false);
      check_round_trip("R64 BC1 array", small_array.data.size, small_array);
    }

    {
      constexpr auto format = Prospero::BufferFormat::k11_11_10UInt;
      constexpr auto tile = Prospero::TileMode::kStandard64KB;
      constexpr u32 width = 128, height = 128, levels = 8;
      TileSurfaceLayout surface{};
      const TileSurfaceDescription description{
          format, tile, TileSurfaceDimension::Dim2D, width, height, 1,
          levels, 1};
      TileSizeAlign total{};
      TileGetTextureTotalSize(format, width, height, 1, levels, tile, false,
                              total);
      const bool built = TileGetTiledTextureLayout(description, surface);
      const auto surface_format = TextureGetSurfaceFormatInfo(format);
      constexpr u32 tail_xy[7][2] = {{64, 0}, {0, 64}, {32, 0}, {0, 32},
                                     {16, 0}, {8, 16}, {0, 24}};
      bool valid = built && Prospero::NumBytesPerElement(format) == 4 &&
                   Prospero::BlockCompressedBytesPerBlock(format) == 0 &&
                   Prospero::SampledTextureNumericClass(format) ==
                       Prospero::TextureNumericClass::Uint &&
                   Prospero::RemapTextureFormat(format) ==
                       Prospero::BufferFormat::k32UInt &&
                   surface_format.vk_format == vk::Format::eR32Uint &&
                   surface_format.conversion_format == format &&
                   VulkanFormat(format) == vk::Format::eUndefined &&
                   surface.texture.block.block_width == 128 &&
                   surface.texture.block.block_height == 128 &&
                   surface.texture.block.block_depth == 1 &&
                   surface.first_tail_level == 1 &&
                   surface.mips[0].offset == 0x10000 && total.size == 0x20000 &&
                   total.align == 0x10000;
      for (u32 level = 1; level < levels && valid; level++) {
        valid &= surface.mips[level].offset == 0 &&
                 surface.mips[level].tail_x == tail_xy[level - 1][0] &&
                 surface.mips[level].tail_y == tail_xy[level - 1][1];
      }
      Require(name, "11_11_10 uint layout", valid,
              "PS5 packed integer texture metadata or Standard64KB mip tail "
              "changed");

      const auto signed_format = Prospero::BufferFormat::k32SInt;
      const auto signed_surface = TextureGetSurfaceFormatInfo(signed_format);
      Require(name, "signed sampled surface mapping",
              Prospero::SampledTextureNumericClass(signed_format) ==
                      Prospero::TextureNumericClass::Sint &&
                  Prospero::RemapTextureFormat(signed_format) ==
                      signed_format &&
                  signed_surface.vk_format == vk::Format::eR32Sint &&
                  signed_surface.conversion_format ==
                      Prospero::BufferFormat::kInvalid,
              "signed sampled texture class or native backing changed");
    }

    {
      constexpr auto format = Prospero::BufferFormat::k32_32_32Float;
      TileTextureElementLayout element{};
      Require(name, "RGB32 texture policy",
              !TileGetTextureElementLayout(format, element),
              "12-byte texel layouts must be rejected");
    }

    {
      constexpr auto format = Prospero::BufferFormat::kBc1UNorm;
      constexpr auto tile = Prospero::TileMode::kStandard64KB;
      constexpr u32 width = 256, height = 256, levels = 9;
      TileSizeAlign total{};
      TileGetTextureSize(format, width, height, levels, tile, &total, nullptr,
                         nullptr);
      const auto layout = MakeTilingImage(format, width, height, levels, 1,
                                          tile, total.size, false);
      const auto regions = layout.BufferCopies();

      const auto linear_size = layout.LinearSize();
      Require(name, "BC1 mip-tail capacities",
              total.size == 0x10000 && layout.first_tail_level == 0 &&
                  linear_size == 0x15560 && linear_size > total.size,
              "BC1 mip tail conflated tiled and linear capacities");
      check_round_trip("BC1 mip tail", total.size, layout);
    }

    {
      constexpr auto format = Prospero::BufferFormat::k32Float;
      constexpr auto tile = Prospero::TileMode::kRenderTarget;
      constexpr u32 width = 129, height = 65, layers = 3;
      TileSizeAlign total{};
      TileGetTextureTotalSize(format, width, height, layers, 1, tile, false,
                              total);
      const auto layout = MakeTilingImage(format, width, height, 1, layers,
                                          tile, total.size, false);
      const auto regions = layout.BufferCopies();

      Require(name, "array infos",
              layout.resources.levels == 1 &&
                  layout.MipExtent(0).depth == layers &&
                  layout.mip_layout[0].surface_z == 0 &&
                  layout.tiled_slice_stride == total.size / layers &&
                  regions.size() == 1 &&
                  regions[0].imageSubresource.layerCount == layers,
              "array mip lost its layer count or absolute surface Z origin");
      check_round_trip("array", total.size, layout);
    }

    for (const auto &mode : texture_modes) {
      constexpr auto format = Prospero::BufferFormat::k32Float;
      constexpr u32 levels = 2;
      TileBlockLayout block{};
      Require(name, "odd mip block", TileGetBlockLayout(mode.family, 4, block),
              "odd multi-mip format has no block layout");
      const u32 width = block.block_width * 2u + 1u;
      const u32 height = block.block_height * 2u + 1u;
      TileSizeAlign total{};
      TileGetTextureSize(format, width, height, levels, mode.tile, &total,
                         nullptr, nullptr);
      const auto layout = MakeTilingImage(format, width, height, levels, 1,
                                          mode.tile, total.size, false);
      const auto regions = layout.BufferCopies();
      Require(name, "odd mip infos",
              layout.resources.levels == 2 &&
                  layout.mip_layout[1].pitch >=
                      layout.mip_layout[1].linear_pitch &&
                  layout.mip_layout[1].height >=
                      ((layout.MipExtent(1).height +
                        layout.tiling.texel_height - 1u) /
                       layout.tiling.texel_height) &&
                  (mode.family == TileBlockFamily::Standard256B ||
                   layout.mip_layout[1].height >
                       ((layout.MipExtent(1).height +
                         layout.tiling.texel_height - 1u) /
                        layout.tiling.texel_height)),
              "odd multi-mip physical stride collapsed to the active linear "
              "extent");
      check_round_trip("odd multi-mip", total.size, layout);
    }

    struct TailMode {
      Prospero::TileMode tile;
      TileBlockFamily family;
    };
    constexpr TailMode tail_modes[] = {
        {Prospero::TileMode::kStandard4KB, TileBlockFamily::Standard4KB},
        {Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB},
        {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB},
        {Prospero::TileMode::kRenderTarget, TileBlockFamily::RenderTarget64KB},
        {Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB},
    };
    for (const auto &mode : tail_modes) {
      constexpr auto format = Prospero::BufferFormat::k32Float;
      constexpr u32 levels = 7;
      TileBlockLayout block{};
      Require(name, "2D tail block", TileGetBlockLayout(mode.family, 4, block),
              "2D tail mode has no block layout");
      const u32 width = block.block_width * 2u;
      const u32 height = block.block_height * 2u;
      TileSizeAlign total{};
      TileGetTextureSize(format, width, height, levels, mode.tile, &total,
                         nullptr, nullptr);
      const auto layout = MakeTilingImage(format, width, height, levels, 1,
                                          mode.tile, total.size, false);
      const auto regions = layout.BufferCopies();
      Require(name, "2D mip tail seam",
              layout.first_tail_level == 2 && layout.resources.levels == levels,
              "2D mip chain lost its linear/tiled tail boundary");
      check_round_trip("2D mip tail seam", total.size, layout);
    }

    {
      constexpr auto format = Prospero::BufferFormat::k32Float;
      constexpr auto tile = Prospero::TileMode::kStandard4KB;
      constexpr u32 width = 65;
      constexpr u32 height = 129;
      constexpr u32 depth = 17;
      constexpr u32 levels = 2;
      TileSizeAlign total{};
      TileGetTextureTotalSize(format, width, height, depth, levels, tile, true,
                              total);
      const auto layout = MakeTilingImage(format, width, height, levels, depth,
                                          tile, total.size, true);
      const auto regions = layout.BufferCopies();

      Require(
          name, "3D mip infos",
          regions.size() == levels && layout.resources.levels == levels &&
              regions[0].imageExtent.depth == depth &&
              regions[1].imageExtent.depth == (depth >> 1u) &&
              layout.MipExtent(0).depth == 17 &&
              layout.MipExtent(1).depth == 8 &&
              layout.mip_layout[0].offset == 0x19000 &&
              layout.tiled_slice_stride == 0x6a000 &&
              layout.mip_layout[0].offset + layout.tiled_slice_stride ==
                  0x83000 &&
              layout.mip_layout[0].offset + 2 * layout.tiled_slice_stride ==
                  0xed000 &&
              layout.mip_layout[1].offset == 0 &&
              layout.mip_layout[1].linear_pitch == 32 &&
              ((layout.MipExtent(1).height + layout.tiling.texel_height - 1u) /
               layout.tiling.texel_height) == 64 &&
              layout.mip_layout[1].pitch == 40 &&
              layout.mip_layout[1].height == 80,
          "Standard4KB3D mip depth, packing, or physical stride changed");
      check_round_trip("3D mip", total.size, layout);
    }

    {
      constexpr auto format = Prospero::BufferFormat::kBc1UNorm;
      constexpr auto tile = Prospero::TileMode::kStandard4KB;
      constexpr u32 width = 65;
      constexpr u32 height = 129;
      constexpr u32 depth = 17;
      constexpr u32 levels = 2;
      TileSizeAlign total{};
      TileGetTextureTotalSize(format, width, height, depth, levels, tile, true,
                              total);
      const auto layout = MakeTilingImage(format, width, height, levels, depth,
                                          tile, total.size, true);
      const auto regions = layout.BufferCopies();
      Require(
          name, "3D BC mip infos",
          layout.resources.levels == levels &&
              layout.MipExtent(0).depth == depth &&
              layout.MipExtent(1).depth == (depth >> 1u) &&
              layout.mip_layout[1].linear_pitch == 8 &&
              ((layout.MipExtent(1).height + layout.tiling.texel_height - 1u) /
               layout.tiling.texel_height) == 16 &&
              layout.mip_layout[1].pitch == 16 &&
              layout.mip_layout[1].height == 24,
          "block-compressed 3D mip lost its physical row or slice stride");
      check_round_trip("3D BC mip", total.size, layout);
    }

    {
      constexpr auto format = Prospero::BufferFormat::kBc3UNorm;
      constexpr auto tile = Prospero::TileMode::kLinear;
      constexpr u32 width = 8, height = 8, levels = 4;
      TileSizeAlign total{};
      TileGetTextureSize(format, width, height, levels, tile, &total, nullptr,
                         nullptr);
      const auto layout = MakeTilingImage(format, width, height, levels, 1,
                                          tile, total.size, false);
      const auto regions = layout.BufferCopies();
      Require(name, "linear BC native regions",
              regions.size() == levels &&
                  std::all_of(regions.begin(), regions.end(),
                              [](const auto &region) {
                                return region.bufferRowLength >=
                                           region.imageExtent.width &&
                                       region.bufferImageHeight >=
                                           region.imageExtent.height &&
                                       region.bufferRowLength % 4 == 0 &&
                                       region.bufferImageHeight % 4 == 0;
                              }),
              "linear BC mip tails emitted invalid Vulkan copy strides");
    }

    struct VolumeModeCase {
      Prospero::TileMode tile;
      TileBlockFamily family;
      Prospero::BufferFormat format;
    };
    constexpr VolumeModeCase volume_modes[] = {
        {Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB3D,
         Prospero::BufferFormat::k32Float},
        {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB3D,
         Prospero::BufferFormat::k32Float},
        {Prospero::TileMode::kRenderTarget, TileBlockFamily::RenderTarget64KB,
         Prospero::BufferFormat::k32Float},
        {Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB,
         Prospero::BufferFormat::k32Float},
        {Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB3D,
         Prospero::BufferFormat::kBc1UNorm},
        {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB3D,
         Prospero::BufferFormat::kBc3UNorm},
    };
    for (const auto test : volume_modes) {
      constexpr u32 width = 65, height = 33, depth = 37, levels = 6;
      TileSizeAlign total{};
      TileGetTextureTotalSize(test.format, width, height, depth, levels,
                              test.tile, true, total);
      const auto layout = MakeTilingImage(test.format, width, height, levels,
                                          depth, test.tile, total.size, true);
      const auto regions = layout.BufferCopies();

      const bool uses_z = test.family == TileBlockFamily::RenderTarget64KB ||
                          test.family == TileBlockFamily::Depth64KB;
      Require(name, "volume family infos",
              layout.resources.levels != 0 &&
                  layout.tiling.block.family == test.family &&
                  layout.resources.levels == levels &&
                  layout.MipExtent(0).depth == depth &&
                  (!uses_z || (layout.mip_layout[0].surface_z == 0 &&
                               layout.tiled_slice_stride != 0)),
              "volume mode selected the wrong family or lost its surface Z");
      check_round_trip("volume family", total.size, layout);
    }

    struct VolumeTailCase {
      Prospero::BufferFormat format;
      u32 bytes;
    };
    constexpr VolumeTailCase volume_tails[] = {
        {Prospero::BufferFormat::k8UNorm, 1},
        {Prospero::BufferFormat::k16UNorm, 2},
        {Prospero::BufferFormat::k32Float, 4},
        {Prospero::BufferFormat::k16_16_16_16Float, 8},
        {Prospero::BufferFormat::k32_32_32_32Float, 16},
        {Prospero::BufferFormat::kBc1UNorm, 8},
        {Prospero::BufferFormat::kBc3UNorm, 16},
    };
    constexpr u32 volume_tail_xy[5][5][2] = {
        {{0, 8}, {8, 4}, {8, 0}, {0, 4}, {0, 0}},
        {{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}},
        {{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}},
        {{0, 4}, {4, 2}, {4, 0}, {0, 2}, {0, 0}},
        {{0, 4}, {2, 2}, {2, 0}, {0, 2}, {0, 0}},
    };
    for (const auto &tail : volume_tails) {
      constexpr auto tile = Prospero::TileMode::kStandard4KB;
      constexpr u32 levels = 5;
      TileBlockLayout block{};
      Require(
          name, "3D tail block",
          TileGetBlockLayout(TileBlockFamily::Standard4KB3D, tail.bytes, block),
          "Standard4KB3D tail format has no block layout");
      const bool compressed =
          Prospero::BlockCompressedBytesPerBlock(tail.format) != 0;
      const u32 width = block.block_width * (compressed ? 4u : 1u);
      const u32 height = block.block_height / 2u * (compressed ? 4u : 1u);
      const u32 depth = block.block_depth + 1u;
      TileSizeAlign total{};
      TileGetTextureTotalSize(tail.format, width, height, depth, levels, tile,
                              true, total);
      const auto layout = MakeTilingImage(tail.format, width, height, levels,
                                          depth, tile, total.size, true);
      const auto regions = layout.BufferCopies();
      bool valid = total.size == 8192 && regions.size() == levels &&
                   layout.resources.levels == levels;
      const u32 table = std::countr_zero(tail.bytes);
      for (u32 level = 0; level < levels && valid; level++) {
        const auto &mip = layout.mip_layout[level];
        valid &=
            level >= layout.first_tail_level &&
            mip.tail_x == volume_tail_xy[table][level][0] &&
            mip.tail_y == volume_tail_xy[table][level][1] && mip.offset == 0 &&
            layout.MipExtent(level).depth == std::max(depth >> level, 1u) &&
            layout.tiled_slice_stride == 4096;
      }
      if (valid) {
        valid = layout.MipExtent(0).depth == block.block_depth + 1u &&
                layout.mip_layout[0].offset + layout.tiled_slice_stride == 4096;
      }
      Require(
          name, "3D mip tail infos", valid,
          "Standard4KB3D mip tail coordinates or block-slice packing changed");
      check_round_trip("3D mip tail", total.size, layout);
    }

    for (const auto family :
         {TileBlockFamily::Standard4KB, TileBlockFamily::Standard64KB,
          TileBlockFamily::Prt64KB, TileBlockFamily::RenderTarget64KB,
          TileBlockFamily::Depth64KB}) {
      for (u32 bpe = 1; bpe <= 16; bpe <<= 1u) {
        if (family == TileBlockFamily::Depth64KB && bpe == 16)
          continue;
        TileBlockLayout block{};
        Require(name, "tail layout", TileGetBlockLayout(family, bpe, block),
                "tail family/BPE has no block layout");
        const u32 width = std::max(block.block_width / 4u, 1u);
        const u32 height = std::max(block.block_height / 4u, 1u);
        const u32 x = block.block_width / 2u;
        const u32 y = block.block_height / 2u;
        const u32 pitch = width;
        const uint64_t linear_size =
            static_cast<uint64_t>(pitch) * height * bpe;
        std::vector<uint8_t> tiled(block.block_size);
        std::vector<uint8_t> cpu(linear_size, 0xcd);
        std::vector<uint8_t> gpu(linear_size, 0xab);
        fill(&tiled, ++case_index);
        auto info =
            block_image(block, width, height, 1, pitch, block.block_size);
        info.first_tail_level = 0;
        info.mip_layout[0].tail_x = x;
        info.mip_layout[0].tail_y = y;
        info.mip_layout[0].surface_z =
            family == TileBlockFamily::RenderTarget64KB ||
                    family == TileBlockFamily::Depth64KB
                ? 2
                : 0;
        convert_reference(false, &cpu, tiled, info);
        gpu_detile(tiled, &gpu, block.block_size, linear_size, info);
        compare("tail bytes", cpu, gpu);

        std::vector<uint8_t> linear(linear_size);
        std::vector<uint8_t> cpu_tiled(block.block_size, 0xab);
        std::vector<uint8_t> gpu_tiled(block.block_size, 0xab);
        fill(&linear, 0x400u + case_index);
        convert_reference(true, &cpu_tiled, linear, info);
        gpu_tile(linear, &gpu_tiled, block.block_size, linear_size, info);
        compare("tail tile bytes", cpu_tiled, gpu_tiled);
      }
    }

    TileBlockLayout small_block{};
    Require(name, "small layout",
            TileGetBlockLayout(TileBlockFamily::Standard256B, 4, small_block),
            "small fixture layout is unavailable");
    std::vector<uint8_t> small_input(small_block.block_size);
    std::vector<uint8_t> small_expected(small_block.block_size, 0);
    std::vector<uint8_t> small_output(small_block.block_size, 0xab);
    fill(&small_input, 0xee);
    auto small_info = block_image(small_block, 1, 1, 1, small_block.block_width,
                                  small_input.size());
    convert_reference(false, &small_expected, small_input, small_info);
    gpu_detile(small_input, &small_output, small_input.size(),
               small_output.size(), small_info);
    compare("small detile", small_expected, small_output);

    std::fill(small_output.begin(), small_output.end(), 0xab);
    gpu_detile(small_input, &small_output, small_input.size(),
               small_output.size(), small_info);
    compare("scheduler-owned reuse", small_expected, small_output);

    constexpr auto volume_format = Prospero::BufferFormat::k32UInt;
    constexpr auto volume_tile = Prospero::TileMode::kLinear;
    constexpr u32 volume_width = 8, volume_height = 4, volume_depth = 5;
    constexpr u32 volume_levels = 3;
    TileSizeAlign volume_size{};
    TileGetTextureTotalSize(volume_format, volume_width, volume_height,
                            volume_depth, volume_levels, volume_tile, true,
                            volume_size);
    const auto volume_layout = MakeTilingImage(
        volume_format, volume_width, volume_height, volume_levels, volume_depth,
        volume_tile, volume_size.size, true);
    const auto volume_copies = volume_layout.BufferCopies();
    std::vector<u32> volume_source(volume_size.size / sizeof(u32), 0);
    std::vector<std::pair<size_t, u32>> volume_probes;
    for (const auto &copy : volume_copies) {
      for (u32 z = 0; z < copy.imageExtent.depth; ++z) {
        for (u32 y = 0; y < copy.imageExtent.height; ++y) {
          for (u32 x = 0; x < copy.imageExtent.width; ++x) {
            const auto index = copy.bufferOffset / sizeof(u32) +
                               (uint64_t{z} * copy.bufferImageHeight + y) *
                                   copy.bufferRowLength +
                               x;
            const u32 value =
                0xa0000000u | (copy.imageSubresource.mipLevel << 24u) |
                ((static_cast<u32>(copy.imageOffset.z) + z) << 16u) |
                (y << 8u) | x;
            Require(name, "linear volume native bounds",
                    index < volume_source.size(),
                    "linear volume copy escaped its padded guest layout");
            volume_source[index] = value;
            volume_probes.emplace_back(static_cast<size_t>(index), value);
          }
        }
      }
    }
    auto volume_upload =
        CreateHostBuffer(name, volume_size.size,
                         vk::BufferUsageFlagBits::eTransferSrc, volume_source);
    auto volume_download = CreateHostBuffer(
        name, volume_size.size, vk::BufferUsageFlagBits::eTransferDst,
        std::vector<u32>(volume_source.size(), 0));
    auto volume_info = volume_layout;
    volume_info.data.address = 0x10000;
    volume_info.pixel_format = vk::Format::eR32Uint;
    Libs::Graphics::Image volume_image(m_runtime_context, scheduler,
                                       volume_info);
    volume_image.Upload(volume_copies, volume_upload.buffer, 0,
                        volume_size.size);
    volume_image.Download(volume_copies, volume_download.buffer, 0,
                          volume_size.size);
    scheduler.Finish();
    const auto volume_observed =
        ReadBuffer(name, volume_download, volume_source.size());
    Require(name, "linear volume native content",
            std::ranges::all_of(volume_probes,
                                [&](const auto &probe) {
                                  return volume_observed[probe.first] ==
                                         probe.second;
                                }),
            "native upload/download did not preserve padded 3D mip rows");
    DestroyBuffer(&volume_download);
    DestroyBuffer(&volume_upload);
    std::printf("[gpu]     %-32s ok (%u cases, %u format/mode pairs)\n", name,
                case_index, format_cases);
  }

private:
  bool m_rasterization_supported = true;
  u32   m_skipped_cases          = 0;

  RenderContext &Renderer() {
    EXIT_IF(m_renderer == nullptr);
    return *m_renderer;
  }

  void EnsureRuntimeContext() {
    if (m_runtime_context.allocator != nullptr) {
      return;
    }
    namespace Exception = Common::HostException;
    Require("VulkanHarness", "guest fault handler",
            Exception::InstallHandler([](const Exception::ExceptionInfo &info) {
              if (info.type != Exception::ExceptionType::AccessViolation ||
                  (info.access_violation_type != Exception::AccessViolationType::Read &&
                   info.access_violation_type != Exception::AccessViolationType::Write)) {
                return false;
              }
              const auto access =
                  info.access_violation_type == Exception::AccessViolationType::Write
                      ? PageFaultAccess::Write : PageFaultAccess::Read;
              return LibKernel::Memory::HandleGpuFault(access, info.access_violation_vaddr);
            }),
            "failed to install the production guest-memory fault route");
    m_runtime_context.instance = m_instance;
    m_runtime_context.physical_device = m_physical_device;
    m_runtime_context.device = m_device;
    m_physical_device.getProperties(
        &m_runtime_context.physical_device_properties);
    m_runtime_context.physical_device_memory_properties = m_memory_properties;
    m_runtime_context.queue_family = m_queue_family;
    m_runtime_context.queue = m_queue;
    m_runtime_context.shader_image_int64_atomics_enabled = true;
    m_runtime_context.maintenance8_enabled = true;
    m_runtime_context.attachment_feedback_loop_enabled = m_rasterization_supported;
    m_runtime_context.provoking_vertex_last_enabled = m_rasterization_supported;
    const vk::PhysicalDeviceImageFormatInfo2 block_texel_view_info{
        .format = vk::Format::eBc1RgbaUnormBlock,
        .type = vk::ImageType::e2D,
        .tiling = vk::ImageTiling::eOptimal,
        .usage = vk::ImageUsageFlagBits::eSampled,
        .flags = vk::ImageCreateFlagBits::eBlockTexelViewCompatible,
    };
    const auto block_texel_view_props =
        m_physical_device.getImageFormatProperties2(block_texel_view_info);
    m_runtime_context.supports_block_texel_view =
        block_texel_view_props.result == vk::Result::eSuccess;

    VmaVulkanFunctions functions{};
    functions.vkGetInstanceProcAddr =
        VULKAN_HPP_DEFAULT_DISPATCHER.vkGetInstanceProcAddr;
    functions.vkGetDeviceProcAddr =
        VULKAN_HPP_DEFAULT_DISPATCHER.vkGetDeviceProcAddr;

    VmaAllocatorCreateInfo allocator_info{};
    allocator_info.instance = m_instance;
    allocator_info.physicalDevice = m_physical_device;
    allocator_info.device = m_device;
    allocator_info.pVulkanFunctions = &functions;
    allocator_info.vulkanApiVersion = VK_API_VERSION_1_3;
    allocator_info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT;
    RequireVk("VulkanHarness", "runtime context",
              static_cast<vk::Result>(vmaCreateAllocator(
                  &allocator_info, &m_runtime_context.allocator)),
              "vmaCreateAllocator");
    m_renderer = std::make_unique<RenderContext>(m_runtime_context);
  }

  void Init() {
    static vk::detail::DynamicLoader loader;
    const auto get_instance_proc_addr =
        loader.getProcAddress<PFN_vkGetInstanceProcAddr>(
            "vkGetInstanceProcAddr");
    Require("VulkanHarness", "dispatch", get_instance_proc_addr != nullptr,
            "could not load the Vulkan loader");
    VULKAN_HPP_DEFAULT_DISPATCHER.init(get_instance_proc_addr);

    vk::ApplicationInfo app{};
    app.sType = vk::StructureType::eApplicationInfo;
    app.pApplicationName = "ShaderRecompilerComputeTests";
    app.apiVersion = VK_API_VERSION_1_3;

    vk::InstanceCreateInfo instance_info{};
    instance_info.sType = vk::StructureType::eInstanceCreateInfo;
    instance_info.pApplicationInfo = &app;
    RequireVk("VulkanHarness", "dispatch",
              vk::createInstance(&instance_info, nullptr, &m_instance),
              "vkCreateInstance");
    VULKAN_HPP_DEFAULT_DISPATCHER.init(m_instance);

    u32 physical_count = 0;
    RequireVk("VulkanHarness", "dispatch",
              m_instance.enumeratePhysicalDevices(&physical_count, nullptr),
              "vkEnumeratePhysicalDevices");
    Require("VulkanHarness", "dispatch", physical_count != 0,
            "no Vulkan physical devices");
    std::vector<vk::PhysicalDevice> physical_devices(physical_count);
    RequireVk("VulkanHarness", "dispatch",
              m_instance.enumeratePhysicalDevices(&physical_count,
                                                  physical_devices.data()),
              "vkEnumeratePhysicalDevices");

    for (auto physical : physical_devices) {
      u32 queue_count = 0;
      physical.getQueueFamilyProperties(&queue_count, nullptr);
      std::vector<vk::QueueFamilyProperties> queues(queue_count);
      physical.getQueueFamilyProperties(&queue_count, queues.data());
      for (u32 i = 0; i < queue_count; i++) {
        if ((queues[i].queueFlags &
             (vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) ==
            (vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) {
          vk::PhysicalDeviceVulkan11Features features11{};
          vk::PhysicalDeviceVulkan12Features features12{};
          features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
          vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{};
          features12.pNext = &features11;
          features11.pNext = &workgroup_layout;
          vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{};
          workgroup_layout.pNext = &image_atomic64;
          vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{};
          barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
          barycentric.pNext = &features12;
          vk::PhysicalDeviceFeatures2 features{};
          features.sType = vk::StructureType::ePhysicalDeviceFeatures2;
          features.pNext = &barycentric;
          physical.getFeatures2(&features);
          if (barycentric.fragmentShaderBarycentric != true ||
              features.features.shaderInt64 != true ||
              features11.storageBuffer16BitAccess != true ||
              features12.storageBuffer8BitAccess != true ||
              features12.samplerMirrorClampToEdge != true ||
              features12.shaderOutputViewportIndex != true ||
              features12.shaderBufferInt64Atomics != true ||
              features12.shaderSampledImageArrayNonUniformIndexing != true ||
              features12.shaderStorageBufferArrayNonUniformIndexing != true ||
              features12.shaderSharedInt64Atomics != true ||
              image_atomic64.shaderImageInt64Atomics != true ||
              workgroup_layout.workgroupMemoryExplicitLayout != true ||
              features12.bufferDeviceAddress != true) {
            continue;
          }
          m_physical_device = physical;
          m_queue_family = i;
          break;
        }
      }
      if (m_physical_device != nullptr) {
        break;
      }
    }
    Require("VulkanHarness", "dispatch", m_physical_device != nullptr,
            "no Vulkan graphics+compute device with fragment barycentrics and 64-bit LDS/image atomics");
    m_physical_device.getMemoryProperties(&m_memory_properties);

    vk::PhysicalDeviceFeatures available_features{};
    m_physical_device.getFeatures(&available_features);
    vk::PhysicalDeviceVulkan11Features available_features11{};
    vk::PhysicalDeviceVulkan12Features available_features12{};
    available_features12.pNext = &available_features11;
    available_features12.sType =
        vk::StructureType::ePhysicalDeviceVulkan12Features;
    vk::PhysicalDeviceVulkan13Features available_features13{};
    available_features13.sType =
        vk::StructureType::ePhysicalDeviceVulkan13Features;
    available_features13.pNext = &available_features12;
    vk::PhysicalDeviceComputeShaderDerivativesFeaturesKHR available_derivatives{};
    available_derivatives.pNext = &available_features13;
    vk::PhysicalDeviceDepthClipEnableFeaturesEXT available_depth_clip{};
    available_depth_clip.pNext = &available_derivatives;
    vk::PhysicalDeviceDepthClipControlFeaturesEXT available_clip_control{};
    available_clip_control.pNext = &available_depth_clip;
    vk::PhysicalDeviceColorWriteEnableFeaturesEXT available_color_write{};
    available_color_write.pNext = &available_clip_control;
    vk::PhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT available_feedback_layout{};
    available_feedback_layout.pNext = &available_color_write;
    vk::PhysicalDeviceAttachmentFeedbackLoopDynamicStateFeaturesEXT available_feedback_dynamic{};
    available_feedback_dynamic.pNext = &available_feedback_layout;
    vk::PhysicalDeviceProvokingVertexFeaturesEXT available_provoking_vertex{};
    available_provoking_vertex.pNext = &available_feedback_dynamic;
    vk::PhysicalDeviceImageViewMinLodFeaturesEXT available_min_lod{};
    available_min_lod.pNext = &available_provoking_vertex;
    vk::PhysicalDeviceMaintenance8FeaturesKHR available_maintenance8{};
    available_maintenance8.pNext = &available_min_lod;
    vk::PhysicalDeviceFeatures2 available_features2{};
    available_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
    available_features2.pNext = &available_maintenance8;
    m_physical_device.getFeatures2(&available_features2);
    Require("VulkanHarness", "dispatch",
            available_features.shaderStorageImageWriteWithoutFormat == true,
            "shaderStorageImageWriteWithoutFormat is not supported");
    Require("VulkanHarness", "dispatch",
            available_features.shaderImageGatherExtended == true &&
                available_derivatives.computeDerivativeGroupQuads == true,
            "image gather or compute derivative quads are not supported");
    Require("VulkanHarness", "dispatch",
            available_features11.storageBuffer16BitAccess == true &&
                available_features12.storageBuffer8BitAccess == true,
            "8-bit and 16-bit storage buffer access are not supported");
    Require("VulkanHarness", "dispatch",
            available_features12.timelineSemaphore == true,
            "timeline semaphores are not supported");
    Require("VulkanHarness", "dispatch",
            available_features13.dynamicRendering == true,
            "dynamic rendering is not supported");
    Require("VulkanHarness", "dispatch",
            available_features13.synchronization2 == true,
            "synchronization2 is not supported");
    Require("VulkanHarness", "dispatch",
            available_features.sampleRateShading == true,
            "sample-rate shading is not supported");
    Require("VulkanHarness", "dispatch", available_features.shaderInt64 == true,
            "shaderInt64 is not supported");
    Require("VulkanHarness", "dispatch",
            available_features12.bufferDeviceAddress == true,
            "bufferDeviceAddress is not supported");
    Require("VulkanHarness", "dispatch",
            available_features12.shaderSampledImageArrayNonUniformIndexing == true,
            "nonuniform sampled image indexing is not supported");
    Require("VulkanHarness", "dispatch",
            available_features12.shaderStorageBufferArrayNonUniformIndexing == true,
            "nonuniform storage buffer indexing is not supported");
    Require("VulkanHarness", "dispatch", available_min_lod.minLod == true,
            "image view minimum LOD is not supported");
    Require("VulkanHarness", "dispatch", available_maintenance8.maintenance8 == true,
            "maintenance8 dynamic image offsets are not supported");
    Require("VulkanHarness", "graphics", available_features12.shaderOutputLayer == true,
            "vertex layer output is not supported");
    m_rasterization_supported = available_features.fillModeNonSolid &&
                                available_features.tessellationShader &&
                                available_features.depthBounds &&
                                available_depth_clip.depthClipEnable &&
                                available_clip_control.depthClipControl &&
                                available_color_write.colorWriteEnable &&
                                available_feedback_layout.attachmentFeedbackLoopLayout &&
                                available_feedback_dynamic.attachmentFeedbackLoopDynamicState &&
                                available_provoking_vertex.provokingVertexLast;
    if (!m_rasterization_supported) {
      std::printf(
          "[host]    ProductionRasterization   unavailable, rasterization cases "
          "will be skipped\n");
    }

    float priority = 1.0f;
    vk::DeviceQueueCreateInfo queue_info{};
    queue_info.sType = vk::StructureType::eDeviceQueueCreateInfo;
    queue_info.queueFamilyIndex = m_queue_family;
    queue_info.queueCount = 1;
    queue_info.pQueuePriorities = &priority;

    vk::DeviceCreateInfo device_info{};
    device_info.sType = vk::StructureType::eDeviceCreateInfo;
    device_info.queueCreateInfoCount = 1;
    device_info.pQueueCreateInfos = &queue_info;
    auto device_features11 = WindowContext::RequiredVulkan11Features();
    auto device_features12 = WindowContext::RequiredVulkan12Features();
    device_features12.shaderSharedInt64Atomics = true;
    vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{};
    workgroup_layout.workgroupMemoryExplicitLayout = true;
    device_features12.pNext = &device_features11;
    device_features11.pNext = &workgroup_layout;
    vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{};
    image_atomic64.shaderImageInt64Atomics = true;
    workgroup_layout.pNext = &image_atomic64;
    vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{};
    barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
    barycentric.pNext = &device_features12;
    barycentric.fragmentShaderBarycentric = true;
    vk::PhysicalDeviceVulkan13Features device_features13{};
    device_features13.sType =
        vk::StructureType::ePhysicalDeviceVulkan13Features;
    device_features13.pNext = &barycentric;
    device_features13.dynamicRendering = true;
    device_features13.synchronization2 = true;
    vk::PhysicalDeviceComputeShaderDerivativesFeaturesKHR derivatives{};
    derivatives.pNext = &device_features13;
    derivatives.computeDerivativeGroupQuads = true;
    // Requesting a feature the device does not support fails device creation, so the
    // rasterization feature chain is only chained in when every part is available.
    vk::PhysicalDeviceDepthClipEnableFeaturesEXT depth_clip{};
    vk::PhysicalDeviceDepthClipControlFeaturesEXT clip_control{};
    vk::PhysicalDeviceColorWriteEnableFeaturesEXT color_write{};
    vk::PhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT feedback_layout{};
    vk::PhysicalDeviceAttachmentFeedbackLoopDynamicStateFeaturesEXT feedback_dynamic{};
    vk::PhysicalDeviceProvokingVertexFeaturesEXT provoking_vertex{};
    if (m_rasterization_supported) {
      depth_clip.pNext = &derivatives;
      depth_clip.depthClipEnable = true;
      clip_control.pNext = &depth_clip;
      clip_control.depthClipControl = true;
      color_write.pNext = &clip_control;
      color_write.colorWriteEnable = true;
      feedback_layout.pNext = &color_write;
      feedback_layout.attachmentFeedbackLoopLayout = true;
      feedback_dynamic.pNext = &feedback_layout;
      feedback_dynamic.attachmentFeedbackLoopDynamicState = true;
      provoking_vertex.pNext = &feedback_dynamic;
      provoking_vertex.provokingVertexLast = available_provoking_vertex.provokingVertexLast;
    }
    vk::PhysicalDeviceImageViewMinLodFeaturesEXT min_lod{};
    min_lod.minLod = true;
    min_lod.pNext = m_rasterization_supported
                        ? static_cast<void *>(&provoking_vertex)
                        : static_cast<void *>(&derivatives);
    vk::PhysicalDeviceMaintenance8FeaturesKHR maintenance8{};
    maintenance8.maintenance8 = true;
    maintenance8.pNext = &min_lod;
    device_info.pNext = &maintenance8;
    vk::PhysicalDeviceFeatures device_features{};
    device_features.shaderStorageImageWriteWithoutFormat = true;
    device_features.shaderImageGatherExtended = true;
    device_features.sampleRateShading = true;
    device_features.shaderInt64 = true;
    device_features.shaderFloat64 = available_features.shaderFloat64;
    device_features.fillModeNonSolid = m_rasterization_supported;
    device_features.tessellationShader = m_rasterization_supported;
    device_features.depthBounds = m_rasterization_supported;
    device_info.pEnabledFeatures = &device_features;
    std::vector<const char *> device_extensions{
        VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME,
        VK_KHR_COMPUTE_SHADER_DERIVATIVES_EXTENSION_NAME,
        VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME,
        VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME,
        VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME,
        VK_KHR_MAINTENANCE_8_EXTENSION_NAME,
        VK_EXT_IMAGE_VIEW_MIN_LOD_EXTENSION_NAME};
    if (m_rasterization_supported) {
      device_extensions.push_back(VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME);
      device_extensions.push_back(VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME);
      device_extensions.push_back(VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
      device_extensions.push_back(VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME);
      device_extensions.push_back(
          VK_EXT_ATTACHMENT_FEEDBACK_LOOP_DYNAMIC_STATE_EXTENSION_NAME);
      device_extensions.push_back(VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME);
    }
    device_info.enabledExtensionCount =
        static_cast<uint32_t>(device_extensions.size());
    device_info.ppEnabledExtensionNames = device_extensions.data();
    RequireVk("VulkanHarness", "dispatch",
              m_physical_device.createDevice(&device_info, nullptr, &m_device),
              "vkCreateDevice");
    VULKAN_HPP_DEFAULT_DISPATCHER.init(m_device);
    m_device.getQueue(m_queue_family, 0, &m_queue);

    vk::CommandPoolCreateInfo pool_info{};
    pool_info.sType = vk::StructureType::eCommandPoolCreateInfo;
    pool_info.queueFamilyIndex = m_queue_family;
    pool_info.flags = vk::CommandPoolCreateFlagBits::eResetCommandBuffer;
    RequireVk("VulkanHarness", "dispatch",
              m_device.createCommandPool(&pool_info, nullptr, &m_command_pool),
              "vkCreateCommandPool");
  }

  void Destroy() {
    if (m_device != nullptr) {
      RequireVulkanSuccess(m_device.waitIdle(), "vkDeviceWaitIdle");
      DestroyBuffer(&m_fault_buffer);
      DestroyBuffer(&m_bda_pagetable_buffer);
      if (m_runtime_context.allocator != nullptr) {
        m_renderer.reset();
        vmaDestroyAllocator(m_runtime_context.allocator);
        m_runtime_context.allocator = nullptr;
      }
      if (m_command_pool != nullptr) {
        m_device.destroyCommandPool(m_command_pool, nullptr);
      }
      m_device.destroy(nullptr);
    }
    if (m_instance != nullptr) {
      m_instance.destroy(nullptr);
    }
  }

  bool FindMemoryType(u32 type_bits, vk::MemoryPropertyFlags required,
                      u32 *index) const {
    for (u32 i = 0; i < m_memory_properties.memoryTypeCount; i++) {
      if ((type_bits & (1u << i)) == 0) {
        continue;
      }
      if ((m_memory_properties.memoryTypes[i].propertyFlags & required) ==
          required) {
        *index = i;
        return true;
      }
    }
    return false;
  }

  static vk::AccessFlags AccessForLayout(vk::ImageLayout layout) {
    switch (layout) {
    case vk::ImageLayout::eTransferDstOptimal:
      return vk::AccessFlagBits::eTransferWrite;
    case vk::ImageLayout::eTransferSrcOptimal:
      return vk::AccessFlagBits::eTransferRead;
    case vk::ImageLayout::eShaderReadOnlyOptimal:
      return vk::AccessFlagBits::eShaderRead;
    case vk::ImageLayout::eGeneral:
      return vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite;
    default:
      return {};
    }
  }

  vk::ShaderModule CreateShaderModule(const char *shader_name, const std::vector<u32> &spirv) {
    vk::ShaderModuleCreateInfo info{};
    info.codeSize = spirv.size() * sizeof(u32);
    info.pCode = spirv.data();
    vk::ShaderModule module{};
    RequireVk(shader_name, "graphics", m_device.createShaderModule(&info, nullptr, &module),
              "vkCreateShaderModule");
    return module;
  }

  vk::CommandBuffer BeginCommands(const char *shader_name, const char *stage) {
    vk::CommandBufferAllocateInfo cmd_alloc{};
    cmd_alloc.sType = vk::StructureType::eCommandBufferAllocateInfo;
    cmd_alloc.commandPool = m_command_pool;
    cmd_alloc.level = vk::CommandBufferLevel::ePrimary;
    cmd_alloc.commandBufferCount = 1;
    vk::CommandBuffer cmd = nullptr;
    RequireVk(shader_name, stage,
              m_device.allocateCommandBuffers(&cmd_alloc, &cmd),
              "vkAllocateCommandBuffers");

    vk::CommandBufferBeginInfo begin{};
    begin.sType = vk::StructureType::eCommandBufferBeginInfo;
    begin.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit;
    RequireVk(shader_name, stage, cmd.begin(&begin), "vkBeginCommandBuffer");
    return cmd;
  }

  void EndSubmitAndFree(const char *shader_name, const char *stage,
                        vk::CommandBuffer cmd) {
    RequireVk(shader_name, stage, cmd.end(), "vkEndCommandBuffer");

    vk::FenceCreateInfo fence_info{};
    fence_info.sType = vk::StructureType::eFenceCreateInfo;
    vk::Fence fence = nullptr;
    RequireVk(shader_name, stage,
              m_device.createFence(&fence_info, nullptr, &fence),
              "vkCreateFence");

    vk::SubmitInfo submit{};
    submit.sType = vk::StructureType::eSubmitInfo;
    submit.commandBufferCount = 1;
    submit.pCommandBuffers = &cmd;
    RequireVk(shader_name, stage, m_queue.submit(1, &submit, fence),
              "vkQueueSubmit");
    RequireVk(shader_name, stage,
              m_device.waitForFences(1, &fence, true, UINT64_MAX),
              "vkWaitForFences");

    m_device.destroyFence(fence, nullptr);
    m_device.freeCommandBuffers(m_command_pool, 1, &cmd);
  }

  void AddImageBarrier(vk::CommandBuffer cmd, vk::Image image,
                       vk::ImageLayout old_layout, vk::ImageLayout new_layout,
                       vk::PipelineStageFlags src_stage,
                       vk::PipelineStageFlags dst_stage,
                       vk::AccessFlags src_access, vk::AccessFlags dst_access,
                       u32 mip_levels = 1, u32 layers = 1) {
    vk::ImageMemoryBarrier barrier{};
    barrier.sType = vk::StructureType::eImageMemoryBarrier;
    barrier.srcAccessMask = src_access;
    barrier.dstAccessMask = dst_access;
    barrier.oldLayout = old_layout;
    barrier.newLayout = new_layout;
    barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
    barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
    barrier.image = image;
    barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor;
    barrier.subresourceRange.baseMipLevel = 0;
    barrier.subresourceRange.levelCount = mip_levels;
    barrier.subresourceRange.baseArrayLayer = 0;
    barrier.subresourceRange.layerCount = layers;
    cmd.pipelineBarrier(src_stage, dst_stage, {}, 0, nullptr, 0, nullptr, 1,
                        &barrier);
  }

  void TransitionImage(const char *shader_name, Image *image,
                       vk::ImageLayout new_layout,
                       vk::PipelineStageFlags src_stage,
                       vk::PipelineStageFlags dst_stage,
                       vk::AccessFlags src_access, vk::AccessFlags dst_access) {
    vk::CommandBuffer cmd = BeginCommands(shader_name, "dispatch");
    AddImageBarrier(cmd, image->image, image->layout, new_layout, src_stage,
                    dst_stage, src_access, dst_access, image->mip_levels,
                    image->layers);
    EndSubmitAndFree(shader_name, "dispatch", cmd);
    image->layout = new_layout;
  }

  void UploadImage(const char *shader_name, Image *image, vk::Buffer staging,
                   vk::ImageLayout final_layout) {
    UploadImageMips(shader_name, image, staging, final_layout);
  }

  void UploadImageMips(const char *shader_name, Image *image,
                       vk::Buffer staging, vk::ImageLayout final_layout) {
    vk::CommandBuffer cmd = BeginCommands(shader_name, "dispatch");
    AddImageBarrier(cmd, image->image, vk::ImageLayout::eUndefined,
                    vk::ImageLayout::eTransferDstOptimal,
                    vk::PipelineStageFlagBits::eTopOfPipe,
                    vk::PipelineStageFlagBits::eTransfer, {},
                    vk::AccessFlagBits::eTransferWrite, image->mip_levels,
                    image->layers);
    std::vector<vk::BufferImageCopy> copies;
    copies.reserve(image->mip_levels);
    vk::DeviceSize offset = 0;
    for (u32 level = 0; level < image->mip_levels; level++) {
      vk::BufferImageCopy copy{};
      copy.bufferOffset = offset;
      copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
      copy.imageSubresource.mipLevel = level;
      copy.imageSubresource.baseArrayLayer = 0;
      copy.imageSubresource.layerCount = image->layers;
      copy.imageExtent.width = MipExtent(image->width, level);
      copy.imageExtent.height = MipExtent(image->height, level);
      copy.imageExtent.depth = 1;
      copies.push_back(copy);
      offset += static_cast<vk::DeviceSize>(
          ImageMipDwordCount(image->width, image->height,
                             image->dwords_per_pixel, level, image->layers) *
          sizeof(u32));
    }
    cmd.copyBufferToImage(staging, image->image,
                          vk::ImageLayout::eTransferDstOptimal,
                          static_cast<u32>(copies.size()), copies.data());
    AddImageBarrier(cmd, image->image, vk::ImageLayout::eTransferDstOptimal,
                    final_layout, vk::PipelineStageFlagBits::eTransfer,
                    vk::PipelineStageFlagBits::eComputeShader,
                    vk::AccessFlagBits::eTransferWrite,
                    AccessForLayout(final_layout), image->mip_levels,
                    image->layers);
    EndSubmitAndFree(shader_name, "dispatch", cmd);
    image->layout = final_layout;
  }

  Buffer CreateDeviceBuffer(const char *shader_name, vk::DeviceSize size,
                            vk::BufferUsageFlags usage) {
    Buffer ret;
    ret.size = size;
    vk::BufferCreateInfo buffer_info{};
    buffer_info.sType = vk::StructureType::eBufferCreateInfo;
    buffer_info.size = size;
    buffer_info.usage = usage;
    buffer_info.sharingMode = vk::SharingMode::eExclusive;
    RequireVk(shader_name, "dispatch",
              m_device.createBuffer(&buffer_info, nullptr, &ret.buffer),
              "vkCreateBuffer");

    vk::MemoryRequirements requirements{};
    m_device.getBufferMemoryRequirements(ret.buffer, &requirements);
    u32 memory_type = 0;
    Require(shader_name, "dispatch",
            FindMemoryType(requirements.memoryTypeBits,
                           vk::MemoryPropertyFlagBits::eDeviceLocal,
                           &memory_type) ||
                FindMemoryType(requirements.memoryTypeBits, {}, &memory_type),
            "no memory type for BDA support buffer");
    vk::MemoryAllocateInfo allocation{};
    allocation.sType = vk::StructureType::eMemoryAllocateInfo;
    allocation.allocationSize = requirements.size;
    allocation.memoryTypeIndex = memory_type;
    RequireVk(shader_name, "dispatch",
              m_device.allocateMemory(&allocation, nullptr, &ret.memory),
              "vkAllocateMemory");
    RequireVk(shader_name, "dispatch",
              m_device.bindBufferMemory(ret.buffer, ret.memory, 0),
              "vkBindBufferMemory");
    return ret;
  }

  void EnsureBdaBuffers(const char *shader_name) {
    if (m_bda_pagetable_buffer.buffer != nullptr) {
      return;
    }
    const auto usage = vk::BufferUsageFlagBits::eStorageBuffer |
                       vk::BufferUsageFlagBits::eTransferDst;
    m_bda_pagetable_buffer =
        CreateDeviceBuffer(shader_name, BufferCache::BDA_PAGETABLE_SIZE, usage);
    m_fault_buffer = CreateDeviceBuffer(
        shader_name, BufferCache::CACHING_NUMPAGES / 8, usage | vk::BufferUsageFlagBits::eTransferSrc);
  }

  Buffer CreateHostBuffer(const char *shader_name, vk::DeviceSize size,
                          vk::BufferUsageFlags usage,
                          const std::vector<u32> &contents) {
    Buffer ret;
    ret.size = std::max<vk::DeviceSize>(size, sizeof(u32));

    vk::BufferCreateInfo buffer_info{};
    buffer_info.sType = vk::StructureType::eBufferCreateInfo;
    buffer_info.size = ret.size;
    buffer_info.usage = usage;
    buffer_info.sharingMode = vk::SharingMode::eExclusive;
    RequireVk(shader_name, "dispatch",
              m_device.createBuffer(&buffer_info, nullptr, &ret.buffer),
              "vkCreateBuffer");

    vk::MemoryRequirements req{};
    m_device.getBufferMemoryRequirements(ret.buffer, &req);
    u32 memory_type = 0;
    ret.coherent = FindMemoryType(req.memoryTypeBits,
                                  vk::MemoryPropertyFlagBits::eHostVisible |
                                      vk::MemoryPropertyFlagBits::eHostCoherent,
                                  &memory_type);
    if (!ret.coherent) {
      Require(shader_name, "dispatch",
              FindMemoryType(req.memoryTypeBits,
                             vk::MemoryPropertyFlagBits::eHostVisible,
                             &memory_type),
              "no host-visible memory type for staging buffer");
    }

    vk::MemoryAllocateInfo alloc{};
    alloc.sType = vk::StructureType::eMemoryAllocateInfo;
    alloc.allocationSize = req.size;
    alloc.memoryTypeIndex = memory_type;
    RequireVk(shader_name, "dispatch",
              m_device.allocateMemory(&alloc, nullptr, &ret.memory),
              "vkAllocateMemory");
    RequireVk(shader_name, "dispatch",
              m_device.bindBufferMemory(ret.buffer, ret.memory, 0),
              "vkBindBufferMemory");
    if (!contents.empty()) {
      WriteBuffer(shader_name, ret, contents);
    }
    return ret;
  }

  void WriteBuffer(const char *shader_name, const Buffer &buffer,
                   const std::vector<u32> &contents) {
    void *data = nullptr;
    RequireVk(shader_name, "dispatch",
              m_device.mapMemory(buffer.memory, 0, buffer.size, {}, &data),
              "vkMapMemory");
    std::memcpy(data, contents.data(), contents.size() * sizeof(u32));
    if (!buffer.coherent) {
      vk::MappedMemoryRange range{};
      range.sType = vk::StructureType::eMappedMemoryRange;
      range.memory = buffer.memory;
      range.offset = 0;
      range.size = VK_WHOLE_SIZE;
      RequireVk(shader_name, "dispatch",
                m_device.flushMappedMemoryRanges(1, &range),
                "vkFlushMappedMemoryRanges");
    }
    m_device.unmapMemory(buffer.memory);
  }

  vk::Instance m_instance = nullptr;
  vk::PhysicalDevice m_physical_device = nullptr;
  vk::Device m_device = nullptr;
  vk::Queue m_queue = nullptr;
  vk::CommandPool m_command_pool = nullptr;
  u32 m_queue_family = 0;
  vk::PhysicalDeviceMemoryProperties m_memory_properties{};
  Buffer m_bda_pagetable_buffer;
  Buffer m_fault_buffer;
  GraphicContext m_runtime_context{};
  std::unique_ptr<RenderContext> m_renderer;
};

void CompareWords(const TestCase &test, const char *stage,
                  const std::vector<u32> &expected,
                  const std::vector<u32> &actual) {
  if (actual == expected) {
    return;
  }
  if (test.expected_float_tolerance > 0.0f &&
      std::equal(expected.begin(), expected.end(), actual.begin(), actual.end(),
                 [&](u32 expected_word, u32 actual_word) {
                   return std::abs(std::bit_cast<float>(expected_word) -
                                   std::bit_cast<float>(actual_word)) <=
                          test.expected_float_tolerance;
                 })) {
    return;
  }
  std::ostringstream out;
  out << "expected [";
  for (size_t i = 0; i < expected.size(); i++) {
    out << (i == 0 ? "" : ", ") << Hex(expected[i]);
  }
  out << "] actual [";
  for (size_t i = 0; i < actual.size(); i++) {
    out << (i == 0 ? "" : ", ") << Hex(actual[i]);
  }
  out << "]";
  Fail(test.name, stage, out.str());
}

void CompareGraphicsWords(const GraphicsCase &test,
                          const std::vector<u32> &actual) {
  if (actual == test.expected_pixel) {
    return;
  }
  std::ostringstream out;
  out << "expected [";
  for (size_t i = 0; i < test.expected_pixel.size(); i++) {
    out << (i == 0 ? "" : ", ") << Hex(test.expected_pixel[i]);
  }
  out << "] actual [";
  for (size_t i = 0; i < actual.size(); i++) {
    out << (i == 0 ? "" : ", ") << Hex(actual[i]);
  }
  out << "]";
  Fail(test.name, "graphics readback", out.str());
}

void RunCase(VulkanHarness *vulkan, const TestCase &test) {
  auto compiled = CompileCase(test, vulkan != nullptr ? vulkan->SubgroupSize() : 64u);
  if (test.image_descriptor_swizzle != DstSel(4, 5, 6, 7) &&
      !compiled.program.info.images.empty()) {
    Require(test.name, "resource specialization",
            compiled.program.info.images[0].shader_swizzle ==
                    test.image_descriptor_swizzle,
            "storage image descriptor swizzle did not reach the specialized "
            "program");
  }
  if (test.expected_force_point_sampler) {
    Require(test.name, "sampler specialization",
            compiled.program.info.samplers.size() == 1u &&
                compiled.program.info.samplers[0].force_point_filtering,
            "bit-packed sampled image did not force point filtering");
  }
  if (test.sampler_filter == vk::Filter::eLinear) {
    Require(test.name, "sampler specialization",
            std::ranges::none_of(compiled.program.info.samplers,
                                [](const auto &sampler) { return sampler.force_point_filtering; }),
            "filterable sampled image incorrectly forced point filtering");
  }
  if (test.compile_only) {
    std::printf("[compute] %-32s ok\n", test.name);
    return;
  }
  using Kind = ShaderRecompiler::IR::DescriptorBindingKind;
  auto Has = [&](Kind kind) {
    return compiled.program.bindings.descriptor_counts[static_cast<size_t>(kind)] != 0;
  };
  const auto dwords = std::max<size_t>(
      {test.initial.size(), test.expected.size(), static_cast<size_t>(1)});
  auto buffer = vulkan->CreateStorageBuffer(
      test.name, test.initial, dwords, Has(Kind::BdaPagetable));
  VulkanHarness::Image sampled_image;
  VulkanHarness::Image storage_image;
  VulkanHarness::Image storage_image_uint;
  VulkanHarness::Buffer gds_buffer;
  vk::Sampler sampler = nullptr;
  bool needs_sampled_image = false;
  bool needs_storage_image = false;
  for (const auto &image : compiled.program.info.images) {
    const auto resource_class = image.resource_class;
    if (resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) {
      needs_sampled_image = true;
    } else if (resource_class ==
               ShaderRecompiler::IR::ImageResourceClass::Storage) {
      needs_storage_image = true;
    }
  }
  const bool needs_sampler = Has(Kind::Samplers);
  const bool needs_gds = Has(Kind::Gds);
  if (needs_gds) {
    const auto gds_dwords = std::max<size_t>(
        {test.gds_initial.size(), test.expected_gds.size(), 1u});
    gds_buffer =
        vulkan->CreateStorageBuffer(test.name, test.gds_initial, gds_dwords);
  }
  if (needs_sampled_image) {
    auto sampled_mips = test.sampled_image_rgba_mips;
    auto sampled_format = test.sampled_image_format;
    auto sampled_dwords_per_pixel = test.sampled_image_dwords_per_pixel;
    if (!test.sampled_image_rgba_mips.empty()) {
      sampled_format = vk::Format::eR32G32B32A32Sfloat;
      sampled_dwords_per_pixel = 4;
    } else if (!test.sampled_image_rgba.empty()) {
      sampled_mips.push_back(test.sampled_image_rgba);
    }
    sampled_image = vulkan->CreateImageMips(
        test.name, test.image_width, test.image_height, sampled_format,
        vk::ImageUsageFlagBits::eSampled, sampled_mips,
        sampled_dwords_per_pixel, vk::ImageLayout::eShaderReadOnlyOptimal,
        test.sampled_image_type, test.sampled_image_view_type,
        test.sampled_image_layers, test.sampled_image_view_base_layer,
        test.sampled_image_view_layers, vk::SampleCountFlagBits::e1,
        vk::Format::eUndefined, test.sampled_image_view_base_mip);
  }
  if (needs_storage_image) {
    const bool atomic64 = std::ranges::any_of(
        compiled.program.info.images,
        [](const auto &image) { return image.atomic64; });
    storage_image = vulkan->CreateImage2D(
        test.name, test.image_width, test.image_height,
        test.storage_image_format, vk::ImageUsageFlagBits::eStorage,
        test.storage_image_rgba, test.storage_image_dwords_per_pixel,
        vk::ImageLayout::eGeneral);
    storage_image_uint = vulkan->CreateImage2D(
        test.name, test.image_width, test.image_height,
        atomic64 ? vk::Format::eR32G32Uint : vk::Format::eR32Uint,
        vk::ImageUsageFlagBits::eStorage, test.storage_image_r32ui, atomic64 ? 2 : 1,
        vk::ImageLayout::eGeneral,
        atomic64 ? vk::Format::eR64Uint : vk::Format::eR32Uint);
  }
  if (needs_sampler) {
    sampler = vulkan->CreateSampler(
        test.name, sampled_image.mip_levels - test.sampled_image_view_base_mip,
        test.sampler_filter);
  }

  vulkan->Dispatch(test, compiled, buffer, needs_gds ? &gds_buffer : nullptr,
                   needs_sampled_image ? &sampled_image : nullptr,
                   needs_storage_image ? &storage_image : nullptr,
                   needs_storage_image ? &storage_image_uint : nullptr,
                   sampler);
  auto actual = vulkan->ReadBuffer(test.name, buffer, test.expected.size());
  if (!test.expected_gds.empty()) {
    const auto gds_actual =
        vulkan->ReadBuffer(test.name, gds_buffer, test.expected_gds.size());
    CompareWords(test, "GDS readback", test.expected_gds, gds_actual);
  }
  if (!test.expected_storage_image_rgba.empty()) {
    auto image_actual = vulkan->ReadImage(test.name, &storage_image);
    image_actual.resize(test.expected_storage_image_rgba.size());
    CompareWords(test, "storage image readback",
                 test.expected_storage_image_rgba, image_actual);
  }
  if (!test.expected_storage_image_r32ui.empty()) {
    auto image_actual = vulkan->ReadImage(test.name, &storage_image_uint);
    image_actual.resize(test.expected_storage_image_r32ui.size());
    CompareWords(test, "uint storage image readback",
                 test.expected_storage_image_r32ui, image_actual);
  }
  if (sampler != nullptr) {
    vulkan->Device().destroySampler(sampler, nullptr);
  }
  vulkan->DestroyImage(&sampled_image);
  vulkan->DestroyImage(&storage_image);
  vulkan->DestroyImage(&storage_image_uint);
  vulkan->DestroyBuffer(&gds_buffer);
  vulkan->DestroyBuffer(&buffer);
  CompareWords(test, "readback", test.expected, actual);
  std::printf("[compute] %-32s ok\n", test.name);
}

void RunGraphicsCase(VulkanHarness *vulkan, const GraphicsCase &test) {
  if (vulkan != nullptr && !vulkan->RasterizationSupported()) {
    vulkan->SkipRasterizationCases(1);
    return;
  }
  auto compiled = CompileFragmentCase(test);
  auto actual = vulkan->RenderFragment(test, compiled);
  CompareGraphicsWords(test, actual);
  std::printf("[graphics] %-31s ok\n", test.name);
}

enum class CoverageClass {
  Covered,
  ControlOrMarker,
  NeedsAluCase,
  NeedsFloatCase,
  NeedsMemoryCase,
  NeedsImageCase,
  NeedsGraphicsStageCase,
};

CoverageClass ClassifyOpcode(ShaderOpcode opcode,
                             const std::set<ShaderOpcode> &covered) {
  using ShaderRecompiler::Decoder::Opcode;

  if (covered.contains(opcode)) {
    return CoverageClass::Covered;
  }

  switch (opcode) {
  case Opcode::S_GETPC_B64:
  case Opcode::S_SETPC_B64:
  case Opcode::S_NOP:
  case Opcode::S_WAITCNT:
  case Opcode::S_BARRIER:
  case Opcode::S_BRANCH:
  case Opcode::S_CBRANCH_SCC0:
  case Opcode::S_CBRANCH_SCC1:
  case Opcode::S_CBRANCH_VCCZ:
  case Opcode::S_CBRANCH_VCCNZ:
  case Opcode::S_CBRANCH_EXECZ:
  case Opcode::S_CBRANCH_EXECNZ:
  case Opcode::S_SENDMSG:
  case Opcode::S_SETREG_B32:
  case Opcode::S_SLEEP:
  case Opcode::S_TTRACEDATA:
  case Opcode::S_INST_PREFETCH:
  case Opcode::S_ENDPGM:
    return CoverageClass::ControlOrMarker;

  case Opcode::V_ADD_F32:
  case Opcode::V_SUB_F32:
  case Opcode::V_SUBREV_F32:
  case Opcode::V_MUL_F32:
  case Opcode::V_MAC_F32:
  case Opcode::V_MADMK_F32:
  case Opcode::V_MADAK_F32:
  case Opcode::V_FMAC_F32:
  case Opcode::V_FMAMK_F32:
  case Opcode::V_FMAAK_F32:
  case Opcode::V_MIN_F32:
  case Opcode::V_MAX_F32:
  case Opcode::V_MAD_F32:
  case Opcode::V_FMA_F32:
  case Opcode::V_MIN3_F32:
  case Opcode::V_MAX3_F32:
  case Opcode::V_MED3_F32:
  case Opcode::V_DOT2C_F32_F16:
  case Opcode::V_CVT_F64_I32:
  case Opcode::V_CVT_F32_F64:
  case Opcode::V_CVT_F64_F32:
  case Opcode::V_CVT_F64_U32:
  case Opcode::V_RCP_F64:
  case Opcode::V_FRACT_F64:
  case Opcode::V_MUL_F64:
  case Opcode::V_FMA_F64:
  case Opcode::V_CVT_F32_I32:
  case Opcode::V_CVT_F32_U32:
  case Opcode::V_CVT_U32_F32:
  case Opcode::V_CVT_I32_F32:
  case Opcode::V_CVT_F16_F32:
  case Opcode::V_CVT_F32_F16:
  case Opcode::V_CVT_U16_F16:
  case Opcode::V_CVT_RPI_I32_F32:
  case Opcode::V_CVT_FLR_I32_F32:
  case Opcode::V_CVT_OFF_F32_I4:
  case Opcode::V_CVT_F32_UBYTE0:
  case Opcode::V_CVT_F32_UBYTE1:
  case Opcode::V_CVT_F32_UBYTE2:
  case Opcode::V_CVT_F32_UBYTE3:
  case Opcode::V_RCP_F32:
  case Opcode::V_FRACT_F32:
  case Opcode::V_TRUNC_F32:
  case Opcode::V_CEIL_F32:
  case Opcode::V_RNDNE_F32:
  case Opcode::V_FLOOR_F32:
  case Opcode::V_EXP_F32:
  case Opcode::V_LOG_F32:
  case Opcode::V_RSQ_F32:
  case Opcode::V_SQRT_F32:
  case Opcode::V_SIN_F32:
  case Opcode::V_COS_F32:
  case Opcode::V_CUBEID_F32:
  case Opcode::V_CUBESC_F32:
  case Opcode::V_CUBETC_F32:
  case Opcode::V_CUBEMA_F32:
  case Opcode::V_LDEXP_F32:
  case Opcode::V_CVT_PK_U8_F32:
  case Opcode::V_CVT_PKNORM_I16_F32:
  case Opcode::V_CVT_PKNORM_U16_F32:
  case Opcode::V_CVT_PKRTZ_F16_F32:
  case Opcode::V_PK_ADD_F16:
  case Opcode::V_PK_MUL_F16:
  case Opcode::V_PK_MIN_F16:
  case Opcode::V_PK_MAX_F16:
  case Opcode::V_PK_FMA_F16:
  case Opcode::V_ADD_F16:
  case Opcode::V_SUB_F16:
  case Opcode::V_SUBREV_F16:
  case Opcode::V_MUL_F16:
  case Opcode::V_MAX_F16:
  case Opcode::V_MIN_F16:
  case Opcode::V_MIN3_F16:
  case Opcode::V_MAX3_F16:
  case Opcode::V_MED3_F16:
  case Opcode::V_RCP_F16:
  case Opcode::V_RSQ_F16:
  case Opcode::V_LOG_F16:
  case Opcode::V_EXP_F16:
  case Opcode::V_SIN_F16:
  case Opcode::V_COS_F16:
  case Opcode::V_MAD_MIXLO_F16:
  case Opcode::V_MAD_MIXHI_F16:
  case Opcode::DS_MIN_F32:
  case Opcode::DS_MAX_F32:
  case Opcode::V_CMP_F_F32:
  case Opcode::V_CMP_LT_F32:
  case Opcode::V_CMP_EQ_F32:
  case Opcode::V_CMP_LE_F32:
  case Opcode::V_CMP_GT_F32:
  case Opcode::V_CMP_LG_F32:
  case Opcode::V_CMP_GE_F32:
  case Opcode::V_CMP_O_F32:
  case Opcode::V_CMP_U_F32:
  case Opcode::V_CMP_NGE_F32:
  case Opcode::V_CMP_NLG_F32:
  case Opcode::V_CMP_NGT_F32:
  case Opcode::V_CMP_NLE_F32:
  case Opcode::V_CMP_NEQ_F32:
  case Opcode::V_CMP_NLT_F32:
  case Opcode::V_CMP_TRU_F32:
  case Opcode::V_CMPX_LT_F32:
  case Opcode::V_CMPX_EQ_F32:
  case Opcode::V_CMPX_LE_F32:
  case Opcode::V_CMPX_GT_F32:
  case Opcode::V_CMPX_LG_F32:
  case Opcode::V_CMPX_GE_F32:
  case Opcode::V_CMPX_O_F32:
  case Opcode::V_CMPX_NGE_F32:
  case Opcode::V_CMPX_NLG_F32:
  case Opcode::V_CMPX_NGT_F32:
  case Opcode::V_CMPX_NLE_F32:
  case Opcode::V_CMPX_NEQ_F32:
  case Opcode::V_CMPX_NLT_F32:
  case Opcode::V_CMP_CLASS_F32:
  case Opcode::V_CMPX_CLASS_F32:
  case Opcode::V_CMP_CLASS_F16:
  case Opcode::V_CMPX_CLASS_F16:
  case Opcode::V_CMP_LT_F16:
  case Opcode::V_CMP_EQ_F16:
  case Opcode::V_CMP_LE_F16:
  case Opcode::V_CMP_GT_F16:
  case Opcode::V_CMP_LG_F16:
  case Opcode::V_CMP_GE_F16:
  case Opcode::V_CMP_NGT_F16:
  case Opcode::V_CMP_NEQ_F16:
  case Opcode::V_CMP_NLT_F16:
  case Opcode::V_CMPX_LT_F16:
  case Opcode::V_CMPX_EQ_F16:
  case Opcode::V_CMPX_LE_F16:
  case Opcode::V_CMPX_GT_F16:
  case Opcode::V_CMPX_GE_F16:
  case Opcode::V_CMPX_NEQ_F16:
  case Opcode::V_CMPX_NLT_F16:
    return CoverageClass::NeedsFloatCase;

  case Opcode::S_LOAD_DWORD:
  case Opcode::S_LOAD_DWORDX2:
  case Opcode::S_LOAD_DWORDX4:
  case Opcode::S_LOAD_DWORDX8:
  case Opcode::S_LOAD_DWORDX16:
  case Opcode::S_BUFFER_LOAD_DWORD:
  case Opcode::S_BUFFER_LOAD_DWORDX2:
  case Opcode::S_BUFFER_LOAD_DWORDX4:
  case Opcode::S_BUFFER_LOAD_DWORDX8:
  case Opcode::S_BUFFER_LOAD_DWORDX16:
  case Opcode::BUFFER_LOAD_FORMAT_X:
  case Opcode::BUFFER_LOAD_FORMAT_XY:
  case Opcode::BUFFER_LOAD_FORMAT_XYZ:
  case Opcode::BUFFER_LOAD_FORMAT_XYZW:
  case Opcode::BUFFER_STORE_FORMAT_X:
  case Opcode::BUFFER_STORE_FORMAT_XY:
  case Opcode::BUFFER_STORE_FORMAT_XYZ:
  case Opcode::BUFFER_STORE_FORMAT_XYZW:
  case Opcode::BUFFER_LOAD_UBYTE:
  case Opcode::BUFFER_LOAD_SBYTE:
  case Opcode::BUFFER_LOAD_USHORT:
  case Opcode::BUFFER_LOAD_SSHORT:
  case Opcode::BUFFER_LOAD_SHORT_D16:
  case Opcode::BUFFER_LOAD_SHORT_D16_HI:
  case Opcode::BUFFER_LOAD_DWORDX2:
  case Opcode::BUFFER_LOAD_DWORDX3:
  case Opcode::BUFFER_LOAD_DWORDX4:
  case Opcode::BUFFER_STORE_BYTE:
  case Opcode::BUFFER_STORE_SHORT:
  case Opcode::BUFFER_STORE_DWORDX2:
  case Opcode::BUFFER_STORE_DWORDX3:
  case Opcode::BUFFER_STORE_DWORDX4:
  case Opcode::TBUFFER_LOAD_FORMAT_X:
  case Opcode::TBUFFER_LOAD_FORMAT_XY:
  case Opcode::TBUFFER_LOAD_FORMAT_XYZ:
  case Opcode::TBUFFER_LOAD_FORMAT_XYZW:
  case Opcode::TBUFFER_STORE_FORMAT_X:
  case Opcode::TBUFFER_STORE_FORMAT_XY:
  case Opcode::TBUFFER_STORE_FORMAT_XYZ:
  case Opcode::TBUFFER_STORE_FORMAT_XYZW:
  case Opcode::BUFFER_ATOMIC_SWAP:
  case Opcode::BUFFER_ATOMIC_CMPSWAP:
  case Opcode::BUFFER_ATOMIC_ADD:
  case Opcode::BUFFER_ATOMIC_SUB:
  case Opcode::BUFFER_ATOMIC_SMIN:
  case Opcode::BUFFER_ATOMIC_UMIN:
  case Opcode::BUFFER_ATOMIC_SMAX:
  case Opcode::BUFFER_ATOMIC_UMAX:
  case Opcode::BUFFER_ATOMIC_AND:
  case Opcode::BUFFER_ATOMIC_AND_X2:
  case Opcode::BUFFER_ATOMIC_OR:
  case Opcode::BUFFER_ATOMIC_XOR:
  case Opcode::BUFFER_ATOMIC_FMIN:
  case Opcode::BUFFER_ATOMIC_FMAX:
  case Opcode::FLAT_LOAD_UBYTE:
  case Opcode::FLAT_LOAD_SBYTE:
  case Opcode::FLAT_LOAD_USHORT:
  case Opcode::FLAT_LOAD_SSHORT:
  case Opcode::FLAT_LOAD_SHORT_D16:
  case Opcode::FLAT_LOAD_DWORD:
  case Opcode::FLAT_LOAD_DWORDX2:
  case Opcode::FLAT_LOAD_DWORDX3:
  case Opcode::FLAT_LOAD_DWORDX4:
  case Opcode::FLAT_STORE_BYTE:
  case Opcode::FLAT_STORE_SHORT:
  case Opcode::FLAT_STORE_DWORD:
  case Opcode::FLAT_STORE_DWORDX2:
  case Opcode::FLAT_STORE_DWORDX3:
  case Opcode::FLAT_STORE_DWORDX4:
  case Opcode::DS_ADD_U32:
  case Opcode::DS_ADD_U64:
  case Opcode::DS_ADD_RTN_U32:
  case Opcode::DS_SUB_U32:
  case Opcode::DS_SUB_RTN_U32:
  case Opcode::DS_INC_RTN_U32:
  case Opcode::DS_DEC_RTN_U32:
  case Opcode::DS_MIN_I32:
  case Opcode::DS_MIN_RTN_I32:
  case Opcode::DS_MAX_I32:
  case Opcode::DS_MAX_RTN_I32:
  case Opcode::DS_MIN_U32:
  case Opcode::DS_MIN_RTN_U32:
  case Opcode::DS_MAX_U32:
  case Opcode::DS_MAX_RTN_U32:
  case Opcode::DS_AND_B32:
  case Opcode::DS_AND_RTN_B32:
  case Opcode::DS_OR_B32:
  case Opcode::DS_OR_RTN_B32:
  case Opcode::DS_XOR_B32:
  case Opcode::DS_XOR_RTN_B32:
  case Opcode::DS_MSKOR_B32:
  case Opcode::DS_WRXCHG_RTN_B32:
  case Opcode::DS_SWIZZLE_B32:
  case Opcode::DS_PERMUTE_B32:
  case Opcode::DS_BPERMUTE_B32:
  case Opcode::DS_READ_I8:
  case Opcode::DS_READ_U8:
  case Opcode::DS_READ_I16:
  case Opcode::DS_READ_U16:
  case Opcode::DS_READ_U16_D16:
  case Opcode::DS_READ_U16_D16_HI:
  case Opcode::DS_READ2_B32:
  case Opcode::DS_READ_B32:
  case Opcode::DS_READ_B64:
  case Opcode::DS_READ2_B64:
  case Opcode::DS_READ_B96:
  case Opcode::DS_READ_B128:
  case Opcode::DS_WRITE_B8:
  case Opcode::DS_WRITE_B16:
  case Opcode::DS_WRITE_B8_D16_HI:
  case Opcode::DS_WRITE_B16_D16_HI:
  case Opcode::DS_WRITE2_B32:
  case Opcode::DS_WRITE2ST64_B32:
  case Opcode::DS_WRITE_B32:
  case Opcode::DS_WRITE_B64:
  case Opcode::DS_WRITE_B96:
  case Opcode::DS_WRITE_B128:
  case Opcode::DS_WRITE_ADDTID_B32:
  case Opcode::DS_READ_ADDTID_B32:
    return CoverageClass::NeedsMemoryCase;

  case Opcode::IMAGE_GET_RESINFO:
  case Opcode::IMAGE_GET_LOD:
  case Opcode::IMAGE_LOAD:
  case Opcode::IMAGE_LOAD_MIP:
  case Opcode::IMAGE_STORE:
  case Opcode::IMAGE_STORE_MIP:
  case Opcode::IMAGE_ATOMIC_SWAP:
  case Opcode::IMAGE_ATOMIC_ADD:
  case Opcode::IMAGE_ATOMIC_SMIN:
  case Opcode::IMAGE_ATOMIC_SMAX:
  case Opcode::IMAGE_ATOMIC_UMIN:
  case Opcode::IMAGE_ATOMIC_AND:
  case Opcode::IMAGE_ATOMIC_OR:
  case Opcode::IMAGE_ATOMIC_XOR:
  case Opcode::IMAGE_ATOMIC_FMIN:
  case Opcode::IMAGE_ATOMIC_FMAX:
  case Opcode::IMAGE_SAMPLE:
  case Opcode::IMAGE_GATHER4_LZ:
  case Opcode::IMAGE_GATHER4_C:
  case Opcode::IMAGE_GATHER4_C_LZ:
  case Opcode::IMAGE_GATHER4_LZ_O:
  case Opcode::IMAGE_GATHER4_C_O:
  case Opcode::IMAGE_GATHER4_C_LZ_O:
    return CoverageClass::NeedsImageCase;

  case Opcode::V_INTERP_P1_F32:
  case Opcode::V_INTERP_P2_F32:
  case Opcode::V_INTERP_MOV_F32:
  case Opcode::EXP:
    return CoverageClass::NeedsGraphicsStageCase;

  default:
    return CoverageClass::NeedsAluCase;
  }
}

const char *CoverageClassName(CoverageClass status) {
  switch (status) {
  case CoverageClass::Covered:
    return "covered";
  case CoverageClass::ControlOrMarker:
    return "control";
  case CoverageClass::NeedsAluCase:
    return "alu";
  case CoverageClass::NeedsFloatCase:
    return "float";
  case CoverageClass::NeedsMemoryCase:
    return "memory";
  case CoverageClass::NeedsImageCase:
    return "image";
  case CoverageClass::NeedsGraphicsStageCase:
    return "graphics";
  default:
    return "unknown";
  }
}

void PrintPendingOpcodes(CoverageClass status,
                         const std::vector<ShaderOpcode> &opcodes) {
  if (opcodes.empty()) {
    return;
  }

  std::printf("[coverage] pending_%s:", CoverageClassName(status));
  for (auto opcode : opcodes) {
    const auto name = magic_enum::enum_name(opcode);
    std::printf(" %.*s", static_cast<int>(name.size()), name.data());
  }
  std::printf("\n");
}

void CheckOpcodeCoverage(const std::vector<TestCase> &tests,
                         const std::vector<GraphicsCase> &graphics_tests) {
  using ShaderRecompiler::Decoder::Opcode;

  std::set<Opcode> covered;
  for (const auto &test : tests) {
    for (auto opcode : test.opcodes) {
      covered.insert(opcode);
    }
  }
  for (const auto &test : graphics_tests) {
    for (auto opcode : test.opcodes) {
      covered.insert(opcode);
    }
  }

  uint32_t counts[7] = {};
  std::vector<ShaderOpcode> pending[7];
  for (auto value = static_cast<int>(Opcode::S_MOV_B32);
       value < static_cast<int>(Opcode::COUNT); value++) {
    const auto opcode = static_cast<Opcode>(value);
    const auto status = ClassifyOpcode(opcode, covered);
    counts[static_cast<uint32_t>(status)]++;
    if (status != CoverageClass::Covered &&
        status != CoverageClass::ControlOrMarker) {
      pending[static_cast<uint32_t>(status)].push_back(opcode);
    }
  }

  std::printf(
      "[coverage] decoder opcodes: covered=%u control=%u alu_pending=%u "
      "float_pending=%u memory_pending=%u image_pending=%u "
      "graphics_pending=%u\n",
      counts[static_cast<uint32_t>(CoverageClass::Covered)],
      counts[static_cast<uint32_t>(CoverageClass::ControlOrMarker)],
      counts[static_cast<uint32_t>(CoverageClass::NeedsAluCase)],
      counts[static_cast<uint32_t>(CoverageClass::NeedsFloatCase)],
      counts[static_cast<uint32_t>(CoverageClass::NeedsMemoryCase)],
      counts[static_cast<uint32_t>(CoverageClass::NeedsImageCase)],
      counts[static_cast<uint32_t>(CoverageClass::NeedsGraphicsStageCase)]);
  PrintPendingOpcodes(
      CoverageClass::NeedsAluCase,
      pending[static_cast<uint32_t>(CoverageClass::NeedsAluCase)]);
  PrintPendingOpcodes(
      CoverageClass::NeedsFloatCase,
      pending[static_cast<uint32_t>(CoverageClass::NeedsFloatCase)]);
  PrintPendingOpcodes(
      CoverageClass::NeedsMemoryCase,
      pending[static_cast<uint32_t>(CoverageClass::NeedsMemoryCase)]);
  PrintPendingOpcodes(
      CoverageClass::NeedsImageCase,
      pending[static_cast<uint32_t>(CoverageClass::NeedsImageCase)]);
  PrintPendingOpcodes(
      CoverageClass::NeedsGraphicsStageCase,
      pending[static_cast<uint32_t>(CoverageClass::NeedsGraphicsStageCase)]);
}

TestCase IntegerAddSubMul() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeSMovB32(0, InlineU32(7)),
      EncodeSMovB32(1, InlineU32(3)),
      EncodeSop2(0x00, 2, 0, 1),
      EncodeSop2(0x01, 3, 2, InlineU32(1)),
      EncodeSop2(0x26, 4, 3, InlineU32(2)),
      EncodeVop1(0x01, 0, 4),
  };
  AppendBufferStoreDword(&code, 0, 30);
  AppendEnd(&code);
  return {"IntegerAddSubMul",
          code,
          {},
          {18},
          {O::S_MOV_B32, O::S_ADD_U32, O::S_SUB_U32, O::S_MUL_I32, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase BitwiseOps() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeSMovB32(0, InlineU32(60)), EncodeSMovB32(1, InlineU32(15)),
      EncodeSop2(0x0e, 2, 0, 1),       EncodeSop2(0x10, 3, 0, 1),
      EncodeSop2(0x12, 4, 3, 2),       EncodeVop1(0x37, 0, 4),
  };
  AppendBufferStoreDword(&code, 0, 30);
  AppendEnd(&code);
  return {"BitwiseAndOrXorNot",
          code,
          {},
          {~0x33u},
          {O::S_MOV_B32, O::S_AND_B32, O::S_OR_B32, O::S_XOR_B32, O::V_NOT_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase Shifts() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeSMovB32(0, InlineU32(3)),
      EncodeSop2(0x1e, 1, 0, InlineU32(2)),
      EncodeSop2(0x20, 2, 1, InlineU32(1)),
      EncodeVop1(0x01, 0, 2),
  };
  AppendBufferStoreDword(&code, 0, 30);
  AppendEnd(&code);
  return {"Shifts",
          code,
          {},
          {6},
          {O::S_MOV_B32, O::S_LSHL_B32, O::S_LSHR_B32, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarZeroShiftWithRuntimeCount() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeSop2(0x20, 8, InlineU32(0), 4),
      EncodeSop2(0x20, 9, 6, 4),
  };
  AppendStoreSgpr(&code, 8, 0);
  AppendStoreSgpr(&code, 9, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "ScalarZeroShiftWithRuntimeCount";
  test.code = std::move(code);
  test.expected = {0, 0x40000000u};
  test.opcodes = {O::S_LSHR_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(0, 2 * sizeof(u32));
  test.user_data[4] = 33;
  test.user_data[6] = 0x80000000u;
  test.has_user_data = true;
  test.ir_counts = {{" = ShiftRightLogical32 ", 1}};
  test.required_spirv = {"OpShiftRightLogical"};
  return test;
}

TestCase ExactPushConstantExtent() {
  using O = ShaderOpcode;

  constexpr std::array<u32, 10> values{
      0x10203040u, 0x21314151u, 0x32425262u, 0x43536373u, 0x54647484u,
      0x65758595u, 0x768696a6u, 0x8797a7b7u, 0x98a8b8c8u, 0xa9b9c9d9u};
  std::vector<u32> code;
  for (u32 i = 0; i < values.size(); i++) {
    AppendStoreSgpr(&code, 4u + i, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "ExactPushConstantExtent";
  test.code = std::move(code);
  test.initial.resize(values.size());
  test.expected.assign(values.begin(), values.end());
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data =
      MakeStructuredStorageBufferData(0, values.size() * sizeof(u32));
  std::copy(values.begin(), values.end(), test.user_data.begin() + 4);
  test.has_user_data = true;
  return test;
}

TestCase ScalarShiftCountsMaskLowBits() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSMovB32(0, InlineU32(1)));
  AppendSMovLiteral(&code, 1, 0x80000000u);
  code.push_back(EncodeSMovB32(2, InlineU32(32)));
  code.push_back(EncodeSMovB32(3, InlineU32(33)));
  code.push_back(EncodeSop2(0x1e, 10, 0, 2));
  code.push_back(EncodeSop2(0x1e, 11, 0, 3));
  code.push_back(EncodeSop2(0x20, 12, 1, 2));
  code.push_back(EncodeSop2(0x20, 13, 1, 3));
  code.push_back(EncodeSop2(0x22, 14, 1, 2));
  code.push_back(EncodeSop2(0x22, 15, 1, 3));

  for (u32 i = 0; i < 6u; i++) {
    AppendStoreSgpr(&code, 10u + i, i);
  }
  AppendEnd(&code);

  return {"ScalarShiftCountsMaskLowBits",
          code,
          {},
          {1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u},
          {O::S_MOV_B32, O::S_LSHL_B32, O::S_LSHR_B32, O::S_ASHR_I32,
           O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase Rdna2ScalarOpcodes() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSMovB32(2, InlineU32(7)));
  code.push_back(EncodeSMovB32(106, InlineU32(16)));
  code.push_back(EncodeSop1(0x1d, 106, InlineU32(0)));
  AppendStoreSgpr(&code, 106, 0);
  code.push_back(EncodeSopk(0x13, 106, 0x1019u));
  AppendStoreSgpr(&code, 106, 1);
  code.push_back(EncodeSop2(0x02, 106, 2, 239u));
  AppendStoreSgpr(&code, 106, 2);
  code.push_back(EncodeSopp(0x0e, 0));
  code.push_back(EncodeSop2(0x02, 106, 239u, 2));
  AppendStoreSgpr(&code, 106, 3);
  AppendEnd(&code);

  return {"Rdna2ScalarOpcodes",
          code,
          {},
          {0x11u, 0x11u, 7u, 7u},
          {O::S_MOV_B32, O::S_BITSET1_B32, O::S_SETREG_B32, O::S_ADD_I32,
           O::S_SLEEP, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarTrapDisabledFallsThroughExactRaw() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 0, 7);
  code.push_back(0xbf920001u); // s_trap 1 from the game shader
  AppendVMovU32(&code, 0, 42);
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ScalarTrapDisabledFallsThroughExactRaw";
  test.code = code;
  test.expected = {42};
  test.opcodes = {O::V_MOV_B32, O::S_TRAP, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"S_TRAP 0x00000001", 1}};
  test.compute_info.threads_num[0] = 1;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.wave_size = 32;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarWaitcntDepctrCapturedVmVsrc() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 7, 7);
  AppendVMovU32(&code, 31, 0);
  AppendBufferStoreDword(&code, 7, 31);
  code.push_back(0xbfa3ffe3u); // s_waitcnt_depctr vm_vsrc(0)
  AppendVMovU32(&code, 7, 42);
  AppendStoreVgpr(&code, 7, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "ScalarWaitcntDepctrCapturedVmVsrc";
  test.code = std::move(code);
  test.initial = {0, 0};
  test.expected = {7, 42};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_WAITCNT_DEPCTR,
                  O::S_ENDPGM};
  test.decoded_counts = {{"0x00000010: S_WAITCNT_DEPCTR 0x0000ffe3\n", 1}};
  test.forbidden_spirv = {"OpControlBarrier", "OpMemoryBarrier"};
  return test;
}

TestCase ScalarExtendedArithmetic() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 0, 0xffffffffu);
  code.push_back(EncodeSMovB32(1, InlineU32(1)));
  code.push_back(EncodeSop2(0x00, 2, 0, 1));
  code.push_back(EncodeSMovB32(3, InlineU32(5)));
  code.push_back(EncodeSMovB32(4, InlineU32(6)));
  code.push_back(EncodeSop2(0x04, 5, 3, 4));
  code.push_back(EncodeSop2(0x02, 6, InlineU32(7), InlineU32(8)));
  code.push_back(EncodeSop2(0x03, 7, InlineU32(7), InlineU32(9)));
  AppendSMovLiteral(&code, 8, 0xfffffffbu);
  code.push_back(EncodeSMovB32(9, InlineU32(3)));
  code.push_back(EncodeSop2(0x06, 10, 8, 9));
  code.push_back(EncodeSop2(0x08, 11, 8, 9));
  AppendSMovLiteral(&code, 12, 0xfffffffeu);
  code.push_back(EncodeSMovB32(13, InlineU32(3)));
  code.push_back(EncodeSop2(0x07, 14, 12, 13));
  code.push_back(EncodeSop2(0x09, 15, 12, 13));
  code.push_back(EncodeSMovB32(16, 8));
  code.push_back(EncodeSop1(0x34, 16, 16));
  code.push_back(EncodeSopk(0x00, 17, 0xfff5u));
  code.push_back(EncodeSopk(0x10, 17, 0xfffdu));

  const u32 results[] = {2, 5, 6, 7, 10, 11, 14, 15, 16, 17};
  for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
    AppendStoreSgpr(&code, results[i], i);
  }
  AppendEnd(&code);

  return {"ScalarExtendedArithmetic",
          code,
          {},
          {0, 12, 15, 0xfffffffeu, 0xfffffffbu, 3, 3, 0xfffffffeu, 5, 33},
          {O::S_MOV_B32, O::S_ADD_U32, O::S_ADDC_U32, O::S_ADD_I32,
           O::S_SUB_I32, O::S_MIN_I32, O::S_MAX_I32, O::S_MIN_U32, O::S_MAX_U32,
           O::S_ABS_I32, O::S_MOVK_I32, O::S_MULK_I32, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarArithmeticSccCarryBorrowOverflow() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  auto set_scc = [&](bool value) {
    code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
  };
  auto capture_scc = [&](u32 dst_sgpr) {
    code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
  };
  auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) {
    set_scc(prior_scc);
    code.push_back(instruction);
    capture_scc(out_sgpr);
  };

  code.push_back(EncodeSMovB32(0, InlineU32(0)));
  code.push_back(EncodeSMovB32(1, InlineU32(1)));
  append_case(false, EncodeSop2(0x01, 10, 0, 1), 20);

  code.push_back(EncodeSMovB32(2, InlineU32(5)));
  code.push_back(EncodeSMovB32(3, InlineU32(3)));
  append_case(true, EncodeSop2(0x01, 11, 2, 3), 21);

  AppendSMovLiteral(&code, 4, 0x7fffffffu);
  code.push_back(EncodeSMovB32(5, InlineU32(1)));
  append_case(false, EncodeSop2(0x02, 12, 4, 5), 22);

  code.push_back(EncodeSMovB32(6, InlineU32(1)));
  code.push_back(EncodeSMovB32(7, InlineU32(2)));
  append_case(true, EncodeSop2(0x02, 13, 6, 7), 23);

  AppendSMovLiteral(&code, 8, 0x80000000u);
  code.push_back(EncodeSMovB32(9, InlineU32(1)));
  append_case(false, EncodeSop2(0x03, 14, 8, 9), 24);

  code.push_back(EncodeSMovB32(15, InlineU32(5)));
  code.push_back(EncodeSMovB32(16, InlineU32(3)));
  append_case(true, EncodeSop2(0x03, 17, 15, 16), 25);

  AppendSMovLiteral(&code, 18, 0x7fffffffu);
  set_scc(false);
  code.push_back(EncodeSopk(0x0f, 18, 1));
  capture_scc(26);

  code.push_back(EncodeSMovB32(19, InlineU32(4)));
  set_scc(true);
  code.push_back(EncodeSopk(0x0f, 19, 0xfffeu));
  capture_scc(27);

  for (u32 i = 0; i < 8u; i++) {
    AppendStoreSgpr(&code, 20u + i, i);
  }
  AppendEnd(&code);

  return {"ScalarArithmeticSccCarryBorrowOverflow",
          code,
          {},
          {1, 0, 1, 0, 1, 0, 1, 0},
          {O::S_MOV_B32, O::S_SUB_U32, O::S_ADD_I32, O::S_SUB_I32,
           O::S_CMP_EQ_U32, O::S_CSELECT_B32, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarMinMaxSccComparisonEdges() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  auto set_scc = [&](bool value) {
    code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
  };
  auto capture_scc = [&](u32 dst_sgpr) {
    code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
  };
  auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) {
    set_scc(prior_scc);
    code.push_back(instruction);
    capture_scc(out_sgpr);
  };

  AppendSMovLiteral(&code, 0, 0xfffffffeu);
  code.push_back(EncodeSMovB32(1, InlineU32(3)));
  append_case(false, EncodeSop2(0x06, 10, 0, 1), 20);
  append_case(true, EncodeSop2(0x06, 11, 1, 0), 21);
  append_case(false, EncodeSop2(0x08, 12, 1, 0), 22);
  append_case(true, EncodeSop2(0x08, 13, 0, 1), 23);

  code.push_back(EncodeSMovB32(2, InlineU32(2)));
  code.push_back(EncodeSMovB32(3, InlineU32(3)));
  append_case(false, EncodeSop2(0x07, 14, 2, 3), 24);
  append_case(true, EncodeSop2(0x07, 15, 3, 2), 25);
  append_case(false, EncodeSop2(0x09, 16, 3, 2), 26);
  append_case(true, EncodeSop2(0x09, 17, 2, 3), 27);

  for (u32 i = 0; i < 8u; i++) {
    AppendStoreSgpr(&code, 20u + i, i);
  }
  AppendEnd(&code);

  return {"ScalarMinMaxSccComparisonEdges",
          code,
          {},
          {1, 0, 1, 0, 1, 0, 1, 0},
          {O::S_MOV_B32, O::S_MIN_I32, O::S_MAX_I32, O::S_MIN_U32, O::S_MAX_U32,
           O::S_CMP_EQ_U32, O::S_CSELECT_B32, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarAbsI32UpdatesScc() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  auto set_scc = [&](bool value) {
    code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
  };
  auto capture_scc = [&](u32 dst_sgpr) {
    code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
  };

  AppendSmemLoadOpcode(&code, 0x08, 16, 0);
  set_scc(true);
  code.push_back(EncodeSop1(0x34, 17, 16));
  capture_scc(18);

  AppendSmemLoadOpcode(&code, 0x08, 19, 4);
  set_scc(false);
  code.push_back(EncodeSop1(0x34, 20, 19));
  capture_scc(21);

  AppendSmemLoadOpcode(&code, 0x08, 22, 8);
  set_scc(false);
  code.push_back(EncodeSop1(0x34, 23, 22));
  capture_scc(24);

  AppendStoreSgpr(&code, 17, 0);
  AppendStoreSgpr(&code, 18, 1);
  AppendStoreSgpr(&code, 20, 2);
  AppendStoreSgpr(&code, 21, 3);
  AppendStoreSgpr(&code, 23, 4);
  AppendStoreSgpr(&code, 24, 5);
  AppendEnd(&code);

  return {"ScalarAbsI32UpdatesScc",
          code,
          {0, 0xfffffffbu, 0x80000000u},
          {0, 0, 5, 1, 0x80000000u, 1},
          {O::S_BUFFER_LOAD_DWORD, O::S_CMP_EQ_U32, O::S_ABS_I32, O::S_CSELECT_B32,
           O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarAbsdiffBitset64AndSetprio() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  auto capture_scc = [&](u32 dst_sgpr) {
    code.push_back(
        EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
  };

  code.push_back(EncodeSMovB32(0, InlineU32(2)));
  code.push_back(EncodeSMovB32(1, InlineU32(5)));
  code.push_back(EncodeSop2(0x2c, 10, 0, 1));
  capture_scc(20);
  AppendSMovLiteral(&code, 0, 0xffffffffu);
  code.push_back(EncodeSMovB32(1, InlineU32(0)));
  code.push_back(EncodeSop2(0x2c, 11, 0, 1));
  capture_scc(21);
  AppendSMovLiteral(&code, 0, 0x80000000u);
  code.push_back(EncodeSop2(0x2c, 12, 0, 1));
  capture_scc(22);
  code.push_back(EncodeSMovB32(1, InlineU32(1)));
  code.push_back(EncodeSop2(0x2c, 13, 0, 1));
  capture_scc(23);
  code.push_back(EncodeSop2(0x2c, 14, 0, 0));
  capture_scc(24);

  AppendSMovLiteral(&code, 2, 65u);
  AppendSMovLiteral(&code, 3, 101u);
  code.push_back(EncodeSMovB32(4, InlineU32(0)));
  code.push_back(EncodeSMovB32(5, InlineU32(0)));
  code.push_back(EncodeSop1(0x1e, 4, 2));
  AppendStoreSgprPair(&code, 4, 10);
  code.push_back(EncodeSop1(0x1e, 4, 3));
  AppendStoreSgprPair(&code, 4, 12);
  code.push_back(EncodeSop1(0x1c, 4, 2));
  AppendStoreSgprPair(&code, 4, 14);
  code.push_back(EncodeSop1(0x1c, 4, 3));
  AppendStoreSgprPair(&code, 4, 16);
  code.push_back(EncodeSopp(0x0f, 3));

  AppendStoreSgpr(&code, 10, 0);
  AppendStoreSgpr(&code, 11, 1);
  AppendStoreSgpr(&code, 12, 2);
  AppendStoreSgpr(&code, 13, 3);
  AppendStoreSgpr(&code, 14, 4);
  AppendStoreSgpr(&code, 20, 5);
  AppendStoreSgpr(&code, 21, 6);
  AppendStoreSgpr(&code, 22, 7);
  AppendStoreSgpr(&code, 23, 8);
  AppendStoreSgpr(&code, 24, 9);
  AppendEnd(&code);

  TestCase test;
  test.name = "ScalarAbsdiffBitset64AndSetprio";
  test.code = std::move(code);
  test.expected = {3u, 1u, 0x80000000u, 0x7fffffffu, 0u,
                   1u, 1u, 1u,          1u,          0u,
                   2u, 0u, 2u,          32u,         0u,
                   32u, 0u, 0u};
  test.opcodes = {O::S_MOV_B32,     O::S_ABSDIFF_I32, O::S_CSELECT_B32,
                  O::S_BITSET1_B64, O::S_BITSET0_B64, O::S_SETPRIO,
                  O::V_MOV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"S_ABSDIFF_I32", 5},
                         {"S_BITSET1_B64", 2},
                         {"S_BITSET0_B64", 2},
                         {"S_SETPRIO 0x00000003", 1}};
  return test;
}

TestCase ScalarShiftLeftAddSccCarryEdges() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  auto set_scc = [&](bool value) {
    code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
  };
  auto capture_scc = [&](u32 dst_sgpr) {
    code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
  };
  auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) {
    set_scc(prior_scc);
    code.push_back(instruction);
    capture_scc(out_sgpr);
  };

  AppendSMovLiteral(&code, 0, 0x80000000u);
  AppendSMovLiteral(&code, 1, 0x40000000u);
  AppendSMovLiteral(&code, 2, 0x20000000u);
  AppendSMovLiteral(&code, 3, 0x10000000u);
  code.push_back(EncodeSMovB32(4, InlineU32(1)));
  code.push_back(EncodeSMovB32(5, InlineU32(0)));
  append_case(false, EncodeSop2(0x2e, 10, 0, 5), 20);
  append_case(true, EncodeSop2(0x2e, 11, 4, 4), 21);
  append_case(false, EncodeSop2(0x2f, 12, 1, 5), 22);
  append_case(true, EncodeSop2(0x2f, 13, 4, 4), 23);
  append_case(false, EncodeSop2(0x30, 14, 2, 5), 24);
  append_case(true, EncodeSop2(0x30, 15, 4, 4), 25);
  append_case(false, EncodeSop2(0x31, 16, 3, 5), 26);
  append_case(true, EncodeSop2(0x31, 17, 4, 4), 27);

  for (u32 i = 0; i < 8u; i++) {
    AppendStoreSgpr(&code, 20u + i, i);
  }
  AppendEnd(&code);

  return {"ScalarShiftLeftAddSccCarryEdges",
          code,
          {},
          {1, 0, 1, 0, 1, 0, 1, 0},
          {O::S_MOV_B32, O::S_LSHL1_ADD_U32, O::S_LSHL2_ADD_U32,
           O::S_LSHL3_ADD_U32, O::S_LSHL4_ADD_U32, O::S_CMP_EQ_U32,
           O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarCompareOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 0, 0xfffffffeu);
  code.push_back(EncodeSMovB32(1, InlineU32(1)));
  code.push_back(EncodeSMovB32(2, InlineU32(5)));
  code.push_back(EncodeSMovB32(3, InlineU32(2)));
  code.push_back(EncodeSMovB32(4, InlineU32(3)));
  AppendSMovLiteral(&code, 10, 0x00000001u);
  code.push_back(EncodeSMovB32(11, InlineU32(0)));
  code.push_back(EncodeSMovB32(12, InlineU32(2)));
  code.push_back(EncodeSMovB32(13, InlineU32(0)));
  code.push_back(EncodeSMovB32(14, InlineU32(1)));
  code.push_back(EncodeSMovB32(15, InlineU32(1)));

  u32 dst = 20;
  auto append_compare = [&](u32 opcode, u32 src0, u32 src1) {
    code.push_back(EncodeSopc(opcode, src0, src1));
    code.push_back(EncodeSop2(0x0a, dst++, InlineU32(1), InlineU32(0)));
  };

  append_compare(0x00, 2, 2);
  append_compare(0x01, 0, 1);
  append_compare(0x02, 1, 0);
  append_compare(0x03, 1, 1);
  append_compare(0x04, 0, 1);
  append_compare(0x05, 0, 0);
  append_compare(0x07, 3, 4);
  append_compare(0x08, 4, 3);
  append_compare(0x09, 4, 4);
  append_compare(0x0b, 3, 4);
  append_compare(0x12, 10, 10);
  append_compare(0x12, 10, 12);
  append_compare(0x12, 10, 14);
  append_compare(0x13, 10, 12);
  append_compare(0x13, 10, 14);

  for (u32 i = 0; i < 15u; i++) {
    AppendStoreSgpr(&code, 20u + i, i);
  }
  AppendEnd(&code);

  return {"ScalarCompareOps",
          code,
          {},
          {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1},
          {O::S_MOV_B32, O::S_CMP_EQ_I32, O::S_CMP_LG_I32, O::S_CMP_GT_I32,
           O::S_CMP_GE_I32, O::S_CMP_LT_I32, O::S_CMP_LE_I32, O::S_CMP_LG_U32,
           O::S_CMP_GT_U32, O::S_CMP_GE_U32, O::S_CMP_LE_U32, O::S_CMP_EQ_U64,
           O::S_CMP_LG_U64, O::S_CSELECT_B32, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarShiftAddAndMaskOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSMovB32(0, InlineU32(2)));
  code.push_back(EncodeSMovB32(1, InlineU32(3)));
  code.push_back(EncodeSop2(0x2e, 2, 0, 1));
  code.push_back(EncodeSop2(0x2f, 3, 0, 1));
  code.push_back(EncodeSop2(0x30, 4, 0, 1));
  code.push_back(EncodeSop2(0x31, 5, 0, 1));
  AppendSMovLiteral(&code, 6, 0xfffffff8u);
  code.push_back(EncodeSop2(0x22, 7, 6, InlineU32(2)));
  AppendSMovLiteral(&code, 8, 0xffffffffu);
  code.push_back(EncodeSMovB32(9, InlineU32(2)));
  code.push_back(EncodeSop2(0x35, 10, 8, 9));
  AppendSMovLiteral(&code, 12, 0x80000011u);
  AppendSMovLiteral(&code, 13, 0x80000001u);
  code.push_back(EncodeSop1(0x08, 14, 12));
  code.push_back(EncodeSop1(0x0a, 16, 12));

  AppendStoreSgpr(&code, 2, 0);
  AppendStoreSgpr(&code, 3, 1);
  AppendStoreSgpr(&code, 4, 2);
  AppendStoreSgpr(&code, 5, 3);
  AppendStoreSgpr(&code, 7, 4);
  AppendStoreSgpr(&code, 10, 5);
  AppendStoreSgprPair(&code, 14, 6);
  AppendStoreSgprPair(&code, 16, 8);
  AppendEnd(&code);

  return {"ScalarShiftAddAndMaskOps",
          code,
          {},
          {7, 11, 19, 35, 0xfffffffeu, 1, 0x7fffffeeu, 0x7ffffffeu, 0xf00000ffu,
           0xf000000fu},
          {O::S_MOV_B32, O::S_LSHL1_ADD_U32, O::S_LSHL2_ADD_U32,
           O::S_LSHL3_ADD_U32, O::S_LSHL4_ADD_U32, O::S_ASHR_I32,
           O::S_MUL_HI_U32, O::S_NOT_B64, O::S_WQM_B64, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarNotB64UpdatesScc() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  auto set_scc = [&](bool value) {
    code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
  };
  auto capture_scc = [&](u32 dst_sgpr) {
    code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
  };

  AppendSMovLiteral(&code, 0, 0xffffffffu);
  AppendSMovLiteral(&code, 1, 0xffffffffu);
  set_scc(true);
  code.push_back(EncodeSop1(0x08, 2, 0));
  capture_scc(4);

  AppendSMovLiteral(&code, 6, 0xfffffffeu);
  AppendSMovLiteral(&code, 7, 0xffffffffu);
  set_scc(false);
  code.push_back(EncodeSop1(0x08, 8, 6));
  capture_scc(10);

  code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
  code.push_back(EncodeSop1(0x04, 12, 106));
  code.push_back(EncodeSop1(0x08, 14, 12));
  capture_scc(16);

  AppendStoreSgpr(&code, 4, 0);
  AppendStoreSgpr(&code, 10, 1);
  AppendStoreSgprPair(&code, 2, 2);
  AppendStoreSgprPair(&code, 8, 4);
  AppendStoreSgpr(&code, 16, 6);
  AppendStoreSgprPair(&code, 14, 7);
  AppendEnd(&code);

  return {"ScalarNotB64UpdatesScc",
          code,
          {},
          {0, 1, 0, 0, 1, 0, 1, 0xfffffffeu, 0xffffffffu},
          {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_NOT_B64, O::S_CSELECT_B32,
           O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase ScalarQuadmaskB64() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  auto capture_scc = [&](u32 dst_sgpr) {
    code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
  };

  AppendSMovLiteral(&code, 106, 0x80000011u);
  AppendSMovLiteral(&code, 107, 0x000000f0u);
  code.push_back(EncodeSop1(0x2d, 106, 106));
  capture_scc(0);
  AppendStoreSgprPair(&code, 106, 0);
  AppendStoreSgpr(&code, 0, 2);

  code.push_back(EncodeSMovB32(106, InlineU32(0)));
  code.push_back(EncodeSMovB32(107, InlineU32(0)));
  code.push_back(EncodeSop1(0x2d, 106, 106));
  capture_scc(0);
  AppendStoreSgprPair(&code, 106, 3);
  AppendStoreSgpr(&code, 0, 5);
  AppendEnd(&code);

  return {"ScalarQuadmaskB64",
          code,
          {},
          {0x00000283u, 0, 1, 0, 0, 0},
          {O::S_MOV_B32, O::S_QUADMASK_B64, O::S_CSELECT_B32, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarFlbitI32B64Gpu() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 0, 0x00000000u);
  AppendSMovLiteral(&code, 1, 0x80000000u);
  code.push_back(EncodeSop1(0x16, 20, 0));

  AppendSMovLiteral(&code, 2, 0x00008000u);
  AppendSMovLiteral(&code, 3, 0x00000000u);
  code.push_back(EncodeSop1(0x16, 21, 2));

  AppendSMovLiteral(&code, 4, 0x00000000u);
  AppendSMovLiteral(&code, 5, 0x00000000u);
  code.push_back(EncodeSop1(0x16, 22, 4));

  AppendSMovLiteral(&code, 14, 0x00000008u);
  AppendSMovLiteral(&code, 15, 0x00000000u);
  code.push_back(EncodeSop1(0x16, 106, 14));

  AppendStoreSgpr(&code, 20, 0);
  AppendStoreSgpr(&code, 21, 1);
  AppendStoreSgpr(&code, 22, 2);
  AppendStoreSgpr(&code, 106, 3);
  AppendEnd(&code);

  return {"ScalarFlbitI32B64Gpu",
          code,
          {},
          {0, 48, 0xffffffffu, 60},
          {O::S_MOV_B32, O::S_FLBIT_I32_B64, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarFlbitI32B32Gpu() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 0, 0x00000000u);
  code.push_back(EncodeSop1(0x15, 20, 0));

  AppendSMovLiteral(&code, 0, 0x0000ccccu);
  code.push_back(EncodeSop1(0x15, 21, 0));

  AppendSMovLiteral(&code, 0, 0x80000000u);
  code.push_back(EncodeSop1(0x15, 22, 0));

  AppendSMovLiteral(&code, 0, 0x7fffffffu);
  code.push_back(EncodeSop1(0x15, 23, 0));

  AppendStoreSgpr(&code, 20, 0);
  AppendStoreSgpr(&code, 21, 1);
  AppendStoreSgpr(&code, 22, 2);
  AppendStoreSgpr(&code, 23, 3);
  AppendEnd(&code);

  return {"ScalarFlbitI32B32Gpu",
          code,
          {},
          {0xffffffffu, 16, 0, 1},
          {O::S_MOV_B32, O::S_FLBIT_I32_B32, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarSaveExecOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 106, 0xffffffffu);
  AppendSMovLiteral(&code, 107, 0xffffffffu);
  code.push_back(0xbe80246au);
  code.push_back(EncodeSop1(0x28, 2, 106));
  code.push_back(EncodeSMovB32(106, InlineU32(0)));
  code.push_back(EncodeSMovB32(107, InlineU32(0)));
  code.push_back(EncodeSop1(0x37, 4, 106));
  AppendSMovLiteral(&code, 106, 0x00000003u);
  code.push_back(EncodeSop1(0x3c, 8, 106));
  code.push_back(EncodeSop1(0x04, 10, 126));
  code.push_back(EncodeSMovB32(106, InlineU32(1)));
  code.push_back(EncodeSop1(0x44, 12, 106));
  code.push_back(EncodeSop1(0x04, 14, 126));
  AppendSMovLiteral(&code, 126, 0xffffffffu);
  AppendSMovLiteral(&code, 127, 0xffffffffu);

  AppendStoreSgprPair(&code, 0, 0);
  AppendStoreSgprPair(&code, 2, 2);
  AppendStoreSgprPair(&code, 4, 4);
  AppendStoreSgpr(&code, 8, 6);
  AppendStoreSgprPair(&code, 10, 7);
  AppendStoreSgpr(&code, 12, 9);
  AppendStoreSgprPair(&code, 14, 10);
  AppendStoreSgpr(&code, 253, 12);
  AppendEnd(&code);

  TestCase test;
  test.name = "ScalarSaveExecOps";
  test.code = code;
  // A full wave enters with EXEC set for all 64 lanes, so the two saved EXEC
  // pairs are 0xffffffff/0xffffffff rather than the 1/0 a single-invocation
  // dispatch would report.
  test.expected = {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu,
                   0xffffffffu, 0xffffffffu, 0xffffffffu, 3, 0xffffffffu,
                   3, 2, 0xffffffffu, 1};
  test.opcodes = {O::S_MOV_B32, O::S_AND_SAVEEXEC_B64, O::S_ORN2_SAVEEXEC_B64,
                  O::S_ANDN1_SAVEEXEC_B64, O::S_AND_SAVEEXEC_B32,
                  O::S_ANDN1_SAVEEXEC_B32, O::S_MOV_B64, O::V_MOV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 64;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarOrn2SaveexecUsesSourceOrNotExec() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 126, 0x0000000cu);
  AppendSMovLiteral(&code, 127, 0x80000000u);
  AppendSMovLiteral(&code, 0, 0x00000001u);
  AppendSMovLiteral(&code, 1, 0x00000001u);
  code.push_back(EncodeSop1(0x28, 2, 0));
  code.push_back(EncodeSop1(0x04, 4, 126));
  code.push_back(EncodeSMovB32(126, InlineU32(1)));
  code.push_back(EncodeSMovB32(127, InlineU32(0)));
  AppendStoreSgprPair(&code, 2, 0);
  AppendStoreSgprPair(&code, 4, 2);
  AppendStoreSgpr(&code, 253, 4);
  AppendEnd(&code);

  TestCase test;
  test.name = "ScalarOrn2SaveexecUsesSourceOrNotExec";
  test.code = code;
  test.expected = {0x0000000cu, 0x80000000u, 0xfffffff3u, 0x7fffffffu, 1};
  test.opcodes = {O::S_MOV_B32, O::S_ORN2_SAVEEXEC_B64, O::S_MOV_B64, O::V_MOV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  // The case reads a whole 64-bit EXEC mask, so it needs a populated subgroup.
  // A 1x1x1 dispatch leaves OpGroupNonUniformBallot with a single invocation
  // and every lane above zero reads back as zero.
  test.compute_info.threads_num[0] = 64;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarOrn2SaveexecB32(u32 wave_size, u32 threads) {
  using O = ShaderOpcode;
  constexpr u32 exec_hi = 0xa5a55a5au;
  constexpr u32 vcc_hi = 0x13579bdfu;
  struct Case {
    u32 exec_lo, source, destination, result;
  };
  constexpr Case cases[] = {
      {0x8000000cu, 126, 106, 0xffffffffu}, // Captured vcc_lo, exec_lo.
      {0x0000000fu, 6, 6, 0xfffffff3u},    // Aliased SGPR source/destination.
      {0xffffffffu, 128, 8, 0},            // SCC ignores the nonzero high half.
      {0xffffffffu, 107, 8, vcc_hi},
      {0xffffffffu, 127, 8, exec_hi},
      {0xffffffffu, 253, 8, 1},            // SCC is a scalar value, not a mask.
      {0xffffffffu, 251, 8, 0},            // VCCZ summarizes the whole mask.
      {0x00000001u, 252, 8, 0xfffffffeu},  // EXECZ is uniform across lanes.
  };
  TestCase test;
  test.name = wave_size == 64 ? "ScalarOrn2SaveexecB32Wave64"
              : threads == 32 ? "ScalarOrn2SaveexecB32Wave32"
                              : "ScalarOrn2SaveexecB32PartialWave";
  auto &code = test.code;
  // Zero flags ignore high bits only in wave32 and include inactive low bits.
  constexpr u32 masks[][2] = {{0, 0}, {0x80000000u, 0}, {0, 0x80000000u}};
  for (const auto &mask : masks) {
    AppendSMovLiteral(&code, 106, mask[0]);
    AppendSMovLiteral(&code, 107, mask[1]);
    AppendSMovLiteral(&code, 126, mask[0]);
    AppendSMovLiteral(&code, 127, mask[1]);
    code.push_back(EncodeSMovB32(20, 251));
    code.push_back(EncodeSMovB32(21, 252));
    code.push_back(EncodeSop1(0x04, 126, 193u));
    const u32 zero = mask[0] == 0 && (wave_size == 32 || mask[1] == 0);
    for (u32 reg = 20; reg <= 21; ++reg) {
      AppendStoreSgprAtLaneDwordOffset(&code, reg, 0,
                                      static_cast<u32>(test.expected.size()));
      test.expected.insert(test.expected.end(), threads, zero);
    }
  }
  for (const auto &item : cases) {
    AppendVMovU32(&code, 2, 0);
    AppendSMovLiteral(&code, 126, item.exec_lo);
    AppendSMovLiteral(&code, 127, exec_hi);
    AppendSMovLiteral(&code, 106, 1);
    AppendSMovLiteral(&code, 107, vcc_hi);
    AppendSMovLiteral(&code, 6, 3);
    code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(0)));
    code.push_back(item.source == 126 ? 0xbeea407eu
                                     : EncodeSop1(0x40, item.destination, item.source));
    const u32 snapshots[] = {item.destination, 126, 127, 107, 253};
    for (u32 i = 0; i < std::size(snapshots); ++i) {
      code.push_back(EncodeSMovB32(20 + i, snapshots[i]));
    }
    AppendVMovU32(&code, 2, 1);
    code.push_back(EncodeSop1(0x04, 126, 193u)); // Restore EXEC for readback.
    const u32 expected[] = {item.exec_lo, item.result, exec_hi, vcc_hi,
                            item.result != 0 ? 1u : 0u};
    for (u32 i = 0; i < std::size(expected); ++i) {
      AppendStoreSgprAtLaneDwordOffset(&code, 20 + i, 0,
                                      static_cast<u32>(test.expected.size()));
      test.expected.insert(test.expected.end(), threads, expected[i]);
    }
    AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
                                    static_cast<u32>(test.expected.size()));
    for (u32 lane = 0; lane < threads; ++lane) {
      const u32 mask = lane < 32 ? item.result : exec_hi;
      test.expected.push_back((mask >> (lane % 32)) & 1u);
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_CMP_EQ_U32,
                  O::S_ORN2_SAVEEXEC_B32, O::V_MOV_B32, O::V_LSHLREV_B32,
                  O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"S_ORN2_SAVEEXEC_B32 vcc_lo, exec_lo", 1}};
  test.compute_info.threads_num[0] = threads;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarSubvectorLoops(u32 wave_size) {
  using O = ShaderOpcode;
  using namespace ShaderRecompiler::Decoder;
  TestCase test;
  test.name = wave_size == 64 ? "ScalarSubvectorLoopsWave64"
                              : "ScalarSubvectorLoopsWave32";
  // The captured mesh shader's signed SOPK targets are relative to PC + 4.
  std::array<u32, 47> captured{};
  captured[0x30 / 4] = 0xbda60022u;
  captured[0xb8 / 4] = 0xbe26ffdeu;
  Instruction begin, end;
  DecodeInstruction(captured, 0x30 / 4, begin);
  DecodeInstruction(captured, 0xb8 / 4, end);
  Require(test.name, "captured branch decode",
          begin.opcode == O::S_SUBVECTOR_LOOP_BEGIN && begin.dst.reg == 38 &&
              begin.dst.kind == OperandKind::Sgpr && begin.branch_target == 0xbc &&
              end.opcode == O::S_SUBVECTOR_LOOP_END && end.dst.reg == 38 &&
              end.dst.kind == OperandKind::Sgpr && end.branch_target == 0x34 &&
              begin.word_count == 1 && end.word_count == 1,
          "subvector SDST or signed PC-relative target was decoded incorrectly");

  constexpr u32 sentinel = 0x5a13abcdu;
  constexpr u32 low = 0x80000005u, high = 0x42000002u;
  struct Case {
    u32 lo, hi, body_mode, saved, final_lo, final_hi, passes;
  };
  constexpr Case cases[] = {
      {0, 0, 0, sentinel, 0, 0, 0},
      {low, 0, 0, 0, low, 0, 1},
      {0, high, 0, 0, 0, high, 1},
      {low, high, 0, low, low, high, 2},
      {~0u, ~0u, 0, ~0u, ~0u, ~0u, 2},
      // The game resets the full EXEC mask before END, selecting its HI != 0 arm.
      {low, high, 1, high, high, ~0u, 1},
      // END must restore the low mask saved after the body has changed it.
      {low, high, 2, 0, 0, high, 2},
      // BEGIN EXEC_LO first copies HI to its aliased destination, then clears HI.
      {low, high, 3, 0, 0, high, 2},
  };
  auto &code = test.code;
  for (u32 index = 0; index < std::size(cases); ++index) {
    const auto &item = cases[index];
    const u32 scc = index & 1u;
    const u32 saved_reg = item.body_mode == 3 ? 126u : 38u;
    AppendVMovU32(&code, 2, 0);
    AppendVMovU32(&code, 3, 0);
    AppendSMovLiteral(&code, 38, sentinel);
    AppendSMovLiteral(&code, 32, 1); // Scalar 2^passes; MULK preserves SCC.
    AppendSMovLiteral(&code, 33, 0xff);
    AppendSMovLiteral(&code, 126, item.lo);
    AppendSMovLiteral(&code, 127, item.hi);
    code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(scc == 0 ? 1 : 0)));
    const auto begin_word = static_cast<u32>(code.size());
    code.push_back(0);
    const auto body_word = static_cast<u32>(code.size());
    code.push_back(EncodeSMovB32(33, 253)); // Observe SCC immediately after BEGIN.
    code.push_back(EncodeSopk(0x10, 32, 2));
    code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2));
    code.push_back(EncodeVop1(0x01, 3, 32));
    if (item.body_mode == 1) {
      code.push_back(EncodeSop1(0x04, 126, 193)); // S_MOV_B64 EXEC, -1.
    } else if (item.body_mode == 2) {
      code.push_back(EncodeSMovB32(126, InlineU32(0)));
    }
    const auto end_word = static_cast<u32>(code.size());
    code.push_back(EncodeSopk(0x1c, saved_reg, body_word - end_word - 1u));
    code[begin_word] = EncodeSopk(0x1b, saved_reg, end_word - begin_word);
    const u32 sources[] = {saved_reg, 126, 127, 253, 32, 33};
    for (u32 i = 0; i < std::size(sources); ++i) {
      code.push_back(EncodeSMovB32(20 + i, sources[i]));
    }
    code.push_back(EncodeSop1(0x04, 126, 193)); // Restore all lanes for readback.
    const u32 expected[] = {item.saved, item.final_lo, item.final_hi, scc,
                            1u << item.passes, item.passes == 0 ? 0xff : scc};
    for (u32 i = 0; i < std::size(expected); ++i) {
      AppendStoreSgprAtLaneDwordOffset(&code, 20 + i, 0,
                                      static_cast<u32>(test.expected.size()));
      test.expected.insert(test.expected.end(), wave_size, expected[i]);
    }
    for (u32 reg : {2u, 3u}) {
      AppendStoreVgprAtLaneDwordOffset(&code, reg, 0,
                                      static_cast<u32>(test.expected.size()));
      for (u32 lane = 0; lane < wave_size; ++lane) {
        const bool upper = lane >= 32;
        const u32 mask = upper || item.body_mode == 3 ? item.hi : item.lo;
        const bool active = ((mask >> (lane % 32)) & 1u) != 0 &&
                            !(upper && item.body_mode == 1);
        const u32 pass_value = upper && item.lo != 0 ? 4u : 2u;
        test.expected.push_back(active ? (reg == 2 ? 1u : pass_value) : 0u);
      }
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::S_SUBVECTOR_LOOP_BEGIN, O::S_SUBVECTOR_LOOP_END,
                  O::S_MOV_B32, O::S_MOV_B64, O::S_MULK_I32, O::S_CMP_EQ_U32,
                  O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarGetpcWritesNextInstructionPc() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSop1(0x1f, 0, 0));
  AppendStoreSgprPair(&code, 0, 0);
  AppendEnd(&code);

  return {"ScalarGetpcWritesNextInstructionPc",
          code,
          {},
          {4, 0},
          {O::S_GETPC_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarBitcmpB64DynamicOperands() {
  using O = ShaderOpcode;

  struct Input {
    u32 low;
    u32 high;
    u32 index;
  };
  const Input inputs[] = {
      {1u, 0u, 0u}, {0x80000000u, 0u, 31u}, {0u, 1u, 32u},
      {0u, 0x80000000u, 63u}, {1u, 0u, 64u},
      {0x80000000u, 0u, 95u}, {0u, 1u, 96u},
      {0u, 0x80000000u, 127u}, {1u, 0u, 128u},
      {0u, 0x80000000u, 0xffffffffu}, {0u, 0u, 17u},
      {0xffffffffu, 0xffffffffu, 42u},
      {0x89abcdefu, 0x76543210u, 4u},
      {0x89abcdefu, 0x76543210u, 36u},
  };
  TestCase test;
  test.name = "ScalarBitcmpB64DynamicOperands";
  for (const auto &input : inputs) {
    test.initial.insert(test.initial.end(), {input.low, input.high, input.index});
  }
  test.expected = test.initial;
  for (u32 i = 0; i < std::size(inputs); ++i) {
    // Load the value and index on the GPU so the compares cannot constant-fold.
    for (u32 component = 0; component < 3; ++component) {
      AppendVMovU32(&test.code, 30, (i * 3 + component) * 4);
      AppendBufferLoadDword(&test.code, 0, 30);
      test.code.push_back(EncodeVop1(0x02, 20 + component, Vgpr(0)));
    }
    const uint64_t value = uint64_t{inputs[i].low} | (uint64_t{inputs[i].high} << 32);
    const u32 bit = static_cast<u32>((value >> (inputs[i].index & 63u)) & 1u);
    for (u32 expected_bit = 0; expected_bit < 2; ++expected_bit) {
      const bool scc = bit == expected_bit;
      test.code.push_back(EncodeSopc(0x0e + expected_bit, 20, 22));
      test.code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
      AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
      test.expected.push_back(scc ? 1u : 0u);

      // Exercise SCC-driven control flow as well as S_CSELECT_B32.
      test.code.push_back(EncodeSopp(0x05, 2)); // s_cbranch_scc1 to the true arm.
      test.code.push_back(EncodeSMovB32(24, InlineU32(3)));
      test.code.push_back(EncodeSopp(0x02, 1)); // s_branch past the true arm.
      test.code.push_back(EncodeSMovB32(24, InlineU32(7)));
      AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
      test.expected.push_back(scc ? 7u : 3u);
    }
  }
  AppendEnd(&test.code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
                  O::S_BITCMP0_B64, O::S_BITCMP1_B64, O::S_CSELECT_B32,
                  O::S_CBRANCH_SCC1, O::S_BRANCH, O::S_MOV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpBitFieldUExtract", "OpULessThan", "OpSelect"};
  return test;
}

TestCase ScalarBitcmpB64IntegerConstants() {
  using O = ShaderOpcode;

  struct Source {
    u32 encoding;
    uint64_t bits;
  };
  // Negative integer inlines sign-extend; B64 literals zero-extend.
  const Source sources[] = {
      {128u, 0u}, {129u, 1u}, {192u, 64u},
      {193u, 0xffffffffffffffffull}, {208u, 0xfffffffffffffff0ull},
      {255u, 0xffffffffu}, {255u, 0x80000000u},
  };
  const u32 indices[] = {0u, 4u, 6u, 31u, 32u, 63u, 64u, 127u};
  TestCase test;
  test.name = "ScalarBitcmpB64IntegerConstants";
  test.initial.assign(std::begin(indices), std::end(indices));
  test.expected = test.initial;
  for (u32 i = 0; i < std::size(indices); ++i) {
    AppendVMovU32(&test.code, 30, i * 4);
    AppendBufferLoadDword(&test.code, 0, 30);
    test.code.push_back(EncodeVop1(0x02, 22, Vgpr(0)));
    for (const auto &source : sources) {
      const u32 bit = static_cast<u32>((source.bits >> (indices[i] & 63u)) & 1u);
      for (u32 expected_bit = 0; expected_bit < 2; ++expected_bit) {
        test.code.push_back(EncodeSopc(0x0e + expected_bit, source.encoding, 22));
        if (source.encoding == 255u) {
          test.code.push_back(static_cast<u32>(source.bits));
        }
        test.code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
        AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
        test.expected.push_back(bit == expected_bit ? 1u : 0u);
      }
    }
  }
  // An instruction has one literal word even when both sources select it.
  for (u32 expected_bit = 0; expected_bit < 2; ++expected_bit) {
    test.code.push_back(EncodeSopc(0x0e + expected_bit, 255u, 255u));
    test.code.push_back(0x8000001fu); // Select bit 31 of the same zero-extended literal.
    test.code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
    AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
    test.expected.push_back(expected_bit);
  }
  AppendEnd(&test.code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
                  O::S_BITCMP0_B64, O::S_BITCMP1_B64, O::S_CSELECT_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpBitFieldUExtract"};
  return test;
}

TestCase ScalarBitfieldPack() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSMovB32(0, InlineU32(15)));
  code.push_back(EncodeSop1(0x0b, 1, 0));
  AppendSMovLiteral(&code, 2, 0xf0000000u);
  code.push_back(EncodeSop1(0x10, 3, 0));
  AppendSMovLiteral(&code, 19, 0xf0f00001u);
  code.push_back(EncodeSop1(0x0f, 20, 19));
  code.push_back(EncodeSop2(0x0a, 21, InlineU32(1), InlineU32(0)));
  code.push_back(EncodeSMovB32(22, InlineU32(0)));
  code.push_back(EncodeSop1(0x0f, 23, 22));
  code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
  code.push_back(EncodeSMovB32(4, InlineU32(3)));
  code.push_back(EncodeSop1(0x3b, 5, 4));
  code.push_back(EncodeSop2(0x24, 7, InlineU32(4), InlineU32(8)));
  AppendSMovLiteral(&code, 8, 0x00f00000u);
  AppendSMovLiteral(&code, 9, 0x00040014u);
  code.push_back(EncodeSop2(0x27, 10, 8, 9));
  AppendSMovLiteral(&code, 11, 0xaaaabbbbu);
  AppendSMovLiteral(&code, 12, 0xccccddddu);
  code.push_back(EncodeSop2(0x32, 13, 11, 12));
  code.push_back(EncodeSop2(0x33, 14, 11, 12));
  code.push_back(EncodeSop2(0x34, 15, 11, 12));
  code.push_back(EncodeSopc(0x0d, InlineU32(4), InlineU32(2)));
  code.push_back(EncodeSop2(0x0a, 17, InlineU32(1), InlineU32(0)));
  code.push_back(EncodeSopc(0x0c, InlineU32(4), InlineU32(1)));
  code.push_back(EncodeSop2(0x0a, 18, InlineU32(1), InlineU32(0)));
  AppendSMovLiteral(&code, 25, 0u);
  AppendSMovLiteral(&code, 26, 0x00000100u);
  code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(1)));
  code.push_back(EncodeSop1(0x14, 31, 25));
  code.push_back(EncodeSop2(0x0a, 34, InlineU32(1), InlineU32(0)));
  AppendSMovLiteral(&code, 27, 0x00000020u);
  AppendSMovLiteral(&code, 28, 0x00000100u);
  code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1)));
  code.push_back(EncodeSop1(0x14, 32, 27));
  code.push_back(EncodeSop2(0x0a, 35, InlineU32(1), InlineU32(0)));
  AppendSMovLiteral(&code, 29, 0u);
  AppendSMovLiteral(&code, 30, 0u);
  code.push_back(EncodeSop1(0x14, 33, 29));

  AppendStoreSgpr(&code, 1, 0);
  AppendStoreSgpr(&code, 3, 1);
  AppendStoreSgpr(&code, 20, 2);
  AppendStoreSgpr(&code, 21, 3);
  AppendStoreSgpr(&code, 23, 4);
  AppendStoreSgpr(&code, 24, 5);
  AppendStoreSgprPair(&code, 5, 6);
  AppendStoreSgpr(&code, 7, 8);
  AppendStoreSgpr(&code, 10, 9);
  AppendStoreSgpr(&code, 13, 10);
  AppendStoreSgpr(&code, 14, 11);
  AppendStoreSgpr(&code, 15, 12);
  AppendStoreSgpr(&code, 17, 13);
  AppendStoreSgpr(&code, 18, 14);
  AppendStoreSgpr(&code, 31, 15);
  AppendStoreSgpr(&code, 32, 16);
  AppendStoreSgpr(&code, 33, 17);
  AppendStoreSgpr(&code, 34, 18);
  AppendStoreSgpr(&code, 35, 19);
  AppendEnd(&code);

  return {"ScalarBitfieldPack",
          code,
          {},
          {0xf0000000u, 8,           9,           1,           0,
           0,           0x0000000fu, 0,           0x00000f00u, 0x0000000fu,
           0xddddbbbbu, 0xccccbbbbu, 0xccccaaaau, 1,           1,
           40,          5,           0xffffffffu, 0,           1},
          {O::S_MOV_B32, O::S_BREV_B32, O::S_BCNT1_I32_B32, O::S_BCNT1_I32_B64,
           O::S_FF1_I32_B64, O::S_BITREPLICATE_B64_B32, O::S_BFM_B32,
           O::S_BFE_U32, O::S_PACK_LL_B32_B16, O::S_PACK_LH_B32_B16,
           O::S_PACK_HH_B32_B16, O::S_BITCMP0_B32, O::S_BITCMP1_B32,
           O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarBrevB32PreservesScc() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  auto set_scc = [&](bool value) {
    code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
  };
  auto capture_scc = [&](u32 dst_sgpr) {
    code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
  };

  code.push_back(EncodeSMovB32(0, InlineU32(1)));
  set_scc(false);
  code.push_back(EncodeSop1(0x0b, 1, 0));
  capture_scc(2);

  code.push_back(EncodeSMovB32(3, InlineU32(0)));
  set_scc(true);
  code.push_back(EncodeSop1(0x0b, 4, 3));
  capture_scc(5);

  AppendStoreSgpr(&code, 1, 0);
  AppendStoreSgpr(&code, 2, 1);
  AppendStoreSgpr(&code, 4, 2);
  AppendStoreSgpr(&code, 5, 3);
  AppendEnd(&code);

  return {"ScalarBrevB32PreservesScc",
          code,
          {},
          {0x80000000u, 0, 0, 1},
          {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_BREV_B32, O::S_CSELECT_B32,
           O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarZeroBitOps() {
  using O = ShaderOpcode;
  constexpr uint64_t inputs[] = {0, UINT64_MAX, 0xaaaaaaaa55555555ull,
      0xffffffff7fffffffull, 0xfffffffeffffffffull, 0x7fffffffffffffffull,
      0x00000000ffffffffull, 0xffffffff00000000ull};
  TestCase test;
  test.name = "ScalarZeroBitOps";
  for (const auto input : inputs)
    test.initial.insert(test.initial.end(), {u32(input), u32(input >> 32)});
  test.expected = test.initial;
  auto &code = test.code;
  const auto check = [&](u32 opcode, u32 source, uint64_t input, bool scc) {
    const u32 width = opcode == 0x0e || opcode == 0x12 ? 64 : 32;
    u32 count = 0, first = UINT32_MAX;
    for (u32 bit = 0; bit < width; ++bit) {
      if (((input >> bit) & 1u) == 0) {
        ++count;
        if (first == UINT32_MAX) first = bit;
      }
    }
    const bool counting = opcode == 0x0d || opcode == 0x0e;
    code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(scc)));
    code.push_back(EncodeSop1(opcode, 20, source));
    if (source == 255) code.push_back(u32(input));
    code.push_back(EncodeSMovB32(22, 253)); // Capture SCC before readback.
    AppendStoreSgpr(&code, 20, u32(test.expected.size()));
    AppendStoreSgpr(&code, 21, u32(test.expected.size() + 1));
    AppendStoreSgpr(&code, 22, u32(test.expected.size() + 2));
    test.expected.insert(test.expected.end(),
        {counting ? count : first, u32(input >> 32), u32(counting ? count != 0 : scc)});
  };
  for (u32 index = 0; index < std::size(inputs); ++index) {
    for (u32 word = 0; word < 2; ++word) {
      AppendVMovU32(&code, 30, (index * 2 + word) * sizeof(u32));
      AppendBufferLoadDword(&code, 1, 30);
      code.push_back(EncodeVop1(0x02, 8 + word, Vgpr(1)));
    }
    for (const u32 opcode : {0x0du, 0x0eu, 0x11u, 0x12u}) {
      for (const bool scc : {false, true}) {
        code.push_back(EncodeSop1(0x04, 20, 8)); // Destination aliases the source.
        check(opcode, 20, inputs[index], scc);
      }
    }
  }
  for (const u32 opcode : {0x0du, 0x0eu, 0x11u, 0x12u}) {
    for (const u32 source : {128u, 193u, 255u}) {
      const uint64_t input = source == 128 ? 0 : source == 193 ? UINT64_MAX : 0xffffffffull;
      AppendSMovLiteral(&code, 21, u32(input >> 32));
      check(opcode, source, input, true); // Only inline -1 sign extends to B64.
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::S_BCNT0_I32_B32, O::S_BCNT0_I32_B64,
      O::S_FF0_I32_B32, O::S_FF0_I32_B64, O::S_CMP_EQ_U32, O::S_MOV_B32,
      O::S_MOV_B64, O::V_MOV_B32, O::V_READFIRSTLANE_B32,
      O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase ScalarSextMasks(u32 bits, u32 wave_size) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = bits == 8 ? (wave_size == 32 ? "ScalarSextI8MasksWave32"
                                         : "ScalarSextI8MasksWave64")
                       : (wave_size == 32 ? "ScalarSextI16MasksWave32"
                                         : "ScalarSextI16MasksWave64");
  constexpr u32 high = 0x80000001u;
  test.initial = bits == 8
                     ? std::vector<u32>{0xdeadff00u, 0xabcdff7fu, 0x12340080u, 0xabcf00ffu}
                     : std::vector<u32>{0xdead0000u, 0xabcd7fffu, 0x12348000u, 0xabcfffffu};
  const u32 extended_values[] = {0u, bits == 8 ? 0x7fu : 0x7fffu,
                                bits == 8 ? 0xffffff80u : 0xffff8000u, 0xffffffffu};
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < 4u; ++i) {
    const auto extended = extended_values[i];
    for (const bool scc : {false, true}) {
      AppendVMovU32(&code, 30, i * 4u);
      AppendBufferLoadDword(&code, 3, 30);
      code.push_back(EncodeVop1(0x02, 11, Vgpr(3)));
      AppendSMovLiteral(&code, 107, high);
      code.push_back(EncodeSopc(0x06, InlineU32(scc ? 1u : 0u), InlineU32(1)));
      code.push_back(EncodeSop1(0x04, 126, 128)); // Scalar operations execute with empty EXEC.
      // The I16 word is captured s_sext_i32_i16 vcc_lo, s11.
      code.push_back(bits == 8 ? 0xbeea190bu : 0xbeea1a0bu);
      code.push_back(EncodeSMovB32(20, 106));
      code.push_back(EncodeSMovB32(21, 107));
      code.push_back(EncodeSMovB32(22, 253));
      code.push_back(EncodeSop1(0x04, 126, 193));
      AppendVMovU32(&code, 1, 1);
      code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
      const u32 expected[] = {extended, high, scc ? 1u : 0u};
      for (u32 word = 0; word < 3u; ++word) {
        AppendStoreSgprAtLaneDwordOffset(&code, 20u + word, 0, test.expected.size());
        test.expected.insert(test.expected.end(), wave_size, expected[word]);
      }
      AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, test.expected.size());
      const uint64_t mask = (uint64_t{high} << 32u) | extended;
      for (u32 lane = 0; lane < wave_size; ++lane)
        test.expected.push_back(static_cast<u32>((mask >> lane) & 1u));
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_MOV_B32,
                  O::S_MOV_B64, O::S_CMP_EQ_U32,
                  bits == 8 ? O::S_SEXT_I32_I8 : O::S_SEXT_I32_I16,
                  O::V_CNDMASK_B32, O::V_MOV_B32, O::V_ADD_NC_U32,
                  O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{bits == 8 ? "S_SEXT_I32_I8 vcc_lo, s11"
                                  : "S_SEXT_I32_I16 vcc_lo, s11", 8u}};
  test.required_spirv = {"OpBitFieldSExtract"};
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarSextConstantsAndAliases() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "ScalarSextConstantsAndAliases";
  auto &code = test.code;
  for (const u32 opcode : {0x19u, 0x1au}) {
    const u32 literal = opcode == 0x19u ? 0x12345680u : 0x12348000u;
    const u32 extended = opcode == 0x19u ? 0xffffff80u : 0xffff8000u;
    for (const bool scc : {false, true}) {
      code.push_back(EncodeSopc(0x06, InlineU32(scc ? 1u : 0u), InlineU32(1)));
      code.push_back(EncodeSop1(opcode, 10, 255));
      code.push_back(literal);
      code.push_back(EncodeSop1(opcode, 10, 10)); // In-place source/destination.
      code.push_back(EncodeSop1(opcode, 11, InlineU32(0)));
      code.push_back(EncodeSop1(opcode, 12, InlineU32(64)));
      code.push_back(EncodeSop1(opcode, 13, 193)); // Inline -1.
      code.push_back(EncodeSMovB32(14, 253));
      const u32 expected[] = {extended, 0, 64, 0xffffffffu, scc ? 1u : 0u};
      for (u32 word = 0; word < 5u; ++word) {
        AppendStoreSgpr(&code, 10u + word, test.expected.size());
        test.expected.push_back(expected[word]);
      }
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::S_CMP_EQ_U32, O::S_SEXT_I32_I8, O::S_SEXT_I32_I16,
                  O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"S_SEXT_I32_I8", 10u}, {"S_SEXT_I32_I16", 10u}};
  return test;
}

TestCase ScalarBrevB64OperandsAndMasks(u32 wave_size) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = wave_size == 64 ? "ScalarBrevB64OperandsAndMasksWave64"
                              : "ScalarBrevB64OperandsAndMasksWave32";
  constexpr uint64_t values[] = {0, 1, 0x8000000000000000ull,
                                 0x0123456789abcdefull};
  for (const auto value : values) {
    test.initial.push_back(static_cast<u32>(value));
    test.initial.push_back(static_cast<u32>(value >> 32u));
  }
  test.expected = test.initial;
  auto &code = test.code;
  size_t instructions = 0;
  const auto check = [&](u32 dst, u32 src, uint64_t value, bool scc,
                          std::optional<u32> literal = {}) {
    uint64_t reversed = 0;
    for (u32 bit = 0; bit < 64; ++bit) {
      reversed |= ((value >> bit) & 1ull) << (63u - bit);
    }
    AppendVMovU32(&code, 1, 1);
    AppendVMovU32(&code, 2, 0);
    if (dst == 106u || dst == 126u) {
      code.push_back(EncodeSop1(0x04, dst, src));
      src = dst;
    }
    code.push_back(EncodeSopc(0x06, InlineU32(scc ? 1u : 0u), InlineU32(1)));
    code.push_back(EncodeSop1(0x0c, dst, src));
    if (literal) {
      code.push_back(*literal);
    }
    code.push_back(EncodeSMovB32(24, dst));
    code.push_back(EncodeSMovB32(25, dst + 1u));
    code.push_back(EncodeSMovB32(26, 253));
    if (dst == 106u) {
      code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
    } else if (dst == 126u) {
      code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
    }
    code.push_back(EncodeSop1(0x04, 126, 193)); // Restore EXEC for readback.
    const u32 expected[] = {static_cast<u32>(reversed),
                            static_cast<u32>(reversed >> 32u), scc ? 1u : 0u};
    for (u32 i = 0; i < std::size(expected); ++i) {
      AppendStoreSgprAtLaneDwordOffset(
          &code, 24 + i, 0, static_cast<u32>(test.expected.size()));
      test.expected.insert(test.expected.end(), wave_size, expected[i]);
    }
    if (dst == 106u || dst == 126u) {
      AppendStoreVgprAtLaneDwordOffset(
          &code, 2, 0, static_cast<u32>(test.expected.size()));
      for (u32 lane = 0; lane < wave_size; ++lane) {
        test.expected.push_back(static_cast<u32>((reversed >> lane) & 1u));
      }
    }
    ++instructions;
  };
  for (const u32 dst : {20u, 106u, 126u}) {
    for (u32 i = 0; i < std::size(values); ++i) {
      // Runtime inputs exercise the emitter, including in-place mask writes.
      for (u32 word = 0; word < 2; ++word) {
        AppendVMovU32(&code, 30, (2 * i + word) * 4u);
        AppendBufferLoadDword(&code, 3, 30);
        code.push_back(EncodeVop1(0x02, 20 + word, Vgpr(3)));
      }
      check(dst, 20, values[i], values[i] == 0);
    }
  }
  check(20, 129, 1, false);
  check(20, 193, ~uint64_t{0}, false);
  check(20, 125, 0, true);
  check(20, 255, 0x80000001u, true, 0x80000001u);
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_MOV_B32,
                  O::S_MOV_B64, O::S_BREV_B64, O::S_CMP_EQ_U32, O::V_MOV_B32,
                  O::V_CNDMASK_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"S_BREV_B64", instructions}};
  test.required_spirv = {"OpBitReverse"};
  test.forbidden_spirv = {"OpTypeInt 64"};
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarBfeI32CapturedRawSignExtends() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendBufferLoadDword(&code, 0, 30);
  code.push_back(EncodeVop1(0x02, 20, Vgpr(0)));
  code.push_back(0x946aff14u); // s_bfe_i32 vcc_lo, s20, literal
  code.push_back(0x0004001cu); // offset 28, width 4
  code.push_back(EncodeSop2(0x0a, 21, InlineU32(1), InlineU32(0)));
  code.push_back(EncodeSop2(0x28, 22, 20, 255u));
  code.push_back(0u); // zero width produces zero and clears SCC
  code.push_back(EncodeSop2(0x0a, 23, InlineU32(1), InlineU32(0)));
  AppendStoreSgpr(&code, 106, 1);
  AppendStoreSgpr(&code, 21, 2);
  AppendStoreSgpr(&code, 22, 3);
  AppendStoreSgpr(&code, 23, 4);
  AppendEnd(&code);

  TestCase test;
  test.name = "ScalarBfeI32CapturedRawSignExtends";
  test.code = std::move(code);
  test.initial = {0xf0000000u, 0, 0, 0, 0};
  test.expected = {0xf0000000u, 0xffffffffu, 1, 0, 0};
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
                  O::S_BFE_I32,         O::S_CSELECT_B32,
                  O::V_MOV_B32,         O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.required_spirv = {"OpBitFieldSExtract"};
  return test;
}

TestCase BitfieldExtractWidthPastEndEdges() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 0, 0xf0000000u);
  AppendSMovLiteral(&code, 1, (20u << 16u) | 20u);
  code.push_back(EncodeSop2(0x27, 2, 0, 1));
  AppendSMovLiteral(&code, 3, 0x80000000u);
  AppendSMovLiteral(&code, 4, (7u << 16u) | 31u);
  code.push_back(EncodeSop2(0x27, 5, 3, 4));
  AppendVMovLiteral(&code, 6, 0xf0000000u);
  AppendVMovU32(&code, 7, 28);
  AppendVMovU32(&code, 8, 8);
  AppendVop3(&code, 0x148, 9, Vgpr(6), Vgpr(7), Vgpr(8));
  AppendStoreSgpr(&code, 2, 0);
  AppendStoreSgpr(&code, 5, 1);
  AppendStoreVgpr(&code, 9, 2);
  AppendEnd(&code);

  return {"BitfieldExtractWidthPastEndEdges",
          code,
          {},
          {0x00000f00u, 1, 0x0000000fu},
          {O::S_MOV_B32, O::S_BFE_U32, O::V_MOV_B32, O::V_BFE_U32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase Scalar64BitOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 0, 0x0f0f0f0fu);
  AppendSMovLiteral(&code, 1, 0x00ff00ffu);
  AppendSMovLiteral(&code, 2, 0x33333333u);
  AppendSMovLiteral(&code, 3, 0x0f0f0f0fu);
  code.push_back(EncodeSop2(0x0f, 4, 0, 2));
  code.push_back(EncodeSop2(0x15, 6, 0, 2));
  code.push_back(EncodeSop2(0x11, 8, 0, 2));
  code.push_back(EncodeSop2(0x17, 10, 0, 2));
  code.push_back(EncodeSop2(0x13, 12, 0, 2));
  code.push_back(EncodeSop2(0x19, 14, 0, 2));
  code.push_back(EncodeSop2(0x1b, 16, 0, 2));
  code.push_back(EncodeSop2(0x1d, 18, 0, 2));
  code.push_back(EncodeSop1(0x04, 20, 0));
  code.push_back(EncodeSop2(0x1f, 22, 0, InlineU32(4)));
  code.push_back(EncodeSop2(0x21, 24, 0, InlineU32(8)));
  code.push_back(EncodeSop2(0x25, 26, InlineU32(36), InlineU32(4)));
  code.push_back(EncodeSop2(0x21, 34, 193u, InlineU32(1)));
  AppendSMovLiteral(&code, 28, 0x000c0004u);
  code.push_back(EncodeSop2(0x29, 30, 0, 28));
  code.push_back(EncodeSopc(0x06, 0, 0));
  code.push_back(EncodeSop2(0x0b, 32, 0, 2));

  const u32 result_pairs[] = {4,  6,  8,  10, 12, 14, 16, 18,
                              20, 22, 24, 26, 34, 30, 32};
  u32 out = 0;
  for (auto sgpr : result_pairs) {
    AppendStoreSgprPair(&code, sgpr, out);
    out += 2;
  }
  AppendEnd(&code);

  return {"Scalar64BitOps",
          code,
          {},
          {0x03030303u, 0x000f000fu, 0x0c0c0c0cu, 0x00f000f0u, 0x3f3f3f3fu,
           0x0fff0fffu, 0xcfcfcfcfu, 0xf0fff0ffu, 0x3c3c3c3cu, 0x0ff00ff0u,
           0xfcfcfcfcu, 0xfff0fff0u, 0xc0c0c0c0u, 0xf000f000u, 0xc3c3c3c3u,
           0xf00ff00fu, 0x0f0f0f0fu, 0x00ff00ffu, 0xf0f0f0f0u, 0x0ff00ff0u,
           0xff0f0f0fu, 0x0000ff00u, 0xfffffff0u, 0x000000ffu, 0xffffffffu,
           0x7fffffffu, 0x000000f0u, 0,           0x0f0f0f0fu, 0x00ff00ffu},
          {O::S_MOV_B32, O::S_MOV_B64, O::S_AND_B64, O::S_ANDN2_B64,
           O::S_OR_B64, O::S_ORN2_B64, O::S_XOR_B64, O::S_NAND_B64,
           O::S_NOR_B64, O::S_XNOR_B64, O::S_LSHL_B64, O::S_LSHR_B64,
           O::S_BFM_B64, O::S_BFE_U64, O::S_CMP_EQ_U32, O::S_CSELECT_B64,
           O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarDynamic64BitOps() {
  using O = ShaderOpcode;
  struct Input {
    uint64_t lhs;
    uint64_t rhs;
    u32 count;
    u32 offset;
  };
  constexpr Input inputs[] = {
      {0, UINT64_MAX, 0, 0},
      {UINT64_MAX, 0, 63, 0},
      {0x8000000000000000ull, 1, 64, 0},
      {0x0123456789abcdefull, 0xfedcba9876543210ull, 127, 0},
      {UINT64_MAX, 0xffffffffull, 64, 1},
      {0x0123456789abcdefull, 0x0123456789abcdefull, 63, 32},
      {UINT64_MAX, 0x100000000ull, 127, 63},
      {0, 0, 0, 127},
      {0x8000000000000001ull, UINT64_MAX, UINT32_MAX, 64},
  };
  TestCase test;
  test.name = "ScalarDynamic64BitOps";
  for (const auto &input : inputs) {
    test.initial.insert(test.initial.end(),
                        {u32(input.lhs), u32(input.lhs >> 32), u32(input.rhs),
                         u32(input.rhs >> 32), input.count, input.offset,
                         ((input.count & 127u) << 16) | input.offset});
  }
  test.expected = test.initial;
  auto &code = test.code;
  const auto store = [&](uint64_t value, bool scc) {
    const auto output = static_cast<u32>(test.expected.size());
    code.push_back(EncodeSMovB32(18, 253));
    AppendStoreSgprPair(&code, 16, output);
    AppendStoreSgpr(&code, 18, output + 2);
    test.expected.insert(test.expected.end(),
                         {u32(value), u32(value >> 32), u32(scc)});
  };
  for (u32 i = 0; i < std::size(inputs); ++i) {
    const auto [lhs, rhs, count, offset] = inputs[i];
    for (u32 word = 0; word < 7; ++word) {
      AppendVMovU32(&code, 30, (i * 7 + word) * sizeof(u32));
      AppendBufferLoadDword(&code, 1, 30);
      code.push_back(EncodeVop1(0x02, 8 + word, Vgpr(1)));
    }
    for (const auto [opcode, value] :
         std::array<std::pair<u32, uint64_t>, 8>{{{0x0f, lhs & rhs},
                                                  {0x11, lhs | rhs},
                                                  {0x13, lhs ^ rhs},
                                                  {0x15, lhs & ~rhs},
                                                  {0x17, lhs | ~rhs},
                                                  {0x19, ~(lhs & rhs)},
                                                  {0x1b, ~(lhs | rhs)},
                                                  {0x1d, ~(lhs ^ rhs)}}}) {
      code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(value != 0)));
      code.push_back(EncodeSop2(opcode, 16, 8, 10));
      store(value, value != 0);
    }
    for (const bool select_lhs : {false, true}) {
      code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(!select_lhs)));
      code.push_back(EncodeSop2(0x0b, 16, 8, 10));
      store(select_lhs ? lhs : rhs, select_lhs);
    }
    // BFM preserves SCC; BFE sets it from the entire extracted result.
    code.push_back(EncodeSop2(0x25, 16, 12, 13));
    store(((uint64_t{1} << (count & 63u)) - 1u) << (offset & 63u), true);
    code.push_back(EncodeSop2(0x29, 16, 8, 14));
    const auto width = std::min(count & 127u, 64u - (offset & 63u));
    const auto mask = width == 64 ? UINT64_MAX : (uint64_t{1} << width) - 1u;
    const auto extracted = (lhs >> (offset & 63u)) & mask;
    store(extracted, extracted != 0);
    for (const bool set : {false, true}) {
      code.push_back(EncodeSop1(0x04, 16, 8));
      code.push_back(EncodeSop1(set ? 0x1e : 0x1c, 16, 12));
      const auto bit = uint64_t{1} << (count & 63u);
      store(set ? lhs | bit : lhs & ~bit, extracted != 0);
    }
    code.push_back(EncodeSop1(0x2d, 16, 8));
    uint64_t quads = 0;
    for (u32 quad = 0; quad < 16; ++quad) {
      quads |= uint64_t((lhs >> (quad * 4) & 15u) != 0) << quad;
    }
    store(quads, quads != 0);
    code.push_back(EncodeSop1(0x3b, 16, 8));
    uint64_t replicated = 0;
    for (u32 bit = 0; bit < 32; ++bit) {
      replicated |= ((lhs >> bit) & 1u) * (uint64_t{3} << (2 * bit));
    }
    store(replicated, quads != 0);
    code.push_back(EncodeSMovB32(17, InlineU32(0)));
    code.push_back(EncodeSop1(0x10, 16, 8));
    store(std::popcount(lhs), lhs != 0);
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {
      O::V_MOV_B32,       O::BUFFER_LOAD_DWORD,  O::V_READFIRSTLANE_B32,
      O::S_MOV_B32,       O::S_CMP_EQ_U32,       O::S_AND_B64,
      O::S_OR_B64,        O::S_XOR_B64,          O::S_ANDN2_B64,
      O::S_ORN2_B64,      O::S_NAND_B64,         O::S_NOR_B64,
      O::S_XNOR_B64,      O::S_CSELECT_B64,      O::S_BFM_B64,
      O::S_BFE_U64,       O::S_MOV_B64,          O::S_BITSET0_B64,
      O::S_BITSET1_B64,   O::S_QUADMASK_B64,     O::S_BITREPLICATE_B64_B32,
      O::S_BCNT1_I32_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpTypeInt 64 0", "OpBitwiseAnd", "OpBitwiseOr",
                         "OpBitwiseXor"};
  return test;
}

uint64_t ReferenceAshrI64(uint64_t bits, u32 count) {
  const auto shift = count & 63u;
  // Complementing negative values expresses sign fill using unsigned shifts.
  return (bits & (uint64_t{1} << 63u)) != 0 ? ~(~bits >> shift) : bits >> shift;
}

TestCase ScalarAshrI64Edges(bool dynamic) {
  using O = ShaderOpcode;
  TestCase test;
  test.name =
      dynamic ? "ScalarAshrI64DynamicEdges" : "ScalarAshrI64ConstantEdges";
  constexpr uint64_t sources[] = {0,
                                  1,
                                  0x80000000ull,
                                  0x100000000ull,
                                  0x7fffffffffffffffull,
                                  0x8000000000000000ull,
                                  0xffffffffffffffffull,
                                  0x0123456789abcdefull,
                                  0xfedcba9876543210ull};
  constexpr u32 counts[] = {0,  1,  7,  31, 32,  33,         63,
                            64, 65, 95, 96, 127, 0xffffffffu};
  test.initial.assign(std::begin(counts), std::end(counts));
  for (const auto source : sources) {
    test.initial.push_back(static_cast<u32>(source));
    test.initial.push_back(static_cast<u32>(source >> 32u));
  }
  test.expected = test.initial;
  auto &code = test.code;
  const auto load_scalar = [&](u32 reg, u32 index) {
    AppendVMovU32(&code, 30, index * sizeof(u32));
    AppendBufferLoadDword(&code, 1, 30);
    code.push_back(EncodeVop1(0x02, reg, Vgpr(1)));
  };
  for (u32 source_index = 0; source_index < std::size(sources);
       ++source_index) {
    const auto source = sources[source_index];
    // Runtime sources exercise both constant-count and dynamic-count emission.
    load_scalar(20, static_cast<u32>(std::size(counts)) + source_index * 2u);
    load_scalar(21,
                static_cast<u32>(std::size(counts)) + source_index * 2u + 1u);
    for (u32 index = 0; index < std::size(counts); ++index) {
      const auto expected = ReferenceAshrI64(source, counts[index]);
      if (dynamic) {
        load_scalar(22, index);
      }
      // Start SCC at the opposite value so every result must update it.
      code.push_back(
          EncodeSopc(0x06, InlineU32(0), InlineU32(expected != 0 ? 1u : 0u)));
      const auto count_src = dynamic                ? 22u
                             : counts[index] <= 64u ? InlineU32(counts[index])
                                                    : 255u;
      code.push_back(EncodeSop2(0x23, 24, 20, count_src));
      if (!dynamic && count_src == 255u) {
        code.push_back(counts[index]);
      }
      code.push_back(EncodeSMovB32(26, 253));
      const auto output = static_cast<u32>(test.expected.size());
      AppendStoreSgprPair(&code, 24, output);
      AppendStoreSgpr(&code, 26, output + 2u);
      test.expected.push_back(static_cast<u32>(expected));
      test.expected.push_back(static_cast<u32>(expected >> 32u));
      test.expected.push_back(expected != 0 ? 1u : 0u);
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::BUFFER_LOAD_DWORD,  O::V_READFIRSTLANE_B32,
                  O::S_MOV_B32,          O::S_CMP_EQ_U32,
                  O::S_ASHR_I64,         O::V_MOV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {
      {"S_ASHR_I64", std::size(sources) * std::size(counts)}};
  test.required_spirv = {"OpShiftRightArithmetic", "OpTypeInt 64 0"};
  return test;
}

TestCase ScalarAshrI64OperandsAndAliases() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "ScalarAshrI64OperandsAndAliases";
  test.initial = {33}; // Runtime shift for signed literal source tests.
  test.expected = test.initial;
  auto &code = test.code;
  AppendVMovU32(&code, 30, 0);
  AppendBufferLoadDword(&code, 1, 30);
  code.push_back(EncodeVop1(0x02, 22, Vgpr(1)));
  size_t instructions = 0;
  const auto check_result = [&](u32 dst, u32 src0, u32 src1, uint64_t value,
                                u32 count, std::optional<u32> literal = {}) {
    code.push_back(EncodeSop2(0x23, dst, src0, src1));
    if (literal) {
      code.push_back(*literal);
    }
    code.push_back(EncodeSMovB32(26, 253));
    const auto output = static_cast<u32>(test.expected.size());
    AppendStoreSgprPair(&code, dst, output);
    AppendStoreSgpr(&code, 26, output + 2u);
    const auto expected = ReferenceAshrI64(value, count);
    test.expected.push_back(static_cast<u32>(expected));
    test.expected.push_back(static_cast<u32>(expected >> 32u));
    test.expected.push_back(expected != 0 ? 1u : 0u);
    ++instructions;
  };
  for (const u32 literal : {0u, 0x7fffffffu, 0x80000000u, 0xffffffffu}) {
    const uint64_t value =
        literal | ((literal & 0x80000000u) != 0 ? 0xffffffff00000000ull : 0ull);
    for (const u32 count : {0u, 1u, 31u, 32u, 63u, 64u}) {
      check_result(24, 255, InlineU32(count), value, count, literal);
    }
    check_result(24, 255, 22, value, 33, literal);
  }
  for (const u32 source : {128u, 129u, 192u, 193u, 208u, 125u}) {
    const uint64_t value = source == 125u   ? 0ull
                           : source <= 192u ? source - 128u
                                            : uint64_t{192} - source;
    for (const u32 count : {0u, 1u, 32u, 63u}) {
      check_result(24, source, InlineU32(count), value, count);
    }
  }
  for (const u32 scc : {0u, 1u}) {
    // SCC may be either source and must be read before it is overwritten.
    code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
    check_result(24, 253, InlineU32(1), scc, 1);
    AppendSMovLiteral(&code, 20, 1);
    AppendSMovLiteral(&code, 21, 0);
    code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
    check_result(24, 20, 253, 1, scc);
  }
  constexpr uint64_t value = 0xfedcba9889abcdefull;
  for (const u32 count : {0u, 1u, 31u, 32u, 33u, 63u, 65u}) {
    AppendSMovLiteral(&code, 20, static_cast<u32>(value));
    AppendSMovLiteral(&code, 21, static_cast<u32>(value >> 32u));
    AppendSMovLiteral(&code, 23, count);
    check_result(20, 20, 23, value, count); // In-place source/destination pair.
    for (const u32 count_reg : {24u, 25u}) {
      AppendSMovLiteral(&code, 20, static_cast<u32>(value));
      AppendSMovLiteral(&code, 21, static_cast<u32>(value >> 32u));
      AppendSMovLiteral(&code, count_reg, count);
      check_result(24, 20, count_reg, value,
                   count); // Count aliases destination.
    }
  }
  for (const u32 count_reg : {20u, 21u}) {
    const uint64_t alias_value =
        count_reg == 20u ? 0x8000000000000021ull : 0x0000002100000001ull;
    AppendSMovLiteral(&code, 20, static_cast<u32>(alias_value));
    AppendSMovLiteral(&code, 21, static_cast<u32>(alias_value >> 32u));
    check_result(20, 20, count_reg, alias_value, 33);
  }
  // Both operands may refer to the single literal word in the instruction.
  check_result(24, 255, 255, 0xffffffffffffffffull, 0xffffffffu, 0xffffffffu);
  for (const u32 source : {0u, 1u}) {
    // Discarding the destination must still update SCC and leave M0 untouched.
    AppendSMovLiteral(&code, 124, 0x5a13abcdu);
    AppendSMovLiteral(&code, 20, 0);
    AppendSMovLiteral(&code, 21, source);
    code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(source)));
    code.push_back(EncodeSop2(0x23, 125, 20, InlineU32(0)));
    code.push_back(EncodeSMovB32(26, 253));
    const auto output = static_cast<u32>(test.expected.size());
    AppendStoreSgpr(&code, 26, output);
    AppendStoreSgpr(&code, 124, output + 1u);
    AppendStoreSgprPair(&code, 20, output + 2u);
    test.expected.insert(test.expected.end(), {source, 0x5a13abcdu, 0, source});
    ++instructions;
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::BUFFER_LOAD_DWORD,  O::V_READFIRSTLANE_B32,
                  O::S_MOV_B32,          O::S_CMP_EQ_U32,
                  O::S_ASHR_I64,         O::V_MOV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"S_ASHR_I64", instructions}};
  test.required_spirv.push_back("OpTypeInt 64 0");
  return test;
}

TestCase ScalarAshrI64Masks(u32 wave_size) {
  using O = ShaderOpcode;
  TestCase test;
  test.name =
      wave_size == 64 ? "ScalarAshrI64MasksWave64" : "ScalarAshrI64MasksWave32";
  auto &code = test.code;
  struct Case {
    uint64_t source;
    u32 count;
  };
  constexpr Case cases[] = {
      {0x8000000100000001ull, 1},  {0x8000000100000001ull, 32},
      {0x8000000100000001ull, 63}, {0x0000000100000000ull, 0},
      {0x0000000100000000ull, 32}, {1, 1}};
  for (const u32 mask_reg : {106u, 126u}) {
    for (const auto &item : cases) {
      const auto result = ReferenceAshrI64(item.source, item.count);
      AppendVMovU32(&code, 1, 1);
      AppendVMovU32(&code, 2, 0);
      AppendSMovLiteral(&code, mask_reg, static_cast<u32>(item.source));
      AppendSMovLiteral(&code, mask_reg + 1u,
                        static_cast<u32>(item.source >> 32u));
      code.push_back(
          EncodeSopc(0x06, InlineU32(0), InlineU32(result != 0 ? 1u : 0u)));
      code.push_back(
          EncodeSop2(0x23, mask_reg, mask_reg, InlineU32(item.count)));
      code.push_back(EncodeSMovB32(20, mask_reg));
      code.push_back(EncodeSMovB32(21, mask_reg + 1u));
      code.push_back(EncodeSMovB32(22, 253));
      if (mask_reg == 106u) {
        code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
      } else {
        code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
      }
      code.push_back(EncodeSop1(0x04, 126, 193)); // Restore EXEC for readback.
      const u32 expected[] = {static_cast<u32>(result),
                              static_cast<u32>(result >> 32u),
                              result != 0 ? 1u : 0u};
      for (u32 i = 0; i < std::size(expected); ++i) {
        AppendStoreSgprAtLaneDwordOffset(
            &code, 20 + i, 0, static_cast<u32>(test.expected.size()));
        test.expected.insert(test.expected.end(), wave_size, expected[i]);
      }
      AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
                                       static_cast<u32>(test.expected.size()));
      for (u32 lane = 0; lane < wave_size; ++lane) {
        test.expected.push_back(static_cast<u32>((result >> lane) & 1u));
      }
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::S_MOV_B32,     O::S_MOV_B64,    O::S_CMP_EQ_U32,
                  O::S_ASHR_I64,    O::V_MOV_B32,    O::V_CNDMASK_B32,
                  O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.decoded_counts = {{"S_ASHR_I64 vcc_lo, vcc_lo", std::size(cases)},
                         {"S_ASHR_I64 exec_lo, exec_lo", std::size(cases)}};
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}
TestCase ScalarConditionalMoveB64() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "ScalarConditionalMoveB64";
  auto &code = test.code;
  for (u32 scc : {0u, 1u}) {
    AppendSMovLiteral(&code, 0, 0x13579bdfu);
    AppendSMovLiteral(&code, 1, 0x2468ace0u);
    AppendSMovLiteral(&code, 2, 0xabcdef01u);
    AppendSMovLiteral(&code, 3, 0xfedcba98u);
    code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
    code.push_back(EncodeSop1(0x06, 0, 2));
    code.push_back(EncodeSop2(0x0a, 4, InlineU32(1), InlineU32(0)));
    AppendStoreSgprPair(&code, 0, scc * 3);
    AppendStoreSgpr(&code, 4, scc * 3 + 2);
  }
  AppendEnd(&code);
  test.expected = {0x13579bdfu, 0x2468ace0u, 0,
                   0xabcdef01u, 0xfedcba98u, 1};
  test.opcodes = {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CMOV_B64,
                  O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  return test;
}

TestCase ScalarConditionalMoveB64PreservesMasks() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "ScalarConditionalMoveB64PreservesMasks";
  auto &code = test.code;
  code.push_back(EncodeSop1(0x04, 16, 126)); // Save full EXEC.
  AppendVMovU32(&code, 1, 3);
  AppendVMovU32(&code, 2, 9);
  u32 offset = 0;
  for (u32 destination : {126u, 106u, 8u}) {
    for (u32 scc : {0u, 1u}) {
      code.push_back(EncodeVopc(0xc2, Vgpr(0), 1)); // Only lane 3.
      code.push_back(EncodeSop1(0x04, destination, 106));
      code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
      // EXEC encoding is the captured Pathless instruction, 0xbefe06c1.
      code.push_back(EncodeSop1(0x06, destination, 193u));
      code.push_back(EncodeSop2(0x0a, 4, InlineU32(1), InlineU32(0)));
      code.push_back(EncodeSop1(0x04, 126, destination));
      AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
      code.push_back(EncodeSop1(0x04, 126, 16));
      AppendStoreSgprAtLaneDwordOffset(&code, 4, 0, offset + 64);
      for (u32 lane = 0; lane < 64; ++lane) {
        test.expected.push_back(scc || lane == 3 ? 9u : 0xdeadbeefu);
      }
      test.expected.insert(test.expected.end(), 64, scc);
      offset += 128;
    }
  }
  AppendEnd(&code);
  test.initial.assign(test.expected.size(), 0xdeadbeefu);
  test.opcodes = {O::S_MOV_B64, O::V_MOV_B32, O::V_CMP_EQ_U32,
                  O::S_CMP_EQ_U32, O::S_CMOV_B64, O::S_CSELECT_B32,
                  O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = 64;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarAndn2B64SccUsesMaskShadow() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVopc(0xc2, InlineU32(0), 0),
      EncodeSop1(0x04, 0, 106),
      EncodeSop2(0x15, 4, 0, InlineU32(0)),
      EncodeVop1(0x01, 0, InlineU32(1)),
      EncodeSopp(0x04, 1),
      EncodeVop1(0x01, 0, InlineU32(7)),
  };
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  return {"ScalarAndn2B64SccUsesMaskShadow",
          code,
          {},
          {7},
          {O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_ANDN2_B64, O::V_MOV_B32,
           O::S_CBRANCH_SCC0, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarMaskHighWriteInvalidatesProvenance() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVopc(0xc2, Vgpr(0), 0),      EncodeSop1(0x04, 0, 106),
      EncodeSMovB32(1, InlineU32(1)),    EncodeSop1(0x08, 2, 0),
      EncodeVop1(0x01, 0, InlineU32(1)), EncodeSopp(0x04, 1),
      EncodeVop1(0x01, 0, InlineU32(7)),
  };
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  return {"ScalarMaskHighWriteInvalidatesProvenance",
          code,
          {},
          {7},
          {O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_MOV_B32, O::S_NOT_B64,
           O::V_MOV_B32, O::S_CBRANCH_SCC0, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase ScalarSelectB64PreservesMaskProvenance(u32 wave_size) {
  using O = ShaderOpcode;

  TestCase test;
  test.name = wave_size == 32 ? "ScalarSelectB64PreservesMaskProvenanceWave32"
                              : "ScalarSelectB64PreservesMaskProvenanceWave64";
  auto &code = test.code;
  const auto set_scc = [&] {
    code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1)));
  };
  const auto capture_scc = [&](u32 dst) {
    code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
  };

  AppendSMovLiteral(&code, 0, 0xfffffffeu);
  AppendSMovLiteral(&code, 1, 0xffffffffu);
  code.push_back(EncodeSMovB32(2, InlineU32(0)));
  code.push_back(EncodeSMovB32(3, InlineU32(0)));
  set_scc();
  code.push_back(EncodeSop2(0x0b, 4, 0, 2));
  code.push_back(EncodeSop1(0x08, 6, 4));
  capture_scc(8);

  code.push_back(EncodeVopc(0xc2, InlineU32(0), 0)); // Only lane zero is set.
  code.push_back(EncodeSop1(0x04, 10, 106));
  set_scc();
  code.push_back(EncodeSop2(0x0b, 12, 10, 2));
  code.push_back(EncodeSop1(0x08, 14, 12));
  capture_scc(16);

  AppendStoreSgprAtLaneDwordOffset(&code, 8, 0, 0);
  AppendStoreSgprAtLaneDwordOffset(&code, 16, 0, wave_size);
  test.expected.assign(2u * wave_size, 1u);
  code.push_back(EncodeSop1(0x04, 16, 126)); // Save full EXEC.
  AppendVMovU32(&code, 2, 9);
  constexpr std::array<uint32_t, 4> words{
      0x5aa55aa5u, 0xc39ac39au, 0xa55aa55au, 0x3c653c65u};
  for (u32 i = 0; i < words.size(); ++i) AppendSMovLiteral(&code, i, words[i]);
  for (const u32 destination : {106u, 126u}) {
    for (const u32 scc : {0u, 1u}) {
      code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
      code.push_back(EncodeSop2(0x0b, destination, 0, 2));
      code.push_back(EncodeSop1(0x04, 8, destination)); // Preserve both raw words.
      code.push_back(EncodeSop1(0x04, 126, destination));
      AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
                                      static_cast<u32>(test.expected.size()));
      const u32 selected = scc ? 0u : 2u;
      for (u32 lane = 0; lane < wave_size; ++lane)
        test.expected.push_back((words[selected + lane / 32u] >> (lane & 31u)) & 1u
                                    ? 9u : 0xdeadbeefu);
      code.push_back(EncodeSop1(0x04, 126, 16));
      for (u32 component = 0; component < 2; ++component) {
        AppendStoreSgprAtLaneDwordOffset(&code, 8 + component, 0,
                                        static_cast<u32>(test.expected.size()));
        test.expected.insert(test.expected.end(), wave_size, words[selected + component]);
      }
    }
  }
  AppendEnd(&code);

  test.initial.assign(test.expected.size(), 0xdeadbeefu);
  test.opcodes = {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CSELECT_B64, O::S_NOT_B64,
                  O::S_CSELECT_B32, O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32,
                  O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarWqmB32Masks(u32 wave_size) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = wave_size == 64 ? "ScalarWqmB32MasksWave64" : "ScalarWqmB32MasksWave32";
  auto &code = test.code;
  const auto store_scalar = [&](u32 reg, u32 expected) {
    AppendStoreSgprAtLaneDwordOffset(&code, reg, 0,
                                    static_cast<u32>(test.expected.size()));
    test.expected.insert(test.expected.end(), wave_size, expected);
  };
  const auto store_marker = [&](u32 lo, u32 hi) {
    AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
                                    static_cast<u32>(test.expected.size()));
    for (u32 lane = 0; lane < wave_size; ++lane) {
      const u32 mask = lane < 32 ? lo : hi;
      test.expected.push_back(((mask >> (lane % 32)) & 1u) != 0 ? 9u : 0u);
    }
  };
  constexpr u32 other_half = 0x2468ace0u;
  constexpr u32 masks[] = {0, 1, 8, 0x10, 0x100, 0x80000000u, 0x12481248u, ~0u};
  for (u32 mask : masks) {
    u32 expanded = 0;
    for (u32 bit = 0; bit < 32; bit += 4) {
      if ((mask & (0xfu << bit)) != 0) expanded |= 0xfu << bit;
    }
    AppendVMovU32(&code, 2, 0);
    AppendSMovLiteral(&code, 126, mask);
    AppendSMovLiteral(&code, 127, other_half);
    code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(mask == 0 ? 0 : 1)));
    code.push_back(0xbefe097eu); // Captured S_WQM_B32 EXEC_LO, EXEC_LO.
    code.push_back(EncodeSMovB32(20, 126));
    code.push_back(EncodeSMovB32(21, 127));
    code.push_back(EncodeSMovB32(22, 253));
    AppendVMovU32(&code, 2, 9);
    code.push_back(EncodeSop1(0x04, 126, 193));
    store_scalar(20, expanded);
    store_scalar(21, other_half);
    store_scalar(22, mask != 0 ? 1u : 0u);
    store_marker(expanded, other_half);
  }
  // VALU-produced masks keep WQM dynamic and carry provenance through S_MOV_B64.
  // Updating either word must invalidate an old SGPR-pair predicate and preserve
  // the other word. Wave32 still reads/writes the scalar high half independently.
  for (u32 base : {8u, 106u, 126u}) {
    for (u32 half : {0u, 1u}) {
      AppendVMovU32(&code, 2, 0);
      code.push_back(EncodeVop2(0x1b, 1, InlineU32(31), 0));
      code.push_back(EncodeVopc(0xc2, InlineU32(3), 1)); // Lanes 3 and 35.
      code.push_back(EncodeSop1(0x04, base, 106));
      code.push_back(EncodeSop1(0x09, base + half, base + half));
      code.push_back(EncodeSMovB32(20, base));
      code.push_back(EncodeSMovB32(21, base + 1));
      code.push_back(EncodeSMovB32(22, 253));
      code.push_back(EncodeSop1(0x04, 126, base));
      AppendVMovU32(&code, 2, 9);
      code.push_back(EncodeSop1(0x04, 126, 193));
      const u32 lo = half == 0 ? 0xfu : 8u;
      const u32 hi = wave_size == 32 ? 0u : half == 1 ? 0xfu : 8u;
      store_scalar(20, lo);
      store_scalar(21, hi);
      store_scalar(22, wave_size == 32 && half == 1 ? 0u : 1u);
      store_marker(lo, hi);
    }
  }
  for (u32 base : {106u, 126u}) {
    for (u32 half : {0u, 1u}) {
      AppendVMovU32(&code, 2, 0);
      code.push_back(EncodeSMovB32(107, InlineU32(0)));
      code.push_back(EncodeVopc(0xc2, InlineU32(3), 0)); // Only lane 3.
      code.push_back(EncodeSop1(0x04, base, 106));
      code.push_back(EncodeSop1(0x09, base + half, base + (half ^ 1u)));
      code.push_back(EncodeSMovB32(20, base));
      code.push_back(EncodeSMovB32(21, base + 1));
      code.push_back(EncodeSop1(0x04, 126, base));
      AppendVMovU32(&code, 2, 9);
      code.push_back(EncodeSop1(0x04, 126, 193));
      const u32 lo = half == 0 ? 0u : 8u;
      const u32 hi = half == 0 ? 0u : 0xfu;
      store_scalar(20, lo);
      store_scalar(21, hi);
      store_marker(lo, hi);
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::S_WQM_B32, O::S_MOV_B32, O::S_MOV_B64, O::S_CMP_EQ_U32,
                  O::V_CMP_EQ_U32, O::V_MOV_B32, O::V_AND_B32, O::V_ADD_NC_U32,
                  O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"S_WQM_B32 exec_lo, exec_lo", 9}};
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarWqmB64SelectsSccDomain() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  const auto capture_scc = [&](u32 dst) {
    code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
  };

  code.push_back(EncodeSMovB32(0, InlineU32(0)));
  code.push_back(EncodeSMovB32(1, InlineU32(1)));
  code.push_back(EncodeSop1(0x0a, 2, 0));
  capture_scc(4);

  code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
  code.push_back(EncodeSop1(0x04, 6, 106));
  code.push_back(EncodeSop1(0x08, 8, 6));
  code.push_back(EncodeSop1(0x0a, 10, 8));
  capture_scc(12);

  AppendStoreSgpr(&code, 4, 0);
  AppendStoreSgpr(&code, 12, 1);
  AppendEnd(&code);

  return {"ScalarWqmB64SelectsSccDomain",
          code,
          {},
          {1, 1},
          {O::S_MOV_B32, O::S_WQM_B64, O::S_CSELECT_B32, O::V_CMP_EQ_U32,
           O::S_MOV_B64, O::S_NOT_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase ScalarWqmB64PreservesPartialMasks() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "ScalarWqmB64PreservesPartialMasks";
  auto &code = test.code;
  code.push_back(EncodeSop1(0x04, 16, 126)); // Save full EXEC.
  AppendVMovU32(&code, 1, 3);
  AppendVMovU32(&code, 2, 9);
  u32 offset = 0;
  for (const u32 destination : {126u, 106u, 8u}) {
    code.push_back(EncodeVopc(0xc2, Vgpr(0), 1)); // Only lane 3.
    code.push_back(EncodeSop1(0x04, destination, 106));
    code.push_back(EncodeSop1(0x0a, destination, destination));
    code.push_back(EncodeSop1(0x04, 126, destination));
    AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
    code.push_back(EncodeSop1(0x04, 126, 16));
    offset += 64;
  }
  code.push_back(EncodeSMovB32(8, InlineU32(32))); // Raw bits 5 and 33.
  code.push_back(EncodeSMovB32(9, InlineU32(2)));
  code.push_back(EncodeSop1(0x0a, 10, 8));
  code.push_back(EncodeSop1(0x04, 126, 10));
  AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
  code.push_back(EncodeSop1(0x04, 126, 16));
  for (u32 source : {253u, 127u, 107u}) {
    if (source == 253u) {
      code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1))); // SCC=1.
    } else {
      AppendSMovLiteral(&code, source - 1, ~0u);
      code.push_back(EncodeSMovB32(source, InlineU32(1)));
    }
    code.push_back(EncodeSop1(0x0a, 126, source));
    offset += 64;
    AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
    code.push_back(EncodeSop1(0x04, 126, 16));
  }
  AppendEnd(&code);
  test.initial.assign(448, 0xdeadbeef);
  test.expected = test.initial;
  for (u32 lane = 0; lane < 64; ++lane) {
    for (u32 region = 0; region < 3; ++region) {
      if (lane < 4) test.expected[region * 64 + lane] = 9;
    }
    if ((lane >= 4 && lane < 8) || (lane >= 32 && lane < 36)) {
      test.expected[192 + lane] = 9;
    }
    for (u32 region = 4; region < 7; ++region) {
      if (lane < 4) test.expected[region * 64 + lane] = 9;
    }
  }
  test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::S_WQM_B64, O::V_MOV_B32,
                  O::V_CMP_EQ_U32, O::S_CMP_EQ_U32, O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarMaskProvenanceOverlapAndMixedBinary() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  const auto capture_scc = [&](u32 dst) {
    code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
  };

  code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
  code.push_back(EncodeSop1(0x04, 0, 106));
  code.push_back(EncodeSop1(0x04, 1, 0));
  code.push_back(EncodeSop1(0x08, 3, 1));
  capture_scc(5);

  code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
  code.push_back(EncodeSop1(0x04, 8, 106));
  code.push_back(EncodeSMovB32(10, InlineU32(1)));
  code.push_back(EncodeSMovB32(11, InlineU32(0)));
  code.push_back(EncodeSop2(0x19, 12, 8, 10));
  capture_scc(14);

  AppendStoreSgpr(&code, 5, 0);
  AppendStoreSgpr(&code, 14, 1);
  AppendEnd(&code);

  return {"ScalarMaskProvenanceOverlapAndMixedBinary",
          code,
          {},
          {1, 1},
          {O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_NOT_B64, O::S_CSELECT_B32,
           O::S_MOV_B32, O::S_NAND_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase ScalarB32MaskDataflow(u32 wave_size) {
  using O = ShaderOpcode;
  constexpr u32 mask = 0xffff0000u, rhs = 0x0f0f0f0fu;
  constexpr u32 sentinel = 0xdeadbeefu, marker = 0x12345678u;
  struct Operation {
    u32 encoding;
    O opcode;
    u32 expected;
    u32 source = rhs;
  };
  const std::array operations{
      Operation{0x0e, O::S_AND_B32, mask & rhs},
      Operation{0x10, O::S_OR_B32, mask | rhs},
      Operation{0x12, O::S_XOR_B32, mask ^ rhs},
      Operation{0x14, O::S_ANDN2_B32, mask & ~rhs},
      Operation{0x16, O::S_ORN2_B32, mask | ~rhs},
      Operation{0x18, O::S_NAND_B32, ~(mask & rhs)},
      Operation{0x1a, O::S_NOR_B32, ~(mask | rhs)},
      Operation{0x1c, O::S_XNOR_B32, ~(mask ^ rhs)},
      Operation{0x07, O::S_NOT_B32, ~mask},
      Operation{0x12, O::S_XOR_B32, 0, mask}};
  TestCase test;
  test.name = wave_size == 32 ? "ScalarB32MaskDataflowWave32" : "ScalarB32MaskDataflowWave64";
  auto &code = test.code;
  code.push_back(EncodeSop1(0x04, 16, 126)); // Preserve both EXEC halves.
  AppendVMovU32(&code, 2, marker);
  AppendVMovU32(&code, 1, 4);
  for (const bool predicate_source : {false, true}) {
    for (const auto &operation : operations) {
      code.push_back(EncodeVopc(0xc1, InlineU32(15), 0)); // Lanes greater than 15.
      code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1)));
      code.push_back(EncodeSop2(0x0a, 107, 106, InlineU32(0))); // VCC_HI is a wave32 mask temporary.
      code.push_back(EncodeSMovB32(8, 107));
      if (wave_size == 32) {
        code.push_back(EncodeSMovB32(127, 8));
        code.push_back(EncodeSMovB32(8, 127)); // EXEC_HI must preserve its own scalar word.
        code.push_back(EncodeSMovB32(127, InlineU32(0)));
      }
      AppendSMovLiteral(&code, 9, sentinel); // Adjacent write must preserve a wave32 predicate.
      if (predicate_source) {
        if (operation.source == mask) {
          code.push_back(EncodeVopc(0xc1, InlineU32(15), 0));
        } else {
          code.push_back(EncodeVop2(0x1b, 3, InlineU32(7), 0));
          code.push_back(EncodeVopc(0xc1, Vgpr(3), 1)); // (lane & 7) < 4.
        }
        code.push_back(EncodeSMovB32(10, 106));
      } else {
        code.push_back(EncodeSMovB32(10, operation.source == rhs ? 60u : 61u));
      }
      code.push_back(operation.opcode == O::S_NOT_B32
                         ? EncodeSop1(operation.encoding, 8, 8)
                         : EncodeSop2(operation.encoding, 8, 8, 10));
      code.push_back(EncodeSop2(0x0a, 12, InlineU32(1), InlineU32(0))); // Capture scalar SCC.
      code.push_back(EncodeSop2(0x0a, 8, 8, 10)); // Aliased scalar select, including SCC0.
      code.push_back(EncodeSMovB32(126, 8));
      const auto offset = static_cast<u32>(test.expected.size());
      AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
      code.push_back(EncodeSop1(0x04, 126, 16));
      const auto selected = operation.expected != 0 ? operation.expected : operation.source;
      for (u32 lane = 0; lane < wave_size; ++lane) {
        test.expected.push_back(lane >= 32 || (selected & (1u << lane)) != 0
                                    ? marker : sentinel);
      }
      for (const auto [reg, value] : std::array<std::pair<u32, u32>, 3>{{
               {8, selected}, {9, sentinel}, {12, operation.expected != 0}}}) {
        AppendStoreSgprAtLaneDwordOffset(&code, reg, 0, static_cast<u32>(test.expected.size()));
        test.expected.insert(test.expected.end(), wave_size, value);
      }
      test.opcodes.push_back(operation.opcode);
    }
  }
  if (wave_size == 32) {
    for (u32 low : {106u, 126u}) {
      code.push_back(EncodeVopc(0xc1, InlineU32(15), 0));
      code.push_back(EncodeSMovB32(low + 1, 106)); // A previously known high-word predicate.
      code.push_back(EncodeSop1(0x04, low, 193u)); // Full scalar -1 overwrites both words.
      code.push_back(EncodeSMovB32(8, low + 1));
      code.push_back(EncodeSop1(0x04, 126, 16));
      code.push_back(EncodeSMovB32(126, 8));
      AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, static_cast<u32>(test.expected.size()));
      test.expected.insert(test.expected.end(), wave_size, marker);
      code.push_back(EncodeSop1(0x04, 126, 16));
      AppendStoreSgprAtLaneDwordOffset(&code, 8, 0, static_cast<u32>(test.expected.size()));
      test.expected.insert(test.expected.end(), wave_size, UINT32_MAX);
    }
  }
  AppendEnd(&code);
  test.initial.assign(test.expected.size(), sentinel);
  test.user_data = MakeNativeUserData(nullptr);
  test.user_data[50] = static_cast<u32>(test.expected.size() * sizeof(u32));
  test.user_data[60] = rhs; // Raw masks initialized at shader entry.
  test.user_data[61] = mask;
  test.has_user_data = true;
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  test.opcodes.insert(test.opcodes.end(), {O::S_MOV_B32, O::S_MOV_B64, O::S_CSELECT_B32, O::S_CMP_EQ_U32,
      O::V_CMP_LT_U32, O::V_MOV_B32, O::V_AND_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
      O::BUFFER_STORE_DWORD, O::S_ENDPGM});
  return test;
}

TestCase ScalarLiteral() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 0, 0x12345678u);
  code.push_back(EncodeVop1(0x01, 0, 0));
  AppendBufferStoreDword(&code, 0, 30);
  AppendEnd(&code);
  return {"ScalarLiteral",
          code,
          {},
          {0x12345678u},
          {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorMoves() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVop1(0x00, 0, 0),
      EncodeVop1(0x01, 0, InlineU32(5)),
      EncodeVop1(0x01, 2, Vgpr(0)),
  };
  AppendBufferStoreDword(&code, 2, 30);
  AppendEnd(&code);
  return {"VectorRegisterMoves",
          code,
          {},
          {5},
          {O::V_NOP, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorVop3MoveAppliesFloatSourceModifiers() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 24, 0x40000000u);
  AppendVop3(&code, 0x181, 1, 24, 0, 0, 0, 0, false, 0, 0x1);
  AppendStoreVgpr(&code, 1, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorVop3MoveAppliesFloatSourceModifiers";
  test.code = code;
  test.expected = {0xc0000000u};
  test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.required_spirv = {"OpFNegate"};
  return test;
}

TestCase VectorIntegerOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0xfffffff8u);
  code.push_back(EncodeVop1(0x01, 1, InlineU32(5)));
  code.push_back(EncodeVop1(0x01, 4, InlineU32(7)));
  AppendVMovLiteral(&code, 10, 0x01000001u);
  code.push_back(EncodeVop1(0x01, 11, InlineU32(2)));
  AppendVMovLiteral(&code, 12, 0x0f0f0f0fu);
  AppendVMovLiteral(&code, 13, 0x33333333u);
  AppendVMovLiteral(&code, 14, 0xfffffff0u);
  code.push_back(EncodeVop1(0x01, 15, InlineU32(4)));
  code.push_back(EncodeVop1(0x01, 29, InlineU32(15)));

  code.push_back(EncodeVop2(0x11, 2, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x12, 3, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x13, 5, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x14, 6, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x25, 7, Vgpr(4), 1));
  code.push_back(EncodeVop2(0x26, 8, Vgpr(4), 1));
  code.push_back(EncodeVop2(0x27, 9, InlineU32(3), 4));
  code.push_back(EncodeVop2(0x26, 35, 249, 11));
  code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6));
  code.push_back(EncodeVop2(0x27, 36, 249, 11));
  code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6));
  code.push_back(EncodeVop2(0x0b, 16, Vgpr(10), 11));
  code.push_back(EncodeVop2(0x09, 37, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x1b, 17, Vgpr(12), 13));
  code.push_back(EncodeVop2(0x1c, 18, Vgpr(12), 13));
  code.push_back(EncodeVop2(0x1d, 19, Vgpr(12), 13));
  code.push_back(EncodeVop2(0x1e, 20, Vgpr(12), 13));
  code.push_back(EncodeVop2(0x19, 21, Vgpr(11), 15));
  code.push_back(EncodeVop2(0x1a, 22, InlineU32(4), 11));
  code.push_back(EncodeVop2(0x15, 23, Vgpr(14), 15));
  code.push_back(EncodeVop2(0x16, 24, InlineU32(4), 14));
  code.push_back(EncodeVop2(0x17, 25, Vgpr(14), 15));
  code.push_back(EncodeVop2(0x18, 26, InlineU32(4), 14));
  code.push_back(EncodeVop1(0x37, 27, Vgpr(12)));
  code.push_back(EncodeVop1(0x38, 28, Vgpr(29)));
  code.push_back(EncodeVop1(0x3a, 30, Vgpr(11)));
  code.push_back(EncodeVop1(0x39, 32, Vgpr(11)));
  code.push_back(EncodeVopc(0xc2, Vgpr(4), 4));
  code.push_back(EncodeVop2(0x01, 33, InlineU32(3), 4));
  code.push_back(EncodeVop2(0x11, 34, 249, 4));
  code.push_back(EncodeVop2Sdwa(0));

  const u32 results[] = {2,  3,  5,  6,  7,  8,  9,  35, 36, 16, 37, 17, 18, 19,
                         20, 21, 22, 23, 24, 25, 26, 27, 28, 30, 32, 33, 34};
  for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
    AppendStoreVgpr(&code, results[i], i);
  }
  AppendEnd(&code);

  return {"VectorIntegerOps",
          code,
          {},
          {0xfffffff8u, 5,           5,           0xfffffff8u, 12,
           2,           4,           13,          0xfffffff3u, 2,
           0xffffffd8u, 0x03030303u, 0x3f3f3f3fu, 0x3c3c3c3cu, 0xc3c3c3c3u,
           32,          32,          0x0fffffffu, 0x0fffffffu, 0xffffffffu,
           0xffffffffu, 0xf0f0f0f0u, 0xf0000000u, 1,           30,
           7,           0xfffffff8u},
          {O::V_MOV_B32,     O::V_MIN_I32,       O::V_MAX_I32,
           O::V_MIN_U32,     O::V_MAX_U32,       O::V_ADD_NC_U32,
           O::V_SUB_NC_U32,  O::V_SUBREV_NC_U32, O::V_MUL_U32_U24,
           O::V_MUL_I32_I24, O::V_AND_B32,       O::V_OR_B32,
           O::V_XOR_B32,     O::V_XNOR_B32,      O::V_LSHL_B32,
           O::V_LSHLREV_B32, O::V_LSHR_B32,      O::V_LSHRREV_B32,
           O::V_ASHR_I32,    O::V_ASHRREV_I32,   O::V_NOT_B32,
           O::V_BFREV_B32,   O::V_FFBL_B32,      O::V_FFBH_U32,
           O::V_CMP_EQ_U32,  O::V_CNDMASK_B32,   O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase VectorIntegerExtremaRuntimeEdges() {
  using O = ShaderOpcode;
  constexpr std::array<std::array<u32, 3>, 8> inputs{{
      {0x80000000u, 0u, 0x7fffffffu},
      {0xffffffffu, 1u, 0x80000000u},
      {1u, 0x80000000u, 0xffffffffu},
      {1u, 0xffffffffu, 0x80000000u},
      {0x7fffffffu, 0x7fffffffu, 0x80000000u},
      {0x80000000u, 0x80000000u, 0xffffffffu},
      {0xffffffffu, 0xffffffffu, 0xffffffffu},
      {0u, 0u, 0u},
  }};
  TestCase test;
  test.name = "VectorIntegerExtremaRuntimeEdges";
  for (const auto &values : inputs)
    test.initial.insert(test.initial.end(), values.begin(), values.end());
  test.expected = test.initial;
  for (u32 i = 0; i < inputs.size(); ++i) {
    for (u32 reg = 0; reg < 3; ++reg) {
      AppendVMovU32(&test.code, 30, (i * 3u + reg) * 4u);
      AppendBufferLoadDword(&test.code, reg, 30);
    }
    for (u32 op = 0; op < 4; ++op)
      test.code.push_back(EncodeVop2(0x11u + op, 4u + op, Vgpr(0), 1));
    constexpr std::array opcodes{0x152u, 0x155u, 0x158u, 0x153u, 0x156u, 0x159u};
    for (u32 op = 0; op < opcodes.size(); ++op)
      AppendVop3(&test.code, opcodes[op], 8u + op, Vgpr(0), Vgpr(1), Vgpr(2));
    const auto output = static_cast<u32>(test.expected.size());
    for (u32 result = 0; result < 10; ++result)
      AppendStoreVgpr(&test.code, 4u + result, output + result);

    auto unsigned_values = inputs[i];
    std::array<int32_t, 3> signed_values;
    for (u32 j = 0; j < 3; ++j)
      signed_values[j] = std::bit_cast<int32_t>(unsigned_values[j]);
    test.expected.insert(test.expected.end(), {
        static_cast<u32>(std::min(signed_values[0], signed_values[1])),
        static_cast<u32>(std::max(signed_values[0], signed_values[1])),
        std::min(unsigned_values[0], unsigned_values[1]),
        std::max(unsigned_values[0], unsigned_values[1])});
    std::sort(signed_values.begin(), signed_values.end());
    std::sort(unsigned_values.begin(), unsigned_values.end());
    test.expected.insert(test.expected.end(), {
        static_cast<u32>(signed_values[0]), static_cast<u32>(signed_values[2]),
        static_cast<u32>(signed_values[1]), unsigned_values[0],
        unsigned_values[2], unsigned_values[1]});
  }
  AppendEnd(&test.code);
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MIN_I32, O::V_MAX_I32,
                  O::V_MIN_U32, O::V_MAX_U32, O::V_MIN3_I32, O::V_MAX3_I32,
                  O::V_MED3_I32, O::V_MIN3_U32, O::V_MAX3_U32, O::V_MED3_U32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {" SMin ", " SMax ", " UMin ", " UMax "};
  return test;
}

TestCase VectorFfbhI32NativeAndVop3OnGpu() {
  using O = ShaderOpcode;

  constexpr std::array inputs = {0x00000000u, 0x40000000u, 0x80000000u,
                                 0x0fffffffu, 0xffff0000u, 0xfffffffeu,
                                 0xffffffffu, 0x80000000u};
  std::vector<u32> code;
  for (u32 i = 0; i < inputs.size() - 1u; i++) {
    AppendVMovU32(&code, 30, i * 4u);
    AppendBufferLoadDword(&code, 14, 30);
    code.push_back(0x7e1c770eu); // v_ffbh_i32 v14, v14
    AppendStoreVgpr(&code, 14, i);
  }
  AppendVMovU32(&code, 30, (inputs.size() - 1u) * 4u);
  AppendBufferLoadDword(&code, 14, 30);
  AppendVop3(&code, 0x1bbu, 15, Vgpr(14), 0);
  AppendStoreVgpr(&code, 15, inputs.size() - 1u);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorFfbhI32NativeAndVop3OnGpu";
  test.code = std::move(code);
  test.initial.assign(inputs.begin(), inputs.end());
  test.expected = {0xffffffffu, 1u, 1u, 4u, 16u, 31u, 0xffffffffu, 1u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FFBH_I32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_FFBH_I32 v14, v14", inputs.size() - 1u},
                         {"V_FFBH_I32 v15, v14", 1u}};
  return test;
}

TestCase Vop1SdwaBfrevSourceSelectors() {
  using O = ShaderOpcode;
  constexpr u32 expected[2][7] = {
      {0x8f000000u, 0xd5000000u, 0x21000000u, 0x49000000u,
       0x8fd50000u, 0x21490000u, 0x8fd52149u},
      {0x8fffffffu, 0xd5ffffffu, 0x21ffffffu, 0x49ffffffu,
       0x8fd5ffffu, 0x2149ffffu, 0x8fd52149u}};
  TestCase test;
  test.name = "Vop1SdwaBfrevSourceSelectors";
  for (u32 sext = 0; sext < 2; sext++) {
    for (u32 selector = 0; selector < 7; selector++) {
      const u32 index = static_cast<u32>(test.initial.size());
      AppendVMovU32(&test.code, 30, index * 4u);
      AppendBufferLoadDword(&test.code, 0, 30);
      // selector=4, sext=0 reproduces captured words 7e0070f9, 00040600.
      test.code.insert(test.code.end(),
                       {0x7e0070f9u, EncodeVop1Sdwa(0, 6, 0, selector, sext)});
      AppendStoreVgpr(&test.code, 0, index);
      test.initial.push_back(0x9284abf1u);
      test.expected.push_back(expected[sext][selector]);
    }
  }
  AppendEnd(&test.code);
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_BFREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_BFREV_B32 v0, v0.sdwa(sel=4,sext=0)", 1}};
  test.required_spirv = {"OpBitFieldUExtract", "OpBitFieldSExtract", "OpBitReverse"};
  return test;
}

TestCase Vop1SdwaFfbhCapturedScalarLowWordSource() {
  using O = ShaderOpcode;

  constexpr std::array inputs = {0xdead0000u, 0xffff0001u, 0x00008000u,
                                 0x00010008u, 0xffff4000u, 0x0000ffffu};
  std::vector<u32> code;
  for (u32 i = 0; i < inputs.size(); i++) {
    AppendSmemLoadOpcode(&code, 0x08, 39, i * 4u);
    code.insert(code.end(), {0x7e0072f9u, 0x00840627u}); // v_ffbh_u32 v0, s39.word0
    AppendStoreVgpr(&code, 0, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop1SdwaFfbhCapturedScalarLowWordSource";
  test.code = std::move(code);
  test.initial.assign(inputs.begin(), inputs.end());
  test.expected = {0xffffffffu, 31u, 16u, 28u, 17u, 16u};
  test.opcodes = {O::S_BUFFER_LOAD_DWORD, O::V_FFBH_U32, O::V_MOV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_FFBH_U32 v0, s39.sdwa(sel=4,sext=0)", inputs.size()}};
  test.required_spirv = {"OpBitFieldUExtract", "FindUMsb"};
  return test;
}

TestCase Vop1SdwaFfblCapturedHighWordSource() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendBufferLoadDword(&code, 2, 30);
  code.push_back(0x7e2074f9u);
  code.push_back(0x00050602u); // v_ffbl_b32 v16, v2.word1
  AppendStoreVgpr(&code, 16, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop1SdwaFfblCapturedHighWordSource";
  test.code = std::move(code);
  test.initial = {0x00400000u};
  test.expected = {6u};
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_FFBL_B32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.decoded_counts = {{"V_FFBL_B32 v16, v2.sdwa(sel=5,sext=0)", 1}};
  test.ir_counts = {{" = BitFieldUExtract ", 1}, {" = FindILsb32 ", 1}};
  test.required_spirv = {"OpBitFieldUExtract"};
  return test;
}

TestCase Vop1SdwaNotCapturedByte0Source() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendBufferLoadDword(&code, 2, 30);
  code.push_back(0x7e046ef9u);
  code.push_back(0x00000602u); // v_not_b32 v2, v2.byte0
  AppendStoreVgpr(&code, 2, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop1SdwaNotCapturedByte0Source";
  test.code = std::move(code);
  test.initial = {0x12345678u};
  test.expected = {0xffffff87u};
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_NOT_B32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.decoded_counts = {{"V_NOT_B32 v2, v2.sdwa(sel=0,sext=0)", 1}};
  test.ir_counts = {{" = BitFieldUExtract ", 1}, {" = BitwiseNot32 ", 1}};
  test.required_spirv = {"OpBitFieldUExtract", "OpNot"};
  return test;
}

TestCase Vop1SdwaNotPreservesHighWordDestination() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendBufferLoadDword(&code, 0, 30);
  AppendVMovLiteral(&code, 3, 0xabcd5555u);
  code.push_back(0x7e066ef9u);
  code.push_back(0x00061400u);
  AppendStoreVgpr(&code, 3, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop1SdwaNotPreservesHighWordDestination";
  test.code = std::move(code);
  test.initial = {0x12345678u};
  test.expected = {0xabcda987u};
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::V_NOT_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.ir_counts = {{" = BitwiseNot32 ", 1}};
  test.required_spirv = {"OpNot"};
  return test;
}

TestCase Vop1SdwaNotPartialSourcesAndDestinations() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendBufferLoadDword(&code, 8, 30);
  AppendVMovLiteral(&code, 28, 0xabcd5555u);
  code.push_back(EncodeVop1(0x37, 28, 249));
  code.push_back(EncodeVop1Sdwa(8, 4, 2, 5)); // v28.word0 = ~v8.word1, preserve word1
  AppendStoreVgpr(&code, 28, 0);

  AppendVMovLiteral(&code, 3, 0xa1b2c3d4u);
  code.push_back(EncodeVop1(0x37, 3, 249));
  code.push_back(EncodeVop1Sdwa(8, 2, 2, 0)); // v3.byte2 = ~v8.byte0, preserve others
  AppendStoreVgpr(&code, 3, 1);

  code.push_back(EncodeVop1(0x37, 4, 249));
  code.push_back(EncodeVop1Sdwa(8, 0, 0, 0)); // v4.byte0 = ~v8.byte0, zero others
  AppendStoreVgpr(&code, 4, 2);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop1SdwaNotPartialSourcesAndDestinations";
  test.code = std::move(code);
  test.initial = {0x12345678u, 0u, 0u};
  test.expected = {0xabcdedcbu, 0xa187c3d4u, 0x00000087u};
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::V_NOT_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_NOT_B32", 3u}};
  test.required_spirv = {"OpBitFieldUExtract", "OpNot"};
  return test;
}

TestCase Vop2SdwaBitwiseByteDestinations(u32 wave_size) {
  using O = ShaderOpcode;
  // Low result bytes of AND, OR, XOR, XNOR for the two runtime inputs.
  constexpr std::array<u32, 4> result_bytes{0xa4u, 0xbeu, 0x1au, 0xe5u};
  constexpr std::array<u32, 4> alias_expected{
      0xc350e5a6u, 0xc3fce5a6u, 0xc3ace5a6u, 0xc353e5a6u};
  constexpr u32 sentinel = 0xa1b2c3d4u;
  TestCase test;
  test.name = wave_size == 64 ? "Vop2SdwaBitwiseByteDestinationsWave64"
                             : "Vop2SdwaBitwiseByteDestinationsWave32";
  test.initial = {0xc3d4e5a6u, 0x9b785abcu};
  test.expected = test.initial;
  auto &code = test.code;
  const auto store = [&](u32 reg, u32 expected) {
    AppendStoreVgpr(&code, reg, static_cast<u32>(test.expected.size()));
    test.expected.push_back(expected);
  };
  for (u32 op = 0; op < 4; ++op) {
    AppendVMovU32(&code, 30, 0);
    AppendBufferLoadDword(&code, 12, 30);
    AppendVMovU32(&code, 30, 4);
    AppendBufferLoadDword(&code, 13, 30);
    for (u32 selector = 0; selector < 4; ++selector) {
      for (u32 unused = 0; unused < 3; ++unused) {
        AppendVMovLiteral(&code, 26, sentinel);
        code.push_back(EncodeVop2(0x1b + op, 26, 249, 13));
        code.push_back(EncodeVop2Sdwa(12, selector, unused));
        // RDNA2 table 88: pad, sign extend above/zero below, or preserve.
        const u32 shift = selector * 8u;
        u32 expected = result_bytes[op] << shift;
        if (unused == 1 && (result_bytes[op] & 0x80u) != 0) {
          expected = (0xffffff00u | result_bytes[op]) << shift;
        } else if (unused == 2) {
          expected |= sentinel & ~(0xffu << shift);
        }
        store(26, expected);
      }
    }
    // Read source byte2 and word1 before overwriting source0's byte2.
    code.push_back(EncodeVop2(0x1b + op, 12, 249, 13));
    code.push_back(EncodeVop2Sdwa(12, 2, 2, 2, 5));
    store(12, alias_expected[op]);
    // Source1 may also alias the destination.
    AppendVMovU32(&code, 30, 0);
    AppendBufferLoadDword(&code, 12, 30);
    code.push_back(EncodeVop2(0x1b + op, 13, 249, 13));
    code.push_back(EncodeVop2Sdwa(12, 1, 2));
    store(13, 0x9b7800bcu | (result_bytes[op] << 8u));
    for (u32 unused = 0; unused < 3; ++unused) {
      AppendVMovLiteral(&code, 26, sentinel);
      code.push_back(EncodeSMovB32(126, InlineU32(0)));
      code.push_back(EncodeVop2(0x1b + op, 26, 249, 13));
      code.push_back(EncodeVop2Sdwa(12, 2, unused));
      code.push_back(EncodeSMovB32(126, InlineU32(1)));
      store(26, sentinel);
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_AND_B32, O::V_OR_B32, O::V_XOR_B32, O::V_XNOR_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpBitwiseAnd", "OpBitwiseOr", "OpBitwiseXor", "OpNot",
                         "OpBitFieldInsert", "OpBitFieldSExtract"};
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase Vop1SdwaMovByteDestinations() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendBufferLoadDword(&code, 3, 30);
  AppendVMovU32(&code, 30, 4);
  AppendBufferLoadDword(&code, 2, 30);
  code.push_back(EncodeVop1(0x01, 1, Vgpr(3)));
  code.push_back(0x7e0202f9u);
  code.push_back(0x00861280u); // Captured v_mov_b32 v1.byte2, 0, preserve.
  AppendStoreVgpr(&code, 1, 2);

  // RDNA2 table 88: select any byte, then pad, sign extend, or preserve.
  for (u32 dst_u = 0; dst_u < 3; dst_u++) {
    for (u32 dst_sel = 0; dst_sel < 4; dst_sel++) {
      code.push_back(EncodeVop1(0x01, 1, Vgpr(3)));
      code.push_back(EncodeVop1(0x01, 1, 249));
      code.push_back(EncodeVop1Sdwa(2, dst_sel, dst_u));
      AppendStoreVgpr(&code, 1, 3 + dst_u * 4 + dst_sel);
    }
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop1SdwaMovByteDestinations";
  test.code = std::move(code);
  test.initial = {0xa1b2c3d4u, 0x12345680u};
  test.expected = {0xa1b2c3d4u, 0x12345680u, 0xa100c3d4u,
                   0x00000080u, 0x00008000u, 0x00800000u, 0x80000000u,
                   0xffffff80u, 0xffff8000u, 0xff800000u, 0x80000000u,
                   0xa1b2c380u, 0xa1b280d4u, 0xa180c3d4u, 0x80b2c3d4u};
  test.initial.resize(test.expected.size());
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.decoded_counts = {{"V_MOV_B32 v1.sdwa(sel=2,sext=0), 0", 1}};
  test.required_spirv = {"OpBitFieldInsert", "OpBitFieldSExtract", "OpBitwiseOr"};
  return test;
}

TestCase Vop1SdwaMovByteSources(u32 wave_size) {
  using O = ShaderOpcode;
  // Select 0x80 from each byte position, with nonzero neighboring bytes.
  constexpr std::array<u32, 4> sources{
      0x12345680u, 0x12348056u, 0x12803456u, 0x80341256u};
  // RDNA2 table 88 / PS5 SDWA-related options: PAD, SEXT, PRESERVE.
  constexpr std::array<u32, 18> partial_expected{
      0x00000080u, 0xffffff80u, 0xa1b2c380u,
      0x00008000u, 0xffff8000u, 0xa1b280d4u,
      0x00800000u, 0xff800000u, 0xa180c3d4u,
      0x80000000u, 0x80000000u, 0x80b2c3d4u,
      0x00000080u, 0x00000080u, 0xa1b20080u,
      0x00800000u, 0x00800000u, 0x0080c3d4u};
  constexpr std::array<u32, 6> sext_word_expected{
      0x0000ff80u, 0xffffff80u, 0xa1b2ff80u,
      0xff800000u, 0xff800000u, 0xff80c3d4u};
  TestCase test;
  test.name = wave_size == 64 ? "Vop1SdwaMovByteSourcesWave64"
                             : "Vop1SdwaMovByteSourcesWave32";
  test.initial.assign(sources.begin(), sources.end());
  test.expected = test.initial;
  auto &code = test.code;
  const auto store = [&](u32 expected) {
    AppendStoreVgpr(&code, 1, static_cast<u32>(test.expected.size()));
    test.expected.push_back(expected);
  };
  for (u32 src_sel = 0; src_sel < sources.size(); ++src_sel) {
    AppendVMovU32(&code, 30, src_sel * sizeof(u32));
    AppendBufferLoadDword(&code, 2, 30);
    for (u32 sext = 0; sext < 2; ++sext) {
      for (u32 dst_sel = 0; dst_sel < 6; ++dst_sel) {
        for (u32 unused = 0; unused < 3; ++unused) {
          AppendVMovLiteral(&code, 1, 0xa1b2c3d4u);
          code.push_back(EncodeVop1(0x01, 1, 249));
          code.push_back(EncodeVop1Sdwa(2, dst_sel, unused, src_sel, sext));
          store(sext && dst_sel >= 4 ? sext_word_expected[(dst_sel - 4u) * 3u + unused]
                                    : partial_expected[dst_sel * 3u + unused]);
        }
      }
    }
  }
  // Aliasing must read the source byte before replacing the destination byte.
  AppendVMovU32(&code, 30, 0);
  AppendBufferLoadDword(&code, 1, 30);
  code.push_back(EncodeVop1(0x01, 1, 249));
  code.push_back(EncodeVop1Sdwa(1, 3, 2, 1));
  store(0x56345680u);
  // Scalar and inline sources use the same independent selectors.
  AppendSMovLiteral(&code, 18, 0x12348056u);
  code.push_back(EncodeVop1(0x01, 1, 249));
  code.push_back(EncodeVop1Sdwa(18, 5, 0, 1, 1, 0, 0, 1));
  store(0xff800000u);
  AppendVMovLiteral(&code, 1, 0xa1b2c3d4u);
  code.push_back(EncodeVop1(0x01, 1, 249));
  code.push_back(EncodeVop1Sdwa(193, 0, 2, 3, 0, 0, 0, 1)); // Inline -1.
  store(0xa1b2c3ffu);
  for (u32 unused = 0; unused < 3; ++unused) {
    AppendVMovLiteral(&code, 1, 0xa1b2c3d4u);
    code.push_back(EncodeSMovB32(126, InlineU32(0)));
    code.push_back(EncodeVop1(0x01, 1, 249));
    code.push_back(EncodeVop1Sdwa(2, 4, unused, 0));
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    store(0xa1b2c3d4u);
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpBitFieldUExtract", "OpBitFieldSExtract", "OpBitFieldInsert"};
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase Vop2SdwaSubNcExactByte2Destination() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 11, 2);
  code.push_back(0x4c1616f9u);
  code.push_back(0x0686128du);
  AppendStoreVgpr(&code, 11, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop2SdwaSubNcExactByte2Destination";
  test.code = code;
  test.expected = {0x000b0002u};
  test.opcodes = {O::V_MOV_B32, O::V_SUB_NC_U32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  return test;
}

TestCase Vop2SdwaAddNcCapturedHighWordDestination() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendBufferLoadDword(&code, 0, 30);
  code.push_back(EncodeVop1(0x02, 106, Vgpr(0)));
  AppendVMovLiteral(&code, 97, 0x00030004u);
  AppendVMovLiteral(&code, 4, 0xa1b2c3d4u);
  code.push_back(0x4a08c2f9u);
  code.push_back(0x0686156au);
  AppendStoreVgpr(&code, 4, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop2SdwaAddNcCapturedHighWordDestination";
  test.code = std::move(code);
  test.initial = {0x00010002u, 0};
  test.expected = {0x00010002u, 0x0006c3d4u};
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::V_MOV_B32,
                  O::V_ADD_NC_U32,      O::BUFFER_STORE_DWORD,  O::S_ENDPGM};
  test.required_spirv = {"OpIAdd", "OpShiftLeftLogical", "OpBitwiseOr"};
  return test;
}

TestCase Vop2SdwaAshrrevCapturedWord0SignExtends() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendBufferLoadDword(&code, 1, 30);
  code.push_back(0x303202f9u);
  code.push_back(0x0c860686u); // v_ashrrev_i32 v25, 6, sign_extend(v1.word0)
  AppendStoreVgpr(&code, 25, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop2SdwaAshrrevCapturedWord0SignExtends";
  test.code = std::move(code);
  test.initial = {0xabcd8000u, 0};
  test.expected = {0xabcd8000u, 0xfffffe00u};
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_ASHRREV_I32, O::V_MOV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_ASHRREV_I32 v25, 6, v1.sdwa(sel=4,sext=1)", 1}};
  test.ir_counts = {{" = BitFieldSExtract ", 1},
                    {" = ShiftRightArithmetic32 ", 1}};
  test.required_spirv = {"OpBitFieldSExtract", "OpShiftRightArithmetic"};
  return test;
}

TestCase Vop2SdwaMaxI32CapturedHighWord(u32 wave_size) {
  using O = ShaderOpcode;
  struct MaxCase { u32 lhs, packed_rhs, expected, exec = 1; };
  constexpr std::array<MaxCase, 9> cases{{
      {0xfffffffbu, 0xfffe1234u, 0xfffffffeu}, // -5 versus sign-extended -2.
      {0xffffffffu, 0x80007fffu, 0xffffffffu},
      {3, 0x0007ffffu, 7},                  // Ignore the low word.
      {0xffffffffu, 0x0000ffffu, 0},
      {0x80000000u, 0x80000000u, 0xffff8000u},
      {0x7fffffffu, 0x8000ffffu, 0x7fffffffu},
      {0x0000ffffu, 0x7fffffffu, 0x0000ffffu}, // Source 0 stays 32-bit.
      {0x00007fffu, 0x7fff8000u, 0x00007fffu},
      {0x80000001u, 0x7fff0123u, 0x80000001u, 0}, // Inactive destination.
  }};
  TestCase test;
  test.name = wave_size == 64 ? "Vop2SdwaMaxI32HighWordWave64"
                             : "Vop2SdwaMaxI32HighWordWave32";
  for (const auto &entry : cases) {
    test.initial.insert(test.initial.end(), {entry.lhs, entry.packed_rhs});
  }
  test.initial.resize(cases.size() * 3u, 0xdeadbeefu);
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < cases.size(); ++i) {
    AppendVMovU32(&code, 30, i * 8u);
    AppendBufferLoadDword(&code, 11, 30);
    AppendVMovU32(&code, 30, i * 8u + 4u);
    AppendBufferLoadDword(&code, 22, 30);
    code.push_back(EncodeSMovB32(126, InlineU32(cases[i].exec)));
    // Exact game encoding: v11 = max(v11, sign_extend(v22.word1)).
    code.insert(code.end(), {0x24162cf9u, 0x0d06060bu});
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    const u32 out = cases.size() * 2u + i;
    AppendStoreVgpr(&code, 11, out);
    test.expected[out] = cases[i].expected;
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_MAX_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_MAX_I32 v11, v11, v22.sdwa(sel=5,sext=1)", cases.size()}};
  test.required_spirv = {"OpBitFieldSExtract"};
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase Vop2SdwaMinI32CapturedByte0(u32 wave_size) {
  using O = ShaderOpcode;
  struct MinCase { u32 lhs, packed_rhs, expected, exec = 1; };
  constexpr std::array<MinCase, 9> cases{{
      {0xfffffffbu, 0x12345678u, 0xfffffffbu},
      {0x00000064u, 0xffffff80u, 0x00000064u},
      {0x000000c8u, 0xffffff80u, 0x00000080u},
      {0x7fffffffu, 0x000000ffu, 0x000000ffu},
      {0x80000000u, 0x7fffffffu, 0x80000000u},
      {0x00000100u, 0x00000105u, 0x00000005u},
      {0xffffffffu, 0x00000000u, 0xffffffffu},
      {0x00000010u, 0xdeadbe0fu, 0x0000000fu},
      {0x80000001u, 0x7fff0123u, 0x7fff0123u, 0},
  }};
  TestCase test;
  test.name = wave_size == 64 ? "Vop2SdwaMinI32Byte0Wave64"
                             : "Vop2SdwaMinI32Byte0Wave32";
  for (const auto &entry : cases) {
    test.initial.insert(test.initial.end(), {entry.lhs, entry.packed_rhs});
  }
  test.initial.resize(cases.size() * 3u, 0xdeadbeefu);
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < cases.size(); ++i) {
    AppendVMovU32(&code, 30, i * 8u);
    AppendBufferLoadDword(&code, 5, 30);
    AppendVMovU32(&code, 30, i * 8u + 4u);
    AppendBufferLoadDword(&code, 2, 30);
    code.push_back(EncodeSMovB32(126, InlineU32(cases[i].exec)));
    code.insert(code.end(), {0x220404f9u, 0x00060605u});
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    const u32 out = cases.size() * 2u + i;
    AppendStoreVgpr(&code, 2, out);
    test.expected[out] = cases[i].expected;
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_MIN_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_MIN_I32 v2, v5, v2.sdwa(sel=0,sext=0)", cases.size()}};
  test.required_spirv = {"OpBitFieldUExtract"};
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase Vop2SdwaMulI24Destinations(u32 wave_size) {
  using O = ShaderOpcode;
  constexpr std::array<u32, 5> sources{
      0xdead1234u, 0xabcdffffu, 0x98768001u, 0x34560000u, 0x56789abcu};
  constexpr std::array<u32, 10> captured_expected{
      0xdead91a0u, 0xdead3400u, 0xabcdfff8u, 0xabcdff00u,
      0x98760008u, 0x98760100u, 0x34560000u, 0x34560000u,
      0x56789abcu, 0x56789abcu};
  constexpr std::array<u32, 18> partial_expected{
      0x00000080u, 0xffffff80u, 0xa1b2c380u,
      0x00008000u, 0xffff8000u, 0xa1b280d4u,
      0x00800000u, 0xff800000u, 0xa180c3d4u,
      0x80000000u, 0x80000000u, 0x80b2c3d4u,
      0x0000a180u, 0xffffa180u, 0xa1b2a180u,
      0xa1800000u, 0xa1800000u, 0xa180c3d4u};
  TestCase test;
  test.name = wave_size == 64 ? "Vop2SdwaMulI24DestinationsWave64"
                             : "Vop2SdwaMulI24DestinationsWave32";
  test.initial.assign(sources.begin(), sources.end());
  test.initial.push_back(0xabcde080u);
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 index = 0; index < sources.size(); ++index) {
    AppendVMovU32(&code, 30, index * sizeof(u32));
    AppendBufferLoadDword(&code, 12, 30);
    AppendBufferLoadDword(&code, 26, 30);
    AppendSMovLiteral(&code, 106, 256);
    code.push_back(EncodeSMovB32(126, InlineU32(index + 1u == sources.size() ? 0 : 1)));
    // Captured WORD0/PRESERVE encodings alias their source and destination.
    code.insert(code.end(), {0x121818f9u, 0x04861488u,
                             0x123434f9u, 0x0486146au});
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    AppendStoreVgpr(&code, 12, static_cast<u32>(test.expected.size()));
    AppendStoreVgpr(&code, 26, static_cast<u32>(test.expected.size()) + 1u);
    test.expected.push_back(captured_expected[index * 2u]);
    test.expected.push_back(captured_expected[index * 2u + 1u]);
  }
  AppendVMovU32(&code, 30, sources.size() * sizeof(u32));
  AppendBufferLoadDword(&code, 12, 30);
  for (u32 selector = 0; selector < 6; ++selector) {
    for (u32 unused = 0; unused < 3; ++unused) {
      AppendVMovLiteral(&code, 26, 0xa1b2c3d4u);
      code.push_back(EncodeVop2(0x09, 26, 249, 12));
      code.push_back(EncodeVop2Sdwa(InlineU32(3), selector, unused, 6, 4,
                                    0, 0, 0, 0, 0, 0, 1));
      AppendStoreVgpr(&code, 26, static_cast<u32>(test.expected.size()));
      test.expected.push_back(partial_expected[selector * 3u + unused]);
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_MUL_I32_I24, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpIMul", "OpBitFieldUExtract"};
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase Vop2SdwaMulU24Destinations(u32 wave_size) {
  using O = ShaderOpcode;
  // The high source bytes are discarded: 3 * 0xe080 = 0x2a180.
  constexpr std::array<u32, 18> partial_expected{
      0x00000080u, 0xffffff80u, 0xa1b2c380u,
      0x00008000u, 0xffff8000u, 0xa1b280d4u,
      0x00800000u, 0xff800000u, 0xa180c3d4u,
      0x80000000u, 0x80000000u, 0x80b2c3d4u,
      0x0000a180u, 0xffffa180u, 0xa1b2a180u,
      0xa1800000u, 0xa1800000u, 0xa180c3d4u};
  TestCase test;
  test.name = wave_size == 64 ? "Vop2SdwaMulU24DestinationsWave64"
                             : "Vop2SdwaMulU24DestinationsWave32";
  test.initial = {0xab000003u, 0xcd00e080u, 0xffffffffu};
  test.expected = test.initial;
  auto &code = test.code;
  const auto store = [&](u32 reg, u32 expected) {
    AppendStoreVgpr(&code, reg, static_cast<u32>(test.expected.size()));
    test.expected.push_back(expected);
  };
  for (u32 i = 0; i < test.initial.size(); ++i) {
    AppendVMovU32(&code, 30, i * sizeof(u32));
    AppendBufferLoadDword(&code, 12u + i, 30);
  }
  for (u32 selector = 0; selector < 6; ++selector) {
    for (u32 unused = 0; unused < 3; ++unused) {
      AppendVMovLiteral(&code, 26, 0xa1b2c3d4u);
      code.push_back(EncodeVop2(0x0b, 26, 249, 13));
      code.push_back(EncodeVop2Sdwa(12, selector, unused));
      store(26, partial_expected[selector * 3u + unused]);
    }
  }
  // Full result still truncates the 48-bit unsigned product to 32 bits.
  code.push_back(EncodeVop2(0x0b, 26, 249, 14));
  code.push_back(EncodeVop2Sdwa(14));
  store(26, 0xfe000001u);
  // Either multiplicand may alias the partial destination.
  code.push_back(EncodeVop2(0x0b, 12, 249, 13));
  code.push_back(EncodeVop2Sdwa(12, 5, 2));
  store(12, 0xa1800003u);
  code.push_back(EncodeVop2(0x0b, 13, 249, 13));
  code.push_back(EncodeVop2Sdwa(InlineU32(3), 2, 2, 6, 4,
                                0, 0, 0, 0, 0, 0, 1));
  store(13, 0xcd80e080u);
  // Source-byte sign extension happens before the unsigned 24-bit multiply.
  AppendSMovLiteral(&code, 18, 0x12348056u);
  code.push_back(EncodeVop2(0x0b, 26, 249, InlineU32(3)));
  code.push_back(EncodeVop2Sdwa(18, 5, 0, 1, 6, 1, 0, 0, 0, 0, 0, 1, 1));
  store(26, 0xfe800000u);
  for (u32 unused = 0; unused < 3; ++unused) {
    AppendVMovLiteral(&code, 26, 0xa1b2c3d4u);
    code.push_back(EncodeSMovB32(126, InlineU32(0)));
    code.push_back(EncodeVop2(0x0b, 26, 249, 13));
    code.push_back(EncodeVop2Sdwa(12, 4, unused));
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    store(26, 0xa1b2c3d4u);
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_MUL_U32_U24, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_MUL_U32_U24", 25u}};
  test.required_spirv = {"OpIMul", "OpBitFieldUExtract", "OpBitFieldSExtract"};
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase Vop2SdwaLshrrevCapturedByte1Source() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 106, 4);
  AppendBufferLoadDword(&code, 8, 30);
  code.push_back(0x2c1210f9u);
  code.push_back(0x0186066au); // v_lshrrev_b32 v9, vcc_lo, v8.byte1
  AppendStoreVgpr(&code, 9, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop2SdwaLshrrevCapturedByte1Source";
  test.code = std::move(code);
  test.initial = {0x12348678u, 0};
  test.expected = {0x12348678u, 8};
  test.opcodes = {O::S_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHRREV_B32,
                  O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {
      {"V_LSHRREV_B32 v9, vcc_lo, v8.sdwa(sel=1,sext=0)", 1}};
  test.ir_counts = {{" = BitFieldUExtract ", 1},
                    {" = ShiftRightLogical32 ", 1}};
  test.required_spirv = {"OpBitFieldUExtract", "OpShiftRightLogical"};
  return test;
}

TestCase Vop2SdwaSubNcPreservesByteAndWordDestinations() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 0, 3);
  for (u32 dst = 10; dst <= 15; dst++) {
    AppendVMovLiteral(&code, dst, 0xa1b2c3d4u);
  }
  for (u32 sel = 0; sel <= 5; sel++) {
    code.push_back(EncodeVop2(0x26, 10 + sel, 249, 0));
    code.push_back(
        EncodeVop2Sdwa(InlineU32(20), sel, 2, 6, 6, 0, 0, 0, 0, 0, 0, 1));
    AppendStoreVgpr(&code, 10 + sel, sel);
  }
  AppendEnd(&code);

  return {"Vop2SdwaSubNcPreservesByteAndWordDestinations",
          code,
          {},
          {0xa1b2c311u, 0xa1b211d4u, 0xa111c3d4u, 0x11b2c3d4u, 0xa1b20011u,
           0x0011c3d4u},
          {O::V_MOV_B32, O::V_SUB_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase Vop3CvtPkI16I32Captured() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendBufferLoadDword(&code, 7, 30);
  AppendVMovU32(&code, 30, 4);
  AppendBufferLoadDword(&code, 16, 30);
  code.push_back(0xd76b0005u);
  code.push_back(0x00022107u); // v_cvt_pk_i16_i32 v5, v7, v16
  AppendStoreVgpr(&code, 5, 2);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop3CvtPkI16I32Captured";
  test.code = std::move(code);
  test.initial = {0x00018001u, 0xfffe7ffeu, 0};
  test.expected = {0x00018001u, 0xfffe7ffeu, 0x7ffe8001u};
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::V_CVT_PK_I16_I32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"0x00000014: V_CVT_PK_I16_I32 v5, v7, v16\n", 1}};
  test.ir_counts = {{" = BitwiseAnd32 ", 2},
                    {" = ShiftLeftLogical32 ", 1},
                    {" = BitwiseOr32 ", 1}};
  test.required_spirv = {"OpBitwiseAnd", "OpShiftLeftLogical", "OpBitwiseOr"};
  return test;
}

TestCase Vop3MulLoU16CapturedAndSelectors() {
  using O = ShaderOpcode;

  TestCase test;
  test.name = "Vop3MulLoU16CapturedAndSelectors";
  // Distinct runtime halves expose source selection and prevent constant folding.
  test.initial = {0x00138001u, 0x0005ffffu, 0xffffffffu};
  test.expected = {0x00138001u, 0x0005ffffu, 0xffffffffu,
                   0xa5a57fffu, 0xa5a5ffedu, 0xa5a58005u, 0xa5a5005fu,
                   0x7fff1234u, 0xffed1234u, 0x80051234u, 0x005f1234u,
                   0xffff0001u, 0x00231234u};
  auto &code = test.code;
  AppendBufferLoadDword(&code, 12, 30);
  AppendVMovU32(&code, 30, 4);
  AppendBufferLoadDword(&code, 24, 30);
  u32 output = 3;
  for (u32 selectors : {0u, 1u, 2u, 3u, 8u, 9u, 10u, 11u}) {
    AppendVMovLiteral(&code, 2, 0xa5a51234u);
    if (selectors == 8u) {
      code.push_back(0xd7054002u);
      code.push_back(0x0002310cu); // Captured v_mul_lo_u16 v2.hi, v12.lo, v24.lo.
    } else {
      AppendVop3(&code, 0x305, 2, Vgpr(12), Vgpr(24), 0, 0, selectors);
    }
    AppendStoreVgpr(&code, 2, output++);
  }

  // 0xffff * 0xffff wraps to 1; aliasing preserves the original high half.
  AppendVMovU32(&code, 30, 8);
  AppendBufferLoadDword(&code, 12, 30);
  AppendVop3(&code, 0x305, 12, Vgpr(12), Vgpr(12), 0, 0, 1);
  AppendStoreVgpr(&code, 12, output++);

  AppendVMovLiteral(&code, 2, 0xa5a51234u);
  AppendVop3(&code, 0x305, 2, 255, Vgpr(24), 0, 0, 11);
  code.push_back(0x0007deadu); // Select literal high half: 7 * 5 = 35.
  AppendStoreVgpr(&code, 2, output);
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MUL_LO_U16,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_MUL_LO_U16", 10}};
  test.required_spirv = {"OpIMul"};
  return test;
}

TestCase Vop3MadI16CapturedSelectorsAndSaturation() {
  using O = ShaderOpcode;
  struct MadCase {
    u32 lhs, rhs, addend, expected;
    u32 selectors = 0;
    bool clamp = false;
    u32 dst = 1;
  };
  constexpr u32 a = 0x8000fffdu, b = 0xfffe0005u, c = 0xfff90007u;
  const std::array<MadCase, 14> cases{{
      {a, b, c, 0xa5a5fff8u}, // Captured: -3 * 5 + 7 = -8.
      {a, b, c, 0x8007beefu, 9}, // Source 0 high; destination high.
      {a, b, c, 0xa5a5000du, 2}, // Source 1 high: -3 * -2 + 7.
      {a, b, c, 0xa5a5ffeau, 4}, // Source 2 high: -3 * 5 - 7.
      {a, b, c, 0xfff9beefu, 15}, // All high: -32768 * -2 - 7 wraps.
      {a, b, c, 0xfff8fffdu, 8, false, 9}, // Destination aliases source 0.
      {0x7fffu, 1, 1, 0xa5a58000u},
      {0x7fffu, 1, 1, 0x7fffbeefu, 8, true}, // Positive saturation.
      {0x8000u, 1, 0xffffu, 0xa5a57fffu},
      {0x8000u, 1, 0xffffu, 0x8000beefu, 8, true}, // Negative saturation.
      {0x8000u, 0x8000u, 0xffffu, 0xa5a5ffffu},
      {0x8000u, 0x8000u, 0xffffu, 0xa5a57fffu, 0, true},
      {200, 200, 0xd8f0u, 0xa5a57530u, 0, true}, // 40000 - 10000 = 30000.
      {200, 0xff38u, 10000, 0xa5a58ad0u, 0, true}, // -40000 + 10000 = -30000.
  }};
  TestCase test;
  test.name = "Vop3MadI16CapturedSelectorsAndSaturation";
  for (const auto &entry : cases) {
    test.initial.insert(test.initial.end(), {entry.lhs, entry.rhs, entry.addend});
  }
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < cases.size(); ++i) {
    const auto &entry = cases[i];
    u32 offset = i * 12u;
    for (u32 reg : {9u, 6u, 15u}) {
      AppendVMovU32(&code, 30, offset);
      AppendBufferLoadDword(&code, reg, 30); // Runtime inputs prevent folding.
      offset += 4u;
    }
    AppendVMovLiteral(&code, 1, 0xa5a5beefu);
    if (i == 0u) {
      code.insert(code.end(), {0xd75e0001u, 0x043e0d09u});
    } else {
      AppendVop3(&code, 0x35e, entry.dst, Vgpr(9), Vgpr(6), Vgpr(15), 0,
                  entry.selectors, entry.clamp);
    }
    AppendStoreVgpr(&code, entry.dst, static_cast<u32>(test.initial.size()) + i);
    test.expected.push_back(entry.expected);
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_I16,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_MAD_I16", cases.size()}};
  test.required_spirv = {"OpIMul", "OpIAdd"};
  return test;
}

TestCase Vop3Min3U16CapturedAndSelectors() {
  using O = ShaderOpcode;
  struct MinCase {
    u32 lhs, rhs, third, expected;
    u32 selectors = 0;
    u32 dst = 17;
    bool active = true;
  };
  constexpr u32 a = 0x0009ea60u, b = 0x0007c350u, c = 0x00059c40u;
  const std::array<MinCase, 10> cases{{
      {a, b, c, 0x0005beefu, 14}, // Captured: min(60000, 7, 5) into v17.hi.
      {a, b, c, 0xa5a59c40u}, // Unsigned values above INT16_MAX.
      {a, b, c, 0xa5a50009u, 1},
      {a, b, c, 0xa5a50007u, 2},
      {a, b, c, 0xa5a50005u, 4},
      {a, b, c, 0x0005beefu, 15},
      {a, b, c, 0x9c40ea60u, 8, 4}, // Destination aliases source 0.
      {0xabcdffffu, 0xffff0000u, 0x98768000u, 0xa5a50000u},
      {0x0000ffffu, 0x7fffffffu, 0x8000ffffu, 0xa5a5ffffu},
      {a, b, c, 0xa5a5beefu, 14, 17, false},
  }};
  TestCase test;
  test.name = "Vop3Min3U16CapturedAndSelectors";
  for (const auto &entry : cases) {
    test.initial.insert(test.initial.end(), {entry.lhs, entry.rhs, entry.third});
  }
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < cases.size(); ++i) {
    const auto &entry = cases[i];
    u32 offset = i * 12u;
    for (u32 reg : {4u, 0u, 7u}) {
      AppendVMovU32(&code, 30, offset);
      AppendBufferLoadDword(&code, reg, 30); // Runtime inputs prevent folding.
      offset += 4u;
    }
    AppendVMovLiteral(&code, 17, 0xa5a5beefu);
    if (!entry.active) {
      code.push_back(EncodeSop1(0x04, 24, 126)); // Preserve the initial active lanes.
      code.push_back(EncodeSop1(0x04, 126, 128));
    }
    if (i == 0u) {
      code.insert(code.end(), {0xd7537011u, 0x041e0104u});
    } else {
      AppendVop3(&code, 0x353, entry.dst, Vgpr(4), Vgpr(0), Vgpr(7), 0,
                  entry.selectors);
    }
    if (!entry.active) code.push_back(EncodeSop1(0x04, 126, 24));
    AppendStoreVgpr(&code, entry.dst, static_cast<u32>(test.initial.size()) + i);
    test.expected.push_back(entry.expected);
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MIN3_U16,
                  O::S_MOV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_MIN3_U16", cases.size()}};
  test.required_spirv = {" UMin "};
  return test;
}

TestCase Vop3Med3I16Captured() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 5, 0x8ad003e8u);  // high=-30000, low=1000
  AppendSMovLiteral(&code, 25, 0xb1e04e20u); // high=-20000, low=20000
  AppendSMovLiteral(&code, 27, 0x7530fc18u); // high=30000, low=-1000
  AppendVMovLiteral(&code, 3, 0x1234cafeu);
  code.push_back(0xd7584803u);
  code.push_back(0x006c3305u); // v_med3_i16 v3.hi, v5.hi, s25.lo, s27.lo
  AppendStoreVgpr(&code, 3, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop3Med3I16Captured";
  test.code = std::move(code);
  test.expected = {0xfc18cafeu};
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_MED3_I16,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"0x00000020: V_MED3_I16 v3.sdwa(sel=5,sext=0), "
                          "v5.opsel(lo=1,hi=0,neghi=0), s25, s27\n",
                          1}};
  test.required_spirv = {" SMin ", " SMax "};
  return test;
}

TestCase Vop2SdwaMinU32PreservesWordDestination() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 10, 0xa1b2c3d4u);
  AppendVMovU32(&code, 12, 7);
  code.push_back(0x261418f9u);
  code.push_back(0x0686149fu);
  AppendStoreVgpr(&code, 10, 0);
  AppendVMovLiteral(&code, 10, 0xa1b2c3d4u);
  AppendVMovU32(&code, 12, 64);
  code.push_back(0x261418f9u);
  code.push_back(0x0686149fu);
  AppendStoreVgpr(&code, 10, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop2SdwaMinU32PreservesWordDestination";
  test.code = code;
  test.expected = {0xa1b20007u, 0xa1b2001fu};
  test.opcodes = {O::V_MOV_B32, O::V_MIN_U32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.required_spirv = {" UMin ", "OpSelect"};
  return test;
}

TestCase VectorShiftCountsMaskLowBits() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 0, 1);
  AppendVMovU32(&code, 1, 32);
  AppendVMovU32(&code, 2, 33);
  AppendVMovLiteral(&code, 3, 0x80000000u);
  code.push_back(EncodeVop2(0x19, 10, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x1a, 11, Vgpr(2), 0));
  code.push_back(EncodeVop2(0x15, 12, Vgpr(3), 1));
  code.push_back(EncodeVop2(0x16, 13, Vgpr(2), 3));
  code.push_back(EncodeVop2(0x17, 14, Vgpr(3), 1));
  code.push_back(EncodeVop2(0x18, 15, Vgpr(2), 3));

  for (u32 i = 0; i < 6u; i++) {
    AppendStoreVgpr(&code, 10u + i, i);
  }
  AppendEnd(&code);

  return {"VectorShiftCountsMaskLowBits",
          code,
          {},
          {1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u},
          {O::V_MOV_B32, O::V_LSHL_B32, O::V_LSHLREV_B32, O::V_LSHR_B32,
           O::V_LSHRREV_B32, O::V_ASHR_I32, O::V_ASHRREV_I32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorVop3LshrrevB64Captured() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 106, 0x89abcdefu);
  AppendSMovLiteral(&code, 107, 0x01234567u);
  AppendVMovU32(&code, 12, 68);
  code.push_back(0xd7000001u);
  code.push_back(0x0000d50cu); // v_lshrrev_b64 v[1:2], v12, vcc
  AppendStoreVgpr(&code, 1, 0);
  AppendStoreVgpr(&code, 2, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorVop3LshrrevB64Captured";
  test.code = std::move(code);
  test.expected = {0x789abcdeu, 0x00123456u};
  test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::V_LSHRREV_B64,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpShiftRightLogical"};
  return test;
}

TestCase VectorVop3LshlrevB64Captured() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendBufferLoadDword(&code, 3, 30);
  code.push_back(0xd6ff0021u);
  code.push_back(0x00010303u); // v_lshlrev_b64 v[33:34], v3, 1
  AppendStoreVgpr(&code, 33, 1);
  AppendStoreVgpr(&code, 34, 2);
  code.push_back(EncodeVop2(0x03, 3, InlineU32(31), 3));
  code.push_back(0xd6ff0021u);
  code.push_back(0x00010303u);
  AppendStoreVgpr(&code, 33, 3);
  AppendStoreVgpr(&code, 34, 4);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorVop3LshlrevB64Captured";
  test.code = std::move(code);
  test.initial = {33u, 0u, 0u, 0u, 0u};
  test.expected = {33u, 0u, 2u, 1u, 0u};
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64, O::V_ADD_NC_U32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpShiftLeftLogical", "OpTypeInt 64 0"};
  return test;
}

TestCase VectorDynamicU64ShiftEdges() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "VectorDynamicU64ShiftEdges";
  test.initial = {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u, 127u, UINT32_MAX};
  test.expected = test.initial;
  auto& code = test.code;
  for (const uint64_t source : {uint64_t{0}, uint64_t{1},
                               uint64_t{0x8000000000000000ull},
                               uint64_t{0xffffffffffffffffull},
                               uint64_t{0x0123456789abcdefull}}) {
    AppendVMovLiteral(&code, 4, static_cast<u32>(source));
    AppendVMovLiteral(&code, 5, static_cast<u32>(source >> 32u));
    for (u32 index = 0; index < test.initial.size(); ++index) {
      // A buffer load prevents the constant-shift specialization.
      AppendVMovU32(&code, 30, index * sizeof(u32));
      AppendBufferLoadDword(&code, 3, 30);
      for (const bool left : {true, false}) {
        AppendVop3(&code, left ? 0x2ff : 0x300, 33, Vgpr(3), Vgpr(4));
        const auto amount = test.initial[index] & 63u;
        const auto expected = left ? source << amount : source >> amount;
        const auto output = static_cast<u32>(test.expected.size());
        AppendStoreVgpr(&code, 33, output);
        AppendStoreVgpr(&code, 34, output + 1u);
        test.expected.push_back(static_cast<u32>(expected));
        test.expected.push_back(static_cast<u32>(expected >> 32u));
      }
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64,
                  O::V_LSHRREV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpShiftLeftLogical", "OpShiftRightLogical",
                         "OpTypeInt 64 0"};
  return test;
}

TestCase VectorVop3IntegerOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x01, 0, InlineU32(2)));
  code.push_back(EncodeVop1(0x01, 1, InlineU32(3)));
  code.push_back(EncodeVop1(0x01, 2, InlineU32(4)));
  AppendVMovLiteral(&code, 3, 0xfffffff8u);
  code.push_back(EncodeVop1(0x01, 4, InlineU32(5)));
  AppendVMovLiteral(&code, 5, 0x11223344u);
  AppendVMovLiteral(&code, 6, 0x55667788u);
  AppendVMovLiteral(&code, 7, 0x0f0f0f0fu);
  AppendVMovLiteral(&code, 8, 0x33333333u);
  AppendVMovLiteral(&code, 9, 0xaaaaaaaau);
  AppendVMovLiteral(&code, 14, 0x00f00000u);
  code.push_back(EncodeVop1(0x01, 15, InlineU32(8)));
  AppendVMovLiteral(&code, 30, 0x00010000u);
  AppendVMovLiteral(&code, 31, 0x00010000u);

  AppendVop3(&code, 0x36d, 10, Vgpr(0), Vgpr(1), Vgpr(2));
  AppendVop3(&code, 0x152, 11, Vgpr(3), Vgpr(4), Vgpr(0));
  AppendVop3(&code, 0x155, 12, Vgpr(3), Vgpr(4), Vgpr(0));
  AppendVop3(&code, 0x158, 13, Vgpr(3), Vgpr(4), Vgpr(0));
  AppendVop3(&code, 0x153, 16, Vgpr(3), Vgpr(4), Vgpr(0));
  AppendVop3(&code, 0x156, 17, Vgpr(3), Vgpr(4), Vgpr(0));
  AppendVop3(&code, 0x159, 18, Vgpr(3), Vgpr(4), Vgpr(0));
  AppendVop3(&code, 0x15d, 19, Vgpr(0), Vgpr(4), Vgpr(2));
  AppendVop3(&code, 0x346, 20, Vgpr(1), Vgpr(0), Vgpr(2));
  AppendVop3(&code, 0x347, 21, Vgpr(0), Vgpr(1), Vgpr(0));
  AppendVop3(&code, 0x345, 22, Vgpr(7), Vgpr(8), Vgpr(0));
  AppendVop3(&code, 0x36f, 23, Vgpr(1), Vgpr(2), Vgpr(0));
  AppendVop3(&code, 0x371, 24, Vgpr(7), Vgpr(8), Vgpr(2));
  AppendVop3(&code, 0x372, 25, Vgpr(7), Vgpr(8), Vgpr(2));
  AppendVop3(&code, 0x178, 26, Vgpr(7), Vgpr(8), Vgpr(2));
  AppendVop3(&code, 0x148, 27, Vgpr(14), InlineU32(20), Vgpr(2));
  AppendVop3(&code, 0x149, 28, Vgpr(3), Vgpr(2), Vgpr(2));
  AppendVop3(&code, 0x14a, 29, Vgpr(8), Vgpr(7), Vgpr(9));
  AppendVop3(&code, 0x14e, 32, Vgpr(5), Vgpr(6), Vgpr(15));
  AppendVop3(&code, 0x363, 33, Vgpr(2), Vgpr(15));
  AppendVop3(&code, 0x169, 34, Vgpr(30), Vgpr(31));
  AppendVop3(&code, 0x16a, 35, Vgpr(30), Vgpr(31));
  AppendVop3(&code, 0x16b, 36, Vgpr(30), Vgpr(31));
  AppendVop3(&code, 0x16c, 44, Vgpr(3), Vgpr(30));
  AppendVop3(&code, 0x142, 37, Vgpr(0), Vgpr(1), Vgpr(2));
  AppendVop3(&code, 0x143, 38, Vgpr(0), Vgpr(1), Vgpr(2));
  AppendVop3(&code, 0x30f, 39, Vgpr(0), Vgpr(1));
  AppendVop3(&code, 0x310, 40, Vgpr(0), Vgpr(1));
  AppendVop3(&code, 0x319, 41, Vgpr(0), Vgpr(1));

  const u32 results[] = {10, 11, 12, 13, 16, 17, 18, 19, 20, 21,
                         22, 23, 24, 25, 26, 27, 28, 29, 32, 33,
                         34, 35, 36, 44, 37, 38, 39, 40, 41};
  for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
    AppendStoreVgpr(&code, results[i], i);
  }
  AppendEnd(&code);

  return {"VectorVop3IntegerOps",
          code,
          {},
          {9,
           0xfffffff8u,
           5,
           2,
           2,
           0xfffffff8u,
           5,
           7,
           16,
           20,
           0x3c3c3c3eu,
           0x32,
           0x03030307u,
           0x3f3f3f3fu,
           0x3c3c3c38u,
           0x0fu,
           0xffffffffu,
           0x8b8b8b8bu,
           0x44556677u,
           0x00000f00u,
           0,
           1,
           0,
           0xffffffffu,
           10,
           10,
           5,
           0xffffffffu,
           1},
          {O::V_MOV_B32,          O::V_ADD3_U32,     O::V_MIN3_I32,
           O::V_MAX3_I32,         O::V_MED3_I32,     O::V_MIN3_U32,
           O::V_MAX3_U32,         O::V_MED3_U32,     O::V_SAD_U32,
           O::V_LSHL_ADD_U32,     O::V_ADD_LSHL_U32, O::V_XAD_U32,
           O::V_LSHL_OR_B32,      O::V_AND_OR_B32,   O::V_OR3_B32,
           O::V_XOR3_B32,         O::V_BFE_U32,      O::V_BFE_I32,
           O::V_BFI_B32,          O::V_ALIGNBIT_B32, O::V_BFM_B32,
           O::V_MUL_LO_U32,       O::V_MUL_HI_U32,   O::V_MUL_LO_I32,
           O::V_MUL_HI_I32,       O::V_MAD_I32_I24,  O::V_MAD_U32_U24,
           O::V_ADD_I32,          O::V_SUB_I32,      O::V_SUBREV_I32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorBfeI32SignExtendsField() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  for (u32 vgpr = 0; vgpr < 6; vgpr++) {
    AppendVMovU32(&code, 30, vgpr * sizeof(u32));
    AppendBufferLoadDword(&code, vgpr, 30);
  }
  AppendVop3(&code, 0x149, 10, Vgpr(0), Vgpr(2), Vgpr(3));
  AppendVop3(&code, 0x149, 11, Vgpr(1), Vgpr(4), Vgpr(5));
  AppendStoreVgpr(&code, 10, 0);
  AppendStoreVgpr(&code, 11, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorBfeI32SignExtendsField";
  test.code = std::move(code);
  test.initial = {0x80000000u, 0xfffffff8u, 31, 1, 3, 4};
  test.expected = {0xffffffffu, 0xffffffffu};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_BFE_I32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpBitFieldSExtract"};
  return test;
}

TestCase VectorAlignByteUsesTwoBitByteOffset() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x11223344u);
  AppendVMovLiteral(&code, 1, 0x55667788u);
  const std::vector<u32> offsets = {0, 1, 2, 3, 4, 5, 7, 8, 31, 32, 0xffffffffu};
  for (u32 i = 0; i < static_cast<u32>(std::size(offsets)); i++) {
    AppendVMovU32(&code, 2, i * sizeof(u32));
    AppendBufferLoadDword(&code, 2, 2);
    AppendVop3(&code, 0x14f, 10u + i, Vgpr(0), Vgpr(1), Vgpr(2));
    AppendStoreVgpr(&code, 10u + i, i);
  }
  AppendEnd(&code);

  TestCase test{
      "VectorAlignByteUsesTwoBitByteOffset",
      code,
      offsets,
      {0x55667788u, 0x44556677u, 0x33445566u, 0x22334455u, 0x55667788u,
       0x44556677u, 0x22334455u, 0x55667788u, 0x22334455u, 0x55667788u,
       0x22334455u},
      {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_ALIGNBYTE_B32,
       O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.forbidden_spirv = {"OpTypeInt 64"};
  return test;
}

TestCase VectorCarryAndBitCountOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0xffffffffu);
  code.push_back(EncodeVop1(0x01, 1, InlineU32(0)));
  AppendVMovLiteral(&code, 3, 0x0000f0f0u);
  code.push_back(EncodeVop1(0x01, 4, InlineU32(5)));

  code.push_back(EncodeVopc(0xc7, Vgpr(1), 1));
  code.push_back(EncodeVop2(0x28, 2, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x22, 5, Vgpr(3), 4));
  AppendVMovLiteral(&code, 6, 0xffffffffu);
  code.push_back(EncodeVop2(0x23, 7, Vgpr(6), 4));
  code.push_back(EncodeVop2(0x24, 8, Vgpr(6), 4));

  AppendStoreVgpr(&code, 2, 0);
  AppendStoreSgpr(&code, 106, 1);
  AppendStoreVgpr(&code, 5, 2);
  AppendStoreVgpr(&code, 7, 3);
  AppendStoreVgpr(&code, 8, 4);
  AppendEnd(&code);

  return {"VectorCarryAndBitCountOps",
          code,
          {},
          {0, 1, 13, 5, 5},
          {O::V_MOV_B32, O::V_CMP_T_U32, O::V_ADDC_U32, O::V_BCNT_U32_B32,
           O::V_MBCNT_LO_U32_B32, O::V_MBCNT_HI_U32_B32, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase VectorMbcntUsesThreadMask() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVop1(0x01, 1, 0),
      EncodeVop2(0x1a, 1, InlineU32(2), 1),
      EncodeVop2(0x25, 1, Vgpr(0), 1),
      EncodeVop2(0x1a, 3, InlineU32(2), 1),
  };
  AppendVMovLiteral(&code, 2, 0xffffffffu);
  code.push_back(EncodeVop1(0x01, 4, InlineU32(0)));
  code.push_back(EncodeVop2(0x23, 5, Vgpr(2), 4));
  code.push_back(EncodeVop2(0x24, 6, Vgpr(2), 5));
  AppendBufferStoreDword(&code, 6, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorMbcntUsesThreadMask";
  test.code = code;
  test.expected = {0, 1, 2, 3, 0, 1, 2, 3};
  test.opcodes = {O::V_MOV_B32,
                  O::V_LSHLREV_B32,
                  O::V_ADD_NC_U32,
                  O::V_MBCNT_LO_U32_B32,
                  O::V_MBCNT_HI_U32_B32,
                  O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.has_compute_info = true;
  test.dispatch_x = 2;
  return test;
}

TestCase VectorAddcUsesPerLaneCarryIn() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVop1(0x01, 1, 0),
      EncodeVop2(0x1a, 1, InlineU32(2), 1),
      EncodeVop2(0x25, 1, Vgpr(0), 1),
      EncodeVopc(0xc7, Vgpr(0), 0),
      EncodeVop1(0x01, 2, InlineU32(0)),
      EncodeVop2(0x28, 3, Vgpr(2), 2),
      EncodeVop2(0x1a, 4, InlineU32(2), 1),
  };
  AppendBufferStoreDword(&code, 3, 4);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorAddcUsesPerLaneCarryIn";
  test.code = code;
  test.expected = std::vector<u32>(8, 1);
  test.opcodes = {O::V_MOV_B32,   O::V_LSHLREV_B32, O::V_ADD_NC_U32,
                  O::V_CMP_T_U32, O::V_ADDC_U32,    O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.has_compute_info = true;
  test.dispatch_x = 2;
  return test;
}

TestCase VectorAddcWritesPerLaneCarryOut() {
  using O = ShaderOpcode;

  std::vector<u32> code = {EncodeVop1(0x01, 1, Vgpr(0)),
                           EncodeVop2(0x1a, 4, InlineU32(2), 1),
                           EncodeVopc(0xc7, Vgpr(0), 0)};
  AppendBufferLoadDword(&code, 26, 4);
  code.push_back(0x500434f9u);
  code.push_back(0x0c860680u);
  code.push_back(EncodeVop1(0x01, 6, 106));
  AppendBufferStoreDword(&code, 6, 4);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorAddcWritesPerLaneCarryOut";
  test.code = code;
  test.initial = {0x0000ffffu, 0u, 0x0000ffffu, 0u};
  test.expected = {5u, 5u, 5u, 5u};
  test.opcodes = {O::V_MOV_B32,         O::V_LSHLREV_B32, O::V_CMP_T_U32,
                  O::BUFFER_LOAD_DWORD, O::V_ADDC_U32,    O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.decoded_counts = {
      {"V_ADDC_U32 v2, vcc_lo, 0, v26.sdwa(sel=4,sext=1), vcc_lo", 1}};
  test.ir_counts = {{" = BitFieldSExtract ", 1}, {" = IAddCarry32 ", 2}};
  test.required_spirv = {"OpBitFieldSExtract"};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorSubCoCiU32CompactAndVop3() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
  AppendBufferLoadDword(&code, 1, 4);
  code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4));
  AppendBufferLoadDword(&code, 2, 5);
  code.push_back(EncodeVop2(0x1b, 3, InlineU32(1), 0));
  code.push_back(EncodeVopc(0xc5, InlineU32(0), 3));
  code.push_back(EncodeSop1(0x04, 20, 106));
  code.push_back(EncodeVop2(0x29, 10, Vgpr(1), 2));
  AppendVMovU32(&code, 6, 1);
  code.push_back(EncodeVop2(0x01, 11, InlineU32(0), 6));
  AppendStoreVgprAtLaneDwordOffset(&code, 10, 0, 0);
  AppendStoreVgprAtLaneDwordOffset(&code, 11, 0, 4);

  code.push_back(EncodeVopc(0xc2, InlineU32(0), 6));
  AppendVop3B(&code, 0x129u, 12, 22, Vgpr(1), Vgpr(2), 20);
  AppendVop3(&code, 0x101u, 13, InlineU32(0), Vgpr(6), 22);
  code.push_back(EncodeVop2(0x01, 14, InlineU32(0), 6));
  AppendStoreVgprAtLaneDwordOffset(&code, 12, 0, 8);
  AppendStoreVgprAtLaneDwordOffset(&code, 13, 0, 12);
  AppendStoreVgprAtLaneDwordOffset(&code, 14, 0, 16);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorSubCoCiU32CompactAndVop3";
  test.code = std::move(code);
  test.initial = {5u, 5u, 3u, 3u, 3u, 3u, 5u, 3u, 0u, 0u,
                  0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u};
  test.expected = {2u, 1u, 0xfffffffeu, 0xffffffffu, 0u, 0u, 1u,
                   1u, 2u, 1u, 0xfffffffeu, 0xffffffffu, 0u, 0u,
                   1u, 1u, 0u, 0u, 0u, 0u};
  test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
                  O::V_AND_B32, O::V_CMP_NE_U32, O::V_CMP_EQ_U32,
                  O::V_SUB_CO_CI_U32, O::V_CNDMASK_B32, O::S_MOV_B64,
                  O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_SUB_CO_CI_U32", 2}};
  test.required_spirv = {"OpISub", "OpUGreaterThan", "OpLogicalOr"};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = 32;
  test.has_compute_info = true;
  return test;
}

TestCase VectorSubrevCoCiU32ExactRawOnGpu() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x01, 24, Vgpr(0)));
  code.push_back(EncodeSop1(0x04, 20, 126)); // s_mov_b64 s[20:21], exec
  code.push_back(EncodeVop1(0x01, 1, InlineU32(2)));
  code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
  code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
  code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4));
  code.push_back(EncodeVopc(0xc1, Vgpr(0), 1));
  code.push_back(EncodeSop1(0x04, 22, 106)); // s_mov_b64 s[22:23], vcc
  code.push_back(EncodeVopc(0xc5, InlineU32(0), 0));
  AppendVMovLiteral(&code, 0, 0xdeadbeefu);
  code.push_back(EncodeSop1(0x04, 126, 106)); // s_mov_b64 exec, vcc
  code.push_back(EncodeSop1(0x04, 106, 22));  // s_mov_b64 vcc, s[22:23]
  code.push_back(0x54003080u); // v_subrev_co_ci_u32 v0, vcc, 0, v24, vcc
  code.push_back(EncodeSop1(0x04, 126, 20)); // s_mov_b64 exec, s[20:21]
  code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 2));
  AppendBufferStoreDword(&code, 0, 4);
  AppendBufferStoreDword(&code, 3, 5);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorSubrevCoCiU32ExactRawOnGpu";
  test.code = code;
  test.expected = {0xdeadbeefu, 0, 2, 3, 0, 0, 0, 0};
  test.opcodes = {O::V_MOV_B32,          O::V_LSHLREV_B32,
                  O::V_ADD_NC_U32,       O::V_CMP_LT_U32,
                  O::S_MOV_B64,          O::V_CMP_NE_U32,
                  O::V_SUBREV_CO_CI_U32, O::V_CNDMASK_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpISub", "OpUGreaterThan"};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.wave_size = 32;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.has_compute_info = true;
  return test;
}

TestCase VectorVop3BSubrevCoCiUsesEncodedMasks() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x01, 24, Vgpr(0)));
  code.push_back(EncodeVop2(0x25, 24, InlineU32(3), 24));
  code.push_back(EncodeVop1(0x01, 1, InlineU32(2)));
  code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
  code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
  code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4));
  code.push_back(EncodeVopc(0xc2, InlineU32(1), 0));
  code.push_back(EncodeSop1(0x04, 20, 106)); // s_mov_b64 s[20:21], vcc
  AppendSMovLiteral(&code, 106, 0);
  AppendSMovLiteral(&code, 107, 0);
  AppendVop3B(&code, 0x12au, 10, 22, InlineU32(5), Vgpr(24), 20);
  AppendVop3(&code, 0x101u, 3, InlineU32(0), Vgpr(2), 22);
  AppendBufferStoreDword(&code, 10, 4);
  AppendBufferStoreDword(&code, 3, 5);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorVop3BSubrevCoCiUsesEncodedMasks";
  test.code = code;
  test.expected = {0xfffffffeu, 0xfffffffeu, 0, 1, 1, 1, 0, 0};
  test.opcodes = {O::V_MOV_B32,          O::V_LSHLREV_B32,
                  O::V_ADD_NC_U32,       O::V_CMP_EQ_U32,
                  O::S_MOV_B64,          O::S_MOV_B32,
                  O::V_SUBREV_CO_CI_U32, O::V_CNDMASK_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.wave_size = 32;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.has_compute_info = true;
  return test;
}

TestCase VectorVop3BCarryOutWritesSgprMask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 1, 0xffffffffu);
  AppendVMovU32(&code, 2, 1);
  AppendVMovU32(&code, 3, 0);

  AppendVop3B(&code, 0x30fu, 10, 0, Vgpr(1), Vgpr(2));
  AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 0);
  AppendVop3B(&code, 0x310u, 11, 0, Vgpr(3), Vgpr(2));
  AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 4);
  AppendVop3B(&code, 0x319u, 12, 0, Vgpr(2), Vgpr(3));
  AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 8);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorVop3BCarryOutWritesSgprMask";
  test.code = code;
  test.expected = std::vector<u32>(12, 15u);
  test.opcodes = {O::V_MOV_B32,    O::V_ADD_I32,          O::V_SUB_I32,
                  O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.has_compute_info = true;
  return test;
}

TestCase VectorVop3BCarryOutUsesEncodedSdst() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 1, 0xffffffffu);
  AppendVMovU32(&code, 2, 1);
  AppendVMovU32(&code, 3, 0);

  AppendVop3B(&code, 0x30fu, 10, 20, Vgpr(1), Vgpr(2));
  AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 0);
  AppendVop3B(&code, 0x310u, 11, 22, Vgpr(3), Vgpr(2));
  AppendStoreSgprAtLaneDwordOffset(&code, 22, 0, 4);
  AppendVop3B(&code, 0x319u, 12, 24, Vgpr(2), Vgpr(3));
  AppendStoreSgprAtLaneDwordOffset(&code, 24, 0, 8);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorVop3BCarryOutUsesEncodedSdst";
  test.code = code;
  test.expected = std::vector<u32>(12, 15u);
  test.opcodes = {O::V_MOV_B32,    O::V_ADD_I32,          O::V_SUB_I32,
                  O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.has_compute_info = true;
  return test;
}

TestCase VectorVop3BSubCoU32UsesRdna2Opcode310() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovU32(&code, 2, 1);
  AppendVMovU32(&code, 3, 1);
  AppendVMovLiteral(&code, 4, 0xffffffffu);
  AppendVMovLiteral(&code, 5, 0x80000000u);

  code.push_back(EncodeVopc(0xc2, InlineU32(1), 0));
  code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(3)));
  code.push_back(EncodeVopc(0xc2, InlineU32(2), 0));
  code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(4)));
  code.push_back(EncodeVopc(0xc2, InlineU32(3), 0));
  code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(5)));
  code.push_back(EncodeVop2(0x01, 2, Vgpr(2), Vgpr(4)));

  AppendVop3B(&code, 0x310u, 10, 20, Vgpr(1), Vgpr(2));
  AppendStoreVgprAtLaneDwordOffset(&code, 10, 0, 0);
  AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 4);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorVop3BSubCoU32UsesRdna2Opcode310";
  test.code = code;
  test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 9, 9, 9, 9};
  test.opcodes = {O::V_MOV_B32, O::V_CMP_EQ_U32,       O::V_CNDMASK_B32,
                  O::V_SUB_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.has_compute_info = true;
  return test;
}

TestCase VectorMadU64U32UnsignedCarryOut() {
  using O = ShaderOpcode;
  struct Input {
    u32 lhs;
    u32 rhs;
    uint64_t addend;
  };
  constexpr Input inputs[] = {
      {0, UINT32_MAX, UINT64_MAX},
      {UINT32_MAX, 2, 0xffffffff00000002ull},
      {UINT32_MAX, UINT32_MAX, UINT64_MAX},
      {0x80000000u, 0x80000000u, 0x4000000000000000ull},
      {3, 5, 7},
  };
  TestCase test;
  test.name = "VectorMadU64U32UnsignedCarryOut";
  for (const auto &input : inputs) {
    test.initial.insert(
        test.initial.end(),
        {input.lhs, input.rhs, u32(input.addend), u32(input.addend >> 32)});
  }
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < std::size(inputs); ++i) {
    for (u32 word = 0; word < 4; ++word) {
      AppendVMovU32(&code, 30, (i * 4 + word) * sizeof(u32));
      AppendBufferLoadDword(&code, word < 2 ? word + 1 : word + 2, 30);
    }
    AppendVop3B(&code, 0x176u, 10, 20, Vgpr(1), Vgpr(2), Vgpr(4));
    const auto output = static_cast<u32>(test.expected.size());
    AppendStoreVgpr(&code, 10, output);
    AppendStoreVgpr(&code, 11, output + 1);
    AppendStoreSgprPair(&code, 20, output + 2);
    const auto product = uint64_t(inputs[i].lhs) * inputs[i].rhs;
    const auto sum = product + inputs[i].addend;
    test.expected.insert(test.expected.end(),
                         {u32(sum), u32(sum >> 32), u32(sum < product), 0});
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_U64_U32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  // NVIDIA rejected the native widening multiply in a large Astro's Playroom
  // vertex shader. Keep its 32x32 product portable and its 64-bit addition
  // native.
  test.required_spirv = {"OpTypeInt 64 0", "OpUMulExtended", "OpIMul %uint",
                         "OpIAdd %ulong", "OpULessThan"};
  test.forbidden_spirv = {"OpIMul %ulong", "OpIMul %long"};
  return test;
}

TestCase VectorLaneAndPackedOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x12345678u);
  code.push_back(EncodeVop1(0x01, 1, InlineU32(0)));
  AppendVMovLiteral(&code, 4, 0x40003c00u);
  AppendVMovLiteral(&code, 5, 0x44004200u);
  AppendVMovLiteral(&code, 6, 0x3c003c00u);
  AppendVMovLiteral(&code, 7, 0x3f800000u);
  AppendVMovLiteral(&code, 8, 0x40000000u);
  AppendVMovLiteral(&code, 9, 0x12345678u);
  AppendVMovLiteral(&code, 10, 0xabcdef01u);

  code.push_back(EncodeVop1(0x02, 0, Vgpr(0)));
  AppendVop3(&code, 0x360, 1, Vgpr(0), InlineU32(0));
  AppendVop3(&code, 0x361, 1, 0, InlineU32(0));
  AppendVop3(&code, 0x377, 2, Vgpr(0), InlineU32(0), InlineU32(0));
  code.push_back(EncodeVop2(0x2f, 3, Vgpr(7), 8));
  AppendVop3p(&code, 0x0f, 11, Vgpr(4), Vgpr(5), 0, 0x3);
  AppendVop3p(&code, 0x10, 12, Vgpr(4), Vgpr(5), 0, 0x3);
  AppendVop3p(&code, 0x11, 13, Vgpr(4), Vgpr(5), 0, 0x3);
  AppendVop3p(&code, 0x12, 14, Vgpr(4), Vgpr(5), 0, 0x3);
  AppendVop3p(&code, 0x0e, 15, Vgpr(4), Vgpr(5), Vgpr(6), 0x7);
  AppendVop3(&code, 0x362, 16, Vgpr(7), InlineU32(1));
  AppendVop3(&code, 0x36a, 17, Vgpr(9), Vgpr(10));

  AppendStoreSgpr(&code, 0, 0);
  AppendStoreSgpr(&code, 1, 1);
  const u32 results[] = {1, 2, 3, 11, 12, 13, 14, 15, 16, 17};
  for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
    AppendStoreVgpr(&code, results[i], i + 2u);
  }
  AppendEnd(&code);

  return {"VectorLaneAndPackedOps",
          code,
          {},
          {0x12345678u, 0x12345678u, 0x12345678u, 0x12345678u, 0x40003c00u,
           0x46004400u, 0x48004200u, 0x40003c00u, 0x44004200u, 0x48804400u,
           0x40000000u, 0xef015678u},
          {O::V_MOV_B32, O::V_READFIRSTLANE_B32, O::V_READLANE_B32,
           O::V_WRITELANE_B32, O::V_PERMLANE16_B32, O::V_CVT_PKRTZ_F16_F32,
           O::V_PK_ADD_F16, O::V_PK_MUL_F16, O::V_PK_MIN_F16, O::V_PK_MAX_F16,
           O::V_PK_FMA_F16, O::V_LDEXP_F32, O::V_CVT_PK_U16_U32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase Vop2PkFmacF16AccumulatesPackedHalvesIndependently() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x50004c00u); // low=16.0h, high=32.0h
  AppendVMovLiteral(&code, 1, 0x42003c00u); // low=1.0h, high=3.0h
  AppendVMovLiteral(&code, 2, 0x48004400u); // low=4.0h, high=8.0h

  code.push_back(EncodeVop2(0x3c, 0, Vgpr(1), 2));
  AppendStoreVgpr(&code, 0, 0);
  code.push_back(EncodeVop2(0x3c, 0, Vgpr(1), 2));
  AppendStoreVgpr(&code, 0, 1);

  AppendVMovLiteral(&code, 3, 0x50004c00u);
  code.push_back(EncodeVop2(0x3c, 3, 242, 2)); // Inline 1.0h is {1.0h, 0.0h}.
  AppendStoreVgpr(&code, 3, 2);
  AppendEnd(&code);

  return {"Vop2PkFmacF16AccumulatesPackedHalvesIndependently",
          code,
          {},
          {0x53004d00u, 0x55004e00u, 0x50004d00u},
          {O::V_MOV_B32, O::V_PK_FMAC_F16, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase Vop2PkFmacF16DppNegatesBothPackedHalves() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x50004c00u); // low=16.0h, high=32.0h
  AppendVMovLiteral(&code, 1, 0x42003c00u); // low=1.0h, high=3.0h
  AppendVMovLiteral(&code, 2, 0x48004400u); // low=4.0h, high=8.0h

  code.push_back(EncodeVop2(0x3c, 0, 250, 2));
  code.push_back(EncodeVop2Dpp(1, 0, 0xf, 0xf, 1));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  return {"Vop2PkFmacF16DppNegatesBothPackedHalves",
          code,
          {},
          {0x48004a00u},
          {O::V_MOV_B32, O::V_PK_FMAC_F16, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase Vop3pOpselHiUsesArchitecturalSourceBits() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x42003c00u); // low=1.0h, high=3.0h
  AppendVMovLiteral(&code, 1, 0x48004400u); // low=4.0h, high=8.0h
  AppendVMovLiteral(&code, 2, 0x50004c00u); // low=16.0h, high=32.0h

  // Raw V_PK_FMA_F16 words keep this test independent of the encoders above.
  code.insert(code.end(), {0xcc0e000au, 0x0c0a0300u}); // src0 high: 3*4+16=28
  code.insert(code.end(), {0xcc0e000bu, 0x140a0300u}); // src1 high: 1*8+16=24
  code.insert(code.end(), {0xcc0e400cu, 0x040a0300u}); // src2 high: 1*4+32=36

  for (u32 i = 0; i < 3; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  return {"Vop3pOpselHiUsesArchitecturalSourceBits",
          code,
          {},
          {0x4f004d00u, 0x4e004d00u, 0x50804d00u},
          {O::V_MOV_B32, O::V_PK_FMA_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase Vop3pIntegerNegationCapturedAndSelectedHalves() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 6, 0xfed4ff38u); // (-200, -300)
  AppendVMovLiteral(&code, 3, 0x012c00c8u); // (200, 300)
  // Captured temporal shader: NEG_LO/HI turns 0x7fff into 0xffff (-1).
  // Adding that clamped offset must select (199,299), not collapse to (0,0).
  code.insert(code.end(), {0xcc074107u, 0x30020cffu, 0x00007fffu});
  code.insert(code.end(), {0xcc0a4010u, 0x18020f03u});
  AppendStoreVgpr(&code, 7, 0);
  AppendStoreVgpr(&code, 16, 1);

  AppendVMovLiteral(&code, 0, 0x80011234u);
  AppendVMovLiteral(&code, 1, 0xbeef5678u);
  // Independently negate src0 low and src1 high, then reverse both the
  // modified sources and src1 half selection. NEG flips the selected sign bit.
  AppendVop3p(&code, 0x0a, 10, Vgpr(0), Vgpr(1), 0, 0x3, 0, 0x2, 0x1);
  AppendVop3p(&code, 0x0a, 11, Vgpr(0), Vgpr(1), 0, 0x1, 0x2, 0x1, 0x2);
  AppendStoreVgpr(&code, 10, 2);
  AppendStoreVgpr(&code, 11, 3);
  AppendEnd(&code);

  return {"Vop3pIntegerNegationCapturedAndSelectedHalves",
          code,
          {},
          {0xffffffffu, 0x012b00c7u, 0xbef0e8acu, 0x56795123u},
          {O::V_MOV_B32, O::V_PK_MAX_I16, O::V_PK_ADD_U16,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase Vop3pInt16Ops() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 1, 0x80007fffu);
  AppendVMovLiteral(&code, 2, 0x00010001u);
  AppendVMovLiteral(&code, 3, 0xfffe0003u);
  AppendVMovLiteral(&code, 4, 0x00130004u);
  AppendVMovLiteral(&code, 5, 0x8010f00fu);
  AppendVMovLiteral(&code, 6, 0x00050007u);

  AppendVop3p(&code, 0x00u, 10, Vgpr(1), Vgpr(3), Vgpr(6), 0x7u);
  AppendVop3p(&code, 0x09u, 11, Vgpr(1), Vgpr(3), Vgpr(6), 0x7u);
  AppendVop3p(&code, 0x01u, 12, Vgpr(1), Vgpr(3), 0, 0x3u);
  AppendVop3p(&code, 0x02u, 13, Vgpr(1), Vgpr(2), 0, 0x3u);
  AppendVop3p(&code, 0x0bu, 14, Vgpr(2), Vgpr(3), 0, 0x3u);
  AppendVop3p(&code, 0x04u, 15, Vgpr(4), Vgpr(5), 0, 0x3u);
  AppendVop3p(&code, 0x05u, 16, Vgpr(4), Vgpr(5), 0, 0x3u);
  AppendVop3p(&code, 0x06u, 17, Vgpr(4), Vgpr(5), 0, 0x3u);
  AppendVop3p(&code, 0x08u, 18, Vgpr(1), Vgpr(2), 0, 0x3u);
  AppendVop3p(&code, 0x0cu, 19, Vgpr(1), Vgpr(2), 0, 0x3u);
  AppendVop3p(&code, 0x0du, 20, Vgpr(1), Vgpr(2), 0, 0x3u);

  for (u32 i = 0; i < 11; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop3pInt16Ops";
  test.code = std::move(code);
  test.expected = {0x00058004u, 0x00058004u, 0x00007ffdu, 0x80018000u,
                   0x0003fffeu, 0x008000f0u, 0x10020f00u, 0xf002ff00u,
                   0x80000001u, 0x80007fffu, 0x00010001u};
  test.opcodes = {O::V_MOV_B32,        O::V_PK_MAD_I16,     O::V_PK_MAD_U16,
                  O::V_PK_MUL_LO_U16,  O::V_PK_ADD_I16,     O::V_PK_SUB_U16,
                  O::V_PK_LSHLREV_B16, O::V_PK_LSHRREV_B16, O::V_PK_ASHRREV_I16,
                  O::V_PK_MIN_I16,     O::V_PK_MAX_U16,     O::V_PK_MIN_U16,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase CvtPkU8F32PacksSelectedByte() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  // RDNA2 V_CVT_PK_U8_F32: uint8(S0) replaces byte S1[1:0] in S2.
  AppendVMovLiteral(&code, 0, 0x414c0000u); // 12.75f truncates to 12.
  AppendVMovLiteral(&code, 1, 0x43960000u); // 300.0f saturates to 255.
  AppendVMovLiteral(&code, 2, 0xbf800000u); // Negative values saturate to zero.
  AppendVMovLiteral(&code, 3, 0x7fc00000u); // NaN saturates to zero.
  AppendVMovLiteral(&code, 4, 0x11223344u);

  AppendVop3(&code, 0x15eu, 10, Vgpr(0), InlineU32(0), Vgpr(4));
  AppendVop3(&code, 0x15eu, 11, Vgpr(1), InlineU32(5), Vgpr(4));
  AppendVop3(&code, 0x15eu, 12, Vgpr(2), InlineU32(2), Vgpr(4));
  AppendVop3(&code, 0x15eu, 13, Vgpr(3), InlineU32(3), Vgpr(4));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  return {
      "CvtPkU8F32PacksSelectedByte",
      code,
      {},
      {0x1122330cu, 0x1122ff44u, 0x11003344u, 0x00223344u},
      {O::V_MOV_B32, O::V_CVT_PK_U8_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase CvtPkrtzF16F32SubnormalRoundsTowardZero() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x33c00000u);
  AppendVMovLiteral(&code, 1, 0xb3c00000u);
  code.push_back(EncodeVop2(0x2f, 2, Vgpr(0), 1));
  AppendStoreVgpr(&code, 2, 0);
  AppendEnd(&code);

  return {"CvtPkrtzF16F32SubnormalRoundsTowardZero",
          code,
          {},
          {0x80010001u},
          {O::V_MOV_B32, O::V_CVT_PKRTZ_F16_F32, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase CvtPkrtzF16F32SdwaAndOutputModifiers() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x3f000000u); // 0.5f
  AppendVMovLiteral(&code, 1, 0x40000000u); // 2.0f
  AppendVMovLiteral(&code, 2, 0x3f802000u); // exactly 0x3c01h
  AppendVMovLiteral(&code, 3, 0x00000000u);
  AppendVMovLiteral(&code, 4, 0xbf000000u); // -0.5f
  for (u32 dst = 10; dst <= 18; dst++) {
    AppendVMovLiteral(&code, dst, 0xa1b2c3d4u);
  }

  code.push_back(EncodeVop2(0x2f, 10, 249, 1));
  code.push_back(EncodeVop2Sdwa(0, 6, 2, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0,
                                1));
  code.push_back(EncodeVop2(0x2f, 11, 249, 3));
  code.push_back(EncodeVop2Sdwa(2, 0, 0));
  code.push_back(EncodeVop2(0x2f, 12, 249, 1));
  code.push_back(EncodeVop2Sdwa(0, 6, 0, 3, 5));
  code.push_back(EncodeVop2(0x2f, 13, 249, 1));
  code.push_back(EncodeVop2Sdwa(4, 6, 0, 6, 6, 0, 0, 0, 1, 1));

  code.push_back(EncodeVop2(0x2f, 14, 250, 1));
  code.push_back(EncodeVop2Dpp(0, 0, 0xf, 0xf, 1));
  AppendVop3(&code, 0x12f, 15, Vgpr(0), Vgpr(1), 0, 0, 0, true);
  code.push_back(EncodeVop2(0x2f, 16, 249, 1));
  code.push_back(EncodeVop2Sdwa(0, 4, 2));
  code.push_back(EncodeVop2(0x2f, 17, 249, 1));
  code.push_back(EncodeVop2Sdwa(0, 6, 2, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0,
                                0, 1));
  AppendVop3(&code, 0x12f, 18, Vgpr(0), Vgpr(1), 0, 0, 0, false, 1);

  for (u32 dst = 10; dst <= 18; dst++) {
    AppendStoreVgpr(&code, dst, dst - 10);
  }
  AppendEnd(&code);

  return {"CvtPkrtzF16F32SdwaAndOutputModifiers",
          code,
          {},
          {0x3c003800u, 0x00000001u, 0x00000000u, 0xc0003800u,
           0x4000b800u, 0x3c003800u, 0xa1b23800u, 0x44003c00u,
           0x44003c00u},
          {O::V_MOV_B32, O::V_CVT_PKRTZ_F16_F32, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase PackedMinMaxF16NanAndSignedZeroEdges() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x80004000u);
  AppendVMovLiteral(&code, 1, 0x00007e00u);
  AppendVMovLiteral(&code, 2, 0x00004000u);
  AppendVMovLiteral(&code, 3, 0x80007e00u);
  AppendVMovLiteral(&code, 4, 0x40007d01u);
  AppendVMovLiteral(&code, 5, 0x7d014000u);

  AppendVop3p(&code, 0x11, 10, Vgpr(0), Vgpr(1), 0, 0x3);
  AppendVop3p(&code, 0x12, 11, Vgpr(2), Vgpr(3), 0, 0x3);
  AppendVop3p(&code, 0x11, 12, Vgpr(4), Vgpr(5), 0, 0x3);
  AppendVop3p(&code, 0x12, 13, Vgpr(4), Vgpr(5), 0, 0x3);

  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  return {"PackedMinMaxF16NanAndSignedZeroEdges",
          code,
          {},
          {0x80004000u, 0x00004000u, 0x7f017f01u, 0x7f017f01u},
          {O::V_MOV_B32, O::V_PK_MIN_F16, O::V_PK_MAX_F16,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorMinMaxF16Ops() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0xaaaa4000u); // low=2.0h
  AppendVMovLiteral(&code, 1, 0xbbbb3c00u); // low=1.0h
  AppendVMovLiteral(&code, 2, 0x12345678u);
  AppendVMovLiteral(&code, 3, 0x87654321u);
  AppendVMovLiteral(&code, 4, 0x5555aaaau);
  AppendVMovLiteral(&code, 5, 0x6666bbbbu);
  AppendVMovLiteral(&code, 6, 0x7777bbbbu);
  AppendVMovLiteral(&code, 7, 0x8888bbbbu);
  AppendVMovLiteral(&code, 8, 0x9999bbbbu);
  AppendVMovLiteral(&code, 9, 0x99990000u);
  code.push_back(EncodeVop2(0x32, 5, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x33, 6, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x34, 7, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x35, 4, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x39, 2, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x3a, 3, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x35, 9, 249, 1));
  code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4));
  code.push_back(0x660000f9u);
  code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4));
  AppendStoreVgpr(&code, 5, 0);
  AppendStoreVgpr(&code, 6, 1);
  AppendStoreVgpr(&code, 7, 2);
  AppendStoreVgpr(&code, 0, 3);
  AppendStoreVgpr(&code, 4, 4);
  AppendStoreVgpr(&code, 2, 5);
  AppendStoreVgpr(&code, 3, 6);
  AppendStoreVgpr(&code, 9, 7);
  AppendEnd(&code);

  return {"VectorMinMaxF16Ops",
          code,
          {},
          {0x66664200u, 0x77773c00u, 0x8888bc00u, 0xaaaa0000u, 0x55554000u,
           0x12344000u, 0x87653c00u, 0x99994000u},
          {O::V_MOV_B32, O::V_ADD_F16, O::V_SUB_F16, O::V_SUBREV_F16,
           O::V_MUL_F16, O::V_MAX_F16, O::V_MIN_F16, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase VectorCvtF16U16ByteSource() {
  using O = ShaderOpcode;

  constexpr std::array inputs = {0xff123456u, 0x80123456u, 0x00123456u};
  std::vector<u32> code;
  AppendVMovLiteral(&code, 16, 0xa5a55a5au);
  for (u32 i = 0; i < inputs.size(); i++) {
    AppendVMovU32(&code, 30, i * 4u);
    AppendBufferLoadDword(&code, 18, 30);
    // Convert unsigned byte3 to FP16 word0 while preserving word1 of v16.
    code.insert(code.end(), {0x7e20a0f9u, 0x00031412u});
    AppendStoreVgpr(&code, 16, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorCvtF16U16ByteSource";
  test.code = std::move(code);
  test.initial.assign(inputs.begin(), inputs.end());
  test.expected = {0xa5a55bf8u, 0xa5a55800u, 0xa5a50000u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_CVT_F16_U16,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpBitFieldUExtract", "OpConvertUToF"};
  return test;
}

TestCase VectorCvtU16F16Sdwa() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x45004200u); // low=3.0h, high=5.0h
  AppendVMovLiteral(&code, 1, 0xbc007c00u); // low=+inf, high=-1.0h
  AppendVMovLiteral(&code, 2, 0x00007e00u); // low=qNaN
  AppendVMovLiteral(&code, 10, 0x12345678u);
  AppendVMovLiteral(&code, 11, 0x87654321u);
  AppendVMovLiteral(&code, 12, 0xabcd1111u);
  AppendVMovLiteral(&code, 13, 0x77772222u);
  AppendVMovLiteral(&code, 14, 0x5555aaaau);
  code.push_back(EncodeVop1(0x52, 10, 249));
  code.push_back(EncodeVop1Sdwa(0, 4, 2, 4));
  code.push_back(EncodeVop1(0x52, 11, 249));
  code.push_back(EncodeVop1Sdwa(0, 5, 2, 5));
  code.push_back(EncodeVop1(0x52, 12, 249));
  code.push_back(EncodeVop1Sdwa(1, 4, 2, 4));
  code.push_back(EncodeVop1(0x52, 13, 249));
  code.push_back(EncodeVop1Sdwa(1, 4, 2, 5));
  code.push_back(EncodeVop1(0x52, 14, 249));
  code.push_back(EncodeVop1Sdwa(2, 4, 2, 4));
  for (u32 i = 0; i < 5; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  return {"VectorCvtU16F16Sdwa",
          code,
          {},
          {0x12340003u, 0x00054321u, 0xabcdffffu, 0x77770000u, 0x55550000u},
          {O::V_MOV_B32, O::V_CVT_U16_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase NativeAndSdwa16BitDestinationWrites() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x3fc00000u); // 1.5f -> 0x3e00h
  AppendVMovLiteral(&code, 1, 0x00004200u); // low=3.0h
  AppendVMovLiteral(&code, 2, 0x0000c000u); // low=-2.0h
  AppendVMovLiteral(&code, 3, 0x00003c00u); // low=1.0h
  AppendVMovLiteral(&code, 4, 0x00004000u); // low=2.0h

  const u32 sentinels[] = {
      0x3c001234u, 0xbbbb2345u, 0xcccc3456u, 0xdddd4567u, 0xeeee5678u,
      0xffff6789u, 0xf00d789au, 0x11118888u, 0x22229999u, 0x3333aaaau,
      0xaaaa0000u, 0xbbbb0000u, 0xcccc0000u, 0xabcd1234u, 0xbcde2345u,
      0xcdef3456u, 0xdef01234u, 0xef012345u, 0xf0123456u, 0x01234567u,
      0xaaaa1234u, 0xbbbb5678u, 0x24681234u, 0x13572468u, 0x89abcdefu,
      0xfedcba98u, 0x0bad1234u,
  };
  // AppendStoreVgpr uses v31 as its address scratch register.
  const auto output_vgpr = [](u32 index) { return 10u + index + (index >= 21u ? 1u : 0u); };
  for (u32 i = 0; i < static_cast<u32>(std::size(sentinels)); i++) {
    AppendVMovLiteral(&code, output_vgpr(i), sentinels[i]);
  }

  code.push_back(EncodeVop1(0x0a, 10, Vgpr(0)));
  code.push_back(EncodeVop1(0x0a, 11, 250));
  code.push_back(EncodeVop1Dpp(0));
  AppendVop3(&code, 0x18a, 12, Vgpr(0), 0);
  code.push_back(EncodeVop1(0x52, 13, Vgpr(1)));
  code.push_back(EncodeVop1(0x53, 14, Vgpr(2)));

  code.push_back(EncodeVop1(0x0a, 15, 249));
  code.push_back(EncodeVop1Sdwa(0));
  code.push_back(EncodeVop1(0x54, 16, 249));
  code.push_back(EncodeVop1Sdwa(10, 4, 2, 5));
  code.push_back(EncodeVop1(0x52, 17, 249));
  code.push_back(EncodeVop1Sdwa(1, 6, 0, 4));
  code.push_back(EncodeVop1(0x52, 18, 249));
  code.push_back(EncodeVop1Sdwa(1, 4, 0, 4));
  code.push_back(EncodeVop1(0x53, 19, 249));
  code.push_back(EncodeVop1Sdwa(2, 4, 1, 4));

  code.push_back(EncodeVop2(0x32, 20, 250, 4));
  code.push_back(EncodeVop2Dpp(3));
  AppendVop3(&code, 0x132, 21, Vgpr(3), Vgpr(4));
  code.push_back(EncodeVop2(0x32, 22, 249, 4));
  code.push_back(EncodeVop2Sdwa(3, 4, 0, 4, 4));

  code.push_back(EncodeVop1(0x0a, 23, 249));
  code.push_back(EncodeVop1Sdwa(0, 4, 2, 3));
  code.push_back(EncodeVop1(0x0a, 24, 249));
  code.push_back(EncodeVop1Sdwa(0, 4, 2, 5));
  code.push_back(EncodeVop1(0x0a, 25, 249));
  code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 1, 1));

  code.push_back(EncodeVop1(0x0a, 26, 249));
  code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 0, 0, 0, 0, 1));
  code.push_back(EncodeVop1(0x0a, 27, 249));
  code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 0, 0, 0, 1));
  AppendVop3(&code, 0x18a, 28, Vgpr(0), 0, 0, 0, 0, false, 1);
  AppendVop3(&code, 0x18a, 29, Vgpr(0), 0, 0, 0, 0, true);
  AppendVop3(&code, 0x34b, 30, Vgpr(3), Vgpr(4), Vgpr(3));
  AppendVop3(&code, 0x34b, 32, Vgpr(3), Vgpr(4), Vgpr(3), 0, 0x8);
  code.push_back(EncodeVop1(0x52, 33, 249));
  code.push_back(EncodeVop1Sdwa(1, 4, 2, 4, 0, 1, 1, 0, 1));
  AppendVop3(&code, 0x1d3, 34, Vgpr(1), 0, 0, 1, 0, true, 0, 1);
  code.push_back(EncodeVop1(0x07, 35, 249));
  code.push_back(EncodeVop1Sdwa(0, 6, 0, 6, 0, 1, 1, 0, 1));
  AppendVop3(&code, 0x188, 36, Vgpr(0), 0, 0, 1, 0, true, 0, 1);
  code.push_back(EncodeVop1(0x54, 37, 249));
  code.push_back(EncodeVop1Sdwa(4, 4, 2, 4, 0, 1, 1));

  for (u32 i = 0; i < static_cast<u32>(std::size(sentinels)); i++) {
    AppendStoreVgpr(&code, output_vgpr(i), i);
  }
  AppendEnd(&code);

  return {"NativeAndSdwa16BitDestinationWrites",
          code,
          {},
          {0x3c003e00u, 0xbbbb3e00u, 0xcccc3e00u, 0xdddd0003u,
           0xeeeefffeu, 0x00003e00u, 0xf00d3c00u, 0x00000003u,
           0x00000003u, 0xfffffffeu, 0xaaaa4200u, 0xbbbb4200u,
           0x00004200u, 0xabcd0000u, 0xbcde0000u, 0xcdefbe00u,
           0xdef04200u, 0xef013c00u, 0xf0124200u, 0x01233c00u,
           0xaaaa4200u, 0x42005678u, 0x24680000u, 0x1357fffdu,
           0x00000000u, 0xffffffffu, 0x0badb800u},
          {O::V_MOV_B32, O::V_CVT_F16_F32, O::V_CVT_U32_F32,
           O::V_CVT_I32_F32, O::V_CVT_U16_F16, O::V_CVT_I16_F16,
           O::V_RCP_F16, O::V_ADD_F16, O::V_FMA_F16, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase VectorMinMaxMed3F16Ops() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x40003c00u); // low=1.0h, high=2.0h
  AppendVMovLiteral(&code, 1, 0x44004200u); // low=3.0h, high=4.0h
  AppendVMovLiteral(&code, 2, 0xc8004000u); // low=2.0h, high=-8.0h
  AppendVMovLiteral(&code, 3, 0x00007e00u); // low=qNaN
  AppendVMovLiteral(&code, 4, 0x00004000u); // low=2.0h
  AppendVMovLiteral(&code, 5, 0x00003c00u); // low=1.0h
  AppendVMovLiteral(&code, 10, 0xaaaa5555u);
  AppendVMovLiteral(&code, 11, 0x12345678u);
  AppendVMovLiteral(&code, 12, 0x77772222u);

  AppendVop3(&code, 0x351, 10, Vgpr(0), Vgpr(1), Vgpr(2));
  AppendVop3(&code, 0x354, 11, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0xf, false, 0,
             0x4);
  AppendVop3(&code, 0x357, 12, Vgpr(3), Vgpr(4), Vgpr(5));
  for (u32 i = 0; i < 3; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  return {"VectorMinMaxMed3F16Ops",
          code,
          {},
          {0xaaaa3c00u, 0x48005678u, 0x77773c00u},
          {O::V_MOV_B32, O::V_MIN3_F16, O::V_MAX3_F16, O::V_MED3_F16,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorSpecialF16Ops() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0xc000fc00u); // low=-inf, high=-2.0h
  AppendVMovLiteral(&code, 1, 0x00004400u); // low=+4.0h
  AppendVMovLiteral(&code, 2, 0x00003c00u); // low=+1.0h
  AppendVMovLiteral(&code, 3, 0x0000bc00u); // low=-1.0h
  AppendVMovLiteral(&code, 4, 0x0000fc00u); // low=-inf
  AppendVMovLiteral(&code, 11, 0x12345678u);

  code.push_back(EncodeVop1(0x54, 10, Vgpr(0)));
  code.push_back(EncodeVop1(0x54, 11, 249));
  code.push_back(EncodeVop1Sdwa(0, 5, 2, 5));
  code.push_back(EncodeVop1(0x56, 12, Vgpr(1)));
  code.push_back(EncodeVop1(0x57, 13, Vgpr(2)));
  code.push_back(EncodeVop1(0x57, 14, Vgpr(3)));
  code.push_back(EncodeVop1(0x58, 15, Vgpr(4)));
  AppendVop3(&code, 0x1d7, 16, Vgpr(3), 0, 0, 0, 0, true);
  for (u32 i = 0; i < 7; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  return {"VectorSpecialF16Ops",
          code,
          {},
          {0x00008000u, 0xb8005678u, 0x00003800u, 0x00000000u, 0x0000fe00u,
           0x00000000u, 0x00000000u},
          {O::V_MOV_B32, O::V_RCP_F16, O::V_RSQ_F16, O::V_LOG_F16, O::V_EXP_F16,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorFractF16CapturedAndEdges() {
  using O = ShaderOpcode;

  // RDNA2 section 12.8: x - floor(x), rounded to half precision.
  // Load at runtime to exercise the emitted operation, including half subnormals.
  const std::array<u32, 18> inputs{
      0x3d00u, 0xbd00u, 0x0000u, 0x8000u, 0x0001u, 0x8001u,
      0x03ffu, 0x83ffu, 0x7bffu, 0xfbffu, 0x3800u, 0xb800u,
      0x8c00u, 0x8c01u, 0x7c00u, 0xfc00u, 0x7e55u, 0x7d01u};
  const std::array<u32, 18> fractions{
      0x3400u, 0x3a00u, 0x0000u, 0x0000u, 0x0001u, 0x3c00u,
      0x03ffu, 0x3c00u, 0x0000u, 0x0000u, 0x3800u, 0x3800u,
      0x3c00u, 0x3bffu, 0x7e00u, 0x7e00u, 0x7e00u, 0x7e00u};
  TestCase test;
  test.name = "VectorFractF16CapturedAndEdges";
  for (u32 bits : inputs) {
    test.initial.push_back(0x42000000u | bits); // Distinct high half must be ignored.
  }
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < inputs.size(); i++) {
    AppendVMovU32(&code, 30, i * 4u);
    AppendBufferLoadDword(&code, 7, 30);
    AppendVMovLiteral(&code, 9, 0xa5a51234u);
    code.push_back(0x7e12bf07u); // Captured v_fract_f16 v9, v7.
    if (i >= 14u) {
      // Check quiet NaN class without constraining its sign or payload.
      code.push_back(EncodeVop2(0x1b, 9, 255u, 9));
      code.push_back(0xffff7e00u);
    }
    AppendStoreVgpr(&code, 9, static_cast<u32>(inputs.size()) + i);
    test.expected.push_back(0xa5a50000u | fractions[i]);
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FRACT_F16,
                  O::V_AND_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_FRACT_F16 v9, v7", inputs.size()}};
  test.ir_counts = {{" = FPFract32 ", inputs.size()}};
  test.required_spirv = {" Fract ", " PackHalf2x16 ", " UnpackHalf2x16 "};
  test.forbidden_spirv = {"OpCapability Float16"};
  return test;
}

TestCase VectorFractF16Modifiers() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 7, 0xbd003d00u); // high=-1.25h, low=+1.25h
  for (u32 dst = 10; dst <= 19; dst++) {
    AppendVMovLiteral(&code, dst, 0xabcd1234u);
  }
  code.push_back(EncodeVop1(0x5f, 10, 249));
  code.push_back(EncodeVop1Sdwa(7, 5, 2, 5)); // high -> high, preserve low
  code.push_back(EncodeVop1(0x5f, 11, 249));
  code.push_back(EncodeVop1Sdwa(7, 4, 0, 5, 0, 1, 1)); // -abs(-1.25), pad high
  code.push_back(EncodeVop1(0x5f, 12, 249));
  code.push_back(EncodeVop1Sdwa(7, 4, 2, 5, 0, 0, 1)); // abs(-1.25)
  code.push_back(EncodeVop1(0x5f, 13, 249));
  code.push_back(EncodeVop1Sdwa(7, 4, 2, 5, 0, 0, 0, 0, 1, 1)); // *2, clamp
  code.push_back(EncodeVop1(0x5f, 14, 250));
  code.push_back(EncodeVop1Dpp(7, 0x0e4) | (1u << 20u)); // DPP identity, negate
  AppendVop3(&code, 0x1df, 15, Vgpr(7), 0, 0, 1, 0, true, 1, 1);
  code.push_back(EncodeVop1(0x5f, 16, 255));
  code.push_back(0x4200bd00u); // literal -1.25h
  code.push_back(EncodeVop1(0x5f, 17, 240)); // inline +0.5, interpreted as FP16
  AppendSMovLiteral(&code, 20, 0x4200bd00u);
  code.push_back(EncodeVop1(0x5f, 18, 20)); // SGPR source
  AppendVop3(&code, 0x1df, 19, Vgpr(7), 0, 0, 0, 0, false, 3); // *0.5
  for (u32 i = 0; i < 10; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  TestCase test{"VectorFractF16Modifiers", code, {},
                {0x3a001234u, 0x00003a00u, 0xabcd3400u, 0xabcd3c00u,
                 0xabcd3a00u, 0xabcd3c00u, 0xabcd3a00u, 0xabcd3800u,
                 0xabcd3a00u, 0xabcd3000u},
                {O::V_MOV_B32, O::S_MOV_B32, O::V_FRACT_F16,
                 O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.decoded_counts = {{"V_FRACT_F16", 10}};
  test.ir_counts = {{" = FPFract32 ", 10}};
  test.required_spirv = {" Fract ", " PackHalf2x16 "};
  return test;
}

TestCase VectorCosF16CapturedSdwaAndEdges() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 4, 0x34001234u); // high=+0.25h, low sentinel
  code.push_back(0x7e08c2f9u);
  code.push_back(0x00051504u); // captured v_cos_f16 v4.word1, v4.word1
  AppendVMovLiteral(&code, 5, 0x00008000u); // -0.0h
  code.push_back(EncodeVop1(0x61, 6, Vgpr(5)));
  AppendVMovLiteral(&code, 7, 0x00007bffu); // max finite
  code.push_back(EncodeVop1(0x61, 8, Vgpr(7)));
  AppendVMovLiteral(&code, 9, 0x00007c00u); // +inf
  code.push_back(EncodeVop1(0x61, 10, Vgpr(9)));
  AppendVMovLiteral(&code, 11, 0x00000001u); // minimum positive denormal
  code.push_back(EncodeVop1(0x61, 12, Vgpr(11)));
  AppendVMovLiteral(&code, 13, 0x00003800u); // +0.5h
  code.push_back(EncodeVop1(0x61, 14, Vgpr(13)));
  for (u32 i = 0; i < 6; i++) {
    AppendStoreVgpr(&code, 4 + i * 2, i);
  }
  AppendEnd(&code);

  TestCase test{"VectorCosF16CapturedSdwaAndEdges",
                code,
                {},
                {0x00001234u, 0x00003c00u, 0x00003c00u, 0x0000fe00u,
                 0x00003c00u, 0x0000bc00u},
                {O::V_MOV_B32, O::V_COS_F16, O::BUFFER_STORE_DWORD,
                 O::S_ENDPGM}};
  test.decoded_counts = {{"V_COS_F16", 6},
                         {"V_COS_F16 v4.sdwa(sel=5,sext=0), "
                          "v4.sdwa(sel=5,sext=0)",
                          1}};
  test.ir_counts = {{" = FPCos ", 6}};
  test.required_spirv = {" Fract ", " Cos ", " PackHalf2x16 "};
  test.forbidden_spirv = {"OpCapability Float16"};
  return test;
}

TestCase VectorSinF16SdwaAndEdges() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 4, 0x34001234u); // high=+0.25h, low sentinel
  code.push_back(0x7e08c0f9u);
  code.push_back(0x00051504u); // v_sin_f16 v4.word1, v4.word1
  AppendVMovLiteral(&code, 5, 0x00008000u); // -0.0h
  code.push_back(EncodeVop1(0x60, 6, Vgpr(5)));
  AppendVMovLiteral(&code, 7, 0x00007bffu); // max finite
  code.push_back(EncodeVop1(0x60, 8, Vgpr(7)));
  AppendVMovLiteral(&code, 9, 0x0000fbffu); // minimum finite
  code.push_back(EncodeVop1(0x60, 10, Vgpr(9)));
  AppendVMovLiteral(&code, 11, 0x00007c00u); // +inf
  code.push_back(EncodeVop1(0x60, 12, Vgpr(11)));
  AppendVMovLiteral(&code, 13, 0x0000fc00u); // -inf
  code.push_back(EncodeVop1(0x60, 14, Vgpr(13)));
  AppendVMovLiteral(&code, 15, 0x00003800u); // +0.5h
  code.push_back(EncodeVop1(0x60, 16, Vgpr(15)));
  AppendVMovLiteral(&code, 17, 0x00003a00u); // +0.75h
  code.push_back(EncodeVop1(0x60, 18, Vgpr(17)));
  AppendVMovLiteral(&code, 19, 0x00003400u); // +0.25h
  AppendVop3(&code, 0x1e0, 20, Vgpr(19), 0);
  for (u32 i = 0; i < 9; i++) {
    AppendStoreVgpr(&code, 4 + i * 2, i);
  }
  AppendEnd(&code);

  TestCase test{"VectorSinF16SdwaAndEdges",
                code,
                {},
                {0x3c001234u, 0x00008000u, 0x00000000u, 0x00000000u,
                 0x0000fe00u, 0x0000fe00u, 0x00000000u, 0x0000bc00u,
                 0x00003c00u},
                {O::V_MOV_B32, O::V_SIN_F16, O::BUFFER_STORE_DWORD,
                 O::S_ENDPGM}};
  test.decoded_counts = {{"V_SIN_F16", 9},
                         {"V_SIN_F16 v4.sdwa(sel=5,sext=0), "
                          "v4.sdwa(sel=5,sext=0)",
                          1}};
  test.ir_counts = {{" = FPSin ", 9}};
  test.required_spirv = {" Fract ", " Sin ", " PackHalf2x16 "};
  test.forbidden_spirv = {"OpCapability Float16"};
  return test;
}

TestCase VectorWritelaneIgnoresExecMask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 2, 0xaaaaaaaau);
  AppendSMovLiteral(&code, 4, 0x12345678u);
  code.push_back(EncodeSMovB32(126, InlineU32(1)));
  code.push_back(EncodeSMovB32(127, InlineU32(0)));
  AppendVop3(&code, 0x361, 2, 4, InlineU32(1));
  code.push_back(EncodeSMovB32(126, InlineU32(0xf)));
  code.push_back(EncodeSMovB32(127, InlineU32(0)));
  code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
  AppendBufferStoreDword(&code, 2, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorWritelaneIgnoresExecMask";
  test.code = code;
  test.expected = {0xaaaaaaaau, 0x12345678u, 0xaaaaaaaau, 0xaaaaaaaau};
  test.opcodes = {O::V_MOV_B32,     O::S_MOV_B32,          O::V_WRITELANE_B32,
                  O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorReadlaneFromInactiveWrittenLane() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 4, 0x12345678u);
  AppendVop3(&code, 0x361, 2, 4, InlineU32(4));
  AppendSMovLiteral(&code, 4, 0xdeadbeefu);
  AppendVop3(&code, 0x360, 5, Vgpr(2), InlineU32(4));
  AppendStoreSgpr(&code, 5, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorReadlaneInactiveWrittenLane";
  test.code = code;
  test.expected = {0x12345678u};
  test.opcodes = {O::S_MOV_B32, O::V_WRITELANE_B32,    O::V_READLANE_B32,
                  O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.forbidden_spirv = {"OpGroupNonUniformShuffle"};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorLaneWave32RuntimeSelectorWraps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSop2(0x00, 5, 4, InlineU32(33)));
  AppendSMovLiteral(&code, 6, 0x12345678u);
  AppendVMovU32(&code, 2, 0);
  AppendVop3(&code, 0x361, 2, 6, 5);
  AppendVop3(&code, 0x360, 7, Vgpr(2), InlineU32(1));
  AppendStoreSgpr(&code, 7, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorLaneWave32RuntimeSelectorWraps";
  test.code = code;
  test.expected = {0x12345678u};
  test.opcodes = {O::S_ADD_U32,      O::S_MOV_B32, O::V_WRITELANE_B32,
                  O::V_READLANE_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.required_spirv = {"OpGroupNonUniformShuffle"};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.wave_size = 32;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 4;
  test.has_compute_info = true;
  return test;
}

TestCase VectorReadlaneSelectsTwoKeysWithinWave() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop2(0x16, 1, InlineU32(4), 0)); // lanes 0..15: 0, 16..31: 1
  AppendVop3(&code, 0x360, 20, Vgpr(1), InlineU32(0));
  AppendVop3(&code, 0x360, 21, Vgpr(1), InlineU32(16));
  AppendVMovU32(&code, 2, 0);
  AppendVMovU32(&code, 3, 11);
  AppendVMovU32(&code, 4, 22);
  code.push_back(EncodeVopc(0xc2, 20, 1)); // selected lane 0 key == local key
  code.push_back(EncodeVop2(0x01, 2, Vgpr(2), 3));
  code.push_back(EncodeVopc(0xc2, 21, 1)); // selected lane 16 key == local key
  code.push_back(EncodeVop2(0x01, 2, Vgpr(2), 4));
  AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorReadlaneSelectsTwoKeysWithinWave";
  test.code = std::move(code);
  test.expected.resize(32);
  std::fill_n(test.expected.begin(), 16, 11u);
  std::fill_n(test.expected.begin() + 16, 16, 22u);
  test.opcodes = {O::V_LSHRREV_B32, O::V_READLANE_B32, O::V_MOV_B32,
                  O::V_CMP_EQ_U32, O::V_CNDMASK_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_READLANE_B32", 2}};
  test.required_spirv = {"OpGroupNonUniformShuffle"};
  test.compute_info.threads_num[0] = 32;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.wave_size = 32;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorPermlanex16() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 3, 0x76543210u);
  AppendVMovLiteral(&code, 4, 0xfedcba98u);
  code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0));
  AppendVMovLiteral(&code, 5, 0xfeedbabeu);
  AppendVop3(&code, 0x378, 1, Vgpr(5), Vgpr(3), Vgpr(4));
  AppendBufferStoreDword(&code, 1, 2);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorPermlanex16";
  test.code = code;
  test.expected = std::vector<u32>(32, 0xfeedbabeu);
  test.opcodes = {O::V_MOV_B32, O::V_PERMLANEX16_B32, O::V_LSHLREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 32;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorPermlane16FetchInactiveZero() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVop1(0x01, 1, 0),
      EncodeVop2(0x25, 1, InlineU32(10), 1),
      EncodeVopc(0xc2, InlineU32(0), 0),
      EncodeSop1(0x04, 126, 106),
      EncodeSMovB32(0, InlineU32(1)),
      EncodeSMovB32(1, InlineU32(0)),
  };
  AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1);
  AppendStoreVgpr(&code, 2, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorPermlane16FetchInactiveZero";
  test.code = code;
  test.expected = {0};
  test.opcodes = {O::V_MOV_B32,          O::V_ADD_NC_U32, O::V_CMP_EQ_U32,
                  O::S_MOV_B64,          O::S_MOV_B32,    O::V_PERMLANE16_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorPermlane16FetchInactiveFi() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVop1(0x01, 1, Vgpr(0)),      EncodeVop2(0x25, 1, InlineU32(10), 1),
      EncodeVopc(0xc2, InlineU32(0), 0), EncodeSop1(0x04, 126, 106),
      EncodeSMovB32(0, InlineU32(1)),    EncodeSMovB32(1, InlineU32(0)),
  };
  AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1, 0, 1);
  AppendStoreVgpr(&code, 2, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorPermlane16FetchInactiveFi";
  test.code = code;
  test.expected = {11};
  test.opcodes = {O::V_MOV_B32,          O::V_ADD_NC_U32, O::V_CMP_EQ_U32,
                  O::S_MOV_B64,          O::S_MOV_B32,    O::V_PERMLANE16_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorDpp8Captured(bool masked_exec, u32 wave_size = 64, bool fi = false) {
  using O = ShaderOpcode;
  constexpr u32 sentinel = 0xaaaaaaaau;
  // Both wave halves contain active and inactive lane-3 sources for the ADD.
  const u32 masks[] = {masked_exec ? 0x0f0ff0f0u : 0xffffffffu,
                      masked_exec ? 0xf0f00f0fu : 0xffffffffu};
  std::vector<u32> code;
  AppendVMovU32(&code, 5, 100);
  code.push_back(EncodeVop2(0x25, 5, Vgpr(0), 5));
  code.push_back(EncodeVop1(0x01, 7, Vgpr(5)));
  AppendVMovLiteral(&code, 2, sentinel);
  AppendVMovLiteral(&code, 3, sentinel);
  AppendVMovLiteral(&code, 8, sentinel);
  AppendSMovLiteral(&code, 126, masks[0]);
  if (wave_size == 64) AppendSMovLiteral(&code, 127, masks[1]);
  // Captured DPP8 selectors broadcast each group's upper/lower four lanes.
  const u32 encoding = fi ? 234u : 233u;
  code.push_back(EncodeVop1(0x01, 2, encoding));
  code.push_back(0xfacfac05u);
  code.push_back(EncodeVop1(0x01, 3, encoding));
  code.push_back(0x68868805u);
  // CS 6babb2a8915d35c4, pc 0xcc: add lane 3 of each group to its own v7.
  code.push_back(EncodeVop2(0x25, 8, encoding, 7));
  code.push_back(0x6db6db07u);
  AppendSMovLiteral(&code, 126, 0xffffffffu);
  if (wave_size == 64) AppendSMovLiteral(&code, 127, 0xffffffffu);
  code.push_back(EncodeVop2(0x1a, 6, InlineU32(2), 0));
  AppendBufferStoreDword(&code, 2, 6);
  AppendVMovU32(&code, 7, wave_size * sizeof(u32));
  code.push_back(EncodeVop2(0x25, 6, Vgpr(7), 6));
  AppendBufferStoreDword(&code, 3, 6);
  code.push_back(EncodeVop2(0x25, 6, Vgpr(7), 6));
  AppendBufferStoreDword(&code, 8, 6);
  AppendEnd(&code);

  TestCase test;
  const char *names[2][2] = {
      {"VectorDpp8Wave32MaskedExec", "VectorDpp8Wave64MaskedExec"},
      {"VectorDpp8FiWave32MaskedExec", "VectorDpp8FiWave64MaskedExec"}};
  test.name = masked_exec ? names[fi][wave_size == 64] : "VectorDpp8Captured";
  test.code = std::move(code);
  test.expected.resize(3 * wave_size, sentinel);
  for (u32 lane = 0; lane < wave_size; ++lane) {
    const auto mask = masks[lane / 32u];
    if ((mask & (1u << (lane % 32u))) == 0) {
      continue;
    }
    const u32 sources[] = {(lane & ~7u) | 4u | (lane & 3u),
                           (lane & ~7u) | (lane & 3u)};
    for (u32 permutation = 0; permutation < 2; ++permutation) {
      const auto source = sources[permutation];
      test.expected[permutation * wave_size + lane] =
          fi || (mask & (1u << (source % 32u))) != 0 ? 100u + source : 0u;
    }
    const u32 source = (lane & ~7u) | 3u;
    test.expected[2 * wave_size + lane] = 100u + lane;
    if (fi || (mask & (1u << (source % 32u))) != 0) {
      test.expected[2 * wave_size + lane] += 100u + source;
    }
  }
  test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32,
                  O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{".dpp8(", 3}};
  test.compute_info.wave_size = wave_size;
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorDppQuadPermuteReverse() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 100);
  code.push_back(EncodeVop2(0x25, 2, 250, 1));
  code.push_back(EncodeVop2Dpp(0, 0x01b));
  code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
  AppendBufferStoreDword(&code, 2, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorDppQuadPermuteReverse";
  test.code = code;
  test.expected = {103, 102, 101, 100, 107, 106, 105, 104};
  test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 8;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorDppRowXmask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 100);
  code.push_back(EncodeVop2(0x25, 2, 250, 1));
  code.push_back(EncodeVop2Dpp(0, 0x164));
  code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
  AppendBufferStoreDword(&code, 2, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorDppRowXmask";
  test.code = code;
  test.expected = {104, 105, 106, 107, 100, 101, 102, 103};
  test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 8;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorDppRowShare(u32 wave_size) {
  using O = ShaderOpcode;
  constexpr u32 sentinel = 0xaaaaaaaau;
  TestCase test;
  test.name = wave_size == 32 ? "VectorDppRowShareWave32" : "VectorDppRowShareWave64";
  auto &code = test.code;
  AppendVMovU32(&code, 1, 100);
  code.push_back(EncodeVop2(0x25, 1, Vgpr(0), 1)); // v1 = 100 + lane.
  code.push_back(EncodeSop1(0x04, 10, 126));
  for (u32 selector = 0; selector < 16; ++selector) {
    for (const bool masked : {false, true}) {
      const u32 row_mask = masked ? 0xau : 0xfu;
      const u32 bank_mask = masked ? 0x5u : 0xfu;
      // Keep the selected source active in every row while masking destination
      // lanes. Row and bank masks affect destinations, not the shared source.
      const u32 source_lanes = 0x00010001u << selector;
      const u32 exec[] = {masked ? 0xaaaaaaaau | source_lanes : 0xffffffffu,
                          masked ? 0x55555555u | source_lanes : 0xffffffffu};
      AppendVMovLiteral(&code, 2, sentinel);
      AppendVMovLiteral(&code, 3, sentinel);
      AppendSMovLiteral(&code, 126, exec[0]);
      if (wave_size == 64) AppendSMovLiteral(&code, 127, exec[1]);
      code.push_back(EncodeVop1(0x01, 2, 250));
      code.push_back(EncodeVop1Dpp(1, 0x150 + selector, row_mask, bank_mask));
      code.push_back(EncodeVop2(0x25, 3, 250, 0));
      code.push_back(EncodeVop2Dpp(1, 0x150 + selector, row_mask, bank_mask));
      code.push_back(EncodeSop1(0x04, 126, 10));
      for (const u32 reg : {2u, 3u}) {
        AppendStoreVgprAtLaneDwordOffset(&code, reg, 0, u32(test.expected.size()));
        for (u32 lane = 0; lane < wave_size; ++lane) {
          const bool write = ((row_mask >> (lane / 16)) & 1u) &&
              ((bank_mask >> ((lane / 4) % 4)) & 1u) &&
              ((exec[lane / 32] >> (lane % 32)) & 1u);
          const u32 source = (lane / 16) * 16 + selector;
          test.expected.push_back(write ? 100 + source + (reg == 3 ? lane : 0) : sentinel);
        }
      }
    }
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, O::S_MOV_B64,
                  O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.wave_size = wave_size;
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorDppBankMaskPreservesDestination() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 2, 0xaaaaaaaau);
  AppendVMovU32(&code, 1, 100);
  code.push_back(EncodeVop2(0x25, 2, 250, 1));
  code.push_back(EncodeVop2Dpp(0, 0x0e4, 0xf, 0xe));
  code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
  AppendBufferStoreDword(&code, 2, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorDppBankMaskPreservesDestination";
  test.code = code;
  test.expected = {0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau,
                   104,         105,         106,         107};
  test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 8;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorDppBoundsControlZeroPreservesDestination() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 2, 0xaaaaaaaau);
  AppendVMovU32(&code, 1, 100);
  code.push_back(EncodeVop2(0x25, 2, 250, 1));
  code.push_back(EncodeVop2Dpp(0, 0x111));
  code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
  AppendBufferStoreDword(&code, 2, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorDppBoundsControlZeroPreservesDestination";
  test.code = code;
  test.expected = {0xaaaaaaaau, 100, 101, 102, 103, 104, 105, 106};
  test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 8;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorMadF32RoundsProduct() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  for (u32 source = 0; source < 3; source++) {
    AppendVMovU32(&code, 30, source * sizeof(u32));
    AppendBufferLoadDword(&code, source, 30);
  }
  AppendVMovLiteral(&code, 10, 0xbf800000u);
  AppendVMovLiteral(&code, 11, 0xbf800000u);
  AppendVMovLiteral(&code, 15, 0xbf800000u);
  AppendVMovLiteral(&code, 16, 0xbf800000u);
  code.push_back(EncodeVop2(0x1f, 10, Vgpr(0), 1));
  AppendVop3(&code, 0x11f, 11, Vgpr(0), Vgpr(1), 0);
  AppendVop3(&code, 0x141, 12, Vgpr(0), Vgpr(1), Vgpr(2));
  code.push_back(EncodeVop2(0x20, 13, Vgpr(0), 2));
  code.push_back(0x3f7ffffeu);
  code.push_back(EncodeVop2(0x21, 14, Vgpr(0), 1));
  code.push_back(0xbf800000u);
  code.push_back(EncodeVop2(0x2b, 15, Vgpr(0), 1));
  AppendVop3(&code, 0x12b, 16, Vgpr(0), Vgpr(1), 0);
  AppendVop3(&code, 0x14b, 17, Vgpr(0), Vgpr(1), Vgpr(2));
  code.push_back(EncodeVop2(0x2c, 18, Vgpr(0), 2));
  code.push_back(0x3f7ffffeu);
  code.push_back(EncodeVop2(0x2d, 19, Vgpr(0), 1));
  code.push_back(0xbf800000u);
  for (u32 i = 0; i < 10; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorMadF32RoundsProduct";
  test.code = std::move(code);
  test.initial = {0x3f800001u, 0x3f7ffffeu, 0xbf800000u};
  // (1+2^-23)*(1-2^-23)-1 rounds to zero for MAD, but -2^-46 for FMA.
  test.expected = {0u, 0u, 0u, 0u, 0u, 0xa8800000u, 0xa8800000u,
                   0xa8800000u, 0xa8800000u, 0xa8800000u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAC_F32,
                  O::V_MAD_F32, O::V_MADMK_F32, O::V_MADAK_F32,
                  O::V_FMAC_F32, O::V_FMA_F32, O::V_FMAMK_F32, O::V_FMAAK_F32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpFMul", "OpFAdd", "NoContraction", "Fma"};
  return test;
}

TestCase VectorMadF32FlushesDenorms() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  for (u32 sample = 0; sample < 5; sample++) {
    for (u32 source = 0; source < 3; source++) {
      AppendVMovU32(&code, 30, (sample * 3 + source) * sizeof(u32));
      AppendBufferLoadDword(&code, source, 30);
    }
    AppendVop3(&code, 0x141, 10 + sample, Vgpr(0), Vgpr(1), Vgpr(2));
  }
  for (u32 sample = 0; sample < 5; sample++) {
    AppendStoreVgpr(&code, 10 + sample, sample);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorMadF32FlushesDenorms";
  test.code = std::move(code);
  // Input denorms in each operand, an intermediate denorm, and a negative result denorm.
  test.initial = {0x00400000u, 0x40000000u, 0u,
                  0x40000000u, 0x00400000u, 0u,
                  0u, 0x3f800000u, 0x00400000u,
                  0x00800000u, 0x3f000000u, 0x00800000u,
                  0x00800000u, 0x3f800000u, 0x80800001u};
  test.expected = {0u, 0u, 0u, 0x00800000u, 0x80000000u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_F32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  // Legacy MAD always flushes, even when the shader requests denorm preservation.
  test.compute_info.float_mode = 0xf0;
  test.has_compute_info = true;
  return test;
}

TestCase Vop3FmacF32NegatedSourceAccumulates() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  constexpr u32 registers[] = {13, 20, 29};
  for (u32 sample = 0; sample < 2; sample++) {
    for (u32 source = 0; source < 3; source++) {
      AppendVMovU32(&code, 30, (sample * 3 + source) * sizeof(u32));
      AppendBufferLoadDword(&code, registers[source], 30);
    }
    code.push_back(0xd52b001du);
    code.push_back(0x4002290du); // v_fmac_f32_e64 v29, v13, -v20
    AppendStoreVgpr(&code, 29, sample);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop3FmacF32NegatedSourceAccumulates";
  test.code = std::move(code);
  test.initial = {0x40000000u, 0x40400000u, 0x41200000u,
                  0x3f800001u, 0x3f7ffffeu, 0x3f800000u};
  // 10 - 2*3 = 4; 1 - (1+2^-23)*(1-2^-23) = 2^-46, not rounded zero.
  test.expected = {0x40800000u, 0x28800000u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FMAC_F32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpFNegate", "Fma"};
  return test;
}

TestCase Vop3LdexpSourceModifier() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 9, 0xbfc00000u);
  code.push_back(0xd7620107u);
  code.push_back(0x00018509u);
  AppendStoreVgpr(&code, 7, 0);
  AppendEnd(&code);

  return {"Vop3LdexpSourceModifier",
          code,
          {},
          {0x3ec00000u},
          {O::V_MOV_B32, O::V_LDEXP_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase Vop1MoveRelSource() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 12, 0x11111111u);
  AppendVMovLiteral(&code, 13, 0x22222222u);
  AppendVMovLiteral(&code, 14, 0x33333333u);
  code.push_back(EncodeSMovB32(124, InlineU32(2)));
  code.push_back(0x7e6e870cu);
  AppendStoreVgpr(&code, 55, 0);
  AppendEnd(&code);

  return {"Vop1MoveRelSource",
          code,
          {},
          {0x33333333u},
          {O::V_MOV_B32, O::S_MOV_B32, O::V_MOVRELS_B32, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase Vop1MoveRelDestination() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 1, 0x12345678u);
  AppendVMovLiteral(&code, 5, 0xaaaaaaaau);
  AppendVMovLiteral(&code, 6, 0xbbbbbbbbu);
  AppendVMovLiteral(&code, 7, 0xccccccccu);
  code.push_back(EncodeSMovB32(124, InlineU32(2)));
  code.push_back(EncodeSop1(0x04, 8, 126));
  code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
  code.push_back(EncodeSop1(0x04, 126, 106));
  code.push_back(EncodeVop1(0x42, 5, Vgpr(1)));
  code.push_back(EncodeSop1(0x04, 126, 8));
  code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
  AppendBufferStoreDword(&code, 7, 4);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop1MoveRelDestination";
  test.code = code;
  test.expected = {0x12345678u, 0xccccccccu, 0xccccccccu, 0xccccccccu};
  test.opcodes = {O::V_MOV_B32,          O::S_MOV_B32,     O::V_CMP_EQ_U32,
                  O::S_MOV_B64,          O::V_MOVRELD_B32, O::V_LSHLREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase CubeIdCapturedNegationAndOutputScale() {
  using O = ShaderOpcode;

  constexpr std::array directions = {
      std::array{4.0f, 2.0f, 1.0f}, std::array{-4.0f, 2.0f, 1.0f},
      std::array{1.0f, -4.0f, 2.0f}, std::array{1.0f, 4.0f, 2.0f},
      std::array{1.0f, 2.0f, -4.0f}, std::array{1.0f, 2.0f, 4.0f}};
  constexpr std::array registers = {47u, 49u, 48u};
  TestCase test;
  test.name = "CubeIdCapturedNegationAndOutputScale";
  for (u32 i = 0; i < directions.size(); i++) {
    for (u32 component = 0; component < registers.size(); component++) {
      test.initial.push_back(std::bit_cast<u32>(directions[i][component]));
      AppendVMovU32(&test.code, 30, (i * 3u + component) * 4u);
      AppendBufferLoadDword(&test.code, registers[component], 30);
    }
    // The captured instruction negates Y/Z and divides the floating face ID by two.
    test.code.insert(test.code.end(), {0xd544000bu, 0xdcc2632fu});
    AppendStoreVgpr(&test.code, 11, i);
  }
  AppendEnd(&test.code);
  test.expected = {0x00000000u, 0x3f000000u, 0x3f800000u,
                   0x3fc00000u, 0x40000000u, 0x40200000u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_CUBEID_F32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {
      {"V_CUBEID_F32 v11.omod(3), v47, v49.neg, v48.neg", directions.size()}};
  test.required_spirv = {"OpFNegate", "OpFMul"};
  return test;
}

TestCase VectorFloatSpecialOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovLiteral(&code, 1, 0x40000000u);
  AppendVMovLiteral(&code, 2, 0x40400000u);
  AppendVMovLiteral(&code, 3, 0x40800000u);
  AppendVMovLiteral(&code, 4, 0x40003c00u);
  AppendVMovLiteral(&code, 5, 0x44004200u);
  AppendVMovLiteral(&code, 6, 0x3f800000u);
  AppendVMovLiteral(&code, 7, 0x3f800000u);
  AppendVMovLiteral(&code, 8, 0xbf800000u);
  AppendVMovLiteral(&code, 10, 0x3f000000u);
  AppendVMovLiteral(&code, 12, 0xaaaabbbbu);
  AppendVMovLiteral(&code, 13, 0xaaaabbbbu);

  code.push_back(EncodeVop2(0x02, 6, Vgpr(4), 5));
  AppendVop3(&code, 0x368, 9, Vgpr(7), Vgpr(8));
  AppendVop3(&code, 0x369, 11, Vgpr(7), Vgpr(10));
  AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(2), Vgpr(0));
  AppendVop3p(&code, 0x22, 13, Vgpr(1), Vgpr(2), Vgpr(0));
  AppendVop3(&code, 0x144, 14, Vgpr(0), Vgpr(1), Vgpr(2));
  AppendVop3(&code, 0x145, 15, Vgpr(0), Vgpr(1), Vgpr(2));
  AppendVop3(&code, 0x146, 16, Vgpr(0), Vgpr(1), Vgpr(2));
  AppendVop3(&code, 0x147, 17, Vgpr(0), Vgpr(1), Vgpr(2));

  const u32 results[] = {6, 9, 11, 12, 13, 14, 15, 16, 17};
  for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
    AppendStoreVgpr(&code, results[i], i);
  }
  AppendEnd(&code);

  return {"VectorFloatSpecialOps",
          code,
          {},
          {0x41400000u, 0x80017fffu, 0x8000ffffu, 0xaaaa4700u, 0x4700bbbbu,
           0x40800000u, 0x3f800000u, 0xc0000000u, 0x40c00000u},
          {O::V_MOV_B32, O::V_DOT2C_F32_F16, O::V_CVT_PKNORM_I16_F32,
           O::V_CVT_PKNORM_U16_F32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16,
           O::V_CUBEID_F32, O::V_CUBESC_F32, O::V_CUBETC_F32, O::V_CUBEMA_F32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase MadMixF16LiteralHalfSourceUsesOpsel() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovLiteral(&code, 12, 0xaaaabbbbu);
  AppendVMovLiteral(&code, 13, 0x11112222u);

  AppendVop3p(&code, 0x21, 12, 255u, Vgpr(0), Vgpr(0), 0x1, 0x1);
  code.push_back(0x40003c00u);
  AppendVop3p(&code, 0x22, 13, 255u, Vgpr(0), Vgpr(0), 0x1, 0x0);
  code.push_back(0x40003c00u);

  AppendStoreVgpr(&code, 12, 0);
  AppendStoreVgpr(&code, 13, 1);
  AppendEnd(&code);

  return {"MadMixF16LiteralHalfSourceUsesOpsel",
          code,
          {},
          {0xaaaa4200u, 0x40002222u},
          {O::V_MOV_B32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase MadMixF16NegHiIsAbsAndNegIsIndependent() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovLiteral(&code, 1, 0xc0000000u);
  AppendVMovLiteral(&code, 2, 0x40000000u);
  AppendVMovLiteral(&code, 12, 0xaaaabbbbu);
  AppendVMovLiteral(&code, 13, 0x11112222u);

  AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(0), Vgpr(0), 0, 0, 0x1);
  AppendVop3p(&code, 0x22, 13, Vgpr(2), Vgpr(0), Vgpr(0), 0, 0, 0, 0x1);

  AppendStoreVgpr(&code, 12, 0);
  AppendStoreVgpr(&code, 13, 1);
  AppendEnd(&code);

  return {"MadMixF16NegHiIsAbsAndNegIsIndependent",
          code,
          {},
          {0xaaaa4200u, 0xbc002222u},
          {O::V_MOV_B32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorVop3FmaF16UsesRdna2Opcode34b() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 1, 0x40003c00u);
  AppendVMovLiteral(&code, 2, 0x44004200u);
  AppendVMovLiteral(&code, 3, 0x3c003c00u);
  AppendVMovLiteral(&code, 10, 0xaaaa5555u);
  AppendVMovLiteral(&code, 11, 0xbbbb5555u);

  AppendVop3(&code, 0x34bu, 10, Vgpr(1), Vgpr(2), Vgpr(3));
  AppendVop3(&code, 0x34bu, 11, Vgpr(1), Vgpr(2), Vgpr(3), 0, 0xfu);

  AppendStoreVgpr(&code, 10, 0);
  AppendStoreVgpr(&code, 11, 1);
  AppendEnd(&code);

  return {"VectorVop3FmaF16UsesRdna2Opcode34b",
          code,
          {},
          {0xaaaa4400u, 0x48805555u},
          {O::V_MOV_B32, O::V_FMA_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase FloatInlineConstantF16UsesNumericValue() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 1, 0x38003800u); // 0.5h, 0.5h
  AppendVMovLiteral(&code, 2, 0x3c003c00u); // 1.0h, 1.0h
  AppendVMovLiteral(&code, 10, 0xaaaa0000u);
  AppendVMovLiteral(&code, 12, 0xbbbb0000u);
  AppendVMovLiteral(&code, 14, 0xcccc0000u);
  AppendVMovLiteral(&code, 15, 0xdddd0000u);

  AppendVop3(&code, 0x34bu, 10, 246u, Vgpr(1), Vgpr(2));
  AppendVop3p(&code, 0x0eu, 11, 246u, Vgpr(1), Vgpr(2), 0x1u);
  AppendVop3(&code, 0x34bu, 12, 248u, 242u, 128u);
  AppendVop3(&code, 0x311u, 13, 242u, 248u);
  AppendVop3(&code, 0x34bu, 14, 246u, Vgpr(1), Vgpr(2), 0, 1);
  AppendVop3(&code, 0x303u, 15, 240u, InlineU32(1));

  AppendStoreVgpr(&code, 10, 0);
  AppendStoreVgpr(&code, 11, 1);
  AppendStoreVgpr(&code, 12, 2);
  AppendStoreVgpr(&code, 13, 3);
  AppendStoreVgpr(&code, 14, 4);
  AppendStoreVgpr(&code, 15, 5);
  AppendEnd(&code);

  TestCase test;
  test.name = "FloatInlineConstantF16UsesNumericValue";
  test.code = std::move(code);
  test.expected = {0xaaaa4200u, 0x3c004200u, 0xbbbb3118u,
                   0x31183c00u, 0xcccc3c00u, 0xdddd3801u};
  test.opcodes = {O::V_MOV_B32, O::V_FMA_F16, O::V_PK_FMA_F16,
                  O::V_PACK_B32_F16, O::V_ADD_NC_U16,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"4.000000", 3}, {"0.159155", 2}};
  return test;
}

TestCase Vop3Int16Ops() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 1, 0xabcd0001u);
  AppendVMovLiteral(&code, 2, 0x12340002u);
  AppendVMovLiteral(&code, 3, 0x55557fffu);
  AppendVMovLiteral(&code, 4, 0x66668000u);
  AppendVMovLiteral(&code, 5, 0x77770013u);
  AppendVMovLiteral(&code, 6, 0x88888010u);
  AppendVMovLiteral(&code, 10, 0xaaaa0000u);
  AppendVMovLiteral(&code, 11, 0xbbbb0000u);
  AppendVMovLiteral(&code, 12, 0xcccc0000u);
  AppendVMovLiteral(&code, 13, 0xdddd0000u);
  AppendVMovLiteral(&code, 14, 0xeeee0000u);
  AppendVMovLiteral(&code, 15, 0x11110000u);

  AppendVop3(&code, 0x304u, 10, Vgpr(1), Vgpr(2));
  AppendVop3(&code, 0x30du, 11, Vgpr(3), Vgpr(1));
  AppendVop3(&code, 0x308u, 12, Vgpr(5), Vgpr(6));
  AppendVop3(&code, 0x309u, 13, Vgpr(3), Vgpr(4));
  AppendVop3(&code, 0x30bu, 14, Vgpr(3), Vgpr(4));
  AppendVop3(&code, 0x308u, 15, Vgpr(5), Vgpr(3));

  for (u32 i = 0; i < 6; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop3Int16Ops";
  test.code = std::move(code);
  test.expected = {0xaaaaffffu, 0xbbbb8000u, 0xccccf002u,
                   0xdddd8000u, 0xeeee7fffu, 0x11110fffu};
  test.opcodes = {O::V_MOV_B32, O::V_SUB_NC_U16, O::V_ADD_NC_I16,
                  O::V_ASHRREV_I16, O::V_MAX_U16, O::V_MIN_U16,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase VectorFloatControlContractPreservesInfNan() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSop2(0x24, 0, InlineU32(8), InlineU32(23)));
  code.push_back(EncodeVop1(0x01, 0, InlineU32(0)));
  AppendVMovU32(&code, 1, 1);
  AppendVMovU32(&code, 2, 3);
  AppendVop3(&code, 0x108, 10, 0, Vgpr(0));
  code.push_back(EncodeVopc(0x88, Vgpr(10), 2));
  code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
  AppendStoreVgpr(&code, 3, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorFloatControlContractPreservesInfNan";
  test.code = std::move(code);
  test.expected = {1u};
  test.opcodes = {O::S_BFM_B32,       O::V_MOV_B32,     O::V_MUL_F32,
                  O::V_CMP_CLASS_F32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.required_spirv = {"SPV_KHR_float_controls",
                         "SignedZeroInfNanPreserve 32"};
  return test;
}

TestCase VectorFloatArithmeticOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovLiteral(&code, 1, 0x40000000u);
  AppendVMovLiteral(&code, 2, 0x40800000u);
  AppendVMovLiteral(&code, 3, 0x40a00000u);
  AppendVMovLiteral(&code, 4, 0x40400000u);
  AppendVMovLiteral(&code, 11, 0x40800000u);
  AppendVMovLiteral(&code, 17, 0x40800000u);

  code.push_back(EncodeVop2(0x03, 5, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x04, 6, Vgpr(3), 1));
  code.push_back(EncodeVop2(0x05, 7, Vgpr(1), 3));
  code.push_back(EncodeVop2(0x08, 8, Vgpr(1), 2));
  code.push_back(EncodeVop1(0x01, 21, Vgpr(8)));
  code.push_back(EncodeVop2(0x0f, 9, Vgpr(3), 1));
  code.push_back(EncodeVop2(0x10, 10, Vgpr(3), 1));
  code.push_back(EncodeVop2(0x1f, 11, Vgpr(1), 4));
  AppendVop3(&code, 0x141, 12, Vgpr(1), Vgpr(4), Vgpr(2));
  AppendVop3(&code, 0x14b, 13, Vgpr(1), Vgpr(4), Vgpr(2));
  AppendVop3(&code, 0x151, 14, Vgpr(3), Vgpr(1), Vgpr(2));
  AppendVop3(&code, 0x154, 15, Vgpr(3), Vgpr(1), Vgpr(2));
  AppendVop3(&code, 0x157, 16, Vgpr(3), Vgpr(1), Vgpr(2));
  code.push_back(EncodeVop2(0x20, 18, Vgpr(1), 17));
  code.push_back(0x3f800000u);
  code.push_back(EncodeVop2(0x21, 19, Vgpr(1), 4));
  code.push_back(0x40000000u);
  code.push_back(EncodeVop2(0x03, 20, 249, 1));
  code.push_back(EncodeVop2Sdwa(0));
  AppendSMovLiteral(&code, 70, 0x40000000u);
  code.push_back(0x06088cf9u);
  code.push_back(0x868606f2u);
  AppendSMovLiteral(&code, 26, 0x40000000u);
  code.push_back(0x081034f9u);
  code.push_back(0x868606f2u);
  code.push_back(0xd51f8011u);
  code.push_back(0x00020d0bu);
  AppendVMovLiteral(&code, 22, 0xbf800000u);
  code.push_back(EncodeVop2(0x05, 23, 249, 26));
  code.push_back(EncodeVop2Sdwa(22, 6, 0, 6, 6, 0, 0, 0, 1, 0, 0, 0, 1));
  AppendVMovLiteral(&code, 24, 0x3f800000u);
  AppendVMovLiteral(&code, 25, 0x3f800000u);
  code.push_back(EncodeVop2(0x03, 26, 249, 25));
  code.push_back(EncodeVop2Sdwa(24, 6, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1));
  AppendVMovLiteral(&code, 27, 0xbf800000u);
  AppendVMovLiteral(&code, 28, 0x40000000u);
  AppendVop3(&code, 0x104, 29, Vgpr(27), Vgpr(28), 0, 1);
  AppendVop3(&code, 0x103, 30, Vgpr(0), Vgpr(1), 0, 0, 0, false, 1);
  AppendVop3(&code, 0x151, 32, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0, false, 1);
  AppendVMovLiteral(&code, 33, 0xbf800000u);
  AppendVMovLiteral(&code, 34, 0x40000000u);
  code.push_back(EncodeVop2(0x03, 35, 250, 34));
  code.push_back(EncodeVop2Dpp(33, 0, 0xf, 0xf, 0, 1));

  const u32 results[] = {5,  6,  7,  21, 9, 10, 11, 12, 13, 14, 15, 16,
                         18, 19, 20, 4,  8, 17, 23, 26, 29, 30, 32, 35};
  for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
    AppendStoreVgpr(&code, results[i], i);
  }
  AppendEnd(&code);

  return {"VectorFloatArithmeticOps",
          code,
          {},
          {0x40400000u, 0x40400000u, 0x40400000u, 0x41000000u, 0x40000000u,
           0x40a00000u, 0x41200000u, 0x41200000u, 0x41200000u, 0x40000000u,
           0x40a00000u, 0x40800000u, 0x40c00000u, 0x41000000u, 0x40400000u,
           0x40400000u, 0xbf800000u, 0x3f800000u, 0x3f800000u, 0x40800000u,
           0xbf800000u, 0x40c00000u, 0x40000000u, 0x40400000u},
          {O::S_MOV_B32, O::V_MOV_B32, O::V_ADD_F32, O::V_SUB_F32,
           O::V_SUBREV_F32, O::V_MUL_F32, O::V_MIN_F32, O::V_MAX_F32,
           O::V_MAC_F32, O::V_MAD_F32, O::V_FMA_F32, O::V_MIN3_F32,
           O::V_MAX3_F32, O::V_MED3_F32, O::V_MADMK_F32, O::V_MADAK_F32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorMinMaxF32NanAndSignedZeroEdges() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x40000000u);
  AppendVMovLiteral(&code, 1, 0x7fc00000u);
  AppendVMovLiteral(&code, 2, 0x00000000u);
  AppendVMovLiteral(&code, 3, 0x80000000u);
  AppendVMovLiteral(&code, 4, 0x40400000u);
  AppendVMovLiteral(&code, 5, 0x3f800000u);
  AppendVMovLiteral(&code, 6, 0x7fa00001u);

  code.push_back(EncodeVop2(0x0f, 10, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x10, 11, Vgpr(0), 1));
  code.push_back(EncodeVop2(0x0f, 12, Vgpr(3), 2));
  code.push_back(EncodeVop2(0x10, 13, Vgpr(2), 3));
  code.push_back(EncodeVop2(0x0f, 14, Vgpr(6), 0));
  code.push_back(EncodeVop2(0x10, 15, Vgpr(0), 6));
  AppendVop3(&code, 0x151, 16, Vgpr(0), Vgpr(1), Vgpr(4));
  AppendVop3(&code, 0x154, 17, Vgpr(0), Vgpr(1), Vgpr(5));

  for (u32 i = 0; i < 8; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  return {"VectorMinMaxF32NanAndSignedZeroEdges",
          code,
          {},
          {0x40000000u, 0x40000000u, 0x80000000u, 0x00000000u, 0x40000000u,
           0x40000000u, 0x40000000u, 0x40000000u},
          {O::V_MOV_B32, O::V_MIN_F32, O::V_MAX_F32, O::V_MIN3_F32,
           O::V_MAX3_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorMed3F32NanUsesMin3Path() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x40000000u);
  AppendVMovLiteral(&code, 1, 0x40400000u);
  AppendVMovLiteral(&code, 2, 0x7fa00001u);
  AppendVMovLiteral(&code, 3, 0x7fc00000u);

  AppendVop3(&code, 0x157, 10, Vgpr(0), Vgpr(1), Vgpr(2));
  AppendVop3(&code, 0x157, 11, Vgpr(0), Vgpr(1), Vgpr(3));
  AppendVop3(&code, 0x157, 12, Vgpr(2), Vgpr(0), Vgpr(1));

  AppendStoreVgpr(&code, 10, 0);
  AppendStoreVgpr(&code, 11, 1);
  AppendStoreVgpr(&code, 12, 2);
  AppendEnd(&code);

  return {"VectorMed3F32NanUsesMin3Path",
          code,
          {},
          {0x40000000u, 0x40000000u, 0x40000000u},
          {O::V_MOV_B32, O::V_MED3_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorFloatConversionOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0xfffffffdu);
  code.push_back(EncodeVop1(0x01, 1, InlineU32(7)));
  AppendVMovLiteral(&code, 2, 0x40e00000u);
  AppendVMovLiteral(&code, 3, 0xc0400000u);
  AppendVMovLiteral(&code, 4, 0x3f800000u);
  AppendVMovLiteral(&code, 5, 0x00003c00u);
  AppendVMovLiteral(&code, 6, 0x40700000u);
  AppendVMovLiteral(&code, 7, 0x44332211u);
  AppendVMovLiteral(&code, 8, 0x40000000u);
  AppendVMovLiteral(&code, 9, 0x40800000u);
  AppendVMovLiteral(&code, 10, 0x40100000u);
  AppendVMovLiteral(&code, 11, 0x40300000u);
  code.push_back(EncodeVop1(0x01, 12, InlineU32(0)));
  code.push_back(EncodeVop1(0x01, 37, InlineU32(15)));

  code.push_back(EncodeVop1(0x05, 13, Vgpr(0)));
  code.push_back(EncodeVop1(0x06, 14, Vgpr(1)));
  code.push_back(EncodeVop1(0x07, 15, Vgpr(2)));
  code.push_back(EncodeVop1(0x08, 16, Vgpr(3)));
  code.push_back(EncodeVop1(0x0a, 17, Vgpr(4)));
  code.push_back(EncodeVop1(0x0b, 18, Vgpr(5)));
  code.push_back(EncodeVop1(0x0c, 42, Vgpr(11)));
  code.push_back(EncodeVop1(0x0d, 19, Vgpr(6)));
  code.push_back(EncodeVop1(0x11, 20, Vgpr(7)));
  code.push_back(EncodeVop1(0x12, 21, Vgpr(7)));
  code.push_back(EncodeVop1(0x13, 22, Vgpr(7)));
  code.push_back(EncodeVop1(0x14, 23, Vgpr(7)));
  code.push_back(EncodeVop1(0x2a, 24, Vgpr(8)));
  code.push_back(EncodeVop1(0x20, 25, Vgpr(10)));
  code.push_back(EncodeVop1(0x21, 26, Vgpr(11)));
  code.push_back(EncodeVop1(0x22, 27, Vgpr(10)));
  code.push_back(EncodeVop1(0x23, 28, Vgpr(10)));
  code.push_back(EncodeVop1(0x24, 29, Vgpr(11)));
  code.push_back(EncodeVop1(0x25, 30, Vgpr(4)));
  code.push_back(EncodeVop1(0x27, 36, Vgpr(9)));
  code.push_back(EncodeVop1(0x2e, 32, Vgpr(9)));
  code.push_back(EncodeVop1(0x33, 33, Vgpr(9)));
  code.push_back(EncodeVop1(0x35, 34, Vgpr(12)));
  code.push_back(EncodeVop1(0x36, 35, Vgpr(12)));
  code.push_back(EncodeVop1(0x0e, 38, Vgpr(37)));
  code.push_back(EncodeVop1(0x2a, 39, 249));
  code.push_back(EncodeVop1Sdwa(8, 6, 0, 6, 0, 0, 0, 0, 0, 1));
  AppendVMovLiteral(&code, 40, 0xc0000000u);
  AppendVop3(&code, 0x1aa, 41, Vgpr(40), 0, 0, 1, 0, false, 1);
  AppendVMovLiteral(&code, 2, 0xc0003c00u); // low=1.0h, high=-2.0h
  code.push_back(0x7e1016f9u);
  code.push_back(0x00250602u);

  const u32 results[] = {13, 14, 15, 16, 17, 18, 42, 19, 20, 21, 22, 23, 24, 25,
                         26, 27, 28, 29, 30, 36, 32, 33, 34, 35, 38, 39, 41, 8};
  for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
    AppendStoreVgpr(&code, results[i], i);
  }
  AppendEnd(&code);

  return {"VectorFloatConversionOps",
          code,
          {},
          {0xc0400000u, 0x40e00000u, 7,           0xfffffffdu, 0x00003c00u,
           0x3f800000u, 3,           3,           0x41880000u, 0x42080000u,
           0x424c0000u, 0x42880000u, 0x3f000000u, 0x3e800000u, 0x40000000u,
           0x40400000u, 0x40000000u, 0x40000000u, 0x40000000u, 0x40000000u,
           0x3f000000u, 0x40000000u, 0,           0x3f800000u, 0xbd800000u,
           0x3f800000u, 0x3f800000u, 0x40000000u},
          {O::V_MOV_B32,        O::V_CVT_F32_I32,     O::V_CVT_F32_U32,
           O::V_CVT_U32_F32,    O::V_CVT_I32_F32,     O::V_CVT_F16_F32,
           O::V_CVT_F32_F16,    O::V_CVT_RPI_I32_F32, O::V_CVT_FLR_I32_F32,
           O::V_CVT_OFF_F32_I4, O::V_CVT_F32_UBYTE0,  O::V_CVT_F32_UBYTE1,
           O::V_CVT_F32_UBYTE2, O::V_CVT_F32_UBYTE3,  O::V_RCP_F32,
           O::V_FRACT_F32,      O::V_TRUNC_F32,       O::V_CEIL_F32,
           O::V_RNDNE_F32,      O::V_FLOOR_F32,       O::V_EXP_F32,
           O::V_LOG_F32,        O::V_RSQ_F32,         O::V_SQRT_F32,
           O::V_SIN_F32,        O::V_COS_F32,         O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase VectorFrexpF32Edges() {
  using O = ShaderOpcode;

  const std::vector<u32> inputs = {
      0x00000000u, 0x80000000u, 0x3f000000u, 0xbf800000u, 0x40800000u,
      0xc1000000u, 0x00800000u, 0x007fffffu, 0x00000001u, 0x7f800000u,
      0x7fc00000u, 0xff800000u, 0x7fa00001u,
  };
  std::vector<u32> code;
  for (u32 i = 0; i < static_cast<u32>(inputs.size()); i++) {
    AppendVMovU32(&code, 31, i * sizeof(u32));
    AppendBufferLoadDword(&code, i == 3u ? 23u : i, 31);
  }
  for (u32 i = 0; i < static_cast<u32>(inputs.size()); i++) {
    const auto exponent_dst = i == 3u ? 19u : 32u + i;
    const auto mantissa_dst = i == 3u ? 21u : 48u + i;
    if (i == 3u) {
      code.push_back(0x7e267ef9u); // captured v_frexp_exp_i32_f32 v19, abs(v23)
      code.push_back(0x00260617u);
    } else {
      code.push_back(EncodeVop1(0x3f, exponent_dst, Vgpr(i)));
    }
    AppendStoreVgpr(&code, exponent_dst, i);
    if (i == 3u) {
      code.push_back(0x7e2a80f9u); // captured v_frexp_mant_f32 v21, abs(v23)
      code.push_back(0x00260617u);
    } else {
      code.push_back(EncodeVop1(0x40, mantissa_dst, Vgpr(i)));
    }
    AppendStoreVgpr(&code, mantissa_dst, inputs.size() + i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorFrexpF32Edges";
  test.code = std::move(code);
  test.initial = inputs;
  test.initial.resize(inputs.size() * 2u);
  test.expected = {
      0u,          0u,          0u,          1u,          3u,
      4u,          0xffffff83u, 0xffffff82u, 0xffffff6cu, 0u,
      0u,          0u,          0u,          0x00000000u, 0x80000000u,
      0x3f000000u, 0x3f000000u, 0x3f000000u, 0xbf000000u, 0x3f000000u,
      0x3f7ffffeu, 0x3f000000u, 0x7f800000u, 0x7fc00000u, 0xff800000u,
      0x7fa00001u,
  };
  test.opcodes = {O::V_MOV_B32,           O::BUFFER_LOAD_DWORD,
                  O::V_FREXP_EXP_I32_F32, O::V_FREXP_MANT_F32,
                  O::BUFFER_STORE_DWORD,  O::S_ENDPGM};
  test.decoded_counts = {{"V_FREXP_EXP_I32_F32", inputs.size()},
                         {"V_FREXP_MANT_F32", inputs.size()}};
  test.required_spirv = {"FindUMsb", "OpBitFieldUExtract",
                         "OpShiftLeftLogical"};
  return test;
}

TestCase CvtF32ToIntSaturatesNaNAndOutOfRange() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x7fc00000u);
  AppendVMovLiteral(&code, 1, 0xbf800000u);
  AppendVMovLiteral(&code, 2, 0x7f800000u);
  AppendVMovLiteral(&code, 3, 0x4f800000u);
  AppendVMovLiteral(&code, 4, 0xff800000u);
  AppendVMovLiteral(&code, 5, 0x4f000000u);

  code.push_back(EncodeVop1(0x07, 10, Vgpr(0)));
  code.push_back(EncodeVop1(0x07, 11, Vgpr(1)));
  code.push_back(EncodeVop1(0x07, 12, Vgpr(2)));
  code.push_back(EncodeVop1(0x07, 13, Vgpr(3)));
  code.push_back(EncodeVop1(0x08, 14, Vgpr(0)));
  code.push_back(EncodeVop1(0x08, 15, Vgpr(2)));
  code.push_back(EncodeVop1(0x08, 16, Vgpr(4)));
  code.push_back(EncodeVop1(0x08, 17, Vgpr(5)));

  for (u32 i = 0; i < 8; i++) {
    AppendStoreVgpr(&code, 10 + i, i);
  }
  AppendEnd(&code);

  return {"CvtF32ToIntSaturatesNaNAndOutOfRange",
          code,
          {},
          {0, 0, 0xffffffffu, 0xffffffffu, 0, 0x7fffffffu, 0x80000000u,
           0x7fffffffu},
          {O::V_MOV_B32, O::V_CVT_U32_F32, O::V_CVT_I32_F32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorSpecialF32FlushesDenormalInputs() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x00000001u);
  code.push_back(EncodeVop1(0x27, 1, Vgpr(0)));
  code.push_back(EncodeVop1(0x2a, 2, Vgpr(0)));
  code.push_back(EncodeVop1(0x2e, 3, Vgpr(0)));
  code.push_back(EncodeVop1(0x33, 4, Vgpr(0)));
  AppendStoreVgpr(&code, 1, 0);
  AppendStoreVgpr(&code, 2, 1);
  AppendStoreVgpr(&code, 3, 2);
  AppendStoreVgpr(&code, 4, 3);
  AppendEnd(&code);

  return {"VectorSpecialF32FlushesDenormalInputs",
          code,
          {},
          {0xff800000u, 0x7f800000u, 0x7f800000u, 0x00000000u},
          {O::V_MOV_B32, O::V_LOG_F32, O::V_RCP_F32, O::V_RSQ_F32,
           O::V_SQRT_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorCompareInteger64Edges() {
  using O = ShaderOpcode;
  constexpr std::array<std::array<uint64_t, 2>, 10> pairs{{
      {0, 0}, {0, 1}, {1, 0}, {0xffffffffull, 0x100000000ull},
      {0x100000000ull, 0xffffffffull}, {0x8000000000000000ull, 0},
      {0x7fffffffffffffffull, 0x8000000000000000ull},
      {0xffffffffffffffffull, 0xffffffffffffffffull},
      {0xffffffff80000000ull, 0xffffffff7fffffffull},
      {0x1234567880000000ull, 0x123456787fffffffull}}};
  TestCase test;
  test.name = "VectorCompareInteger64Edges";
  for (const auto& pair : pairs)
    for (const auto bits : pair)
      test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
  test.expected = test.initial;
  const auto store_mask = [&](u32 sgpr, bool value) {
    AppendStoreSgprPair(&test.code, sgpr, static_cast<u32>(test.expected.size()));
    test.expected.insert(test.expected.end(), {u32(value), 0});
  };
  for (u32 i = 0; i < pairs.size(); ++i) {
    for (u32 word = 0; word < 4; ++word) {
      AppendVMovU32(&test.code, 31, (i * 4 + word) * 4);
      AppendBufferLoadDword(&test.code, 1 + word, 31);
    }
    test.code.push_back(EncodeVop1(0x02, 8, Vgpr(3)));
    test.code.push_back(EncodeVop1(0x02, 9, Vgpr(4)));
    const auto lhs = pairs[i][0], rhs = pairs[i][1];
    const auto signed_lhs = std::bit_cast<int64_t>(lhs);
    const auto signed_rhs = std::bit_cast<int64_t>(rhs);
    for (const auto [opcode, value] : std::array<std::pair<u32, bool>, 11>{{
             {0xa0, false}, {0xa1, signed_lhs < signed_rhs},
             {0xa3, signed_lhs <= signed_rhs}, {0xa4, signed_lhs > signed_rhs},
             {0xa5, lhs != rhs}, {0xa6, signed_lhs >= signed_rhs}, {0xa7, true},
             {0xe0, false}, {0xe3, lhs <= rhs}, {0xe6, lhs >= rhs}, {0xe7, true}}}) {
      test.code.push_back(EncodeVopc(opcode, Vgpr(1), 3));
      store_mask(106, value);
      AppendVop3(&test.code, opcode, 20, Vgpr(1), 8);
      store_mask(20, value);
    }
    // A negative literal expands by signedness, including equality comparisons.
    for (const auto [opcode, value] : std::array<std::pair<u32, bool>, 7>{{
             {0xa1, -1 < signed_rhs}, {0xa3, -1 <= signed_rhs},
             {0xa4, -1 > signed_rhs}, {0xa5, -1 != signed_rhs},
             {0xa6, -1 >= signed_rhs},
             {0xa2, -1 == signed_rhs}, {0xe3, 0xffffffffull <= rhs}}}) {
      test.code.push_back(EncodeVopc(opcode, 255, 3));
      test.code.push_back(0xffffffffu);
      store_mask(106, value);
      if (opcode == 0xa4 || opcode == 0xa6) {
        test.code.push_back(EncodeSop1(0x04, 10, 126));
        test.code.push_back(EncodeVopc(opcode + 0x10, 255, 3));
        test.code.push_back(0xffffffffu);
        test.code.push_back(EncodeSop1(0x04, 20, 126));
        test.code.push_back(EncodeSop1(0x04, 126, 10));
        store_mask(20, value);
      }
    }
    for (const auto [opcode, value] : std::array<std::pair<u32, bool>, 3>{{
             {0xa3, signed_lhs <= -1}, {0xa4, signed_lhs > -1},
             {0xa6, signed_lhs >= -1}}}) {
      for (const u32 source : {255u, 193u}) { // Literal and inline -1.
        AppendVop3(&test.code, opcode, 20, Vgpr(1), source);
        if (source == 255) test.code.push_back(0xffffffffu);
        store_mask(20, value);
        if (opcode == 0xa4 || opcode == 0xa6) {
          test.code.push_back(EncodeSop1(0x04, 10, 126));
          AppendVop3(&test.code, opcode + 0x10, 22, Vgpr(1), source);
          if (source == 255) test.code.push_back(0xffffffffu);
          test.code.push_back(EncodeSop1(0x04, 20, 126));
          test.code.push_back(EncodeSop1(0x04, 126, 10));
          store_mask(20, value);
        }
      }
    }
    test.code.push_back(EncodeSop1(0x04, 10, 126));
    AppendVop3(&test.code, 0xb2, 126, 255, Vgpr(3)); // CMPX_EQ_I64 sign-extends -1.
    test.code.push_back(0xffffffffu);
    test.code.push_back(EncodeSop1(0x04, 20, 126));
    test.code.push_back(EncodeSop1(0x04, 126, 10));
    store_mask(20, -1 == signed_rhs);
    // Captured 8395e43f382309df PC 0x20cc: s[2:3] = (s[0:1] != 0) & EXEC.
    // Save the test buffer descriptor while its SGPRs hold the captured operands.
    test.code.push_back(EncodeSop1(0x04, 32, 0));
    test.code.push_back(EncodeSop1(0x04, 34, 2));
    test.code.push_back(EncodeVop1(0x02, 0, Vgpr(1)));
    test.code.push_back(EncodeVop1(0x02, 1, Vgpr(2)));
    test.code.insert(test.code.end(), {0xd4a50002u, 0x00010000u});
    test.code.push_back(EncodeSop1(0x04, 20, 2));
    test.code.push_back(EncodeSop1(0x04, 0, 32));
    test.code.push_back(EncodeSop1(0x04, 2, 34));
    store_mask(20, lhs != 0);
  }
  AppendEnd(&test.code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
                  O::V_CMP_F_I64, O::V_CMP_LT_I64, O::V_CMP_LE_I64, O::V_CMP_EQ_I64,
                  O::V_CMP_GT_I64, O::V_CMP_NE_I64, O::V_CMP_GE_I64,
                  O::V_CMP_T_I64, O::V_CMPX_EQ_I64, O::V_CMPX_GT_I64, O::V_CMPX_GE_I64,
                  O::V_CMP_F_U64, O::V_CMP_LE_U64, O::V_CMP_GE_U64, O::V_CMP_T_U64,
                  O::S_MOV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpSLessThan", "OpULessThanEqual", "OpUGreaterThanEqual"};
  return test;
}

TestCase VectorCompareExecWaveMasks(u32 wave_size) {
  using O = ShaderOpcode;
  // Rows exercise signed 16-bit truncation, full-width integer ordering, and
  // ordered/unordered FP64 comparisons (including NaN, signed zero, and subnormals).
  constexpr std::array<std::array<std::array<uint64_t, 2>, 8>, 3> inputs{{
      {{{0x1111ffff, 0x22220000}, {0x33338000, 0x44447fff},
        {0x55550000, 0x66660000}, {0x77777fff, 0x88888000},
        {0x99990001, 0xaaaa0002}, {0xbbbb0002, 0xcccc0001},
        {0xddddffff, 0xeeeeffff}, {0xffff8001, 0x11118000}}},
      {{{0, 0}, {0, 1}, {1, 0}, {0xffffffffull, 0x100000000ull},
        {0x100000000ull, 0xffffffffull}, {0x8000000000000000ull, 0},
        {0xffffffffffffffffull, 0xffffffffffffffffull},
        {0x1234567880000000ull, 0x123456787fffffffull}}},
      {{{0, 0x8000000000000000ull}, {1, 0}, {0x8000000000000001ull, 0},
        {0x7ff8000000000001ull, 0}, {0, 0x7ff0000000000001ull},
        {0xc000000000000000ull, 0xbff0000000000000ull},
        {0x7ff0000000000000ull, 0x7ff0000000000000ull},
        {0x3ff0000000000001ull, 0x3ff0000000000000ull}}}}};
  struct Compare { u32 encoding; O opcode; u32 group; bool update_exec{true}; };
  constexpr std::array comparisons{
      Compare{0x99, O::V_CMPX_LT_I16, 0}, Compare{0x9a, O::V_CMPX_EQ_I16, 0},
      Compare{0x9b, O::V_CMPX_LE_I16, 0}, Compare{0x9c, O::V_CMPX_GT_I16, 0},
      Compare{0x9d, O::V_CMPX_NE_I16, 0}, Compare{0x9e, O::V_CMPX_GE_I16, 0},
      Compare{0xb0, O::V_CMPX_F_I64, 1}, Compare{0xb1, O::V_CMPX_LT_I64, 1},
      Compare{0xb2, O::V_CMPX_EQ_I64, 1}, Compare{0xb3, O::V_CMPX_LE_I64, 1},
      Compare{0xb4, O::V_CMPX_GT_I64, 1}, Compare{0xb6, O::V_CMPX_GE_I64, 1},
      Compare{0xa4, O::V_CMP_GT_I64, 1, false},
      Compare{0xa6, O::V_CMP_GE_I64, 1, false},
      Compare{0xb7, O::V_CMPX_T_I64, 1}, Compare{0xf0, O::V_CMPX_F_U64, 1},
      Compare{0xf1, O::V_CMPX_LT_U64, 1}, Compare{0xf2, O::V_CMPX_EQ_U64, 1},
      Compare{0xf3, O::V_CMPX_LE_U64, 1}, Compare{0xf4, O::V_CMPX_GT_U64, 1},
      Compare{0xf6, O::V_CMPX_GE_U64, 1}, Compare{0xf7, O::V_CMPX_T_U64, 1},
      Compare{0x30, O::V_CMPX_F_F64, 2}, Compare{0x3f, O::V_CMPX_TRU_F64, 2},
      Compare{0x33, O::V_CMPX_LE_F64, 2}, Compare{0x36, O::V_CMPX_GE_F64, 2},
      Compare{0x31, O::V_CMPX_LT_F64, 2}, Compare{0x32, O::V_CMPX_EQ_F64, 2},
      Compare{0x34, O::V_CMPX_GT_F64, 2}, Compare{0x35, O::V_CMPX_LG_F64, 2},
      Compare{0x37, O::V_CMPX_O_F64, 2},  Compare{0x38, O::V_CMPX_U_F64, 2},
      Compare{0x39, O::V_CMPX_NGE_F64, 2}, Compare{0x3a, O::V_CMPX_NLG_F64, 2},
      Compare{0x3b, O::V_CMPX_NGT_F64, 2}, Compare{0x3c, O::V_CMPX_NLE_F64, 2},
      Compare{0x3d, O::V_CMPX_NEQ_F64, 2}, Compare{0x3e, O::V_CMPX_NLT_F64, 2}};
  constexpr u32 low_exec = 0xeeeeeeeeu, high_exec = 0x77777777u;
  constexpr u32 vcc_lo = 0x12345678u, vcc_hi = 0x89abcdefu;
  TestCase test;
  test.name = wave_size == 32 ? "VectorCompareExecWave32Masks" : "VectorCompareExecWave64Masks";
  test.initial.assign(12 * wave_size, 0);
  for (u32 group = 0; group < inputs.size(); ++group)
    for (u32 lane = 0; lane < wave_size; ++lane)
      for (u32 word = 0; word < 4; ++word)
        test.initial[(group * 4 + word) * wave_size + lane] =
            u32(inputs[group][lane % 8][word / 2] >> (32 * (word % 2)));
  test.expected = test.initial;
  auto& code = test.code;
  code.push_back(EncodeSop1(0x04, 10, 126));
  for (const auto compare : comparisons) {
    for (u32 word = 0; word < 4; ++word) {
      code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
      AppendVMovU32(&code, 31, (compare.group * 4 + word) * wave_size * 4);
      code.push_back(EncodeVop2(0x25, 31, Vgpr(31), 30));
      AppendBufferLoadDword(&code, 1 + word, 31);
    }
    std::array<u32, 2> expected_exec{};
    for (u32 lane = 0; lane < wave_size; ++lane) {
      const auto pair = inputs[compare.group][lane % 8];
      const auto a = std::bit_cast<int16_t>(uint16_t(pair[0]));
      const auto b = std::bit_cast<int16_t>(uint16_t(pair[1]));
      const auto lhs = std::bit_cast<double>(pair[0]);
      const auto rhs = std::bit_cast<double>(pair[1]);
      bool result = false;
      switch (compare.encoding) {
        case 0x99: result = a < b; break;
        case 0x9a: result = a == b; break;
        case 0x9b: result = a <= b; break;
        case 0x9c: result = a > b; break;
        case 0x9d: result = a != b; break;
        case 0x9e: result = a >= b; break;
        case 0x30: case 0xb0: case 0xf0: result = false; break;
        case 0xb1: result = std::bit_cast<int64_t>(pair[0]) < std::bit_cast<int64_t>(pair[1]); break;
        case 0xb2: case 0xf2: result = pair[0] == pair[1]; break;
        case 0xb3: result = std::bit_cast<int64_t>(pair[0]) <= std::bit_cast<int64_t>(pair[1]); break;
        case 0xa4: case 0xb4: result = std::bit_cast<int64_t>(pair[0]) > std::bit_cast<int64_t>(pair[1]); break;
        case 0xa6: case 0xb6: result = std::bit_cast<int64_t>(pair[0]) >= std::bit_cast<int64_t>(pair[1]); break;
        case 0x3f: case 0xb7: case 0xf7: result = true; break;
        case 0xf1: result = pair[0] < pair[1]; break;
        case 0xf3: result = pair[0] <= pair[1]; break;
        case 0xf4: result = pair[0] > pair[1]; break;
        case 0xf6: result = pair[0] >= pair[1]; break;
        case 0x33: result = lhs <= rhs; break;
        case 0x36: result = lhs >= rhs; break;
        case 0x31: result = lhs < rhs; break;
        case 0x32: result = lhs == rhs; break;
        case 0x34: result = lhs > rhs; break;
        case 0x35: result = lhs < rhs || lhs > rhs; break;
        case 0x37: result = !std::isnan(lhs) && !std::isnan(rhs); break;
        case 0x38: result = std::isnan(lhs) || std::isnan(rhs); break;
        // The N* predicates are the logical negation of their ordered counterpart,
        // which is true for any comparison involving a NaN (IEEE unordered compares).
        case 0x39: result = !(lhs >= rhs); break;
        case 0x3a: result = !(lhs < rhs || lhs > rhs); break;
        case 0x3b: result = !(lhs > rhs); break;
        case 0x3c: result = !(lhs <= rhs); break;
        case 0x3d: result = !(lhs == rhs); break;
        case 0x3e: result = !(lhs < rhs); break;
      }
      if (result) expected_exec[lane / 32] |= 1u << (lane % 32);
    }
    expected_exec[0] &= low_exec;
    expected_exec[1] &= high_exec;
    for (const bool vop3 : {false, true}) {
      AppendSMovLiteral(&code, 106, vcc_lo);
      AppendSMovLiteral(&code, 107, vcc_hi);
      AppendSMovLiteral(&code, 22, vcc_lo);
      AppendSMovLiteral(&code, 23, vcc_hi);
      AppendSMovLiteral(&code, 126, low_exec);
      if (wave_size == 64) AppendSMovLiteral(&code, 127, high_exec);
      if (vop3) AppendVop3(&code, compare.encoding, 22, Vgpr(1), Vgpr(3));
      else code.push_back(EncodeVopc(compare.encoding, Vgpr(1), 3));
      code.push_back(EncodeSMovB32(20, 126));
      code.push_back(EncodeSMovB32(21, 127));
      code.push_back(EncodeSop1(0x04, 126, 10));
      const bool vcc_dest = !compare.update_exec && !vop3;
      const bool sgpr_dest = !compare.update_exec && vop3;
      // Wave32 writes only the low half of VCC or the selected SGPR pair.
      const u32 mask_hi = wave_size == 64 ? expected_exec[1] : vcc_hi;
      for (const auto [reg, value] : std::array<std::pair<u32, u32>, 6>{{
               {20, compare.update_exec ? expected_exec[0] : low_exec},
               {21, compare.update_exec ? expected_exec[1] : (wave_size == 64 ? high_exec : 0)},
               {106, vcc_dest ? expected_exec[0] : vcc_lo},
               {107, vcc_dest ? mask_hi : vcc_hi},
               {22, sgpr_dest ? expected_exec[0] : vcc_lo},
               {23, sgpr_dest ? mask_hi : vcc_hi}}}) {
        AppendStoreSgprAtLaneDwordOffset(&code, reg, 0, u32(test.expected.size()));
        test.expected.insert(test.expected.end(), wave_size, value);
      }
    }
    test.opcodes.push_back(compare.opcode);
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  test.opcodes.insert(test.opcodes.end(), {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64,
      O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD,
      O::BUFFER_STORE_DWORD, O::S_ENDPGM});
  test.required_spirv = {"OpFOrdLessThanEqual", "OpFOrdGreaterThanEqual", "OpFOrdLessThan",
                         "OpFOrdGreaterThan",   "OpFOrdEqual",           "OpFOrdNotEqual",
                         "OpFUnordEqual",       "OpFUnordNotEqual",      "OpFUnordLessThan",
                         "OpFUnordGreaterThan", "OpFUnordLessThanEqual", "OpFUnordGreaterThanEqual"};
  return test;
}

TestCase VectorCompareF64Edges() {
  using O = ShaderOpcode;
  constexpr std::array<std::array<uint64_t, 2>, 16> pairs{{
      {0xc000000000000000ull, 0xbff0000000000000ull},
      {0x8000000000000001ull, 0}, {0x3fc45f306dc9c882ull, 0x3fc45f306dc9c882ull},
      {0, 0x8000000000000000ull}, {1, 0}, {1, 1}, {1, 2},
      {0x11d04c7b00ull, 0}, {0x3ff0000000000000ull, 0x3ff0000000000001ull},
      {0xbff0000000000000ull, 0xbff0000000000000ull},
      {0x7ff0000000000000ull, 0x7ff0000000000000ull},
      {0x7ff0000000000000ull, 0xfff0000000000000ull},
      {0x7ff8000000000001ull, 0x7ff8000000000001ull},
      {0x7ff0000000000001ull, 0}, {0, 0x7ff8000000000001ull},
      {0x7fefffffffffffffull, 0x7fefffffffffffffull}}};
  TestCase test;
  test.name = "VectorCompareF64Edges";
  for (const auto& pair : pairs)
    for (const auto bits : pair)
      test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
  test.expected = test.initial;
  const auto store_mask = [&](u32 sgpr, bool value) {
    AppendStoreSgprPair(&test.code, sgpr, static_cast<u32>(test.expected.size()));
    test.expected.insert(test.expected.end(), {u32(value), 0});
  };
  for (u32 i = 0; i < pairs.size(); ++i) {
    for (u32 word = 0; word < 4; ++word) {
      AppendVMovU32(&test.code, 31, (i * 4 + word) * 4);
      AppendBufferLoadDword(&test.code, 1 + word, 31);
    }
    const auto lhs = std::bit_cast<double>(pairs[i][0]);
    const auto rhs = std::bit_cast<double>(pairs[i][1]);
    for (const bool vop3 : {false, true}) {
      const auto a = vop3 ? -std::abs(lhs) : lhs;
      const auto b = vop3 ? -std::abs(rhs) : rhs;
      const bool ordered = !std::isnan(a) && !std::isnan(b);
      // Predicates 0x20..0x2f; N* is the negation of its ordered counterpart.
      const std::array results{false, a < b, a == b, a <= b, a > b,
          a < b || a > b, a >= b, ordered, !ordered, !(a >= b),
          !(a < b || a > b), !(a > b), !(a <= b), !(a == b), !(a < b), true};
      for (u32 predicate = 0; predicate < results.size(); ++predicate) {
        if (vop3)
          AppendVop3(&test.code, 0x20 + predicate, 20, Vgpr(1), Vgpr(3),
                     0, 3, 0, false, 0, 3);
        else
          test.code.push_back(EncodeVopc(0x20 + predicate, Vgpr(1), 3));
        store_mask(vop3 ? 20 : 106, results[predicate]);
      }
    }
    test.code.push_back(EncodeVopc(0x23, 255, 3));
    test.code.push_back(0x3ff00000u); // FP64 literals supply the high word.
    store_mask(106, 1.0 <= rhs);
    AppendVop3(&test.code, 0x23, 20, 248, Vgpr(3));
    store_mask(20, std::bit_cast<double>(uint64_t{0x3fc45f306dc9c882ull}) <= rhs);
    test.code.push_back(EncodeVop1(0x02, 100, Vgpr(3)));
    test.code.push_back(EncodeVop1(0x02, 101, Vgpr(4)));
    // Captured e6d76d24f59f8015 PC 0xe4: compare zero with s100:s101.
    test.code.insert(test.code.end(), {0xd422006au, 0x0000c880u});
    store_mask(106, 0.0 == rhs);
    AppendVop3(&test.code, 0x23, 20, Vgpr(1), 100);
    store_mask(20, lhs <= rhs);
  }
  AppendEnd(&test.code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
                  O::V_CMP_F_F64, O::V_CMP_LT_F64, O::V_CMP_EQ_F64, O::V_CMP_LE_F64,
                  O::V_CMP_GT_F64, O::V_CMP_LG_F64, O::V_CMP_GE_F64, O::V_CMP_O_F64,
                  O::V_CMP_U_F64, O::V_CMP_NGE_F64, O::V_CMP_NLG_F64, O::V_CMP_NGT_F64,
                  O::V_CMP_NLE_F64, O::V_CMP_NEQ_F64, O::V_CMP_NLT_F64, O::V_CMP_TRU_F64,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "OpIsNan",
                         "OpFOrdEqual", "OpFOrdNotEqual", "OpFOrdLessThan",
                         "OpFOrdLessThanEqual", "OpFOrdGreaterThan", "OpFOrdGreaterThanEqual",
                         "OpFUnordEqual", "OpFUnordNotEqual", "OpFUnordLessThan",
                         "OpFUnordLessThanEqual", "OpFUnordGreaterThan", "OpFUnordGreaterThanEqual"};
  return test;
}

TestCase VectorCompare64WaveMasks(u32 wave_size, bool integer_ne = false) {
  using O = ShaderOpcode;
  constexpr std::array<uint64_t, 4> values{0, 1, 0x8000000000000000ull,
                                          0x7ff8000000000000ull};
  constexpr u32 low_exec = 0xeeeeeeeeu, high_exec = 0x77777777u;
  constexpr u32 vcc_hi = 0x89abcdefu;
  constexpr u32 vcc_lo = 0x12345678u;
  const u32 result_reg = integer_ne ? 20 : 106;
  const u32 result_bits = integer_ne ? 0xeeeeeeeeu : 0x55555555u;
  TestCase test;
  test.name = integer_ne
      ? (wave_size == 32 ? "VectorCompareNeI64Wave32Masks" : "VectorCompareNeI64Wave64Masks")
      : (wave_size == 32 ? "VectorCompareF64Wave32Masks" : "VectorCompareF64Wave64Masks");
  test.initial.assign((integer_ne ? 8 : 6) * wave_size, 0);
  for (u32 lane = 0; lane < wave_size; ++lane) {
    test.initial[lane] = u32(values[lane % 4]);
    test.initial[wave_size + lane] = u32(values[lane % 4] >> 32);
  }
  test.expected = test.initial;
  for (u32 lane = 0; lane < wave_size; ++lane) {
    test.expected[2 * wave_size + lane] = low_exec & result_bits;
    test.expected[3 * wave_size + lane] = wave_size == 64 ? high_exec & result_bits : vcc_hi;
    test.expected[4 * wave_size + lane] = low_exec;
    test.expected[5 * wave_size + lane] = wave_size == 64 ? high_exec : 0;
    if (integer_ne) {
      test.expected[6 * wave_size + lane] = vcc_lo;
      test.expected[7 * wave_size + lane] = vcc_hi;
    }
  }
  auto& code = test.code;
  code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
  AppendBufferLoadDword(&code, 1, 30);
  AppendVMovU32(&code, 31, wave_size * 4);
  code.push_back(EncodeVop2(0x25, 31, Vgpr(31), 30));
  AppendBufferLoadDword(&code, 2, 31);
  code.push_back(EncodeSop1(0x04, 10, 126));
  AppendSMovLiteral(&code, result_reg + 1, vcc_hi);
  if (integer_ne) {
    AppendSMovLiteral(&code, 106, vcc_lo);
    AppendSMovLiteral(&code, 107, vcc_hi);
  }
  AppendSMovLiteral(&code, 126, low_exec);
  if (wave_size == 64) AppendSMovLiteral(&code, 127, high_exec);
  AppendVop3(&code, integer_ne ? 0xa5 : 0x22, result_reg, InlineU32(0), Vgpr(1));
  code.push_back(EncodeSMovB32(22, 126));
  code.push_back(EncodeSMovB32(23, 127));
  code.push_back(EncodeSop1(0x04, 126, 10));
  for (const auto [reg, offset] : std::array<std::pair<u32, u32>, 4>{{
           {result_reg, 2}, {result_reg + 1, 3}, {22, 4}, {23, 5}}})
    AppendStoreSgprAtLaneDwordOffset(&code, reg, 0, offset * wave_size);
  if (integer_ne) {
    AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, 6 * wave_size);
    AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, 7 * wave_size);
  }
  AppendEnd(&code);
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_LSHLREV_B32,
                  O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD,
                  integer_ne ? O::V_CMP_NE_I64 : O::V_CMP_EQ_F64,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {integer_ne ? "OpINotEqual" : "OpFOrdEqual"};
  return test;
}

TestCase VectorF64CapturedScreenSpaceShadows() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "VectorF64CapturedScreenSpaceShadows";
  const auto store_pair = [&](u32 reg, double value) {
    const auto bits = std::bit_cast<uint64_t>(value);
    AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
    AppendStoreVgpr(&test.code, reg + 1,
                    static_cast<u32>(test.expected.size() + 1));
    test.expected.insert(test.expected.end(), {static_cast<u32>(bits),
                                               static_cast<u32>(bits >> 32)});
  };
  for (const auto values :
       {std::array<int32_t, 3>{63, 127, 101},
        std::array<int32_t, 3>{16777217, -16777217, 123456789},
        std::array<int32_t, 3>{INT32_MIN, INT32_MAX, -31}}) {
    AppendVMovLiteral(&test.code, 7, static_cast<u32>(values[0]));
    AppendVMovLiteral(&test.code, 1, static_cast<u32>(values[1]));
    AppendVMovLiteral(&test.code, 9, static_cast<u32>(values[2]));
    // Exact FP64 instruction words from b90e2024732c6111, PCs 0xc8 through
    // 0x130.
    test.code.insert(test.code.end(), {0x7e060907u, 0x7e0a0901u, 0x7e020909u});
    store_pair(3, double(values[0]));
    store_pair(5, double(values[1]));
    store_pair(1, double(values[2]));
    test.code.push_back(0x7e065f03u);
    const double reciprocal = 1.0 / double(values[0]);
    store_pair(3, reciprocal);
    test.code.insert(test.code.end(), {0xd5650005u, 0x00020305u});
    const double product = double(values[1]) * double(values[2]);
    store_pair(5, product);
    test.code.insert(test.code.end(), {0xd54c0001u, 0x84060705u});
    const double result = std::fma(product, reciprocal, -double(values[2]));
    store_pair(1, result);
    test.code.push_back(0x7e041f01u);
    AppendStoreVgpr(&test.code, 2, static_cast<u32>(test.expected.size()));
    test.expected.push_back(std::bit_cast<u32>(static_cast<float>(result)));
  }
  AppendEnd(&test.code);
  test.opcodes = {O::V_MOV_B32,          O::V_CVT_F64_I32, O::V_RCP_F64,
                  O::V_MUL_F64,          O::V_FMA_F64,     O::V_CVT_F32_F64,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpCapability Float64",
                         "OpTypeFloat 64",
                         "OpFDiv",
                         "Fma",
                         "SignedZeroInfNanPreserve 64"};
  test.ir_counts = {{"ConvertF64S32", 9},
                    {"FPRecip64", 3},
                    {"FPMul64", 3},
                    {"FPFma64", 3},
                    {"ConvertF32F64", 3}};

  ShaderComputeInputInfo info{};
  std::vector<u32> normal_key, other_key;
  BuildStageStaticKey(info, normal_key);
  info.float_mode = 0xc4;
  BuildStageStaticKey(info, other_key);
  Require(test.name, "pipeline identity",
          normal_key != other_key && normal_key.size() == other_key.size(),
          "FP64 rounding modes must not share a compiled shader key");
  return test;
}

TestCase VectorFractF64CapturedAndEdges() {
  using O = ShaderOpcode;
  constexpr std::array<std::array<uint64_t, 2>, 17> cases{{
      {0x400a000000000000ull, 0x3fd0000000000000ull}, // 3.25 -> .25
      {0xbff4000000000000ull, 0x3fe8000000000000ull}, // -1.25 -> .75
      {0x3ff0000000000001ull, 0x3cb0000000000000ull}, // 1 + 2^-52
      {0xbff0000000000001ull, 0x3feffffffffffffeull},
      {0x4320000000000001ull, 0x3fe0000000000000ull}, // 2^51 + .5
      {0, 0}, {0x8000000000000000ull, 0},
      {0x401c000000000000ull, 0}, {0xc01c000000000000ull, 0},
      {1, 1}, {0x8000000000000001ull, 0x3ff0000000000000ull},
      {0x000fffffffffffffull, 0x000fffffffffffffull},
      {0x7fefffffffffffffull, 0},
      {0x7ff0000000000000ull, 0x7ff8000000000000ull},
      {0xfff0000000000000ull, 0x7ff8000000000000ull},
      {0x7ff8123456789abcull, 0x7ff8000000000000ull},
      {0x7ff0000000000001ull, 0x7ff8000000000000ull}}};
  TestCase test;
  test.name = "VectorFractF64CapturedAndEdges";
  const auto store_pair = [&](u32 reg, uint64_t expected) {
    const auto index = static_cast<u32>(test.expected.size());
    AppendStoreVgpr(&test.code, reg, index);
    AppendStoreVgpr(&test.code, reg + 1, index + 1);
    test.expected.insert(test.expected.end(), {u32(expected), u32(expected >> 32)});
  };
  for (u32 i = 0; i < cases.size(); i++) {
    const auto [source, expected] = cases[i];
    test.initial.insert(test.initial.end(), {u32(source), u32(source >> 32)});
    for (u32 word = 0; word < 2; word++) {
      AppendVMovU32(&test.code, 30, (i * 2 + word) * 4);
      AppendBufferLoadDword(&test.code, 4 + word, 30);
    }
    test.code.push_back(0x7e087d04u); // Captured v_fract_f64 v[4:5], v[4:5].
    if ((source & 0x7ff0000000000000ull) == 0x7ff0000000000000ull) {
      // Classify quiet NaNs without requiring a particular payload or sign.
      AppendVMovU32(&test.code, 4, 0);
      AppendVMovLiteral(&test.code, 6, 0x7ff80000u);
      test.code.push_back(EncodeVop2(0x1b, 5, Vgpr(6), 5));
    }
    store_pair(4, expected);
  }
  AppendVMovU32(&test.code, 1, 0);
  AppendVMovLiteral(&test.code, 2, 0xbff40000u);
  AppendVop3(&test.code, 0x1be, 1, Vgpr(1), 0, 0, 1, 0, false, 0, 1);
  store_pair(1, 0x3fe8000000000000ull); // Odd pair; ABS precedes NEG.
  AppendVMovU32(&test.code, 4, 1);
  AppendVMovLiteral(&test.code, 5, 0x3ff00000u);
  test.code.push_back(EncodeSop1(0x04, 12, 126));
  test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  test.code.push_back(0x7e087d04u);
  test.code.push_back(EncodeSop1(0x04, 126, 12));
  store_pair(4, 0x3ff0000000000001ull); // Both words survive inactive EXEC.
  AppendEnd(&test.code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FRACT_F64,
                  O::V_AND_B32, O::S_MOV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_FRACT_F64 v4, v4", cases.size() + 1}};
  test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "Fract"};
  return test;
}

TestCase VectorMinMaxF64CapturedAndEdges() {
  using O = ShaderOpcode;
  // MODE.IEEE=0: select the numeric operand, or the second NaN unchanged.
  constexpr uint64_t positive_zero = 0, negative_zero = 0x8000000000000000ull;
  constexpr uint64_t one = 0x3ff0000000000000ull, minus_two = 0xc000000000000000ull;
  constexpr uint64_t qnan = 0x7ff8123456789abcull, snan = 0xfff0000000000042ull;
  constexpr uint64_t infinity = 0x7ff0000000000000ull;
  constexpr std::array<std::array<uint64_t, 4>, 20> cases{{
      {one, minus_two, minus_two, one}, {minus_two, one, minus_two, one},
      {one, one, one, one},
      {positive_zero, negative_zero, negative_zero, positive_zero},
      {negative_zero, positive_zero, negative_zero, positive_zero},
      {negative_zero, negative_zero, negative_zero, negative_zero},
      {positive_zero, positive_zero, positive_zero, positive_zero},
      {qnan, one, one, one}, {one, qnan, one, one},
      {snan, minus_two, minus_two, minus_two}, {minus_two, snan, minus_two, minus_two},
      {qnan, snan, snan, snan}, {snan, qnan, qnan, qnan},
      {infinity, one, one, infinity},
      {infinity | negative_zero, one, infinity | negative_zero, one},
      {0x3fe0000000000000ull, one, 0x3fe0000000000000ull, one},
      {1, negative_zero | 1, negative_zero | 1, 1}, {1, 0, 0, 1},
      {1, 2, 1, 2}, {2, 1, 1, 2}}};
  TestCase test;
  test.name = "VectorMinMaxF64CapturedAndEdges";
  for (const auto &item : cases) {
    for (u32 source = 0; source < 2; source++) {
      test.initial.insert(test.initial.end(), {u32(item[source]), u32(item[source] >> 32)});
    }
  }
  test.expected = test.initial;
  const auto store_pair = [&](u32 reg, uint64_t bits) {
    AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
    AppendStoreVgpr(&test.code, reg + 1, static_cast<u32>(test.expected.size() + 1));
    test.expected.insert(test.expected.end(), {u32(bits), u32(bits >> 32)});
  };
  for (u32 i = 0; i < cases.size(); i++) {
    for (u32 word = 0; word < 4; word++) {
      AppendVMovU32(&test.code, 30, (i * 4 + word) * 4);
      AppendBufferLoadDword(&test.code, 2 + word, 30);
    }
    AppendVop3(&test.code, 0x166, 7, Vgpr(2), Vgpr(4));
    store_pair(7, cases[i][2]);
    AppendVop3(&test.code, 0x167, 9, Vgpr(2), Vgpr(4));
    store_pair(9, cases[i][3]);
    // CS daced64c72398e80 PCs 0x10c/0x11c clamp the pair to [0, 1].
    test.code.insert(test.code.end(), {0xd5670006u, 0x00020480u,
                                       0xd5660006u, 0x00020cf2u});
    const auto source = std::bit_cast<double>(cases[i][0]);
    uint64_t clamped = 0;
    if (source > 0) clamped = source < 1 ? cases[i][0] : one;
    store_pair(6, clamped);
  }
  AppendSMovLiteral(&test.code, 8, 0);
  AppendSMovLiteral(&test.code, 9, u32(minus_two >> 32));
  // Scalar source and odd destination pairs; ABS must precede NEG.
  AppendVop3(&test.code, 0x166, 1, 8, 243, 0, 1, 0, false, 0, 1);
  store_pair(1, minus_two);
  AppendVop3(&test.code, 0x167, 1, 8, 243, 0, 1, 0, false, 0, 1);
  store_pair(1, 0xbff0000000000000ull);
  test.code.push_back(EncodeSop1(0x04, 12, 126));
  test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  AppendVop3(&test.code, 0x166, 1, 242, 242);
  AppendVop3(&test.code, 0x167, 1, 242, 242);
  test.code.push_back(EncodeSop1(0x04, 126, 12));
  store_pair(1, 0xbff0000000000000ull);
  AppendEnd(&test.code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_DWORD,
                  O::V_MIN_F64, O::V_MAX_F64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64",
                         "SignedZeroInfNanPreserve 64"};
  return test;
}

TestCase VectorRoundF64EdgesAndModifiers() {
  using O = ShaderOpcode;
  constexpr std::array<uint64_t, 14> sources{
      0x400a000000000000ull, 0xbff4000000000000ull,
      0x3fd0000000000000ull, 0xbfd0000000000000ull,
      0, 0x8000000000000000ull, 0x4330000000000001ull, 0xc330000000000001ull,
      0x7ff0000000000000ull, 0xfff0000000000000ull,
      0xfff8123456789abcull, 0x7ff0000000000001ull, 1, 0x8000000000000001ull};
  TestCase test;
  test.name = "VectorRoundF64EdgesAndModifiers";
  for (const auto bits : sources) {
    test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
  }
  test.expected = test.initial;
  const auto rounded = [](u32 opcode, double value) {
    switch (opcode) {
      case 0x17: return std::trunc(value);
      case 0x18: return std::ceil(value);
      case 0x1a: return std::floor(value);
      default: std::abort();
    }
  };
  const auto store_pair = [&](u32 reg, uint64_t bits) {
    AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
    AppendStoreVgpr(&test.code, reg + 1, static_cast<u32>(test.expected.size() + 1));
    test.expected.insert(test.expected.end(), {u32(bits), u32(bits >> 32)});
  };
  for (const u32 opcode : {0x17u, 0x18u, 0x1au}) {
    for (u32 i = 0; i < sources.size(); i++) {
      for (u32 word = 0; word < 2; word++) {
        AppendVMovU32(&test.code, 30, (i * 2 + word) * 4);
        AppendBufferLoadDword(&test.code, 2 + word, 30);
      }
      test.code.push_back(EncodeVop1(opcode, 5, Vgpr(2)));
      AppendVop3(&test.code, 0x180 + opcode, 7, Vgpr(2), 0);
      const auto value = std::bit_cast<double>(sources[i]);
      for (const u32 reg : {5u, 7u}) {
        if (std::isnan(value)) {
          // IEEE=false does not promise a particular NaN payload or quiet bit.
          test.code.push_back(EncodeVopc(0x22, Vgpr(reg), reg));
          AppendStoreSgprPair(&test.code, 106, static_cast<u32>(test.expected.size()));
          test.expected.insert(test.expected.end(), {0, 0});
        } else {
          store_pair(reg, std::bit_cast<uint64_t>(rounded(opcode, value)));
        }
      }
    }
    AppendSMovLiteral(&test.code, 8, 0);
    AppendSMovLiteral(&test.code, 9, 0xbff40000u);
    AppendVop3(&test.code, 0x180 + opcode, 1, 8, 0, 0, 1, 0, false, 0, 1);
    const auto expected = std::bit_cast<uint64_t>(rounded(opcode, -1.25));
    store_pair(1, expected);
    test.code.push_back(EncodeSop1(0x04, 12, 126));
    test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
    test.code.push_back(EncodeVop1(opcode, 1, 242));
    test.code.push_back(EncodeSop1(0x04, 126, 12));
    store_pair(1, expected);
  }
  AppendEnd(&test.code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_DWORD,
                  O::V_TRUNC_F64, O::V_CEIL_F64, O::V_FLOOR_F64, O::V_CMP_EQ_F64,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "Trunc", "Ceil", "Floor"};
  return test;
}

TestCase VectorFractF64CapturedChainRuntimeExec() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "VectorFractF64CapturedChainRuntimeExec";
  auto& code = test.code;
  AppendSMovLiteral(&code, 60, 3);
  AppendSMovLiteral(&code, 61, 3);
  code.push_back(EncodeSop1(0x04, 126, 60)); // Lanes 0,1 and 32,33.
  code.push_back(EncodeVop1(0x01, 20, Vgpr(0)));
  code.push_back(EncodeVop2(0x1b, 30, InlineU32(1), 0));
  code.push_back(EncodeVop2(0x1a, 30, InlineU32(4), 30));
  AppendVMovU32(&code, 7, 192 * 4);
  code.push_back(EncodeVop2(0x25, 30, Vgpr(7), 30));
  constexpr u32 input_regs[] = {6, 4, 10, 11};
  for (u32 word = 0; word < 4; ++word) {
    code.push_back(EncodeMubuf0(0x0c, word * 4));
    code.push_back(EncodeMubuf1(input_regs[word], 0, 30));
  }
  code.push_back(EncodeSmem0(0x08, 16, 0));
  code.push_back(EncodeSmem1(200 * 4, 125));
  constexpr u32 output_regs[] = {12, 22, 24};
  for (const auto reg : output_regs) AppendVMovLiteral(&code, reg, 0xdeadbeefu);
  code.push_back(EncodeVopc(0xc2, InlineU32(1), 6));
  code.push_back(EncodeSop1(0x24, 8, 106));
  code.push_back(EncodeSMovB32(106, 16));
  AppendSMovLiteral(&code, 107, 0x3fea36a9u);
  code.push_back(EncodeVop1(0x2a, 4, Vgpr(4))); // Runtime FP32 reciprocal with FP64 narrowing.
  // Captured 4ee720fc4678fa19 PCs 0x1e8..0x20c, 0x23c..0x258, 0x260..0x270.
  code.insert(code.end(), {0x7e04086au, 0xbeea03f4u, 0x7e082104u, 0xd54c0004u,
                           0x0410d502u, 0xbeea0b86u, 0xbeeb03ffu, 0x3fe3f9e1u,
                           0x7e087d04u, 0x7e181f04u});
  code.push_back(EncodeVop1(0x01, 4, Vgpr(10))); // Second runtime image value.
  code.insert(code.end(), {0x7e082104u, 0xd54c0004u, 0x0410d502u, 0xbeea0b87u,
                           0xbeeb03ffu, 0x3fc46bacu, 0x7e087d04u, 0x7e2c1f04u,
                           0x7e08210bu, 0xd54c0002u, 0x0410d502u, 0x7e047d02u,
                           0x7e301f02u});
  code.push_back(EncodeSop1(0x04, 126, 8));
  for (u32 component = 0; component < 3; ++component)
    AppendStoreVgprAtLaneDwordOffset(&code, output_regs[component], 20, component * 64);
  AppendEnd(&code);

  test.initial.resize(192);
  test.initial.insert(test.initial.end(), {1, 0x3e800000u, 0xbfa00000u, 0x40600000u,
                                          0, 0x3f800000u, 0x40000000u, 0x40400000u, 3});
  test.expected = test.initial;
  constexpr u32 results[] = {0x3eea3fbcu, 0x3f1f6d21u, 0x3f7a860du};
  for (u32 component = 0; component < 3; ++component) {
    for (const u32 lane : {0u, 32u}) test.expected[component * 64 + lane] = results[component];
    for (const u32 lane : {1u, 33u}) test.expected[component * 64 + lane] = 0xdeadbeefu;
  }
  test.compute_info.wave_size = 64;
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  test.required_spirv = {"OpCapability Float64", "OpFDiv", "Fma", "Fract"};
  test.forbidden_spirv = {"RoundingModeRTE"};
  test.ir_counts = {{"FPRecip32", 1}, {"ConvertF64S32", 1}, {"ConvertF64F32", 3},
                    {"FPFma64", 3}, {"FPFract64", 3}, {"ConvertF32F64", 3}};
  test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_MOV_B32, O::V_AND_B32,
                  O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD,
                  O::S_BUFFER_LOAD_DWORD, O::V_CMP_EQ_U32, O::S_AND_SAVEEXEC_B64,
                  O::S_BREV_B32, O::V_CVT_F64_I32, O::V_CVT_F64_F32, O::V_FMA_F64,
                  O::V_FRACT_F64, O::V_RCP_F32, O::V_CVT_F32_F64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase VectorF64WideningConversions() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "VectorF64WideningConversions";
  const auto store_pair = [&](u32 reg, uint64_t bits) {
    AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
    AppendStoreVgpr(&test.code, reg + 1,
                    static_cast<u32>(test.expected.size() + 1));
    test.expected.insert(test.expected.end(), {static_cast<u32>(bits),
                                               static_cast<u32>(bits >> 32)});
  };
  for (const u32 bits :
       {0u, 0x80000000u, 0x3f800001u, 0xbf800001u, 0x00800000u, 0x7f7fffffu,
        0xff7fffffu, 0x7f800000u, 0xff800000u, 1u, 0x80000001u}) {
    AppendSMovLiteral(&test.code, 8, bits);
    test.code.push_back(0x7e002008u);
    const auto source = (bits & 0x7f800000u) == 0 ? bits & 0x80000000u : bits;
    store_pair(0, std::bit_cast<uint64_t>(
                      static_cast<double>(std::bit_cast<float>(source))));
  }
  for (const u32 bits : {0x7fc12345u, 0x7f812345u}) {
    AppendSMovLiteral(&test.code, 8, bits);
    test.code.push_back(0x7e002008u);
    AppendVMovLiteral(&test.code, 3, 0x7ff80000u);
    test.code.push_back(EncodeVop2(0x1b, 1, Vgpr(3), 1));
    AppendStoreVgpr(&test.code, 1, static_cast<u32>(test.expected.size()));
    test.expected.push_back(0x7ff80000u);
  }
  AppendVMovLiteral(&test.code, 1, std::bit_cast<u32>(-3.5f));
  AppendVop3(&test.code, 0x190, 1, Vgpr(1), 0, 0, 1, 0, false, 0, 1);
  store_pair(1, std::bit_cast<uint64_t>(-3.5));
  for (const u32 value :
       {0u, 1u, 16777217u, 0x7fffffffu, 0x80000000u, 0xffffffffu}) {
    AppendVMovLiteral(&test.code, 1, value);
    test.code.push_back(EncodeVop1(0x16, 1, Vgpr(1)));
    store_pair(1, std::bit_cast<uint64_t>(static_cast<double>(value)));
  }
  AppendSMovLiteral(&test.code, 8, 0xffffffffu);
  AppendVop3(&test.code, 0x196, 1, 8u, 0);
  store_pair(1, std::bit_cast<uint64_t>(4294967295.0));
  test.code.push_back(EncodeSop1(0x04, 12, 126));
  test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  test.code.push_back(EncodeVop1(0x10, 1, InlineU32(0)));
  test.code.push_back(EncodeVop1(0x16, 1, InlineU32(0)));
  test.code.push_back(EncodeSop1(0x04, 126, 12));
  store_pair(1, std::bit_cast<uint64_t>(4294967295.0));
  AppendEnd(&test.code);
  test.opcodes = {O::S_MOV_B32,          O::S_MOV_B64,     O::V_MOV_B32,
                  O::V_AND_B32,          O::V_CVT_F64_F32, O::V_CVT_F64_U32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "OpFConvert",
                         "OpConvertUToF"};

  std::vector<u32> vertex_code;
  AppendSMovLiteral(&vertex_code, 8, std::bit_cast<u32>(2.0f));
  vertex_code.insert(vertex_code.end(),
                     {0x7e002008u, 0x7e005f00u, 0x7e401f00u});
  vertex_code.push_back(EncodeExp0(0x0c, 0xf));
  vertex_code.push_back(EncodeExp1(32, 32, 32, 32));
  AppendEnd(&vertex_code);
  ShaderVertexInputInfo vertex{};
  ShaderRecompiler::CompileOptions options;
  options.stage = ShaderType::Vertex;
  options.input_info.vertex = &vertex;
  auto translated = ShaderRecompiler::TranslateProgram(vertex_code, options);
  auto result =
      ShaderRecompiler::CompileProgram(std::move(translated), options, {});
  ValidateSpirv(test.name, result.spirv);
  Require(test.name, "captured vertex conversion chain",
          result.ir_dump.find("ConvertF64F32") != std::string::npos &&
              result.ir_dump.find("FPRecip64") != std::string::npos &&
              result.ir_dump.find("ConvertF32F64") != std::string::npos,
          "vertex compilation lost the captured widening/reciprocal/narrowing "
          "sequence");
  return test;
}

TestCase VectorF64ModesModifiersAndExec() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "VectorF64ModesModifiersAndExec";
  constexpr std::array<std::array<uint64_t, 2>, 8> add_pairs{{
      {1, 1}, {1, 0x8000000000000001ull},
      {0x8000000000000000ull, 0x8000000000000000ull},
      {0x3ff0000000000000ull, 0x3ca0000000000000ull},
      {0x3ff0000000000001ull, 0x3ca0000000000000ull},
      {0x3ff0000000000000ull, 0xbff0000000000000ull},
      {0x7ff0000000000000ull, 0x3ff0000000000000ull},
      {0xfff0000000000000ull, 0xbff0000000000000ull}}};
  for (const auto& pair : add_pairs)
    for (const auto bits : pair)
      test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
  // Runtime F64 values immediately around even/odd F32 halfway points, both signs.
  constexpr std::array<std::pair<uint64_t, u32>, 12> narrowing{{
      {0x3ff000000fffffffull, 0x3f800000u}, {0x3ff0000010000000ull, 0x3f800000u},
      {0x3ff0000010000001ull, 0x3f800001u}, {0x3ff000002fffffffull, 0x3f800001u},
      {0x3ff0000030000000ull, 0x3f800002u}, {0x3ff0000030000001ull, 0x3f800002u},
      {0xbff000000fffffffull, 0xbf800000u}, {0xbff0000010000000ull, 0xbf800000u},
      {0xbff0000010000001ull, 0xbf800001u}, {0xbff000002fffffffull, 0xbf800001u},
      {0xbff0000030000000ull, 0xbf800002u}, {0xbff0000030000001ull, 0xbf800002u}}};
  for (const auto& item : narrowing)
    test.initial.insert(test.initial.end(), {u32(item.first), u32(item.first >> 32)});
  test.expected = test.initial;
  const auto load_pair = [&](u32 reg, uint64_t bits) {
    AppendVMovLiteral(&test.code, reg, static_cast<u32>(bits));
    AppendVMovLiteral(&test.code, reg + 1, static_cast<u32>(bits >> 32));
  };
  const auto store_pair = [&](u32 reg, uint64_t bits) {
    AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
    AppendStoreVgpr(&test.code, reg + 1,
                    static_cast<u32>(test.expected.size() + 1));
    test.expected.insert(test.expected.end(), {static_cast<u32>(bits),
                                               static_cast<u32>(bits >> 32)});
  };
  const auto store_word = [&](u32 reg, u32 bits) {
    AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
    test.expected.push_back(bits);
  };
  for (const auto [source, expected] :
       {std::pair<uint64_t, uint64_t>{0, 0x7ff0000000000000ull},
        {0x8000000000000000ull, 0xfff0000000000000ull},
        {0x7ff0000000000000ull, 0},
        {0xfff0000000000000ull, 0x8000000000000000ull},
        {0x0008000000000000ull, 0x7fe0000000000000ull},
        {0x7fe0000000000000ull, 0x0008000000000000ull}}) {
    load_pair(1, source);
    test.code.push_back(EncodeVop1(0x2f, 1, Vgpr(1)));
    store_pair(1, expected);
  }
  for (u32 i = 0; i < add_pairs.size(); ++i) {
    const auto pair = add_pairs[i];
    for (u32 word = 0; word < 4; ++word) {
      AppendVMovU32(&test.code, 31, (i * 4 + word) * 4);
      AppendBufferLoadDword(&test.code, 1 + word, 31);
    }
    AppendVop3(&test.code, 0x164, 1, Vgpr(1), Vgpr(3));
    store_pair(1, std::bit_cast<uint64_t>(std::bit_cast<double>(pair[0]) +
                                         std::bit_cast<double>(pair[1])));
  }
  // Scalar pair, exact inline 1/(2*pi), literal high word, and ABS before NEG.
  AppendSMovLiteral(&test.code, 8, 0);
  AppendSMovLiteral(&test.code, 9, 0xbff00000u);
  AppendVop3(&test.code, 0x164, 1, 8, 244, 0, 1, 0, false, 0, 1);
  store_pair(1, std::bit_cast<uint64_t>(1.0));
  AppendVop3(&test.code, 0x164, 1, 255, 240);
  test.code.push_back(0x40080000u);
  store_pair(1, std::bit_cast<uint64_t>(3.5));
  AppendVop3(&test.code, 0x164, 1, 248, InlineU32(0));
  store_pair(1, 0x3fc45f306dc9c882ull);
  for (const auto pair : std::array<std::array<uint64_t, 2>, 2>{{
           {0x7ff0000000000000ull, 0xfff0000000000000ull},
           {0x7ff0000000000001ull, 0}}}) {
    load_pair(1, pair[0]);
    load_pair(3, pair[1]);
    AppendVop3(&test.code, 0x164, 1, Vgpr(1), Vgpr(3));
    AppendVMovLiteral(&test.code, 3, 0x7ff80000u);
    test.code.push_back(EncodeVop2(0x1b, 2, Vgpr(3), 2));
    store_word(2, 0x7ff80000u);
  }
  // A fused result retains the residual that separate multiply/add would round
  // to zero.
  load_pair(1, std::bit_cast<uint64_t>(1.0 + 0x1p-27));
  load_pair(3, std::bit_cast<uint64_t>(1.0 - 0x1p-27));
  AppendVop3(&test.code, 0x14c, 1, Vgpr(1), Vgpr(3), 243u);
  store_pair(1, std::bit_cast<uint64_t>(-0x1p-54));
  // Unaligned pairs, SGPR pairs, inline floats, literal highword expansion, ABS
  // then NEG.
  test.code.push_back(EncodeSop1(0x03, 8, 255));
  test.code.push_back(0u);
  test.code.push_back(EncodeSop1(0x03, 9, 255));
  test.code.push_back(0xbff00000u);
  AppendVop3(&test.code, 0x165, 1, 8u, 244u, 0, 1, 0, false, 0, 1);
  store_pair(1, std::bit_cast<uint64_t>(-2.0));
  AppendVop3(&test.code, 0x165, 1, 255u, 242u);
  test.code.push_back(0x40080000u);
  store_pair(1, std::bit_cast<uint64_t>(3.0));
  AppendVop3(&test.code, 0x165, 1, 248u, 242u);
  store_pair(1, 0x3fc45f306dc9c882ull);
  AppendVop3(&test.code, 0x184, 1, InlineU32(3), 0);
  store_pair(1, std::bit_cast<uint64_t>(3.0));
  AppendVop3(&test.code, 0x1af, 1, 244u, 0);
  store_pair(1, std::bit_cast<uint64_t>(0.5));
  // Preserve both destination words when EXEC is clear, restoring the original
  // live lanes.
  test.code.push_back(EncodeSop1(0x04, 12, 126));
  test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  test.code.push_back(EncodeVop1(0x04, 1, InlineU32(9)));
  AppendVop3(&test.code, 0x164, 1, 242, 242);
  test.code.push_back(EncodeSop1(0x04, 126, 12));
  store_pair(1, std::bit_cast<uint64_t>(0.5));
  for (const auto [source, expected] :
       {std::pair<uint64_t, u32>{0x47f0000000000000ull, 0x7f7fffffu},
        {0xc7f0000000000000ull, 0xff7fffffu},
        {0x7ff0000000000000ull, 0x7f800000u},
        {0xfff0000000000000ull, 0xff800000u},
        {0x3800000000000000ull, 0},
        {0xb800000000000000ull, 0x80000000u},
        {0x3810000000000000ull, 0x00800000u},
        {0x3ff0000010000000ull, 0x3f800000u}}) {
    load_pair(1, source);
    AppendVop3(&test.code, 0x18f, 1, Vgpr(1), 0);
    store_word(1, expected);
  }
  for (u32 i = 0; i < narrowing.size(); ++i) {
    for (u32 word = 0; word < 2; ++word) {
      AppendVMovU32(&test.code, 31, (add_pairs.size() * 4 + i * 2 + word) * 4);
      AppendBufferLoadDword(&test.code, 1 + word, 31);
    }
    AppendVop3(&test.code, 0x18f, 1, Vgpr(1), 0);
    store_word(1, narrowing[i].second);
  }
  load_pair(1, 0x7ff8123456789abcull);
  test.code.push_back(EncodeVop1(0x2f, 1, Vgpr(1)));
  AppendVMovLiteral(&test.code, 3, 0x7ff80000u);
  test.code.push_back(EncodeVop2(0x1b, 2, Vgpr(3), 2));
  store_word(2, 0x7ff80000u);
  AppendEnd(&test.code);
  test.opcodes = {O::V_MOV_B32,          O::S_MOV_B32,     O::S_MOV_B64,
                  O::V_CVT_F64_I32,      O::V_RCP_F64,     O::V_MUL_F64,
                  O::V_ADD_F64,          O::V_FMA_F64,          O::V_CVT_F32_F64, O::V_AND_B32,
                  O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.initial.resize(test.expected.size());
  test.required_spirv = {"OpFAdd"};
  return test;
}

TestCase VectorRcpIflagF32IntegerReciprocal() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 0, 0);
  AppendVMovU32(&code, 1, 1);
  AppendVMovU32(&code, 2, 2);
  AppendVMovLiteral(&code, 3, 0xffffffffu);
  for (u32 i = 0; i < 4; i++) {
    code.push_back(EncodeVop1(0x06, 4 + i, Vgpr(i)));
    code.push_back(EncodeVop1(0x2b, 8 + i, Vgpr(4 + i)));
  }
  AppendVMovLiteral(&code, 12, 0xc0000000u);
  AppendVop3(&code, 0x1ab, 13, Vgpr(12), 0, 0, 1, 0, false, 1);
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, 8 + i, i);
  }
  AppendStoreVgpr(&code, 13, 4);
  AppendEnd(&code);

  TestCase test;
  test.name = "VectorRcpIflagF32IntegerReciprocal";
  test.code = std::move(code);
  test.expected = {0x7f800000u, 0x3f800000u, 0x3f000000u, 0x2f800000u,
                   0x3f800000u};
  test.opcodes = {O::V_MOV_B32, O::V_CVT_F32_U32, O::V_RCP_IFLAG_F32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_RCP_IFLAG_F32", 5}};
  test.ir_counts = {{"FPRecipIFlag32", 5}};
  test.required_spirv = {"OpConvertUToF", "OpFDiv"};
  test.forbidden_spirv = {"8388608"};
  return test;
}

TestCase VectorSinCosMaxFiniteSpecialCases() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0xff7fffffu);
  code.push_back(EncodeVop1(0x35, 1, Vgpr(0)));
  code.push_back(EncodeVop1(0x36, 2, Vgpr(0)));
  AppendStoreVgpr(&code, 1, 0);
  AppendStoreVgpr(&code, 2, 1);
  AppendEnd(&code);

  return {"VectorSinCosMaxFiniteSpecialCases",
          code,
          {},
          {0x00000000u, 0x3f800000u},
          {O::V_MOV_B32, O::V_SIN_F32, O::V_COS_F32, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase VectorCompareF32ExposureGuard() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "VectorCompareF32ExposureGuard";
  test.initial = {0x00000001u, 0x80000001u, 0x007fffffu, 0x807fffffu,
                  0x00000000u, 0x80000000u, 0x3f800000u, 0x40000000u};
  test.expected = test.initial;
  AppendVMovLiteral(&test.code, 1, 0x3f800000u);
  for (u32 i = 0; i < test.initial.size(); i++) {
    AppendVMovU32(&test.code, 31, i * sizeof(u32));
    AppendBufferLoadDword(&test.code, 0, 31);
    // Captured exposure guard: replace zero with 1.0 before taking its reciprocal.
    test.code.push_back(EncodeVopc(0x02, InlineU32(0), 0));
    test.code.push_back(EncodeVop2(0x01, 2, Vgpr(0), 1));
    test.code.push_back(EncodeVop1(0x2a, 3, Vgpr(2)));
    AppendStoreVgpr(&test.code, 2, static_cast<u32>(test.expected.size()));
    AppendStoreVgpr(&test.code, 3, static_cast<u32>(test.expected.size() + 1));
    test.expected.push_back(i == 7 ? 0x40000000u : 0x3f800000u);
    test.expected.push_back(i == 7 ? 0x3f000000u : 0x3f800000u);
  }
  test.initial.resize(test.expected.size());
  AppendEnd(&test.code);
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_CMP_EQ_F32, O::V_CNDMASK_B32,
                  O::V_RCP_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase VectorCompareF32DenormalModes(u32 mode) {
  using O = ShaderOpcode;
  enum Relation : u32 { Less, Equal, Greater, Unordered };
  struct Pair {
    u32 lhs;
    u32 rhs;
    Relation relation;
  };
  constexpr Pair pairs[] = {
      {0x00000001u, 0x00000000u, Equal},
      {0x80000001u, 0x80000000u, Equal},
      {0x00000000u, 0x00000001u, Equal},
      {0x80000000u, 0x80000001u, Equal},
      {0x007fffffu, 0x80000001u, Equal},
      {0x00800000u, 0x007fffffu, Greater},
      {0xbf800000u, 0x80000001u, Less},
      {0xff800000u, 0xbf800000u, Less},
      {0x7f800000u, 0x7f800000u, Equal},
      {0x7fc12345u, 0x00000000u, Unordered},
      {0x00000000u, 0x7fa12345u, Unordered},
      {0x00000000u, 0x80000000u, Equal},
  };
  TestCase test;
  constexpr const char* names[] = {"VectorCompareF32DenormalModeC0",
                                   "VectorCompareF32DenormalModeE0"};
  test.name = names[(mode >> 5u) & 1u];
  test.compute_info.float_mode = static_cast<uint8_t>(mode);
  test.has_compute_info = true;
  for (const auto& pair : pairs) {
    test.initial.insert(test.initial.end(), {pair.lhs, pair.rhs});
  }
  test.expected = test.initial;
  AppendVMovU32(&test.code, 1, 1);
  for (u32 i = 0; i < std::size(pairs); i++) {
    AppendVMovU32(&test.code, 31, i * 2u * sizeof(u32));
    AppendBufferLoadDword(&test.code, 2, 31);
    AppendVMovU32(&test.code, 31, (i * 2u + 1u) * sizeof(u32));
    AppendBufferLoadDword(&test.code, 3, 31);
    // The sixteen architectural predicates select subsets of LT/EQ/GT/unordered.
    for (u32 predicate = 0; predicate < 16; predicate++) {
      test.code.push_back(EncodeVopc(predicate, Vgpr(2), 3));
      test.code.push_back(EncodeVop2(0x01, 4, InlineU32(0), 1));
      AppendStoreVgpr(&test.code, 4, static_cast<u32>(test.expected.size()));
      test.expected.push_back((predicate >> pairs[i].relation) & 1u);
    }
  }
  test.initial.resize(test.expected.size());
  AppendEnd(&test.code);
  test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_CMP_EQ_F32, O::V_CMP_LE_F32,
                  O::V_CMP_GE_F32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  return test;
}

TestCase VectorCompareOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  std::vector<u32> results;
  std::vector<u32> expected;
  u32 dst = 10;

  code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
  AppendVMovLiteral(&code, 2, 0x40000000u);
  AppendVMovLiteral(&code, 3, 0x40400000u);
  code.push_back(EncodeVop1(0x01, 4, InlineU32(1)));
  AppendVMovLiteral(&code, 5, 0xffffffffu);

  auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) {
    if (dst == 31u) {
      dst++;
    }
    code.push_back(EncodeVopc(opcode, src0, src1));
    code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1));
    results.push_back(dst);
    expected.push_back(value ? 1u : 0u);
    dst++;
  };

  append_compare(0x00, Vgpr(2), 3, false);
  append_compare(0x01, Vgpr(2), 3, true);
  append_compare(0x02, Vgpr(2), 3, false);
  append_compare(0x03, Vgpr(2), 3, true);
  append_compare(0x04, Vgpr(2), 3, false);
  append_compare(0x05, Vgpr(2), 3, true);
  append_compare(0x06, Vgpr(2), 3, false);
  append_compare(0x07, Vgpr(2), 3, true);
  append_compare(0x08, Vgpr(2), 3, false);
  append_compare(0x09, Vgpr(2), 3, true);
  append_compare(0x0a, Vgpr(2), 3, false);
  append_compare(0x0b, Vgpr(2), 3, true);
  append_compare(0x0c, Vgpr(2), 3, false);
  append_compare(0x0d, Vgpr(2), 3, true);
  append_compare(0x0e, Vgpr(2), 3, false);
  append_compare(0x0f, Vgpr(2), 3, true);

  append_compare(0x80, Vgpr(5), 4, false);
  append_compare(0x81, Vgpr(5), 4, true);
  append_compare(0x82, Vgpr(5), 4, false);
  append_compare(0x83, Vgpr(5), 4, true);
  append_compare(0x84, Vgpr(5), 4, false);
  append_compare(0x85, Vgpr(5), 4, true);
  append_compare(0x86, Vgpr(5), 4, false);
  append_compare(0x87, Vgpr(5), 4, true);

  append_compare(0xc0, Vgpr(4), 5, false);
  append_compare(0xc1, Vgpr(4), 5, true);
  append_compare(0xc2, Vgpr(4), 5, false);
  append_compare(0xc3, Vgpr(4), 5, true);
  append_compare(0xc4, Vgpr(4), 5, false);
  append_compare(0xc5, Vgpr(4), 5, true);
  append_compare(0xc6, Vgpr(4), 5, false);
  append_compare(0xc7, Vgpr(4), 5, true);

  code.push_back(0x7c1d00f9u);
  code.push_back(0x86069201u);
  code.push_back(EncodeVopc(0x84, 249, 4));
  code.push_back(EncodeVopcSdwa(5, 19, 1));
  code.push_back(EncodeVopc(0x01, 250, 3));
  code.push_back(EncodeVop2Dpp(2));
  AppendVMovLiteral(&code, 6, 0xc0000000u);
  AppendVMovLiteral(&code, 7, 0x3f800000u);
  AppendVop3(&code, 0x04, 20, Vgpr(6), Vgpr(7), 0, 1);
  for (u32 i = 0; i < static_cast<u32>(results.size()); i++) {
    AppendStoreVgpr(&code, results[i], i);
  }
  AppendStoreSgpr(&code, 18, static_cast<u32>(results.size()));
  expected.push_back(1u);
  AppendStoreSgpr(&code, 19, static_cast<u32>(results.size() + 1u));
  expected.push_back(0u);
  AppendStoreSgpr(&code, 106, static_cast<u32>(results.size() + 2u));
  expected.push_back(1u);
  AppendStoreSgpr(&code, 20, static_cast<u32>(results.size() + 3u));
  expected.push_back(1u);
  AppendEnd(&code);

  return {"VectorCompareOps",
          code,
          {},
          expected,
          {O::V_MOV_B32,     O::V_CNDMASK_B32,      O::V_CMP_F_F32,
           O::V_CMP_LT_F32,  O::V_CMP_EQ_F32,       O::V_CMP_LE_F32,
           O::V_CMP_GT_F32,  O::V_CMP_LG_F32,       O::V_CMP_GE_F32,
           O::V_CMP_O_F32,   O::V_CMP_U_F32,        O::V_CMP_NGE_F32,
           O::V_CMP_NLG_F32, O::V_CMP_NGT_F32,      O::V_CMP_NLE_F32,
           O::V_CMP_NEQ_F32, O::V_CMP_NLT_F32,      O::V_CMP_TRU_F32,
           O::V_CMP_F_I32,   O::V_CMP_LT_I32,       O::V_CMP_EQ_I32,
           O::V_CMP_LE_I32,  O::V_CMP_GT_I32,       O::V_CMP_NE_I32,
           O::V_CMP_GE_I32,  O::V_CMP_T_I32,        O::V_CMP_F_U32,
           O::V_CMP_LT_U32,  O::V_CMP_EQ_U32,       O::V_CMP_LE_U32,
           O::V_CMP_GT_U32,  O::V_CMP_NE_U32,       O::V_CMP_GE_U32,
           O::V_CMP_T_U32,   O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorVop3CompareEqU64OnGpu() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));

  code.push_back(EncodeSop1(0x04, 106, 126)); // s_mov_b64 vcc, exec
  code.push_back(0xd4e2006au);
  code.push_back(0x0000d47eu); // v_cmp_eq_u64 vcc, exec, vcc
  code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
  AppendStoreVgpr(&code, 2, 0);

  AppendSMovLiteral(&code, 107, 1); // high-dword mismatch
  code.push_back(0xd4e2006au);
  code.push_back(0x0000d47eu); // v_cmp_eq_u64 vcc, exec, vcc
  code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
  AppendStoreVgpr(&code, 3, 1);
  AppendEnd(&code);

  TestCase test{"VectorVop3CompareEqU64OnGpu",
                code,
                {},
                {1, 0},
                {O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_U64,
                 O::V_CNDMASK_B32, O::S_MOV_B32, O::BUFFER_STORE_DWORD,
                 O::S_ENDPGM}};
  test.decoded_counts = {{"V_CMP_EQ_U64 vcc_lo, exec_lo, vcc_lo", 2}};
  return test;
}

TestCase VectorVop3CompareNeU64OnGpu() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));

  code.push_back(EncodeSop1(0x04, 106, 126)); // s_mov_b64 vcc, exec
  code.push_back(0xd4e5006au);
  code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc
  code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
  AppendStoreVgpr(&code, 2, 0);

  AppendSMovLiteral(&code, 106, 0);
  AppendSMovLiteral(&code, 107, 0);
  code.push_back(0xd4e5006au);
  code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc
  code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
  AppendStoreVgpr(&code, 3, 1);
  AppendEnd(&code);

  return {"VectorVop3CompareNeU64OnGpu",
          code,
          {},
          {0, 1},
          {O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_NE_U64, O::V_CNDMASK_B32,
           O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorVopcCmpxNeU64CapturedExecMask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 7, 0);
  AppendVMovU32(&code, 8, 0);
  AppendVMovU32(&code, 31, 0);
  code.push_back(0x7dea0e80u); // v_cmpx_ne_u64 exec, 0, v[7:8]
  AppendBufferStoreDword(&code, 8, 31);

  code.push_back(EncodeSMovB32(126, InlineU32(1)));
  AppendVMovU32(&code, 8, 1);
  AppendVMovU32(&code, 31, 4);
  code.push_back(0x7dea0e80u); // v_cmpx_ne_u64 exec, 0, v[7:8]
  AppendBufferStoreDword(&code, 8, 31);

  AppendVMovU32(&code, 2, 0);
  AppendVMovU32(&code, 3, 0);
  AppendVMovU32(&code, 31, 8);
  code.push_back(0x7dea0480u); // v_cmpx_ne_u64 exec, 0, v[2:3]
  AppendBufferStoreDword(&code, 3, 31);

  code.push_back(EncodeSMovB32(126, InlineU32(1)));
  AppendVMovU32(&code, 3, 1);
  AppendVMovU32(&code, 31, 12);
  code.push_back(0x7dea0480u); // v_cmpx_ne_u64 exec, 0, v[2:3]
  AppendBufferStoreDword(&code, 3, 31);
  AppendEnd(&code);

  TestCase test{"VectorVopcCmpxNeU64CapturedExecMask",
                code,
                {0x11223344u, 0x55667788u, 0xaabbccddu, 0xeeff0011u},
                {0x11223344u, 1u, 0xaabbccddu, 1u},
                {O::V_MOV_B32, O::V_CMPX_NE_U64, O::BUFFER_STORE_DWORD,
                 O::S_MOV_B32, O::S_ENDPGM}};
  test.decoded_counts = {{"V_CMPX_NE_U64 exec_lo, 0, v7", 2},
                         {"V_CMPX_NE_U64 exec_lo, 0, v2", 2}};
  return test;
}

TestCase VectorVop3CmpxNeI64CapturedExecMask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 7, 0);
  AppendVMovU32(&code, 8, 0);
  AppendVMovU32(&code, 31, 0);
  code.push_back(0xd4b5007eu);
  code.push_back(0x00020e80u); // v_cmpx_ne_i64 exec, 0, v[7:8]
  AppendBufferStoreDword(&code, 8, 31);

  code.push_back(EncodeSMovB32(126, InlineU32(1)));
  AppendVMovU32(&code, 8, 1);
  AppendVMovU32(&code, 31, 4);
  code.push_back(0xd4b5007eu);
  code.push_back(0x00020e80u); // v_cmpx_ne_i64 exec, 0, v[7:8]
  AppendBufferStoreDword(&code, 8, 31);
  AppendEnd(&code);

  TestCase test{"VectorVop3CmpxNeI64CapturedExecMask",
                code,
                {0x11223344u, 0x55667788u},
                {0x11223344u, 1u},
                {O::V_MOV_B32, O::V_CMPX_NE_I64, O::BUFFER_STORE_DWORD,
                 O::S_MOV_B32, O::S_ENDPGM}};
  test.decoded_counts = {{"V_CMPX_NE_I64 exec_lo, 0, v7", 2}};
  return test;
}

TestCase VectorVop3CompareGtU64OnGpu() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));

  AppendSMovLiteral(&code, 4, 0);
  AppendSMovLiteral(&code, 5, 0x80000000u);
  code.push_back(0xd4e4006au);
  code.push_back(0x00010004u); // v_cmp_gt_u64 vcc, s[4:5], 0
  code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
  AppendStoreVgpr(&code, 2, 0);

  code.push_back(0xd4e4006au);
  code.push_back(0x0001006au); // v_cmp_gt_u64 vcc, vcc, 0
  code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
  AppendStoreVgpr(&code, 3, 1);

  AppendSMovLiteral(&code, 106, 0);
  AppendSMovLiteral(&code, 107, 0);
  code.push_back(0xd4e4006au);
  code.push_back(0x0001006au); // v_cmp_gt_u64 vcc, vcc, 0
  code.push_back(EncodeVop2(0x01, 4, InlineU32(0), 1));
  AppendStoreVgpr(&code, 4, 2);
  AppendEnd(&code);

  TestCase test{"VectorVop3CompareGtU64OnGpu",
                code,
                {},
                {1, 1, 0},
                {O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_GT_U64, O::V_CNDMASK_B32,
                 O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.wave_size = 32;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase VectorVopcCompareLtU64OnGpu() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 0, 5);
  AppendVMovU32(&code, 1, 0);
  AppendVMovU32(&code, 2, 1);

  AppendSMovLiteral(&code, 4, 4);
  AppendSMovLiteral(&code, 5, 0);
  code.push_back(EncodeVopc(0xe1, 4, 0));
  code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 2));
  AppendStoreVgpr(&code, 3, 0);

  AppendSMovLiteral(&code, 4, 6);
  code.push_back(EncodeVopc(0xe1, 4, 0));
  code.push_back(EncodeVop2(0x01, 4, InlineU32(0), 2));
  AppendStoreVgpr(&code, 4, 1);
  AppendEnd(&code);

  TestCase test{"VectorVopcCompareLtU64OnGpu",
                code,
                {},
                {1, 0},
                {O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_LT_U64,
                 O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.decoded_counts = {{"V_CMP_LT_U64 vcc_lo, s4, v0", 2}};
  return test;
}

TestCase VectorVop3CompareEqI64OnGpu() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
  code.push_back(EncodeSop1(0x04, 4, 126)); // s_mov_b64 s[4:5], exec

  code.push_back(0xd4a2006au);
  code.push_back(0x0000087eu); // v_cmp_eq_i64 vcc, exec, s[4:5]
  code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
  AppendStoreVgpr(&code, 2, 0);

  AppendSMovLiteral(&code, 4, 0);
  code.push_back(0xd4a2006au);
  code.push_back(0x0000087eu); // low dword mismatch
  code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
  AppendStoreVgpr(&code, 3, 1);
  AppendEnd(&code);

  return {"VectorVop3CompareEqI64OnGpu",
          code,
          {},
          {1, 0},
          {O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_I64, O::V_CNDMASK_B32,
           O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorCompareClassF32() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  std::vector<u32> expected;
  AppendVMovU32(&code, 1, 1);

  u32 out = 0;
  auto append_class = [&](u32 bits, u32 class_mask, bool match) {
    AppendVMovLiteral(&code, 0, bits);
    AppendVMovU32(&code, 2, class_mask);
    code.push_back(EncodeVopc(0x88, Vgpr(0), 2));
    code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
    AppendStoreVgpr(&code, 3, out++);
    expected.push_back(match ? 1u : 0u);
  };

  append_class(0x7fc00000u, 1u << 1u, true);
  append_class(0xff800000u, 1u << 2u, true);
  append_class(0xbf800000u, 1u << 3u, true);
  append_class(0x80000000u, 1u << 5u, true);
  append_class(0x00000000u, 1u << 6u, true);
  append_class(0x40000000u, 1u << 8u, true);
  append_class(0x7f800000u, 1u << 9u, true);
  append_class(0x40000000u, 1u << 3u, false);
  append_class(0x00000001u, 1u << 7u, true);
  append_class(0x80000001u, 1u << 4u, true);

  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovU32(&code, 2, 1u << 3u);
  AppendVop3(&code, 0x88, 20, Vgpr(0), Vgpr(2), 0, 0, 0, false, 0, 1);
  AppendStoreSgprPair(&code, 20, out);
  expected.push_back(1u);
  expected.push_back(0u);
  AppendEnd(&code);

  return {"VectorCompareClassF32",
          code,
          {},
          expected,
          {O::V_MOV_B32, O::V_CMP_CLASS_F32, O::V_CNDMASK_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorCompareClassF32ImmediateAndDynamicMasks() {
  using O = ShaderOpcode;
  // One input for each architectural class bit, in bit order.
  constexpr std::array<u32, 10> values{
      0x7f800001u, 0x7fc00001u, 0xff800000u, 0xbf800000u, 0x80000001u,
      0x80000000u, 0x00000000u, 0x00000001u, 0x3f800000u, 0x7f800000u};
  constexpr std::array<u32, 8> masks{
      0u, 0x3ffu, 3u, 7u, 0xfffffc00u, 0xffffffffu, 0xfffffc03u, 0xfffffc07u};
  TestCase test;
  test.name = "VectorCompareClassF32ImmediateAndDynamicMasks";
  test.initial.assign(values.begin(), values.end());
  test.initial.insert(test.initial.end(), masks.begin(), masks.end());
  test.expected = test.initial;
  auto &code = test.code;
  AppendVMovU32(&code, 1, 1);
  for (u32 index = 0; index < values.size(); ++index) {
    AppendVMovU32(&code, 30, index * 4u);
    AppendBufferLoadDword(&code, 0, 30);
    for (u32 mask_index = 0; mask_index < masks.size(); ++mask_index) {
      for (const bool dynamic : {false, true}) {
        if (dynamic) {
          AppendVMovU32(&code, 30, (values.size() + mask_index) * 4u);
          AppendBufferLoadDword(&code, 2, 30);
          code.push_back(EncodeVopc(0x88, Vgpr(0), 2));
        } else {
          AppendVop3(&code, 0x88, 106, Vgpr(0), 255);
          code.push_back(masks[mask_index]);
        }
        code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
        AppendStoreVgpr(&code, 3, static_cast<u32>(test.expected.size()));
        test.expected.push_back((masks[mask_index] & (1u << index)) != 0u);
      }
    }
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_CMP_CLASS_F32,
                  O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpIsNan"};
  return test;
}

TestCase VectorVopcSdwaCmpxClassF32CapturedExecMask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 13, 0x7fc00000u); // Quiet NaN.
  AppendVMovU32(&code, 3, 7);
  AppendVMovU32(&code, 31, 0);
  code.push_back(EncodeSMovB32(106, InlineU32(2))); // qNaN class mask.
  code.insert(code.end(), {0x7d30d4f9u, 0x8606000du});
  AppendBufferStoreDword(&code, 3, 31);

  code.push_back(EncodeSMovB32(126, InlineU32(1))); // Restore lane-zero EXEC.
  AppendVMovLiteral(&code, 13, 0x3f800000u);        // Positive normal.
  AppendVMovU32(&code, 3, 9);
  AppendVMovU32(&code, 31, 4);
  code.push_back(EncodeSMovB32(106, InlineU32(2))); // qNaN class mask.
  code.insert(code.end(), {0x7d30d4f9u, 0x8606000du});
  AppendBufferStoreDword(&code, 3, 31);
  AppendEnd(&code);

  TestCase test{
      "VectorVopcSdwaCmpxClassF32CapturedExecMask",
      code,
      {0x11111111u, 0x22222222u},
      {7u, 0x22222222u},
      {O::S_MOV_B32, O::V_MOV_B32, O::V_CMPX_CLASS_F32,
       O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.decoded_counts = {
      {"V_CMPX_CLASS_F32 exec_lo, v13, vcc_lo", 2}};
  return test;
}

TestCase VectorCmpClassF16(u32 wave_size) {
  using O = ShaderOpcode;
  constexpr std::array<u32, 10> values{
      0x7c01u, 0x7e01u, 0xfc00u, 0xbc00u, 0x8001u,
      0x8000u, 0x0000u, 0x0001u, 0x3c00u, 0x7c00u};
  constexpr u32 guard = 0x87654321u;
  TestCase test;
  test.name = wave_size == 64 ? "VectorCmpClassF16Wave64" : "VectorCmpClassF16Wave32";
  for (const auto value : values) {
    test.initial.insert(test.initial.end(), {0xa55a0000u | value, (value << 16u) | 0x7e01u});
  }
  test.expected = test.initial;
  auto &code = test.code;
  const auto compare = [&](std::initializer_list<u32> words, u32 expected,
                           u32 dst = 106u, u32 exec = 1u) {
    AppendSMovLiteral(&code, 106, 0x12345678u);
    AppendSMovLiteral(&code, 107, guard);
    AppendSMovLiteral(&code, dst + 1u, guard);
    code.push_back(EncodeSopc(0x06, InlineU32(expected), InlineU32(0)));
    code.insert(code.end(), words);
    const u32 sources[] = {dst, dst + 1u, 126, 127, 253};
    for (u32 i = 0; i < std::size(sources); ++i) {
      code.push_back(EncodeSMovB32(30u + i, sources[i]));
    }
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    const u32 results[] = {expected, wave_size == 32 ? guard : 0u, exec, 0u, 1u - expected};
    for (u32 i = 0; i < std::size(results); ++i) {
      AppendStoreSgpr(&code, 30u + i, static_cast<u32>(test.expected.size()));
      test.expected.push_back(results[i]);
    }
    if (dst != 106u) {
      AppendStoreSgpr(&code, 106, static_cast<u32>(test.expected.size()));
      AppendStoreSgpr(&code, 107, static_cast<u32>(test.expected.size()) + 1u);
      test.expected.insert(test.expected.end(), {0x12345678u, guard});
    }
  };
  for (u32 index = 0; index < values.size(); ++index) {
    AppendVMovU32(&code, 30, index * 8u);
    AppendBufferLoadDword(&code, 0, 30);
    AppendVMovU32(&code, 30, index * 8u + 4u);
    AppendBufferLoadDword(&code, 2, 30);
    for (u32 bit = 0; bit < 12; ++bit) {
      AppendVMovU32(&code, 1, bit == 11 ? 0 : 1u << bit);
      const u32 expected = bit == index ? 1u : 0u;
      compare({EncodeVopc(0x8f, Vgpr(0), 1)}, expected);
      compare({EncodeVop3Word0(0x8f, 20), EncodeVop3Word1(Vgpr(0), Vgpr(1))},
              expected, 20);
    }
    // SDWA selects the high half, applies -abs, and writes an explicit SGPR pair.
    compare({EncodeVopc(0x8f, 249, InlineU32(56)),
             EncodeVopcSdwa(2, 20, 1, 5, 6, 0, 0, 1, 1, 0, 0, 0, 1)},
            index >= 3 && index <= 8 ? 1u : 0u, 20);
    compare({EncodeVop3Word0(0x8f, 20, 1),
             EncodeVop3Word1(Vgpr(0), InlineU32(56), 0, 0, 1)},
            index >= 3 && index <= 8 ? 1u : 0u, 20);
  }
  AppendVMovU32(&code, 1, 1u << 8u);
  compare({EncodeVopc(0x8f, 242, 1)}, 1); // Inline 1.0 becomes half 0x3c00.
  compare({EncodeVopc(0x8f, 255, 1), 0xdead3c00u}, 1); // Literal low half.
  compare({EncodeVop3Word0(0x8f, 20), EncodeVop3Word1(242, 248)}, 1, 20);
  // Inline INV_2PI supplies half bits 0x3118, including the positive-normal class.
  code.push_back(EncodeSMovB32(126, InlineU32(0)));
  compare({EncodeVopc(0x8f, 242, 1)}, 0, 106, 0); // Inactive lanes clear the result.
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_CMP_EQ_U32, O::BUFFER_LOAD_DWORD,
                  O::V_CMP_CLASS_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_CMP_CLASS_F16", 264u}};
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase VectorCmpxClassF16(u32 wave_size) {
  using O = ShaderOpcode;
  // One raw half value for each architectural class bit, in bit order.
  constexpr std::array<u32, 10> values{
      0x7c01u, 0x7e01u, 0xfc00u, 0xbc00u, 0x8001u,
      0x8000u, 0x0000u, 0x0001u, 0x3c00u, 0x7c00u};
  TestCase test;
  test.name = wave_size == 64 ? "VectorCmpxClassF16Wave64" : "VectorCmpxClassF16Wave32";
  for (const auto value : values) {
    test.initial.insert(test.initial.end(), {0xa55a0000u | value, (value << 16u) | 0x7e01u});
  }
  test.expected = test.initial;
  auto &code = test.code;
  AppendSMovLiteral(&code, 106, 0x12345678u);
  AppendSMovLiteral(&code, 107, 0x87654321u);
  code.push_back(EncodeSMovB32(127, InlineU32(0)));
  const auto compare = [&](std::initializer_list<u32> words, u32 expected) {
    code.insert(code.end(), words);
    code.push_back(EncodeSMovB32(20, 126));
    code.push_back(EncodeSMovB32(21, 127));
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    AppendStoreSgpr(&code, 20, static_cast<u32>(test.expected.size()));
    AppendStoreSgpr(&code, 21, static_cast<u32>(test.expected.size()) + 1u);
    test.expected.insert(test.expected.end(), {expected, 0});
  };
  for (u32 index = 0; index < values.size(); ++index) {
    AppendVMovU32(&code, 30, index * 8u);
    AppendBufferLoadDword(&code, 0, 30);
    AppendVMovU32(&code, 30, index * 8u + 4u);
    AppendBufferLoadDword(&code, 2, 30);
    for (u32 bit = 0; bit < 12; ++bit) {
      AppendVMovU32(&code, 1, bit == 11 ? 0 : 1u << bit);
      const u32 expected = bit == index ? 1 : 0;
      if ((bit & 1u) == 0) {
        compare({EncodeVopc(0x9f, Vgpr(0), 1)}, expected);
      } else {
        compare({EncodeVop3Word0(0x9f, 126), EncodeVop3Word1(Vgpr(0), Vgpr(1))}, expected);
      }
    }
    const u32 finite = index >= 3 && index <= 8 ? 1 : 0;
    compare({0x7d3f70f9u, 0x86360000u}, finite); // Captured -abs(v0), inline56.
    compare({0x7d3f70f9u, 0x86350002u}, finite); // Captured -abs(v2.word1).
  }
  AppendVMovU32(&code, 1, 1u << 8u);
  compare({EncodeVopc(0x9f, 242, 1)}, 1); // Inline1.0 encodes half0x3c00.
  AppendVMovU32(&code, 1, 1u << 6u);
  compare({EncodeVopc(0x9f, 249, 1),
           EncodeVopcSdwa(242, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 1)}, 1);
  AppendVMovU32(&code, 1, 1u << 7u);
  compare({EncodeVopc(0x9f, 249, 1),
           EncodeVopcSdwa(242, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 1)}, 1);
  AppendVMovU32(&code, 30, 8u * 8u);
  AppendBufferLoadDword(&code, 0, 30);
  compare({EncodeVopc(0x9f, 249, 248),
           EncodeVopcSdwa(0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 1)}, 1);
  // INV_2PI as a half class mask is0x3118, including positive normal bit8.
  AppendVMovU32(&code, 1, 1u << 8u);
  code.push_back(EncodeSMovB32(126, InlineU32(0)));
  compare({EncodeVopc(0x9f, Vgpr(0), 1)}, 0);
  AppendStoreSgpr(&code, 106, static_cast<u32>(test.expected.size()));
  AppendStoreSgpr(&code, 107, static_cast<u32>(test.expected.size()) + 1u);
  test.expected.insert(test.expected.end(), {0x12345678u, 0x87654321u});
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_CMPX_CLASS_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase Wave32VccMasksPreserveOtherHalf() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "Wave32VccMasksPreserveOtherHalf";
  auto &code = test.code;
  AppendSMovLiteral(&code, 106, 0x12345678u);
  AppendSMovLiteral(&code, 107, 0x2468ace0u);
  AppendVMovU32(&code, 1, 20);
  u32 offset = 0;
  const auto check_masks = [&](u32 low, u32 high) {
    AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, offset);
    AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, offset + 4);
    AppendVop3(&code, 0x101, 2, InlineU32(10), Vgpr(1), 107);
    code.push_back(EncodeVop2(0x01, 3, InlineU32(10), 1));
    AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset + 8);
    AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, offset + 12);
    test.expected.insert(test.expected.end(), 4, low);
    test.expected.insert(test.expected.end(), 4, high);
    for (u32 mask : {high, low}) {
      for (u32 lane = 0; lane < 4; ++lane) {
        test.expected.push_back((mask & (1u << lane)) != 0 ? 20u : 10u);
      }
    }
    offset += 16;
  };
  // PPSA10737 keeps a table offset in VCC_LO while comparing into VCC_HI.
  AppendVop3(&code, 0xc2, 107, InlineU32(1), Vgpr(0));
  check_masks(0x12345678u, 2u);
  AppendVop3(&code, 0xc2, 106, InlineU32(2), Vgpr(0));
  check_masks(4u, 2u);
  code.push_back(EncodeSop2(0x10, 106, 106, 107)); // s_or_b32
  check_masks(6u, 2u);
  code.push_back(EncodeSop1(0x04, 106, 193u)); // Explicit s_mov_b64 writes both.
  check_masks(0xffffffffu, 0xffffffffu);
  AppendEnd(&code);
  test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_OR_B32, O::V_MOV_B32,
                  O::V_CMP_EQ_U32, O::V_CNDMASK_B32, O::V_LSHLREV_B32,
                  O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = 32;
  test.has_compute_info = true;
  return test;
}

TestCase VectorCompareF16Ops() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  std::vector<u32> expected;
  AppendVMovU32(&code, 1, 1);
  AppendVMovLiteral(&code, 2, 0x40003c00u); // low=1.0h, high=2.0h
  AppendVMovLiteral(&code, 3, 0x44004200u); // low=3.0h, high=4.0h
  AppendVMovLiteral(&code, 4, 0x00007e00u); // low=qNaN

  u32 dst = 10;
  u32 out = 0;
  auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) {
    code.push_back(EncodeVopc(opcode, src0, src1));
    code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1));
    AppendStoreVgpr(&code, dst, out++);
    expected.push_back(value ? 1u : 0u);
    dst++;
  };

  append_compare(0xc9, Vgpr(2), 3, true);
  append_compare(0xca, Vgpr(2), 2, true);
  append_compare(0xcb, Vgpr(2), 3, true);
  append_compare(0xcc, Vgpr(3), 2, true);
  append_compare(0xcd, Vgpr(2), 3, true);
  append_compare(0xce, Vgpr(3), 2, true);
  append_compare(0xe9, Vgpr(2), 3, true);
  append_compare(0xe9, Vgpr(3), 2, false);
  append_compare(0xe9, Vgpr(2), 2, false);
  append_compare(0xe9, Vgpr(4), 2, true);
  append_compare(0xe9, Vgpr(2), 4, true);
  append_compare(0xed, Vgpr(4), 2, true);

  auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) {
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    code.push_back(EncodeVopc(opcode, src0, src1));
    AppendStoreVgpr(&code, 1, out++);
    expected.push_back(1u);
  };

  append_cmpx(0xd9, Vgpr(2), 3);
  append_cmpx(0xda, Vgpr(2), 2);
  append_cmpx(0xdb, Vgpr(2), 3);
  append_cmpx(0xdc, Vgpr(3), 2);
  append_cmpx(0xde, Vgpr(3), 2);
  append_cmpx(0xfd, Vgpr(4), 2);
  append_cmpx(0xfe, Vgpr(2), 2);
  AppendEnd(&code);

  return {"VectorCompareF16Ops",
          code,
          {},
          expected,
          {O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_LT_F16, O::V_CMP_EQ_F16,
           O::V_CMP_LE_F16, O::V_CMP_GT_F16, O::V_CMP_LG_F16, O::V_CMP_GE_F16,
           O::V_CMP_NGE_F16, O::V_CMP_NEQ_F16, O::V_CMPX_LT_F16, O::V_CMPX_EQ_F16,
           O::V_CMPX_LE_F16, O::V_CMPX_GT_F16, O::V_CMPX_GE_F16,
           O::V_CMPX_NEQ_F16, O::V_CMPX_NLT_F16, O::V_CNDMASK_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorCompareI16U16Ops() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  std::vector<u32> expected;
  AppendVMovU32(&code, 1, 1);
  AppendVMovLiteral(&code, 2, 0xabcd8000u);
  AppendVMovLiteral(&code, 3, 0x00007fffu);
  AppendVMovLiteral(&code, 4, 0x12348000u);
  AppendVMovLiteral(&code, 5, 0xffff0001u);
  AppendVMovLiteral(&code, 6, 0x00000001u);

  u32 dst = 10;
  u32 out = 0;
  auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) {
    code.push_back(EncodeVopc(opcode, src0, src1));
    code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1));
    AppendStoreVgpr(&code, dst, out++);
    expected.push_back(value ? 1u : 0u);
    dst++;
  };

  append_compare(0x8a, Vgpr(2), 4, true);
  append_compare(0x8a, Vgpr(2), 3, false);
  append_compare(0x8b, Vgpr(2), 3, true);
  append_compare(0x8b, Vgpr(3), 2, false);
  append_compare(0x8b, Vgpr(5), 6, true);
  append_compare(0x8c, Vgpr(3), 2, true);
  append_compare(0x8c, Vgpr(2), 3, false);
  append_compare(0x8d, Vgpr(2), 3, true);
  append_compare(0x8d, Vgpr(5), 6, false);

  append_compare(0xaa, Vgpr(2), 4, true);
  append_compare(0xaa, Vgpr(2), 3, false);
  append_compare(0xab, Vgpr(3), 2, true);
  append_compare(0xab, Vgpr(2), 3, false);
  append_compare(0xac, Vgpr(2), 3, true);
  append_compare(0xac, Vgpr(3), 2, false);
  append_compare(0xad, Vgpr(2), 3, true);
  append_compare(0xad, Vgpr(5), 6, false);
  append_compare(0xae, Vgpr(2), 3, true);
  append_compare(0xae, Vgpr(3), 2, false);
  append_compare(0xae, Vgpr(5), 6, true);
  AppendEnd(&code);

  return {"VectorCompareI16U16Ops",
          code,
          {},
          expected,
          {O::V_MOV_B32, O::V_CMP_EQ_I16, O::V_CMP_LE_I16, O::V_CMP_GT_I16,
           O::V_CMP_NE_I16, O::V_CMP_EQ_U16, O::V_CMP_LE_U16, O::V_CMP_GT_U16,
           O::V_CMP_NE_U16, O::V_CMP_GE_U16, O::V_CNDMASK_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase Vop2SdwaCndmaskSourceModifier() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 6, 0x00000000u);
  AppendVMovLiteral(&code, 47, 0x12345678u);
  AppendVMovLiteral(&code, 53, 0x3f800000u);
  code.push_back(EncodeVopc(0xc7, Vgpr(6), 6)); // v_cmp_t_u32
  code.push_back(0x025e6af9u);
  code.push_back(0x16060635u);
  AppendStoreVgpr(&code, 47, 0);
  AppendEnd(&code);

  return {"Vop2SdwaCndmaskSourceModifier",
          code,
          {},
          {0xbf800000u},
          {O::V_MOV_B32, O::V_CMP_T_U32, O::V_CNDMASK_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase Vop2SdwaCndmaskCapturedByte3Source() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x55667788u);
  AppendBufferLoadDword(&code, 3, 30);
  code.push_back(EncodeVopc(0xc0, Vgpr(0), 0)); // v_cmp_f_u32
  code.push_back(0x023a00f9u);
  code.push_back(0x06030603u);
  AppendStoreVgpr(&code, 29, 1);
  code.push_back(EncodeVopc(0xc7, Vgpr(0), 0)); // v_cmp_t_u32
  code.push_back(0x023a00f9u);
  code.push_back(0x06030603u);
  AppendStoreVgpr(&code, 29, 2);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop2SdwaCndmaskCapturedByte3Source";
  test.code = code;
  test.initial = {0xabcd1234u, 0, 0};
  test.expected = {0xabcd1234u, 0x000000abu, 0x55667788u};
  test.opcodes = {O::V_MOV_B32,   O::BUFFER_LOAD_DWORD, O::V_CMP_F_U32,
                  O::V_CMP_T_U32, O::V_CNDMASK_B32,     O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.decoded_counts = {{"V_CNDMASK_B32 v29, v3.sdwa(sel=3,sext=0), v0", 2}};
  test.required_spirv = {"OpBitFieldUExtract", "OpSelect"};
  return test;
}

TestCase Vop3CndmaskUsesSgprMaskLaneBits() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVop1(0x01, 1, 0),          EncodeVop2(0x1a, 1, InlineU32(2), 1),
      EncodeVop2(0x25, 1, Vgpr(0), 1), EncodeVopc(0xc2, InlineU32(0), 0),
      EncodeSop1(0x04, 4, 106),
  };
  AppendVop3(&code, 0x101, 2, InlineU32(10), InlineU32(20), 4);
  code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 1));
  AppendBufferStoreDword(&code, 2, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "Vop3CndmaskUsesSgprMaskLaneBits";
  test.code = code;
  test.expected = {20, 10, 10, 10, 20, 10, 10, 10};
  test.opcodes = {O::V_MOV_B32,          O::V_LSHLREV_B32, O::V_ADD_NC_U32,
                  O::V_CMP_EQ_U32,       O::S_MOV_B64,     O::V_CNDMASK_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.has_compute_info = true;
  test.dispatch_x = 2;
  return test;
}

TestCase Vop3CndmaskAllowsDataSourceModifier() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 7, 0x3f800000u);
  AppendSMovLiteral(&code, 34, 1);
  AppendSMovLiteral(&code, 35, 0);
  code.push_back(0xd5010004u);
  code.push_back(0x408a0f07u);
  AppendStoreVgpr(&code, 4, 0);
  AppendEnd(&code);

  return {"Vop3CndmaskAllowsDataSourceModifier",
          code,
          {},
          {0xbf800000u},
          {O::V_MOV_B32, O::S_MOV_B32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase VectorCompareExecOps() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
  AppendVMovLiteral(&code, 2, 0x40000000u);
  AppendVMovLiteral(&code, 3, 0x40400000u);
  code.push_back(EncodeVop1(0x01, 4, InlineU32(1)));
  AppendVMovLiteral(&code, 5, 0xffffffffu);

  u32 out = 0;
  auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) {
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    code.push_back(EncodeVopc(opcode, src0, src1));
    AppendStoreVgpr(&code, 1, out++);
  };

  append_cmpx(0x11, Vgpr(2), 3);
  append_cmpx(0x12, Vgpr(3), 3);
  append_cmpx(0x13, Vgpr(2), 3);
  append_cmpx(0x14, Vgpr(3), 2);
  append_cmpx(0x15, Vgpr(2), 3);
  append_cmpx(0x16, Vgpr(3), 3);
  append_cmpx(0x19, Vgpr(2), 3);
  append_cmpx(0x1a, Vgpr(3), 3);
  append_cmpx(0x1b, Vgpr(2), 3);
  append_cmpx(0x1c, Vgpr(3), 2);
  append_cmpx(0x1d, Vgpr(2), 3);
  append_cmpx(0x1e, Vgpr(3), 3);
  append_cmpx(0x91, Vgpr(5), 4);
  append_cmpx(0x92, Vgpr(4), 4);
  append_cmpx(0x93, Vgpr(5), 4);
  append_cmpx(0x94, Vgpr(4), 5);
  append_cmpx(0x95, Vgpr(5), 4);
  append_cmpx(0x96, Vgpr(4), 4);
  append_cmpx(0xd1, Vgpr(4), 5);
  append_cmpx(0xd2, Vgpr(4), 4);
  append_cmpx(0xd3, Vgpr(4), 5);
  append_cmpx(0xd4, Vgpr(5), 4);
  append_cmpx(0xd5, Vgpr(4), 5);
  append_cmpx(0xd6, Vgpr(4), 4);
  AppendEnd(&code);

  return {"VectorCompareExecOps",
          code,
          {},
          std::vector<u32>(out, 1),
          {O::S_MOV_B32,      O::V_MOV_B32,      O::V_CMPX_LT_F32,
           O::V_CMPX_EQ_F32,  O::V_CMPX_LE_F32,  O::V_CMPX_GT_F32,
           O::V_CMPX_LG_F32,  O::V_CMPX_GE_F32,  O::V_CMPX_NGE_F32,
           O::V_CMPX_NLG_F32, O::V_CMPX_NGT_F32, O::V_CMPX_NLE_F32,
           O::V_CMPX_NEQ_F32, O::V_CMPX_NLT_F32, O::V_CMPX_LT_I32,
           O::V_CMPX_EQ_I32,  O::V_CMPX_LE_I32,  O::V_CMPX_GT_I32,
           O::V_CMPX_NE_I32,  O::V_CMPX_GE_I32,  O::V_CMPX_LT_U32,
           O::V_CMPX_EQ_U32,  O::V_CMPX_LE_U32,  O::V_CMPX_GT_U32,
           O::V_CMPX_NE_U32,  O::V_CMPX_GE_U32,  O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase VectorVopcCmpxOrderedCapturedExecMask() {
  using O = ShaderOpcode;
  struct CompareCase {
    u32 lhs;
    u32 rhs;
    u32 initial_exec;
    u32 expected_exec;
    bool captured;
  };
  constexpr std::array<CompareCase, 8> cases{{
      {0x3f800000u, 0x3f800000u, 1u, 1u, true}, // Finite value.
      {0x7fc00000u, 0x7fc00000u, 1u, 0u, true}, // Quiet NaN.
      {0x7f800001u, 0x7f800001u, 1u, 0u, true}, // Signaling NaN.
      {0x7f800000u, 0x7f800000u, 1u, 1u, true}, // Infinity is ordered.
      {0x80000000u, 0x80000000u, 1u, 1u, true}, // Signed zero is ordered.
      {0x3f800000u, 0x3f800000u, 0u, 0u, true}, // Inactive lane.
      {0x3f800000u, 0x7fc00000u, 1u, 0u, false}, // NaN in SRC1.
      {0x7fc00000u, 0x3f800000u, 1u, 0u, false}, // NaN in SRC0.
  }};
  constexpr u32 vcc_lo = 0x13579bdfu;
  constexpr u32 vcc_hi = 0x2468ace0u;

  TestCase test;
  test.name = "VectorVopcCmpxOrderedCapturedExecMask";
  auto &code = test.code;
  for (const auto &entry : cases) {
    test.initial.push_back(entry.lhs);
    test.initial.push_back(entry.rhs);
  }
  test.expected = test.initial;
  for (u32 i = 0; i < cases.size(); ++i) {
    const auto &entry = cases[i];
    AppendVMovU32(&code, 30, i * 8u);
    AppendBufferLoadDword(&code, 1, 30);
    AppendVMovU32(&code, 30, i * 8u + 4u);
    AppendBufferLoadDword(&code, 2, 30);
    AppendSMovLiteral(&code, 106, vcc_lo);
    AppendSMovLiteral(&code, 107, vcc_hi);
    code.push_back(EncodeSMovB32(126, InlineU32(entry.initial_exec)));
    code.push_back(entry.captured ? 0x7c2e0301u
                                  : EncodeVopc(0x17u, Vgpr(1), 2));
    code.push_back(EncodeSMovB32(20, 126));
    code.push_back(EncodeSMovB32(21, 127));
    code.push_back(EncodeSMovB32(22, 106));
    code.push_back(EncodeSMovB32(23, 107));
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    const u32 out = static_cast<u32>(cases.size()) * 2u + i * 4u;
    AppendStoreSgprPair(&code, 20, out);
    AppendStoreSgprPair(&code, 22, out + 2u);
    test.expected.insert(test.expected.end(),
                         {entry.expected_exec, 0u, vcc_lo, vcc_hi});
  }
  test.initial.resize(test.expected.size());
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_CMPX_O_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_CMPX_O_F32 exec_lo, v1, v1", 6u},
                         {"V_CMPX_O_F32 exec_lo, v1, v2", 2u}};
  test.compute_info.wave_size = 32;
  test.has_compute_info = true;
  return test;
}

TestCase VectorVop3FloatCompareNegSourceModifier() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovU32(&code, 1, 0);
  AppendVop3(&code, 0x01, 20, Vgpr(0), Vgpr(1), 0, 0, 0, false, 0,
             0x1); // -1.0 < 0.0
  AppendStoreSgprPair(&code, 20, 0);
  AppendEnd(&code);

  return {"VectorVop3FloatCompareNegSourceModifier",
          code,
          {},
          {1, 0},
          {O::V_MOV_B32, O::V_CMP_LT_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorVop3CmpxWritesExecMask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 0, 2);
  AppendVMovU32(&code, 1, 1);
  AppendVMovU32(&code, 2, 0);
  AppendVMovU32(&code, 30, 0);
  AppendVop3(&code, 0xd1, 5, Vgpr(0), Vgpr(1)); // v_cmpx_lt_u32, false
  AppendVMovU32(&code, 2, 7);
  AppendBufferStoreDword(&code, 2, 30);
  AppendEnd(&code);

  return {"VectorVop3CmpxWritesExecMask",
          code,
          {0},
          {0},
          {O::V_MOV_B32, O::V_CMPX_LT_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorVopcSdwaCmpxWritesExecMask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 0, 2);
  AppendVMovU32(&code, 1, 1);
  AppendVMovU32(&code, 2, 0);
  AppendVMovU32(&code, 30, 0);
  code.push_back(EncodeVopc(0xd1, 249, 1)); // v_cmpx_lt_u32.sdwa, false
  code.push_back(EncodeVopcSdwa(0));
  AppendVMovU32(&code, 2, 7);
  AppendBufferStoreDword(&code, 2, 30);
  AppendEnd(&code);

  return {"VectorVopcSdwaCmpxWritesExecMask",
          code,
          {0},
          {0},
          {O::V_MOV_B32, O::V_CMPX_LT_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase VectorVopcCmpxGtU16CapturedSdwaExecMask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 2, 0x00008000u);
  AppendVMovU32(&code, 3, 7);
  AppendVMovU32(&code, 31, 0);
  code.insert(code.end(), {0x7d7900f9u, 0x86060002u});
  AppendBufferStoreDword(&code, 3, 31);
  AppendVMovU32(&code, 2, 0x00010000u);
  AppendVMovU32(&code, 3, 9);
  AppendVMovU32(&code, 31, 4);
  code.insert(code.end(), {0x7d7900f9u, 0x86060002u});
  AppendBufferStoreDword(&code, 3, 31);
  AppendEnd(&code);

  TestCase test{"VectorVopcCmpxGtU16CapturedSdwaExecMask",
                code,
                {0x11111111u, 0x22222222u},
                {7, 0x22222222u},
                {O::V_MOV_B32, O::V_CMPX_GT_U16, O::BUFFER_STORE_DWORD,
                 O::S_ENDPGM}};
  test.decoded_counts = {{"V_CMPX_GT_U16", 2}};
  return test;
}

TestCase VectorVopcCmpxLtU16CapturedSdwaExecMask() {
  using O = ShaderOpcode;
  struct CompareCase {
    u32 lhs;
    u32 rhs;
    u32 incoming_exec;
    u32 expected_exec;
    u32 encoding = 0; // Captured SDWA, compact, VOP3, WORD_1 src0/src1.
  };
  const std::array<CompareCase, 12> cases{{
      {0xffff0000u, 0x12340001u, 1, 1}, // Ignore high halves.
      {0xabcd0001u, 0x12340001u, 1, 0}, // Strict comparison: equal is false.
      {0x0000ffffu, 0x12340001u, 1, 0}, // Unsigned, not signed -1.
      {0x00007fffu, 0x12348000u, 1, 1},
      {0x00008000u, 0x12347fffu, 1, 0},
      {0x12340000u, 0x56780000u, 1, 0},
      {0x0000fffeu, 0x1234ffffu, 1, 1},
      {0x00000000u, 0x12340001u, 0, 0}, // CMPX cannot reactivate an inactive lane.
      {0x00008000u, 0x0000ffffu, 1, 1, 1},
      {0x0000ffffu, 0x00008000u, 1, 0, 2},
      {0x0001ffffu, 0x12340002u, 1, 1, 3},
      {0x00000001u, 0x00020000u, 1, 1, 4},
  }};
  constexpr u32 vcc_hi = 0x89abcdefu;
  TestCase test;
  test.name = "VectorVopcCmpxLtU16CapturedSdwaExecMask";
  for (const auto &entry : cases) {
    test.initial.push_back(entry.lhs);
  }
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < cases.size(); ++i) {
    const auto &entry = cases[i];
    AppendVMovU32(&code, 30, i * 4u);
    AppendBufferLoadDword(&code, 0, 30); // Runtime input prevents constant folding.
    AppendVMovU32(&code, 1, entry.rhs);
    AppendSMovLiteral(&code, 106, entry.rhs);
    AppendSMovLiteral(&code, 107, vcc_hi);
    code.push_back(EncodeSMovB32(126, InlineU32(entry.incoming_exec)));
    switch (entry.encoding) {
    case 1: code.push_back(0x7d720300u); break; // compact v0, v1
    case 2: code.insert(code.end(), {0xd4b9007eu, 0x00020300u}); break;
    case 3: code.insert(code.end(), {0x7d72d4f9u, 0x86050000u}); break;
    case 4: code.insert(code.end(), {0x7d72d4f9u, 0x85060000u}); break;
    default: code.insert(code.end(), {0x7d72d4f9u, 0x86060000u}); break;
    }
    code.push_back(EncodeSMovB32(20, 126)); // Snapshot EXEC before restoring it.
    code.push_back(EncodeSMovB32(21, 127));
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    const u32 out = static_cast<u32>(cases.size()) + i * 4u;
    AppendStoreSgprPair(&code, 20, out);
    AppendStoreSgprPair(&code, 106, out + 2u); // CMPX must not overwrite either VCC half.
    test.expected.insert(test.expected.end(),
                         {entry.expected_exec, 0u, entry.rhs, vcc_hi});
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_CMPX_LT_U16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_CMPX_LT_U16", cases.size()}};
  test.required_spirv = {"OpULessThan"};
  return test;
}

TestCase VectorVopcCmpxLtI16CapturedSdwaExecMask() {
  using O = ShaderOpcode;
  enum Encoding { Captured, Compact, Vop3, HighLhs, HighRhs, SdwaOperands };
  struct CompareCase {
    u32 lhs;
    u32 rhs;
    u32 expected_exec;
    Encoding encoding = Captured;
    u32 incoming_exec = 1;
    u32 src0_sel = 6;
    u32 src1_sel = 6;
    u32 src0_sext = 0;
    u32 src1_sext = 0;
  };
  const std::array<CompareCase, 30> cases{{
      {0xffff000eu, 15u, 1}, // Ignore the high half: 14 < 15.
      {0x8000000fu, 15u, 0}, // Equality is false despite a negative high half.
      {0x00000010u, 15u, 0},
      {0x7fff0000u, 15u, 1},
      {0x0000ffffu, 15u, 1}, // Signed -1, not unsigned 65535.
      {0x00008000u, 15u, 1}, // Minimum signed halfword.
      {0x00007fffu, 15u, 0}, // Maximum signed halfword.
      {0x0000fff0u, 15u, 1},
      {0x0000ffffu, 15u, 0, Captured, 0}, // Cannot reactivate an inactive lane.
      {0xffff000fu, 15u, 0, Captured, 0},
      {0x1234ffffu, 0xabcd0000u, 1, Compact}, // -1 < 0.
      {0xffff0000u, 0x0000ffffu, 0, Compact}, // 0 < -1 is false.
      {0xabcd8000u, 0x12347fffu, 1, Vop3},
      {0x12347fffu, 0xabcd8000u, 0, Vop3},
      {0xffff0010u, 15u, 1, HighLhs}, // Select -1 in WORD_1.
      {0x0010ffffu, 15u, 0, HighLhs}, // Select 16, not the negative low half.
      {0x1234fffeu, 0xffff8000u, 1, HighRhs}, // -2 < selected -1.
      {0x1234ffffu, 0xabcd0000u, 1, SdwaOperands, 1, 4, 4, 1, 1},
      {0x12345680u, 0u, 1, SdwaOperands, 1, 0, 6, 1, 0}, // BYTE_0 SEXT: -128.
      {0x1234567fu, 0u, 0, SdwaOperands, 1, 0, 6, 1, 0}, // +127.
      {0x12345680u, 0u, 0, SdwaOperands, 1, 0, 6, 0, 0}, // Without SEXT: +128.
      {0x12348000u, 0u, 1, SdwaOperands, 1, 1, 6, 1, 0},
      {0x12803456u, 0u, 1, SdwaOperands, 1, 2, 6, 1, 0},
      {0x80345678u, 0u, 1, SdwaOperands, 1, 3, 6, 1, 0},
      {0x0000ffffu, 0x12345680u, 0, SdwaOperands, 1, 6, 0, 0, 1}, // -1 < -128 is false.
      {0x0000ff7fu, 0x12345680u, 1, SdwaOperands, 1, 6, 0, 0, 1}, // -129 < -128.
      {0x0000ffffu, 0x12347f00u, 1, SdwaOperands, 1, 6, 1, 0, 1},
      {0u, 0x12803456u, 0, SdwaOperands, 1, 6, 2, 0, 1},
      {0u, 0x80345678u, 0, SdwaOperands, 1, 6, 3, 0, 1},
      {0xff000001u, 0x000100ffu, 1, SdwaOperands, 1, 3, 2, 1, 1},
  }};
  constexpr u32 vcc_lo = 0x76543210u;
  constexpr u32 vcc_hi = 0x89abcdefu;
  TestCase test;
  test.name = "VectorVopcCmpxLtI16CapturedSdwaExecMask";
  for (const auto &entry : cases) {
    test.initial.push_back(entry.lhs);
  }
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < cases.size(); ++i) {
    const auto &entry = cases[i];
    AppendVMovU32(&code, 30, i * 4u);
    AppendBufferLoadDword(&code, 1, 30); // Runtime input prevents constant folding.
    AppendVMovU32(&code, 2, entry.rhs);
    AppendSMovLiteral(&code, 106, vcc_lo);
    AppendSMovLiteral(&code, 107, vcc_hi);
    code.push_back(EncodeSMovB32(126, InlineU32(entry.incoming_exec)));
    switch (entry.encoding) {
    case Compact: code.push_back(0x7d320501u); break;
    case Vop3: code.insert(code.end(), {0xd499007eu, 0x00020501u}); break;
    case HighLhs: code.insert(code.end(), {0x7d331ef9u, 0x86050001u}); break;
    case HighRhs:
      code.push_back(EncodeVopc(0x99u, 249u, 2u));
      code.push_back(EncodeVopcSdwa(1u, 0u, 0u, 6u, 5u));
      break;
    case SdwaOperands:
      code.push_back(EncodeVopc(0x99u, 249u, 2u));
      code.push_back(EncodeVopcSdwa(1u, 0u, 0u, entry.src0_sel, entry.src1_sel,
                                  entry.src0_sext, entry.src1_sext));
      break;
    default: code.insert(code.end(), {0x7d331ef9u, 0x86060001u}); break;
    }
    code.push_back(EncodeSMovB32(20, 126)); // Snapshot EXEC before restoring it.
    code.push_back(EncodeSMovB32(21, 127));
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    const u32 out = static_cast<u32>(cases.size()) + i * 4u;
    AppendStoreSgprPair(&code, 20, out);
    AppendStoreSgprPair(&code, 106, out + 2u); // CMPX preserves both VCC halves.
    test.expected.insert(test.expected.end(),
                         {entry.expected_exec, 0u, vcc_lo, vcc_hi});
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_CMPX_LT_I16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_CMPX_LT_I16", cases.size()}};
  test.required_spirv = {"OpSLessThan", "OpBitFieldSExtract"};
  return test;
}

TestCase VectorVopcCmpxLtI16WaveMasks(u32 wave_size) {
  using O = ShaderOpcode;
  constexpr u32 marker = 0xfeedabcdu;
  constexpr u32 vcc_lo = 0x01234567u;
  constexpr u32 vcc_hi = 0x89abcdefu;
  constexpr u32 expected_exec_lo = 0xaaaaaaaau;
  const u32 expected_exec_hi = wave_size == 64u ? 0x33333333u : 0u;
  TestCase test;
  test.name = wave_size == 32u ? "VectorVopcCmpxLtI16Wave32Masks"
                              : "VectorVopcCmpxLtI16Wave64Masks";
  test.initial.assign(6u * wave_size, 0xdeadbeefu);
  constexpr u32 inputs[] = {0xffff000eu, 0x0000ffffu, 0x8000000fu, 0x12348000u};
  for (u32 lane = 0; lane < wave_size; ++lane) {
    test.initial[lane] = inputs[lane % 4u];
  }
  test.expected = test.initial;
  for (u32 lane = 0; lane < wave_size; ++lane) {
    test.expected[wave_size + lane] = expected_exec_lo;
    test.expected[2u * wave_size + lane] = expected_exec_hi;
    test.expected[3u * wave_size + lane] = vcc_lo;
    test.expected[4u * wave_size + lane] = vcc_hi;
    const u32 mask = lane < 32u ? expected_exec_lo : expected_exec_hi;
    if ((mask & (1u << (lane % 32u))) != 0u) {
      test.expected[5u * wave_size + lane] = marker;
    }
  }
  auto &code = test.code;
  code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
  AppendBufferLoadDword(&code, 1, 30);
  AppendVMovLiteral(&code, 4, marker);
  code.push_back(EncodeSop1(0x04, 10, 126)); // Save the full incoming EXEC.
  AppendSMovLiteral(&code, 106, vcc_lo);
  AppendSMovLiteral(&code, 107, vcc_hi);
  AppendSMovLiteral(&code, 126, 0xeeeeeeeeu);
  AppendSMovLiteral(&code, 127, 0x77777777u);
  code.insert(code.end(), {0x7d331ef9u, 0x86060001u});
  code.push_back(EncodeSMovB32(20, 126));
  code.push_back(EncodeSMovB32(21, 127));
  AppendStoreVgprAtLaneDwordOffset(&code, 4, 0, 5u * wave_size);
  code.push_back(EncodeSop1(0x04, 126, 10)); // Restore lanes to inspect the raw masks.
  AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, wave_size);
  AppendStoreSgprAtLaneDwordOffset(&code, 21, 0, 2u * wave_size);
  AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, 3u * wave_size);
  AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, 4u * wave_size);
  AppendEnd(&code);
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_LSHLREV_B32,
                  O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD, O::V_CMPX_LT_I16,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpSLessThan", "OpBitFieldSExtract"};
  return test;
}

TestCase VectorVopcCmpxU16SdwaCompactVop3ExecMask(bool not_equal) {
  using O = ShaderOpcode;
  struct CompareCase {
    u32 lhs;
    u32 rhs;
    u32 incoming_exec;
    u32 expected_exec;
    u32 encoding = 0; // SDWA, compact, VOP3, sign-extended words/bytes, literal.
  };
  const std::array<CompareCase, 21> cases{{
      {0x12340000u, 0x56780000u, 1, 1}, // Equal low half; ignore high half.
      {0xffff0001u, 0xabcd0001u, 1, 1},
      {0x00000000u, 0x12340000u, 1, 1},
      {0xaaaa5555u, 0xbbbb5555u, 1, 1},
      {0x00000000u, 0x00000001u, 1, 0},
      {0x0000ffffu, 0x00000000u, 1, 0},
      {0x7fff8000u, 0x00018000u, 1, 1}, // Equal low halves despite different high halves.
      {0x00000000u, 0x00010000u, 1, 1},
      {0x12340001u, 0x12340002u, 1, 0, 1},
      {0x00008000u, 0x0000ffffu, 1, 0, 2},
      {0x0001ffffu, 0x12340001u, 1, 0, 3}, // Sign-extended low halves differ.
      {0x00000001u, 0x00010001u, 0, 0}, // CMPX cannot reactivate an inactive lane.
      {0x12345680u, 0x0000ff80u, 1, 1, 4}, // SEXT BYTE_0 is unsigned U16 0xff80.
      {0x123456ffu, 0x0000ffffu, 1, 1, 4},
      {0x0000ff80u, 0x12348000u, 1, 1, 5}, // SEXT BYTE_1 on src1.
      {0x1234ffffu, 0x5678ffffu, 1, 1, 6}, // SEXT WORD_0 versus DWORD low half.
      {0xabcd0001u, 1u, 1, 1, 7}, // Literal source, equal low halves.
      {0xabcd0000u, 1u, 1, 0, 7},
      {0x00000000u, 1u, 0, 0}, // Neither EQ nor NE may reactivate an inactive lane.
      {0x00010000u, 0xffff0001u, 1, 1, 8}, // Select the high word of src0.
      {0xabcd0002u, 0x0002ffffu, 1, 1, 9}, // Select the high word of src1.
  }};
  const u32 opcode = not_equal ? 0xbdu : 0xbau;
  constexpr u32 vcc_hi = 0x89abcdefu;
  TestCase test;
  test.name = not_equal ? "VectorVopcCmpxNeU16SdwaCompactVop3ExecMask"
                        : "VectorVopcCmpxEqU16SdwaCompactVop3ExecMask";
  for (const auto &entry : cases) {
    test.initial.push_back(entry.lhs);
  }
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < cases.size(); ++i) {
    const auto &entry = cases[i];
    AppendVMovU32(&code, 30, i * 4u);
    AppendBufferLoadDword(&code, 0, 30); // Runtime input prevents constant folding.
    AppendVMovU32(&code, 1, entry.rhs);
    AppendSMovLiteral(&code, 106, entry.rhs);
    AppendSMovLiteral(&code, 107, vcc_hi);
    code.push_back(EncodeSMovB32(126, InlineU32(entry.incoming_exec)));
    switch (entry.encoding) {
    case 1: code.push_back(EncodeVopc(opcode, Vgpr(0), 1u)); break;
    case 2: AppendVop3(&code, opcode, 126u, Vgpr(0), Vgpr(1)); break;
    case 3:
      code.push_back(EncodeVopc(opcode, 249u, 1u));
      code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 6u, 6u, 1u, 1u));
      break;
    case 4:
      code.push_back(EncodeVopc(opcode, 249u, 1u));
      code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 0u, 6u, 1u, 0u));
      break;
    case 5:
      code.push_back(EncodeVopc(opcode, 249u, 1u));
      code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 6u, 1u, 0u, 1u));
      break;
    case 6:
      code.push_back(EncodeVopc(opcode, 249u, 1u));
      code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 4u, 6u, 1u, 0u));
      break;
    case 7:
      code.push_back(EncodeVopc(opcode, 255u, 0u));
      code.push_back(entry.rhs);
      break;
    case 8:
    case 9:
      code.push_back(EncodeVopc(opcode, 249u, 1u));
      code.push_back(EncodeVopcSdwa(0u, 0u, 0u, entry.encoding == 8u ? 5u : 6u,
                                  entry.encoding == 9u ? 5u : 6u));
      break;
    default:
      code.push_back(EncodeVopc(opcode, 249u, 1u));
      code.push_back(EncodeVopcSdwa(0u));
      break;
    }
    code.push_back(EncodeSMovB32(20, 126)); // Snapshot EXEC before restoring it.
    code.push_back(EncodeSMovB32(21, 127));
    code.push_back(EncodeSMovB32(126, InlineU32(1u)));
    const u32 out = static_cast<u32>(cases.size()) + i * 4u;
    AppendStoreSgprPair(&code, 20, out);
    AppendStoreSgprPair(&code, 106, out + 2u); // CMPX must not overwrite either VCC half.
    const u32 expected_exec = not_equal ? entry.incoming_exec & (entry.expected_exec ^ 1u)
                                        : entry.expected_exec;
    test.expected.insert(test.expected.end(), {expected_exec, 0u, entry.rhs, vcc_hi});
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  not_equal ? O::V_CMPX_NE_U16 : O::V_CMPX_EQ_U16,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{not_equal ? "V_CMPX_NE_U16" : "V_CMPX_EQ_U16", cases.size()}};
  test.required_spirv = {not_equal ? "OpINotEqual" : "OpIEqual"};
  return test;
}

TestCase VectorVopcCmpxEqU16SdwaCompactVop3ExecMask() {
  return VectorVopcCmpxU16SdwaCompactVop3ExecMask(false);
}

TestCase VectorVopcCmpxNeU16SdwaCompactVop3ExecMask() {
  return VectorVopcCmpxU16SdwaCompactVop3ExecMask(true);
}

TestCase VectorVopcCmpNgtF16CapturedSdwaAndEdges() {
  using O = ShaderOpcode;
  enum Encoding { Captured, Compact, Vop3NegAbs, HighWord, ScalarDst, SdwaNegAbs };
  struct CompareCase {
    u32 lhs;
    u32 rhs;
    u32 expected;
    Encoding encoding = Captured;
    u32 exec = 1;
  };
  const std::array<CompareCase, 17> cases{{
      {0x7e003c00u, 0, 0}, // +1; ignore NaN in the upper half.
      {0x3c00bc00u, 0, 1}, // -1
      {0x3c000000u, 0, 1}, // +0
      {0x3c008000u, 0, 1}, // -0
      {0x3c007e00u, 0, 1}, // Quiet NaN: NGT differs from ordered LE.
      {0x3c007c01u, 0, 1}, // Signaling NaN
      {0x00007c00u, 0, 0}, // +infinity
      {0x0000fc00u, 0, 1}, // -infinity
      {0x0000bc00u, 0, 0, Captured, 0}, // Inactive lane contributes no VCC bit.
      {0x00003c00u, 0x00007e00u, 1, Compact}, // NaN in src1.
      {0x00004000u, 0x00003c00u, 0, Compact}, // +2 > +1
      {0x0000bc00u, 0, 1, Vop3NegAbs}, // -abs(-1)
      {0x00003c00u, 0, 1, Vop3NegAbs}, // -abs(+1)
      {0xbc004000u, 0x3c00c000u, 1, HighWord}, // Select both upper halves.
      {0x00004000u, 0x00003c00u, 0, ScalarDst},
      {0x00003c00u, 0x00004000u, 1, ScalarDst},
      {0x0000bc00u, 0, 1, SdwaNegAbs},
  }};
  constexpr u32 vcc_sentinel = 0x89abcdefu;
  TestCase test;
  test.name = "VectorVopcCmpNgtF16CapturedSdwaAndEdges";
  for (const auto &entry : cases) {
    test.initial.insert(test.initial.end(), {entry.lhs, entry.rhs});
  }
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < cases.size(); ++i) {
    const auto &entry = cases[i];
    AppendVMovU32(&code, 30, i * 8u);
    AppendBufferLoadDword(&code, 21, 30); // Runtime inputs prevent constant folding.
    AppendVMovU32(&code, 30, i * 8u + 4u);
    AppendBufferLoadDword(&code, 1, 30);
    AppendSMovLiteral(&code, 106, vcc_sentinel);
    AppendSMovLiteral(&code, 107, vcc_sentinel);
    code.push_back(EncodeSMovB32(126, InlineU32(entry.exec)));
    switch (entry.encoding) {
    case Captured: code.insert(code.end(), {0x7dd700f9u, 0x86060015u}); break;
    case Compact: code.push_back(EncodeVopc(0xeb, Vgpr(21), 1)); break;
    case Vop3NegAbs:
      AppendVop3(&code, 0xeb, 106, Vgpr(21), Vgpr(1), 0, 1, 0, false, 0, 1);
      break;
    case HighWord:
      code.push_back(EncodeVopc(0xeb, 249, 1));
      code.push_back(EncodeVopcSdwa(21, 0, 0, 5, 5));
      break;
    case ScalarDst:
      code.push_back(EncodeVopc(0xeb, 249, 1));
      code.push_back(EncodeVopcSdwa(21, 22, 1));
      break;
    case SdwaNegAbs:
      code.push_back(EncodeVopc(0xeb, 249, 1));
      code.push_back(EncodeVopcSdwa(21, 0, 0, 6, 6, 0, 0, 1, 1));
      break;
    }
    code.push_back(EncodeSMovB32(20, entry.encoding == ScalarDst ? 22 : 106));
    code.push_back(EncodeSMovB32(21, 126)); // CMP must preserve EXEC even if false.
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    const u32 out = static_cast<u32>(test.initial.size()) + i * 4u;
    AppendStoreSgprPair(&code, 20, out);
    AppendStoreSgprPair(&code, 106, out + 2u);
    test.expected.insert(test.expected.end(),
                         {entry.expected, entry.exec,
                          entry.encoding == ScalarDst ? vcc_sentinel : entry.expected,
                          vcc_sentinel});
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_CMP_NGT_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_CMP_NGT_F16", cases.size()}};
  test.required_spirv = {"OpFUnordLessThanEqual"};
  test.compute_info.wave_size = 32;
  test.has_compute_info = true;
  return test;
}

TestCase VectorVopcCmpNltF16CapturedSdwaAndEdges() {
  using O = ShaderOpcode;
  enum Encoding { Captured, Compact, Vop3NegAbs, SdwaNegAbs };
  struct CompareCase {
    u32 packed; // Captured instruction compares low half against high half.
    u32 expected;
    Encoding encoding = Captured;
    u32 exec = 1;
  };
  const std::array<CompareCase, 16> cases{{
      {0x40003c00u, 0}, // +1 < +2
      {0x3c004000u, 1}, // +2 > +1
      {0x3c003c00u, 1}, // Equal
      {0x80000000u, 1}, // +0 versus -0
      {0x00008000u, 1}, // -0 versus +0
      {0x3c007e00u, 1}, // Quiet NaN in src0: unordered GE, not ordered GE.
      {0x7e003c00u, 1}, // Quiet NaN in src1
      {0x3c007c01u, 1}, // Signaling NaN in src0
      {0x7c013c00u, 1}, // Signaling NaN in src1
      {0xfc007c00u, 1}, // +infinity versus -infinity
      {0x7c00fc00u, 0}, // -infinity versus +infinity
      {0x00003c00u, 0, Captured, 0}, // Inactive lane contributes no mask bit.
      {0x40003c00u, 0, Compact},
      {0x7e003c00u, 1, Compact},
      {0x0000bc00u, 0, Vop3NegAbs}, // -abs(-1) < 0
      {0x0000bc00u, 0, SdwaNegAbs},
  }};
  constexpr u32 sentinel = 0x89abcdefu;
  TestCase test;
  test.name = "VectorVopcCmpNltF16CapturedSdwaAndEdges";
  for (const auto &entry : cases) {
    test.initial.push_back(entry.packed);
  }
  test.expected = test.initial;
  auto &code = test.code;
  code.push_back(EncodeSMovB32(22, 2)); // Preserve the input buffer descriptor.
  code.push_back(EncodeSMovB32(23, 3));
  for (u32 i = 0; i < cases.size(); ++i) {
    const auto &entry = cases[i];
    AppendVMovU32(&code, 30, i * 4u);
    AppendBufferLoadDword(&code, 3, 30); // Runtime input prevents constant folding.
    AppendVMovU32(&code, 1, entry.packed >> 16u);
    for (u32 reg : {3u, 106u, 107u}) {
      AppendSMovLiteral(&code, reg, sentinel);
    }
    code.push_back(EncodeSMovB32(126, InlineU32(entry.exec)));
    switch (entry.encoding) {
    case Captured: code.insert(code.end(), {0x7ddc06f9u, 0x05068203u}); break;
    case Compact: code.push_back(EncodeVopc(0xee, Vgpr(3), 1)); break;
    case Vop3NegAbs:
      AppendVop3(&code, 0xee, 2, Vgpr(3), Vgpr(1), 0, 1, 0, false, 0, 1);
      break;
    case SdwaNegAbs:
      code.insert(code.end(), {0x7ddc06f9u, 0x05368203u});
      break;
    }
    code.push_back(EncodeSMovB32(20, entry.encoding == Compact ? 106 : 2));
    code.push_back(EncodeSMovB32(21, 3)); // Wave32 must preserve adjacent SGPR.
    code.push_back(EncodeSMovB32(24, 126));
    code.push_back(EncodeSMovB32(25, 127));
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    code.push_back(EncodeSMovB32(2, 22));
    code.push_back(EncodeSMovB32(3, 23));
    const u32 out = static_cast<u32>(cases.size()) + i * 6u;
    AppendStoreSgprPair(&code, 20, out);
    AppendStoreSgprPair(&code, 24, out + 2u);
    AppendStoreSgprPair(&code, 106, out + 4u);
    test.expected.insert(test.expected.end(),
                         {entry.expected, sentinel, entry.exec, 0u,
                          entry.encoding == Compact ? entry.expected : sentinel,
                          sentinel});
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_CMP_NLT_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_CMP_NLT_F16", cases.size()}};
  test.required_spirv = {"OpFUnordGreaterThanEqual"};
  test.compute_info.wave_size = 32;
  test.has_compute_info = true;
  return test;
}

TestCase VectorVopcCmpxNgtF16CapturedSdwaExecMask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 31, 0x7e003c00u);
  AppendVMovU32(&code, 3, 7);
  AppendVMovU32(&code, 30, 0);
  code.insert(code.end(), {0x7df700f9u, 0x8605001fu});
  AppendBufferStoreDword(&code, 3, 30);
  AppendVMovLiteral(&code, 31, 0x3c007e00u);
  AppendVMovU32(&code, 3, 9);
  AppendVMovU32(&code, 30, 4);
  code.insert(code.end(), {0x7df700f9u, 0x8605001fu});
  AppendBufferStoreDword(&code, 3, 30);
  AppendEnd(&code);

  TestCase test{"VectorVopcCmpxNgtF16CapturedSdwaExecMask",
                code,
                {0x11111111u, 0x22222222u},
                {7, 0x22222222u},
                {O::V_MOV_B32, O::V_CMPX_NGT_F16, O::BUFFER_STORE_DWORD,
                 O::S_ENDPGM}};
  test.decoded_counts = {{"V_CMPX_NGT_F16", 2}};
  return test;
}

TestCase VectorVopcCmpxNleF16CapturedSdwaExecMask() {
  using O = ShaderOpcode;
  struct CompareCase { u32 value, expected, exec = 1; bool vop3 = false; };
  const std::array<CompareCase, 10> cases{{
      {0x3c007e00u, 1}, {0xbc003c00u, 0}, // High half +1/-1; opposite low halves.
      {0x00003c00u, 0}, {0x80003c00u, 0}, // Signed zeros.
      {0x7e003c00u, 1}, {0x7c013c00u, 1}, // Quiet/signaling NaN are unordered.
      {0x7c003c00u, 1}, {0xfc003c00u, 0}, // Infinities.
      {0x3c000000u, 0, 0},               // Inactive lane cannot enter EXEC.
      {0x0000bc00u, 1, 1, true},         // VOP3 negates -1 before comparison.
  }};
  constexpr u32 sentinel = 0x89abcdefu;
  TestCase test;
  test.name = "VectorVopcCmpxNleF16CapturedSdwaExecMask";
  for (const auto &entry : cases) test.initial.push_back(entry.value);
  test.expected = test.initial;
  auto &code = test.code;
  for (u32 i = 0; i < cases.size(); ++i) {
    const auto &entry = cases[i];
    AppendVMovU32(&code, 30, i * 4u);
    AppendBufferLoadDword(&code, 5, 30);
    AppendSMovLiteral(&code, 106, sentinel);
    AppendSMovLiteral(&code, 107, sentinel);
    code.push_back(EncodeSMovB32(126, InlineU32(entry.exec)));
    if (entry.vop3) {
      AppendVop3(&code, 0xfc, 106, Vgpr(5), InlineU32(0), 0, 0, 0, false, 0, 1);
    } else {
      code.insert(code.end(), {0x7df900f9u, 0x86050005u});
    }
    code.push_back(EncodeSMovB32(8, 126));
    code.push_back(EncodeSMovB32(126, InlineU32(1)));
    const u32 out = test.expected.size();
    code.push_back(EncodeVop1(0x01, 0, 8));
    AppendStoreVgpr(&code, 0, out);
    AppendStoreSgprPair(&code, 106, out + 1u);
    test.expected.insert(test.expected.end(), {entry.expected, sentinel, sentinel});
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_CMPX_NLE_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"V_CMPX_NLE_F16", cases.size()}};
  test.required_spirv = {"OpFUnordGreaterThan"};
  return test;
}

TestCase VectorCompareInvertedMaskSelect() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x12345678u);
  code.push_back(EncodeVopc(0xc7, Vgpr(0), 0));      // v_cmp_t_u32 v0, v0
  code.push_back(EncodeSop1(0x08, 106, 106));        // s_not_b64 vcc, vcc
  code.push_back(EncodeVop2(0x01, 1, Vgpr(0), 128)); // v_cndmask_b32 v1, v0, 0
  AppendBufferStoreDword(&code, 1, 30);
  AppendEnd(&code);

  return {"VectorCompareInvertedMaskSelect",
          code,
          {},
          {0x12345678u},
          {O::V_MOV_B32, O::V_CMP_T_U32, O::S_NOT_B64, O::V_CNDMASK_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase BranchSelect() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeSMovB32(0, InlineU32(0)),
      EncodeSopc(0x06, 0, InlineU32(0)),
      EncodeSopp(0x05, 2),
      EncodeSMovB32(1, InlineU32(1)),
      EncodeSopp(0x02, 1),
      EncodeSMovB32(1, InlineU32(7)),
      EncodeSop2(0x0a, 2, 1, InlineU32(2)),
      EncodeVop1(0x01, 0, 2),
  };
  AppendBufferStoreDword(&code, 0, 30);
  AppendEnd(&code);
  return {"BranchSelect",
          code,
          {},
          {7},
          {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CBRANCH_SCC1, O::S_BRANCH,
           O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase SimpleLoop() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeSMovB32(0, InlineU32(0)),
      EncodeSopc(0x0a, 0, InlineU32(4)),
      EncodeSopp(0x04, 2),
      EncodeSop2(0x00, 0, 0, InlineU32(1)),
      EncodeSopp(0x02, 0xfffcu),
      EncodeVop1(0x01, 0, 0),
  };
  AppendBufferStoreDword(&code, 0, 30);
  AppendEnd(&code);
  return {"SimpleLoop",
          code,
          {},
          {4},
          {O::S_MOV_B32, O::S_CMP_LT_U32, O::S_CBRANCH_SCC0, O::S_ADD_U32,
           O::S_BRANCH, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase SharedReturnKeepsSelectedValues() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVop1(0x01, 5, 4),
      EncodeVop2(0x1a, 5, InlineU32(6), 5),
      EncodeVop2(0x25, 5, Vgpr(0), 5),
      EncodeSMovB32(8, InlineU32(10)),
      EncodeSopc(0x06, 4, InlineU32(0)),
      0,
      EncodeSMovB32(8, InlineU32(20)),
      EncodeSopc(0x06, 4, InlineU32(1)),
      0,
      EncodeSMovB32(8, InlineU32(30)),
  };
  AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
  AppendEnd(&code);
  const auto shared_return = static_cast<u32>(code.size());
  AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
  AppendEnd(&code);
  code[5] = EncodeSopp(0x05, shared_return - 6u);
  code[8] = EncodeSopp(0x05, shared_return - 9u);

  TestCase test;
  test.name = "SharedReturnKeepsSelectedValues";
  test.code = code;
  for (u32 group = 0; group < 3; group++) {
    test.expected.insert(test.expected.end(), 64, (group + 1u) * 10u);
  }
  test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
                  O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CBRANCH_SCC1,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpSelectionMerge"};
  test.forbidden_spirv = {"OpSwitch"};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 4;
  test.has_compute_info = true;
  test.dispatch_x = 3;
  return test;
}

TestCase SiblingSharedExitKeepsCapturedConditions() {
  using O = ShaderOpcode;
  std::vector<u32> code = {
      EncodeVop1(0x01, 5, 4),
      EncodeVop2(0x1a, 5, InlineU32(6), 5),
      EncodeVop2(0x25, 5, Vgpr(0), 5), // group * 64 + lane
      EncodeSop2(0x0e, 6, 4, InlineU32(1)),
      0, // choose left or right
      EncodeSMovB32(8, InlineU32(20)),
      EncodeSop2(0x0e, 6, 4, InlineU32(2)),
      0, // left: skip nested exit
      EncodeSop2(0x0e, 6, 4, InlineU32(4)),
      0, // nested left: shared exit
      0, // left -> normal return
      EncodeSMovB32(8, InlineU32(30)),
      EncodeSop2(0x0e, 6, 4, InlineU32(8)),
      0, // first right shared exit
      EncodeSMovB32(8, InlineU32(40)),
      EncodeSop2(0x0e, 6, 4, InlineU32(16)),
      0, // second right shared exit
      EncodeSMovB32(8, InlineU32(50)),
      EncodeSopc(0x06, InlineU32(0), InlineU32(1)), // overwrite SCC
  };
  AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
  AppendEnd(&code);
  const auto shared_exit = static_cast<u32>(code.size());
  code.push_back(EncodeSop2(0x00, 8, 8, InlineU32(1))); // overwrite SCC again
  AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
  AppendEnd(&code);
  code[4] = EncodeSopp(0x04, 11u - 5u);
  code[7] = EncodeSopp(0x04, 10u - 8u);
  code[9] = EncodeSopp(0x04, shared_exit - 10u);
  code[10] = EncodeSopp(0x02, 17u - 11u);
  code[13] = EncodeSopp(0x04, shared_exit - 14u);
  code[16] = EncodeSopp(0x04, shared_exit - 17u);

  TestCase test;
  test.name = "SiblingSharedExitKeepsCapturedConditions";
  test.code = std::move(code);
  for (u32 group = 0; group < 32; ++group) {
    u32 result = 50;
    if (group & 1u) {
      if ((group & 2u) && !(group & 4u)) result = 21;
    } else if (!(group & 8u)) {
      result = 31;
    } else if (!(group & 16u)) {
      result = 41;
    }
    test.expected.insert(test.expected.end(), 64, result);
  }
  test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
                  O::S_AND_B32, O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_ADD_U32,
                  O::S_CBRANCH_SCC0, O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpSelectionMerge"};
  test.forbidden_spirv = {"OpSwitch"};
  test.compute_info.wave_size = 64;
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 4;
  test.has_compute_info = true;
  test.dispatch_x = 32;
  return test;
}

TestCase BranchVccnzUsesWaveMask() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVop1(0x01, 1, 0),
      EncodeVop2(0x1a, 1, InlineU32(2), 1),
      EncodeVop2(0x25, 1, Vgpr(0), 1),
      EncodeVop2(0x1a, 3, InlineU32(2), 1),
      EncodeVopc(0xc2, InlineU32(0), 0),
      EncodeSopp(0x07, 2),
      EncodeVop1(0x01, 2, InlineU32(11)),
      EncodeSopp(0x02, 1),
  };
  AppendVMovU32(&code, 2, 42);
  AppendBufferStoreDword(&code, 2, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "BranchVccnzUsesWaveMask";
  test.code = code;
  test.expected = std::vector<u32>(8, 42);
  test.opcodes = {O::V_MOV_B32,          O::V_LSHLREV_B32,   O::V_ADD_NC_U32,
                  O::V_CMP_EQ_U32,       O::S_CBRANCH_VCCNZ, O::S_BRANCH,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.has_compute_info = true;
  test.dispatch_x = 2;
  return test;
}

TestCase ScalarBranchRestoresInactiveLanes(u32 wave_size, u32 threads) {
  using O = ShaderOpcode;
  TestCase test;
  if (threads < wave_size) {
    test.name = wave_size == 32 ? "ScalarBranchRestoresPartialWave32"
                               : "ScalarBranchRestoresPartialWave64";
  } else {
    test.name = wave_size == 32 ? "ScalarBranchRestoresInactiveWave32"
                               : "ScalarBranchRestoresInactiveWave64";
  }
  auto &code = test.code;
  code = {EncodeVop1(0x01, 1, InlineU32(32)),
          EncodeVop2(0x25, 1, Vgpr(0), 1),
          EncodeSop1(0x04, 8, 126), // Save all participating lanes.
          EncodeSMovB32(20, InlineU32(0)),
          EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)),
          EncodeSopp(0x08, 3), // EXEC is nonzero for the whole wave.
          EncodeSop1(0x04, 126, 8)};
  // The branch restores inactive lanes before reading the last lane's value.
  AppendVop3(&code, 0x360, 20, Vgpr(1), InlineU32(threads - 1));
  code.push_back(EncodeSop1(0x04, 126, 8));
  AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 0);
  AppendEnd(&code);
  test.initial.assign(threads, 0);
  test.expected.assign(threads, 32 + threads - 1);
  test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, O::S_MOV_B64,
                  O::S_CBRANCH_EXECZ, O::V_READLANE_B32, O::V_LSHLREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpGroupNonUniformBallot", "OpAll"};
  test.compute_info.wave_size = wave_size;
  test.compute_info.threads_num[0] = threads;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarMemRealtimeCapturedPlaceholder() {
  using O = ShaderOpcode;
  namespace D = ShaderRecompiler::Decoder;

  std::vector<u32> code = {0xf4940300u, 0xfa000000u};
  D::Instruction decoded;
  D::DecodeInstruction(code, 0, decoded);
  Require("ScalarMemRealtimeCapturedPlaceholder", "decode",
          decoded.opcode == O::S_MEMREALTIME && decoded.word_count == 2 &&
              decoded.dst.kind == D::OperandKind::Sgpr && decoded.dst.reg == 12 &&
              decoded.data_dwords == 2 && decoded.src_count == 0 &&
              decoded.src0.kind == D::OperandKind::Unknown &&
              decoded.src1.kind == D::OperandKind::Unknown,
          "captured clock instruction must write s12:s13 without memory operands");
  AppendStoreSgprPair(&code, 12, 0);
  code.insert(code.end(), {0xf4940300u, 0xfa000000u});
  AppendStoreSgprPair(&code, 12, 2);
  AppendEnd(&code);
  return {"ScalarMemRealtimeCapturedPlaceholder",
          code,
          {},
          {UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX},
          {O::S_MEMREALTIME, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarMemoryLoadVariants() {
  using O = ShaderOpcode;

  struct Load {
    u32 opcode;
    u32 dst;
    u32 byte_offset;
    u32 dwords;
  };

  const Load s_loads[] = {
      {0x00, 20, 0, 1},  {0x01, 22, 4, 2},   {0x02, 24, 12, 4},
      {0x03, 28, 28, 8}, {0x04, 36, 60, 16},
  };
  const Load s_buffer_loads[] = {
      {0x08, 52, 0, 1},  {0x09, 54, 4, 2},   {0x0a, 56, 12, 4},
      {0x0b, 60, 28, 8}, {0x0c, 68, 60, 16},
  };

  std::vector<u32> code;
  code.push_back(EncodeSMovB32(0, InlineU32(0)));
  for (const auto &load : s_loads) {
    AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset);
  }
  for (const auto &load : s_buffer_loads) {
    AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset);
  }

  u32 out = 0;
  for (const auto &load : s_loads) {
    for (u32 i = 0; i < load.dwords; i++) {
      AppendStoreSgpr(&code, load.dst + i, out++);
    }
  }
  for (const auto &load : s_buffer_loads) {
    for (u32 i = 0; i < load.dwords; i++) {
      AppendStoreSgpr(&code, load.dst + i, out++);
    }
  }
  AppendEnd(&code);

  std::vector<u32> initial;
  for (u32 i = 0; i < 31u; i++) {
    initial.push_back(0x10000000u + i);
  }
  std::vector<u32> expected = initial;
  expected.insert(expected.end(), initial.begin(), initial.end());

  return {"ScalarMemoryLoadVariants",
          code,
          initial,
          expected,
          {O::S_MOV_B32, O::S_LOAD_DWORD, O::S_LOAD_DWORDX2, O::S_LOAD_DWORDX4,
           O::S_LOAD_DWORDX8, O::S_LOAD_DWORDX16, O::S_BUFFER_LOAD_DWORD,
           O::S_BUFFER_LOAD_DWORDX2, O::S_BUFFER_LOAD_DWORDX4,
           O::S_BUFFER_LOAD_DWORDX8, O::S_BUFFER_LOAD_DWORDX16, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarBufferOffsetAlignmentAndCarry() {
  using O = ShaderOpcode;
  std::vector<u32> code;
  AppendSMovLiteral(&code, 12, 0xfffffffcu);
  code.push_back(EncodeSop2(0x00, 12, 12, 4));
  code.push_back(EncodeSmem0(0x08, 20));
  code.push_back(EncodeSmem1(4, 12));
  AppendSMovLiteral(&code, 12, 3u);
  code.push_back(EncodeSop2(0x00, 12, 12, 4));
  code.push_back(EncodeSmem0(0x08, 21));
  code.push_back(EncodeSmem1(1, 12));
  AppendStoreSgpr(&code, 20, 0);
  AppendStoreSgpr(&code, 21, 1);
  AppendEnd(&code);

  TestCase test{"ScalarBufferOffsetAlignmentAndCarry", code,
                {0x11111111u, 0x22222222u}, {0u, 0x11111111u},
                {O::S_MOV_B32, O::S_ADD_U32, O::S_BUFFER_LOAD_DWORD,
                 O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.forbidden_spirv = {"flattened_srt"};
  test.compute_info.threads_num[0] = 1;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.workgroup_register = 4;
  test.has_compute_info = true;
  return test;
}

TestCase ScalarLoadSignedImmediateOffsetAddsSoffset() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSMovB32(0, InlineU32(8)));
  code.push_back(EncodeSMovB32(2, InlineU32(0)));
  code.push_back(EncodeSMovB32(3, InlineU32(0)));
  code.push_back(EncodeSmem0(0x00, 1, 1));
  code.push_back(EncodeSmem1(0x1ffffcu, 0));
  AppendStoreSgpr(&code, 1, 0);
  AppendEnd(&code);

  return {"ScalarLoadSignedImmediateOffsetAddsSoffset",
          code,
          {0x11111111u, 0x22222222u},
          {0x22222222u, 0x22222222u},
          {O::S_MOV_B32, O::S_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase BufferLoadStore() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendBufferLoadDword(&code, 0, 30);
  code.push_back(EncodeVop1(0x01, 31, InlineU32(4)));
  AppendBufferStoreDword(&code, 0, 31);
  AppendEnd(&code);
  return {
      "BufferLoadStore",
      code,
      {0x11223344u, 0},
      {0x11223344u, 0x11223344u},
      {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovU32(&code, 21, 8);
  code.push_back(EncodeMubuf0(0x0cu, 0, true, true));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  return {
      "BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset",
      code,
      {0x11111111u, 0x22222222u, 0x33333333u},
      {0x33333333u, 0x22222222u, 0x33333333u},
      {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovU32(&code, 21, 12);
  AppendVMovLiteral(&code, 0, 0xabcdef01u);
  code.push_back(EncodeMubuf0(0x1cu, 0, true, true));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  return {"BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset",
          code,
          {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u},
          {0x11111111u, 0x22222222u, 0x33333333u, 0xabcdef01u},
          {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase BufferLoadDwordNoAddressFlagsIgnoresVaddr() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 8);
  code.push_back(EncodeMubuf0(0x0cu, 0, false, false));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendStoreVgpr(&code, 0, 3);
  AppendEnd(&code);

  return {
      "BufferLoadDwordNoAddressFlagsIgnoresVaddr",
      code,
      {0x11111111u, 0x22222222u, 0x33333333u, 0},
      {0x11111111u, 0x22222222u, 0x33333333u, 0x11111111u},
      {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase BufferLoadDwordIdxenUsesDescriptorStride() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  code.push_back(EncodeMubuf0(0x0cu, 0, true, false));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadDwordIdxenUsesDescriptorStride";
  test.code = code;
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u,
                  0x44444444u, 0x55555555u, 0x66666666u,
                  0x77777777u, 0x88888888u, 0x99aabbccu};
  test.expected = {0x99aabbccu, 0x22222222u, 0x33333333u,
                   0x44444444u, 0x55555555u, 0x66666666u,
                   0x77777777u, 0x88888888u, 0x99aabbccu};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(16, 3);
  test.has_user_data = true;
  return test;
}

TestCase BufferStoreDwordIdxenUsesDescriptorStride() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovLiteral(&code, 0, 0xabcdef01u);
  code.push_back(EncodeMubuf0(0x1cu, 0, true, false));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreDwordIdxenUsesDescriptorStride";
  test.code = code;
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u,
                  0x44444444u, 0x55555555u, 0x66666666u,
                  0x77777777u, 0x88888888u, 0x99aabbccu};
  test.expected = {0x11111111u, 0x22222222u, 0x33333333u,
                   0x44444444u, 0x55555555u, 0x66666666u,
                   0x77777777u, 0x88888888u, 0xabcdef01u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(16, 3);
  test.has_user_data = true;
  return test;
}

TestCase BufferStoreDwordAppliesHostOffset() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0xabcdef01u);
  code.push_back(EncodeMubuf0(0x1cu, 2, false, false));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreDwordAppliesHostOffset";
  test.code = code;
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0};
  test.expected = {0x11111111u, 0xabcdef01u, 0x33333333u, 0};
  test.storage_buffer_range_bytes = 8;
  test.storage_buffer_offsets = {2};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(0, 8);
  test.has_user_data = true;
  return test;
}

TestCase BufferOffsetsUsePackedWords(bool storage) {
  using O = ShaderOpcode;
  constexpr std::array offsets{0u, 12u, 20u, 28u, 36u};
  TestCase test;
  test.name = storage ? "BufferOffsetsPackedWordsStorage" : "BufferOffsetsPackedWordsPush";
  test.initial.resize(10);
  test.expected = test.initial;
  for (u32 i = 0; i < offsets.size(); ++i) {
    const auto base = i * 4u;
    AppendSMovLiteral(&test.code, base, 0x1000u + base * 0x1000u);
    AppendSMovLiteral(&test.code, base + 1u, 4u << 16u);
    AppendSMovLiteral(&test.code, base + 2u, 1u);
    AppendSMovLiteral(&test.code, base + 3u, 1u << 24u);
    const auto value = 0x12345678u + i;
    AppendVMovLiteral(&test.code, i, value);
    test.code.push_back(EncodeMubuf0(0x1cu, 0, false, false));
    test.code.push_back(EncodeMubuf1(i, i, 20));
    test.expected[offsets[i] / 4u] = value;
  }
  AppendEnd(&test.code);
  test.storage_buffer_range_bytes = 4;
  test.storage_buffer_offsets.assign(offsets.begin(), offsets.end());
  test.expand_shader_data_storage = storage;
  test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  return test;
}

TestCase BufferLoadVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendBufferLoadOpcode(&code, 0x08, 0, 20);
  AppendVMovU32(&code, 20, 2);
  AppendBufferLoadOpcode(&code, 0x09, 1, 20);
  AppendVMovU32(&code, 20, 0);
  AppendBufferLoadOpcode(&code, 0x0a, 2, 20);
  AppendVMovU32(&code, 20, 2);
  AppendBufferLoadOpcode(&code, 0x0b, 3, 20);
  AppendVMovU32(&code, 20, 8);
  AppendBufferLoadOpcode(&code, 0x0d, 4, 20);
  AppendBufferLoadOpcode(&code, 0x0f, 6, 20);
  AppendBufferLoadOpcode(&code, 0x0e, 9, 20);

  for (u32 i = 0; i < 13u; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  return {"BufferLoadVariants",
          code,
          {0x80ff7f01u, 0x00008001u, 0x11223344u, 0x55667788u, 0x99aabbccu,
           0xddeeff00u},
          {0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u, 0x55667788u,
           0x11223344u, 0x55667788u, 0x99aabbccu, 0x11223344u, 0x55667788u,
           0x99aabbccu, 0xddeeff00u},
          {O::BUFFER_LOAD_UBYTE, O::BUFFER_LOAD_SBYTE, O::BUFFER_LOAD_USHORT,
           O::BUFFER_LOAD_SSHORT, O::BUFFER_LOAD_DWORDX2,
           O::BUFFER_LOAD_DWORDX3, O::BUFFER_LOAD_DWORDX4, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase BufferLoadShortD16Captured(bool high) {
  TestCase test;
  test.name = high ? "BufferLoadShortD16HiCaptured" : "BufferLoadShortD16Captured";
  auto& code = test.code;
  const u32 data_reg = high ? 20u : 1u, index_reg = high ? 21u : 15u;
  const std::array<u32, 2> load = high ? std::array{0xe0942006u, 0x80021415u}
                                      : std::array{0xe090200eu, 0x8000010fu};
  AppendVMovU32(&code, index_reg, 1);
  AppendVMovLiteral(&code, data_reg, 0xabcd1234u);
  // Load both halves of v20 through s8:s11.
  if (high) code.insert(code.end(), {0xe090200eu, 0x80021415u});
  code.insert(code.end(), load.begin(), load.end());
  AppendStoreVgpr(&code, data_reg, 0);
  AppendVMovLiteral(&code, data_reg, 0xdeaf4321u);
  code.push_back(EncodeSop1(0x04, 32, 126));
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  code.insert(code.end(), load.begin(), load.end());
  code.push_back(EncodeSop1(0x04, 126, 32));
  AppendStoreVgpr(&code, data_reg, 1);
  AppendVMovU32(&code, index_reg, 2);
  AppendVMovLiteral(&code, data_reg, 0xbeef5678u);
  code.insert(code.end(), load.begin(), load.end());
  AppendStoreVgpr(&code, data_reg, 2);
  AppendEnd(&code);
  const u32 stride = high ? 16u : 32u;
  test.initial.assign(high ? 12u : 20u, 0xa5a5a5a5u);
  test.initial[high ? 5u : 11u] = 0x8001cafeu;
  if (high) test.initial[7] = 0xcafefaceu;
  test.expected = test.initial;
  test.expected[0] = high ? 0x8001cafeu : 0xabcd8001u;
  test.expected[1] = 0xdeaf4321u;
  test.expected[2] = high ? 0x00005678u : 0xbeef0000u;
  test.user_data = MakeStructuredStorageBufferData(stride, 2);
  if (high) std::copy_n(test.user_data.begin(), 4, test.user_data.begin() + 8);
  test.user_data[50] = static_cast<u32>(test.initial.size() * sizeof(u32));
  test.user_data[51] = 3u << 28u;
  test.has_user_data = true;
  test.storage_buffer_range_bytes = stride * 2u;
  test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
      ShaderOpcode::BUFFER_LOAD_SHORT_D16, ShaderOpcode::BUFFER_STORE_DWORD,
      ShaderOpcode::S_ENDPGM};
  if (high) test.opcodes.push_back(ShaderOpcode::BUFFER_LOAD_SHORT_D16_HI);
  test.decoded_counts = {{high ? "BUFFER_LOAD_SHORT_D16_HI v20.sdwa(sel=5,sext=0)"
                               : "BUFFER_LOAD_SHORT_D16 v1.sdwa(sel=4,sext=0)", 3}};
  return test;
}

TestCase BufferSubwordLoadsAtHostOffset(u32 bytes, u32 component, bool sign) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = bytes == 1 ? (sign ? "BufferSbyteHostOffsetTail" : "BufferUbyteHostOffsetTail")
              : sign ? "BufferSshortHostOffsetTail"
              : component == 0 ? "BufferUshortHostOffsetFirst"
              : component == 2 ? "BufferUshortHostOffsetCarry"
                               : "BufferUshortHostOffsetTail";
  // The captured stride-six index buffer uses USHORT IDXEN offsets 4, 2, 0.
  test.code = {EncodeMubuf0((bytes == 1 ? 0x08u : 0x0au) + sign,
                            component, true, false),
               EncodeMubuf1(1, 0, 0),
               EncodeMubuf0(0x1cu, 0, true, false), EncodeMubuf1(1, 12, 0)};
  AppendEnd(&test.code);
  test.initial = {0x800112abu, 0xfedc2345u, 0x80007fffu, 0xfaceffffu,
                  0xccccccccu, 0xccccccccu, 0xccccccccu};
  test.expected = test.initial;
  for (u32 record = 0; record < 2; ++record) {
    u32 value = 0;
    std::memcpy(&value, reinterpret_cast<const uint8_t *>(test.initial.data()) +
                           bytes + record * 6 + component, bytes);
    if (sign && (value & (1u << (bytes * 8u - 1u))) != 0) {
      value |= UINT32_MAX << (bytes * 8u);
    }
    test.expected[4 + record] = value;
  }
  test.expected[6] = 0;
  test.storage_buffer_range_bytes = 12;
  test.storage_buffer_offsets = {bytes, 16};
  test.user_data = MakeStructuredStorageBufferData(6, 2);
  test.user_data[0] = 0x1000u + bytes;
  test.user_data[48] = 0x2000u;
  test.user_data[49] = 4u << 16u;
  test.user_data[50] = 3;
  test.user_data[51] = 0x5204;
  test.has_user_data = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.threads_num[0] = 3;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.has_compute_info = true;
  test.opcodes = {bytes == 1 ? (sign ? O::BUFFER_LOAD_SBYTE : O::BUFFER_LOAD_UBYTE)
                            : (sign ? O::BUFFER_LOAD_SSHORT : O::BUFFER_LOAD_USHORT),
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase BufferSubwordHostOffsetOverflow() {
  auto test = BufferSubwordLoadsAtHostOffset(2, 0, false);
  test.name = "BufferSubwordHostOffsetOverflow";
  test.code.clear();
  AppendVMovLiteral(&test.code, 20, UINT32_MAX);
  test.code.insert(test.code.end(),
                   {EncodeMubuf0(0x0au), EncodeMubuf1(1, 0, 20),
                    EncodeMubuf0(0x1cu, 0, true, false), EncodeMubuf1(1, 12, 0)});
  AppendVMovLiteral(&test.code, 1, 0xbeef);
  test.code.insert(test.code.end(), {EncodeMubuf0(0x1au), EncodeMubuf1(1, 0, 20)});
  AppendEnd(&test.code);
  test.expected[4] = test.expected[5] = 0;
  test.opcodes.push_back(ShaderOpcode::V_MOV_B32);
  test.opcodes.push_back(ShaderOpcode::BUFFER_STORE_SHORT);
  return test;
}

TestCase BufferSubwordStoresAtHostOffset(u32 bytes) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = bytes == 1 ? "BufferByteStoreHostOffsetTail" : "BufferShortStoreHostOffsetTail";
  AppendVMovLiteral(&test.code, 1, bytes == 1 ? 0xa5u : 0xcafeu);
  test.code.push_back(EncodeMubuf0(bytes == 1 ? 0x18u : 0x1au,
                                  6u - bytes, true, false));
  test.code.push_back(EncodeMubuf1(1, 0, 0));
  AppendEnd(&test.code);
  test.initial = {0x800112abu, 0xfedc2345u, 0x80007fffu, 0xfaceffffu,
                  0x13579bdfu, 0x2468ace0u};
  test.expected = bytes == 1
      ? std::vector<u32>{0x800112abu, 0xfea52345u, 0x80007fffu, 0xfaceffa5u,
                         0x13579bdfu, 0x2468ace0u}
      : std::vector<u32>{0x800112abu, 0xcafe2345u, 0x80007fffu, 0xfacecafeu,
                         0x13579bdfu, 0x2468ace0u};
  test.storage_buffer_range_bytes = 12;
  test.storage_buffer_offsets = {bytes};
  test.user_data = MakeStructuredStorageBufferData(6, 2);
  test.user_data[0] = 0x1000u + bytes;
  test.has_user_data = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.threads_num[0] = 3;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.has_compute_info = true;
  test.opcodes = {O::V_MOV_B32, bytes == 1 ? O::BUFFER_STORE_BYTE : O::BUFFER_STORE_SHORT,
                  O::S_ENDPGM};
  return test;
}

TestCase BufferLoadDwordx4SnapshotsOverlappingAddress() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 21, 0);
  AppendVMovU32(&code, 22, 0);
  code.push_back(EncodeMubuf0(0x0eu, 0, true, true));
  code.push_back(EncodeMubuf1(21, 0, 21));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, 21 + i, 4 + i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadDwordx4SnapshotsOverlappingAddress";
  test.code = std::move(code);
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
                  0,           0,           0,           0};
  test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
                   0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX4, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(16, 2);
  test.has_user_data = true;
  return test;
}

TestCase BufferLoadDwordx2SnapshotsOverlappingAddress() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 21, 0);
  AppendVMovU32(&code, 22, 0);
  code.push_back(EncodeMubuf0(0x0du, 0, true, true));
  code.push_back(EncodeMubuf1(21, 0, 21));
  for (u32 i = 0; i < 2; i++) {
    AppendStoreVgpr(&code, 21 + i, 2 + i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadDwordx2SnapshotsOverlappingAddress";
  test.code = std::move(code);
  test.initial = {0x11111111u, 0x22222222u, 0, 0};
  test.expected = {0x11111111u, 0x22222222u, 0x11111111u, 0x22222222u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX2, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(8, 2);
  test.has_user_data = true;
  return test;
}

TestCase BufferLoadDwordx3SnapshotsOverlappingAddress() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 21, 0);
  AppendVMovU32(&code, 22, 0);
  code.push_back(EncodeMubuf0(0x0fu, 0, true, true));
  code.push_back(EncodeMubuf1(21, 0, 21));
  for (u32 i = 0; i < 3; i++) {
    AppendStoreVgpr(&code, 21 + i, 3 + i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadDwordx3SnapshotsOverlappingAddress";
  test.code = std::move(code);
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0, 0, 0};
  test.expected = {0x11111111u, 0x22222222u, 0x33333333u,
                   0x11111111u, 0x22222222u, 0x33333333u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX3, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(12, 2);
  test.has_user_data = true;
  return test;
}

TestCase BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 8);
  AppendBufferLoadOpcode(&code, 0x0e, 0, 20);
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail";
  test.code = std::move(code);
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
  test.expected = {0x33333333u, 0x44444444u, 0u, 0u};
  test.storage_buffer_range_bytes = 16;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX4, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  return test;
}

TestCase BufferLoadsGpuSelectedDescriptors(u32 width) {
  const bool scalar = width == 1u;
  const bool xyz = width == 3u;
  using O = ShaderOpcode;
  constexpr uint64_t GuestBase = 0x0000000110000000ull;
  struct DescriptorCase {
    u32 stride, records, mode, soffset;
    bool swizzle, bound;
    std::array<u32, 6> expected;
  };
  const DescriptorCase cases[] = {
      {24, 2, 0, 0, false, true, {7, 8, 9, 10, 11, 12}},
      // PPSA01417 selects XYZ vertices from a table of stride-16 descriptors.
      {16, 2, 0, 0, false, true, {5, 6, 7, 8, 0, 0}},
      {12, 2, 0, 0, false, true, {4, 5, 6, 0, 0, 0}},
      {12, 2, 1, 0, false, true, {4, 5, 6, 7, 8, 9}},
      {12, 1, 2, 0, false, true, {4, 5, 6, 7, 8, 9}},
      {12, 0, 2, 0, false, true, {}},
      // OOB3 checks offset + soffset, independently of index * stride.
      {12, 20, 3, 4, false, true, {5, 6, 7, 8, 0, 0}},
      {12, 2, 0, 0, false, false, {}},
      // RDNA2 swizzling uses four-byte elements; soffset is added afterward.
      {12, 2, 0, 4, true, true, {3, 4, 5, 0, 0, 0}},
      {12, 8, 3, 4, true, true, {3, 4, 5, 0, 0, 0}},
      {12, 4, 3, 4, true, true, {}},
  };
  // OFFEN byte offset 5 aligns to DWORD; swizzled offsets use eight-index groups.
  constexpr u32 scalar_expected[] = {2, 2, 2, 2, 2, 0, 3, 0, 10, 10, 0};
  TestCase test;
  test.name = scalar ? "BufferLoadDwordGpuSelectedDescriptors"
              : xyz ? "BufferLoadDwordx3GpuSelectedDescriptors"
                    : "BufferLoadsGpuSelectedDescriptors";
  test.initial.resize(2048);
  for (u32 i = 0; i < std::size(cases); ++i) {
    const auto &input = cases[i];
    const u32 data_offset = 4096 + i * 128;
    // 120-byte table entries with a descriptor at byte 8, as in PPSA04677.
    const std::array<u32, 4> descriptor{
        static_cast<u32>(GuestBase + data_offset),
        (input.stride << 16u) | (input.swizzle ? 1u << 31u : 0u) | 1u,
        input.records, (input.bound ? 0x5204u : 0x204u) | (input.mode << 28u)};
    std::copy(descriptor.begin(), descriptor.end(), test.initial.begin() + 130 + i * 30);
    for (u32 word = 0; word < 32; ++word) {
      test.initial[data_offset / 4 + word] = i * 100 + word + 1;
    }
    // Reverse the table order through a GPU load and readfirstlane, not host constants.
    const u32 selected = static_cast<u32>(std::size(cases)) - 1 - i;
    test.initial[64 + i] = selected;
    AppendVMovU32(&test.code, 30, (64 + i) * 4);
    AppendBufferLoadDword(&test.code, 0, 30);
    test.code.push_back(EncodeVop1(0x02, 20, Vgpr(0)));
    test.code.push_back(EncodeSop2(0x26, 20, 20, 255)); // s_mul_i32 s20, s20, 120
    test.code.push_back(120);
    test.code.push_back(EncodeSmem0(0x0a, 8, 0));
    test.code.push_back(EncodeSmem1(520, 20));
    AppendSMovLiteral(&test.code, 22, cases[selected].soffset);
    AppendVMovU32(&test.code, 21, scalar ? 5u : 1u);
    test.code.push_back(EncodeMubuf0(scalar ? 0x0c : xyz ? 0x0f : 0x0e, 0, !scalar, scalar));
    test.code.push_back(EncodeMubuf1(0, 2, 21, 22));
    if (!scalar) {
      test.code.push_back(EncodeMubuf0(xyz ? 0x0f : 0x0d, xyz ? 12 : 16, true, false));
      test.code.push_back(EncodeMubuf1(xyz ? 3 : 4, 2, 21, 22));
    }
    const u32 outputs = scalar ? 1u : 6u;
    for (u32 component = 0; component < outputs; ++component) {
      AppendStoreVgpr(&test.code, component, i * outputs + component);
      const u32 expected = scalar ? scalar_expected[selected] : cases[selected].expected[component];
      test.expected.push_back(expected == 0 ? 0 : selected * 100 + expected);
    }
  }
  AppendEnd(&test.code);
  test.bda_mappings = {{GuestBase, 0}};
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
                  O::V_READFIRSTLANE_B32, O::S_MUL_I32, O::S_BUFFER_LOAD_DWORDX4,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  if (scalar) {
    test.decoded_counts = {{"BUFFER_LOAD_DWORD ", 2u * std::size(cases)}};
  } else if (xyz) {
    test.opcodes.push_back(O::BUFFER_LOAD_DWORDX3);
  } else {
    test.opcodes.insert(test.opcodes.end(), {O::BUFFER_LOAD_DWORDX4, O::BUFFER_LOAD_DWORDX2});
  }
  test.required_spirv = {"OpConvertUToPtr", "PhysicalStorageBuffer"};
  return test;
}

TestCase BufferLoadsGpuSelectedDescriptors() {
  return BufferLoadsGpuSelectedDescriptors(4u);
}

TestCase BufferLoadDwordx3GpuSelectedDescriptors() {
  return BufferLoadsGpuSelectedDescriptors(3u);
}

TestCase BufferLoadDwordGpuSelectedDescriptors() {
  return BufferLoadsGpuSelectedDescriptors(1u);
}

TestCase BufferLoadFormatXGpuSelectedDescriptors(u32 wave) {
  using O = ShaderOpcode;
  using F = Prospero::BufferFormat;
  constexpr uint64_t GuestBase = 0x0000000110000000ull;
  struct DescriptorCase {
    F format;
    u32 selector, stride, records, mode, index, offset, soffset;
    bool swizzle;
    u32 expected;
    bool zero_payload = false;
    bool unmapped = false;
  };
  std::vector<DescriptorCase> cases = {
      {F::k8UInt, 4, 1, 4, 3, 1, 0, 0, false, 0x7f},
      {F::k8UInt, 4, 0, 4, 3, 0, 3, 0, false, 0x80},
      {F::k16UInt, 4, 2, 4, 0, 1, 0, 0, false, 0x80ff},
      {F::k32UInt, 4, 4, 2, 0, 1, 0, 0, false, 0x8001ff00},
      {F::k32_32_32_32Float, 7, 16, 1, 0, 0, 0, 0, false, 0x3fc00000},
      {F::k8SInt, 4, 0, 16, 3, 0, 3, 0, false, 0xffffff80},
      {F::k16SInt, 4, 0, 16, 3, 0, 2, 0, false, 0xffff80ff},
      {F::k8UNorm, 4, 0, 16, 3, 0, 2, 0, false, 0x3f800000},
      {F::k8SNorm, 4, 0, 16, 3, 0, 3, 0, false, 0xbf800000},
      {F::k8UScaled, 4, 0, 16, 3, 0, 1, 0, false, 0x42fe0000},
      {F::k8SScaled, 4, 0, 16, 3, 0, 3, 0, false, 0xc3000000},
      {F::k16Float, 4, 0, 16, 3, 0, 8, 0, false, 0x3f800000},
      {F::k11_11_10UInt, 5, 0, 16, 3, 0, 0, 0, false, 0x7ef},
      {F::k10_10_10_2UInt, 7, 0, 16, 3, 0, 0, 0, false, 2},
      {F::k8_8_8_8UInt, 6, 0, 16, 3, 0, 0, 0, false, 0xff},
      // Align the whole format first, then advance components from one swizzled base.
      {F::k16UInt, 4, 0, 16, 3, 0, 1, 0, false, 0x7f01},
      {F::k16UInt, 4, 0, 4, 3, 0, 3, 0, false, 0},
      {F::k32_32Float, 5, 8, 2, 0, 1, 0, 4, true, 0x3fc00000},
      // FORMAT_X still transfers a complete RGBA32 record for range checking.
      {F::k32_32_32_32Float, 4, 0, 16, 3, 0, 4, 0, false, 0},
      {F::k32_32_32_32Float, 4, 8, 1, 0, 0, 0, 0, false, 0},
      {F::k32_32_32_32Float, 1, 0, 16, 3, 0, 4, 0, false, 0x3f800000},
      {F::k32UInt, 1, 4, 0, 0, 0, 0, 0, false, 1},
      {F::k32UInt, 0, 4, 1, 0, 0, 0, 0, false, 0},
      {F::kInvalid, 1, 4, 1, 0, 0, 0, 0, false, 0},
      {F::k32UInt, 4, 4, 0, 2, 0, 0, 0, false, 0},
      {F::k32UInt, 4, 0, 4, 3, 0, 0, 8, false, 0},
      {F::k8_8UInt, 5, 0, 16, 3, 0, 2, 0, false, 0x80},
      {F::k16_16UInt, 5, 0, 16, 3, 0, 1, 0, false, 0x80ff},
      {F::k16_16_16_16UInt, 7, 0, 16, 3, 0, 0, 0, false, 0x8001},
      {F::k16_16_16_16Float, 6, 0, 16, 3, 0, 4, 0, false, 0x3f800000},
      {F::k32_32_32UInt, 7, 0, 16, 3, 0, 0, 0, false, 0x80ff7f01},
      {F::k32_32_32UInt, 6, 0, 16, 3, 0, 0, 0, false, 0x40003c00},
      {F::k11_11_10Float, 6, 0, 16, 3, 0, 0, 0, false, 0x400c0000},
      {F::k10_11_11Float, 4, 0, 16, 3, 0, 0, 0, false, 0x44040000},
      // Whole-record OOB and constant/reserved selectors must not touch this absent BDA page.
      {F::k32_32_32_32UInt, 4, 4, 1, 0, 0, 0, 0, false, 0, false, true},
      {F::k32UInt, 0, 4, 1, 0, 0, 0, 0, false, 0, false, true},
      {F::k32UInt, 1, 4, 1, 0, 0, 0, 0, false, 1, false, true},
      {F::k32Float, 1, 4, 1, 0, 0, 0, 0, false, 0x3f800000, false, true},
      {F::k32UInt, 2, 4, 1, 0, 0, 0, 0, false, 0, false, true},
      {F::k32UInt, 3, 4, 1, 0, 0, 0, 0, false, 0, false, true},
      {static_cast<F>(127), 4, 4, 1, 0, 0, 0, 0, false, 0, false, true},
  };
  // Visit every geometry entry and every selector. Zero payloads have exact zero
  // results in all numeric classes; selector ONE depends only on integer vs float.
  for (u32 format = 0; format <= static_cast<u32>(F::k32_32_32_32Float); ++format) {
    const auto info = ShaderRecompiler::Format::GetFormatInfo(static_cast<F>(format));
    using T = ShaderRecompiler::Format::ComponentType;
    for (u32 selector = 0; selector < 8u; ++selector) {
      const u32 one = info.type == T::Uint || info.type == T::Sint ? 1u : 0x3f800000u;
      cases.push_back({static_cast<F>(format), selector, 16, 1, 0, 0, 0, 0, false,
                       selector == 1u && info.type != T::Unknown ? one : 0u, true});
    }
  }
  TestCase test;
  test.name = wave == 32 ? "BufferLoadFormatXGpuSelectedDescriptors32"
                        : "BufferLoadFormatXGpuSelectedDescriptors64";
  const auto align256 = [](u32 bytes) { return (bytes + 255u) & ~255u; };
  const auto selection_base = align256(static_cast<u32>(cases.size()) * 4u);
  const auto descriptor_base = align256(selection_base + static_cast<u32>(cases.size()) * 4u);
  const auto payload_base = align256(descriptor_base + static_cast<u32>(cases.size()) * 32u);
  test.initial.resize((payload_base + cases.size() * 64u) / 4u);
  constexpr std::array payload{0x80ff7f01u, 0x8001ff00u, 0x40003c00u, 0x3fc00000u};
  for (u32 i = 0; i < std::size(cases); ++i) {
    const auto &input = cases[i];
    const u32 data_offset = payload_base + i * 64;
    const uint64_t data_address = GuestBase + (input.unmapped ? 0x100000u : data_offset);
    const std::array<u32, 8> row{
        static_cast<u32>(data_address),
        1u | (input.stride << 16u) | (input.swizzle ? 1u << 31u : 0u),
        input.records, input.selector | (static_cast<u32>(input.format) << 12u) |
                           (input.mode << 28u),
        input.index, input.offset, input.soffset, 0};
    std::copy(row.begin(), row.end(), test.initial.begin() + descriptor_base / 4u + i * 8);
    if (!input.zero_payload)
      std::copy(payload.begin(), payload.end(), test.initial.begin() + data_offset / 4);
    // Read the selection on the GPU, as GTA's descriptor waterfall does.
    test.initial[selection_base / 4u + i] = std::size(cases) - 1 - i;
    test.expected.push_back(cases[std::size(cases) - 1 - i].expected);
  }
  auto &code = test.code;
  AppendSMovLiteral(&code, 28, 0);
  AppendSMovLiteral(&code, 29, static_cast<u32>(cases.size()));
  const auto loop = static_cast<u32>(code.size());
  code.push_back(EncodeVop1(0x01, 30, 28));
  code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 30));
  code.push_back(EncodeMubuf0(0x0c, selection_base));
  code.push_back(EncodeMubuf1(2, 0, 30));
  code.push_back(EncodeVop1(0x02, 20, Vgpr(2)));
  code.push_back(EncodeSop2(0x1e, 20, 20, InlineU32(5)));
  code.push_back(EncodeSmem0(0x0b, 8, 0));
  code.push_back(EncodeSmem1(descriptor_base, 20));
  code.push_back(EncodeVop1(0x01, 21, 12));
  code.push_back(EncodeVop1(0x01, 22, 13));
  code.push_back(EncodeMubuf0(0x00, 0, true, true));
  code.push_back(EncodeMubuf1(4, 2, 21, 14));
  AppendBufferStoreDword(&code, 4, 30);
  code.push_back(EncodeSop2(0x00, 28, 28, InlineU32(1)));
  code.push_back(EncodeSopc(0x0a, 28, 29));
  code.push_back(EncodeSopp(0x05, loop - static_cast<u32>(code.size()) - 1u));
  AppendEnd(&code);
  test.bda_mappings = {{GuestBase, 0}};
  test.fault_free_address = GuestBase + 0x100000u;
  test.compute_info.threads_num[0] = 1;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.wave_size = wave;
  test.has_compute_info = true;
  test.required_spirv = {"PhysicalStorageBuffer", "OpSwitch", "OpLoopMerge"};
  test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::V_LSHLREV_B32,
                  O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_LSHL_B32,
                  O::S_BUFFER_LOAD_DWORDX8, O::BUFFER_LOAD_FORMAT_X,
                  O::BUFFER_STORE_DWORD, O::S_ADD_U32, O::S_CMP_LT_U32,
                  O::S_CBRANCH_SCC1, O::S_ENDPGM};
  return test;
}

TestCase BufferLoadFormatXGpuSelectedDescriptors() {
  return BufferLoadFormatXGpuSelectedDescriptors(32u);
}

TestCase BufferLoadFormatXGpuSelectedDescriptors64() {
  return BufferLoadFormatXGpuSelectedDescriptors(64u);
}

TestCase BufferStoreDwordx4DropsOnlyOutOfBoundsTail() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
  AppendVMovLiteral(&code, 1, 0xbbbbbbbbu);
  AppendVMovLiteral(&code, 2, 0xccccccccu);
  AppendVMovLiteral(&code, 3, 0xddddddddu);
  AppendVMovU32(&code, 20, 8);
  code.push_back(EncodeMubuf0(0x1eu));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreDwordx4DropsOnlyOutOfBoundsTail";
  test.code = std::move(code);
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
  test.expected = {0x11111111u, 0x22222222u, 0xaaaaaaaau, 0xbbbbbbbbu};
  test.storage_buffer_range_bytes = 16;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORDX4, O::S_ENDPGM};
  return test;
}

TestCase BufferLoadFormatXyzwRejectsPartialRecord() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 8);
  AppendBufferLoadOpcode(&code, 0x03, 0, 20);
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadFormatXyzwRejectsPartialRecord";
  test.code = std::move(code);
  test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
  test.expected = {0u, 0u, 0u, 0u};
  test.storage_buffer_range_bytes = 16;
  test.user_data = MakeStructuredStorageBufferData(
      0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
  test.has_user_data = true;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase BufferStoreFormatXyzwDropsPartialRecord() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
  AppendVMovLiteral(&code, 1, 0xbbbbbbbbu);
  AppendVMovLiteral(&code, 2, 0xccccccccu);
  AppendVMovLiteral(&code, 3, 0xddddddddu);
  AppendVMovU32(&code, 20, 8);
  code.push_back(EncodeMubuf0(0x07u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXyzwDropsPartialRecord";
  test.code = std::move(code);
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
  test.expected = test.initial;
  test.storage_buffer_range_bytes = 16;
  test.user_data = MakeStructuredStorageBufferData(
      0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
  test.has_user_data = true;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
  return test;
}

TestCase BufferLoadFormatXRejectsPartialRecord() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 12);
  AppendBufferLoadOpcode(&code, 0x00, 0, 20);
  // Bounds precede alignment: rounding offset 1 down must not make its full format fit.
  AppendVMovU32(&code, 20, 1);
  AppendBufferLoadOpcode(&code, 0x00, 1, 20);
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadFormatXRejectsPartialRecord";
  test.code = std::move(code);
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
  test.expected = {0u, 0u, 0x33333333u, 0x44444444u};
  test.storage_buffer_range_bytes = 16;
  test.user_data = MakeStructuredStorageBufferData(
      0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
  test.has_user_data = true;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  return test;
}

TestCase BufferStoreFormatXRejectsPartialRecord() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
  AppendVMovU32(&code, 20, 12);
  code.push_back(EncodeMubuf0(0x04u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXRejectsPartialRecord";
  test.code = std::move(code);
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
  test.expected = test.initial;
  test.storage_buffer_range_bytes = 16;
  test.user_data = MakeStructuredStorageBufferData(
      0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
  test.has_user_data = true;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
  return test;
}

TestCase BufferLoadFormatXyRejectsPartialRecord() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 8);
  AppendBufferLoadOpcode(&code, 0x01, 0, 20);
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadFormatXyRejectsPartialRecord";
  test.code = std::move(code);
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
  test.expected = {0u, 0u, 0x33333333u, 0x44444444u};
  test.storage_buffer_range_bytes = 16;
  test.user_data = MakeStructuredStorageBufferData(
      0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
  test.has_user_data = true;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  return test;
}

TestCase BufferStoreFormatXyRejectsPartialRecord() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
  AppendVMovLiteral(&code, 1, 0xbbbbbbbbu);
  AppendVMovU32(&code, 20, 8);
  code.push_back(EncodeMubuf0(0x05u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXyRejectsPartialRecord";
  test.code = std::move(code);
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
  test.expected = test.initial;
  test.storage_buffer_range_bytes = 16;
  test.user_data = MakeStructuredStorageBufferData(
      0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
  test.has_user_data = true;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XY, O::S_ENDPGM};
  return test;
}

TestCase BufferStoreVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 1);
  AppendVMovU32(&code, 0, 0xaa);
  AppendBufferStoreOpcode(&code, 0x18, 0, 20);
  AppendVMovU32(&code, 20, 2);
  AppendVMovLiteral(&code, 1, 0x0000bbccu);
  AppendBufferStoreOpcode(&code, 0x1a, 1, 20);
  AppendVMovU32(&code, 20, 4);
  AppendVMovLiteral(&code, 2, 0x11111111u);
  AppendVMovLiteral(&code, 3, 0x22222222u);
  AppendBufferStoreOpcode(&code, 0x1d, 2, 20);
  AppendVMovU32(&code, 20, 12);
  AppendVMovLiteral(&code, 4, 0x33333333u);
  AppendVMovLiteral(&code, 5, 0x44444444u);
  AppendVMovLiteral(&code, 6, 0x55555555u);
  AppendBufferStoreOpcode(&code, 0x1f, 4, 20);
  AppendVMovU32(&code, 20, 24);
  AppendVMovLiteral(&code, 7, 0x66666666u);
  AppendVMovLiteral(&code, 8, 0x77777777u);
  AppendVMovLiteral(&code, 9, 0x88888888u);
  AppendVMovLiteral(&code, 10, 0x99999999u);
  AppendBufferStoreOpcode(&code, 0x1e, 7, 20);
  AppendEnd(&code);

  return {"BufferStoreVariants",
          code,
          std::vector<u32>(10, 0),
          {0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
           0x55555555u, 0x66666666u, 0x77777777u, 0x88888888u, 0x99999999u},
          {O::V_MOV_B32, O::BUFFER_STORE_BYTE, O::BUFFER_STORE_SHORT,
           O::BUFFER_STORE_DWORDX2, O::BUFFER_STORE_DWORDX3,
           O::BUFFER_STORE_DWORDX4, O::S_ENDPGM}};
}

TestCase BufferFormatVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendBufferLoadOpcode(&code, 0x00, 0, 20);
  AppendBufferLoadOpcode(&code, 0x01, 1, 20);
  AppendBufferLoadOpcode(&code, 0x02, 3, 20);
  AppendBufferLoadOpcode(&code, 0x03, 6, 20);
  for (u32 i = 0; i < 10u; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  TestCase load;
  load.name = "BufferFormatLoadVariants";
  load.code = code;
  load.initial = {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u};
  load.expected = {0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u,
                   0x11121314u, 0x21222324u, 0x01020304u, 0x11121314u,
                   0x21222324u, 0x31323334u};
  load.opcodes = {O::BUFFER_LOAD_FORMAT_X,
                  O::BUFFER_LOAD_FORMAT_XY,
                  O::BUFFER_LOAD_FORMAT_XYZ,
                  O::BUFFER_LOAD_FORMAT_XYZW,
                  O::V_MOV_B32,
                  O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  return load;
}

TestCase BufferLoadFormatXyzwSnapshotsOverlappingAddress() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 21, 0);
  AppendVMovU32(&code, 22, 0);
  code.push_back(EncodeMubuf0(0x03u, 0, true, true));
  code.push_back(EncodeMubuf1(21, 0, 21));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, 21 + i, 4 + i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadFormatXyzwSnapshotsOverlappingAddress";
  test.code = std::move(code);
  test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
                  0,           0,           0,           0};
  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
                   0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(
      16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
  test.has_user_data = true;
  return test;
}

TestCase BufferLoadFormatXyzwResource32_32FloatAppliesXy01() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendBufferLoadOpcode(&code, 0x03, 0, 20);
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadFormatXyzwResource32_32FloatAppliesXy01";
  test.code = std::move(code);
  test.initial = {0x3f800000u, 0x40000000u, 0, 0};
  test.expected = {0x3f800000u, 0x40000000u, 0, 0x3f800000u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(
      16, 1, false, BufferFormat(Prospero::BufferFormat::k32_32Float));
  test.user_data[3] = (test.user_data[3] & ~0xfffu) | DstSel(4, 5, 0, 1);
  test.has_user_data = true;
  return test;
}

TestCase BufferLoadFormatXyzwResource323232FloatRepeatsSource() {
  using O = ShaderOpcode;
  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendBufferLoadOpcode(&code, 0x03, 0, 20);
  for (u32 i = 0; i < 4; ++i) AppendStoreVgpr(&code, i, i + 4);
  AppendEnd(&code);
  TestCase test;
  test.name = "BufferLoadFormatXyzwResource323232FloatRepeatsSource";
  test.code = std::move(code);
  test.initial = {0x3e800000u, 0x3f000000u, 0x3f400000u, 0xdeadbeefu, 0, 0, 0, 0};
  test.expected = {0x3e800000u, 0x3f000000u, 0x3f400000u, 0xdeadbeefu,
                   0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3e800000u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(
      0, 32, false, BufferFormat(Prospero::BufferFormat::k32_32_32Float));
  test.has_user_data = true;
  return test;
}

TestCase BufferLoadFormatXyzwResource8_8_8_8UintAppliesWzy1() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendBufferLoadOpcode(&code, 0x03, 0, 20);
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadFormatXyzwResource8_8_8_8UintAppliesWzy1";
  test.code = std::move(code);
  test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u};
  test.expected = {0x11u, 0x22u, 0x33u, 1u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(
      16, 1, false, BufferFormat(Prospero::BufferFormat::k8_8_8_8UInt));
  test.user_data[3] = (test.user_data[3] & ~0xfffu) | DstSel(7, 6, 5, 1);
  test.has_user_data = true;
  return test;
}

TestCase BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 21, 0x11111111u);
  AppendVMovLiteral(&code, 22, 0x22222222u);
  AppendVMovLiteral(&code, 23, 0x33333333u);
  AppendVMovLiteral(&code, 24, 0x44444444u);
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  code.push_back(EncodeMubuf0(0x03u, 0, true, true));
  code.push_back(EncodeMubuf1(21, 0, 21));
  code.push_back(EncodeSMovB32(126, InlineU32(1)));
  code.push_back(EncodeSMovB32(127, InlineU32(0)));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, 21 + i, 4 + i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress";
  test.code = std::move(code);
  test.initial = {0xaaaaaaaa, 0xbbbbbbbb, 0xcccccccc, 0xdddddddd, 0, 0, 0, 0};
  test.expected = {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu,
                   0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
  test.opcodes = {
      O::V_MOV_B32, O::S_MOV_B64,          O::BUFFER_LOAD_FORMAT_XYZW,
      O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(
      16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
  test.has_user_data = true;
  return test;
}

TestCase BufferFormatStoreVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovLiteral(&code, 0, 0x01020304u);
  AppendBufferStoreOpcode(&code, 0x04, 0, 20);
  AppendVMovU32(&code, 20, 4);
  AppendVMovLiteral(&code, 1, 0x11121314u);
  AppendVMovLiteral(&code, 2, 0x21222324u);
  AppendBufferStoreOpcode(&code, 0x05, 1, 20);
  AppendVMovU32(&code, 20, 12);
  AppendVMovLiteral(&code, 3, 0x31323334u);
  AppendVMovLiteral(&code, 4, 0x41424344u);
  AppendVMovLiteral(&code, 5, 0x51525354u);
  AppendBufferStoreOpcode(&code, 0x06, 3, 20);
  AppendVMovU32(&code, 20, 24);
  AppendVMovLiteral(&code, 6, 0x61626364u);
  AppendVMovLiteral(&code, 7, 0x71727374u);
  AppendVMovLiteral(&code, 8, 0x81828384u);
  AppendVMovLiteral(&code, 9, 0x91929394u);
  AppendBufferStoreOpcode(&code, 0x07, 6, 20);
  AppendEnd(&code);

  return {"BufferFormatStoreVariants",
          code,
          std::vector<u32>(10, 0),
          {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u,
           0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u},
          {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::BUFFER_STORE_FORMAT_XY,
           O::BUFFER_STORE_FORMAT_XYZ, O::BUFFER_STORE_FORMAT_XYZW,
           O::S_ENDPGM}};
}

TestCase BufferLoadFormatXResource8UintZeroExtendsByte() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMubuf0(0x00u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendStoreVgpr(&code, 0, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadFormatXResource8UintZeroExtendsByte";
  test.code = code;
  test.initial = {0x11223344u, 0};
  test.expected = {0x11223344u, 0x00000044u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5);
  test.has_user_data = true;
  return test;
}

TestCase BufferStoreFormatXResource16UintWritesHalfword() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovLiteral(&code, 0, 0x0000aaaau);
  code.push_back(EncodeMubuf0(0x04u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXResource16UintWritesHalfword";
  test.code = code;
  test.initial = {0x11223344u};
  test.expected = {0x1122aaaau};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(0, 4, false, 11);
  test.has_user_data = true;
  return test;
}

TestCase BufferStoreFormatXResource16UintPreservesAdjacentLanes() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop2(0x25, 1, InlineU32(1), 0));
  code.push_back(EncodeMubuf0(0x04u, 0, true, false));
  code.push_back(EncodeMubuf1(1, 0, 0));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXResource16UintPreservesAdjacentLanes";
  test.code = code;
  test.initial = std::vector<u32>(32, 0);
  for (u32 index = 0; index < 32; index++) {
    test.expected.push_back(((index * 2 + 2) << 16) | (index * 2 + 1));
  }
  test.opcodes = {O::V_ADD_NC_U32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = 64;
  test.has_compute_info = true;
  test.user_data = MakeStructuredStorageBufferData(2, 64, false, 11);
  test.has_user_data = true;
  test.forbidden_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange"};
  return test;
}

TestCase BufferStoreFormatXyzwPackedUnormSkinningVectors() {
  using O = ShaderOpcode;

  // The captured skinning dispatch stores a compressed quaternion with W = 1.
  constexpr std::array<float, 12> values = {
      0.25f, 0.5f, 0.75f, 1.0f,
      -1.0f, 0.0f, 2.0f, 1.0f,
      0.5f, 0.5f, 0.5f, 0.5f};
  std::vector<u32> code;
  for (u32 record = 0; record < 3; record++) {
    for (u32 component = 0; component < 4; component++) {
      AppendVMovLiteral(&code, component,
                        std::bit_cast<u32>(values[record * 4 + component]));
    }
    AppendVMovU32(&code, 20, record);
    code.push_back(EncodeMubuf0(0x07u, 0, true, false));
    code.push_back(EncodeMubuf1(0, 0, 20));
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXyzwPackedUnormSkinningVectors";
  test.code = std::move(code);
  test.initial = std::vector<u32>(4, 0xdeadbeefu);
  test.expected = {0xeff80100u, 0xfff00000u, 0xa0080200u, 0xdeadbeefu};
  test.user_data = MakeStructuredStorageBufferData(
      4, 3, false, BufferFormat(Prospero::BufferFormat::k10_10_10_2UNorm));
  test.has_user_data = true;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
  return test;
}

TestCase BufferStoreFormatXZeroFillsPackedRecord() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, std::bit_cast<u32>(0.5f));
  AppendVMovU32(&code, 20, 4);
  code.push_back(EncodeMubuf0(0x04u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXZeroFillsPackedRecord";
  test.code = std::move(code);
  test.initial = {0x11111111u, 0xffffffffu, 0x33333333u};
  test.expected = {0x11111111u, 0x00000200u, 0x33333333u};
  test.user_data = MakeStructuredStorageBufferData(
      0, 12, false, BufferFormat(Prospero::BufferFormat::k10_10_10_2UNorm));
  test.has_user_data = true;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
  return test;
}

TestCase BufferStoreFormatXyzwFloat16ConvertsComponents() {
  using O = ShaderOpcode;

  // This finite F32 has low bits 0x7e63, which form an F16 NaN if truncated.
  constexpr std::array<float, 8> values = {
      std::bit_cast<float>(0x3f807e63u), -2.0f, 0.5f, 0.25f,
      -4.0f, 3.0f, 0.0f, 2.0f};
  std::vector<u32> code;
  for (u32 record = 0; record < 2; record++) {
    for (u32 component = 0; component < 4; component++) {
      AppendVMovLiteral(&code, component,
                        std::bit_cast<u32>(values[record * 4 + component]));
    }
    AppendVMovU32(&code, 20, record);
    code.push_back(EncodeMubuf0(0x07u, 0, true, false));
    code.push_back(EncodeMubuf1(0, 0, 20));
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXyzwFloat16ConvertsComponents";
  test.code = std::move(code);
  test.initial = std::vector<u32>(4, 0xdeadbeefu);
  test.expected = {0xc0003c04u, 0x34003800u, 0x4200c400u, 0x40000000u};
  test.user_data = MakeStructuredStorageBufferData(
      8, 2, false, BufferFormat(Prospero::BufferFormat::k16_16_16_16Float));
  test.has_user_data = true;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
  test.required_spirv = {"PackHalf2x16"};
  return test;
}

TestCase BufferStoreFormatXyzwSnorm16CapturedSkinningVectors() {
  using O = ShaderOpcode;

  // Normal and tangent from the character skinning dispatch (eadfd178c07feebd).
  constexpr std::array<u32, 8> values = {
      0x3d98f7c5u, 0xbedae712u, 0xbf66a19eu, 0x3f800000u,
      0x3f7e6baeu, 0xbd339a7du, 0x3dd0a2e2u, 0xbf800000u};
  std::vector<u32> code;
  for (u32 record = 0; record < 2; record++) {
    for (u32 component = 0; component < 4; component++) {
      AppendVMovLiteral(&code, component, values[record * 4 + component]);
    }
    AppendVMovU32(&code, 20, record);
    code.push_back(EncodeMubuf0(0x07u, 0, true, false));
    code.push_back(EncodeMubuf1(0, 0, 20));
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXyzwSnorm16CapturedSkinningVectors";
  test.code = std::move(code);
  test.initial = std::vector<u32>(4, 0xdeadbeefu);
  test.expected = {0xc947098fu, 0x7fff8cb0u, 0xfa637f35u, 0x80010d0au};
  test.user_data = MakeStructuredStorageBufferData(
      8, 2, false, BufferFormat(Prospero::BufferFormat::k16_16_16_16SNorm));
  test.has_user_data = true;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
  return test;
}

TestCase BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords() {
  using O = ShaderOpcode;

  constexpr std::array<float, 9> values = {
      -2.0f, -1.0f, -0.75f, -0.25f, 0.0f, 0.25f, 0.75f, 1.0f, 2.0f};
  std::vector<u32> code;
  for (u32 i = 0; i < values.size(); i++) {
    AppendVMovLiteral(&code, 0, std::bit_cast<u32>(values[i]));
    AppendVMovU32(&code, 20, i * 2 + 2);
    code.push_back(EncodeMubuf0(0x04u));
    code.push_back(EncodeMubuf1(0, 0, 20));
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords";
  test.code = std::move(code);
  test.initial = std::vector<u32>(6, 0xdeadbeefu);
  test.expected = {0x8001beefu, 0xa0018001u, 0x0000e000u,
                   0x5fff2000u, 0x7fff7fffu, 0xdeadbeefu};
  test.user_data = MakeStructuredStorageBufferData(
      0, 24, false, BufferFormat(Prospero::BufferFormat::k16SNorm));
  test.has_user_data = true;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
  return test;
}

std::vector<TestCase> BufferLoadFormatD16XCases() {
  using F = Prospero::BufferFormat;
  using O = ShaderOpcode;
  struct LoadCase {
    const char *name;
    F format;
    u32 input;
    u32 expected;
    u32 selector = 4;
    u32 index = 0;
    bool active = true;
    bool overlap = false;
  };
  constexpr LoadCase inputs[] = {
      {"BufferLoadFormatD16XFloat16", F::k16Float, 0x7e63c000u, 0xabcdc000u},
      {"BufferLoadFormatD16XFloat32", F::k32Float, 0x3f000000u, 0xabcd3800u},
      {"BufferLoadFormatD16XUint32", F::k32UInt, 0x1234fffeu, 0xabcdffffu},
      {"BufferLoadFormatD16XUintAtMax", F::k32UInt, 0x0000ffffu, 0xabcdffffu},
      {"BufferLoadFormatD16XUintMax32", F::k32UInt, 0xffffffffu, 0xabcdffffu},
      {"BufferLoadFormatD16XSint32", F::k32SInt, 0xffff8001u, 0xabcd8001u},
      {"BufferLoadFormatD16XSintAboveMax", F::k32SInt, 0x00008000u, 0xabcd7fffu},
      {"BufferLoadFormatD16XSintBelowMin", F::k32SInt, 0xffff7fffu, 0xabcd8000u},
      {"BufferLoadFormatD16XSintAtMax", F::k32SInt, 0x00007fffu, 0xabcd7fffu},
      {"BufferLoadFormatD16XSintAtMin", F::k32SInt, 0xffff8000u, 0xabcd8000u},
      {"BufferLoadFormatD16XSintMax32", F::k32SInt, 0x7fffffffu, 0xabcd7fffu},
      {"BufferLoadFormatD16XSintMin32", F::k32SInt, 0x80000000u, 0xabcd8000u},
      {"BufferLoadFormatD16XUnorm8", F::k8UNorm, 0xffu, 0xabcd3c00u},
      {"BufferLoadFormatD16XSnorm16", F::k16SNorm, 0x8001u, 0xabcdbc00u},
      {"BufferLoadFormatD16XFloatOne", F::k32Float, 0u, 0xabcd3c00u, 1},
      {"BufferLoadFormatD16XOobFloatOne", F::k32Float, 0u, 0xabcd3c00u, 1, 4},
      {"BufferLoadFormatD16XOobUintOne", F::k32UInt, 0u, 0xabcd0001u, 1, 4},
      {"BufferLoadFormatD16XOobZero", F::k32Float, 0u, 0xabcd0000u, 4, 4},
      {"BufferLoadFormatD16XInactive", F::k32Float, 0u, 0xabcd1234u, 4, 0, false},
      {"BufferLoadFormatD16XOverlappingAddress", F::k16UInt, 7u, 7u, 4, 0, true, true},
  };
  std::vector<TestCase> cases;
  for (const auto &input : inputs) {
    TestCase test;
    test.name = input.name;
    AppendVMovLiteral(&test.code, 0, input.overlap ? input.index : 0xabcd1234u);
    AppendVMovU32(&test.code, 1, input.index);
    if (!input.active) {
      test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
    }
    test.code.push_back(EncodeMubuf0(0x80u, 0, true, false));
    test.code.push_back(EncodeMubuf1(0, 0, input.overlap ? 0 : 1));
    if (!input.active) {
      test.code.push_back(EncodeSMovB32(126, InlineU32(1)));
      test.code.push_back(EncodeSMovB32(127, InlineU32(0)));
    }
    AppendStoreVgpr(&test.code, 0, 2);
    AppendEnd(&test.code);
    test.initial = {input.input, 0xdeadbeefu, 0u, 0xcafef00du};
    test.expected = {input.input, 0xdeadbeefu, input.expected, 0xcafef00du};
    test.user_data = MakeStructuredStorageBufferData(4, 4, false, BufferFormat(input.format));
    test.user_data[3] = (test.user_data[3] & ~7u) | input.selector;
    test.has_user_data = true;
    test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_D16_X,
                    O::BUFFER_STORE_DWORD, O::S_ENDPGM};
    cases.push_back(std::move(test));
  }
  return cases;
}

std::vector<TestCase> BufferStoreFormatD16XCases() {
  using F = Prospero::BufferFormat;
  using O = ShaderOpcode;
  struct StoreCase {
    const char *name;
    F format;
    u32 input;
    u32 expected;
  };
  constexpr StoreCase inputs[] = {
      {"BufferStoreFormatD16XFloat16", F::k16Float, 0x7e63c000u, 0xffffc000u},
      {"BufferStoreFormatD16XFloat32", F::k32Float, 0x7e633800u, 0x3f000000u},
      {"BufferStoreFormatD16XUint32", F::k32UInt, 0x7e63fffeu, 0x0000fffeu},
      {"BufferStoreFormatD16XSint32", F::k32SInt, 0x7e63fffeu, 0xfffffffeu},
      {"BufferStoreFormatD16XUnorm8", F::k8UNorm, 0x7e633800u, 0xffffff80u},
      {"BufferStoreFormatD16XSnorm16", F::k16SNorm, 0x7e63b800u, 0xffffc000u},
      {"BufferStoreFormatD16XZeroFill", F::k16_16Float, 0x7e633800u, 0x00003800u},
  };
  std::vector<TestCase> cases;
  for (const auto &input : inputs) {
    TestCase test;
    test.name = input.name;
    AppendVMovLiteral(&test.code, 66, input.input);
    AppendVMovU32(&test.code, 65, 1);
    // Match the captured CS824586f35db5c80d instruction at pc 0x484.
    test.code.push_back(EncodeMubuf0(0x84u, 0, true, false));
    test.code.push_back(EncodeMubuf1(66, 0, 65));
    AppendEnd(&test.code);
    test.initial = {0xdeadbeefu, 0xffffffffu, 0xcafef00du};
    test.expected = {0xdeadbeefu, input.expected, 0xcafef00du};
    test.user_data = MakeStructuredStorageBufferData(4, 3, false, BufferFormat(input.format));
    test.has_user_data = true;
    test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_D16_X, O::S_ENDPGM};
    cases.push_back(std::move(test));
  }
  return cases;
}

std::vector<TestCase> FormattedStoreConversionCases() {
  using F = Prospero::BufferFormat;
  using O = ShaderOpcode;
  struct ConversionCase {
    const char *name[2];
    F format;
    u32 components;
    std::array<float, 4> values;
    u32 expected;
  };
  constexpr float nan = std::bit_cast<float>(0x7fc12345u);
  constexpr float inf = std::bit_cast<float>(0x7f800000u);
  constexpr ConversionCase inputs[] = {
      {{"BufferStoreSnorm8Range", "TBufferStoreSnorm8Range"},
       F::k8_8_8_8SNorm, 4, {-2.0f, 0.25f, 0.75f, 2.0f}, 0x7f5f2081u},
      {{"BufferStoreSnorm8Ties", "TBufferStoreSnorm8Ties"},
       F::k8_8SNorm, 2, {0.5f, -0.5f}, 0xffffc040u},
      {{"BufferStoreSnormPackedAlpha", "TBufferStoreSnormPackedAlpha"},
       F::k10_10_10_2SNorm, 4, {-1.0f, -0.5f, 0.5f, 1.0f}, 0xd00c0201u},
      {{"BufferStoreSnormPackedFirstChannel", "TBufferStoreSnormPackedFirstChannel"},
       F::k2_10_10_10SNorm, 4, {1.0f, -1.0f, 0.5f, -0.5f}, 0xc0100807u},
      {{"BufferStoreSnormPackedZeroFillX", "TBufferStoreSnormPackedZeroFillX"},
       F::k10_10_10_2SNorm, 1, {-1.0f}, 0x00000201u},
      {{"BufferStoreSnormPackedZeroFillXy", "TBufferStoreSnormPackedZeroFillXy"},
       F::k10_10_10_2SNorm, 2, {-1.0f, 1.0f}, 0x0007fe01u},
      {{"BufferStoreSnormPackedZeroFillXyz", "TBufferStoreSnormPackedZeroFillXyz"},
       F::k10_10_10_2SNorm, 3, {-1.0f, 1.0f, -1.0f}, 0x2017fe01u},
      {{"BufferStoreSnorm8ZeroFillX", "TBufferStoreSnorm8ZeroFillX"},
       F::k8_8_8_8SNorm, 1, {-1.0f}, 0x00000081u},
      {{"BufferStoreSnorm16ZeroFillX", "TBufferStoreSnorm16ZeroFillX"},
       F::k16_16SNorm, 1, {-1.0f}, 0x00008001u},
      {{"BufferStoreUscaledZeroFillXy", "TBufferStoreUscaledZeroFillXy"},
       F::k8_8_8_8UScaled, 2, {42.0f, 99.0f}, 0x0000632au},
      {{"BufferStoreSscaledZeroFillXyz", "TBufferStoreSscaledZeroFillXyz"},
       F::k8_8_8_8SScaled, 3, {-42.0f, 127.0f, -128.0f}, 0x00807fd6u},
      {{"BufferStoreUscaled8Fraction", "TBufferStoreUscaled8Fraction"},
       F::k8UScaled, 1, {42.875f}, 0xffffff2au},
      {{"BufferStoreSscaled8Fraction", "TBufferStoreSscaled8Fraction"},
       F::k8SScaled, 1, {-42.875f}, 0xffffffd6u},
      {{"BufferStoreUscaled8Range", "TBufferStoreUscaled8Range"},
       F::k8_8_8_8UScaled, 4, {-1.0f, 255.875f, 256.0f, 0.0f}, 0x00ffff00u},
      {{"BufferStoreSscaled8Range", "TBufferStoreSscaled8Range"},
       F::k8_8_8_8SScaled, 4, {-129.0f, -128.75f, 127.875f, 128.0f}, 0x7f7f8080u},
      {{"BufferStoreUscaled16Range", "TBufferStoreUscaled16Range"},
       F::k16_16UScaled, 2, {65536.0f, 42.875f}, 0x002affffu},
      {{"BufferStoreSscaled16Range", "TBufferStoreSscaled16Range"},
       F::k16_16SScaled, 2, {-32769.0f, 32768.0f}, 0x7fff8000u},
      {{"BufferStoreUscaledPacked", "TBufferStoreUscaledPacked"},
       F::k10_10_10_2UScaled, 4, {1024.0f, 3.875f, -1.0f, 4.0f}, 0xc0000fffu},
      {{"BufferStoreSscaledPackedAlpha", "TBufferStoreSscaledPackedAlpha"},
       F::k10_10_10_2SScaled, 4, {-512.75f, 511.875f, -3.875f, 3.875f}, 0xffd7fe00u},
      {{"BufferStoreSscaledPackedFirstChannel", "TBufferStoreSscaledPackedFirstChannel"},
       F::k2_10_10_10SScaled, 4, {3.0f, -512.0f, 511.0f, -1.0f}, 0xffdff803u},
      {{"BufferStoreSnorm8SpecialValues", "TBufferStoreSnorm8SpecialValues"},
       F::k8_8_8_8SNorm, 4, {nan, inf, -inf, -0.0f}, 0x00817f00u},
      {{"BufferStoreUscaled8SpecialValues", "TBufferStoreUscaled8SpecialValues"},
       F::k8_8_8_8UScaled, 4, {nan, inf, -inf, -0.0f}, 0x0000ff00u},
      {{"BufferStoreSscaled8SpecialValues", "TBufferStoreSscaled8SpecialValues"},
       F::k8_8_8_8SScaled, 4, {nan, inf, -inf, -0.0f}, 0x00807f00u},
  };
  constexpr O store_opcodes[2][4] = {
      {O::BUFFER_STORE_FORMAT_X, O::BUFFER_STORE_FORMAT_XY,
       O::BUFFER_STORE_FORMAT_XYZ, O::BUFFER_STORE_FORMAT_XYZW},
      {O::TBUFFER_STORE_FORMAT_X, O::TBUFFER_STORE_FORMAT_XY,
       O::TBUFFER_STORE_FORMAT_XYZ, O::TBUFFER_STORE_FORMAT_XYZW},
  };
  std::vector<TestCase> cases;
  for (const auto &input : inputs) {
    for (const bool typed : {false, true}) {
      TestCase test;
      test.name = input.name[typed];
      for (u32 component = 0; component < input.components; ++component) {
        AppendVMovLiteral(&test.code, component,
                          std::bit_cast<u32>(input.values[component]));
      }
      AppendVMovU32(&test.code, 20, 4);
      const u32 opcode = 3 + input.components;
      if (typed) {
        const u32 format = BufferFormat(input.format);
        test.code.push_back(EncodeMtbuf0(opcode, format & 0xfu, format >> 4u));
        test.code.push_back(EncodeMtbuf1(opcode, 0, 0, 20));
      } else {
        test.code.push_back(EncodeMubuf0(opcode));
        test.code.push_back(EncodeMubuf1(0, 0, 20));
      }
      AppendEnd(&test.code);
      test.initial = {0xdeadbeefu, 0xffffffffu, 0xcafef00du};
      test.expected = {0xdeadbeefu, input.expected, 0xcafef00du};
      test.user_data = MakeStructuredStorageBufferData(
          0, 12, false, BufferFormat(input.format));
      test.has_user_data = true;
      test.opcodes = {O::V_MOV_B32, store_opcodes[typed][input.components - 1],
                      O::S_ENDPGM};
      cases.push_back(std::move(test));
    }
  }
  return cases;
}

TestCase BufferLoadFormatXyResource88UintExtractsBytes() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMubuf0(0x01u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendStoreVgpr(&code, 0, 2);
  AppendStoreVgpr(&code, 1, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferLoadFormatXyResource8_8UintExtractsBytes";
  test.code = code;
  test.initial = {0x0000807fu, 0x55667788u, 0, 0};
  test.expected = {0x0000807fu, 0x55667788u, 0x0000007fu, 0x00000080u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18);
  test.has_user_data = true;
  return test;
}

TestCase BufferLoadFormatXyResource8888UnormConvertsFirstTwoComponents() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMubuf0(0x01u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  TestCase test;
  test.name =
      "BufferLoadFormatXyResource8_8_8_8UnormConvertsFirstTwoComponents";
  test.code = code;
  test.initial = {0x44332211u, 0xdeadbeefu};
  test.expected = {0x3d888889u, 0x3e088889u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(0, 8, false, 56);
  test.has_user_data = true;
  return test;
}

TestCase BufferStoreFormatXyResource88UintWritesBytes() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovLiteral(&code, 0, 0x000000aau);
  AppendVMovLiteral(&code, 1, 0x000000bbu);
  code.push_back(EncodeMubuf0(0x05u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXyResource8_8UintWritesBytes";
  test.code = code;
  test.initial = {0x11223344u, 0x55667788u};
  test.expected = {0x1122bbaau, 0x55667788u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XY, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18);
  test.has_user_data = true;
  return test;
}

TestCase BufferStoreFormatXyResource32UintWritesOneDword() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovLiteral(&code, 0, 0xabcdef01u);
  AppendVMovLiteral(&code, 1, 0x12345678u);
  code.push_back(EncodeMubuf0(0x05u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXyResource32UintWritesOneDword";
  test.code = code;
  test.initial = {0x11111111u, 0x22222222u};
  test.expected = {0xabcdef01u, 0x22222222u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XY, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(0, 8, false, 20);
  test.has_user_data = true;
  return test;
}

TestCase BufferStoreFormatXyzResource3232UintWritesTwoDwords() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovLiteral(&code, 0, 0xabcdef01u);
  AppendVMovLiteral(&code, 1, 0x12345678u);
  AppendVMovLiteral(&code, 2, 0x0badc0deu);
  code.push_back(EncodeMubuf0(0x06u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXyzResource32_32UintWritesTwoDwords";
  test.code = code;
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u};
  test.expected = {0xabcdef01u, 0x12345678u, 0x33333333u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZ, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(0, 8, false, 62);
  test.has_user_data = true;
  return test;
}

TestCase BufferStoreFormatXyzwResource323232UintWritesThreeDwords() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovLiteral(&code, 0, 0xabcdef01u);
  AppendVMovLiteral(&code, 1, 0x12345678u);
  AppendVMovLiteral(&code, 2, 0x0badc0deu);
  AppendVMovLiteral(&code, 3, 0xfeedfaceu);
  code.push_back(EncodeMubuf0(0x07u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXyzwResource32_32_32UintWritesThreeDwords";
  test.code = code;
  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
  test.expected = {0xabcdef01u, 0x12345678u, 0x0badc0deu, 0x44444444u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(0, 8, false, 72);
  test.has_user_data = true;
  return test;
}

TestCase BufferStoreFormatXyzResource8UintWritesOneByte() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovLiteral(&code, 0, 0x000000aau);
  AppendVMovLiteral(&code, 1, 0x000000bbu);
  AppendVMovLiteral(&code, 2, 0x000000ccu);
  code.push_back(EncodeMubuf0(0x06u));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXyzResource8UintWritesOneByte";
  test.code = code;
  test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu};
  test.expected = {0x112233aau, 0x55667788u, 0x99aabbccu};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZ, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5);
  test.has_user_data = true;
  return test;
}

TestCase BufferStoreFormatXAddTidUsesLaneIndex() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x12345678u);
  code.push_back(EncodeMubuf0(0x04u, 0, false, false));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXAddTidUsesLaneIndex";
  test.code = code;
  test.initial = std::vector<u32>(64, 0);
  test.expected = std::vector<u32>(64, 0);
  std::fill_n(test.expected.begin(), 32, 0x12345678u);
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 0;
  test.compute_info.wave_size = 32;
  test.has_compute_info = true;
  test.user_data = MakeStructuredStorageBufferData(4, 64, true);
  test.has_user_data = true;
  test.required_spirv = {"BuiltIn SubgroupLocalInvocationId"};
  test.forbidden_spirv = {"BuiltIn LocalInvocationId",
                          "OpGroupNonUniformBallot"};
  return test;
}

TestCase BufferStoreFormatXDropsOutOfRangeRecord() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 1);
  AppendVMovLiteral(&code, 0, 0xabcdef01u);
  code.push_back(EncodeMubuf0(0x04u, 0, true, false));
  code.push_back(EncodeMubuf1(0, 0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferStoreFormatXDropsOutOfRangeRecord";
  test.code = code;
  test.initial = {0x11111111u, 0x22222222u};
  test.expected = {0x11111111u, 0x22222222u};
  test.storage_buffer_range_bytes = 4;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(4, 1, false, 20);
  test.has_user_data = true;
  return test;
}

TestCase TBufferCapturedZeroStrideOob(bool raw_bounds = false, bool scalar = false) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = scalar ? "TBufferZeroStrideOobPreservesScalarRead"
                    : raw_bounds ? "TBufferZeroStrideRawBoundsStillLoads"
                                 : "TBufferCapturedZeroStrideOob";
  test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u,
                  0x12345678u, 0xdeadbeefu, 0xdeadbeefu};
  test.expected = test.initial;
  for (u32 i = 0; i < 4; i++) {
    test.expected[1 + i] = raw_bounds ? test.initial[i] : 0u;
  }
  test.expected[5] = test.initial[4];
  if (scalar) test.expected[6] = test.initial[0];
  test.user_data = MakeStructuredStorageBufferData(4, test.initial.size());
  std::copy_n(test.user_data.begin(), 4, test.user_data.begin() + 4);
  test.user_data[4] = 0x2000u;
  // Captured HITMAN descriptor in s[32:35]: mode 0, zero stride, arbitrary base.
  test.user_data[32] = 0x4a398620u;
  test.user_data[33] = 0;
  test.user_data[34] = 0x2b5u;
  test.user_data[35] = 0x0004d000u | (raw_bounds ? 3u << 28u : 0u);
  test.has_user_data = true;
  test.storage_buffer_offsets = {0, 0, 16};
  test.expected_buffer_resources = raw_bounds || scalar
                                       ? std::vector<u32>{0, 1, 2}
                                       : std::vector<u32>{0, 2};
  auto &code = test.code;
  AppendVMovLiteral(&code, 0, test.initial[0]);
  AppendStoreVgpr(&code, 0, 0); // Keep resource zero before the removed middle slot.
  AppendVMovU32(&code, 30, 0);
  code.push_back(0xea6b2000u);
  code.push_back(0x8008191eu); // Exact pc 0x47c: typed float4, s[32:35], v30.
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMubuf0(0x0cu));
  code.push_back(EncodeMubuf1(33, 1, 20));
  if (scalar) {
    code.push_back(EncodeSmem0(0x08u, 40, 16));
    code.push_back(EncodeSmem1(0));
    code.push_back(EncodeVop1(0x01u, 32, 40));
  }
  for (u32 i = 0; i < 4; i++) AppendStoreVgpr(&code, 25 + i, 1 + i);
  AppendStoreVgpr(&code, 33, 5);
  if (scalar) AppendStoreVgpr(&code, 32, 6);
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW,
                  O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  if (scalar) test.opcodes.push_back(O::S_BUFFER_LOAD_DWORD);
  test.ir_counts = {{"LoadBufferU32x4", raw_bounds ? 1u : 0u}};
  return test;
}

TestCase BufferZeroStrideOobFormatsAndWidths() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "BufferZeroStrideOobFormatsAndWidths";
  test.initial = std::vector<u32>(20, 0xdeadbeefu);
  test.expected = std::vector<u32>(20, 0);
  test.expected[10] = 0x3f800000u;
  test.expected[12] = 0x3f800000u;
  test.expected[14] = 1;
  test.expected[16] = 1;
  test.expected[18] = 0xabcdef01u;
  test.expected[19] = 0xdeadbeefu;
  test.user_data = MakeStructuredStorageBufferData(4, 20);
  for (u32 base : {4u, 8u}) {
    test.user_data[base] = 0x4a398620u;
    test.user_data[base + 2] = 0x2b5u;
    const u32 format = base == 4 ? 77u : 75u; // Four 32-bit float or unsigned integer components.
    test.user_data[base + 3] = (format << 12u) | DstSel(1, 4, 1, 0);
  }
  test.has_user_data = true;
  test.expected_buffer_resources = std::vector<u32>{0};
  auto &code = test.code;
  AppendVMovU32(&code, 0, 0);
  AppendStoreVgpr(&code, 0, 0);
  AppendVMovU32(&code, 20, 0);
  u32 destination = 0;
  for (const auto [opcode, width] :
       {std::pair{0x08u, 1u}, {0x0au, 1u}, {0x0cu, 1u}, {0x0du, 2u}, {0x0fu, 3u}}) {
    code.push_back(EncodeMubuf0(opcode));
    code.push_back(EncodeMubuf1(destination, 1, 20));
    destination += width;
  }
  AppendVMovU32(&code, 8, 0);
  AppendVMovU32(&code, 9, 0);
  for (u32 base : {1u, 2u}) {
    code.push_back(EncodeMubuf0(0x03u));
    code.push_back(EncodeMubuf1(base == 1 ? 10 : 14, base, 20));
  }
  AppendVMovLiteral(&code, 18, 0xabcdef01u);
  code.push_back(EncodeSop1(0x04u, 60, 126u)); // Preserve the partial wave's active lanes.
  code.push_back(EncodeSop1(0x04u, 126, InlineU32(0)));
  code.push_back(EncodeMubuf0(0x00u));
  code.push_back(EncodeMubuf1(18, 1, 20));
  code.push_back(EncodeSop1(0x04u, 126, 60u));
  for (u32 i = 0; i < 19; i++) AppendStoreVgpr(&code, i, i);
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_UBYTE,
                  O::BUFFER_LOAD_USHORT, O::BUFFER_LOAD_DWORD,
                  O::BUFFER_LOAD_DWORDX2, O::BUFFER_LOAD_DWORDX3,
                  O::BUFFER_LOAD_FORMAT_X, O::BUFFER_LOAD_FORMAT_XYZW,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.ir_counts = {{"LoadBuffer", 0}};
  return test;
}

TestCase TBufferLoadVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendTBufferLoadOpcode(&code, 0x00, 0, 20);
  AppendTBufferLoadOpcode(&code, 0x01, 1, 20);
  AppendTBufferLoadOpcode(&code, 0x02, 3, 20);
  AppendTBufferLoadOpcode(&code, 0x03, 6, 20);
  for (u32 i = 0; i < 10u; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  return {"TBufferLoadVariants",
          code,
          {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u},
          {0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u, 0x11121314u,
           0x21222324u, 0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u},
          {O::TBUFFER_LOAD_FORMAT_X, O::TBUFFER_LOAD_FORMAT_XY,
           O::TBUFFER_LOAD_FORMAT_XYZ, O::TBUFFER_LOAD_FORMAT_XYZW,
           O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXyzwSnapshotsOverlappingAddress() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 21, 0);
  AppendVMovU32(&code, 22, 0);
  constexpr auto format =
      BufferFormat(Prospero::BufferFormat::k32_32_32_32Float);
  code.push_back(
      EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
  code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, 21 + i, 4 + i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "TBufferLoadFormatXyzwSnapshotsOverlappingAddress";
  test.code = std::move(code);
  test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
                  0,           0,           0,           0};
  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
                   0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
  test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(16, 2);
  test.has_user_data = true;
  return test;
}

TestCase TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 21, 0);
  AppendVMovU32(&code, 22, 0);
  constexpr auto format = BufferFormat(Prospero::BufferFormat::k8_8_8_8UInt);
  code.push_back(
      EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
  code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, 21 + i, 4 + i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress";
  test.code = std::move(code);
  test.initial = {0x44332211u, 0, 0, 0, 0, 0, 0, 0};
  test.expected = {0x44332211u, 0, 0, 0, 0x11u, 0x22u, 0x33u, 0x44u};
  test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(4, 8);
  test.has_user_data = true;
  return test;
}

TestCase TBufferStoreFormatX8UintWritesOneByte() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovU32(&code, 0, 0xaa);
  code.push_back(EncodeMtbuf0(0x04, 5, 0));
  code.push_back(EncodeMtbuf1(0x04, 0, 0, 20));
  AppendEnd(&code);

  return {"TBufferStoreFormatX8UintWritesOneByte",
          code,
          {0x11223344u},
          {0x112233aau},
          {O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_X, O::S_ENDPGM}};
}

TestCase TBufferLoadFormatX8UintZeroExtendsByte() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMtbuf0(0x00, 5, 0));
  code.push_back(EncodeMtbuf1(0x00, 0, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  return {"TBufferLoadFormatX8UintZeroExtendsByte",
          code,
          {0x11223344u, 0},
          {0x00000044u, 0},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferLoadFormatX8888UintExtractsFirstByte() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20,
                                Prospero::BufferFormat::k8_8_8_8UInt);
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  return {"TBufferLoadFormatX8_8_8_8UintExtractsFirstByte",
          code,
          {0x44332211u, 0},
          {0x00000011u, 0},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXIdxenUsesDescriptorStride() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  code.push_back(EncodeMtbuf0(0x00, 5, 0, 0, true, false));
  code.push_back(EncodeMtbuf1(0x00, 0, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "TBufferLoadFormatXIdxenUsesDescriptorStride";
  test.code = code;
  test.initial = {0x01020304u, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu};
  test.expected = {0x0000007eu, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu};
  test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.user_data = MakeStructuredStorageBufferData(16, 3);
  test.has_user_data = true;
  return test;
}

TestCase TBufferLoadFormatX16FloatConvertsToFloat() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20,
                                Prospero::BufferFormat::k16Float);
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  return {"TBufferLoadFormatX16FloatConvertsToFloat",
          code,
          {0x00003c00u, 0},
          {0x3f800000u, 0},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXSintSignExtendsSubDword() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMtbuf0(0x00, 6, 0));
  code.push_back(EncodeMtbuf1(0x00, 0, 0, 20));
  AppendVMovU32(&code, 20, 4);
  code.push_back(EncodeMtbuf0(0x00, 12, 0));
  code.push_back(EncodeMtbuf1(0x00, 1, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  return {"TBufferLoadFormatXSintSignExtendsSubDword",
          code,
          {0x00000080u, 0x00008001u},
          {0xffffff80u, 0xffff8001u},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferStoreFormatXSintWritesSubDword() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovLiteral(&code, 0, 0xffffff80u);
  code.push_back(EncodeMtbuf0(0x04, 6, 0));
  code.push_back(EncodeMtbuf1(0x04, 0, 0, 20));
  AppendVMovU32(&code, 20, 4);
  AppendVMovLiteral(&code, 1, 0xffff8001u);
  code.push_back(EncodeMtbuf0(0x04, 12, 0));
  code.push_back(EncodeMtbuf1(0x04, 1, 0, 20));
  AppendEnd(&code);

  return {"TBufferStoreFormatXSintWritesSubDword",
          code,
          {0x11223344u, 0x55667788u},
          {0x11223380u, 0x55668001u},
          {O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_X, O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXy88IntegerComponents() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMtbuf0(0x01, 2, 1));
  code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
  AppendVMovU32(&code, 20, 4);
  code.push_back(EncodeMtbuf0(0x01, 3, 1));
  code.push_back(EncodeMtbuf1(0x01, 2, 0, 20));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  return {"TBufferLoadFormatXy8_8IntegerComponents",
          code,
          {0x0000807fu, 0x00007f80u, 0, 0},
          {0x0000007fu, 0x00000080u, 0xffffff80u, 0x0000007fu},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferStoreFormatXy88IntegerComponents() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovU32(&code, 0, 0xaa);
  AppendVMovU32(&code, 1, 0xbb);
  code.push_back(EncodeMtbuf0(0x05, 2, 1));
  code.push_back(EncodeMtbuf1(0x05, 0, 0, 20));
  AppendVMovU32(&code, 20, 4);
  AppendVMovLiteral(&code, 2, 0xffffff80u);
  AppendVMovU32(&code, 3, 0x7f);
  code.push_back(EncodeMtbuf0(0x05, 3, 1));
  code.push_back(EncodeMtbuf1(0x05, 2, 0, 20));
  AppendEnd(&code);

  return {"TBufferStoreFormatXy8_8IntegerComponents",
          code,
          {0x11223344u, 0x55667788u},
          {0x1122bbaau, 0x55667f80u},
          {O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_XY, O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXy1616IntegerComponents() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMtbuf0(0x01, 11, 1));
  code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
  AppendVMovU32(&code, 20, 4);
  code.push_back(EncodeMtbuf0(0x01, 12, 1));
  code.push_back(EncodeMtbuf1(0x01, 2, 0, 20));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  return {"TBufferLoadFormatXy16_16IntegerComponents",
          code,
          {0x80017fffu, 0x7fff8000u, 0, 0},
          {0x00007fffu, 0x00008001u, 0xffff8000u, 0x00007fffu},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferStoreFormatXy1616IntegerComponents() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovU32(&code, 0, 0xaaaa);
  AppendVMovU32(&code, 1, 0xbbbb);
  code.push_back(EncodeMtbuf0(0x05, 11, 1));
  code.push_back(EncodeMtbuf1(0x05, 0, 0, 20));
  AppendVMovU32(&code, 20, 4);
  AppendVMovLiteral(&code, 2, 0xffff8000u);
  AppendVMovU32(&code, 3, 0x7fff);
  code.push_back(EncodeMtbuf0(0x05, 12, 1));
  code.push_back(EncodeMtbuf1(0x05, 2, 0, 20));
  AppendEnd(&code);

  return {"TBufferStoreFormatXy16_16IntegerComponents",
          code,
          {0x11223344u, 0x55667788u},
          {0xbbbbaaaau, 0x7fff8000u},
          {O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_XY, O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXyz16161616UintLoadsHalfwords() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20,
                                Prospero::BufferFormat::k16_16_16_16UInt);
  for (u32 i = 0; i < 3; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  return {"TBufferLoadFormatXyz16_16_16_16UintLoadsHalfwords",
          code,
          {0x22221111u, 0x44443333u, 0},
          {0x00001111u, 0x00002222u, 0x00003333u},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZ, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXy88UnormConvertsToFloat() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMtbuf0(0x01, 14, 0));
  code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  return {"TBufferLoadFormatXy8_8UnormConvertsToFloat",
          code,
          {0x0000ff80u, 0},
          {0x3f008081u, 0x3f800000u},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXy88SnormConvertsToFloat() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendTBufferLoadFormatOpcode(&code, 0x01, 0, 20,
                                Prospero::BufferFormat::k8_8SNorm);
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  return {"TBufferLoadFormatXy8_8SnormConvertsToFloat",
          code,
          {0x00007f80u, 0},
          {0xbf800000u, 0x3f800000u},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXy1616UnormConvertsToFloat() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMtbuf0(0x01, 7, 1));
  code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  return {"TBufferLoadFormatXy16_16UnormConvertsToFloat",
          code,
          {0xffff8000u, 0},
          {0x3f000080u, 0x3f800000u},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMtbuf0(0x01, 8, 3));
  code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  return {"TBufferLoadFormatXy8_8_8_8UnormConvertsFirstTwoComponents",
          code,
          {0x44332211u, 0xdeadbeefu},
          {0x3d888889u, 0x3e088889u},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXyzw8888UintExtractsBytes() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20,
                                Prospero::BufferFormat::k8_8_8_8UInt);
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  return {"TBufferLoadFormatXyzw8_8_8_8UintExtractsBytes",
          code,
          {0x44332211u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu},
          {0x00000011u, 0x00000022u, 0x00000033u, 0x00000044u},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXyzw1010102SnormConvertsToFloat() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20,
                                Prospero::BufferFormat::k10_10_10_2SNorm);
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  return {"TBufferLoadFormatXyzw10_10_10_2SnormConvertsToFloat",
          code,
          {0x800801ffu, 0, 0, 0},
          {0x3f800000u, 0xbf800000u, 0, 0xbf800000u},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXyz111110FloatUnpacks() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20,
                                Prospero::BufferFormat::k11_11_10Float);
  for (u32 i = 0; i < 3; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  return {"TBufferLoadFormatXyz11_11_10FloatUnpacks",
          code,
          {0x781e03c0u, 0, 0},
          {0x3f800000u, 0x3f800000u, 0x3f800000u},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZ, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferLoadFormatXyzw3232FloatZerosMissingComponents() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeMtbuf0(0x03, 0, 4));
  code.push_back(EncodeMtbuf1(0x03, 0, 0, 20));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  return {"TBufferLoadFormatXyzw32_32FloatZerosMissingComponents",
          code,
          {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u},
          {0x11111111u, 0x22222222u, 0, 0},
          {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovLiteral(&code, 1, 0x40000000u);
  AppendVMovLiteral(&code, 2, 0x40400000u);
  AppendVMovLiteral(&code, 3, 0x40800000u);
  code.push_back(EncodeMtbuf0(0x07, 0, 4));
  code.push_back(EncodeMtbuf1(0x07, 0, 0, 20));
  AppendEnd(&code);

  return {"TBufferStoreFormatXyzw32_32FloatWritesOnlyPresentComponents",
          code,
          {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu},
          {0x3f800000u, 0x40000000u, 0xccccccccu, 0xddddddddu},
          {O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM}};
}

TestCase TBufferStoreVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovLiteral(&code, 0, 0x01020304u);
  AppendTBufferStoreOpcode(&code, 0x04, 0, 20);
  AppendVMovU32(&code, 20, 4);
  AppendVMovLiteral(&code, 1, 0x11121314u);
  AppendVMovLiteral(&code, 2, 0x21222324u);
  AppendTBufferStoreOpcode(&code, 0x05, 1, 20);
  AppendVMovU32(&code, 20, 12);
  AppendVMovLiteral(&code, 3, 0x31323334u);
  AppendVMovLiteral(&code, 4, 0x41424344u);
  AppendVMovLiteral(&code, 5, 0x51525354u);
  AppendTBufferStoreOpcode(&code, 0x06, 3, 20);
  AppendVMovU32(&code, 20, 24);
  AppendVMovLiteral(&code, 6, 0x61626364u);
  AppendVMovLiteral(&code, 7, 0x71727374u);
  AppendVMovLiteral(&code, 8, 0x81828384u);
  AppendVMovLiteral(&code, 9, 0x91929394u);
  AppendTBufferStoreOpcode(&code, 0x07, 6, 20);
  AppendEnd(&code);

  return {"TBufferStoreVariants",
          code,
          std::vector<u32>(10, 0),
          {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u,
           0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u},
          {O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_X, O::TBUFFER_STORE_FORMAT_XY,
           O::TBUFFER_STORE_FORMAT_XYZ, O::TBUFFER_STORE_FORMAT_XYZW,
           O::S_ENDPGM}};
}

TestCase FlatLoadDlcCaptured(u32 wave_size) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = wave_size == 64 ? "FlatLoadDlcCapturedWave64" : "FlatLoadDlcCapturedWave32";
  auto &code = test.code;
  code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
  AppendVMovU32(&code, 4, 0);
  AppendVMovU32(&code, 5, 31);
  // PPSA04288 CS 05b58beb842bacbf:0x638, FLAT_LOAD_DWORDX2 v[0:1], v[4:5] DLC.
  code.insert(code.end(), {0xdc341000u, 0x007d0004u});
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  // The descriptor store aliases the physical address read by the next load.
  AppendStoreVgpr(&code, 1, 8);
  code.push_back(0xbbfd0000u); // S_WAITCNT_VSCNT null, 0.
  code.insert(code.end(), {0xdc341000u, 0x007d0004u});
  AppendStoreVgpr(&code, 0, 2);
  AppendStoreVgpr(&code, 1, 3);
  AppendEnd(&code);
  test.initial = {0, 0, 0, 0, 0, 0, 0, 0, 0x11223344u, 0x55667788u};
  test.expected = {0x11223344u, 0x55667788u, 0x55667788u, 0x55667788u,
                   0, 0, 0, 0, 0x55667788u, 0x55667788u};
  test.bda_mappings = {{0x1f00000000ull, 32}};
  test.opcodes = {O::S_MOV_B64, O::V_MOV_B32, O::FLAT_LOAD_DWORDX2,
                  O::BUFFER_STORE_DWORD, O::S_WAITCNT_VSCNT, O::S_ENDPGM};
  test.required_spirv = {"OpDecorate %buffers Coherent", " 0 Coherent", "Volatile|Aligned 4"};
  test.decoded_counts = {{"dlc=1", 2}};
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  return test;
}

TestCase FlatStoreSlcCaptured() {
  TestCase test;
  test.name = "FlatStoreSlcCaptured";
  auto& code = test.code;
  code.push_back(EncodeSop1(0x0b, 26, InlineU32(1))); // s26 = 0x80000000.
  AppendVMovLiteral(&code, 12, 0x000c0008u); // Packed halfwords 8 and 12 select byte offset 8.
  code.insert(code.end(), {0x260c18f9u, 0x0504060cu, // Unsigned minimum of the two halfwords.
      0xd5590007u, 0x0431fe1au, 0x70000000u});
  AppendVMovLiteral(&code, 3, 0x12345678u);
  // Exercise an SLC store through v[6:7].
  code.insert(code.end(), {0xdc720000u, 0x007d0306u});
  AppendVMovLiteral(&code, 3, 0xdeadbeefu);
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  code.insert(code.end(), {0xdc720000u, 0x007d0306u});
  code.push_back(EncodeSop1(0x04, 126, 193)); // Restore EXEC before readback.
  code.insert(code.end(), {0xd5590007u, 0x0431fe1au, 0x70000000u,
      EncodeFlat0(0x0c, 0), EncodeFlat1(4, 0x7d, 0, 6)});
  AppendStoreVgprAtLaneDwordOffset(&code, 4, 0, 0);
  AppendEnd(&code);
  test.initial.assign(65, 0xa5a5a5a5u);
  test.expected.assign(64, 0x12345678u);
  test.expected.push_back(0xa5a5a5a5u);
  test.opcodes = {ShaderOpcode::S_BREV_B32, ShaderOpcode::V_MIN_U32,
      ShaderOpcode::V_MED3_U32, ShaderOpcode::V_MOV_B32, ShaderOpcode::FLAT_STORE_DWORD,
      ShaderOpcode::S_MOV_B64, ShaderOpcode::FLAT_LOAD_DWORD,
      ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
  test.compute_info.scratch_size_dwords = 4;
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  test.decoded_counts = {{"slc=1", 2}};
  test.forbidden_spirv = {"get_bda_pointer"};
  return test;
}

TestCase FlatLoadVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovU32(&code, 21, 0);
  code.push_back(EncodeFlat0(0x08, 0, 0));
  code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
  code.push_back(EncodeFlat0(0x09, 0, 2));
  code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
  code.push_back(EncodeFlat0(0x0a, 0, 0));
  code.push_back(EncodeFlat1(2, 0x7d, 0, 20));
  code.push_back(EncodeFlat0(0x0b, 0, 2));
  code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
  code.push_back(EncodeFlat0(0x0c, 0, 8));
  code.push_back(EncodeFlat1(13, 0x7d, 0, 20));
  code.push_back(EncodeFlat0(0x0d, 0, 8));
  code.push_back(EncodeFlat1(4, 0x7d, 0, 20));
  code.push_back(EncodeFlat0(0x0f, 0, 8));
  code.push_back(EncodeFlat1(6, 0x7d, 0, 20));
  code.push_back(EncodeFlat0(0x0e, 0, 8));
  code.push_back(EncodeFlat1(9, 0x7d, 0, 20));

  for (u32 i = 0; i < 14u; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  const std::vector<u32> guest_memory{0x80ff7f01u, 0x00008001u, 0x11223344u,
                                      0x55667788u, 0x99aabbccu, 0xddeeff00u};
  constexpr size_t byte_offset = 8;
  std::vector<u32> host_memory(
      (byte_offset + guest_memory.size() * sizeof(u32) + sizeof(u32) - 1u) /
      sizeof(u32));
  std::memcpy(reinterpret_cast<uint8_t *>(host_memory.data()) + byte_offset,
              guest_memory.data(), guest_memory.size() * sizeof(u32));

  TestCase test{"FlatLoadVariants",
                code,
                std::move(host_memory),
                {0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u,
                 0x55667788u, 0x11223344u, 0x55667788u, 0x99aabbccu,
                 0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u,
                 0x11223344u},
                {O::V_MOV_B32, O::FLAT_LOAD_UBYTE, O::FLAT_LOAD_SBYTE,
                 O::FLAT_LOAD_USHORT, O::FLAT_LOAD_SSHORT, O::FLAT_LOAD_DWORD,
                 O::FLAT_LOAD_DWORDX2, O::FLAT_LOAD_DWORDX3,
                 O::FLAT_LOAD_DWORDX4, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.bda_mappings = {{0, static_cast<u32>(byte_offset)}};
  return test;
}

TestCase FlatSubdwordLoadsApplyByteOffset() {
  using O = ShaderOpcode;

  constexpr size_t byte_offset = 3;
  std::vector<u32> code;
  AppendVMovU32(&code, 20, static_cast<u32>(byte_offset));
  AppendVMovU32(&code, 21, 0);
  code.push_back(EncodeFlat0(0x08, 0, 0));
  code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
  code.push_back(EncodeFlat0(0x0a, 0, 1));
  code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  constexpr u32 guest_word = 0x80ff7f01u;
  std::vector<u32> host_memory(
      (byte_offset + sizeof(guest_word) + sizeof(u32) - 1u) / sizeof(u32));
  std::memcpy(reinterpret_cast<uint8_t *>(host_memory.data()) + byte_offset,
              &guest_word, sizeof(guest_word));

  TestCase test;
  test.name = "FlatSubdwordLoadsApplyByteOffset";
  test.code = std::move(code);
  test.initial = std::move(host_memory);
  test.expected = {0x01u, 0x0000ff7fu};
  test.bda_mappings = {{0, 0}};
  test.opcodes = {O::V_MOV_B32, O::FLAT_LOAD_UBYTE, O::FLAT_LOAD_USHORT,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase GlobalLoadShortD16Captured(u32 wave_size) {
  using O = ShaderOpcode;
  constexpr uint64_t GuestBase = 0x0000000110000000ull;
  constexpr u32 Marker = 0xa5a50000u;
  constexpr uint64_t ScalarBase = GuestBase - Marker - 0x70u;
  constexpr uint64_t ExecMask = 0xaaaaaaaa55555555ull;
  constexpr u32 Values[] = {0u, 0x7fffu, 0x8000u, 0xffffu};
  TestCase test;
  test.name = wave_size == 32u ? "GlobalLoadShortD16CapturedWave32"
                              : "GlobalLoadShortD16CapturedWave64";
  test.initial.resize(2u * wave_size + wave_size / 2u);
  test.expected.resize(2u * wave_size);
  for (u32 lane = 0; lane < wave_size; ++lane) {
    const u32 value = Values[lane % std::size(Values)];
    test.initial[2u * wave_size + lane / 2u] |= value << (16u * (lane % 2u));
    test.expected[lane] = Marker | value;
    test.expected[wave_size + lane] = Marker |
        ((ExecMask & (uint64_t{1} << lane)) != 0u ? value : lane * 2u);
  }
  auto &code = test.code;
  AppendSMovLiteral(&code, 2, static_cast<u32>(ScalarBase));
  AppendSMovLiteral(&code, 3, static_cast<u32>(ScalarBase >> 32u));
  code.push_back(EncodeVop2(0x1a, 6, InlineU32(1), 0));
  AppendVMovLiteral(&code, 5, Marker);
  code.push_back(EncodeVop2(0x25, 6, Vgpr(5), 6));
  for (bool masked : {false, true}) {
    code.push_back(EncodeVop1(0x01, 5, Vgpr(6)));
    if (masked) {
      code.push_back(EncodeSop1(0x04, 8, 126));
      AppendSMovLiteral(&code, 126, static_cast<u32>(ExecMask));
      AppendSMovLiteral(&code, 127, static_cast<u32>(ExecMask >> 32u));
    }
    // Captured GLOBAL_LOAD_SHORT_D16: v5 supplies the byte offset and destination.
    code.insert(code.end(), {0xdc908070u, 0x05020005u});
    if (masked) code.push_back(EncodeSop1(0x04, 126, 8));
    AppendStoreVgprAtLaneDwordOffset(&code, 5, 0, masked ? wave_size : 0u);
  }
  AppendEnd(&code);
  test.bda_mappings = {{GuestBase, 8u * wave_size}};
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_MOV_B32, O::V_LSHLREV_B32,
                  O::V_ADD_NC_U32, O::FLAT_LOAD_SHORT_D16,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.decoded_counts = {{"FLAT_LOAD_SHORT_D16", 2}};
  return test;
}

TestCase FlatLoadShortD16AddressSegments() {
  using O = ShaderOpcode;
  constexpr uint64_t GuestBase = 0x0000000110000000ull;
  std::vector<u32> code;
  AppendVMovLiteral(&code, 20, static_cast<u32>(GuestBase) + 2u);
  AppendVMovLiteral(&code, 21, static_cast<u32>(GuestBase >> 32u));
  for (u32 segment : {0u, 2u}) {
    AppendVMovLiteral(&code, 3, 0x12345678u);
    code.push_back(EncodeFlat0(0x24, segment, segment == 0u ? 0u : 0xffeu));
    code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
    AppendStoreVgpr(&code, 3, segment == 0u ? 0u : 1u);
  }
  AppendVMovU32(&code, 20, 0u);
  AppendVMovLiteral(&code, 3, 0xbeef5678u);
  code.push_back(EncodeFlat0(0x1c, 1, 0));
  code.push_back(EncodeFlat1(0, 0x7d, 3, 20));
  AppendVMovLiteral(&code, 3, 0x98761234u);
  code.push_back(EncodeFlat0(0x24, 1, 2));
  code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
  AppendStoreVgpr(&code, 3, 2);
  AppendEnd(&code);
  TestCase test{"FlatLoadShortD16AddressSegments", code,
                {0, 0, 0, 0, 0x80017fffu},
                {0x12348001u, 0x12347fffu, 0x9876beefu},
                {O::V_MOV_B32, O::FLAT_LOAD_SHORT_D16, O::FLAT_STORE_DWORD,
                 O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.bda_mappings = {{GuestBase, 16}};
  test.compute_info.scratch_size_dwords = 1;
  return test;
}

TestCase BranchVccnzUsesCarryProducedWaveMask() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVop1(0x01, 1, 0),          EncodeVop2(0x1a, 1, InlineU32(2), 1),
      EncodeVop2(0x25, 1, Vgpr(0), 1), EncodeVop2(0x1a, 3, InlineU32(2), 1),
      EncodeVopc(0xc0, Vgpr(0), 0),
  };
  AppendVMovLiteral(&code, 2, 0xffffffffu);
  code.push_back(EncodeVop2(0x28, 5, Vgpr(2), 1));
  code.push_back(EncodeSopp(0x07, 2));
  code.push_back(EncodeVop1(0x01, 2, InlineU32(11)));
  code.push_back(EncodeSopp(0x02, 1));
  AppendVMovU32(&code, 2, 42);
  AppendBufferStoreDword(&code, 2, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "BranchVccnzUsesCarryProducedWaveMask";
  test.code = code;
  test.expected = std::vector<u32>(8, 42);
  test.opcodes = {O::V_MOV_B32,   O::V_LSHLREV_B32,      O::V_ADD_NC_U32,
                  O::V_CMP_F_U32, O::V_ADDC_U32,         O::S_CBRANCH_VCCNZ,
                  O::S_BRANCH,    O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.has_compute_info = true;
  test.dispatch_x = 2;
  return test;
}

TestCase ScalarLoadAlignsComponentsAndMasksAddress() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 0, 1u);
  AppendSMovLiteral(&code, 2, 3u);
  AppendSMovLiteral(&code, 3, 0xffff0000u);
  code.push_back(EncodeSmem0(0x00, 1, 1));
  code.push_back(EncodeSmem1(3, 0));
  AppendStoreSgpr(&code, 1, 0);
  AppendEnd(&code);

  return {"ScalarLoadAlignsComponentsAndMasksAddress",
          code,
          {0x11111111u, 0x22222222u},
          {0x11111111u, 0x22222222u},
          {O::S_MOV_B32, O::S_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase ScalarLoadAlignsDynamicBase() {
  using O = ShaderOpcode;
  constexpr uint64_t GuestBase = 0x10000;
  std::vector<u32> code;
  AppendSMovLiteral(&code, 8, GuestBase);
  AppendSMovLiteral(&code, 9, 0);
  code.push_back(EncodeSop2(0x00, 8, 8, 4)); // group ID supplies each base low-bit pattern
  AppendSMovLiteral(&code, 10, 7);
  code.push_back(EncodeSmem0(0x01, 12, 4));
  code.push_back(EncodeSmem1(7, 10)); // all three address components align independently
  code.push_back(EncodeVop1(0x01, 0, 12));
  code.push_back(EncodeVop1(0x01, 1, 13));
  code.push_back(EncodeVop1(0x01, 2, 4));
  code.push_back(EncodeVop2(0x1a, 3, InlineU32(3), 2));
  AppendBufferStoreDword(&code, 0, 3);
  code.push_back(EncodeVop2(0x25, 3, InlineU32(4), 3));
  AppendBufferStoreDword(&code, 1, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "ScalarLoadAlignsDynamicBase";
  test.code = std::move(code);
  test.initial.resize(20);
  test.initial[18] = 0x12345678;
  test.initial[19] = 0x9abcdef0;
  test.expected = test.initial;
  for (u32 group = 0; group < 4; ++group) {
    test.expected[group * 2] = test.initial[18];
    test.expected[group * 2 + 1] = test.initial[19];
  }
  test.bda_mappings = {{GuestBase, 64}};
  test.required_spirv = {"get_bda_pointer"};
  test.forbidden_spirv = {"flattened_srt"};
  test.opcodes = {O::S_MOV_B32, O::S_ADD_U32, O::S_LOAD_DWORDX2, O::V_MOV_B32,
                  O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.wave_size = 32;
  test.compute_info.threads_num[0] = 1;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.workgroup_register = 4;
  test.has_compute_info = true;
  test.dispatch_x = 4;
  return test;
}

TestCase ScalarBufferFromLoopReadlane(u32 wave_size) {
  using O = ShaderOpcode;
  constexpr u32 GuestBase = 0x10000;
  constexpr u32 DescriptorCount = 5;
  constexpr u32 PayloadBase = GuestBase + DescriptorCount * 32;
  std::vector<u32> code;
  code.push_back(EncodeVop2(0x1a, 1, InlineU32(2), 0));
  AppendBufferLoadDword(&code, 2, 1);
  AppendSMovLiteral(&code, 8, GuestBase + 3);
  AppendSMovLiteral(&code, 9, 0);
  AppendSMovLiteral(&code, 28, 0);
  const auto loop = static_cast<u32>(code.size());
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); // SMEM ignores EXEC.
  code.push_back(EncodeSmem0(0x03, 16, 4)); // descriptor and runtime soffset
  code.push_back(EncodeSmem1(3, 125)); // null keeps this a flattening candidate.
  code.push_back(EncodeSmem0(0x0a, 32, 8));
  code.push_back(EncodeSmem1(7, 20));
  code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
  code.push_back(EncodeVop1(0x01, 3, 28));
  code.push_back(EncodeVop2(0x1a, 31, InlineU32(4), 3));
  for (u32 component = 0; component < 4; ++component) {
    code.push_back(EncodeVop1(0x01, 4, 32 + component));
    AppendBufferStoreDword(&code, 4, 31);
    code.push_back(EncodeVop2(0x25, 31, InlineU32(4), 31));
  }
  code.push_back(EncodeSop1(0x04, 126, 193));
  code.push_back(EncodeSop2(0x00, 29, 28, InlineU32(wave_size == 64 ? 32 : 0)));
  AppendVop3(&code, 0x360, 8, Vgpr(2), 29);
  code.push_back(EncodeSop2(0x00, 28, 28, InlineU32(1)));
  code.push_back(EncodeSopc(0x0a, 28, InlineU32(DescriptorCount)));
  code.push_back(EncodeSopp(0x05, loop - static_cast<u32>(code.size()) - 1u));
  AppendEnd(&code);

  TestCase test;
  test.name = wave_size == 32 ? "ScalarBufferFromLoopReadlane32"
                             : "ScalarBufferFromLoopReadlane64";
  test.code = std::move(code);
  test.initial.resize(128 + DescriptorCount * 8 + 8);
  constexpr u32 records[] = {20, 5, 3, 0x40000005u, 20};
  for (u32 i = 0; i < DescriptorCount; ++i) {
    test.initial[(wave_size == 64 ? 32 : 0) + i] =
        GuestBase + ((i + 1) % DescriptorCount) * 32 + 3;
    const auto row = 128 + i * 8;
    test.initial[row] = PayloadBase + 3;
    test.initial[row + 1] = (i == 1 || i == 3 ? 4u << 16u : 0u) |
                            (i == 1 ? 1u << 31u : 0u);
    test.initial[row + 2] = records[i];
    // Scalar addressing ignores FORMAT, swizzle, ADD_TID, INDEX_STRIDE and OOB_SELECT.
    test.initial[row + 3] = i == 1 ? (3u << 21u) | (1u << 23u) | (3u << 28u) : 0u;
    test.initial[row + 4] = i == 4 ? 0xffffffffu : 11u;
  }
  for (u32 i = 0; i < 8; ++i)
    test.initial[128 + DescriptorCount * 8 + i] = 0x11111111u * (i + 1u);
  test.expected = {0x44444444u, 0x55555555u, 0, 0,
                   0x44444444u, 0x55555555u, 0, 0,
                   0, 0, 0, 0,
                   0x44444444u, 0x55555555u, 0x66666666u, 0x77777777u,
                   0, 0, 0, 0};
  test.bda_mappings = {{GuestBase, 512}};
  test.required_spirv = {"get_bda_pointer", "OpLoopMerge", "OpGroupNonUniformShuffle"};
  test.forbidden_spirv = {"flattened_srt"};
  test.opcodes = {O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD, O::S_MOV_B32,
                  O::S_MOV_B64, O::S_LOAD_DWORDX8, O::S_BUFFER_LOAD_DWORDX4,
                  O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::V_ADD_NC_U32,
                  O::S_ADD_U32, O::V_READLANE_B32, O::S_CMP_LT_U32,
                  O::S_CBRANCH_SCC1, O::S_ENDPGM};
  test.compute_info.wave_size = wave_size;
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase FlatVirtualAddressRebasesGuestAllocation() {
  using O = ShaderOpcode;

  constexpr uint64_t GuestBase = 0x0000000110000000ull;
  constexpr uint64_t ExtendedBase = Libs::LibKernel::Memory::kExtendedMemoryBase + GuestBase;
  static_assert(BufferCache::PageIndex(ExtendedBase) ==
                ((Libs::Graphics::LOWER_ADDRESS_SIZE + GuestBase) >> BufferCache::CACHING_PAGEBITS));
  static_assert(BufferCache::GuestAddress(BufferCache::PageIndex(ExtendedBase) <<
                                        BufferCache::CACHING_PAGEBITS) == ExtendedBase);
  std::vector<u32> code;
  AppendVMovLiteral(&code, 20, static_cast<u32>(GuestBase + 4u));
  AppendVMovLiteral(&code, 21, static_cast<u32>(GuestBase >> 32u));
  code.push_back(EncodeFlat0(0x0c, 0, 0));
  code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
  code.push_back(EncodeFlat0(0x0c, 2, 0));
  code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
  AppendVMovLiteral(&code, 21, static_cast<u32>((GuestBase >> 32u) + 1u));
  code.push_back(EncodeFlat0(0x0c, 0, 0));
  code.push_back(EncodeFlat1(2, 0x7d, 0, 20));
  AppendVMovLiteral(&code, 20, static_cast<u32>(ExtendedBase + 4u));
  AppendVMovLiteral(&code, 21, static_cast<u32>(ExtendedBase >> 32u));
  code.push_back(EncodeFlat0(0x0c, 0, 0));
  code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendStoreVgpr(&code, 2, 2);
  AppendStoreVgpr(&code, 3, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "FlatVirtualAddressRebasesGuestAllocation";
  test.code = std::move(code);
  test.initial = {0xfeedfaceu, 0xcafebabeu, 0, 0x12345678u};
  test.expected = {0x12345678u, 0x12345678u, 0, 0xcafebabeu};
  test.bda_mappings = {{GuestBase, 8}, {ExtendedBase, 0}};
  test.opcodes = {O::V_MOV_B32, O::FLAT_LOAD_DWORD, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  return test;
}

TestCase GlobalSignedImmediateRebasesBeforeSaddr() {
  using O = ShaderOpcode;

  constexpr uint64_t GuestBase = 0x0000000110000000ull;
  std::vector<u32> code;
  AppendSMovLiteral(&code, 0, static_cast<u32>(GuestBase));
  AppendSMovLiteral(&code, 1, static_cast<u32>(GuestBase >> 32u));
  AppendVMovU32(&code, 20, 8);
  code.push_back(EncodeFlat0(0x0c, 2, 0xffcu));
  code.push_back(EncodeFlat1(0, 0, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "GlobalSignedImmediateRebasesBeforeSaddr";
  test.code = std::move(code);
  test.initial = {0xfeedfaceu, 0xcafebabeu, 0x11111111u, 0x12345678u};
  test.expected = {0x12345678u};
  test.bda_mappings = {{GuestBase, 8}};
  test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase FlatSegmentIgnoresSaddrAndMasksOffsetMsb() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSMovB32(0, InlineU32(4)));
  AppendVMovU32(&code, 20, 0);
  AppendVMovU32(&code, 21, 0);
  code.push_back(EncodeFlat0(0x0c, 0, 4));
  code.push_back(EncodeFlat1(0, 0, 0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeFlat0(0x0c, 0, 0x804));
  code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  std::vector<u32> initial(514, 0);
  initial[1] = 0x11111111u;
  initial[2] = 0x22222222u;
  initial[513] = 0x33333333u;

  TestCase test;
  test.name = "FlatSegmentIgnoresSaddrAndMasksOffsetMsb";
  test.code = code;
  test.initial = initial;
  test.expected = {0x11111111u, 0x11111111u};
  test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.bda_mappings = {{0, 0}};
  return test;
}

TestCase ScratchIsPrivatePerInvocation() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSMovB32(20, InlineU32(0)));
  AppendVMovU32(&code, 21, 0);
  code.push_back(EncodeFlat0(0x1c, 1, 4));
  code.push_back(EncodeFlat1(0, 20, 0, 21));
  code.push_back(EncodeFlat0(0x0c, 1, 4));
  code.push_back(EncodeFlat1(5, 20, 0, 21));
  AppendStoreVgprAtLaneDwordOffset(&code, 5, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ScratchIsPrivatePerInvocation";
  test.code = std::move(code);
  for (u32 lane = 0; lane < 64; lane++) {
    test.expected.push_back(lane);
  }
  test.opcodes = {O::S_MOV_B32,          O::V_MOV_B32,
                  O::FLAT_STORE_DWORD,   O::FLAT_LOAD_DWORD,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.compute_info.scratch_size_dwords = 2;
  test.has_compute_info = true;
  return test;
}

#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
TestCase FlatStoreVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 21, 0);
  AppendVMovU32(&code, 20, 1);
  AppendVMovU32(&code, 0, 0xaa);
  code.push_back(EncodeFlat0(0x18, 0, 0));
  code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
  AppendVMovU32(&code, 20, 2);
  AppendVMovLiteral(&code, 1, 0x0000bbccu);
  code.push_back(EncodeFlat0(0x1a, 0, 0));
  code.push_back(EncodeFlat1(0, 0x7d, 1, 20));
  AppendVMovU32(&code, 20, 4);
  AppendVMovLiteral(&code, 2, 0x11111111u);
  AppendVMovLiteral(&code, 3, 0x22222222u);
  code.push_back(EncodeFlat0(0x1d, 0, 0));
  code.push_back(EncodeFlat1(0, 0x7d, 2, 20));
  AppendVMovU32(&code, 20, 12);
  AppendVMovLiteral(&code, 4, 0x33333333u);
  AppendVMovLiteral(&code, 5, 0x44444444u);
  AppendVMovLiteral(&code, 6, 0x55555555u);
  code.push_back(EncodeFlat0(0x1f, 0, 0));
  code.push_back(EncodeFlat1(0, 0x7d, 4, 20));
  AppendVMovU32(&code, 20, 24);
  AppendVMovLiteral(&code, 7, 0x66666666u);
  AppendVMovLiteral(&code, 8, 0x77777777u);
  AppendVMovLiteral(&code, 9, 0x88888888u);
  AppendVMovLiteral(&code, 10, 0x99999999u);
  code.push_back(EncodeFlat0(0x1e, 0, 0));
  code.push_back(EncodeFlat1(0, 0x7d, 7, 20));
  AppendVMovU32(&code, 20, 0);
  code.push_back(EncodeFlat0(0x1c, 0, 40));
  code.push_back(EncodeFlat1(0, 0x7d, 2, 20));
  AppendEnd(&code);

  TestCase test{"FlatStoreVariants",
                code,
                std::vector<u32>(12, 0),
                {0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u,
                 0x44444444u, 0x55555555u, 0x66666666u, 0x77777777u,
                 0x88888888u, 0x99999999u, 0x11111111u, 0},
                {O::V_MOV_B32, O::FLAT_STORE_BYTE, O::FLAT_STORE_SHORT,
                 O::FLAT_STORE_DWORD, O::FLAT_STORE_DWORDX2,
                 O::FLAT_STORE_DWORDX3, O::FLAT_STORE_DWORDX4, O::S_ENDPGM}};
  return test;
}
#endif

TestCase DsReadWriteVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 2, 0x11223344u);
  code.push_back(EncodeDs0(0x0d, 0));
  code.push_back(EncodeDs1(0, 2, 1));
  code.push_back(EncodeDs0(0x36, 0));
  code.push_back(EncodeDs1(3, 0, 1));
  AppendVMovU32(&code, 4, 0xaa);
  code.push_back(EncodeDs0(0x1e, 4));
  code.push_back(EncodeDs1(0, 4, 1));
  code.push_back(EncodeDs0(0x3a, 4));
  code.push_back(EncodeDs1(5, 0, 1));
  code.push_back(EncodeDs0(0x39, 4));
  code.push_back(EncodeDs1(6, 0, 1));
  AppendVMovLiteral(&code, 7, 0x000080ffu);
  code.push_back(EncodeDs0(0x1f, 8));
  code.push_back(EncodeDs1(0, 7, 1));
  code.push_back(EncodeDs0(0x3c, 8));
  code.push_back(EncodeDs1(8, 0, 1));
  code.push_back(EncodeDs0(0x3b, 8));
  code.push_back(EncodeDs1(9, 0, 1));
  AppendVMovLiteral(&code, 10, 0x10101010u);
  AppendVMovLiteral(&code, 11, 0x11111111u);
  code.push_back(EncodeDs0(0x4d, 12));
  code.push_back(EncodeDs1(0, 10, 1));
  code.push_back(EncodeDs0(0x76, 12));
  code.push_back(EncodeDs1(14, 0, 1));
  AppendVMovLiteral(&code, 16, 0x20202020u);
  AppendVMovLiteral(&code, 17, 0x21212121u);
  AppendVMovLiteral(&code, 18, 0x22222222u);
  code.push_back(EncodeDs0(0xde, 20));
  code.push_back(EncodeDs1(0, 16, 1));
  code.push_back(EncodeDs0(0xfe, 20));
  code.push_back(EncodeDs1(19, 0, 1));
  AppendVMovLiteral(&code, 22, 0x30303030u);
  AppendVMovLiteral(&code, 23, 0x31313131u);
  AppendVMovLiteral(&code, 24, 0x32323232u);
  AppendVMovLiteral(&code, 25, 0x33333333u);
  code.push_back(EncodeDs0(0xdf, 32));
  code.push_back(EncodeDs1(0, 22, 1));
  code.push_back(EncodeDs0(0xff, 32));
  code.push_back(EncodeDs1(26, 0, 1));

  const u32 results[] = {3, 5, 6, 8, 9, 14, 15, 19, 20, 21, 26, 27, 28, 29};
  for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
    AppendStoreVgpr(&code, results[i], i);
  }
  AppendEnd(&code);

  return {"DsReadWriteVariants",
          code,
          std::vector<u32>(14, 0),
          {0x11223344u, 0xaau, 0xffffffaau, 0x80ffu, 0xffff80ffu, 0x10101010u,
           0x11111111u, 0x20202020u, 0x21212121u, 0x22222222u, 0x30303030u,
           0x31313131u, 0x32323232u, 0x33333333u},
          {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_B32, O::DS_WRITE_B8,
           O::DS_READ_U8, O::DS_READ_I8, O::DS_WRITE_B16, O::DS_READ_U16,
           O::DS_READ_I16, O::DS_WRITE_B64, O::DS_READ_B64, O::DS_WRITE_B96,
           O::DS_READ_B96, O::DS_WRITE_B128, O::DS_READ_B128,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase DsReadWriteFullRegionAddress(u32 wave_size, bool gds) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = gds ? (wave_size == 32u ? "DsGdsFullAddress32" : "DsGdsFullAddress64")
                  : (wave_size == 32u ? "DsLdsFullAddress32" : "DsLdsFullAddress64");
  auto &code = test.code;
  code.push_back(EncodeVop2(0x1a, 1, InlineU32(2), 0));
  code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0));
  code.insert(code.end(), {EncodeDs0(0x0d, 4, gds), EncodeDs1(0, 2, 1)});
  AppendVMovLiteral(&code, 3, 0x10000u);
  code.push_back(EncodeVop2(0x25, 3, Vgpr(1), 3));
  AppendVMovLiteral(&code, 4, 0xdeadbeefu);
  // A high region address must not alias the valid word in the same lane.
  code.insert(code.end(), {EncodeDs0(0x0d, 4, gds), EncodeDs1(0, 4, 3),
                           EncodeDs0(0x36, 4, gds), EncodeDs1(5, 0, 1),
                           EncodeDs0(0x36, 4, gds), EncodeDs1(6, 0, 3)});
  AppendStoreVgprAtLaneDwordOffset(&code, 5, 0, 0);
  AppendStoreVgprAtLaneDwordOffset(&code, 6, 0, wave_size);
  AppendEnd(&code);
  test.initial.assign(wave_size * 2u, 0xdeadbeefu);
  test.expected.resize(wave_size * 2u);
  std::iota(test.expected.begin(), test.expected.begin() + wave_size, 1u);
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.compute_info.lds_size_dwords = wave_size + 1u;
  test.has_compute_info = true;
  if (gds) {
    test.gds_initial.resize(wave_size + 1u);
    test.expected_gds.resize(wave_size + 1u);
    std::iota(test.expected_gds.begin(), test.expected_gds.end(), 0u);
  }
  test.opcodes = {O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::V_MOV_B32, O::DS_WRITE_B32,
                  O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.forbidden_spirv = {"%uint_65535"};
  return test;
}

TestCase DsReadU16D16CapturedPreservesHighHalf() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 2, 0x89abcdefu);
  code.push_back(EncodeDs0(0x0d));
  code.push_back(EncodeDs1(0, 2, 1)); // ds_write_b32 v2, v1

  AppendVMovLiteral(&code, 3, 0x12345678u);
  code.push_back(EncodeDs0(0xa6));
  code.push_back(EncodeDs1(3, 0, 1)); // ds_read_u16_d16 v3, v1

  AppendVMovU32(&code, 7, 0);
  code.push_back(0xda980000u);
  code.push_back(0x07000007u); // captured ds_read_u16_d16 v7, v7

  AppendStoreVgpr(&code, 3, 0);
  AppendStoreVgpr(&code, 7, 1);
  AppendEnd(&code);

  TestCase test{"DsReadU16D16CapturedPreservesHighHalf",
                code,
                std::vector<u32>(2, 0),
                {0x1234cdefu, 0x0000cdefu},
                {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_U16_D16,
                 O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.decoded_counts = {{"DS_READ_U16_D16", 2}};
  return test;
}

TestCase DsWriteB16D16HiCapturedUsesHighHalf() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 3, 0xdeadbeefu);
  code.push_back(EncodeDs0(0x0d));
  code.push_back(EncodeDs1(0, 3, 1)); // ds_write_b32 v1, v3

  AppendVMovU32(&code, 20, 2);
  AppendVMovLiteral(&code, 2, 0x1234abcdu);
  AppendVMovLiteral(&code, 0, 0x87654321u);
  code.push_back(0xda840000u);
  code.push_back(0x00000214u); // captured ds_write_b16_d16_hi v20, v2

  code.push_back(EncodeDs0(0x36));
  code.push_back(EncodeDs1(4, 0, 1)); // ds_read_b32 v4, v1
  AppendStoreVgpr(&code, 4, 0);
  AppendStoreVgpr(&code, 0, 1);
  AppendEnd(&code);

  TestCase test{"DsWriteB16D16HiCapturedUsesHighHalf",
                code,
                std::vector<u32>(2, 0),
                {0x1234beefu, 0x87654321u},
                {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_WRITE_B16_D16_HI,
                 O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.decoded_counts = {{"DS_WRITE_B16_D16_HI", 1}};
  return test;
}

TestCase DsWriteB8D16HiWritesByteTwo() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 3, 0xdeadbeefu);
  code.push_back(EncodeDs0(0x0d));
  code.push_back(EncodeDs1(0, 3, 1)); // ds_write_b32 v1, v3

  // Every byte of the source differs, so selecting any byte but 23:16 fails.
  AppendVMovU32(&code, 20, 1);
  AppendVMovLiteral(&code, 2, 0x11a22233u);
  AppendVMovLiteral(&code, 0, 0x87654321u);
  code.push_back(EncodeDs0(0xa0));
  code.push_back(EncodeDs1(0, 2, 20)); // ds_write_b8_d16_hi v20, v2

  code.push_back(EncodeDs0(0x36));
  code.push_back(EncodeDs1(4, 0, 1)); // ds_read_b32 v4, v1
  AppendStoreVgpr(&code, 4, 0);
  AppendStoreVgpr(&code, 0, 1);
  AppendEnd(&code);

  TestCase test{"DsWriteB8D16HiWritesByteTwo",
                code,
                std::vector<u32>(2, 0),
                {0xdeada2efu, 0x87654321u},
                {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_WRITE_B8_D16_HI,
                 O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.decoded_counts = {{"DS_WRITE_B8_D16_HI", 1}};
  return test;
}

TestCase DsReadU16D16HiCapturedPreservesLowHalf() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 2, 0x89abcdefu);
  code.push_back(EncodeDs0(0x0d));
  code.push_back(EncodeDs1(0, 2, 1)); // ds_write_b32 v2, v1

  AppendVMovLiteral(&code, 5, 0x12345678u);
  AppendVMovU32(&code, 6, 0);
  code.push_back(0xda9c0000u);
  code.push_back(0x05000006u); // captured ds_read_u16_d16_hi v5, v6

  AppendStoreVgpr(&code, 5, 0);
  AppendEnd(&code);

  TestCase test{"DsReadU16D16HiCapturedPreservesLowHalf",
                code,
                std::vector<u32>(1, 0),
                {0xcdef5678u},
                {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_U16_D16_HI,
                 O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.decoded_counts = {{"DS_READ_U16_D16_HI", 1}};
  return test;
}

TestCase DsReadWrite2Variants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 2, 0x11111111u);
  AppendVMovLiteral(&code, 3, 0x22222222u);
  code.push_back(EncodeDs0(0x0e, (3u << 8u) | 1u));
  code.push_back(EncodeDs1Ex(0, 3, 2, 1));
  code.push_back(EncodeDs0(0x37, (3u << 8u) | 1u));
  code.push_back(EncodeDs1Ex(4, 0, 0, 1));
  AppendVMovLiteral(&code, 6, 0x33333333u);
  AppendVMovLiteral(&code, 7, 0x44444444u);
  AppendVMovLiteral(&code, 8, 0x55555555u);
  AppendVMovLiteral(&code, 9, 0x66666666u);
  code.push_back(EncodeDs0(0x4d, 32));
  code.push_back(EncodeDs1(0, 6, 1));
  code.push_back(EncodeDs0(0x4d, 48));
  code.push_back(EncodeDs1(0, 8, 1));
  code.push_back(EncodeDs0(0x77, (6u << 8u) | 4u));
  code.push_back(EncodeDs1Ex(10, 0, 0, 1));
  AppendVMovLiteral(&code, 14, 0x77777777u);
  AppendVMovLiteral(&code, 15, 0x88888888u);
  code.push_back(EncodeDs0(0x0f, (2u << 8u) | 1u));
  code.push_back(EncodeDs1Ex(0, 15, 14, 1));
  code.push_back(EncodeDs0(0x38, (2u << 8u) | 1u));
  code.push_back(EncodeDs1Ex(16, 0, 0, 1));
  AppendVMovLiteral(&code, 18, 0x99999999u);
  AppendVMovLiteral(&code, 19, 0xaaaaaaaau);
  AppendVMovLiteral(&code, 20, 0xbbbbbbbbu);
  AppendVMovLiteral(&code, 21, 0xccccccccu);
  code.push_back(EncodeDs0(0x4e, (10u << 8u) | 8u));
  code.push_back(EncodeDs1Ex(0, 20, 18, 1));
  code.push_back(EncodeDs0(0x77, (10u << 8u) | 8u));
  code.push_back(EncodeDs1Ex(22, 0, 0, 1));
  AppendVMovLiteral(&code, 26, 0xddddddddu);
  AppendVMovLiteral(&code, 27, 0xeeeeeeeeu);
  AppendVMovLiteral(&code, 28, 0xf0f0f0f0u);
  AppendVMovLiteral(&code, 29, 0x12345678u);
  code.push_back(EncodeDs0(0x4f, (2u << 8u) | 1u));
  code.push_back(EncodeDs1Ex(0, 28, 26, 1));
  code.push_back(EncodeDs0(0x78, (2u << 8u) | 1u));
  code.push_back(EncodeDs1Ex(34, 0, 0, 1));

  const u32 results[] = {4,  5,  10, 11, 12, 13, 22, 23,
                         24, 25, 16, 17, 34, 35, 36, 37};
  for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
    AppendStoreVgpr(&code, results[i], i);
  }
  AppendEnd(&code);

  return {"DsReadWrite2Variants",
          code,
          std::vector<u32>(16, 0),
          {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u,
           0x66666666u, 0x99999999u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu,
           0x77777777u, 0x88888888u, 0xddddddddu, 0xeeeeeeeeu, 0xf0f0f0f0u,
           0x12345678u},
          {O::V_MOV_B32, O::DS_WRITE2_B32, O::DS_READ2_B32, O::DS_WRITE_B64,
           O::DS_READ2_B64, O::DS_WRITE2_B64, O::DS_WRITE2ST64_B32,
           O::DS_READ2ST64_B32, O::DS_WRITE2ST64_B64, O::DS_READ2ST64_B64,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase DsWideReadSnapshotsOverlappingAddress() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 6, 0x11111111u);
  AppendVMovLiteral(&code, 7, 0x22222222u);
  code.push_back(EncodeDs0(0x4d, 0));
  code.push_back(EncodeDs1(0, 6, 1));
  AppendVMovLiteral(&code, 8, 0x33333333u);
  AppendVMovLiteral(&code, 9, 0x44444444u);
  code.push_back(EncodeDs0(0x4d, 16));
  code.push_back(EncodeDs1(0, 8, 1));
  code.push_back(EncodeDs0(0x77, 2u << 8u));
  code.push_back(EncodeDs1Ex(1, 0, 0, 1));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, 1u + i, i);
  }
  AppendEnd(&code);

  return {"DsWideReadSnapshotsOverlappingAddress",
          code,
          std::vector<u32>(4, 0),
          {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u},
          {O::V_MOV_B32, O::DS_WRITE_B64, O::DS_READ2_B64,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase DsReadWrite2EqualOffsetsUseData0() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 2, 0x11111111u);
  AppendVMovLiteral(&code, 3, 0x22222222u);
  code.push_back(EncodeDs0(0x0e, (1u << 8u) | 1u));
  code.push_back(EncodeDs1Ex(0, 3, 2, 1));
  AppendVMovLiteral(&code, 4, 0x33333333u);
  AppendVMovLiteral(&code, 5, 0x44444444u);
  AppendVMovLiteral(&code, 6, 0x55555555u);
  AppendVMovLiteral(&code, 7, 0x66666666u);
  code.push_back(EncodeDs0(0x4e, (2u << 8u) | 2u));
  code.push_back(EncodeDs1Ex(0, 6, 4, 1));
  code.push_back(EncodeDs0(0x37, (1u << 8u) | 1u));
  code.push_back(EncodeDs1Ex(8, 0, 0, 1));
  code.push_back(EncodeDs0(0x77, (2u << 8u) | 2u));
  code.push_back(EncodeDs1Ex(10, 0, 0, 1));
  for (u32 i = 0; i < 6; i++) {
    AppendStoreVgpr(&code, 8u + i, i);
  }
  AppendEnd(&code);

  return {"DsReadWrite2EqualOffsetsUseData0",
          code,
          std::vector<u32>(6, 0),
          {0x11111111u, 0x11111111u, 0x33333333u, 0x44444444u, 0x33333333u,
           0x44444444u},
          {O::V_MOV_B32, O::DS_WRITE2_B32, O::DS_WRITE2_B64, O::DS_READ2_B32,
           O::DS_READ2_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase DsWideLdsPartialBounds() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 2, 0x11111111u);
  AppendVMovLiteral(&code, 3, 0x22222222u);
  AppendVMovLiteral(&code, 4, 0x33333333u);
  AppendVMovLiteral(&code, 5, 0x44444444u);
  code.push_back(EncodeDs0(0xdf));
  code.push_back(EncodeDs1(0, 2, 1));
  code.push_back(EncodeDs0(0xff));
  code.push_back(EncodeDs1(6, 0, 1));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, 6u + i, i);
  }
  AppendEnd(&code);

  TestCase test{"DsWideLdsPartialBounds",
                code,
                std::vector<u32>(4, 0),
                {0x11111111u, 0x22222222u, 0x33333333u, 0},
                {O::V_MOV_B32, O::DS_WRITE_B128, O::DS_READ_B128,
                 O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.compute_info.lds_size_dwords = 3;
  return test;
}

TestCase DsWideGdsPartialBounds() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 2, 0x11111111u);
  AppendVMovLiteral(&code, 3, 0x22222222u);
  AppendVMovLiteral(&code, 4, 0x33333333u);
  AppendVMovLiteral(&code, 5, 0x44444444u);
  code.push_back(EncodeDs0(0xdf, 0, true));
  code.push_back(EncodeDs1(0, 2, 1));
  code.push_back(EncodeDs0(0xff, 0, true));
  code.push_back(EncodeDs1(6, 0, 1));
  for (u32 i = 0; i < 4; i++) {
    AppendStoreVgpr(&code, 6u + i, i);
  }
  AppendEnd(&code);

  TestCase test{"DsWideGdsPartialBounds",
                code,
                std::vector<u32>(4, 0),
                {0x11111111u, 0x22222222u, 0x33333333u, 0},
                {O::V_MOV_B32, O::DS_WRITE_B128, O::DS_READ_B128,
                 O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.gds_initial = {0, 0, 0};
  test.expected_gds = {0x11111111u, 0x22222222u, 0x33333333u};
  return test;
}

TestCase DsAtomicNoReturnVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  const u32 initial[] = {10, 10, 2, 1, 0, 3, 9, 0xfffffff0u, 0xfffffff0u,
                         10, 10, 0xf0f0u, 0xf000u, 0xf00fu};
  const u32 values[] = {5, 3, 2, 5, 2, 5, 5, 5, 5,
                        5, 20, 0x0ff0u, 0x0f00u, 0x00ffu};
  const u32 ops[] = {0x00, 0x01, 0x03, 0x03, 0x04, 0x04, 0x04,
                     0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b};
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendVMovLiteral(&code, 2, initial[i]);
    code.push_back(EncodeDs0(0x0d, i * 4u));
    code.push_back(EncodeDs1(0, 2, 1));
    AppendVMovLiteral(&code, 3, values[i]);
    code.push_back(EncodeDs0(ops[i], i * 4u));
    code.push_back(EncodeDs1(0, 3, 1));
    code.push_back(EncodeDs0(0x36, i * 4u));
    code.push_back(EncodeDs1(10u + i, 0, 1));
  }
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendStoreVgpr(&code, 10u + i, i);
  }
  AppendEnd(&code);

  return {"DsAtomicNoReturnVariants",
          code,
          std::vector<u32>(14, 0),
          {15, 7, 0, 2, 2, 2, 5, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u,
           0xf0f0u},
          {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_ADD_U32, O::DS_SUB_U32,
           O::DS_INC_U32, O::DS_DEC_U32, O::DS_MIN_I32, O::DS_MAX_I32,
           O::DS_MIN_U32, O::DS_MAX_U32, O::DS_AND_B32, O::DS_OR_B32,
           O::DS_XOR_B32, O::DS_READ_B32, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase DsMaskedOrBounds(bool gds) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = gds ? "DsMaskedOrGdsBounds" : "DsMaskedOrLdsBounds";
  auto &code = test.code;
  constexpr u32 initial[] = {0xf0f00ff0u, 0x12345678u, 0x87654321u};
  constexpr u32 masks[] = {0x00ff00ffu, 0u, 0xffffffffu};
  constexpr u32 values[] = {0x33000300u, 0x80000001u, 0xabcdef01u};
  AppendVMovU32(&code, 0, 0x1234);
  for (u32 i = 0; i < std::size(initial); ++i) {
    AppendVMovU32(&code, 2, i * 4u);
    AppendVMovLiteral(&code, 8, initial[i]);
    code.push_back(EncodeDs0(0x0d, 0x144, gds));
    code.push_back(EncodeDs1(0, 8, 2));
    // Captured mask v8 and data v10 are distinct; LDS wraps and aligns the address.
    AppendVMovU32(&code, 2, (gds ? 0u : 0x10000u) + i * 4u + 3u);
    AppendVMovLiteral(&code, 8, masks[i]);
    AppendVMovLiteral(&code, 10, values[i]);
    code.insert(code.end(), {0xd8300144u | (gds ? 1u << 17u : 0u), 0x000a0802u});
  }
  AppendVMovLiteral(&code, 8, 0xffffffffu);
  AppendVMovU32(&code, 10, 0);
  AppendVMovU32(&code, 2, 12); // First DWORD beyond the allocation.
  code.insert(code.end(), {EncodeDs0(0x0c, 0x144, gds), 0x000a0802u});
  AppendVMovU32(&code, 2, 0);
  code.push_back(EncodeSMovB32(126, InlineU32(0)));
  code.insert(code.end(), {EncodeDs0(0x0c, 0x144, gds), 0x000a0802u});
  code.push_back(EncodeSMovB32(126, InlineU32(1)));
  for (u32 i = 0; i < std::size(initial); ++i) {
    code.push_back(EncodeDs0(0x36, 0x144u + i * 4u, gds));
    code.push_back(EncodeDs1(11, 0, 2));
    AppendStoreVgpr(&code, 11, i);
    test.expected.push_back((initial[i] & ~masks[i]) | values[i]);
  }
  AppendStoreVgpr(&code, 0, 3); // The encoded VDst must stay untouched.
  test.expected.push_back(0x1234);
  AppendEnd(&code);
  test.compute_info.lds_size_dwords = 0x144 / 4 + 3;
  if (gds) {
    test.gds_initial.resize(0x144 / 4 + 3);
    test.expected_gds = test.gds_initial;
    std::copy_n(test.expected.begin(), 3, test.expected_gds.begin() + 0x144 / 4);
  }
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, O::DS_MSKOR_B32,
                  O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpAtomicCompareExchange"};
  return test;
}

TestCase DsMaskedOrContention(u32 wave_size) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = wave_size == 32 ? "DsMaskedOrContentionWave32" : "DsMaskedOrContentionWave64";
  auto &code = test.code;
  AppendVMovU32(&code, 2, 0);
  AppendVMovLiteral(&code, 8, 0xffffffffu);
  code.push_back(EncodeSop1(0x04, 10, 126));
  code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
  code.push_back(EncodeSop1(0x04, 126, 106));
  code.push_back(EncodeDs0(0x0d));
  code.push_back(EncodeDs1(0, 8, 2));
  code.push_back(EncodeSop1(0x04, 126, 10));
  code.push_back(EncodeSopp(0x0a, 0));
  AppendVMovU32(&code, 8, 1);
  code.push_back(EncodeVop2(0x1a, 8, Vgpr(0), 8));
  AppendVMovU32(&code, 9, 0);
  AppendSMovLiteral(&code, 126, 0x55555555u);
  AppendSMovLiteral(&code, 127, 0xaaaaaaaau);
  code.push_back(EncodeDs0(0x0c));
  code.push_back(EncodeDs1Ex(0, 9, 8, 2));
  code.push_back(EncodeSop1(0x04, 126, 10));
  code.push_back(EncodeSopp(0x0a, 0));
  code.push_back(EncodeDs0(0x36));
  code.push_back(EncodeDs1(11, 0, 2));
  AppendStoreVgprAtLaneDwordOffset(&code, 11, 0, 0);
  AppendEnd(&code);
  test.expected.assign(128, wave_size == 32 ? 0xaaaaaaaau : 0u);
  test.compute_info.threads_num[0] = 128;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.compute_info.lds_size_dwords = 1;
  test.has_compute_info = true;
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_U32,
                  O::V_LSHLREV_B32, O::S_BARRIER, O::DS_WRITE_B32, O::DS_MSKOR_B32,
                  O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpAtomicCompareExchange"};
  return test;
}

TestCase DsAtomicReturnVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  const u32 initial[] = {10, 10,      0xfffffff0u, 0xfffffff0u, 10,
                         10, 0xf0f0u, 0xf000u,     0xf00fu,     10};
  const u32 values[] = {5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu, 99};
  const u32 ops[] = {0x20, 0x21, 0x25, 0x26, 0x27,
                     0x28, 0x29, 0x2a, 0x2b, 0x2d};
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendVMovLiteral(&code, 2, initial[i]);
    code.push_back(EncodeDs0(0x0d, i * 4u));
    code.push_back(EncodeDs1(0, 2, 1));
    AppendVMovLiteral(&code, 3, values[i]);
    code.push_back(EncodeDs0(ops[i], i * 4u));
    code.push_back(EncodeDs1(10u + i, 3, 1));
    code.push_back(EncodeDs0(0x36, i * 4u));
    code.push_back(EncodeDs1(20u + i, 0, 1));
  }
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendStoreVgpr(&code, 10u + i, i);
  }
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendStoreVgpr(&code, 20u + i, i + 10u);
  }
  AppendEnd(&code);

  return {
      "DsAtomicReturnVariants",
      code,
      std::vector<u32>(20, 0),
      {10, 10, 0xfffffff0u, 0xfffffff0u, 10, 10, 0xf0f0u, 0xf000u, 0xf00fu, 10,
       15, 7,  0xfffffff0u, 5,           5,  20, 0x00f0u, 0xff00u, 0xf0f0u, 99},
      {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_ADD_RTN_U32, O::DS_SUB_RTN_U32,
       O::DS_MIN_RTN_I32, O::DS_MAX_RTN_I32, O::DS_MIN_RTN_U32,
       O::DS_MAX_RTN_U32, O::DS_AND_RTN_B32, O::DS_OR_RTN_B32,
       O::DS_XOR_RTN_B32, O::DS_WRXCHG_RTN_B32, O::DS_READ_B32,
       O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase DsAtomic64Bounds(bool add) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = add ? "DsAddU64Bounds" : "DsOrB64Bounds";
  const u32 atomic_word = add ? 0xd9000000u : 0xd9280000u;
  auto &code = test.code;
  AppendVMovU32(&code, 0, 0x1234);
  AppendVMovU32(&code, 7, 0);
  const std::array<u32, 5> initial = add
      ? std::array<u32, 5>{0xffffffffu, 1, 0xffffffffu, 0xfffffffeu, 0x13579bdf}
      : std::array<u32, 5>{0x100, 0x200, 0x400, 0x800, 0x13579bdf};
  for (u32 i = 0; i < std::size(initial); ++i) {
    AppendVMovLiteral(&code, 3, initial[i]);
    code.push_back(EncodeDs0(0x0d, i * 4u));
    code.push_back(EncodeDs1(0, 3, 7));
  }
  AppendVMovU32(&code, 63, 1);
  AppendVMovU32(&code, 64, 2);
  AppendVMovU32(&code, 7, 3); // The 64-bit address ignores the low three bits.
  code.insert(code.end(), {atomic_word, 0x00003f07u});
  AppendVMovLiteral(&code, 7, 0x10007);
  code.push_back(EncodeDs0(add ? 0x40 : 0x4a, 2)); // Offset before wrap/alignment -> 8.
  code.push_back(EncodeDs1(0, 63, 7));
  AppendVMovLiteral(&code, 63, 0xffffffffu);
  AppendVMovLiteral(&code, 64, 0xffffffffu);
  AppendVMovU32(&code, 7, 16); // Only one DWORD remains: ignore the entire atomic.
  code.insert(code.end(), {atomic_word, 0x00003f07u});
  AppendVMovU32(&code, 7, 0);
  code.push_back(EncodeSMovB32(126, InlineU32(0)));
  code.insert(code.end(), {atomic_word, 0x00003f07u});
  code.push_back(EncodeSMovB32(126, InlineU32(1)));
  for (u32 i = 0; i < std::size(initial); ++i) {
    code.push_back(EncodeDs0(0x36, i * 4u));
    code.push_back(EncodeDs1(8, 0, 7));
    AppendStoreVgpr(&code, 8, i);
  }
  AppendStoreVgpr(&code, 0, 5); // No-return DS must not overwrite the encoded VDst.
  AppendEnd(&code);
  test.expected = add ? std::vector<u32>{0, 4, 0, 1, 0x13579bdf, 0x1234}
                      : std::vector<u32>{0x101, 0x202, 0x401, 0x802, 0x13579bdf, 0x1234};
  test.compute_info.lds_size_dwords = 5;
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32,
                  add ? O::DS_ADD_U64 : O::DS_OR_B64,
                  O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {add ? "OpAtomicIAdd %ulong" : "OpAtomicOr %ulong",
                         "WorkgroupMemoryExplicitLayoutKHR", "Aliased"};
  return test;
}

TestCase DsAtomic64Contention(bool add, u32 wave_size) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = add ? (wave_size == 32 ? "DsAddU64ContentionWave32" : "DsAddU64ContentionWave64")
                  : (wave_size == 32 ? "DsOrB64ContentionWave32" : "DsOrB64ContentionWave64");
  auto &code = test.code;
  AppendVMovU32(&code, 7, 0);
  code.push_back(EncodeSop1(0x04, 10, 126));
  code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
  code.push_back(EncodeSop1(0x04, 126, 106));
  for (u32 offset : {0, 4}) {
    code.push_back(EncodeDs0(0x0d, offset));
    code.push_back(EncodeDs1(0, 7, 7));
  }
  code.push_back(EncodeSop1(0x04, 126, 10));
  code.push_back(EncodeSopp(0x0a, 0));
  if (add) {
    AppendVMovLiteral(&code, 3, 0xffffffffu);
    AppendVMovU32(&code, 4, 1);
  } else {
    AppendVMovU32(&code, 5, 1);
    AppendVMovU32(&code, 6, 2);
    code.push_back(EncodeVop2(0x1a, 3, Vgpr(0), 5));
    code.push_back(EncodeVop2(0x1a, 4, Vgpr(0), 6));
  }
  AppendSMovLiteral(&code, 126, 0x55555555u);
  AppendSMovLiteral(&code, 127, 0xaaaaaaaau);
  code.insert(code.end(), {add ? 0xd9000000u : 0xd9280000u, 0x00000307u});
  code.push_back(EncodeSop1(0x04, 126, 10));
  code.push_back(EncodeSopp(0x0a, 0));
  for (u32 half = 0; half < 2; ++half) {
    code.push_back(EncodeDs0(0x36, half * 4));
    code.push_back(EncodeDs1(8, 0, 7));
    AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, half * 128);
  }
  AppendEnd(&code);
  test.expected.assign(128, add ? 0xffffffc0u : wave_size == 32 ? 0x55555555u : 0xffffffffu);
  test.expected.resize(256, add ? 0x7fu : wave_size == 32 ? 0xaaaaaaaau : 0xfffffffeu);
  test.compute_info.threads_num[0] = 128;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.compute_info.lds_size_dwords = 2;
  test.has_compute_info = true;
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::S_BARRIER,
                  O::V_CMP_EQ_U32, O::V_ADD_NC_U32,
                  O::DS_WRITE_B32, add ? O::DS_ADD_U64 : O::DS_OR_B64, O::DS_READ_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  if (!add) {
    test.opcodes.push_back(O::V_LSHLREV_B32);
  }
  test.required_spirv = {add ? "OpAtomicIAdd %ulong" : "OpAtomicOr %ulong",
                         "WorkgroupMemoryExplicitLayoutKHR", "Aliased"};
  return test;
}

TestCase DsBoundedAtomicBoundaries(bool decrement, bool gds) {
  using O = ShaderOpcode;
  constexpr u32 guard = 0x13579bdfu;
  struct Boundary {
    u32 before, limit, incremented, decremented;
  };
  constexpr Boundary boundaries[] = {
      {4, 5, 5, 3},
      {5, 5, 0, 4},
      {6, 5, 0, 5},
      {0, 5, 1, 5},
      {0, 0, 0, 0},
      {1, 0, 0, 0},
      {0xffffffffu, 0xffffffffu, 0, 0xfffffffeu},
      {0xfffffffeu, 0xffffffffu, 0xffffffffu, 0xfffffffdu},
      {0, 0xffffffffu, 1, 0xffffffffu},
      {0x80000000u, 0x7fffffffu, 0, 0x7fffffffu},
      {0x7fffffffu, 0x80000000u, 0x80000000u, 0x7ffffffeu},
      {0xffffffffu, 0x80000000u, 0, 0x80000000u}};
  constexpr u32 count = static_cast<u32>(std::size(boundaries));
  const u32 opcode = decrement ? 0x24u : 0x23u;
  TestCase test;
  test.name =
      decrement
          ? (gds ? "DsDecReturnGdsBoundaries" : "DsDecReturnLdsBoundaries")
          : (gds ? "DsIncReturnGdsBoundaries" : "DsIncReturnLdsBoundaries");
  auto &code = test.code;
  AppendVMovU32(&code, 2, 0);
  std::vector<u32> memory(68, guard);
  for (u32 i = 0; i < count; ++i)
    memory[i + 1] = boundaries[i].before;
  if (!gds) {
    for (u32 i = 0; i < memory.size(); ++i) {
      AppendVMovLiteral(&code, 3, memory[i]);
      code.push_back(EncodeDs0(0x0d, i * 4));
      code.push_back(EncodeDs1(0, 3, 2));
    }
  }
  auto StoreResult = [&](u32 vgpr, u32 expected) {
    AppendStoreVgpr(&code, vgpr, static_cast<u32>(test.expected.size()));
    test.expected.push_back(expected);
  };
  for (u32 i = 0; i < count; ++i) {
    AppendVMovU32(&code, 2, i * 4);
    AppendVMovLiteral(&code, 3, boundaries[i].limit);
    // Preserve the captured DATA0/VDST alias and exact increment words in GDS.
    code.push_back(decrement ? EncodeDs0(opcode, 4, gds)
                             : (gds ? 0xd88e0004u : 0xd88c0004u));
    code.push_back(0x03000302u);
    StoreResult(3, boundaries[i].before);
    code.push_back(EncodeDs0(0x36, 4, gds));
    code.push_back(EncodeDs1(4, 0, 2));
    StoreResult(4, decrement ? boundaries[i].decremented
                             : boundaries[i].incremented);
  }

  // EXEC=0 preserves memory and the aliased destination register.
  AppendVMovU32(&code, 2, 0);
  AppendVMovLiteral(&code, 3, 0x12345678u);
  code.push_back(EncodeSop1(0x04, 10, 126));
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  code.push_back(EncodeDs0(opcode, 4, gds));
  code.push_back(0x03000302u);
  code.push_back(EncodeSop1(0x04, 126, 10));
  StoreResult(3, 0x12345678u);

  // Use both offset bytes and alias VDST with ADDR instead of DATA0.
  AppendVMovU32(&code, 2, 4);
  AppendVMovLiteral(&code, 3, 0xffffffffu);
  code.push_back(EncodeDs0(opcode, 0x104, gds));
  code.push_back(EncodeDs1(2, 3, 2));
  StoreResult(2, guard);

  // Overwriting an unused return must not let DCE discard the atomic update.
  AppendVMovU32(&code, 2, 0);
  code.push_back(EncodeDs0(opcode, 0x108, gds));
  code.push_back(EncodeDs1(4, 3, 2));
  AppendVMovU32(&code, 4, 0);

  // An active access just outside the resource returns zero.
  AppendVMovU32(&code, 2, static_cast<u32>(memory.size() * 4 - 4));
  code.push_back(EncodeDs0(opcode, 4, gds));
  code.push_back(0x03000302u);
  StoreResult(3, 0);

  // Read back all words, including guards around both address ranges.
  AppendVMovU32(&code, 2, 0);
  for (u32 i = 0; i < memory.size(); ++i) {
    u32 expected = memory[i];
    if (i >= 1 && i <= count) {
      expected = decrement ? boundaries[i - 1].decremented
                           : boundaries[i - 1].incremented;
    } else if (i == 66) {
      expected = decrement ? guard - 2 : guard + 2;
    }
    code.push_back(EncodeDs0(0x36, i * 4, gds));
    code.push_back(EncodeDs1(4, 0, 2));
    StoreResult(4, expected);
    if (gds)
      test.expected_gds.push_back(expected);
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  test.compute_info.lds_size_dwords = static_cast<u32>(memory.size());
  test.opcodes = {
      O::S_MOV_B64,          O::V_MOV_B32,
      O::DS_READ_B32,        decrement ? O::DS_DEC_RTN_U32 : O::DS_INC_RTN_U32,
      O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange",
                         "OpLoopMerge"};
  if (gds)
    test.gds_initial = memory;
  else
    test.opcodes.push_back(O::DS_WRITE_B32);
  return test;
}

TestCase DsBoundedAtomicContention(bool decrement, bool gds, u32 wave_size) {
  using O = ShaderOpcode;
  constexpr u32 guard = 0x13579bdfu;
  constexpr u32 limit = 15;
  TestCase test;
  if (wave_size == 32) {
    test.name = decrement ? (gds ? "DsDecReturnGdsContentionWave32"
                                 : "DsDecReturnLdsContentionWave32")
                          : (gds ? "DsIncReturnGdsContentionWave32"
                                 : "DsIncReturnLdsContentionWave32");
  } else {
    test.name = decrement ? (gds ? "DsDecReturnGdsContentionWave64"
                                 : "DsDecReturnLdsContentionWave64")
                          : (gds ? "DsIncReturnGdsContentionWave64"
                                 : "DsIncReturnLdsContentionWave64");
  }
  auto &code = test.code;
  AppendVMovU32(&code, 2, 0);
  AppendVMovU32(&code, 3, limit);
  AppendVMovU32(&code, 5, 1);
  AppendVMovLiteral(&code, 6, guard);
  if (!gds) {
    // One lane initializes LDS before every wave in the workgroup uses it.
    code.push_back(EncodeSop1(0x04, 10, 126));
    code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
    code.push_back(EncodeSop1(0x04, 126, 106));
    code.push_back(EncodeDs0(0x0d, 4));
    code.push_back(EncodeDs1(0, 2, 2));
    code.push_back(EncodeSop1(0x04, 126, 10));
    code.push_back(EncodeSopp(0x0a, 0));
  }
  // Distinct masks in the wave64 halves also check inactive return
  // preservation.
  code.push_back(EncodeSop1(0x04, 10, 126));
  AppendSMovLiteral(&code, 126, 0x55555555u);
  AppendSMovLiteral(&code, 127, 0xaaaaaaaau);
  code.push_back(EncodeDs0(decrement ? 0x24 : 0x23, 4, gds));
  code.push_back(EncodeDs1(6, 3, 2));
  // Returned old values form a histogram; no lane ordering is assumed.
  code.push_back(EncodeVop2(0x1a, 7, InlineU32(2), 6));
  AppendBufferStoreOpcode(&code, 0x32, 5, 7);
  code.push_back(EncodeSop1(0x04, 126, 10));
  AppendVMovLiteral(&code, 7, guard);
  code.push_back(EncodeVopc(0xc2, Vgpr(7), 6));
  code.push_back(EncodeVop2(0x01, 8, InlineU32(0), 5));
  AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, 16);
  code.push_back(EncodeSopp(0x0a, 0));
  // S_BARRIER orders LDS; use atomic add-zero to read GDS consistently.
  code.push_back(EncodeDs0(gds ? 0x20 : 0x36, 4, gds));
  code.push_back(EncodeDs1(9, 2, 2));
  AppendStoreVgprAtLaneDwordOffset(&code, 9, 0, 144);
  AppendEnd(&code);
  test.expected.assign(16, 4); // 64 active lanes, four complete wraps.
  for (u32 lane = 0; lane < 128; ++lane) {
    const uint64_t mask = 0xaaaaaaaa55555555ull;
    test.expected.push_back(
        (mask & (uint64_t{1} << (lane % wave_size))) == 0 ? 1 : 0);
  }
  test.expected.resize(272, 0);
  test.initial.resize(test.expected.size());
  test.compute_info.threads_num[0] = 128;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.compute_info.lds_size_dwords = 3;
  test.has_compute_info = true;
  test.opcodes = {O::S_MOV_B32,
                  O::S_MOV_B64,
                  O::S_BARRIER,
                  O::V_MOV_B32,
                  O::V_LSHLREV_B32,
                  O::V_ADD_NC_U32,
                  O::V_CMP_EQ_U32,
                  O::V_CNDMASK_B32,
                  decrement ? O::DS_DEC_RTN_U32 : O::DS_INC_RTN_U32,
                  gds ? O::DS_ADD_RTN_U32 : O::DS_READ_B32,
                  O::BUFFER_ATOMIC_ADD,
                  O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange",
                         "OpLoopMerge"};
  if (gds) {
    test.gds_initial = {guard, 0, guard};
    test.expected_gds = test.gds_initial;
  } else
    test.opcodes.push_back(O::DS_WRITE_B32);
  return test;
}

TestCase DsMiscVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 0, 0);
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 2, 0x40800000u);
  code.push_back(EncodeDs0(0x0d, 0));
  code.push_back(EncodeDs1(0, 2, 1));
  AppendVMovLiteral(&code, 3, 0x3f800000u);
  code.push_back(EncodeDs0(0x0d, 4));
  code.push_back(EncodeDs1(0, 3, 1));
  AppendVMovLiteral(&code, 4, 0x40000000u);
  code.push_back(EncodeDs0(0x12, 0));
  code.push_back(EncodeDs1(0, 4, 1));
  AppendVMovLiteral(&code, 5, 0x40400000u);
  code.push_back(EncodeDs0(0x13, 4));
  code.push_back(EncodeDs1(0, 5, 1));
  code.push_back(EncodeDs0(0x36, 0));
  code.push_back(EncodeDs1(6, 0, 1));
  code.push_back(EncodeDs0(0x36, 4));
  code.push_back(EncodeDs1(7, 0, 1));
  AppendVMovLiteral(&code, 8, 0x12345678u);
  code.push_back(EncodeDs0(0x35, 0x001f));
  code.push_back(EncodeDs1(9, 0, 8));
  code.push_back(EncodeSMovB32(124, InlineU32(0)));
  AppendVMovLiteral(&code, 10, 0xabcdef01u);
  code.push_back(EncodeDs0(0xb0, 8));
  code.push_back(EncodeDs1(0, 10, 0));
  code.push_back(EncodeDs0(0xb1, 8));
  code.push_back(EncodeDs1(11, 0, 0));

  const u32 results[] = {6, 7, 9, 11};
  for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
    AppendStoreVgpr(&code, results[i], i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "DsMiscVariants";
  test.code = code;
  test.initial = std::vector<u32>(4, 0);
  test.expected = {0x40000000u, 0x40400000u, 0x12345678u, 0xabcdef01u};
  test.opcodes = {O::V_MOV_B32,
                  O::S_MOV_B32,
                  O::DS_WRITE_B32,
                  O::DS_MIN_F32,
                  O::DS_MAX_F32,
                  O::DS_READ_B32,
                  O::DS_SWIZZLE_B32,
                  O::DS_WRITE_ADDTID_B32,
                  O::DS_READ_ADDTID_B32,
                  O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.compute_info.threads_num[0] = 1;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.has_compute_info = true;
  return test;
}

TestCase DsFloatMinMaxIgnoresData1() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 0);
  AppendVMovLiteral(&code, 2, 0x40800000u);
  code.push_back(EncodeDs0(0x0d, 0));
  code.push_back(EncodeDs1(0, 2, 1));
  AppendVMovLiteral(&code, 3, 0x40800000u);
  code.push_back(EncodeDs0(0x0d, 4));
  code.push_back(EncodeDs1(0, 3, 1));
  AppendVMovLiteral(&code, 4, 0x41100000u);
  AppendVMovLiteral(&code, 5, 0x40000000u);
  code.push_back(EncodeDs0(0x12, 0));
  code.push_back(EncodeDs1Ex(0, 5, 4, 1));
  AppendVMovLiteral(&code, 6, 0x3f800000u);
  AppendVMovLiteral(&code, 7, 0x40400000u);
  code.push_back(EncodeDs0(0x13, 4));
  code.push_back(EncodeDs1Ex(0, 7, 6, 1));
  code.push_back(EncodeDs0(0x36, 0));
  code.push_back(EncodeDs1(8, 0, 1));
  code.push_back(EncodeDs0(0x36, 4));
  code.push_back(EncodeDs1(9, 0, 1));
  AppendStoreVgpr(&code, 8, 0);
  AppendStoreVgpr(&code, 9, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "DsFloatMinMaxIgnoresData1";
  test.code = code;
  test.initial = std::vector<u32>(2, 0);
  test.expected = {0x40800000u, 0x40800000u};
  test.opcodes = {O::V_MOV_B32,  O::DS_WRITE_B32, O::DS_MIN_F32,
                  O::DS_MAX_F32, O::DS_READ_B32,  O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.compute_info.threads_num[0] = 1;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.has_compute_info = true;
  return test;
}

TestCase DsFloatMinMaxClasses(bool gds) {
  using O = ShaderOpcode;
  // Rows are memory, columns source: -inf, -normal, -denorm, -0, +0,
  // +denorm, +normal, +inf, qNaN, sNaN. O/N select old/new; Q/R quiet them.
  constexpr std::string_view minimum[] = {
      "OOOOOOOOOR", "NNOOOOOOOR", "NNNNOOOOOR", "NNNOOOOOOR", "NNNNOOOOOR",
      "NNNNNNOOOR", "NNNNNNOOOR", "NNNNNNNOOR", "NNNNNNNNOR", "QQQQQQQQQQ"};
  constexpr std::string_view maximum[] = {
      "ONNNNNNNOR", "OONNNNNNOR", "OOOONNNNOR", "OOOONNNNOR", "OOOOONNNOR",
      "OOOOOONNOR", "OOOOOONNOR", "OOOOOOOOOR", "NNNNNNNNOR", "QQQQQQQQQQ"};
  constexpr u32 old_values[] = {
      0xff800000u, 0xc0000000u, 0x80000100u, 0x80000000u, 0u,
      0x00000100u, 0x40000000u, 0x7f800000u, 0x7fc12345u, 0x7f812345u};
  constexpr u32 sources[] = {
      0xff800000u, 0xc0400000u, 0x80000001u, 0x80000000u, 0u,
      0x00000001u, 0x40400000u, 0x7f800000u, 0xffc54321u, 0xff854321u};
  const auto select = [](char choice, u32 old, u32 source) {
    const u32 bits = choice == 'O' || choice == 'Q' ? old : source;
    return choice == 'Q' || choice == 'R' ? bits | 0x00400000u : bits;
  };
  TestCase test;
  test.name = gds ? "DsFloatMinMaxGdsClasses" : "DsFloatMinMaxLdsClasses";
  for (u32 row = 0; row < std::size(old_values); ++row) {
    for (u32 column = 0; column < std::size(sources); ++column) {
      const auto old = old_values[row];
      const auto source = sources[column];
      test.initial.insert(test.initial.end(), {old, source, 0u, 0u});
      test.expected.insert(test.expected.end(),
                           {old, source, select(minimum[row][column], old, source),
                            select(maximum[row][column], old, source)});
    }
  }
  // Reversing positive denorm magnitudes still selects source for min and old for max.
  test.initial.insert(test.initial.end(), {1u, 0x100u, 0u, 0u});
  test.expected.insert(test.expected.end(), {1u, 0x100u, 0x100u, 1u});
  const auto count = static_cast<u32>(test.initial.size() / 4u);
  auto &code = test.code;
  code.push_back(EncodeVop2(0x1a, 1, InlineU32(3), 0));
  code.push_back(EncodeVop2(0x1a, 8, InlineU32(4), 0));
  code.push_back(EncodeMubuf0(0x0d));
  code.push_back(EncodeMubuf1(2, 0, 8));
  for (u32 offset : {0u, 4u}) {
    code.push_back(EncodeDs0(0x0d, offset, gds));
    code.push_back(EncodeDs1(0, 2, 1));
  }
  code.push_back(EncodeDs0(0x12, 0, gds));
  code.push_back(EncodeDs1Ex(0, 250, 3, 1)); // DATA1 is unused and uninitialized.
  code.push_back(EncodeDs0(0x13, 4, gds));
  code.push_back(EncodeDs1Ex(0, 251, 3, 1));
  AppendVMovLiteral(&code, 6, 0xff800000u);
  AppendVMovLiteral(&code, 7, 0x7f800000u);
  code.push_back(EncodeSop1(0x04, 4, 126));
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  // Inactive updates must leave the table results untouched.
  code.push_back(EncodeDs0(0x12, 0, gds));
  code.push_back(EncodeDs1(0, 6, 1));
  code.push_back(EncodeDs0(0x13, 4, gds));
  code.push_back(EncodeDs1(0, 7, 1));
  code.push_back(EncodeSop1(0x04, 126, 4));
  // Active accesses immediately beyond the allocation must also be discarded.
  AppendVMovU32(&code, 9, count * 8u);
  code.push_back(EncodeDs0(0x12, 0, gds));
  code.push_back(EncodeDs1(0, 6, 9));
  code.push_back(EncodeDs0(0x13, 0, gds));
  code.push_back(EncodeDs1(0, 7, 9));
  for (u32 component = 0; component < 2u; ++component) {
    code.push_back(EncodeDs0(0x36, component * 4u, gds));
    code.push_back(EncodeDs1(4u + component, 0, 1));
  }
  code.push_back(EncodeMubuf0(0x1d, 8));
  code.push_back(EncodeMubuf1(4, 12, 8));
  AppendEnd(&code);
  test.compute_info.threads_num[0] = count;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.lds_size_dwords = count * 2u;
  test.has_compute_info = true;
  test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::S_MOV_B64,
                  O::BUFFER_LOAD_DWORDX2, O::DS_WRITE_B32, O::DS_MIN_F32,
                  O::DS_MAX_F32, O::DS_READ_B32, O::BUFFER_STORE_DWORDX2,
                  O::S_ENDPGM};
  test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange", "OpLoopMerge"};
  test.forbidden_spirv = {"OpFOrdLessThan", "OpFOrdGreaterThan"};
  if (gds) {
    test.gds_initial.resize(count * 2u);
    for (u32 i = 0; i < count; ++i) {
      test.expected_gds.push_back(test.expected[i * 4u + 2u]);
      test.expected_gds.push_back(test.expected[i * 4u + 3u]);
    }
  }
  return test;
}

TestCase DsSwizzleInvalidSourceLaneZero() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 1, 100);
  code.push_back(EncodeDs0(0x35, 0x00e0));
  code.push_back(EncodeDs1(2, 0, 1));
  code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
  AppendBufferStoreDword(&code, 2, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "DsSwizzleInvalidSourceLaneZero";
  test.code = code;
  test.expected = {0, 0, 0, 0};
  test.opcodes = {O::V_MOV_B32, O::DS_SWIZZLE_B32, O::V_LSHLREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase DsPermuteCapturedExecOffsetAndWrap() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
  AppendBufferLoadDword(&code, 10, 30);
  AppendVMovLiteral(&code, 1, 0xdeadbeefu);
  AppendVMovU32(&code, 9, 100);
  code.push_back(EncodeVop2(0x25, 9, Vgpr(0), 9));
  code.push_back(EncodeSop1(0x04, 4, 126));
  code.push_back(EncodeVop2(0x1b, 18, InlineU32(1), 0));
  code.push_back(EncodeVopc(0xc2, InlineU32(1), 18));
  code.push_back(EncodeSop1(0x04, 126, 106));
  code.push_back(0xdac80000u);
  code.push_back(0x0100090au); // Captured SH2 instruction: ds_permute_b32 v1, v10, v9
  code.push_back(EncodeSop1(0x04, 126, 4));
  AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 4);

  code.push_back(0xdac80000u);
  code.push_back(0x0100090au);
  AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 8);
  AppendVMovU32(&code, 10, 127);
  code.push_back(EncodeDs0(0xb2, 1));
  code.push_back(EncodeDs1(1, 9, 10));
  AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 12);
  AppendEnd(&code);

  TestCase test;
  test.name = "DsPermuteCapturedExecOffsetAndWrap";
  test.code = code;
  test.initial = {0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
  test.expected = {0, 0, 12, 4, 0xdeadbeefu, 103, 0xdeadbeefu, 0,
                   101, 103, 0, 102, 103, 0, 0, 0};
  test.opcodes = {O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD, O::V_MOV_B32,
                  O::V_ADD_NC_U32, O::V_AND_B32, O::V_CMP_EQ_U32, O::S_MOV_B64,
                  O::DS_PERMUTE_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpGroupNonUniformShuffle", "OpGroupNonUniformBallot"};
  test.forbidden_spirv = {" Workgroup", "OpControlBarrier", "OpMemoryBarrier"};
  test.decoded_counts = {{"DS_PERMUTE_B32", 3}};
  test.ir_counts = {{"PermuteU32", 3}};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.wave_size = 32;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase DsPermuteWave64UsesIndependentHalves() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 9, 100);
  code.push_back(EncodeVop2(0x25, 9, Vgpr(0), 9));
  code.push_back(EncodeVop2(0x1d, 10, InlineU32(31), 0));
  code.push_back(EncodeVop2(0x1a, 10, InlineU32(2), 10));
  code.push_back(0xdac80000u);
  code.push_back(0x0100090au);
  AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "DsPermuteWave64UsesIndependentHalves";
  test.code = code;
  test.initial = std::vector<u32>(64, 0);
  for (u32 lane = 0; lane < 64; ++lane) {
    test.expected.push_back(100 + (lane ^ 31));
  }
  test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_XOR_B32, O::V_LSHLREV_B32,
                  O::DS_PERMUTE_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpGroupNonUniformShuffle", "OpGroupNonUniformBallot"};
  test.forbidden_spirv = {" Workgroup", "OpControlBarrier", "OpMemoryBarrier"};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.wave_size = 64;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase DsBpermuteCapturedExecOffsetAndWrap() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
  AppendBufferLoadDword(&code, 17, 30);
  AppendVMovLiteral(&code, 1, 0xdeadbeefu);
  AppendVMovU32(&code, 3, 100);
  code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3));
  code.push_back(EncodeSop1(0x04, 4, 126));
  code.push_back(EncodeVop2(0x1b, 18, InlineU32(1), 0));
  code.push_back(EncodeVopc(0xc2, InlineU32(1), 18));
  code.push_back(EncodeSop1(0x04, 126, 106));
  code.push_back(0xdacc0000u);
  code.push_back(0x01000311u); // ds_bpermute_b32 v1, v17, v3
  code.push_back(EncodeSop1(0x04, 126, 4));
  AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 4);

  AppendVMovU32(&code, 17, 124);
  code.push_back(EncodeDs0(0xb3, 4));
  code.push_back(EncodeDs1(2, 3, 17));
  AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 8);
  AppendEnd(&code);

  TestCase test;
  test.name = "DsBpermuteCapturedExecOffsetAndWrap";
  test.code = code;
  test.initial = {0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0};
  test.expected = {0,          0, 12,         4,
                   0xdeadbeefu, 0, 0xdeadbeefu, 101,
                   100,        100, 100,        100};
  test.opcodes = {O::V_LSHLREV_B32,     O::BUFFER_LOAD_DWORD,
                  O::V_MOV_B32,         O::V_ADD_NC_U32,
                  O::V_AND_B32,         O::V_CMP_EQ_U32,
                  O::S_MOV_B64,
                  O::DS_BPERMUTE_B32,   O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.required_spirv = {"OpGroupNonUniformShuffle",
                         "OpGroupNonUniformBallot"};
  test.forbidden_spirv = {" Workgroup", "OpControlBarrier",
                          "OpMemoryBarrier"};
  test.decoded_counts = {{"DS_BPERMUTE_B32", 2}};
  test.ir_counts = {{"BpermuteU32", 2}};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.wave_size = 32;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase DsBpermuteWave64UsesIndependentHalves() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 3, 100);
  code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3));
  AppendVMovU32(&code, 17, 128);
  code.push_back(0xdacc0000u);
  code.push_back(0x01000311u); // ds_bpermute_b32 v1, v17, v3
  AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "DsBpermuteWave64UsesIndependentHalves";
  test.code = code;
  test.initial = std::vector<u32>(64, 0);
  test.expected = std::vector<u32>(32, 100);
  test.expected.insert(test.expected.end(), 32, 132);
  test.opcodes = {O::V_MOV_B32,       O::V_ADD_NC_U32,
                  O::DS_BPERMUTE_B32, O::V_LSHLREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpGroupNonUniformShuffle"};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.wave_size = 64;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase Wave64CrossHalfLaneAndLds() {
  using O = ShaderOpcode;
  std::vector<u32> code;
  AppendVMovU32(&code, 1, 100);
  code.push_back(EncodeVop2(0x25, 1, Vgpr(0), 1));
  AppendVop3(&code, 0x360, 8, Vgpr(1), InlineU32(47));
  AppendStoreSgprAtLaneDwordOffset(&code, 8, 0, 0);
  code.push_back(EncodeSop1(0x04, 10, 126));
  code.push_back(EncodeVop2(0x1b, 2, InlineU32(63), 0));
  code.push_back(EncodeVopc(0xc2, InlineU32(40), 2));
  code.push_back(EncodeSop1(0x04, 126, 106));
  code.push_back(EncodeVop1(0x02, 9, Vgpr(1)));
  code.push_back(EncodeSop1(0x04, 126, 10));
  AppendStoreSgprAtLaneDwordOffset(&code, 9, 0, 128);
  AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, 256);
  AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, 384);
  code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0));
  code.push_back(EncodeDs0(0x0d));
  code.push_back(EncodeDs1(0, 1, 2));
  code.push_back(EncodeSopp(0x0a, 0));
  code.push_back(EncodeVop2(0x1d, 3, InlineU32(64), 0));
  code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 3));
  code.push_back(EncodeDs0(0x36));
  code.push_back(EncodeDs1(4, 0, 3));
  AppendStoreVgprAtLaneDwordOffset(&code, 4, 0, 512);
  AppendEnd(&code);

  TestCase test;
  test.name = "Wave64CrossHalfLaneAndLds";
  test.code = std::move(code);
  test.initial.resize(640);
  test.expected.insert(test.expected.end(), 64, 147);
  test.expected.insert(test.expected.end(), 64, 211);
  test.expected.insert(test.expected.end(), 64, 140);
  test.expected.insert(test.expected.end(), 64, 204);
  test.expected.insert(test.expected.end(), 128, 0);
  test.expected.insert(test.expected.end(), 128, 256);
  for (u32 lane = 0; lane < 128; lane++) {
    test.expected.push_back(100 + (lane ^ 64));
  }
  test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_READLANE_B32,
                  O::V_READFIRSTLANE_B32, O::V_CMP_EQ_U32, O::S_MOV_B64,
                  O::V_LSHLREV_B32, O::V_AND_B32, O::V_XOR_B32, O::DS_WRITE_B32,
                  O::S_BARRIER, O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpGroupNonUniformBallot", "OpGroupNonUniformShuffle",
                         "OpControlBarrier"};
  test.ir_counts = {{"Barrier", 1}};
  test.compute_info.threads_num[0] = 128;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

void CheckWave64WholeWaveResults() {
  const auto test = Wave64CrossHalfLaneAndLds();
  const auto native = CompileCase(test, 64);
  const auto emulated = CompileCase(test, 32);
  spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
  std::string native_text, emulated_text;
  Require(test.name, "whole-wave disassembly",
          tools.Disassemble(native.spirv, &native_text) &&
              tools.Disassemble(emulated.spirv, &emulated_text),
          "failed to disassemble whole-wave operations");
  const auto ballots = CountText(native_text, "OpGroupNonUniformBallot ");
  Require(test.name, "whole-wave ballot reuse",
          ballots != 0 &&
              CountText(emulated_text, "OpGroupNonUniformBallot ") == ballots * 2,
          "emulated halves repeated a whole-wave ballot");
  // READLANE still emits two shuffles per half; READFIRSTLANE needs one shared pair.
  Require(test.name, "whole-wave first-lane reuse",
          CountText(emulated_text, "OpGroupNonUniformShuffle ") == 6,
          "emulated halves repeated the first-active-lane broadcast");
}

TestCase Wave64RawMasksAndScalarBranch() {
  using O = ShaderOpcode;
  std::vector<u32> code;
  AppendVMovU32(&code, 1, 7);
  AppendSMovLiteral(&code, 8, 33u << 16u);
  code.push_back(EncodeSop2(0x29, 126, 193u, 8)); // captured ES BFE EXEC,-1,width
  AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
  code.push_back(EncodeSop2(0x1f, 126, InlineU32(1), InlineU32(32)));
  const auto exec_branch = code.size();
  code.push_back(0);
  code.push_back(EncodeSopp(0x0a, 0));
  code.push_back(EncodeSop1(0x24, 10, 193u)); // s_and_saveexec_b64 s[10:11],-1
  const auto scc_branch = code.size();
  code.push_back(0);
  AppendVMovU32(&code, 1, 9);
  AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 64);
  const auto restore = code.size();
  code[exec_branch] = EncodeSopp(0x08, restore - exec_branch - 1);
  code[scc_branch] = EncodeSopp(0x04, restore - scc_branch - 1);
  code.push_back(EncodeSop1(0x04, 126, 193u));
  code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
  code.push_back(EncodeSop1(0x04, 12, 106));
  code.push_back(EncodeSMovB32(13, InlineU32(1)));
  AppendVop3(&code, 0x101, 2, InlineU32(0), InlineU32(11), 12);
  AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 128);
  code.push_back(EncodeSMovB32(127, InlineU32(1)));
  AppendVMovU32(&code, 3, 13);
  AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, 192);
  code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
  AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, 256);
  code.push_back(EncodeSMovB32(107, InlineU32(2)));
  code.push_back(EncodeSMovB32(127, 107));
  AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 320);
  AppendEnd(&code);

  TestCase test;
  test.name = "Wave64RawMasksAndScalarBranch";
  test.code = std::move(code);
  test.initial.assign(384, 0xdeadbeef);
  test.expected = test.initial;
  for (u32 lane = 0; lane < 64; lane++) {
    if (lane < 33) {
      test.expected[lane] = 7;
      test.expected[192 + lane] = 13;
    }
    test.expected[128 + lane] = lane < 33 ? 11 : 0;
  }
  test.expected[64 + 32] = 9;
  test.expected[256] = 13;
  test.expected[320] = 7;
  test.expected[320 + 33] = 7;
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_BFE_U64, O::S_LSHL_B64,
                  O::S_CBRANCH_EXECZ, O::S_AND_SAVEEXEC_B64, O::S_CBRANCH_SCC0,
                  O::S_BARRIER, O::S_MOV_B64, O::V_CMP_EQ_U32, O::V_CNDMASK_B32,
                  O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase Wave64PartialMultidimensionalWorkgroup() {
  using O = ShaderOpcode;
  std::vector<u32> code;
  code.push_back(EncodeVop2(0x25, 3, Vgpr(2), 2));
  code.push_back(EncodeVop2(0x25, 3, Vgpr(2), 3));
  code.push_back(EncodeVop2(0x25, 3, Vgpr(1), 3));
  code.push_back(EncodeVop2(0x1a, 3, InlineU32(3), 3));
  code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3));
  code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 3));
  AppendVMovU32(&code, 5, 1);
  AppendBufferStoreOpcode(&code, 0x32, 5, 4, false);
  AppendEnd(&code);
  TestCase test;
  test.name = "Wave64PartialMultidimensionalWorkgroup";
  test.code = std::move(code);
  test.initial.resize(128);
  test.expected.assign(96, 1);
  test.expected.resize(128);
  test.opcodes = {O::V_ADD_NC_U32, O::V_LSHLREV_B32, O::V_MOV_B32,
                  O::BUFFER_ATOMIC_ADD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 8;
  test.compute_info.threads_num[1] = 3;
  test.compute_info.threads_num[2] = 4;
  test.compute_info.thread_ids_num = 3;
  test.has_compute_info = true;
  return test;
}

TestCase Wave64AppendConsumeHighHalf() {
  using O = ShaderOpcode;
  std::vector<u32> code;
  AppendSMovLiteral(&code, 124, 4);
  code.push_back(EncodeSop2(0x1f, 126, 193u, InlineU32(32)));
  code.push_back(EncodeDs0(0x3e, 0, true));
  code.push_back(EncodeDs1(1, 0, 0));
  AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
  code.push_back(EncodeDs0(0x3d, 0, true));
  code.push_back(EncodeDs1(2, 0, 0));
  AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 64);
  AppendEnd(&code);
  TestCase test;
  test.name = "Wave64AppendConsumeHighHalf";
  test.code = std::move(code);
  test.initial.assign(128, 0xdeadbeef);
  test.expected = test.initial;
  for (u32 lane = 32; lane < 64; lane++) {
    test.expected[lane] = 10;
    test.expected[64 + lane] = 42;
  }
  test.gds_initial = {10};
  test.expected_gds = {10};
  test.opcodes = {O::S_MOV_B32, O::S_LSHL_B64, O::DS_APPEND, O::DS_CONSUME,
                  O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false,
                                  bool store_completion = false) {
  using O = ShaderOpcode;
  constexpr u32 groups = 31;
  TestCase test;
  test.name = store_completion ? (wave_size == 64 ? "Wave64StoreCompletionPublication"
                                                 : "Wave32StoreCompletionPublication")
             : dlc_only ? "BufferWorkgroupPublicationDlc"
             : wave_size == 64 ? "BufferWorkgroupPublicationWave64"
                               : "BufferWorkgroupPublicationWave32";
  auto &code = test.code;
  // Allocate work in execution order so every polled predecessor has started.
  // Only one lane per workgroup owns a ticket and publishes its prefix total.
  code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
  AppendVMovU32(&code, 1, 1);
  AppendVMovU32(&code, 20, 0);
  AppendBufferStoreOpcode(&code, 0x32, 1, 20, true);
  code.push_back(EncodeVop1(0x02, 20, Vgpr(1)));
  code.push_back(EncodeVop2(0x1a, 20, InlineU32(2), 1));
  AppendVMovU32(&code, 2, 0);
  code.push_back(EncodeSMovB32(21, InlineU32(0)));
  AppendSMovLiteral(&code, 22, 1000000);
  if (store_completion) AppendSMovLiteral(&code, 26, groups * 4);
  code.push_back(EncodeSopc(0x06, 20, InlineU32(0)));
  const auto first_ticket = code.size();
  code.push_back(0);

  const auto poll = code.size();
  if (store_completion) {
    code.push_back(EncodeVop2(0x25, 4, 26, 20));
    AppendVMovU32(&code, 3, 0);
    AppendBufferStoreOpcode(&code, 0x32, 3, 4, true);
  }
  code.push_back(EncodeMubuf0(0x0c, 0, false, true, !dlc_only) | (1u << 15u));
  code.push_back(EncodeMubuf1(2, 12, 20));
  code.push_back(EncodeSopp(0x0c, 0)); // S_WAITCNT vmcnt(0).
  code.push_back(EncodeVop1(0x02, 23, Vgpr(store_completion ? 3 : 2)));
  code.push_back(EncodeSopc(0x07, 23, InlineU32(0)));
  const auto ready = code.size();
  code.push_back(0);
  code.push_back(EncodeSop2(0x00, 21, 21, InlineU32(1)));
  code.push_back(EncodeSopc(0x0a, 21, 22));
  code.push_back(EncodeSopp(0x05, static_cast<u32>(poll - code.size() - 1)));
  // A broken poll reports an incorrect result instead of hanging the test GPU.
  AppendVMovLiteral(&code, 2, 0x80000000u);
  const auto publish = code.size();
  code[first_ticket] = EncodeSopp(0x05, publish - first_ticket - 1);
  code[ready] = EncodeSopp(0x05, publish - ready - 1);
  code.push_back(EncodeVop2(0x25, 2, Vgpr(1), 2));
  code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2));
  code.push_back(EncodeVop2(0x25, 20, InlineU32(4), 20));
  if (store_completion) {
    // The last lane writes the payload; lane zero publishes readiness after the
    // native wave-wide store completion, including both emulated wave64 halves.
    code.push_back(EncodeVop1(0x02, 24, Vgpr(2)));
    code.push_back(EncodeVop1(0x02, 25, Vgpr(20)));
    code.push_back(EncodeSop1(0x04, 126, 193u));
    code.push_back(EncodeVop1(0x01, 2, 24));
    code.push_back(EncodeVop1(0x01, 20, 25));
    code.push_back(EncodeSop2(0x1f, 126, InlineU32(1), InlineU32(wave_size - 1)));
  }
  // RDNA2 stores publish to L2 even without GLC/DLC on the producer.
  AppendBufferStoreDword(&code, 2, 20);
  if (store_completion) {
    code.push_back(0xbbfd0000u); // Captured S_WAITCNT_VSCNT null, 0 at 5b10:0x2464.
    code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
    code.push_back(EncodeVop2(0x25, 4, 26, 20));
    AppendVMovU32(&code, 3, 1);
    AppendBufferStoreOpcode(&code, 0x32, 3, 4, true);
  }
  AppendEnd(&code);

  test.initial.assign(groups * (store_completion ? 2 : 1) + 1, 0);
  test.expected = {groups};
  for (u32 ticket = 0; ticket < groups; ++ticket) {
    test.expected.push_back((ticket + 1) * (ticket + 2) / 2);
  }
  if (store_completion) test.expected.resize(groups * 2 + 1, 1);
  test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::V_MOV_B32,
                  O::BUFFER_ATOMIC_ADD, O::V_READFIRSTLANE_B32,
                  O::V_LSHLREV_B32, O::S_CMP_EQ_U32, O::S_CMP_LG_U32,
                  O::S_CMP_LT_U32, O::S_CBRANCH_SCC1, O::S_ADD_U32,
                  O::S_WAITCNT, O::BUFFER_LOAD_DWORD, O::V_ADD_NC_U32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"Coherent", "Volatile", "OpAtomicIAdd", "OpLoopMerge"};
  if (store_completion) {
    test.opcodes.push_back(O::S_WAITCNT_VSCNT);
    test.opcodes.push_back(O::S_LSHL_B64);
    test.required_spirv.push_back("OpControlBarrier %uint_3 %uint_1 %uint_2116");
  }
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  test.dispatch_x = groups;
  // DLC alone does not bypass RDNA2's L0, so only check its translated policy.
  test.compile_only = dlc_only;
  return test;
}

TestCase BufferAtomicVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  const u32 values[] = {100, 5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu};
  const u32 ops[] = {0x30, 0x32, 0x33, 0x35, 0x36,
                     0x37, 0x38, 0x39, 0x3a, 0x3b};
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendVMovU32(&code, 20, i * 4u);
    AppendVMovU32(&code, i, values[i]);
    AppendBufferStoreOpcode(&code, ops[i], i, 20, true);
  }
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendStoreVgpr(&code, i, i + 10u);
  }
  AppendEnd(&code);

  return {
      "BufferAtomicVariants",
      code,
      {10, 10, 10, 0xfffffff0u, 10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu,
       0,  0,  0,  0,           0,  0,           0,  0,       0,       0},
      {100,     15,          7,       0xfffffff0u, 5,       5,      20,
       0x00f0u, 0xff00u,     0xf0f0u, 10,          10,      10,     0xfffffff0u,
       10,      0xfffffff0u, 10,      0xf0f0u,     0xf000u, 0xf00fu},
      {O::V_MOV_B32, O::BUFFER_ATOMIC_SWAP, O::BUFFER_ATOMIC_ADD,
       O::BUFFER_ATOMIC_SUB, O::BUFFER_ATOMIC_SMIN, O::BUFFER_ATOMIC_UMIN,
       O::BUFFER_ATOMIC_SMAX, O::BUFFER_ATOMIC_UMAX, O::BUFFER_ATOMIC_AND,
       O::BUFFER_ATOMIC_OR, O::BUFFER_ATOMIC_XOR, O::BUFFER_STORE_DWORD,
       O::S_ENDPGM}};
}

TestCase BufferAtomicAndX2GlcAndExec() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "BufferAtomicAndX2GlcAndExec";
  auto &code = test.code;
  code.push_back(EncodeSop1(0x04, 12, 126));
  AppendVMovU32(&code, 20, 0);
  const u32 values[][2] = {{0x0ff00ff0u, 0xf00ff00fu},
                           {0xff00ff00u, 0x00ff00ffu},
                           {0, 0}};
  for (u32 i = 0; i < std::size(values); ++i) {
    AppendVMovLiteral(&code, 2, values[i][0]);
    AppendVMovLiteral(&code, 3, values[i][1]);
    if (i == 2) {
      code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
    }
    AppendBufferStoreOpcode(&code, 0x59, 2, 20, i != 1);
    code.push_back(EncodeSop1(0x04, 126, 12));
    AppendStoreVgpr(&code, 2, 2 + i * 2);
    AppendStoreVgpr(&code, 3, 3 + i * 2);
  }
  AppendEnd(&code);
  test.initial = {0x12345678u, 0x87654321u, 0, 0, 0, 0, 0, 0};
  test.expected = {0x02000600u, 0x00050001u, 0x12345678u, 0x87654321u,
                   0xff00ff00u, 0x00ff00ffu, 0, 0};
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_ATOMIC_AND_X2,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpAtomicAnd %ulong", "Int64Atomics"};
  return test;
}

TestCase BufferAtomicCmpSwapExactRaw() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  const auto Store = [&](u32 value, u32 index) {
    AppendVMovU32(&code, 31, index * sizeof(u32));
    code.push_back(EncodeMubuf0(0x1cu));
    code.push_back(EncodeMubuf1(value, 8, 31));
  };
  AppendVMovU32(&code, 4, 1);
  AppendVMovLiteral(&code, 1, 0x22222222u);
  AppendVMovLiteral(&code, 2, 0x11111111u);
  code.push_back(0xe0c46000u);
  code.push_back(0x80080104u);
  Store(1, 2);

  AppendVMovLiteral(&code, 1, 0x33333333u);
  AppendVMovLiteral(&code, 2, 0x44444444u);
  code.push_back(0xe0c46000u);
  code.push_back(0x80080104u);
  Store(1, 3);
  Store(2, 4);

  AppendVMovLiteral(&code, 1, 0x55555555u);
  AppendVMovLiteral(&code, 2, 0x22222222u);
  code.push_back(EncodeMubuf0(0x31u, 0, true, false));
  code.push_back(0x80080104u);
  Store(1, 5);
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferAtomicCmpSwapExactRaw";
  test.code = std::move(code);
  test.initial = {0xaaaaaaaau, 0x11111111u, 0, 0, 0, 0};
  test.expected = {0xaaaaaaaau, 0x55555555u, 0x11111111u,
                   0x22222222u, 0x44444444u, 0x55555555u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_ATOMIC_CMPSWAP,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpAtomicCompareExchange"};
  const auto descriptor = MakeStructuredStorageBufferData(
      sizeof(u32), static_cast<u32>(test.initial.size()));
  std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 32);
  test.has_user_data = true;
  return test;
}

TestCase BufferAtomicGlc0DoesNotReturnOldValue() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovU32(&code, 0, 5);
  AppendBufferStoreOpcode(&code, 0x32, 0, 20);
  AppendStoreVgpr(&code, 0, 1);
  AppendEnd(&code);

  return {
      "BufferAtomicGlc0DoesNotReturnOldValue",
      code,
      {10, 0},
      {15, 5},
      {O::V_MOV_B32, O::BUFFER_ATOMIC_ADD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase BufferAtomicFMinExactRawGlcModes() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x40000000u); // 2.0
  code.push_back(0xe0fc0000u);
  code.push_back(0x80010000u); // exact failing buffer_atomic_fmin, GLC=0
  AppendStoreVgpr(&code, 0, 1);
  AppendVMovLiteral(&code, 0, 0x3f800000u); // 1.0
  code.push_back(0xe0fc4000u);
  code.push_back(0x80010000u); // same instruction with GLC=1
  AppendStoreVgpr(&code, 0, 2);
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferAtomicFMinExactRawGlcModes";
  test.code = code;
  test.initial = {0x40800000u, 0, 0}; // 4.0
  test.expected = {0x3f800000u, 0x40000000u, 0x40000000u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  const auto descriptor = MakeStructuredStorageBufferData(
      0, static_cast<u32>(test.initial.size() * sizeof(u32)));
  std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4);
  test.user_data[50] = 1u << 20u;
  test.has_user_data = true;
  return test;
}

TestCase BufferAtomicFMinSpecialValues() {
  using O = ShaderOpcode;

  const u32 values[] = {
      0x40000000u, // 2.0
      0x7f800000u, // +infinity
      0xff800000u, // -infinity
      0x3f800000u, // 1.0
      0x7fc00000u, // quiet NaN
      0x00000000u, // +0.0
      0x80000000u, // -0.0
      0x00000000u, // +0.0
      0x80000001u, // smallest negative denorm
  };
  std::vector<u32> code;
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendVMovU32(&code, 20, i * 4u);
    AppendVMovLiteral(&code, i, values[i]);
    AppendBufferStoreOpcode(&code, 0x3f, i, 20, true);
  }
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendStoreVgpr(&code, i, i + static_cast<u32>(std::size(values)));
  }
  AppendEnd(&code);

  return {"BufferAtomicFMinSpecialValues",
          code,
          {0x40800000u, 0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u,
           0x80000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0, 0, 0, 0, 0, 0,
           0, 0, 0},
          {0x40000000u, 0xbf800000u, 0xff800000u, 0x7fc00000u, 0x3f800000u,
           0x80000000u, 0x00000000u, 0x00000000u, 0x80000001u, 0x40800000u,
           0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u, 0x80000000u,
           0x00000000u, 0x00000001u, 0x00000000u},
          {O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase BufferAtomicFMinContendedWorkgroup() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // v_cvt_f32_u32 v1, thread_id.x
  AppendVMovU32(&code, 20, 0);
  AppendBufferStoreOpcode(&code, 0x3f, 1, 20);
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferAtomicFMinContendedWorkgroup";
  test.code = code;
  test.initial = {0x42c80000u};  // 100.0
  test.expected = {0x00000000u}; // min(100.0, 0.0 .. 63.0)
  test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN,
                  O::S_ENDPGM};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase BufferAtomicFMaxExactRawGlcModes() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 3, 0x40000000u); // 2.0
  code.push_back(0xe100000cu);
  code.push_back(
      0x80010300u); // exact failing buffer_atomic_fmax v3, offset 12, GLC=0
  AppendStoreVgpr(&code, 3, 4);
  AppendVMovLiteral(&code, 0, 0x40800000u); // 4.0
  code.push_back(0xe100400cu);
  code.push_back(0x80010000u); // same address with GLC=1
  AppendStoreVgpr(&code, 0, 5);
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferAtomicFMaxExactRawGlcModes";
  test.code = code;
  test.initial = {0, 0, 0, 0x3f800000u, 0, 0}; // memory[3] = 1.0
  test.expected = {0, 0, 0, 0x40800000u, 0x40000000u, 0x40000000u};
  test.opcodes = {O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  const auto descriptor = MakeStructuredStorageBufferData(
      0, static_cast<u32>(test.initial.size() * sizeof(u32)));
  std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4);
  test.user_data[50] = 1u << 20u;
  test.has_user_data = true;
  return test;
}

TestCase BufferAtomicFMaxSpecialValues() {
  using O = ShaderOpcode;

  const u32 values[] = {
      0x40000000u, // 2.0, finite update
      0x40000000u, // 2.0, finite no-update
      0x7f800000u, // incoming +infinity
      0xff800000u, // incoming -infinity
      0x3f800000u, // finite against old +infinity
      0x3f800000u, // finite against old -infinity
      0x3f800000u, // finite against old quiet NaN
      0x7fcabcdeu, // incoming quiet NaN
      0x3f800000u, // finite against old signaling NaN
      0x7faabcdeu, // incoming signaling NaN
      0x00000000u, // +0.0 against old -0.0
      0x80000000u, // -0.0 against old +0.0
      0x80000000u, // -0.0 against a negative denorm
      0x80000001u, // negative denorm against -0.0
      0x00000001u, // positive denorm against +0.0
      0x00000000u, // +0.0 against a positive denorm
  };
  std::vector<u32> code;
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendVMovU32(&code, 20, i * 4u);
    AppendVMovLiteral(&code, i, values[i]);
    AppendBufferStoreOpcode(&code, 0x40, i, 20, true);
  }
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendStoreVgpr(&code, i, i + static_cast<u32>(std::size(values)));
  }
  AppendEnd(&code);

  return {"BufferAtomicFMaxSpecialValues",
          code,
          {0x3f800000u, 0x40800000u, 0xbf800000u, 0x3f800000u,
           0x7f800000u, 0xff800000u, 0x7fc12345u, 0x3f800000u,
           0x7fa54321u, 0x3f800000u, 0x80000000u, 0x00000000u,
           0x80000001u, 0x80000000u, 0x00000000u, 0x00000001u,
           0,           0,           0,           0,
           0,           0,           0,           0,
           0,           0,           0,           0,
           0,           0,           0,           0},
          {0x40000000u, 0x40800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u,
           0x3f800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u, 0x3f800000u,
           0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000001u,
           0x00000001u, 0x3f800000u, 0x40800000u, 0xbf800000u, 0x3f800000u,
           0x7f800000u, 0xff800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u,
           0x3f800000u, 0x80000000u, 0x00000000u, 0x80000001u, 0x80000000u,
           0x00000000u, 0x00000001u},
          {O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX, O::BUFFER_STORE_DWORD,
           O::S_ENDPGM}};
}

TestCase BufferAtomicFMaxContendedWorkgroup() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // v_cvt_f32_u32 v1, thread_id.x
  AppendVMovU32(&code, 20, 0);
  AppendBufferStoreOpcode(&code, 0x40, 1, 20);
  AppendEnd(&code);

  TestCase test;
  test.name = "BufferAtomicFMaxContendedWorkgroup";
  test.code = code;
  test.initial = {0xc2c80000u};  // -100.0
  test.expected = {0x427c0000u}; // max(-100.0, 0.0 .. 63.0)
  test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX,
                  O::S_ENDPGM};
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

std::vector<u32> MakeRgbaImage(u32 width, u32 height, u32 value = 0) {
  return std::vector<u32>(static_cast<size_t>(width) * height * 4u, value);
}

void SetRgbaPixel(std::vector<u32> *image, u32 width, u32 x, u32 y, u32 r,
                  u32 g, u32 b, u32 a) {
  const auto base = static_cast<size_t>((y * width + x) * 4u);
  (*image)[base + 0] = r;
  (*image)[base + 1] = g;
  (*image)[base + 2] = b;
  (*image)[base + 3] = a;
}

TestCase ImageLoadVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  AppendVMovU32(&code, 22, 0);
  AppendVMovU32(&code, 23, 0);
  code.push_back(EncodeMimg0(0x00, 0xf));
  code.push_back(EncodeMimg1(0, 20));
  code.push_back(EncodeMimg0(0x01, 0xf));
  code.push_back(EncodeMimg1(4, 20));
  AppendVMovU32(&code, 24, 0);
  code.push_back(EncodeMimg0(0x0e, 0x1));
  code.push_back(EncodeMimg1(8, 24));
  for (u32 i = 0; i < 9u; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  auto image = MakeRgbaImage(4, 4);
  SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
               0x40800000u);

  TestCase test;
  test.name = "ImageLoadVariants";
  test.code = code;
  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u,
                   0x40800000u, 0x3f800000u, 0x40000000u,
                   0x40400000u, 0x40800000u, 4};
  test.opcodes = {O::V_MOV_B32,          O::IMAGE_LOAD,
                  O::IMAGE_LOAD_MIP,     O::IMAGE_GET_RESINFO,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.sampled_image_rgba = image;
  return test;
}

TestCase DsAppendConsumeUsesEncodedLdsSelector() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 124, 0xffff0000u);
  AppendVMovU32(&code, 1, 0);
  AppendVMovU32(&code, 2, 10);
  code.push_back(EncodeDs0(0x0d, 0));
  code.push_back(EncodeDs1(0, 2, 1));
  code.push_back(EncodeDs0(0x3e, 1));
  code.push_back(EncodeDs1(3, 0, 0));
  code.push_back(EncodeDs0(0x3d, 2));
  code.push_back(EncodeDs1(4, 0, 0));
  AppendSMovLiteral(&code, 124, 0);
  code.push_back(EncodeDs0(0x3e, 3));
  code.push_back(EncodeDs1(5, 0, 0));
  code.push_back(EncodeDs0(0x36, 0));
  code.push_back(EncodeDs1(6, 0, 1));
  AppendStoreVgpr(&code, 3, 0);
  AppendStoreVgpr(&code, 4, 1);
  AppendStoreVgpr(&code, 5, 2);
  AppendStoreVgpr(&code, 6, 3);
  AppendEnd(&code);

  return {"DsAppendConsumeLdsSelector",
          code,
          {},
          {10, 11, 10, 11},
          {O::S_MOV_B32, O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_B32,
           O::DS_APPEND, O::DS_CONSUME, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}

TestCase DsAppendAllocatesAcrossWaves(u32 wave_size) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = wave_size == 64 ? "DsAppendAllocatesAcrossWaves64"
                             : "DsAppendAllocatesAcrossWaves32";
  auto &code = test.code;
  AppendVMovU32(&code, 2, 0);
  AppendVMovU32(&code, 5, 1);
  // Only workgroup lane zero initializes the shared allocation counter.
  code.push_back(EncodeSop1(0x04, 10, 126));
  code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
  code.push_back(EncodeSop1(0x04, 126, 106));
  code.push_back(EncodeDs0(0x0d, 0));
  code.push_back(EncodeDs1(0, 2, 2));
  code.push_back(EncodeSop1(0x04, 126, 10));
  code.push_back(EncodeSopp(0x0a, 0));
  for (u32 batch = 0; batch < 2; ++batch) {
    // Captured CS ea0aceac sets M0=0 and uses OFFSET=1. LDS ignores M0
    // and aligns OFFSET before allocating once for every active wave.
    AppendSMovLiteral(&code, 124, batch == 0 ? 0 : 0xffff0000u);
    code.push_back(EncodeDs0(0x3e, batch == 0 ? 1 : 3));
    code.push_back(EncodeDs1(3, 0, 0));
    code.push_back(EncodeVop2(0x24, 6, 127, 3));
    code.push_back(EncodeVop2(0x23, 6, 126, 6));
    code.push_back(EncodeVop2(0x1a, 7, InlineU32(2), 6));
    // A histogram detects duplicate allocations without assuming wave order.
    AppendBufferStoreOpcode(&code, 0x32, 5, 7);
  }
  code.push_back(EncodeSopp(0x0a, 0));
  code.push_back(EncodeDs0(0x36, 0));
  code.push_back(EncodeDs1(8, 0, 2));
  AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, 512);
  AppendEnd(&code);
  test.expected.assign(512, 1);
  test.expected.resize(768, 512);
  test.initial.resize(test.expected.size());
  test.compute_info.threads_num[0] = 256;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.compute_info.lds_size_dwords = 1;
  test.has_compute_info = true;
  test.forbidden_spirv = {"\"gds\""};
  test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_BARRIER,
                  O::V_MOV_B32, O::V_CMP_EQ_U32, O::DS_WRITE_B32,
                  O::DS_APPEND, O::V_MBCNT_HI_U32_B32,
                  O::V_MBCNT_LO_U32_B32, O::V_LSHLREV_B32,
                  O::BUFFER_ATOMIC_ADD, O::DS_READ_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase DsAppendUsesEncodedGdsSelector() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendSMovLiteral(&code, 124, 0x00000008u);
  code.push_back(EncodeDs0(0x3e, 0, true));
  code.push_back(EncodeDs1(0, 0, 0));
  code.push_back(EncodeDs0(0x3d, 0, true));
  code.push_back(EncodeDs1(1, 0, 0));
  code.push_back(EncodeDs0(0x3e, 4, true));
  code.push_back(EncodeDs1(2, 0, 0));
  code.push_back(EncodeDs0(0x3d, 4, true));
  code.push_back(EncodeDs1(3, 0, 0));
  AppendSMovLiteral(&code, 124, 0x00080008u);
  code.push_back(EncodeDs0(0x3e, 4, true));
  code.push_back(EncodeDs1(4, 0, 0));
  code.push_back(EncodeDs0(0x3d, 4, true));
  code.push_back(EncodeDs1(5, 0, 0));
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendStoreVgpr(&code, 2, 2);
  AppendStoreVgpr(&code, 3, 3);
  AppendStoreVgpr(&code, 4, 4);
  AppendStoreVgpr(&code, 5, 5);
  AppendEnd(&code);

  TestCase test{"DsAppendGdsSelector",
                code,
                {},
                {10, 11, 20, 21, 40, 41},
                {O::S_MOV_B32, O::DS_APPEND, O::DS_CONSUME,
                 O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
  test.gds_initial = {10, 20, 30, 40};
  test.expected_gds = {10, 20, 30, 40};
  return test;
}

TestCase DsOrderedCountAddressAndExec(u32 wave_size, bool pixel_counter) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = pixel_counter
                  ? (wave_size == 64 ? "DsOrderedCountPixelAddressWave64"
                                     : "DsOrderedCountPixelAddressWave32")
                  : (wave_size == 64 ? "DsOrderedCountAddressAndExecWave64"
                                     : "DsOrderedCountAddressAndExecWave32");
  auto &code = test.code;
  // Both the base and OFFSET0 are aligned independently; M0's low bits are
  // a wave index, or a pixel packer ID, rather than a byte address or size.
  AppendSMovLiteral(&code, 124, 0x01030003u);
  code.push_back(EncodeSop1(0x04, 10, 126)); // Save the full EXEC mask.
  code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0)); // ADDR value = lane + 1.
  AppendVMovU32(&code, 9, 99); // DATA0 must not supply the count.
  const u32 selected_lane = wave_size == 64 ? 47u : 7u;
  const std::array<uint64_t, 3> masks = {
      wave_size == 64 ? UINT64_MAX : uint64_t{UINT32_MAX},
      uint64_t{1} << selected_lane, 0};
  u32 counter = 1000;
  for (bool exchange : {false, true}) {
    for (size_t mode = 0; mode < masks.size(); ++mode) {
      AppendVMovU32(&code, 3, 0xdeadbeefu);
      AppendSMovLiteral(&code, 126, static_cast<u32>(masks[mode]));
      AppendSMovLiteral(&code, 127, static_cast<u32>(masks[mode] >> 32u));
      const u32 release_done = exchange && mode == 2 ? 3u : 0u;
      const u32 offset1 = release_done | (pixel_counter ? 4u : 0u) |
                          (exchange ? 0x10u : 0u);
      code.push_back(EncodeDs0(0x3f, (offset1 << 8u) | 7u, true));
      code.push_back(EncodeDs1(3, 9, 2));
      code.push_back(EncodeSopp(0x0c, 0)); // Wait for the returned GDS value.
      code.push_back(EncodeSop1(0x04, 126, 10)); // Restore EXEC before reading all lanes.
      AppendStoreVgprAtLaneDwordOffset(&code, 3, 0,
                                      static_cast<u32>(test.expected.size()));
      test.expected.insert(test.expected.end(), wave_size, counter);
      if (mode != 2) {
        const u32 value = mode == 0 ? 1u : selected_lane + 1u;
        counter = exchange ? value : counter + value;
      }
    }
  }
  AppendEnd(&code);
  test.initial.assign(test.expected.size(), 0xdeadbeefu);
  const u32 counter_address = pixel_counter ? 0x110u : 0x104u;
  test.gds_initial.assign(0x120u / sizeof(u32), 100);
  test.gds_initial[counter_address / sizeof(u32)] = 1000;
  test.expected_gds = test.gds_initial;
  test.expected_gds[counter_address / sizeof(u32)] = counter;
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_ADD_NC_U32, O::V_MOV_B32,
                  O::DS_ORDERED_COUNT, O::S_WAITCNT, O::V_LSHLREV_B32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase DsAppendConsumeGdsRegionBounds() {
  using O = ShaderOpcode;
  struct Access {
    u32 m0, offset;
    bool valid;
  };
  constexpr Access accesses[] = {
      {0x00000004u, 3, true},
      {0x00000004u, 4, false},
      {0x00000003u, 0, false},
      {0xbffc0004u, 3, true},
      {0xbffc0008u, 0, false},
      {0xc0000004u, 0, false},
      {0x0000c000u, 0xbfff, true},
      {0x0000c000u, 0xc000, false},
  };
  TestCase test;
  test.name = "DsAppendConsumeGdsRegionBounds";
  auto &code = test.code;
  for (const auto &access : accesses) {
    AppendSMovLiteral(&code, 124, access.m0);
    code.push_back(EncodeDs0(0x3e, access.offset, true));
    code.push_back(EncodeDs1(0, 0, 0));
    code.push_back(EncodeDs0(0x3d, access.offset, true));
    code.push_back(EncodeDs1(1, 0, 0));
    AppendStoreVgpr(&code, 0, static_cast<u32>(test.expected.size()));
    test.expected.push_back(access.valid ? 10 : 0);
    AppendStoreVgpr(&code, 1, static_cast<u32>(test.expected.size()));
    test.expected.push_back(access.valid ? 11 : 0);
  }
  AppendEnd(&code);
  test.initial.resize(test.expected.size());
  // Backing beyond the guest partition must not make an invalid region valid.
  test.gds_initial.assign(0x10000u / sizeof(u32), 10);
  test.expected_gds = test.gds_initial;
  test.opcodes = {O::S_MOV_B32, O::DS_APPEND, O::DS_CONSUME,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase DsGdsSubdwordAndAtomicWrites() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x01, 1, Vgpr(0)));
  code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0));
  code.push_back(EncodeDs0(0x1e, 0, true));
  code.push_back(EncodeDs1(0, 2, 1));
  code.push_back(EncodeVop2(0x1a, 8, InlineU32(1), 0));
  code.push_back(EncodeVop2(0x25, 8, InlineU32(12), 8));
  code.push_back(EncodeDs0(0x1f, 0, true));
  code.push_back(EncodeDs1(0, 2, 8));
  AppendVMovU32(&code, 3, 4);
  AppendVMovU32(&code, 4, 1);
  code.push_back(EncodeDs0(0x00, 0, true));
  code.push_back(EncodeDs1(0, 4, 3));
  AppendVMovU32(&code, 5, 8);
  AppendVMovLiteral(&code, 6, 0x40a00000u);
  AppendVMovLiteral(&code, 7, 0x40a00000u);
  code.push_back(EncodeDs0(0x12, 0, true));
  code.push_back(EncodeDs1Ex(0, 7, 6, 5));
  AppendEnd(&code);

  TestCase test;
  test.name = "DsGdsSubdwordAndAtomics";
  test.code = code;
  test.opcodes = {O::V_MOV_B32,   O::V_ADD_NC_U32, O::V_LSHLREV_B32,
                  O::DS_WRITE_B8, O::DS_WRITE_B16, O::DS_ADD_U32,
                  O::DS_MIN_F32,  O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  test.gds_initial = {0, 0, 0x42c80000u, 0, 0};
  test.expected_gds = {0x04030201u, 4, 0x40a00000u, 0x00020001u, 0x00040003u};
  return test;
}

TestCase ImageLoadR32UintUsesIntegerSampledImage() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  code.push_back(EncodeMimg0(0x00, 0x1));
  code.push_back(EncodeMimg1(0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageLoadR32UintUsesIntegerSampledImage";
  test.code = code;
  test.expected = {0xdeadbeefu};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.sampled_image_rgba.resize(16);
  test.sampled_image_rgba[6] = 0xdeadbeefu;
  test.sampled_image_format = vk::Format::eR32Uint;
  test.sampled_image_dwords_per_pixel = 1;
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
  test.has_user_data = true;
  test.required_spirv = {"image_10", "OpTypeImage %uint",
                         "OpImageFetch %v4uint"};
  return test;
}

TestCase ImageLoadFmaskUsesNativeSampleMapping() {
  auto test = ImageLoadR32UintUsesIntegerSampledImage();
  test.name = "ImageLoadFmaskUsesNativeSampleMapping";
  test.expected = {0x76543210u};
  test.user_data = {0x303ac300u, 0xca100000u, 0x021bc3bfu, 0x91800004u,
                    0u, 0x00700000u, 0u, 0u};
  test.image_descriptor_swizzle = DstSel(4, 0, 0, 0);
  test.sampled_image_rgba.clear();
  test.required_spirv = {"OpConstant %uint 1985229328"};
  test.forbidden_spirv = {"OpTypeImage", "OpImageFetch"};
  return test;
}

TestCase ImageLoadR32SintUsesSignedSampledImage() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  code.push_back(EncodeMimg0(0x00, 0x1));
  code.push_back(EncodeMimg1(0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageLoadR32SintUsesSignedSampledImage";
  test.code = code;
  test.expected = {0xffffff80u};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.sampled_image_rgba.resize(16);
  test.sampled_image_rgba[6] = 0xffffff80u;
  test.sampled_image_format = vk::Format::eR32Sint;
  test.sampled_image_dwords_per_pixel = 1;
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32SInt);
  test.has_user_data = true;
  test.required_spirv = {"image_17", "OpTypeImage %int", "OpImageFetch %v4int",
                         "OpBitcast %v4uint"};
  return test;
}

TestCase ImageLoadPackedUintUnpacksAndSwizzles() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  code.push_back(EncodeMimg0(0x00, 0xf));
  code.push_back(EncodeMimg1(0, 20));
  for (u32 component = 0; component < 4; component++) {
    AppendStoreVgpr(&code, component, component);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageLoadPackedUintUnpacksAndSwizzles";
  test.code = std::move(code);
  test.expected = {0x2abu, 0x456u, 0x321u, 0x321u};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.sampled_image_rgba.resize(16);
  test.sampled_image_rgba[6] = 0xaae2b321u;
  test.sampled_image_format = vk::Format::eR32Uint;
  test.sampled_image_dwords_per_pixel = 1;
  test.user_data =
      MakeSampledTextureData(Prospero::BufferFormat::k11_11_10UInt);
  test.has_user_data = true;
  test.image_descriptor_swizzle = DstSel(6, 5, 4, 7);
  test.required_spirv = {"OpImageFetch %v4uint", "OpBitFieldUExtract"};
  return test;
}

TestCase ImageSamplePackedUintConvertsSampleAndGather() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 20, 0x3f200000u);
  AppendVMovLiteral(&code, 21, 0x3ec00000u);
  code.push_back(EncodeMimg0(0x20, 0xf));
  code.push_back(EncodeMimg1(0, 20));
  code.push_back(EncodeMimg0(0x47, 0x2));
  code.push_back(EncodeMimg1(4, 20));
  for (u32 component = 0; component < 8; component++) {
    AppendStoreVgpr(&code, component, component);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageSamplePackedUintConvertsSampleAndGather";
  test.code = std::move(code);
  test.expected = {0x2abu, 0x456u, 0x321u, 0x321u,
                   0x456u, 0x456u, 0x456u, 0x456u};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::IMAGE_GATHER4_LZ,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.sampled_image_rgba = std::vector<u32>(16, 0xaae2b321u);
  test.sampled_image_format = vk::Format::eR32Uint;
  test.sampled_image_dwords_per_pixel = 1;
  test.user_data =
      MakeSampledTextureData(Prospero::BufferFormat::k11_11_10UInt);
  test.has_user_data = true;
  test.image_descriptor_swizzle = DstSel(6, 5, 4, 7);
  test.expected_force_point_sampler = true;
  test.required_spirv = {"OpImageSampleExplicitLod", "OpImageGather",
                         "OpBitFieldUExtract"};
  return test;
}

template <bool rg, bool snorm = false> TestCase ImageSamplePacked8() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = snorm ? "ImageSampleR8SNorm"
                   : rg ? "ImageSampleRG8UScaled" : "ImageSampleR8UScaled";
  AppendVMovLiteral(&test.code, 20, std::bit_cast<u32>(0.375f));
  AppendVMovLiteral(&test.code, 21, std::bit_cast<u32>(0.375f));
  test.code.push_back(EncodeMimg0(0x20, 0xf));
  test.code.push_back(EncodeMimg1(0, 20));
  test.code.push_back(EncodeMimg0(0x47, 0x1));
  test.code.push_back(EncodeMimg1(4, 20));
  test.code.push_back(EncodeMimg0(0x47, 0x8));
  test.code.push_back(EncodeMimg1(8, 20));
  for (u32 component = 0; component < 12; ++component) {
    AppendStoreVgpr(&test.code, component, component);
  }
  AppendEnd(&test.code);
  test.image_width = test.image_height = 2;
  test.sampled_image_rgba =
      snorm ? std::vector<u32>{0x7fc08180u}
            : rg ? std::vector<u32>{0xffffff00u, 0x00ff0000u}
                 : std::vector<u32>{0xff00ff00u};
  test.sampled_image_format =
      snorm ? vk::Format::eR8Snorm
            : rg ? vk::Format::eR8G8Unorm : vk::Format::eR8Unorm;
  test.sampled_image_dwords_per_pixel = 1;
  test.sampler_filter = snorm ? vk::Filter::eNearest : vk::Filter::eLinear;
  test.expected_float_tolerance = snorm ? 0.00002f : 0.01f;
  test.user_data = MakeSampledTextureData(
      snorm ? Prospero::BufferFormat::k8SNorm
            : rg ? Prospero::BufferFormat::k8_8UScaled
                 : Prospero::BufferFormat::k8UScaled);
  test.user_data[1] |= 1u << 30u;
  test.user_data[2] = 1u << 14u;
  test.user_data[50] = 12u * sizeof(u32);
  test.user_data[51] = 3u << 28u;
  test.has_user_data = true;
  test.image_descriptor_swizzle = snorm ? DstSel(4, 5, 6, 7)
                                      : DstSel(rg ? 5 : 4, 4, 0, 1);
  constexpr float midpoint = -64.0f / 127.0f;
  const std::array<float, 12> expected = snorm
      // -128 and -127 both clamp to -1; the interior signed value normalizes by 127.
      ? std::array<float, 12>{-1, 0, 0, 1, midpoint, 1, -1, -1, 1, 1, 1, 1}
      // R varies horizontally; G varies vertically in the opposite direction.
      : std::array<float, 12>{rg ? 191.25f : 63.75f, 63.75f, 0.0f, 1.0f,
                             0.0f, rg ? 0.0f : 255.0f, 255.0f, rg ? 255.0f : 0.0f,
                             1.0f, 1.0f, 1.0f, 1.0f};
  for (float value : expected) {
    test.expected.push_back(std::bit_cast<u32>(value));
  }
  test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::IMAGE_GATHER4_LZ,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

TestCase ImageLoadR128IgnoresAdjacentMaskSgprs() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  code.push_back(EncodeSop1(0x04, 4, 126)); // s_mov_b64 s[4:5], exec
  code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 1, true));
  code.push_back(EncodeMimg1(0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageLoadR128IgnoresAdjacentMaskSgprs";
  test.code = code;
  test.expected = {0xdeadbeefu};
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::IMAGE_LOAD,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.sampled_image_rgba.resize(16);
  test.sampled_image_rgba[6] = 0xdeadbeefu;
  test.sampled_image_format = vk::Format::eR32Uint;
  test.sampled_image_dwords_per_pixel = 1;
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
  test.has_user_data = true;
  test.decoded_counts = {{"r128=1", 1}};
  return test;
}

TestCase ImageLoad1DUsesScalarCoordinate() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 0));
  code.push_back(EncodeMimg1(0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageLoad1DUsesScalarCoordinate";
  test.code = code;
  test.expected = {0xdeadbeefu};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.image_width = 4;
  test.image_height = 1;
  test.sampled_image_rgba = {0, 0, 0xdeadbeefu, 0};
  test.sampled_image_format = vk::Format::eR32Uint;
  test.sampled_image_dwords_per_pixel = 1;
  test.sampled_image_type = vk::ImageType::e1D;
  test.sampled_image_view_type = vk::ImageViewType::e1D;
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
  test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1D)
                      << 28u;
  test.has_user_data = true;
  test.required_spirv = {"image_8"};
  return test;
}

TestCase ImageGather2DInstructionWith1DDescriptor() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 20, 0x3f000000u);
  AppendVMovLiteral(&code, 21, 0x3f000000u);
  code.push_back(EncodeMimg0(0x47, 0x1));
  code.push_back(EncodeMimg1(0, 20));
  for (u32 i = 0; i < 4u; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  auto image = MakeRgbaImage(4, 1);
  SetRgbaPixel(&image, 4, 1, 0, 0x40000000u, 0, 0, 0);
  SetRgbaPixel(&image, 4, 2, 0, 0x40400000u, 0, 0, 0);

  TestCase test;
  test.name = "ImageGather2DInstructionWith1DDescriptor";
  test.code = code;
  test.expected = {0x40000000u, 0x40400000u, 0x40400000u, 0x40000000u};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_GATHER4_LZ, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.image_width = 4;
  test.image_height = 1;
  test.sampled_image_rgba = image;
  test.sampled_image_type = vk::ImageType::e1D;
  test.sampled_image_view_type = vk::ImageViewType::e1D;
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32Float);
  test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1D)
                      << 28u;
  test.has_user_data = true;
  test.required_spirv = {"image_1", "OpImageQuerySizeLod",
                         "OpImageSampleExplicitLod", "Floor"};
  test.forbidden_spirv = {"OpImageGather"};
  return test;
}

TestCase ImageLoad1DArrayUsesLayerCoordinate() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 4));
  code.push_back(EncodeMimg1(0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageLoad1DArrayUsesLayerCoordinate";
  test.code = code;
  test.expected = {0xcafebabeu};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.image_width = 4;
  test.image_height = 1;
  test.sampled_image_rgba = {0, 0, 0, 0, 0, 0, 0xcafebabeu, 0};
  test.sampled_image_format = vk::Format::eR32Uint;
  test.sampled_image_dwords_per_pixel = 1;
  test.sampled_image_type = vk::ImageType::e1D;
  test.sampled_image_view_type = vk::ImageViewType::e1DArray;
  test.sampled_image_layers = 2;
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
  test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1DArray)
                      << 28u;
  test.has_user_data = true;
  test.required_spirv = {"image_9"};
  return test;
}

TestCase ImageLoad1DArrayDescriptorUsesSelectedLayer() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 0));
  code.push_back(EncodeMimg1(0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageLoad1DArrayDescriptorUsesSelectedLayer";
  test.code = code;
  test.expected = {0xcafebabeu};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.image_width = 4;
  test.image_height = 1;
  test.sampled_image_rgba = {
      0, 0, 0xdeadbeefu, 0, 0, 0, 0xcafebabeu, 0,
  };
  test.sampled_image_format = vk::Format::eR32Uint;
  test.sampled_image_dwords_per_pixel = 1;
  test.sampled_image_type = vk::ImageType::e1D;
  test.sampled_image_view_type = vk::ImageViewType::e1D;
  test.sampled_image_layers = 2;
  test.sampled_image_view_base_layer = 1;
  test.sampled_image_view_layers = 1;
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
  test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1DArray)
                      << 28u;
  test.user_data[4] = 1u | (1u << 16u);
  test.has_user_data = true;
  test.required_spirv = {"image_8"};
  return test;
}

TestCase ImageLoadMipUsesVaddr2Lod2D() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 1);
  AppendVMovU32(&code, 21, 1);
  AppendVMovU32(&code, 22, 1);
  AppendVMovU32(&code, 23, 0);
  code.push_back(EncodeMimg0(0x01, 0x1));
  code.push_back(EncodeMimg1(0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  auto base = MakeRgbaImage(4, 4);
  SetRgbaPixel(&base, 4, 1, 1, 0x3f800000u, 0, 0, 0);
  auto mip1 = MakeRgbaImage(2, 2);
  SetRgbaPixel(&mip1, 2, 1, 1, 0x40000000u, 0, 0, 0);

  TestCase test;
  test.name = "ImageLoadMipUsesVaddr2Lod2D";
  test.code = code;
  test.expected = {0x40000000u};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD_MIP, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.sampled_image_rgba_mips = {base, mip1};
  return test;
}

TestCase ImageLoadMipNsaUsesSelectedAddressVgprs() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 1);
  AppendVMovU32(&code, 21, 0);
  AppendVMovU32(&code, 22, 0);
  AppendVMovU32(&code, 30, 1);
  AppendVMovU32(&code, 31, 1);
  code.push_back(EncodeMimg0(0x01, 0x1, 1));
  code.push_back(EncodeMimg1(0, 20));
  code.push_back((30u << 0u) | (31u << 8u));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  auto base = MakeRgbaImage(4, 4);
  SetRgbaPixel(&base, 4, 1, 0, 0x3f800000u, 0, 0, 0);
  auto mip1 = MakeRgbaImage(2, 2);
  SetRgbaPixel(&mip1, 2, 1, 1, 0x40a00000u, 0, 0, 0);

  TestCase test;
  test.name = "ImageLoadMipNsaUsesSelectedAddressVgprs";
  test.code = code;
  test.expected = {0x40a00000u};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD_MIP, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.sampled_image_rgba_mips = {base, mip1};
  return test;
}

TestCase ImageLoadA16UintCoordsOnGpu() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 20, 0x00010002u); // x=2, y=1 packed as u16.
  AppendVMovU32(&code, 21, 0);
  code.push_back(EncodeMimg0(0x00, 0xf));
  code.push_back(EncodeMimg1(0, 20, 0, 0, true));
  for (u32 i = 0; i < 4u; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  auto image = MakeRgbaImage(4, 4);
  SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
               0x40800000u);

  TestCase test;
  test.name = "ImageLoadA16UintCoordsOnGpu";
  test.code = code;
  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.sampled_image_rgba = image;
  test.required_spirv = {"OpShiftRightLogical", "OpBitwiseAnd"};
  test.forbidden_spirv = {"UnpackHalf2x16"};
  return test;
}

TestCase ImageGetResinfoDmaskWidthHeight() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 24, 0);
  code.push_back(EncodeMimg0(0x0e, 0x3));
  code.push_back(EncodeMimg1(0, 24));
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageGetResinfoDmaskWidthHeight";
  test.code = code;
  test.expected = {4, 2};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_GET_RESINFO, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.image_width = 4;
  test.image_height = 2;
  test.sampled_image_rgba = MakeRgbaImage(test.image_width, test.image_height);
  return test;
}

TestCase ImageGetResinfoDmaskMipLevels() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 24, 0);
  code.push_back(EncodeMimg0(0x0e, 0x8));
  code.push_back(EncodeMimg1(0, 24));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageGetResinfoDmaskMipLevels";
  test.code = code;
  test.expected = {2};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_GET_RESINFO, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  auto base = MakeRgbaImage(4, 4);
  auto mip1 = MakeRgbaImage(2, 2);
  test.sampled_image_rgba_mips = {base, mip1};
  return test;
}

TestCase ImageSampleAndGather() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 20, 0x3f200000u);
  AppendVMovLiteral(&code, 21, 0x3ec00000u);
  AppendVMovU32(&code, 22, 0);
  code.push_back(EncodeMimg0(0x20, 0xf));
  code.push_back(EncodeMimg1(0, 20));
  code.push_back(EncodeMimg0(0x47, 0x1));
  code.push_back(EncodeMimg1(4, 20));
  AppendVMovU32(&code, 24, 0);
  AppendVMovLiteral(&code, 25, 0x3f200000u);
  AppendVMovLiteral(&code, 26, 0x3ec00000u);
  AppendVMovU32(&code, 27, 0);
  code.push_back(EncodeMimg0(0x57, 0x1));
  code.push_back(EncodeMimg1(8, 24));
  code.push_back(EncodeMimg0(0x60, 0x1));
  code.push_back(EncodeMimg1(12, 20));
  for (u32 i = 0; i < 13u; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  auto image = MakeRgbaImage(4, 4);
  for (u32 y = 0; y < 4u; y++) {
    for (u32 x = 0; x < 4u; x++) {
      SetRgbaPixel(&image, 4, x, y, 0x3f800000u, 0, 0, 0);
    }
  }
  SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
               0x40800000u);

  TestCase test;
  test.name = "ImageSampleAndGather";
  test.code = code;
  test.expected = {0x3f800000u,
                   0x40000000u,
                   0x40400000u,
                   0x40800000u,
                   0x3f800000u,
                   0x3f800000u,
                   0x3f800000u,
                   0x3f800000u,
                   0x3f800000u,
                   0x3f800000u,
                   0x3f800000u,
                   0x3f800000u,
                   0};
  test.opcodes = {
      O::V_MOV_B32,        O::IMAGE_SAMPLE,       O::IMAGE_GET_LOD,
      O::IMAGE_GATHER4_LZ, O::IMAGE_GATHER4_LZ_O, O::BUFFER_STORE_DWORD,
      O::S_ENDPGM};
  test.sampled_image_rgba = image;
  return test;
}

TestCase ImageCubeGradientsPreserveDerivatives() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = "ImageCubeGradientsPreserveDerivatives";
  for (u32 component = 0; component < 7u; component++) {
    constexpr std::array values{0.125f, 0.0f, 0.0f, 0.125f,
                                1.375f, 1.625f, 10.0f};
    AppendVMovLiteral(&test.code, 20u + component,
                      std::bit_cast<u32>(values[component]));
  }
  test.code.push_back(EncodeMimg0(0x22, 0x1, 0, false, 3));
  test.code.push_back(EncodeMimg1(0, 20));
  AppendStoreVgpr(&test.code, 0, 0);
  AppendEnd(&test.code);
  test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
  test.user_data[1] |= 3u << 30u;
  test.user_data[2] = 3u << 14u;
  test.user_data[3] =
      (static_cast<u32>(Prospero::ImageType::kCube) << 28u) | (2u << 16u);
  test.user_data[4] = 11u;
  test.user_data[5] = 2u << 4u;
  test.user_data[50] = sizeof(u32);
  test.has_user_data = true;
  test.sampled_image_view_type = vk::ImageViewType::e2DArray;
  test.sampled_image_layers = 12;
  for (u32 mip = 0; mip < 3u; mip++) {
    const auto layer_dwords = (4u >> mip) * (4u >> mip) * 4u;
    auto &pixels = test.sampled_image_rgba_mips.emplace_back(layer_dwords * 12u);
    for (u32 layer = 0; layer < 12u; layer++) {
      std::fill_n(pixels.begin() + layer * layer_dwords, layer_dwords,
                  std::bit_cast<u32>(static_cast<float>(1u + layer + 100u * mip)));
    }
  }
  // Guest face 10 is host layer 8. Unmodified gradients select mip 0;
  // applying the cube coordinate bias to gradients instead selects mip 2.
  test.expected = {std::bit_cast<u32>(9.0f)};
  test.required_spirv = {"OpImageSampleExplicitLod", "Grad"};
  return test;
}

TestCase ImageGatherExplicitLod() {
  using O = ShaderOpcode;
  std::vector<u32> code;
  // PPSA29343 uses NSA coordinates v81/v84 with its explicit LOD in v78.
  AppendVMovLiteral(&code, 81, 0x3f000000u);
  AppendVMovLiteral(&code, 84, 0x3f000000u);
  AppendVMovLiteral(&code, 78, 0x3f800000u);
  code.push_back(EncodeMimg0(0x44, 0x1, 1));
  code.push_back(EncodeMimg1(9, 81, 0, 2));
  code.push_back(84u | (78u << 8u));
  code.push_back(EncodeMimg0(0x24, 0x1, 1));
  code.push_back(EncodeMimg1(13, 81, 0, 2));
  code.push_back(84u | (78u << 8u));
  for (u32 i = 0; i < 5u; i++) {
    AppendStoreVgpr(&code, i + 9, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageGatherExplicitLod";
  test.code = std::move(code);
  test.opcodes = {O::V_MOV_B32, O::IMAGE_GATHER4_L, O::IMAGE_SAMPLE,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  // A gather's single-mip descriptors must not replace the ordinary sample's
  // full mip chain when both instructions use the same guest descriptor.
  test.expected = std::vector<u32>(5, 0x41000000u);
  test.sampled_image_rgba_mips = {std::vector<u32>(4 * 4 * 4, 0x3f800000u),
                                 std::vector<u32>(2 * 2 * 4, 0x41000000u)};
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
  test.user_data[1] |= 3u << 30u;
  test.user_data[2] = 3u << 14u;
  test.user_data[3] |= 1u << 16u;
  test.user_data[8] = 0x92u;
  test.user_data[9] = 0xfff000u;
  test.user_data[10] = 0x05000000u; // Captured point sampler, preclamp disabled.
  test.user_data[50] = test.expected.size() * sizeof(u32);
  test.user_data[51] = 3u << 28u;
  test.has_user_data = true;
  test.expected_mip_descriptors = 2;
  test.required_spirv = {"OpImageGather", "OpImageSampleExplicitLod"};
  return test;
}

template <u32 mode> TestCase ImageGatherLodPerLane() {
  using O = ShaderOpcode;
  constexpr std::array names{
      "ImageGatherLodPerLane", "ImageGatherLodBaseMip",
      "ImageGatherLodSamplerBounds", "ImageGatherLodTextureMinimum",
      "ImageGatherLodMipNone", "ImageGatherLodPointPostclampMaximum",
      "ImageGatherLodPointPreclampMaximum", "ImageGatherLodPointPostclampMinimum",
      "ImageGatherLodPointPreclampMinimum"};
  constexpr std::array lods = mode < 5
      ? std::array{-1.0f, 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 9.0f}
      : std::array{-1.0f, 0.0f, 0.25f, 0.75f, 1.25f, 1.75f, 2.75f, 9.0f};
  constexpr std::array<std::array<u32, 8>, 9> selected_mips{{
      {0, 0, 1, 2, 3, 4, 4, 4},
      {1, 1, 2, 3, 4, 4, 4, 4},
      {2, 2, 2, 3, 3, 3, 3, 3},
      {3, 3, 3, 3, 3, 3, 3, 3},
      {1, 1, 1, 1, 1, 1, 1, 1},
      {0, 0, 0, 1, 1, 2, 2, 2},
      {0, 0, 0, 1, 1, 1, 1, 1},
      {2, 2, 2, 2, 2, 2, 3, 4},
      {1, 1, 1, 1, 1, 2, 3, 4},
  }};
  constexpr u32 lanes = lods.size();
  constexpr u32 sampler_min = mode == 2 ? 256 : mode >= 7 ? 448 : 0;
  constexpr u32 sampler_max = mode == 2 ? 512 : mode == 3 ? 256
      : mode == 5 || mode == 6 ? 448 : 1024;
  TestCase test;
  test.name = names[mode];
  test.image_width = test.image_height = 16;
  test.sampled_image_view_base_mip = mode > 0 && mode < 5 ? 1 : 0;
  for (u32 mip = 0; mip < 5; mip++) {
    test.sampled_image_rgba_mips.emplace_back(
        (16u >> mip) * (16u >> mip) * 4u,
        std::bit_cast<u32>(static_cast<float>(mip + 1)));
  }
  for (const auto lod : lods) {
    test.initial.push_back(std::bit_cast<u32>(lod));
  }
  test.expected = test.initial;
  for (u32 component = 0; component < 4; component++) {
    for (const auto mip : selected_mips[mode]) {
      test.expected.push_back(std::bit_cast<u32>(static_cast<float>(mip + 1)));
    }
  }
  auto &code = test.code;
  code.push_back(EncodeVop1(0x01, 16, Vgpr(0)));
  code.push_back(EncodeVop2(0x1a, 31, InlineU32(2), 16));
  code.push_back(EncodeMubuf0(0x0c));
  code.push_back(EncodeMubuf1(22, 4, 31));
  AppendVMovLiteral(&code, 20, 0x3f000000u);
  AppendVMovLiteral(&code, 21, 0x3f000000u);
  code.push_back(EncodeMimg0(0x44, 0x1));
  code.push_back(EncodeMimg1(0, 20, 0, 2));
  for (u32 component = 0; component < 4; component++) {
    AppendStoreVgprAtLaneDwordOffset(&code, component, 16, lanes * (component + 1));
  }
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD,
                  O::IMAGE_GATHER4_L, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
  test.user_data[1] |= (3u << 30u) | (mode == 3 ? 768u << 8u : 0u);
  test.user_data[2] = 3u | (15u << 14u);
  test.user_data[3] |= (test.sampled_image_view_base_mip << 12u) | (4u << 16u);
  test.user_data[8] = 0x92u;
  test.user_data[9] = sampler_min | (sampler_max << 12u);
  // Point preclamp selects whether the half-level rounding adjustment precedes
  // or follows the clamps. The streaming T# minimum follows the sampler bounds.
  test.user_data[10] = mode == 4 ? 0x4001u : (1u << 26u);
  if constexpr (mode == 6 || mode == 8) {
    test.user_data[10] |= 1u << 28u;
  }
  test.user_data[18] = lanes * sizeof(u32);
  test.user_data[19] = 3u << 28u;
  test.user_data[50] = test.expected.size() * sizeof(u32);
  test.user_data[51] = 3u << 28u;
  test.has_user_data = true;
  test.compute_info.threads_num[0] = lanes;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  test.expected_mip_descriptors = 5 - test.sampled_image_view_base_mip;
  test.required_spirv = {"OpImageGather"};
  return test;
}

TestCase ImageD16GatherPacksHalfPairs() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 20, 0x3f000000u);
  AppendVMovLiteral(&code, 21, 0x3f000000u);
  AppendVMovU32(&code, 22, 0);
  code.push_back(EncodeMimg0(0x47, 0x1));
  code.push_back(EncodeMimg1(0, 20, 0, 0, false, true));
  AppendStoreVgpr(&code, 0, 0);
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  auto image = MakeRgbaImage(4, 4);
  for (u32 y = 0; y < 4u; y++) {
    for (u32 x = 0; x < 4u; x++) {
      SetRgbaPixel(&image, 4, x, y, 0x3f000000u, 0, 0, 0);
    }
  }

  TestCase test;
  test.name = "ImageD16GatherPacksHalfPairs";
  test.code = code;
  test.expected = {0x38003800u, 0x38003800u};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_GATHER4_LZ, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.sampled_image_rgba = image;
  test.decoded_counts = {{"d16=1", 1}};
  test.required_spirv = {"OpImageGather", "PackHalf2x16"};
  return test;
}

TestCase ImageSampleA16SamplerCoordsOnGpu() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 20, 0x36003900u); // x=0.625, y=0.375 packed as f16.
  AppendVMovU32(&code, 21, 0);
  code.push_back(EncodeMimg0(0x20, 0xf));
  code.push_back(EncodeMimg1(0, 20, 0, 0, true));
  for (u32 i = 0; i < 4u; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  auto image = MakeRgbaImage(4, 4);
  SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
               0x40800000u);

  TestCase test;
  test.name = "ImageSampleA16SamplerCoordsOnGpu";
  test.code = code;
  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.sampled_image_rgba = image;
  test.required_spirv = {"UnpackHalf2x16"};
  return test;
}

TestCase ImageSampleOpcodeAliasUsesNormalCoords() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 2, 0x3f000000u);
  AppendVMovLiteral(&code, 3, 0x3f000000u);
  code.push_back(0xf0800109u); // observed image_sample_a v6, v2, s0, s24
  code.push_back(0x00c00602u);
  AppendStoreVgpr(&code, 6, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageSampleOpcodeAliasUsesNormalCoords";
  test.code = code;
  test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.required_spirv = {"OpImageSampleExplicitLod"};
  test.forbidden_spirv = {"UnpackHalf2x16"};
  test.compile_only = true;
  return test;
}

template <u32 wave_size> TestCase ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 1); // +1 X remains a full dword with A16 coordinates.
  AppendVMovLiteral(&code, 21, 0x36003900u); // x=0.625, y=0.375 packed as f16.
  AppendVMovU32(&code, 22, 0);
  code.push_back(EncodeMimg0(0x30, 0xf));
  code.push_back(EncodeMimg1(0, 20, 0, 0, true));
  for (u32 i = 0; i < 4u; i++) {
    AppendStoreVgpr(&code, i, i);
  }
  AppendEnd(&code);

  auto image = MakeRgbaImage(4, 4);
  SetRgbaPixel(&image, 4, 2, 1, 0x3f000000u, 0, 0, 0);
  SetRgbaPixel(&image, 4, 3, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
               0x40800000u);

  TestCase test;
  test.name = wave_size == 32 ? "ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu32"
                              : "ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu64";
  test.code = code;
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.sampled_image_rgba = image;
  test.required_spirv = {"UnpackHalf2x16", "Lod|ConstOffset"};
  test.forbidden_spirv = {"OpBitFieldSExtract", "OpImageQuerySizeLod", "OpFDiv"};
  return test;
}

template <u32 wave_size> TestCase ImageSampleOffsetsReadFourNeighbors() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = wave_size == 32 ? "ImageSampleOffsetsReadFourNeighbors32"
                              : "ImageSampleOffsetsReadFourNeighbors64";
  test.compute_info.wave_size = wave_size;
  test.has_compute_info = true;
  constexpr std::array offsets{0x01u, 0x3fu, 0x3f00u, 0x0100u};
  constexpr std::array opcodes{0x37u, 0x37u, 0x34u, 0x32u};
  for (u32 sample = 0; sample < offsets.size(); ++sample) {
    AppendVMovLiteral(&test.code, 20, offsets[sample]);
    u32 coordinate = 21;
    if (opcodes[sample] == 0x32u) {
      for (float gradient : {0.25f, 0.f, 0.f, 0.25f}) {
        AppendVMovLiteral(&test.code, coordinate++, std::bit_cast<u32>(gradient));
      }
    }
    // All instructions address the center of texel (2, 1) before applying the offset.
    AppendVMovLiteral(&test.code, coordinate, std::bit_cast<u32>(0.625f));
    AppendVMovLiteral(&test.code, coordinate + 1, std::bit_cast<u32>(0.375f));
    if (opcodes[sample] == 0x34u) AppendVMovU32(&test.code, coordinate + 2, 0);
    test.code.push_back(EncodeMimg0(opcodes[sample], 0x1));
    test.code.push_back(EncodeMimg1(sample, 20));
    AppendStoreVgpr(&test.code, sample, sample);
  }
  AppendEnd(&test.code);
  test.sampled_image_rgba = MakeRgbaImage(4, 4);
  SetRgbaPixel(&test.sampled_image_rgba, 4, 2, 1, std::bit_cast<u32>(16.f), 0, 0, 0);
  constexpr std::array<std::array<u32, 2>, 4> neighbors{{{3, 1}, {1, 1}, {2, 0}, {2, 2}}};
  for (u32 sample = 0; sample < neighbors.size(); ++sample) {
    const auto value = std::bit_cast<u32>(float(sample + 1));
    SetRgbaPixel(&test.sampled_image_rgba, 4, neighbors[sample][0], neighbors[sample][1],
                 value, 0, 0, 0);
    test.expected.push_back(value);
  }
  test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"Lod|ConstOffset", "Grad|ConstOffset"};
  test.forbidden_spirv = {"OpImageQuerySizeLod", "OpFDiv", "OpBitFieldSExtract", "OpSwitch"};
  return test;
}

void CheckDynamicSampleOffsets(VulkanHarness &vulkan, u32 wave_size) {
  using O = ShaderOpcode;
  TestCase test;
  test.name = wave_size == 32u ? "DynamicSampleOffsets32" : "DynamicSampleOffsets64";
  test.has_compute_info = test.has_user_data = true;
  test.compute_info.wave_size = wave_size;
  test.compute_info.host_subgroup_size = vulkan.SubgroupSize();
  test.compute_info.threads_num[0] = wave_size;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
  test.compute_info.thread_ids_num = 1;
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
  test.user_data[2] = 7u | (7u << 14u);
  test.user_data[3] |= 1u << 16u;
  test.user_data[5] = 1u << 4u;
  test.user_data[50] = wave_size * sizeof(u32);
  // Read the offset selector from a texture, as the native shader does.
  test.code = {EncodeVop2(0x1b, 20, InlineU32(7), 0),
               EncodeVop2(0x16, 21, InlineU32(3), 0),
               EncodeVop2(0x1b, 21, InlineU32(1), 21),
               EncodeMimg0(0x00, 0x1), EncodeMimg1(4, 20),
               EncodeVop2(0x1b, 20, InlineU32(3), 4),
               EncodeVop2(0x16, 22, InlineU32(2), 4),
               EncodeVop2(0x1b, 22, InlineU32(3), 22)};
  for (u32 reg : {20u, 22u}) {
    test.code.push_back(EncodeVop2(0x25, reg, InlineU32(63), reg));
    test.code.push_back(EncodeVop2(0x1b, reg, InlineU32(63), reg));
  }
  test.code.push_back(EncodeVop2(0x1a, 22, InlineU32(8), 22));
  test.code.push_back(EncodeVop2(0x1c, 20, Vgpr(22), 20));
  AppendVMovLiteral(&test.code, 21, std::bit_cast<u32>(0.375f));
  AppendVMovLiteral(&test.code, 22, std::bit_cast<u32>(0.375f));
  AppendVMovLiteral(&test.code, 23, std::bit_cast<u32>(1.0f));
  test.code.push_back(EncodeMimg0(0x34, 0x1)); // SAMPLE_L_O: offset is applied at mip 1.
  test.code.push_back(EncodeMimg1(5, 20));
  AppendStoreVgprAtLaneDwordOffset(&test.code, 5, 0, 0);
  AppendEnd(&test.code);
  test.opcodes = {O::V_AND_B32, O::V_LSHRREV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
                  O::V_OR_B32, O::V_MOV_B32, O::IMAGE_LOAD, O::IMAGE_SAMPLE,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"ImageGatherExtended", "Lod|Offset", "OpBitFieldSExtract %int"};
  test.forbidden_spirv = {"OpSwitch", "OpImageQuerySizeLod", "OpFDiv"};
  test.initial.assign(wave_size, 0xdeadbeefu);
  for (u32 lane = 0; lane < wave_size; ++lane)
    test.expected.push_back(std::bit_cast<u32>(float(100u + (lane & 15u))));
  auto compiled = CompileCase(test, vulkan.SubgroupSize());
  spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
  std::string text;
  const auto halves = wave_size > vulkan.SubgroupSize() ? 2u : 1u;
  Require(test.name, "direct dynamic offset", tools.Disassemble(compiled.spirv, &text) &&
              CountText(text, "OpImageSampleExplicitLod") == halves &&
              CountText(text, "OpBitFieldSExtract %int") == 2u * halves,
          "dynamic offsets expanded samples or lost signed six-bit components");
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
  if (wave_size == 32u) {
    Require(test.name, "default feature gate", !ShaderRecompiler::CompileOptions{}.maintenance8_enabled,
            "maintenance8 must reflect the enabled device feature");
    ExpectFatal("DynamicSampleOffsetsRequireMaintenance8", [&] {
      ShaderRecompiler::Spirv::EmitProgram(compiled.program, {.compute = &test.compute_info});
    });
  }
#endif
  auto base = MakeRgbaImage(8, 8);
  auto mip = MakeRgbaImage(4, 4);
  for (u32 selector = 0; selector < 16u; ++selector) {
    SetRgbaPixel(&base, 8, selector & 7u, selector >> 3u, 0x3f800000u | selector, 0, 0, 0);
    SetRgbaPixel(&mip, 4, selector & 3u, selector >> 2u,
                 std::bit_cast<u32>(float(100u + selector)), 0, 0, 0);
  }
  auto texture = vulkan.CreateImageMips(test.name, 8, 8, vk::Format::eR32G32B32A32Sfloat,
      vk::ImageUsageFlagBits::eSampled, {base, mip}, 4u, vk::ImageLayout::eShaderReadOnlyOptimal,
      vk::ImageType::e2D, vk::ImageViewType::e2D, 1u);
  const auto sampler = vulkan.CreateSampler(test.name, 2u);
  auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
  vulkan.Dispatch(test, compiled, output, nullptr, &texture, nullptr, nullptr, sampler);
  CompareWords(test, "image-derived offsets -1..2 at mip 1", test.expected,
                vulkan.ReadBuffer(test.name, output, test.expected.size()));
  vulkan.DestroyBuffer(&output);
  vulkan.Device().destroySampler(sampler);
  vulkan.DestroyImage(&texture);
  std::printf("[compute] %-32s ok\n", test.name);
}

TestCase ImageSampleA16CompareBiasRdna2AddressOrder() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 20, 0x00003800u); // bias=0.5 as low f16.
  AppendVMovLiteral(&code, 21, 0x3f000000u); // PCF reference stays 32-bit.
  AppendVMovLiteral(&code, 22, 0x36003900u); // x=0.625, y=0.375 packed as f16.
  code.push_back(EncodeMimg0(0x2d, 0x1));
  code.push_back(EncodeMimg1(0, 20, 0, 0, true));
  AppendStoreVgpr(&code, 0, 0);
  // Ordinary and comparison reads of one sharp require distinct SPIR-V image types.
  code.push_back(EncodeMimg0(0x27, 0x1));
  code.push_back(EncodeMimg1(1, 22, 0, 0, true));
  AppendStoreVgpr(&code, 1, 1);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageSampleA16CompareBiasRdna2AddressOrder";
  test.code = code;
  test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.required_spirv = {"OpTypeImage %float 2D 0", "OpTypeImage %float 2D 1",
                         "OpImageSampleDrefExplicitLod", "OpImageSampleExplicitLod",
                         "UnpackHalf2x16"};
  test.compile_only = true;
  return test;
}

TestCase ImageGatherCompareOpcodes() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovLiteral(&code, 21, 0x3f000000u);
  AppendVMovLiteral(&code, 22, 0x3f200000u);
  AppendVMovLiteral(&code, 23, 0x3ec00000u);
  AppendVMovLiteral(&code, 24, 0x3f000000u);
  AppendVMovLiteral(&code, 25, 0x3f200000u);
  AppendVMovLiteral(&code, 26, 0x3ec00000u);
  AppendVMovLiteral(&code, 28, 0x3f000000u);
  AppendVMovLiteral(&code, 29, 0x3f200000u);
  AppendVMovLiteral(&code, 30, 0x3ec00000u);
  AppendVMovU32(&code, 32, 0);
  AppendVMovLiteral(&code, 33, 0x3f000000u);
  AppendVMovLiteral(&code, 34, 0x3f200000u);
  AppendVMovLiteral(&code, 35, 0x3ec00000u);
  code.push_back(EncodeMimg0(0x48, 0x1));
  code.push_back(EncodeMimg1(4, 24));
  code.push_back(EncodeMimg0(0x4f, 0x1));
  code.push_back(EncodeMimg1(8, 28));
  code.push_back(EncodeMimg0(0x58, 0x1));
  code.push_back(EncodeMimg1(0, 20));
  code.push_back(EncodeMimg0(0x5f, 0x1));
  code.push_back(EncodeMimg1(12, 32));
  AppendStoreVgpr(&code, 4, 0);
  AppendStoreVgpr(&code, 8, 1);
  AppendStoreVgpr(&code, 0, 2);
  AppendStoreVgpr(&code, 12, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageGatherCompareOpcodes";
  test.code = code;
  test.opcodes = {
      O::V_MOV_B32,         O::IMAGE_GATHER4_C,      O::IMAGE_GATHER4_C_LZ,
      O::IMAGE_GATHER4_C_O, O::IMAGE_GATHER4_C_LZ_O, O::BUFFER_STORE_DWORD,
      O::S_ENDPGM};
  test.required_spirv = {"OpImageDrefGather", "ConstOffset"};
  test.forbidden_spirv = {"OpBitFieldSExtract"};
  test.compile_only = true;
  return test;
}

TestCase ImageStoreVariants() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 1);
  AppendVMovU32(&code, 21, 2);
  AppendVMovU32(&code, 22, 0);
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovLiteral(&code, 1, 0x40000000u);
  AppendVMovLiteral(&code, 2, 0x40400000u);
  AppendVMovLiteral(&code, 3, 0x40800000u);
  code.push_back(EncodeMimg0(0x08, 0xf));
  code.push_back(EncodeMimg1(0, 20));
  AppendVMovU32(&code, 24, 3);
  AppendVMovU32(&code, 25, 2);
  AppendVMovU32(&code, 26, 0);
  AppendVMovU32(&code, 27, 0);
  AppendVMovLiteral(&code, 4, 0x40a00000u);
  AppendVMovLiteral(&code, 5, 0x40c00000u);
  AppendVMovLiteral(&code, 6, 0x40e00000u);
  AppendVMovLiteral(&code, 7, 0x41000000u);
  code.push_back(EncodeMimg0(0x09, 0xf));
  code.push_back(EncodeMimg1(4, 24));
  AppendVMovLiteral(&code, 8, 0x12345678u);
  AppendStoreVgpr(&code, 8, 0);
  AppendEnd(&code);

  auto expected_image = MakeRgbaImage(4, 4);
  SetRgbaPixel(&expected_image, 4, 1, 2, 0x3f800000u, 0x40000000u, 0x40400000u,
               0x40800000u);
  SetRgbaPixel(&expected_image, 4, 3, 2, 0x40a00000u, 0x40c00000u, 0x40e00000u,
               0x41000000u);

  TestCase test;
  test.name = "ImageStoreVariants";
  test.code = code;
  test.expected = {0x12345678u};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_STORE_MIP,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.storage_image_rgba = MakeRgbaImage(4, 4);
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.expected_storage_image_rgba = expected_image;
  return test;
}

TestCase ImageD16StoreUnpacksHalfPairs() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 1);
  AppendVMovU32(&code, 21, 2);
  AppendVMovU32(&code, 22, 0);
  AppendVMovLiteral(&code, 0, 0x38003400u);
  AppendVMovLiteral(&code, 1, 0x3c003a00u);
  code.push_back(EncodeMimg0(0x08, 0xf));
  code.push_back(EncodeMimg1(0, 20, 0, 0, false, true));
  AppendEnd(&code);

  auto expected_image = MakeRgbaImage(4, 4);
  SetRgbaPixel(&expected_image, 4, 1, 2, 0x3e800000u, 0x3f000000u, 0x3f400000u,
               0x3f800000u);

  TestCase test;
  test.name = "ImageD16StoreUnpacksHalfPairs";
  test.code = code;
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
  test.storage_image_rgba = MakeRgbaImage(4, 4);
  test.expected_storage_image_rgba = expected_image;
  test.decoded_counts = {{"d16=1", 1}};
  test.required_spirv = {"UnpackHalf2x16", "OpImageWrite"};
  return test;
}

void CheckComputeThreadDimensions(VulkanHarness &vulkan) {
  using O = ShaderOpcode;
  constexpr u32 grid_size = 16 * 8 * 4;
  const std::array extents = {std::array{9u, 5u, 3u}, std::array{13u, 7u, 4u},
                              std::array{3u, 2u, 1u}, std::array{16u, 8u, 4u}};
  for (const u32 wave_size : {32u, 64u}) {
    for (const bool thread_dimensions : {false, true}) {
      TestCase test;
      test.name = thread_dimensions ? (wave_size == 32 ? "ThreadDimensions32" : "ThreadDimensions64")
                                    : (wave_size == 32 ? "GroupDimensions32" : "GroupDimensions64");
      auto &code = test.code;
      code.push_back(EncodeSop1(0x04, 20, 126)); // Save the SPI-provided initial EXEC.
      AppendSMovLiteral(&code, 24, grid_size * 4);
      for (u32 axis = 0; axis < 3; ++axis) {
        code.push_back(EncodeVop1(0x01, 3 + axis, 4 + axis));
        code.push_back(EncodeVop2(0x1a, 3 + axis, InlineU32(3 - axis), 3 + axis));
        code.push_back(EncodeVop2(0x25, 3 + axis, Vgpr(axis), 3 + axis));
      }
      code.push_back(EncodeVop2(0x1a, 4, InlineU32(4), 4));
      code.push_back(EncodeVop2(0x1a, 5, InlineU32(7), 5));
      code.push_back(EncodeVop2(0x25, 3, Vgpr(4), 3));
      code.push_back(EncodeVop2(0x25, 3, Vgpr(5), 3));
      code.push_back(EncodeVop2(0x25, 20, InlineU32(1), 3));
      code.push_back(EncodeVop2(0x1a, 20, InlineU32(2), 20));
      AppendVMovU32(&code, 6, 1);
      AppendVMovU32(&code, 7, 0);
      AppendBufferStoreDword(&code, 6, 20);
      AppendBufferStoreOpcode(&code, 0x32, 6, 7);
      code.push_back(EncodeSop2(0x0f, 126, 20, InlineU32(1)));
      AppendVMovU32(&code, 6, 2); // Divergence changes only lane zero of each wave.
      code.push_back(EncodeSop1(0x04, 126, 20));
      code.push_back(EncodeVop2(0x25, 20, 24, 20));
      AppendBufferStoreDword(&code, 6, 20);
      AppendVMovU32(&code, 8, 1);
      AppendBufferStoreOpcode(&code, 0x32, 8, 7);
      AppendEnd(&code);
      test.initial.resize(1 + 2 * grid_size);
      test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::S_AND_B64, O::V_MOV_B32,
                      O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
                      O::BUFFER_ATOMIC_ADD, O::S_ENDPGM};
      test.compute_info.wave_size = wave_size;
      test.compute_info.host_subgroup_size = vulkan.SubgroupSize();
      test.compute_info.threads_num[0] = 8;
      test.compute_info.threads_num[1] = 4;
      test.compute_info.threads_num[2] = 2;
      std::fill_n(test.compute_info.group_id, 3, true);
      test.compute_info.thread_ids_num = 3;
      test.compute_info.workgroup_register = 4;
      test.compute_info.dispatch_thread_dimensions = thread_dimensions;
      test.has_compute_info = true;
      test.expand_shader_data_storage = thread_dimensions && wave_size == 32;
      const auto compiled = CompileCase(test, vulkan.SubgroupSize());
      std::vector<u32> compiled_key;
      BuildStageStaticKey(test.compute_info, compiled_key);
      for (const auto &extent : extents) {
        std::copy(extent.begin(), extent.end(), test.compute_info.dispatch_threads_num);
        std::vector<u32> runtime_key;
        BuildStageStaticKey(test.compute_info, runtime_key);
        Require(test.name, "runtime shader reuse", compiled_key == runtime_key,
                "dispatch thread counts changed the compiled shader cache key");
        test.dispatch_x = thread_dimensions ? (extent[0] + 7) / 8 : 2;
        test.dispatch_y = thread_dimensions ? (extent[1] + 3) / 4 : 2;
        test.dispatch_z = thread_dimensions ? (extent[2] + 1) / 2 : 2;
        test.expected.assign(test.initial.size(), 0);
        for (u32 z = 0; z < 4; ++z) {
          for (u32 y = 0; y < 8; ++y) {
            for (u32 x = 0; x < 16; ++x) {
              if (thread_dimensions && (x >= extent[0] || y >= extent[1] || z >= extent[2])) continue;
              const auto index = x + 16 * y + 128 * z;
              const auto local_index = x % 8 + 8 * (y % 4) + 32 * (z % 2);
              test.expected[0] += 2;
              test.expected[1 + index] = 1;
              test.expected[1 + grid_size + index] = local_index % wave_size == 0 ? 2 : 1;
            }
          }
        }
        auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
        vulkan.Dispatch(test, compiled, output);
        const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
        vulkan.DestroyBuffer(&output);
        CompareWords(test, "initial EXEC, restored EXEC and padded-lane atomics", test.expected, actual);
        if (!thread_dimensions) break;
      }
      std::printf("[compute] %-32s ok\n", test.name);
    }
  }
}

void CheckRuntimeBufferRecords(VulkanHarness &vulkan) {
  using namespace ShaderRecompiler::IR;
  TestCase test;
  test.name = "RuntimeBufferRecords";
  // Captured b71ef7c579835a0d: read an input count, divide it into chunks,
  // construct a structured descriptor, then clear each live record's last DWORD.
  test.code = {
    0xbfa00003u, 0xd7460000u, 0x04010c0eu, 0x8805ff0du, 0x00040000u, 0xbe84030cu,
    0xbe860381u, 0xbe8703ffu, 0x00016204u, 0xb06a0100u, 0xe0300000u, 0x80010400u,
    0x92049281u, 0x7e0a026au, 0x996a9480u, 0xb0062000u, 0xb00a0080u, 0x380212f9u,
    0x8686066au, 0x7e040208u, 0x7e0602ffu, 0x00016204u, 0xbf8c3f70u, 0x7d8808ffu,
    0x00000200u, 0x7d8808f9u, 0x06868404u, 0x7d8808f9u, 0x06868606u, 0x020d80f9u,
    0x8686060au, 0xd5010005u, 0x00120affu, 0x00000200u, 0xd5010006u, 0x001a0d05u,
    0x7e0a0d06u, 0x7e0c0506u, 0xd76d0004u, 0x041a08c1u, 0x7e0a5705u, 0x81ea0680u,
    0x7e080504u, 0x100a0affu, 0x4f7ffffeu, 0x7e0a0f05u, 0x7ed60505u, 0x936a6b6au,
    0x9aea6b6au, 0x816a6a6bu, 0x9a87046au, 0x936a0607u, 0x4e08086au, 0x4e0a0806u,
    0x7d8608f9u, 0x06868406u, 0x7d860a06u, 0x87ea046au, 0x020902f9u, 0x86860680u,
    0xd5286a04u, 0x00120807u, 0xbeea047eu, 0x7da40080u, 0xbf880003u, 0x7e0a0280u,
    0xe0700000u, 0x80000500u, 0xbefe046au, 0x7e0a0280u, 0x7e000502u, 0x7e020501u,
    0x7e040504u, 0x7e060503u, 0xe0702010u, 0x80000500u, 0xbf810000u,
  };
  constexpr u32 count_address = 4;
  test.user_data[0] = 16;
  test.user_data[1] = 4u << 16;
  test.user_data[2] = 1;
  test.user_data[3] = 0x5204;
  test.user_data[8] = 20;
  test.user_data[12] = count_address;
  test.has_user_data = true;
  test.storage_buffer_offsets = {count_address, 16, 20};
  test.initial.assign(5 + 5 * 130, 0xdeadbeefu);
  test.initial[1] = 72;
  test.has_compute_info = true;
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.lds_size_dwords = 0;
  test.compute_info.wave_size = 64;
  test.compute_info.float_mode = 0xc0;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 14;
  test.compute_info.dispatch_thread_dimensions = true;
  test.compute_info.dispatch_threads_num[0] = 2;
  test.compute_info.dispatch_threads_num[1] = test.compute_info.dispatch_threads_num[2] = 1;
  test.forbidden_spirv = {"get_bda_pointer", "flattened_srt", "OpFDiv", "OpUMulExtended"};
  const auto compiled = CompileCase(test, vulkan.SubgroupSize());
  Require(test.name, "runtime record count",
          compiled.program.info.buffers.size() == 3 &&
              compiled.resources.buffers[2].dwords[2] == 2 &&
              compiled.resources.specialization_reads ==
                  std::vector<std::pair<uint64_t, uint64_t>>{{count_address, 4}},
          "descriptor capacity lost its exact clean input read");
  ShaderRecompiler::CompileOptions options;
  options.stage = ShaderType::Compute;
  options.wave_size = 64;
  options.user_data = test.user_data;
  options.input_info.compute = &test.compute_info;
  auto translated = ShaderRecompiler::TranslateProgram(test.code, options);
  auto plan = ExtractResourcePlan(translated.program);
  struct CountMemory { u32 value = 72; u32 reads = 0; bool clean = true; } memory;
  SrtRuntime runtime{.user_data = test.user_data, .userdata = &memory};
  ResourceSnapshot snapshot;
  ResourceSpecialization specialization;
  Require(test.name, "strict reader required",
          !MaterializeResources(plan, runtime, snapshot, specialization),
          "vector descriptor input bypassed the clean-memory reader");
  runtime.read_specialization_memory = [](void *data, uint64_t address, std::span<u32> words) {
    auto &memory = *static_cast<CountMemory *>(data);
    ++memory.reads;
    if (!memory.clean || address != count_address || words.size() != 1) return false;
    words[0] = memory.value;
    return true;
  };
  memory.clean = false;
  Require(test.name, "dirty input rejected",
          !MaterializeResources(plan, runtime, snapshot, specialization),
          "GPU-dirty count was accepted for host descriptor evaluation");
  memory.clean = true;
  ResourceSpecialization initial_specialization;
  // Reuse the same resource plan and compiled shader as the count and coverage change.
  for (const u32 count : {0u, 72u, 512u, 8192u, 262144u}) {
    memory.value = count;
    memory.reads = 0;
    const u32 chunk = count < 512 ? 64 : count < 8192 ? 128 : count < 262144 ? 256 : 512;
    const u32 records = (count + chunk - 1) / chunk;
    Require(test.name, "cached descriptor refresh",
            MaterializeResources(plan, runtime, snapshot, specialization) && memory.reads == 1 &&
                snapshot.buffers[2].dwords[2] == records &&
                snapshot.specialization_reads ==
                    std::vector<std::pair<uint64_t, uint64_t>>{{count_address, 4}},
            "count refresh repeated a memory read or retained an old descriptor capacity");
    if (count == 0) initial_specialization = specialization;
    Require(test.name, "capacity is not specialization",
            specialization == initial_specialization,
            "changing record capacity would recompile the cached shader");
    test.initial.assign(5 + 5 * records, 0xdeadbeefu);
    for (const u32 threads : {2u, 3u, 65u}) {
      test.initial[1] = count;
      test.compute_info.dispatch_threads_num[0] = threads;
      test.dispatch_x = (threads + 63) / 64;
      test.expected = test.initial;
      test.expected[4] = 0;
      for (u32 index = 0; index < std::min(threads, records); ++index) {
        test.expected[9 + index * 5] = 0;
      }
      auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
      vulkan.Dispatch(test, compiled, output);
      const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
      vulkan.DestroyBuffer(&output);
      CompareWords(test, "record capacity and padded invocation bounds", test.expected, actual);
    }
  }
  std::printf("[compute] %-32s ok\n", test.name);
}

void CheckBoundedFormattedBufferCopy(VulkanHarness &vulkan) {
  using namespace ShaderRecompiler::IR;
  constexpr std::array<u32, 8> lengths{16, 1, 4, 32, 48, 4, 32, 48};
  constexpr std::array<u32, 8> source_records{16, 1, 2, 32, 40, 0, 32, 48};
  constexpr std::array<u32, 8> destination_records{16, 1, 4, 24, 48, 4, 32, 44};
  const auto alignment = static_cast<u32>(vulkan.RuntimeContext().GetPhysicalDeviceProperties()
                                             .limits.minStorageBufferOffsetAlignment);
  const auto slot_bytes = (512u + alignment - 1u) / alignment * alignment;
  for (const u32 wave : {32u, 64u}) {
    for (const u32 count : {0u, 1u, 8u}) {
      TestCase test;
      test.name = wave == 32 ? "BoundedFormattedBufferCopy32" : "BoundedFormattedBufferCopy64";
      // Captured b3ade16b0485b3bd: uniform loop over two V# arrays; copy only lanes
      // below each row's count. Scalar+vector guards and register reuse are intact.
      test.code = {0xbfa00001u, 0xd7460000u, 0x04010c02u, 0xbeeb0380u,
          0xf4000080u, 0xfa000120u, 0xbf8cc07fu, 0xbf0a026bu, 0xbf840015u,
          0x97eaff6bu, 0x00000100u, 0xf40000c0u, 0xd4000000u, 0xbf8cc07fu,
          0x7da80003u, 0xbf88000bu, 0x8f6a846bu, 0xf4080100u, 0xd4000000u,
          0xbf8cc07fu, 0xe0002000u, 0x80010100u, 0xf4080200u, 0xd4000080u,
          0xbf8c0070u, 0xe0102000u, 0x80020100u, 0x816b816bu, 0xbefe04c1u,
          0xbf82ffe8u, 0xbf810000u};
      constexpr u32 table_base = 256u;
      test.initial.assign(slot_bytes * 19u / 4u, 0xdeadbeefu);
      test.user_data[0] = table_base;
      test.has_user_data = true;
      test.has_compute_info = true;
      test.compute_info.wave_size = wave;
      test.compute_info.threads_num[0] = wave;
      test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
      test.compute_info.lds_size_dwords = 0;
      test.compute_info.group_id[0] = true;
      test.compute_info.workgroup_register = 2;
      test.compute_info.thread_ids_num = 1;
      test.bda_mappings = {{0u, 0u}};
      for (u32 key = 0; key < 8u; ++key) {
        const auto src = slot_bytes * (key + 2u);
        const auto dst = slot_bytes * (key + 10u);
        for (const auto [row, address, records] :
             {std::tuple{key * 4u, src, source_records[key]},
              std::tuple{32u + key * 4u, dst, destination_records[key]}}) {
          const auto word = table_base / 4u + row;
          test.initial[word] = address;
          test.initial[word + 1u] = 4u << 16u;
          test.initial[word + 2u] = records;
          test.initial[word + 3u] =
              static_cast<u32>(Prospero::BufferFormat::k32UInt) << 12u | 4u;
        }
        test.initial[table_base / 4u + 64u + key] = lengths[key];
        for (u32 lane = 0; lane < 64u; ++lane)
          test.initial[src / 4u + lane] = 0x80000001u + key * 0x10000u + lane;
      }
      test.initial[table_base / 4u + 72u] = count;
      test.expected = test.initial;
      for (u32 key = 0; key < count; ++key)
        for (u32 lane = 0; lane < std::min({wave, lengths[key], destination_records[key]}); ++lane)
          test.expected[slot_bytes * (key + 10u) / 4u + lane] =
              lane < source_records[key] ? 0x80000001u + key * 0x10000u + lane : 0u;
      auto compiled = CompileCase(test, vulkan.SubgroupSize());
      Require(test.name, "finite writable descriptors",
              compiled.resources.buffers.size() == std::max(1u, count),
              "the bounded loop lost or overprobed a destination");
      test.storage_buffer_ranges.resize(compiled.resources.buffers.size());
      for (u32 resource = 0; resource < compiled.resources.buffers.size(); ++resource) {
        const auto& descriptor = compiled.resources.buffers[resource].dwords;
        test.storage_buffer_ranges[resource] =
            {descriptor[0], std::max(4u, descriptor[2] * 4u)};
      }
      if (count == 8u) {
        std::set<u32> nonuniform;
        std::vector<u32> stores;
        u32 bda_calls = 0;
        for (size_t offset = 5; offset < compiled.spirv.size();) {
          const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u);
          const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
          if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform)
            nonuniform.insert(words[1]);
          if (opcode == spv::OpStore) stores.push_back(words[1]);
          if (opcode == spv::OpFunctionCall) ++bda_calls;
          offset += words.size();
        }
        Require(test.name, "one shared indirect store per virtual half",
                std::ranges::count_if(stores, [&](u32 pointer) { return nonuniform.contains(pointer); }) ==
                    (wave > vulkan.SubgroupSize() ? 2 : 1),
                "formatted identity lowering duplicated the store per descriptor");
        Require(test.name, "shared formatted payload read", bda_calls <= 12u,
                "formatted source load duplicated BDA reads per format or component");
        std::printf("[compute] BoundedFormattedBufferCopy%u: %zu SPIR-V words, %u BDA calls\n",
                    wave, compiled.spirv.size(), bda_calls);
      }
      auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size(), true);
      vulkan.Dispatch(test, compiled, output);
      const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
      vulkan.DestroyBuffer(&output);
      CompareWords(test, "bounded rows, formatted raw bits, source OOB zero and destination canaries",
                   test.expected, actual);
    }
  }
  std::printf("[compute] %-32s ok\n", "BoundedFormattedBufferCopy");
}

void CheckIndirectBufferStore(VulkanHarness &vulkan) {
  using namespace ShaderRecompiler::IR;
  TestCase test;
  test.name = "IndirectBufferStore";
  test.initial.assign(64, 0xdeadbeefu);
  test.storage_buffer_range_bytes = 64;
  test.storage_buffer_offsets = {12, 108};

  CompiledShader compiled;
  auto &program = compiled.program;
  program.stage = ShaderType::Compute;
  program.wave_size = 32;
  program.srt_plan_complete = true;
  program.resource_tracking_complete = true;
  program.shader_info_complete = true;
  program.block_storage.push_back(std::make_unique<Block>());
  auto *block = program.block_storage.back().get();
  program.blocks.push_back(block);

  auto &lane = block->AppendNewInst(ValueOpcode::LaneId);
  auto &selector = block->AppendNewInst(ValueOpcode::BitwiseAnd32,
                                      {Value(&lane), Value(7u)});
  auto &key = block->AppendNewInst(ValueOpcode::IAdd32,
                                 {Value(&selector), Value(8u)});
  auto &handle = block->AppendNewInst(
      ValueOpcode::GetBufferResource,
      {Value(&key), Value(0u), Value(0u), Value(0u)});
  handle.SetFlags<uint32_t>(0u);
  auto &lane_offset = block->AppendNewInst(ValueOpcode::IMul32,
                                         {Value(&lane), Value(4u)});
  auto &overflow = block->AppendNewInst(ValueOpcode::IEqual32,
                                       {Value(&lane), Value(6u)});
  auto &offset = block->AppendNewInst(ValueOpcode::SelectU32,
                                    {Value(&overflow), Value(UINT32_MAX), Value(&lane_offset)});
  auto &index = block->AppendNewInst(ValueOpcode::SelectU32,
                                   {Value(&overflow), Value(0x40000000u), Value(&lane)});
  auto &data = block->AppendNewInst(ValueOpcode::IAdd32,
                                  {Value(&lane), Value(0x1000u)});
  auto &exec = block->AppendNewInst(ValueOpcode::INotEqual32,
                                  {Value(&lane), Value(4u)});
  auto &store = block->AppendNewInst(ValueOpcode::StoreBufferU32,
                      {Value(&handle), Value(0u), Value(&offset), Value(0u),
                       Value(&data), Value(&exec)});
  program.memory_info.push_back({.kind = ResourceKind::Buffer, .offen = true});

  BufferResource root;
  root.packed_stride = 1;
  root.written = true;
  root.indirect_root = 0;
  root.indirect_mapping_offset = 4;
  root.indirect_search_iterations = std::bit_width(3u);
  root.indirect_resources = {0u, 1u};
  auto candidate = root;
  candidate.indirect_search_iterations = 0;
  candidate.indirect_resources.clear();
  program.descriptor_sources.resize(1);
  program.descriptor_sources[0].indirect_descriptor =
      DescriptorSource::IndirectDescriptor{.table_stride = 16};
  ShaderComputeInputInfo compute;
  compute.wave_size = 32;
  compute.host_subgroup_size = vulkan.SubgroupSize();
  compute.threads_num[0] = 32;
  compute.threads_num[1] = compute.threads_num[2] = 1;
  enum class Layout { Shared, NullChild, AllNull, DifferentStride, ShiftedPushData, StorageData,
                      Indexed, IndexedNullChild, IndexedSingle };
  for (const auto layout : {Layout::Shared, Layout::NullChild, Layout::AllNull,
                           Layout::DifferentStride, Layout::ShiftedPushData, Layout::StorageData,
                           Layout::Indexed, Layout::IndexedNullChild, Layout::IndexedSingle}) {
    const bool indexed = layout == Layout::Indexed || layout == Layout::IndexedNullChild ||
                         layout == Layout::IndexedSingle;
    const bool null_child = layout == Layout::NullChild || layout == Layout::IndexedNullChild;
    const bool single = layout == Layout::AllNull || layout == Layout::IndexedSingle;
    program.info.buffers = {root, candidate};
    if (indexed) {
      for (auto &buffer : program.info.buffers) buffer.packed_stride = 4;
    }
    program.descriptor_sources[0].indirect_descriptor->indexed_store = indexed;
    program.memory_info[0].idxen = indexed;
    program.memory_info[0].offen = !indexed;
    store.SetArg(1, indexed ? Value(&index) : Value(0u));
    store.SetArg(2, indexed ? Value(0u) : Value(&offset));
    if (null_child) program.info.buffers[1].packed_stride = 0;
    if (layout == Layout::DifferentStride) program.info.buffers[1].packed_stride = 2;
    if (single) {
      // Materialization deduplicates all null destinations into one unmapped root.
      program.info.buffers.resize(1);
      if (layout == Layout::AllNull) program.info.buffers[0].packed_stride = 0;
      program.info.buffers[0].indirect_root = BufferResource::NoIndirectBuffer;
      program.info.buffers[0].indirect_search_iterations = 0;
      program.info.buffers[0].indirect_resources.clear();
    }
    program.shader_info_complete = false;
    CollectShaderInfo(program, {.compute = &compute});
    const auto push_start = layout == Layout::ShiftedPushData ? 3u :
                            layout == Layout::StorageData ? PushData::NoStart : 0u;
    compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}, push_start);
    Require(test.name, "candidate bindings",
            program.bindings.descriptor_counts[0] == (single ? 1u : 2u) &&
                program.info.buffers[0].descriptor_index == 0u &&
                (single || program.info.buffers[1].descriptor_index == 1u),
            "indirect store lost a buffer binding");
    compiled.packed_user_data.resize(program.bindings.ShaderDataDwords());
    compiled.packed_user_data[program.bindings.memory_offset_dword] = 12u | (108u << 8u);
    ValidateSpirv(test.name, compiled.spirv);
    test.required_spirv.clear();
    test.forbidden_spirv = {"PhysicalStorageBuffer"};
    if (!single && !null_child && layout != Layout::DifferentStride) {
      test.required_spirv = {"StorageBufferArrayNonUniformIndexing"};
      test.forbidden_spirv.push_back("OpSwitch");
      std::set<u32> nonuniform;
      std::vector<u32> accessed_pointers;
      u32 prefix_extracts = 0;
      for (size_t offset = 5; offset < compiled.spirv.size();) {
        const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u);
        const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
        if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform)
          nonuniform.insert(words[1]);
        if (opcode == spv::OpArrayLength) accessed_pointers.push_back(words[3]);
        if (opcode == spv::OpStore) accessed_pointers.push_back(words[1]);
        if (opcode == spv::OpBitFieldUExtract) ++prefix_extracts;
        offset += words.size();
      }
      Require(test.name, "shared store and nonuniform pointers",
              prefix_extracts == 1u && accessed_pointers.size() == 2u &&
                  std::ranges::all_of(accessed_pointers, [&](u32 pointer) {
                    return nonuniform.contains(pointer);
                  }),
              "indirect store retained static prefixes, was duplicated or lost a NonUniform pointer decoration");
    } else if (single) {
      test.forbidden_spirv.push_back("OpSwitch");
    } else {
      test.required_spirv = {"OpSwitch"};
    }
    CheckSpirvText(test, compiled.spirv);

    for (const bool swapped : {false, true}) {
      if (layout != Layout::Shared && layout != Layout::IndexedNullChild && swapped) continue;
      // Repeated keys select two distinct logical buffers within one native allocation.
      // Keys below, between and above the live keys must not select either buffer.
      compiled.resources.flattened_srt = {0u, 0u, 0u, 0u, 3u,
                                         10u, swapped ? 0u : 1u,
                                         12u, swapped ? 1u : 0u,
                                         14u, swapped ? 0u : 1u};
      test.expected = test.initial;
      for (u32 lane = 0; lane < 16u; ++lane) {
        if (lane == 4u || lane == 6u) continue;
        if (layout == Layout::IndexedSingle) {
          test.expected[test.storage_buffer_offsets[0] / 4u + lane] = 0x1000u + lane;
          continue;
        }
        if (lane != 2u && lane != 10u && lane != 12u && lane != 14u) continue;
        const u32 resource = (lane & 7u) == 4u ? swapped : !swapped;
        if (layout == Layout::AllNull || (null_child && resource == 1u)) continue;
        test.expected[test.storage_buffer_offsets[resource] / 4u + lane] = 0x1000u + lane;
      }
      auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
      vulkan.Dispatch(test, compiled, output);
      const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
      vulkan.DestroyBuffer(&output);
      CompareWords(test, "key selection, EXEC, bounds and null descriptors", test.expected, actual);
    }
  }
  std::printf("[compute] %-32s ok\n", test.name);
}

void CheckFiniteInlineSamplerPhi(VulkanHarness &vulkan) {
  TestCase test;
  test.name = "FiniteInlineSamplerPhi";
  auto &code = test.code;
  code.insert(code.end(), {EncodeSop1(0x04, 36, InlineU32(0)),
      EncodeSop1(0x04, 38, InlineU32(0)), EncodeSop1(0x04, 24, 126),
      EncodeVop1(0x01, 10, 16), EncodeVopc(0xd5, InlineU32(0), 10)});
  const auto skip_sampler = code.size();
  code.push_back(EncodeSopp(0x08));
  // Actual correlated inline sampler words from 905b2e2c: zero or {0,fff000,a500000,0}.
  code.insert(code.end(), {EncodeSop2(0x24, 37, InlineU32(12), InlineU32(12)),
      EncodeSop1(0x04, 38, 255), 0x0a500000u});
  code[skip_sampler] = EncodeSopp(0x08, code.size() - skip_sampler - 1u);
  code.push_back(EncodeSop1(0x04, 126, 24));
  AppendVMovLiteral(&code, 20, std::bit_cast<u32>(0.5f));
  AppendVMovLiteral(&code, 21, std::bit_cast<u32>(0.5f));
  AppendVMovLiteral(&code, 1, 0x12345678u);
  AppendVMovLiteral(&code, 2, 0x87654321u);
  code.insert(code.end(), {EncodeVopc(0xd4, InlineU32(2), 10),
      EncodeMimg0(0x27, 1), EncodeMimg1(1, 20, 0, 9),
      EncodeMimg0(0x27, 1), EncodeMimg1(2, 20, 2, 9),
      EncodeSop1(0x04, 126, 24), EncodeVop1(0x01, 30, 16),
      EncodeVop2(0x1a, 31, InlineU32(3), 30),
      EncodeMubuf0(0x1c), EncodeMubuf1(1, 12, 31),
      EncodeMubuf0(0x1c, 4), EncodeMubuf1(2, 12, 31)});
  AppendEnd(&code);
  test.initial.assign(7, 0xa5a5a5a5u);
  test.expected = {std::bit_cast<u32>(4.0f), 0xfffffff9u,
      std::bit_cast<u32>(2.0f), 0xfffffff9u, 0x12345678u, 0x87654321u, 0xa5a5a5a5u};
  test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
  test.user_data[3] |= static_cast<u32>(Prospero::ImageType::kColor2D) << 28u;
  const auto signed_image = MakeSampledTextureData(Prospero::BufferFormat::k32SInt);
  std::copy_n(signed_image.begin(), 8, test.user_data.begin() + 8);
  test.user_data[11] |= static_cast<u32>(Prospero::ImageType::kColor2D) << 28u;
  test.user_data[50] = test.initial.size() * sizeof(u32);
  test.user_data[51] = 3u << 28u;
  test.has_user_data = true;
  test.compute_info.threads_num[0] = test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
  test.compute_info.wave_size = 32;
  test.compute_info.group_id[0] = true;
  test.compute_info.workgroup_register = 16;
  test.has_compute_info = true;
  test.dispatch_x = 3;
  test.opcodes = {ShaderOpcode::S_MOV_B64, ShaderOpcode::S_BFM_B32,
      ShaderOpcode::S_CBRANCH_EXECZ, ShaderOpcode::V_MOV_B32, ShaderOpcode::V_CMPX_NE_U32,
      ShaderOpcode::V_CMPX_GT_U32, ShaderOpcode::V_LSHLREV_B32,
      ShaderOpcode::IMAGE_SAMPLE, ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
  test.required_spirv = {"OpPhi", "NonUniform", "OpImageSampleExplicitLod"};
  auto compiled = CompileCase(test, vulkan.SubgroupSize());
  std::set<u32> nonuniform_ids;
  std::vector<std::pair<u32, u32>> sampled_ids;
  for (size_t offset = 5; offset < compiled.spirv.size();) {
    const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u);
    if (static_cast<spv::Op>(words[0] & 0xffffu) == spv::OpDecorate &&
        words[2] == spv::DecorationNonUniform) nonuniform_ids.insert(words[1]);
    if (static_cast<spv::Op>(words[0] & 0xffffu) == spv::OpSampledImage)
      sampled_ids.emplace_back(words[2], words[4]);
    offset += words.size();
  }
  Require(test.name, "dynamic sampler decorations", sampled_ids.size() >= 2 &&
              std::ranges::all_of(sampled_ids, [&](const auto &ids) {
                return nonuniform_ids.contains(ids.first) && nonuniform_ids.contains(ids.second);
              }),
          "fetched sampler or sampled image lost its NonUniform decoration");
  const auto &samplers = compiled.program.info.samplers;
  u32 roots = 0;
  for (const auto &root : samplers) {
    if (root.indirect_resources.empty()) continue;
    ++roots;
    Require(test.name, "correlated sampler candidates", root.indirect_resources.size() == 2,
            "finite sampler was folded or lost a correlated branch state");
    for (const auto child : root.indirect_resources)
      Require(test.name, "sampler class remapping", child < samplers.size() &&
                  samplers[child].force_point_filtering == root.force_point_filtering &&
                  samplers[child].integer_border == root.integer_border,
              "a finite candidate retained another image class's sampler");
  }
  Require(test.name, "finite sampler class variants", roots == 2 && samplers.size() == 4 &&
              compiled.resources.samplers.size() == 2,
          "the two-state sampler did not retain both float and signed image classes");
  constexpr std::array<u32, 4> zero_sampler{};
  constexpr std::array<u32, 4> linear_sampler{0, 0x00fff000u, 0x0a500000u, 0};
  Require(test.name, "inline sampler states",
          std::ranges::all_of(compiled.resources.samplers, [&](const auto &value) {
            return std::equal(zero_sampler.begin(), zero_sampler.end(), value.dwords.begin()) ||
                std::equal(linear_sampler.begin(), linear_sampler.end(), value.dwords.begin());
          }) && compiled.resources.samplers[0] != compiled.resources.samplers[1],
          "descriptor words were mixed between the two native sampler states");
  auto pixels = MakeRgbaImage(4, 4);
  for (u32 y = 0; y < 4; ++y)
    for (u32 x = 2; x < 4; ++x) SetRgbaPixel(&pixels, 4, x, y, std::bit_cast<u32>(4.0f), 0, 0, 0);
  std::array<VulkanHarness::Image, 2> images{
      vulkan.CreateImageMips(test.name, 4, 4, vk::Format::eR32G32B32A32Sfloat,
          vk::ImageUsageFlagBits::eSampled, {pixels}, 4, vk::ImageLayout::eShaderReadOnlyOptimal,
          vk::ImageType::e2D, vk::ImageViewType::e2D, 1),
      vulkan.CreateImageMips(test.name, 4, 4, vk::Format::eR32Sint,
          vk::ImageUsageFlagBits::eSampled, {std::vector<u32>(16, 0xfffffff9u)}, 1,
          vk::ImageLayout::eShaderReadOnlyOptimal, vk::ImageType::e2D, vk::ImageViewType::e2D, 1)};
  std::vector<vk::Sampler> native_samplers;
  for (const auto &resource : samplers) {
    ShaderSamplerResource descriptor{};
    std::copy_n(compiled.resources.samplers[resource.snapshot_index].dwords.begin(), 4, descriptor.fields);
    const auto filter = !resource.force_point_filtering && descriptor.XyMagFilter() == 1
        ? vk::Filter::eLinear : vk::Filter::eNearest;
    native_samplers.push_back(vulkan.CreateSampler(test.name, 1, filter));
  }
  auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
  vulkan.Dispatch(test, compiled, output, nullptr, nullptr, nullptr, nullptr, nullptr, images, native_samplers);
  CompareWords(test, "point, linear, remapped signed sampler and inactive output", test.expected,
      vulkan.ReadBuffer(test.name, output, test.expected.size()));
  vulkan.DestroyBuffer(&output);
  for (const auto sampler : native_samplers) vulkan.Device().destroySampler(sampler);
  for (auto &image : images) vulkan.DestroyImage(&image);
  std::printf("[compute] %-32s ok\n", test.name);
}

void CheckCountedRowImageSelector(VulkanHarness &vulkan) {
  using O = ShaderOpcode;
  using F = Prospero::BufferFormat;
  constexpr u32 poison = 0x345d1746u;
  const auto alignment = static_cast<u32>(vulkan.RuntimeContext().GetPhysicalDeviceProperties()
                                             .limits.minStorageBufferOffsetAlignment);
  for (const bool masked : {false, true})
  for (const u32 wave : {32u, 64u}) {
    const auto output_words = masked ? 48u * wave : 64u;
    const auto slot = (output_words * 4u + alignment - 1u) / alignment * alignment;
    const u32 srt = slot, counts = slot * 2u, keys = slot * 3u, material = slot * 4u;
    const auto tile_list = slot * 5u;
    TestCase test;
    test.name = masked ? (wave == 32u ? "MaskedRowImageSelector32" : "MaskedRowImageSelector64")
                       : (wave == 32u ? "CountedRowImageSelector32" : "CountedRowImageSelector64");
    test.initial.resize((masked ? tile_list + 12u : material + 396u * 4u) / 4u, 0u);
    std::fill_n(test.initial.begin(), output_words, 0xdeadbeefu);
    const std::array<u32, 12> descriptors{counts, 4u << 16u, 3u, 0x14004u,
        keys, 4u << 16u, 48u, 0x14004u, material, 396u << 16u, 4u, 0x14004u};
    std::copy(descriptors.begin(), descriptors.end(), test.initial.begin() + srt / 4u);
    test.initial[counts / 4u] = 3u;
    test.initial[counts / 4u + 2u] = 1u;
    std::fill_n(test.initial.begin() + keys / 4u, 48u, poison);
    test.initial[keys / 4u] = 2u;
    test.initial[keys / 4u + 1u] = 0x80000001u;
    test.initial[keys / 4u + 2u] = 0x1890cedeu; // Dynamic +184 wraps to material byte32.
    test.initial[keys / 4u + 32u] = 3u;
    const auto put_image = [&](u32 offset, u32 identity, F format) {
      auto at = test.initial.begin() + (material + offset) / 4u;
      at[0] = identity;
      at[1] = static_cast<u32>(format) << 20u;
      at[3] = DstSel(4, 5, 6, 7) |
          (static_cast<u32>(Prospero::ImageType::kColor2D) << 28u);
    };
    for (u32 key = 0; key < 4u; ++key)
      put_image(key * 396u + 184u, 0x100u + key, F::k32_32_32_32Float);
    put_image(32u, 0x104u, F::k32_32_32_32Float);
    put_image(256u, 0x200u, F::k8SScaled);
    if (masked) {
      // The scalar tile list reverses workgroups; row selection must remain on the GPU.
      test.initial[tile_list / 4u] = 2u;
      test.initial[tile_list / 4u + 1u] = 1u;
      test.user_data[52] = tile_list;
      test.user_data[53] = 4u << 16u;
      test.user_data[54] = 3u;
      test.user_data[55] = 0x14004u;
    }
    test.expected = test.initial;
    for (const auto [index, bits] : std::array<std::pair<u32, u32>, 4>{{
             {0u, 0x3f800002u}, {1u, 0x3f800001u}, {2u, 0x3f800004u}, {32u, 0x3f800003u}}}) {
      if (masked) {
        for (u32 lane = 1u; lane < wave; lane += 2u)
          test.expected[index * wave + lane] = bits;
      } else test.expected[index] = bits;
    }
    test.user_data[0] = srt;
    test.user_data[50] = output_words * 4u;
    test.user_data[51] = 3u << 28u;
    test.has_user_data = true;
    test.has_compute_info = true;
    test.compute_info.wave_size = wave;
    test.compute_info.threads_num[0] = masked ? wave : 1u;
    test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
    test.compute_info.thread_ids_num = masked ? 1 : 0;
    test.compute_info.group_id[0] = true;
    test.compute_info.workgroup_register = 44u;
    test.dispatch_x = 3u;
    auto &code = test.code;
    code.insert(code.end(), {EncodeSmem0(0x03, 8), EncodeSmem1(0, 125),
                            EncodeSmem0(0x02, 16), EncodeSmem1(32, 125)});
    code.push_back(EncodeSop2(0x1e, 22, 44, InlineU32(2)));
    if (masked) {
      code.push_back(EncodeVop1(0x01, 2, Vgpr(0))); // Preserve the lane for disjoint output addresses.
      code.insert(code.end(), {EncodeSmem0(0x08, 23, 26), EncodeSmem1(0, 22)});
      code.push_back(EncodeVop1(0x01, 18, 23));
      code.push_back(EncodeVop1(0x02, 22, Vgpr(18)));
      code.push_back(EncodeSop2(0x1e, 22, 22, InlineU32(2)));
    }
    code.insert(code.end(), {EncodeSmem0(0x08, 21, 4), EncodeSmem1(0, 22)});
    AppendSMovLiteral(&code, 20, 0u);
    AppendVMovU32(&code, 0, 0x3f000000u);
    AppendVMovU32(&code, 1, 0x3f000000u);
    const auto masked_exit_branch = masked ? code.size() + 2u : 0u;
    if (masked) {
      code.push_back(EncodeVop2(0x1b, 6, InlineU32(1), 2));
      code.push_back(EncodeVopc(0xd5, InlineU32(0), 6));
      code.push_back(0u); // Skip the loop when the new EXEC is empty.
    }
    const auto header = code.size();
    code.push_back(EncodeSopc(0x0a, 20, 21));
    const auto exit_branch = code.size();
    code.push_back(0u);
    if (masked) {
      code.insert(code.end(), {EncodeVop3Word0(0x346, 3),
                              EncodeVop3Word1(Vgpr(18), InlineU32(4), 20)});
      code.push_back(EncodeVop1(0x02, 22, Vgpr(3)));
    } else {
      code.push_back(EncodeSop2(0x1e, 22, 44, InlineU32(4)));
      code.push_back(EncodeSop2(0x02, 22, 22, 20));
    }
    code.push_back(EncodeSop2(0x1e, 22, 22, InlineU32(2)));
    code.insert(code.end(), {EncodeSmem0(0x08, 23, 6), EncodeSmem1(0, 22)});
    code.push_back(EncodeSop2(0x26, 24, 23, 255)); code.push_back(396u);
    code.push_back(EncodeSop2(0x02, 24, 24, 255)); code.push_back(184u);
    code.insert(code.end(), {EncodeSmem0(0x0b, 32, 8), EncodeSmem1(0, 24),
                            EncodeMimg0(0x27, 1), EncodeMimg1(4, 0, 8, 10)});
    if (masked) {
      code.push_back(EncodeVop2(0x1a, 5, InlineU32(2), 2));
      code.insert(code.end(), {EncodeVop3Word0(0x346, 30),
          EncodeVop3Word1(22, InlineU32(std::countr_zero(wave)), Vgpr(5))});
    } else code.push_back(EncodeVop1(0x01, 30, 22));
    AppendBufferStoreDword(&code, 4, 30);
    code.push_back(EncodeSop2(0x02, 20, 20, InlineU32(1)));
    code.push_back(EncodeSopp(0x02, header - code.size() - 1u));
    code[exit_branch] = EncodeSopp(0x04, code.size() - exit_branch - 1u);
    if (masked) code[masked_exit_branch] = EncodeSopp(0x08, code.size() - masked_exit_branch - 1u);
    AppendEnd(&code);
    test.opcodes = {O::S_LOAD_DWORDX8, O::S_LOAD_DWORDX4, O::S_BUFFER_LOAD_DWORD,
        O::S_BUFFER_LOAD_DWORDX8, O::S_MOV_B32, O::S_LSHL_B32, O::S_ADD_I32,
        O::S_MUL_I32, O::S_CMP_LT_U32, O::S_CBRANCH_SCC0, O::S_BRANCH,
        O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
    if (masked) test.opcodes.insert(test.opcodes.end(), {O::V_READFIRSTLANE_B32,
        O::V_LSHL_ADD_U32, O::V_LSHLREV_B32, O::V_AND_B32, O::V_CMPX_NE_U32, O::S_CBRANCH_EXECZ});
    const auto compiled = CompileCase(test, vulkan.SubgroupSize());
    Require(test.name, "exact active images", compiled.resources.images.size() == 4u &&
                std::ranges::all_of(compiled.resources.images, [](const auto &image) {
                  return image.dwords[0] >= 0x101u && image.dwords[0] <= 0x104u;
                }), "unused row entries selected malformed material fields");
    for (u32 word = 0; word < 4u; ++word)
      Require(test.name, "count descriptor capture",
              std::ranges::any_of(compiled.resources.specialization_reads, [&](const auto &read) {
                return read.first <= srt + word * 4u &&
                    read.first + read.second >= srt + (word + 1u) * 4u;
              }), "count V# loaded from SRT escaped writable-alias capture");
    for (const auto &buffer : compiled.resources.buffers) {
      ShaderBufferResource descriptor{};
      std::copy_n(buffer.dwords.begin(), 4u, descriptor.fields);
      test.storage_buffer_ranges.emplace_back(static_cast<u32>(descriptor.Base48()),
                                              static_cast<u32>(descriptor.GetSize()));
    }
    std::vector<VulkanHarness::Image> images;
    for (const auto &image : compiled.resources.images)
      images.push_back(vulkan.CreateImageMips(test.name, 1, 1, vk::Format::eR32G32B32A32Sfloat,
          vk::ImageUsageFlagBits::eSampled, {{0x3f800000u + image.dwords[0] - 0x100u, 0u, 0u, 0u}},
          4u, vk::ImageLayout::eShaderReadOnlyOptimal, vk::ImageType::e2D, vk::ImageViewType::e2D, 1));
    auto sampler = vulkan.CreateSampler(test.name, 1);
    auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
    vulkan.Dispatch(test, compiled, output, nullptr, nullptr, nullptr, nullptr, sampler, images);
    CompareWords(test, "active row images, wrapped keys and untouched unused slots", test.expected,
                 vulkan.ReadBuffer(test.name, output, test.expected.size()));
    vulkan.DestroyBuffer(&output);
    vulkan.Device().destroySampler(sampler);
    for (auto &image : images) vulkan.DestroyImage(&image);
    std::printf("[compute] %-32s ok\n", test.name);
  }
}

void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) {
  constexpr const char *name = "IndirectImageKeySwitch";
  constexpr uint32_t mapping_capacity = 1793u;
  using namespace ShaderRecompiler::IR;

  Program program{};
  program.stage = ShaderType::Compute;
  program.wave_size = 32;
  program.srt_plan_complete = true;
  program.resource_tracking_complete = true;
  program.shader_info_complete = true;
  program.block_storage.push_back(std::make_unique<Block>());
  auto *block = program.block_storage.back().get();
  program.blocks.push_back(block);

  auto &key = block->AppendNewInst(ValueOpcode::LaneId);
  auto &image =
      block->AppendNewInst(ValueOpcode::GetImageResource,
                           {Value(&key), Value(0u), Value(0u), Value(0u),
                            Value(0u), Value(0u), Value(0u), Value(0u)});
  image.SetFlags<uint32_t>(0u);
  auto &sampler =
      block->AppendNewInst(ValueOpcode::GetSamplerResource,
                           {Value(0u), Value(0u), Value(0u), Value(0u)});
  sampler.SetFlags<uint32_t>(0u);
  auto &address = block->AppendNewInst(
      ValueOpcode::MakeImageAddress,
      {Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
       Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
       Value(0u)});
  MemoryInfo memory{};
  memory.kind = ResourceKind::Image;
  memory.resource = 0;
  memory.sampler = 0;
  memory.dmask = 0xf;
  memory.image_sample_flags = ShaderRecompiler::Decoder::ImageSampleFlagLod;
  memory.image_dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
  memory.image_address_components = 3;
  program.memory_info.push_back(memory);
  const MemoryFlags memory_flags{0u, 0x10f0u};
  uint64_t memory_flag_bits = 0;
  std::memcpy(&memory_flag_bits, &memory_flags, sizeof(memory_flags));
  auto &sample = block->AppendNewInst(
      ValueOpcode::ImageSampleRaw,
      {Value(&image), Value(&sampler), Value(&address), Value(true)}, memory_flag_bits);
  auto &sample_x = block->AppendNewInst(ValueOpcode::CompositeExtractU32x4,
                                        {Value(&sample), Value(0u)});
  block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(&sample_x)});

  program.descriptor_sources.resize(2);
  program.descriptor_sources[0].dword_count = 8;
  program.descriptor_sources[0].indirect_descriptor =
      DescriptorSource::IndirectDescriptor{
          .selector = DescriptorSource::IndirectDescriptor::SelectorRead{
              .source = 0, .stride = 224, .offset = 12},
          .table_source = 0, .table_stride = 32};
  program.descriptor_sources[1].dword_count = 4;

  ImageResource root{};
  root.source = 0;
  root.first_use_pc = 0x10f0u;
  root.resource_class = ImageResourceClass::Sampled;
  root.numeric_class = Prospero::TextureNumericClass::Float;
  root.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
  root.read = true;
  root.indirect_root = 0;
  root.indirect_mapping_offset = 0;
	root.indirect_search_iterations = std::bit_width(mapping_capacity);
  root.indirect_resources = {0u, 1u};
  auto candidate = root;
	candidate.indirect_search_iterations = 0;
  candidate.indirect_resources.clear();
  program.info.images = {root, candidate};
  program.info.samplers.push_back({1u, 0x10f0u});
  program.info.sampled_pairs.push_back({0u, 0u, 0x10f0u});

  ShaderComputeInputInfo compute{};
  program.shader_info_complete = false;
  CollectShaderInfo(program, {.compute = &compute});
  auto spirv = ShaderRecompiler::Spirv::EmitProgram(program,
                                                    {.compute = &compute});
  ValidateSpirv(name, spirv);
  spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
  std::string text;
  Require(name, "SPIR-V disassembly", tools.Disassemble(spirv, &text),
          "failed to disassemble indirect image shader");
  Require(name, "homogeneous image array",
          text.find("OpSwitch") == std::string::npos &&
              CountText(text, "OpImageSampleExplicitLod") == 1 &&
              text.find("SampledImageArrayNonUniformIndexing") != std::string::npos,
          "homogeneous image keys did not select one descriptor array sample");
  u32 sampled_image = 0;
  std::vector<u32> nonuniform;
  for (size_t offset = 5; offset < spirv.size();) {
    const auto words = std::span<const u32>(spirv).subspan(offset, spirv[offset] >> 16u);
    const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
    if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform)
      nonuniform.push_back(words[1]);
    if (opcode == spv::OpImageSampleExplicitLod) sampled_image = words[3];
    offset += words.size();
  }
  Require(name, "nonuniform sampled image operand",
          sampled_image != 0u && std::ranges::find(nonuniform, sampled_image) != nonuniform.end(),
          "the sample operand lacks its nonuniform decoration");

  program.info.images[0].indirect_search_iterations = 0;
  program.shader_info_complete = false;
  CollectShaderInfo(program, {.compute = &compute});
  spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
  ValidateSpirv("DenseIndirectImageTable", spirv);
  Require(name, "dense image map disassembly", tools.Disassemble(spirv, &text),
          "failed to disassemble dense image table shader");
  Require(name, "bounded dense image selection",
          CountText(text, "OpULessThan") == 1 &&
              CountText(text, "OpImageSampleExplicitLod") == 1 &&
              text.find("SampledImageArrayNonUniformIndexing") != std::string::npos,
          "dense image selection lost its bounds check or native nonuniform array");

  // Materialized children follow all native roots, including unrelated images.
  root.indirect_resources = {0u};
  program.info.images = {root, candidate};
  program.info.images[1].indirect_root = ImageResource::NoIndirectImage;
  for (u32 ordinal = 1; ordinal < 189u; ++ordinal) {
    root.indirect_resources.push_back(static_cast<u32>(program.info.images.size()));
    program.info.images.push_back(candidate);
  }
  program.info.images[0] = root;
  program.shader_info_complete = false;
  CollectShaderInfo(program, {.compute = &compute});
  spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
  ValidateSpirv(name, spirv);
  Require(name, "large image array disassembly", tools.Disassemble(spirv, &text),
          "failed to disassemble the materialized image array");
  Require(name, "large homogeneous image array",
          text.find("OpSwitch") == std::string::npos &&
              CountText(text, "OpImageSampleExplicitLod") == 1,
          "materialized images expanded into per-candidate samples");

  program.info.images[root.indirect_resources[103]].dimension =
      ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
  program.info.images[root.indirect_resources[103]].cube = true;
  program.shader_info_complete = false;
  CollectShaderInfo(program, {.compute = &compute});
  spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
  ValidateSpirv(name, spirv);
  Require(name, "mixed contiguous image runs", tools.Disassemble(spirv, &text) &&
              CountText(text, "OpImageSampleExplicitLod") == 3,
          "one cube split the native arrays into per-image samples");

  program.memory_info[0].image_dimension =
      ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
  program.memory_info[0].image_address_components = 4;
  for (u32 component = 0; component < 4u; component++) {
    constexpr std::array values{1.375f, 1.625f, 10.0f, 2.0f};
    address.SetArg(component, Value(std::bit_cast<u32>(values[component])));
  }
  root.indirect_resources = {0u, 1u, 2u};
  program.info.images = {root, candidate, candidate};
  for (const bool cube_first : {true, false}) {
    for (u32 resource = 0; resource < 3u; resource++) {
      auto &image_resource = program.info.images[resource];
      image_resource.cube = resource == (cube_first ? 0u : 1u);
      image_resource.dimension =
          resource == (cube_first ? 1u : 0u)
              ? ShaderRecompiler::Decoder::ImageDimension::Dim2D
              : ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
    }
    program.shader_info_complete = false;
    CollectShaderInfo(program, {.compute = &compute});
    spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
    ValidateSpirv(name, spirv);
    Require(name, "mixed SPIR-V disassembly", tools.Disassemble(spirv, &text),
            "failed to disassemble mixed indirect image shader");
    Require(name, "mixed key switch",
            CountText(text, "OpImageSampleExplicitLod") == 3 &&
                CountText(text, "OpFSub") == 2,
            "mixed candidates lost a sample or shared cube coordinate conversion");

    std::vector<std::span<const u32>> definitions(spirv[3]);
    u32 samples = 0;
    for (size_t offset = 5; offset < spirv.size();) {
      const auto words = std::span<const u32>(spirv).subspan(offset, spirv[offset] >> 16u);
      const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
      if (opcode == spv::OpTypeVector) {
        definitions[words[1]] = words;
      } else if (opcode == spv::OpCompositeConstruct || opcode == spv::OpBitcast ||
                 opcode == spv::OpConstant) {
        definitions[words[2]] = words;
      } else if (opcode == spv::OpImageSampleExplicitLod) {
        const auto coord = definitions[words[4]];
        const auto lod = definitions[words[6]];
        const auto components = samples == (cube_first ? 1u : 0u) ? 2u : 3u;
        Require(name, "mixed coordinate and LOD layout",
                !coord.empty() && definitions[coord[1]][3] == components &&
                    words[5] == spv::ImageOperandsLodMask && lod.size() == 4u &&
                    definitions[lod[3]].size() == 4u &&
                    definitions[lod[3]][3] == std::bit_cast<u32>(2.0f),
                "candidate coordinates changed the instruction's LOD operand");
        samples++;
      }
      offset += words.size();
    }
    Require(name, "mixed sample count", samples == 3u,
            "mixed candidate switch did not retain every image");
  }
  // Keep an unrelated native root between the null image and appended children,
  // then select both sides of a cube boundary and the unmapped suffix on the GPU.
  root.indirect_resources = {0u, 2u, 3u, 4u, 5u, 6u};
  root.indirect_search_iterations = 0;
  program.info.images.assign(7, candidate);
  program.info.images[0] = root;
  program.info.images[1].indirect_root = ImageResource::NoIndirectImage;
  program.info.images[4].dimension =
      ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
  program.info.images[4].cube = true;
  for (u32 component = 0; component < 4u; ++component) {
    constexpr std::array values{1.5f, 1.5f, 0.0f, 0.0f};
    address.SetArg(component, Value(std::bit_cast<u32>(values[component])));
  }
  auto &byte_offset = *block->PrependNewInst(
      std::next(block->begin()), ValueOpcode::IMul32, {Value(&key), Value(4u)});
  image.SetArg(0, Value(&byte_offset));
  auto &output = block->AppendNewInst(ValueOpcode::GetBufferResource,
                                      {Value(0u), Value(0u), Value(0u), Value(0u)});
  output.SetFlags<u32>(0u);
  program.memory_info.push_back({.kind = ResourceKind::Buffer, .offen = true});
  auto &store = block->AppendNewInst(
      ValueOpcode::StoreBufferU32,
      {Value(&output), Value(0u), Value(&byte_offset), Value(0u), Value(&sample_x), Value(true)});
  store.SetFlags(MemoryFlags{1u, 0u});
  program.info.buffers.push_back({.packed_stride = 1, .written = true});
  program.shader_info_complete = false;
  CollectShaderInfo(program, {.compute = &compute});

  TestCase test;
  test.name = "IndirectImageContiguousRuns";
  CompiledShader compiled;
  compiled.program = std::move(program);
  constexpr std::array ordinals{0u, 1u, 2u, 3u, 4u, 5u, 2u, 0u};
  compiled.resources.flattened_srt.push_back(ordinals.size());
  compiled.resources.flattened_srt.insert(compiled.resources.flattened_srt.end(),
                                          ordinals.begin(), ordinals.end());
  std::vector<VulkanHarness::Image> textures;
  for (u32 resource = 0; resource < 7u; ++resource) {
    const auto layers = resource == 4u ? 6u : 1u;
    std::vector<u32> pixels(layers * 16u, std::bit_cast<u32>(float(resource)));
    std::vector<u32> mip(layers * 4u,
                         std::bit_cast<u32>(resource == 0u ? 0.0f : float(resource + 10u)));
    textures.push_back(vulkan.CreateImageMips(
        test.name, 2, 2, vk::Format::eR32G32B32A32Sfloat,
        vk::ImageUsageFlagBits::eSampled, {pixels, mip}, 4u,
        vk::ImageLayout::eShaderReadOnlyOptimal, vk::ImageType::e2D,
        resource == 4u ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D, layers));
  }
  const auto native_sampler = vulkan.CreateSampler(test.name);
  enum class Access { Sample, Gather, GatherHomogeneous, Fetch };
  auto *access = &sample;
  for (const auto mode : {Access::Sample, Access::Gather,
                          Access::GatherHomogeneous, Access::Fetch}) {
    const bool fetch = mode == Access::Fetch;
    const bool gather = mode == Access::Gather || mode == Access::GatherHomogeneous;
    const bool homogeneous = mode == Access::GatherHomogeneous;
    if (mode == Access::Gather) {
      test.name = "IndirectImageGatherContiguousRuns";
      const auto position = std::ranges::find_if(*block, [&](const Inst &inst) {
        return &inst == access;
      });
      auto &read = *block->PrependNewInst(
          position, ValueOpcode::ImageGatherRaw,
          {Value(&image), Value(&sampler), Value(&address), Value(0u), Value(0u),
           Value(0u), Value(0u), Value(true)}, memory_flag_bits);
      access->ReplaceUsesWith(Value(&read), false);
      access = &read;
      compiled.program.info.images[4].cube = false;
      auto &gather_memory = compiled.program.memory_info[0];
      gather_memory.dmask = 1;
      gather_memory.image_sample_flags = ShaderRecompiler::Decoder::ImageSampleFlagLevelZero |
          ShaderRecompiler::Decoder::ImageSampleFlagOffset;
      // GATHER4_LZ_O places its packed signed offset before the coordinates.
      address.SetArg(0, Value(0x3fu));
      address.SetArg(1, Value(std::bit_cast<u32>(0.75f)));
      address.SetArg(2, Value(std::bit_cast<u32>(0.25f)));
      address.SetArg(3, Value(0u));
    }
    auto &selected_resources = compiled.program.info.images[0].indirect_resources;
    selected_resources = homogeneous ? std::vector<u32>{0u, 2u, 3u, 5u, 6u}
                                     : root.indirect_resources;
    if (homogeneous) test.name = "IndirectImageGatherHomogeneous";
    compiled.resources.flattened_srt[6] = homogeneous ? 4u : ordinals[5];
    if (fetch) {
      test.name = "IndirectImageFetchContiguousRuns";
      const auto position = std::ranges::find_if(*block, [&](const Inst &inst) {
        return &inst == access;
      });
      auto &read = *block->PrependNewInst(
          position, ValueOpcode::ImageRead,
          {Value(&image), Value(&address), Value(true)}, memory_flag_bits);
      access->ReplaceUsesWith(Value(&read), false);
      sampler.Invalidate();
      compiled.program.info.samplers.clear();
      compiled.program.info.sampled_pairs.clear();
      compiled.program.info.images[4].cube = false;
      compiled.program.memory_info[0].dmask = 0xf;
      compiled.program.memory_info[0].image_sample_flags = 0;
      compiled.program.memory_info[0].image_has_mip = true;
      for (u32 component = 0; component < 4u; ++component)
        address.SetArg(component, Value(0u));
      // Native DIM=2DArray places LOD after the layer, even for a 2D candidate.
      address.SetArg(3, Value(1u));
    }
    compiled.program.shader_info_complete = false;
    CollectShaderInfo(compiled.program, {.compute = &compute});
    for (const u32 wave_size : {32u, 64u}) {
      compute.wave_size = wave_size;
      compute.host_subgroup_size = vulkan.SubgroupSize();
      compute.threads_num[0] = wave_size;
      compute.threads_num[1] = compute.threads_num[2] = 1u;
      compiled.program.wave_size = wave_size;
      compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(compiled.program, {.compute = &compute});
      compiled.packed_user_data.resize(compiled.program.bindings.ShaderDataDwords());
      ValidateSpirv(test.name, compiled.spirv);
      const auto halves = wave_size > compute.host_subgroup_size ? 2u : 1u;
      const auto image_opcode = fetch ? "OpImageFetch" : gather ? "OpImageGather"
                                                          : "OpImageSampleExplicitLod";
      Require(test.name, "native array image runs", tools.Disassemble(compiled.spirv, &text) &&
                  CountText(text, image_opcode) == (homogeneous ? 1u : 3u) * halves,
              "mixed runs expanded into per-image accesses");
      if (homogeneous) {
        Require(test.name, "homogeneous gather array", text.find("OpSwitch") == std::string::npos,
                "homogeneous gather emitted a descriptor switch");
      }
      if (fetch || gather) {
        std::vector<u32> decorated;
        std::vector<u32> image_operands;
        for (size_t offset = 5; offset < compiled.spirv.size();) {
          const auto words = std::span<const u32>(compiled.spirv).subspan(
              offset, compiled.spirv[offset] >> 16u);
          const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
          if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform)
            decorated.push_back(words[1]);
          if (opcode == (fetch ? spv::OpImageFetch : spv::OpImageGather))
            image_operands.push_back(words[3]);
          offset += words.size();
        }
        Require(test.name, "nonuniform image operands",
                std::ranges::count_if(image_operands, [&](u32 operand) {
                  return std::ranges::find(decorated, operand) != decorated.end();
                }) == (homogeneous ? 1u : 2u) * halves,
                "a dynamically selected image operand lacks its nonuniform decoration");
      }
      test.initial.assign(wave_size, 0xdeadbeefu);
      test.expected.assign(wave_size, 0u);
      for (u32 lane = 0; lane < ordinals.size(); ++lane) {
        const auto resource = selected_resources[compiled.resources.flattened_srt[1u + lane]];
        test.expected[lane] = std::bit_cast<u32>(float(resource + (fetch && resource != 0u ? 10u : 0u)));
      }
      auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
      vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr,
                      fetch ? vk::Sampler{} : native_sampler, textures);
      const auto actual = vulkan.ReadBuffer(test.name, buffer, test.expected.size());
      vulkan.DestroyBuffer(&buffer);
      CompareWords(test, "root, native slots, array runs and mapping bounds", test.expected, actual);
    }
    std::printf("[compute] %-32s ok\n", test.name);
  }
  vulkan.Device().destroySampler(native_sampler);
  for (auto &texture : textures) vulkan.DestroyImage(&texture);
}

TestCase ImageStoreMipSelectsPpsa01340Descriptor() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovU32(&code, 21, 0);
  AppendVMovU32(&code, 22, 2);
  AppendVMovU32(&code, 23, 0);
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovLiteral(&code, 1, 0x40000000u);
  AppendVMovLiteral(&code, 2, 0x40400000u);
  AppendVMovLiteral(&code, 3, 0x40800000u);
  code.push_back(EncodeMimg0(0x09, 0xf));
  code.push_back(EncodeMimg1(0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageStoreMipSelectsPpsa01340Descriptor";
  test.code = std::move(code);
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE_MIP, O::S_ENDPGM};
  const std::array<u32, 8> descriptor{0x0294dc00u, 0xc4700000u, 0x00b3c13fu,
                                      0x91b31facu, 0x00000000u, 0x00700030u,
                                      0xb07b0000u, 0x0002ac3cu};
  std::copy(descriptor.begin(), descriptor.end(), test.user_data.begin());
  test.has_user_data = true;
  test.compile_only = true;
  test.expected_mip_descriptors = 3;
  test.required_spirv = {"OpSwitch", "OpImageWrite"};
  return test;
}

template <u32 mode> TestCase ImageStoreSingleMipBounds() {
  using O = ShaderOpcode;
  constexpr std::array names{"ImageStoreSingleMipValid", "ImageStoreSingleMipBeyondEnd",
                             "ImageStoreSingleMipNegative"};
  constexpr std::array lods{0u, 1u, UINT32_MAX};
  TestCase test;
  test.name = names[mode];
  AppendVMovU32(&test.code, 20, 0);
  AppendVMovU32(&test.code, 21, 0);
  AppendVMovU32(&test.code, 22, lods[mode]);
  for (u32 component = 0; component < 4; component++) {
    AppendVMovLiteral(&test.code, component, 0x40000000u);
  }
  test.code.push_back(EncodeMimg0(0x09, 0xf));
  test.code.push_back(EncodeMimg1(0, 20));
  AppendEnd(&test.code);
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE_MIP, O::S_ENDPGM};
  test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32_32_32_32Float);
  test.has_user_data = true;
  test.storage_image_rgba = std::vector<u32>(4 * 4 * 4, 0x3f800000u);
  test.expected_storage_image_rgba = test.storage_image_rgba;
  if constexpr (mode == 0) {
    std::fill_n(test.expected_storage_image_rgba.begin(), 4, 0x40000000u);
  }
  test.expected_mip_descriptors = 1;
  if constexpr (mode == 0) {
    test.required_spirv = {"OpImageWrite"};
  }
  return test;
}

TestCase ImageStoreRgbOneUsesInverseSwizzle() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 1);
  AppendVMovU32(&code, 21, 2);
  AppendVMovU32(&code, 22, 0);
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovLiteral(&code, 1, 0x3f008081u);
  AppendVMovLiteral(&code, 2, 0x3e808081u);
  AppendVMovLiteral(&code, 3, 0x3f40c0c1u);
  code.push_back(EncodeMimg0(0x08, 0xf));
  code.push_back(EncodeMimg1(0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageStoreRgbOneUsesInverseSwizzle";
  test.code = code;
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
  test.has_user_data = true;
  test.user_data[0] = 0x1000u;
  test.user_data[1] =
      static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u;
  test.image_descriptor_swizzle = DstSel(4, 5, 6, 1);
  test.storage_image_format = vk::Format::eR8G8B8A8Unorm;
  test.storage_image_dwords_per_pixel = 1;
  test.storage_image_rgba = std::vector<u32>(16, 0);
  test.expected_storage_image_rgba = std::vector<u32>(16, 0);
  test.expected_storage_image_rgba[2 * 4 + 1] = 0x004080ffu;
  return test;
}

TestCase ImageStoreDuplicateSelectorUsesInverseSwizzle() {
  auto test = ImageStoreRgbOneUsesInverseSwizzle();
  test.name = "ImageStoreDuplicateSelectorUsesInverseSwizzle";
  test.image_descriptor_swizzle = DstSel(4, 4, 6, 7);
  test.expected_storage_image_rgba[2 * 4 + 1] = 0xc04000ffu;
  return test;
}

TestCase ImageStoreBgraUsesInverseSwizzle() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 1);
  AppendVMovU32(&code, 21, 2);
  AppendVMovU32(&code, 22, 0);
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovLiteral(&code, 1, 0x3f008081u); // 128/255
  AppendVMovLiteral(&code, 2, 0x3e808081u); // 64/255
  AppendVMovLiteral(&code, 3, 0x3f3fbfc0u); // 191/255
  code.push_back(EncodeMimg0(0x08, 0xf));
  code.push_back(EncodeMimg1(0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageStoreBgraUsesInverseSwizzle";
  test.code = code;
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
  test.has_user_data = true;
  test.user_data[0] = 0x1000u;
  test.user_data[1] =
      static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u;
  test.image_descriptor_swizzle = DstSel(6, 5, 4, 7);
  test.storage_image_format = vk::Format::eR8G8B8A8Unorm;
  test.storage_image_dwords_per_pixel = 1;
  test.storage_image_rgba = std::vector<u32>(16, 0);
  test.expected_storage_image_rgba = std::vector<u32>(16, 0);
  test.expected_storage_image_rgba[2 * 4 + 1] = 0xbfff8040u;
  return test;
}

TestCase ImageStoreYzwxUsesInverseSwizzle() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 1);
  AppendVMovU32(&code, 21, 2);
  AppendVMovU32(&code, 22, 0);
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  AppendVMovLiteral(&code, 1, 0x3f000000u);
  AppendVMovLiteral(&code, 2, 0x3e800000u);
  AppendVMovLiteral(&code, 3, 0x3f400000u);
  code.push_back(EncodeMimg0(0x08, 0xf));
  code.push_back(EncodeMimg1(0, 20));
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageStoreYzwxUsesInverseSwizzle";
  test.code = code;
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
  test.user_data =
      MakeStorageTextureData(Prospero::BufferFormat::k32_32_32_32Float);
  test.has_user_data = true;
  test.image_descriptor_swizzle = DstSel(5, 6, 7, 4);
  test.storage_image_rgba = MakeRgbaImage(4, 4);
  test.expected_storage_image_rgba = MakeRgbaImage(4, 4);
  SetRgbaPixel(&test.expected_storage_image_rgba, 4, 1, 2, 0x3f400000u,
               0x3f800000u, 0x3f000000u, 0x3e800000u);
  return test;
}

TestCase ImageStoreR32FloatUsesFormatlessStorageImage() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  AppendVMovU32(&code, 22, 0);
  AppendVMovLiteral(&code, 0, 0x3f800000u);
  code.push_back(EncodeMimg0(0x08, 0x1));
  code.push_back(EncodeMimg1(0, 20));
  AppendEnd(&code);

  std::vector<u32> expected_image(16, 0);
  expected_image[1 * 4 + 2] = 0x3f800000u;

  TestCase test;
  test.name = "ImageStoreR32FloatUsesFormatlessStorageImage";
  test.code = code;
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
  test.storage_image_format = vk::Format::eR32Sfloat;
  test.storage_image_dwords_per_pixel = 1;
  test.storage_image_rgba = std::vector<u32>(16, 0);
  test.expected_storage_image_rgba = expected_image;
  test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat"};
  test.forbidden_spirv = {"OpCapability StorageImageReadWithoutFormat",
                          "Rgba32f", "OpTypeSampler", "OpTypeSampledImage"};
  return test;
}

TestCase ImageStoreR32SintUsesRawUintView() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  AppendVMovU32(&code, 22, 0);
  AppendVMovLiteral(&code, 0, 0x80000001u);
  code.push_back(EncodeMimg0(0x08, 0x1));
  code.push_back(EncodeMimg1(0, 20));
  AppendEnd(&code);

  std::vector<u32> expected_image(16, 0);
  expected_image[1 * 4 + 2] = 0x80000001u;

  TestCase test;
  test.name = "ImageStoreR32SintUsesRawUintView";
  test.code = code;
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
  test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32SInt);
  test.has_user_data = true;
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.expected_storage_image_r32ui = expected_image;
  test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
                         StorageUint2DImageBindingName(false)};
  test.forbidden_spirv = {"R32ui"};
  return test;
}

TestCase ImageStoreR32UintUsesFormatlessStorageImage() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  AppendVMovU32(&code, 22, 0);
  AppendVMovLiteral(&code, 0, 0x12345678u);
  code.push_back(EncodeMimg0(0x08, 0x1));
  code.push_back(EncodeMimg1(0, 20));
  AppendEnd(&code);

  std::vector<u32> expected_image(16, 0);
  expected_image[1 * 4 + 2] = 0x12345678u;

  TestCase test;
  test.name = "ImageStoreR32UintUsesFormatlessStorageImage";
  test.code = code;
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
  test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
  test.has_user_data = true;
  test.storage_image_rgba = MakeRgbaImage(4, 4);
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.expected_storage_image_r32ui = expected_image;
  test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
                         StorageUint2DImageBindingName(false)};
  test.forbidden_spirv = {"R32ui"};
  return test;
}

TestCase ImageStorePackedUintSaturatesChannels() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  AppendVMovU32(&code, 22, 0);
  AppendVMovU32(&code, 0, 0x800u);
  AppendVMovLiteral(&code, 1, UINT32_MAX);
  AppendVMovU32(&code, 2, 0x400u);
  AppendVMovU32(&code, 3, 0x55u);
  code.push_back(EncodeMimg0(0x08, 0xf));
  code.push_back(EncodeMimg1(0, 20));
  AppendEnd(&code);

  std::vector<u32> expected_image(16, 0);
  expected_image[1 * 4 + 2] = UINT32_MAX;

  TestCase test;
  test.name = "ImageStorePackedUintSaturatesChannels";
  test.code = std::move(code);
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
  test.user_data =
      MakeStorageTextureData(Prospero::BufferFormat::k11_11_10UInt);
  test.has_user_data = true;
  test.image_descriptor_swizzle = DstSel(4, 5, 6, 0);
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.expected_storage_image_r32ui = std::move(expected_image);
  test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
                         StorageUint2DImageBindingName(false), "OpULessThan",
                         "OpSelect"};
  test.forbidden_spirv = {"R32ui"};
  return test;
}

TestCase ImageStorePackedUintHonorsSparseDmask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  AppendVMovU32(&code, 22, 0);
  AppendVMovU32(&code, 0, 0x321u);
  AppendVMovU32(&code, 1, 0x2abu);
  code.push_back(EncodeMimg0(0x08, 0x5));
  code.push_back(EncodeMimg1(0, 20));
  AppendEnd(&code);

  std::vector<u32> expected_image(16, 0);
  expected_image[1 * 4 + 2] = 0xaac00321u;

  TestCase test;
  test.name = "ImageStorePackedUintHonorsSparseDmask";
  test.code = std::move(code);
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
  test.user_data =
      MakeStorageTextureData(Prospero::BufferFormat::k11_11_10UInt);
  test.has_user_data = true;
  test.image_descriptor_swizzle = DstSel(4, 5, 6, 0);
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.expected_storage_image_r32ui = std::move(expected_image);
  test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
                         StorageUint2DImageBindingName(false)};
  test.forbidden_spirv = {"R32ui"};
  return test;
}

TestCase ComputeTgSizeSgprUsesWaveMetadata() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendStoreSgpr(&code, 2, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ComputeTgSizeSgprUsesWaveMetadata";
  test.code = code;
  test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 4;
  test.compute_info.threads_num[2] = 16;
  test.compute_info.group_id[0] = true;
  test.compute_info.group_id[1] = true;
  test.compute_info.wave_size = 32;
  test.compute_info.thread_ids_num = 3;
  test.compute_info.workgroup_register = 0;
  test.compute_info.tg_size_en = true;
  test.has_compute_info = true;
  test.compile_only = true;
  test.required_spirv = {"OpUDiv", "OpShiftLeftLogical", "2147483648"};
  return test;
}

TestCase ImageStoreAndAtomicShareTypedBinding() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  AppendVMovU32(&code, 22, 0);
  AppendVMovU32(&code, 0, 5);
  code.push_back(EncodeMimg0(0x08, 0x1));
  code.push_back(EncodeMimg1(0, 20));
  AppendVMovU32(&code, 0, 3);
  code.push_back(EncodeMimg0(0x11, 0x1, 0, true));
  code.push_back(EncodeMimg1(0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  std::vector<u32> expected_image(16, 0);
  expected_image[1 * 4 + 2] = 8;

  TestCase test;
  test.name = "ImageStoreAndAtomicShareTypedBinding";
  test.code = std::move(code);
  test.expected = {5};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_ATOMIC_ADD,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
  test.has_user_data = true;
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.expected_storage_image_r32ui = std::move(expected_image);
  test.required_spirv = {"OpImageWrite", "OpImageTexelPointer", "R32ui",
                         StorageUint2DImageBindingName(true)};
  test.forbidden_spirv = {"StorageImageReadWithoutFormat",
                          "StorageImageWriteWithoutFormat",
                          StorageUint2DImageBindingName(false)};
  return test;
}

// RDNA 2 MIMG opcode 0x10: replacement at VDATA, comparison at VDATA+1.
TestCase ImageAtomicCompareSwapGlcAndExec() {
  TestCase test;
  test.name = "ImageAtomicCompareSwapGlcAndExec";
  auto &code = test.code;
  code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve EXEC.
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  const auto exchange = [&](u32 replacement, u32 comparison, bool glc, u32 output) {
    AppendVMovLiteral(&code, 0, replacement);
    AppendVMovLiteral(&code, 1, comparison);
    code.push_back(EncodeMimg0(0x10, 3, 0, glc));
    code.push_back(EncodeMimg1(0, 20));
    AppendStoreVgpr(&code, 0, output);
    AppendStoreVgpr(&code, 1, output + 1);
  };
  exchange(0x12345678u, 0xdeadbeefu, true, 0); // Match; return original bits.
  exchange(0x87654321u, 0xdeadbeefu, true, 2); // Mismatch; preserve texel.
  exchange(0xabcdef01u, 0x12345678u, false, 4); // GLC=0 preserves VDATA.
  AppendVMovLiteral(&code, 0, 0x11111111u);
  AppendVMovLiteral(&code, 1, 0xabcdef01u);
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); // Disable EXEC.
  code.push_back(EncodeMimg0(0x10, 3, 0, true));
  code.push_back(EncodeMimg1(0, 20));
  code.push_back(EncodeSop1(0x04, 126, 12));
  AppendStoreVgpr(&code, 0, 6);
  AppendStoreVgpr(&code, 1, 7);
  AppendEnd(&code);
  test.expected = {0xdeadbeefu, 0xdeadbeefu, 0x12345678u, 0xdeadbeefu,
                   0xabcdef01u, 0x12345678u, 0x11111111u, 0xabcdef01u};
  test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
                 ShaderOpcode::IMAGE_ATOMIC_CMPSWAP, ShaderOpcode::BUFFER_STORE_DWORD,
                 ShaderOpcode::S_ENDPGM};
  test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
  test.has_user_data = true;
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.storage_image_r32ui[6] = 0xdeadbeefu;
  test.expected_storage_image_r32ui = test.storage_image_r32ui;
  test.expected_storage_image_r32ui[6] = 0xabcdef01u;
  // Device scope, AcquireRelease | ImageMemory (0x808).
  test.required_spirv = {"OpAtomicCompareExchange", "OpImageTexelPointer",
                         "OpMemoryBarrier %uint_1 %uint_2056"};
  return test;
}

TestCase ImageAtomicSwapReturnsPreviousTexel() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  AppendVMovU32(&code, 22, 0);
  AppendVMovU32(&code, 0, 5);
  code.push_back(EncodeMimg0(0x08, 0x1));
  code.push_back(EncodeMimg1(0, 20));
  AppendVMovU32(&code, 0, 3);
  code.push_back(EncodeMimg0(0x0f, 0x1, 0, true));
  code.push_back(EncodeMimg1(0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  std::vector<u32> expected_image(16, 0);
  expected_image[1 * 4 + 2] = 3;

  TestCase test;
  test.name = "ImageAtomicSwapReturnsPreviousTexel";
  test.code = std::move(code);
  test.expected = {5};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_ATOMIC_SWAP,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
  test.has_user_data = true;
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.expected_storage_image_r32ui = std::move(expected_image);
  test.required_spirv = {"OpAtomicExchange", "OpImageTexelPointer"};
  return test;
}

TestCase ImageStoreAndAtomicUseSeparateBindings() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 2);
  AppendVMovU32(&code, 21, 1);
  AppendVMovU32(&code, 22, 0);
  AppendVMovU32(&code, 0, 5);
  code.push_back(EncodeMimg0(0x08, 0x1));
  code.push_back(EncodeMimg1(0, 20, 0));
  AppendVMovU32(&code, 0, 3);
  code.push_back(EncodeMimg0(0x11, 0x1, 0, true));
  code.push_back(EncodeMimg1(0, 20, 2));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  auto user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
  std::copy_n(user_data.begin(), 8, user_data.begin() + 8);
  std::vector<u32> expected_image(16, 0);
  expected_image[1 * 4 + 2] = 8;

  TestCase test;
  test.name = "ImageStoreAndAtomicUseSeparateBindings";
  test.code = std::move(code);
  test.expected = {5};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_ATOMIC_ADD,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.user_data = user_data;
  test.has_user_data = true;
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.expected_storage_image_r32ui = std::move(expected_image);
  test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
                         "OpImageWrite",
                         "OpImageTexelPointer",
                         "R32ui",
                         StorageUint2DImageBindingName(false),
                         StorageUint2DImageBindingName(true)};
  return test;
}

TestCase ImageAtomicVariants() {
  using O = ShaderOpcode;

  const u32 initial[] = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu};
  const u32 values[] = {5, 5, 0x0ff0u, 0x0f00u, 0x00ffu};
  const u32 ops[] = {0x11, 0x15, 0x18, 0x19, 0x1a};

  std::vector<u32> code;
  for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
    AppendVMovU32(&code, 20, i & 3u);
    AppendVMovU32(&code, 21, i >> 2u);
    AppendVMovU32(&code, 22, 0);
    AppendVMovLiteral(&code, 0, values[i]);
    code.push_back(EncodeMimg0(ops[i], 0x1, 0, true));
    code.push_back(EncodeMimg1(0, 20));
    AppendStoreVgpr(&code, 0, i);
  }
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageAtomicVariants";
  test.code = code;
  test.expected = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu};
  test.opcodes = {O::V_MOV_B32,          O::IMAGE_ATOMIC_ADD,
                  O::IMAGE_ATOMIC_UMIN,  O::IMAGE_ATOMIC_AND,
                  O::IMAGE_ATOMIC_OR,    O::IMAGE_ATOMIC_XOR,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {"OpImageTexelPointer", "R32ui",
                         StorageUint2DImageBindingName(true)};
  test.forbidden_spirv = {"OpTypeSampler", "OpTypeSampledImage",
                          "StorageImageReadWithoutFormat",
                          "StorageImageWriteWithoutFormat",
                          StorageUint2DImageBindingName(false)};
  test.storage_image_rgba = MakeRgbaImage(4, 4);
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  for (u32 i = 0; i < static_cast<u32>(std::size(initial)); i++) {
    test.storage_image_r32ui[i] = initial[i];
  }
  test.expected_storage_image_r32ui = std::vector<u32>(16, 0);
  test.expected_storage_image_r32ui[0] = 15;
  test.expected_storage_image_r32ui[1] = 5;
  test.expected_storage_image_r32ui[2] = 0x00f0u;
  test.expected_storage_image_r32ui[3] = 0xff00u;
  test.expected_storage_image_r32ui[4] = 0xf0f0u;
  return test;
}

// Input and old texel cover high-DWORD ordering, carry/wrap, the unsigned sign
// boundary, and equal-high-DWORD ordering through the RG32_UINT -> R64_UINT view.
constexpr uint64_t ImageAtomicInputs[][2] = {
    {0x0000000100000001ull, 0x00000000ffffffffull},
    {0x00000000ffffffffull, 0x0000000100000001ull},
    {0x8000000000000000ull, 0x7fffffffffffffffull},
    {0x12345678ffffffffull, 0x1234567800000001ull},
    {0xffffffffffffffffull, 0x8000000000000000ull},
};

struct ImageAtomicIntegerCase {
  u32 encoding;
  ShaderOpcode opcode;
  const char *name32;
  const char *name64;
  const char *spirv;
  uint64_t expected64[std::size(ImageAtomicInputs)];
  u32 expected32[std::size(ImageAtomicInputs)];
};

constexpr ImageAtomicIntegerCase ImageAtomicIntegerCases[] = {
    {0x0fu, ShaderOpcode::IMAGE_ATOMIC_SWAP,
     "ImageAtomicSwap32GlcAndExec", "ImageAtomicSwap64GlcAndExec", "OpAtomicExchange",
     {0x0000000100000001ull, 0x00000000ffffffffull, 0x8000000000000000ull,
      0x12345678ffffffffull, 0xffffffffffffffffull},
     {0x00000001u, 0xffffffffu, 0x00000000u, 0xffffffffu, 0xffffffffu}},
    {0x11u, ShaderOpcode::IMAGE_ATOMIC_ADD,
     "ImageAtomicAdd32GlcAndExec", "ImageAtomicAdd64GlcAndExec", "OpAtomicIAdd",
     {0x0000000200000000ull, 0x0000000200000000ull, 0xffffffffffffffffull,
      0x2468acf100000000ull, 0x7fffffffffffffffull},
     {0x00000000u, 0x00000000u, 0xffffffffu, 0x00000000u, 0xffffffffu}},
    {0x15u, ShaderOpcode::IMAGE_ATOMIC_UMIN,
     "ImageAtomicUMin32GlcAndExec", "ImageAtomicUMin64GlcAndExec", "OpAtomicUMin",
     {0x00000000ffffffffull, 0x00000000ffffffffull, 0x7fffffffffffffffull,
      0x1234567800000001ull, 0x8000000000000000ull},
     {0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}},
    {0x17u, ShaderOpcode::IMAGE_ATOMIC_UMAX,
     "ImageAtomicUMax32GlcAndExec", "ImageAtomicUMax64GlcAndExec", "OpAtomicUMax",
     {0x0000000100000001ull, 0x0000000100000001ull, 0x8000000000000000ull,
      0x12345678ffffffffull, 0xffffffffffffffffull},
     {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}},
    {0x18u, ShaderOpcode::IMAGE_ATOMIC_AND,
     "ImageAtomicAnd32GlcAndExec", "ImageAtomicAnd64GlcAndExec", "OpAtomicAnd",
     {0x0000000000000001ull, 0x0000000000000001ull, 0x0000000000000000ull,
      0x1234567800000001ull, 0x8000000000000000ull},
     {0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}},
    {0x19u, ShaderOpcode::IMAGE_ATOMIC_OR,
     "ImageAtomicOr32GlcAndExec", "ImageAtomicOr64GlcAndExec", "OpAtomicOr",
     {0x00000001ffffffffull, 0x00000001ffffffffull, 0xffffffffffffffffull,
      0x12345678ffffffffull, 0xffffffffffffffffull},
     {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}},
    {0x1au, ShaderOpcode::IMAGE_ATOMIC_XOR,
     "ImageAtomicXor32GlcAndExec", "ImageAtomicXor64GlcAndExec", "OpAtomicXor",
     {0x00000001fffffffeull, 0x00000001fffffffeull, 0xffffffffffffffffull,
      0x00000000fffffffeull, 0x7fffffffffffffffull},
     {0xfffffffeu, 0xfffffffeu, 0xffffffffu, 0xfffffffeu, 0xffffffffu}},
};

TestCase ImageAtomicIntegerGlcAndExec(const ImageAtomicIntegerCase &atomic, bool wide) {
  const u32 texel_dwords = wide ? 2 : 1;
  TestCase test;
  test.name = wide ? atomic.name64 : atomic.name32;
  test.user_data = MakeStorageTextureData(wide ? Prospero::BufferFormat::k32_32UInt
                                              : Prospero::BufferFormat::k32UInt);
  test.has_user_data = true;
  test.storage_image_r32ui.assign(16 * texel_dwords, 0);
  test.expected_storage_image_r32ui = test.storage_image_r32ui;
  auto &code = test.code;
  code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve EXEC.
  for (u32 i = 0; i < std::size(ImageAtomicInputs); ++i) {
    const auto input = ImageAtomicInputs[i][0];
    const auto old = ImageAtomicInputs[i][1];
    const bool glc = i != 0 && i != 4;
    AppendVMovU32(&code, 0, i & 3u);
    AppendVMovU32(&code, 1, i >> 2u);
    AppendVMovLiteral(&code, 2, static_cast<u32>(input));
    AppendVMovLiteral(&code, 3, static_cast<u32>(input >> 32u));
    code.push_back(EncodeMimg0(atomic.encoding, wide ? 3u : 1u, 0, glc));
    code.push_back(EncodeMimg1(2, 0));
    AppendStoreVgpr(&code, 2, i * 2);
    AppendStoreVgpr(&code, 3, i * 2 + 1);
    test.expected.push_back(static_cast<u32>(glc ? old : input));
    test.expected.push_back(static_cast<u32>((wide && glc ? old : input) >> 32u));
    const uint64_t result = wide ? atomic.expected64[i] : atomic.expected32[i];
    for (u32 word = 0; word < texel_dwords; ++word) {
      test.storage_image_r32ui[i * texel_dwords + word] =
          static_cast<u32>(old >> (word * 32u));
      test.expected_storage_image_r32ui[i * texel_dwords + word] =
          static_cast<u32>(result >> (word * 32u));
    }
  }
  // A leaked update must change the inactive texel for every operation.
  const bool clear_bits = atomic.opcode == ShaderOpcode::IMAGE_ATOMIC_AND ||
                          atomic.opcode == ShaderOpcode::IMAGE_ATOMIC_UMIN;
  const u32 inactive = clear_bits ? 0xffffffffu : 0u;
  for (u32 word = 0; word < texel_dwords; ++word) {
    test.storage_image_r32ui[5u * texel_dwords + word] = inactive;
    test.expected_storage_image_r32ui[5u * texel_dwords + word] = inactive;
  }
  AppendVMovU32(&code, 0, 1);
  AppendVMovU32(&code, 1, 1);
  AppendVMovLiteral(&code, 2, 0xabcdef01u);
  AppendVMovLiteral(&code, 3, 0x87654321u);
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  code.push_back(EncodeMimg0(atomic.encoding, wide ? 3u : 1u, 0, true));
  code.push_back(EncodeMimg1(2, 0));
  code.push_back(EncodeSop1(0x04, 126, 12));
  AppendStoreVgpr(&code, 2, std::size(ImageAtomicInputs) * 2);
  AppendStoreVgpr(&code, 3, std::size(ImageAtomicInputs) * 2 + 1);
  test.expected.insert(test.expected.end(), {0xabcdef01u, 0x87654321u});
  AppendEnd(&code);
  test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
                 atomic.opcode, ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
  test.required_spirv = {atomic.spirv, "OpImageTexelPointer", wide ? "R64ui" : "R32ui"};
  if (wide) {
    test.required_spirv.push_back("Int64ImageEXT");
    test.required_spirv.push_back("SPV_EXT_shader_image_int64");
  }
  return test;
}

template <bool max_value> TestCase ImageAtomicSignedMinMax() {
  using O = ShaderOpcode;
  // Input, old value, signed minimum, signed maximum.
  constexpr u32 values[][4] = {
      {0xfffffffbu, 7, 0xfffffffbu, 7},
      {7, 0xfffffffbu, 0xfffffffbu, 7},
      {0x80000000u, 0x7fffffffu, 0x80000000u, 0x7fffffffu},
      {0x7fffffffu, 0x80000000u, 0x80000000u, 0x7fffffffu},
      {0xfffffff6u, 0xfffffffbu, 0xfffffff6u, 0xfffffffbu},
      {0xfffffffbu, 0xfffffff6u, 0xfffffff6u, 0xfffffffbu},
      {2, 8, 2, 8}, {8, 2, 2, 8},
  };
  TestCase test;
  test.name = max_value ? "ImageAtomicSMaxGlcAndExec" : "ImageAtomicSMinGlcAndExec";
  test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
  test.has_user_data = true;
  test.storage_image_r32ui.assign(16, 0);
  test.expected_storage_image_r32ui = test.storage_image_r32ui;
  auto &code = test.code;
  code.push_back(EncodeSop1(0x04, 12, 126));
  for (u32 i = 0; i < std::size(values); ++i) {
    AppendVMovU32(&code, 20, i & 3u);
    AppendVMovU32(&code, 21, i >> 2u);
    AppendVMovLiteral(&code, 2, values[i][0]);
    const bool glc = (i & 1u) == 0;
    code.push_back(EncodeMimg0(max_value ? 0x16 : 0x14, 1, 0, glc));
    code.push_back(EncodeMimg1(2, 20));
    AppendStoreVgpr(&code, 2, i);
    test.expected.push_back(values[i][glc ? 1 : 0]);
    test.storage_image_r32ui[i] = values[i][1];
    test.expected_storage_image_r32ui[i] = values[i][max_value ? 3 : 2];
  }
  AppendVMovLiteral(&code, 2, max_value ? 0x7fffffffu : 0x80000000u);
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  code.push_back(EncodeMimg0(max_value ? 0x16 : 0x14, 1, 0, true));
  code.push_back(EncodeMimg1(2, 20));
  code.push_back(EncodeSop1(0x04, 126, 12));
  AppendStoreVgpr(&code, 2, std::size(values));
  test.expected.push_back(max_value ? 0x7fffffffu : 0x80000000u);
  AppendEnd(&code);
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B64,
                  max_value ? O::IMAGE_ATOMIC_SMAX : O::IMAGE_ATOMIC_SMIN,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.required_spirv = {max_value ? "OpAtomicSMax" : "OpAtomicSMin", "R32ui"};
  return test;
}

template <bool max_value> TestCase ImageAtomicFloatGlcAndExec() {
  using O = ShaderOpcode;
  std::vector<u32> code;
  constexpr u32 atomic_word = max_value ? 0xf07c0108u : 0xf0780108u;
  code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve active workgroup lanes.
  AppendVMovU32(&code, 8, 2);
  AppendVMovU32(&code, 9, 1);
  AppendVMovLiteral(&code, 5, 0x40000000u); // 2.0
  // FMAX uses the captured GLC=0 instruction word.
  code.insert(code.end(), {atomic_word, 0x00010508u});
  AppendStoreVgpr(&code, 5, 0);
  AppendVMovLiteral(&code, 5, max_value ? 0x40800000u : 0x3f800000u);
  code.insert(code.end(), {atomic_word | 0x2000u, 0x00010508u}); // GLC=1.
  AppendStoreVgpr(&code, 5, 1);
  AppendVMovLiteral(&code, 5,
                   max_value ? 0x41000000u : 0x3f000000u); // Inactive lane.
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  code.insert(code.end(), {atomic_word | 0x2000u, 0x00010508u});
  code.push_back(EncodeSop1(0x04, 126, 12));
  AppendStoreVgpr(&code, 5, 2);
  AppendEnd(&code);

  TestCase test;
  test.name = max_value ? "ImageAtomicFMaxCapturedGlcAndExec"
                       : "ImageAtomicFMinGlcAndExec";
  test.code = std::move(code);
  test.expected = {0x40000000u, 0x40000000u,
                  max_value ? 0x41000000u : 0x3f000000u};
  test.opcodes = {O::V_MOV_B32, O::S_MOV_B64,
                 max_value ? O::IMAGE_ATOMIC_FMAX : O::IMAGE_ATOMIC_FMIN,
                 O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  const std::array<u32, 8> descriptor = {
      0x504a7c00u, 0xc1600000u, 0x0086c0efu, 0x91b80924u,
      0x00000000u, 0x00700080u, 0x00000000u, 0x00000000u};
  std::copy_n(descriptor.begin(), 8, test.user_data.begin() + 4);
  test.has_user_data = true;
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.storage_image_r32ui[6] = max_value ? 0x3f800000u : 0x40800000u;
  test.expected_storage_image_r32ui = test.storage_image_r32ui;
  test.expected_storage_image_r32ui[6] = max_value ? 0x40800000u : 0x3f800000u;
  test.required_spirv = {"OpImageTexelPointer", "OpAtomicCompareExchange", "R32ui"};
  return test;
}

template <bool max_value> TestCase ImageAtomicFloatSpecialValues() {
  using O = ShaderOpcode;
  // source, previous texel, expected minimum, expected maximum.
  const u32 cases[][4] = {
      {0x40000000u, 0x3f800000u, 0x3f800000u, 0x40000000u},
      {0xc0000000u, 0xbf800000u, 0xc0000000u, 0xbf800000u},
      {0x7f800000u, 0x3f800000u, 0x3f800000u, 0x7f800000u},
      {0xff800000u, 0x3f800000u, 0xff800000u, 0x3f800000u},
      {0x3f800000u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u},
      {0x7fcabcdeu, 0x3f800000u, 0x3f800000u, 0x3f800000u},
      {0x3f800000u, 0x7fa54321u, 0x7fa54321u, 0x7fa54321u},
      {0x7faabcdeu, 0x3f800000u, 0x3f800000u, 0x3f800000u},
      {0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u},
      {0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u},
      {0x80000000u, 0x80000001u, 0x80000001u, 0x80000000u},
      {0x80000001u, 0x80000000u, 0x80000001u, 0x80000000u},
      {0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u},
      {0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u},
  };
  TestCase test;
  test.name = max_value ? "ImageAtomicFMaxSpecialValues"
                       : "ImageAtomicFMinSpecialValues";
  test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
  test.has_user_data = true;
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.expected_storage_image_r32ui = test.storage_image_r32ui;
  for (u32 i = 0; i < std::size(cases); i++) {
    AppendVMovU32(&test.code, 20, i & 3u);
    AppendVMovU32(&test.code, 21, i >> 2u);
    AppendVMovLiteral(&test.code, 0, cases[i][0]);
    test.code.push_back(EncodeMimg0(max_value ? 0x1f : 0x1e, 1, 0, true));
    test.code.push_back(EncodeMimg1(0, 20));
    AppendStoreVgpr(&test.code, 0, i);
    test.expected.push_back(cases[i][1]);
    test.storage_image_r32ui[i] = cases[i][1];
    test.expected_storage_image_r32ui[i] = cases[i][max_value ? 3 : 2];
  }
  AppendEnd(&test.code);
  test.opcodes = {O::V_MOV_B32,
                 max_value ? O::IMAGE_ATOMIC_FMAX : O::IMAGE_ATOMIC_FMIN,
                 O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  return test;
}

template <bool max_value> TestCase ImageAtomicFloatContendedWorkgroup() {
  using O = ShaderOpcode;
  TestCase test;
  test.name = max_value ? "ImageAtomicFMaxContendedWorkgroup"
                       : "ImageAtomicFMinContendedWorkgroup";
  test.code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // Float thread_id.x.
  AppendVMovU32(&test.code, 20, 0);
  AppendVMovU32(&test.code, 21, 0);
  test.code.push_back(EncodeMimg0(max_value ? 0x1f : 0x1e, 1));
  test.code.push_back(EncodeMimg1(1, 20));
  AppendEnd(&test.code);
  test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32,
                 max_value ? O::IMAGE_ATOMIC_FMAX : O::IMAGE_ATOMIC_FMIN,
                 O::S_ENDPGM};
  test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
  test.has_user_data = true;
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.storage_image_r32ui[0] =
      max_value ? 0xc2c80000u : 0x42c80000u; // -/+100.0
  test.expected_storage_image_r32ui = test.storage_image_r32ui;
  test.expected_storage_image_r32ui[0] =
      max_value ? 0x427c0000u : 0u; // 63.0 / 0.0
  test.compute_info.threads_num[0] = 64;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.thread_ids_num = 1;
  test.has_compute_info = true;
  return test;
}

TestCase ImageAtomicGlc0DoesNotReturnOldValue() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovU32(&code, 20, 0);
  AppendVMovU32(&code, 21, 0);
  AppendVMovU32(&code, 22, 0);
  AppendVMovU32(&code, 0, 5);
  code.push_back(EncodeMimg0(0x11, 0x1));
  code.push_back(EncodeMimg1(0, 20));
  AppendStoreVgpr(&code, 0, 0);
  AppendEnd(&code);

  TestCase test;
  test.name = "ImageAtomicGlc0DoesNotReturnOldValue";
  test.code = code;
  test.expected = {5};
  test.opcodes = {O::V_MOV_B32, O::IMAGE_ATOMIC_ADD, O::BUFFER_STORE_DWORD,
                  O::S_ENDPGM};
  test.storage_image_rgba = MakeRgbaImage(4, 4);
  test.storage_image_r32ui = std::vector<u32>(16, 0);
  test.storage_image_r32ui[0] = 10;
  test.expected_storage_image_r32ui = std::vector<u32>(16, 0);
  test.expected_storage_image_r32ui[0] = 15;
  return test;
}

GraphicsCase GraphicsInterpolationExport() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVintrp(0x00, 0, 0, 0, 0));
  code.push_back(EncodeVintrp(0x01, 0, 0, 0, 0));
  code.push_back(EncodeVintrp(0x02, 1, 0, 1, 2));
  AppendVMovLiteral(&code, 2, 0x3f400000u);
  AppendVMovLiteral(&code, 3, 0x3f800000u);
  code.push_back(EncodeExp0(0x00, 0xf));
  code.push_back(EncodeExp1(0, 1, 2, 3));
  AppendEnd(&code);

  return {"GraphicsInterpolationExport",
          code,
          {0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u},
          {O::V_INTERP_P1_F32, O::V_INTERP_P2_F32, O::V_INTERP_MOV_F32,
           O::V_MOV_B32, O::EXP, O::S_ENDPGM}};
}

GraphicsCase GraphicsPositionWExport() {
  GraphicsCase test;
  test.name = "GraphicsPositionWExport";
  test.pixel_position_w = true;
  test.vertex_clip_w = 4.0f;
  // POS_W occupies v2; the rasterizer supplies FragCoord.w=1/4 for this triangle.
  test.fragment_code = {EncodeExp0(0x00, 0xf), EncodeExp1(2, 2, 2, 2)};
  AppendEnd(&test.fragment_code);
  test.expected_pixel = {0x40800000u, 0x40800000u, 0x40800000u, 0x40800000u};
  test.opcodes = {ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
  return test;
}

GraphicsCase GraphicsPackedHalfCentroid() {
  GraphicsCase test;
  test.name = "GraphicsPackedHalfCentroid";
  test.pixel_perspective_centroid_vgpr = 0;
  test.pixel_custom_interpolation_mask = 1;
  test.pixel_interpolator_settings = {0x420u};
  // Captured ab810715011baef3 interpolation, with attr3.x remapped to attr0.x.
  test.fragment_code = {
      0xc8120002u, 0xc80e0000u,             // raw vertex0/vertex1 packed halves
      0xcc204007u, 0x9c1206f2u,             // low(vertex1) - low(vertex0)
      0xcc207005u, 0x9c1206f2u,             // high(vertex1) - high(vertex0)
      0xc80a0001u,                         // raw vertex2
      0xcc204006u, 0x9c1204f2u,
      0xcc207003u, 0x9c1204f2u,
      0xcc20400eu, 0x04120f00u,             // low = I * delta10 + vertex0
      0xcc20600fu, 0x04120b00u,             // high = I * delta10 + vertex0
      0x3e1c0306u, 0x3e1e0303u,             // += J * delta20
      EncodeExp0(0x00, 0xf), EncodeExp1(14, 15, 14, 15)};
  AppendEnd(&test.fragment_code);
  // At the pixel center the vertex weights are (5/8, 1/8, 1/4).
  // Packed values are (0,1), (1,2), (3,3): expected result (0.875, 1.625).
  test.vertices = {
      0xbf800000u, 0xbf800000u, 0x3c000000u, 0, 0, 0,
      0x40e00000u, 0xbf800000u, 0x40003c00u, 0, 0, 0,
      0xbf800000u, 0x40400000u, 0x42004200u, 0, 0, 0};
  test.expected_pixel = {0x3f600000u, 0x3fd00000u, 0x3f600000u, 0x3fd00000u};
  test.opcodes = {ShaderOpcode::V_INTERP_MOV_F32, ShaderOpcode::V_FMA_F32, ShaderOpcode::V_MAC_F32,
                  ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
  return test;
}

GraphicsCase GraphicsPerspectiveSample(bool multisample) {
  GraphicsCase test;
  test.name = multisample ? "GraphicsPerspectiveSample4x"
                         : "GraphicsPerspectiveSample1x";
  test.pixel_perspective_sample_vgpr = 0;
  test.samples = multisample ? vk::SampleCountFlagBits::e4
                            : vk::SampleCountFlagBits::e1;
  test.fragment_code = {
      EncodeVop2(0x08, 2, Vgpr(0), 0), // I squared
      EncodeVop2(0x08, 3, Vgpr(1), 1), // J squared
      EncodeExp0(0x00, 0xf), EncodeExp1(2, 3, 0, 1)};
  AppendEnd(&test.fragment_code);
  // The triangle gives I=x/2, J=y/2. Squaring before the 4x resolve
  // distinguishes sample interpolation (21/256) from pixel-center (1/16).
  const auto square = std::bit_cast<u32>(multisample ? 21.0f / 256.0f : 1.0f / 16.0f);
  test.expected_pixel = {square, square, 0x3e800000u, 0x3e800000u};
  test.opcodes = {ShaderOpcode::V_MUL_F32, ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
  return test;
}

GraphicsCase GraphicsPackedHalfInputAlias(bool second_weights) {
  auto test = GraphicsPackedHalfCentroid();
  test.name = second_weights ? "GraphicsPackedHalfAliasSecondWeights"
                            : "GraphicsPackedHalfInputAlias";
  test.pixel_custom_interpolation_mask = 2u;
  test.pixel_interpolator_settings = {0u, 0x420u};
  // The head shader reads one exported vec4 through two logical inputs:
  // ordinary interpolation and raw per-vertex packed-half interpolation.
  // Move the captured raw attr0.x loads to logical attr1.x, still mapped to 0.
  for (const auto word : {0u, 1u, 6u}) {
    test.fragment_code[word] |= 1u << 10u;
  }
  test.fragment_code.resize(test.fragment_code.size() - 3u);
  test.fragment_code.push_back(EncodeVintrp(0x00, 16, 0, 1, 0));
  test.fragment_code.push_back(EncodeVintrp(0x01, 16, 0, 1, 1));
  AppendVMovLiteral(&test.fragment_code, 17, 0x3f800000u);
  test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
  test.fragment_code.push_back(EncodeExp1(14, 15, 16, 17));
  AppendEnd(&test.fragment_code);
  // The smooth component (2,4,8) must vary independently of packed UVs.
  test.vertices[3] = 0x40000000u;
  test.vertices[9] = 0x40800000u;
  test.vertices[15] = 0x41000000u;
  if (second_weights) {
    // Change the right vertex from x=7 to x=3: weights become (1/2,1/4,1/4).
    test.vertices[6] = 0x40400000u;
    test.expected_pixel = {0x3f800000u, 0x3fe00000u, 0x40800000u, 0x3f800000u};
  } else {
    test.expected_pixel = {0x3f600000u, 0x3fd00000u, 0x40700000u, 0x3f800000u};
  }
  test.opcodes.insert(test.opcodes.end(),
                      {ShaderOpcode::V_INTERP_P1_F32, ShaderOpcode::V_INTERP_P2_F32,
                       ShaderOpcode::V_MOV_B32});
  return test;
}

GraphicsCase GraphicsSmoothRawInputAlias() {
  auto test = GraphicsPackedHalfInputAlias(false);
  test.name = "GraphicsSmoothRawInputAlias";
  test.pixel_perspective_centroid_vgpr = UINT32_MAX;
  // No explicit barycentric builtin: alias promotion must discover that the
  // ordinary input needs weights while the raw alias returns vertex zero.
  test.fragment_code = {EncodeVintrp(0x00, 0, 0, 1, 0),
                        EncodeVintrp(0x01, 0, 0, 1, 1),
                        EncodeVintrp(0x02, 1, 1, 1, 2),
                        EncodeExp0(0x00, 0xf), EncodeExp1(0, 1, 0, 1)};
  AppendEnd(&test.fragment_code);
  test.expected_pixel = {0x40700000u, 0x40000000u, 0x40700000u, 0x40000000u};
  test.opcodes = {ShaderOpcode::V_INTERP_P1_F32, ShaderOpcode::V_INTERP_P2_F32,
                  ShaderOpcode::V_INTERP_MOV_F32, ShaderOpcode::EXP,
                  ShaderOpcode::S_ENDPGM};
  return test;
}

GraphicsCase GraphicsSmoothFlatInputAlias(bool flat_first, bool provoking_last = false) {
  auto test = GraphicsSmoothRawInputAlias();
  test.name = provoking_last
                  ? (flat_first ? "GraphicsFlatSmoothAliasLast" : "GraphicsSmoothFlatAliasLast")
                  : (flat_first ? "GraphicsFlatSmoothInputAlias" : "GraphicsSmoothFlatInputAlias");
  test.pixel_custom_interpolation_mask = 0;
  test.provoking_vtx_last = provoking_last;
  test.pixel_interpolator_settings = flat_first ? std::vector<u32>{0x400u, 0u}
                                                : std::vector<u32>{0u, 0x400u};
  const u32 smooth = flat_first ? 1u : 0u;
  const u32 flat = smooth ^ 1u;
  // One export carries smooth values (2,4,8) and packed light IDs (5,7,11).
  // Flat P0 preserves the ID bits; flat P1/P2 uses the same provoking vertex.
  test.vertices[4] = 0x3f800005u;
  test.vertices[10] = 0x3f800007u;
  test.vertices[16] = 0x3f80000bu;
  test.fragment_code = {EncodeVintrp(0x00, 0, smooth, 1, 0),
                        EncodeVintrp(0x01, 0, smooth, 1, 1),
                        EncodeVintrp(0x02, 1, flat, 2, 2),
                        EncodeVop2(0x1b, 1, InlineU32(63), 1),
                        EncodeVop1(0x06, 1, Vgpr(1)),
                        EncodeVintrp(0x00, 2, flat, 1, 0),
                        EncodeVintrp(0x01, 2, flat, 1, 1),
                        EncodeExp0(0x00, 0xf), EncodeExp1(0, 1, 2, 0)};
  AppendEnd(&test.fragment_code);
  test.expected_pixel = {0x40700000u, provoking_last ? 0x41300000u : 0x40a00000u,
                         provoking_last ? 0x41000000u : 0x40000000u, 0x40700000u};
  test.opcodes.insert(test.opcodes.end(),
                      {ShaderOpcode::V_AND_B32, ShaderOpcode::V_CVT_F32_U32});
  return test;
}

GraphicsCase GraphicsAncillaryLayer(bool front_face) {
  GraphicsCase test;
  test.name = front_face ? "GraphicsAncillaryAfterFrontFace" : "GraphicsAncillaryLayer";
  test.pixel_ancillary = true;
  test.pixel_front_face = front_face;
  test.layers = 8;
  const auto ancillary = Vgpr(front_face ? 3 : 2);
  test.fragment_code.push_back(EncodeSop1(0x04, 8, 126));
  test.fragment_code.push_back(EncodeSop1(0x0a, 126, 126));
  AppendVop3(&test.fragment_code, 0x148, 4, ancillary, InlineU32(16), InlineU32(11));
  AppendVop3(&test.fragment_code, 0x149, 5, ancillary, InlineU32(16), InlineU32(3));
  test.fragment_code.push_back(EncodeSop1(0x04, 126, 8));
  AppendVMovU32(&test.fragment_code, 6, test.layers);
  test.fragment_code.push_back(EncodeVopc(0xc1, Vgpr(4), 6)); // Dynamic layer < count.
  test.fragment_code.push_back(EncodeSop1(0x04, 126, 106));
  test.fragment_code.push_back(EncodeVop1(0x06, 0, Vgpr(4))); // float(unsigned layer)
  test.fragment_code.push_back(EncodeVop1(0x05, 1, Vgpr(5))); // float(signed low 3 bits)
  AppendVMovLiteral(&test.fragment_code, 2, 0);
  AppendVMovLiteral(&test.fragment_code, 3, 0x3f800000u);
  test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
  test.fragment_code.push_back(EncodeExp1(0, 1, 2, 3));
  AppendEnd(&test.fragment_code);
  test.opcodes = {ShaderOpcode::V_BFE_U32, ShaderOpcode::V_BFE_I32,
                  ShaderOpcode::V_CVT_F32_U32, ShaderOpcode::V_CVT_F32_I32,
                  ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
                  ShaderOpcode::S_WQM_B64, ShaderOpcode::V_CMP_LT_U32,
                  ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
  for (u32 layer = 0; layer < test.layers; ++layer) {
    const auto signed_bits = layer < 4 ? static_cast<int>(layer) : static_cast<int>(layer) - 8;
    test.expected_pixel.insert(test.expected_pixel.end(),
        {std::bit_cast<u32>(static_cast<float>(layer)),
         std::bit_cast<u32>(static_cast<float>(signed_bits)), 0, 0x3f800000u});
  }
  return test;
}

GraphicsCase GraphicsAncillarySampleId() {
  GraphicsCase test;
  test.name = "GraphicsAncillarySampleId";
  test.pixel_ancillary = true;
  test.samples = vk::SampleCountFlagBits::e4;
  AppendVop3(&test.fragment_code, 0x148, 4, Vgpr(2), InlineU32(8), InlineU32(4));
  AppendVop3(&test.fragment_code, 0x149, 5, Vgpr(2), InlineU32(8), InlineU32(2));
  AppendVop3(&test.fragment_code, 0x148, 6, Vgpr(2), InlineU32(9), InlineU32(1));
  test.fragment_code.push_back(EncodeVop1(0x06, 0, Vgpr(4)));
  test.fragment_code.push_back(EncodeVop1(0x05, 1, Vgpr(5)));
  test.fragment_code.push_back(EncodeVop1(0x06, 2, Vgpr(6)));
  AppendVMovLiteral(&test.fragment_code, 3, 0x3f800000u);
  test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
  test.fragment_code.push_back(EncodeExp1(0, 1, 2, 3));
  AppendEnd(&test.fragment_code);
  test.opcodes = {ShaderOpcode::V_BFE_U32, ShaderOpcode::V_BFE_I32,
                  ShaderOpcode::V_CVT_F32_U32, ShaderOpcode::V_CVT_F32_I32,
                  ShaderOpcode::V_MOV_B32, ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
  // Average samples 0..3, their signed low two bits, and their second bit.
  test.expected_pixel = {0x3fc00000u, 0xbf000000u, 0x3f000000u, 0x3f800000u};
  return test;
}

GraphicsCase GraphicsFlatInterpolatorExport() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeVintrp(0x02, 0, 0, 0, 2));
  AppendVMovLiteral(&code, 1, 0x00000000u);
  AppendVMovLiteral(&code, 2, 0x00000000u);
  AppendVMovLiteral(&code, 3, 0x3f800000u);
  code.push_back(EncodeExp0(0x00, 0xf));
  code.push_back(EncodeExp1(0, 1, 2, 3));
  AppendEnd(&code);

  GraphicsCase test;
  test.name = "GraphicsFlatInterpolatorExport";
  test.fragment_code = code;
  test.expected_pixel = {0x3e800000u, 0x00000000u, 0x00000000u, 0x3f800000u};
  test.opcodes = {O::V_INTERP_MOV_F32, O::V_MOV_B32, O::EXP, O::S_ENDPGM};
  test.pixel_interpolator_settings = {0x00000400u};
  test.vertices = {
      0xbf800000u, 0xbf800000u, 0x3e800000u, 0x00000000u, 0x00000000u,
      0x3f800000u, 0x40400000u, 0xbf800000u, 0x3f400000u, 0x00000000u,
      0x00000000u, 0x3f800000u, 0xbf800000u, 0x40400000u, 0x3e800000u,
      0x00000000u, 0x00000000u, 0x3f800000u,
  };
  return test;
}

GraphicsCase GraphicsDsAddtidScratchExport() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSMovB32(124, InlineU32(0)));
  AppendVMovLiteral(&code, 0, 0x3f000000u);
  code.push_back(EncodeDs0(0xb0, 0));
  code.push_back(EncodeDs1(0, 0, 0));
  code.push_back(EncodeDs0(0xb1, 0));
  code.push_back(EncodeDs1(4, 0, 0));
  code.push_back(EncodeExp0(0x00, 0xf));
  code.push_back(EncodeExp1(4, 4, 4, 4));
  AppendEnd(&code);

  return {"GraphicsDsAddtidScratchExport",
          code,
          {0x3f000000u, 0x3f000000u, 0x3f000000u, 0x3f000000u},
          {O::S_MOV_B32, O::V_MOV_B32, O::DS_WRITE_ADDTID_B32,
           O::DS_READ_ADDTID_B32, O::EXP, O::S_ENDPGM}};
}

GraphicsCase GraphicsDirectSgprPushConstantExport() {
  using O = ShaderOpcode;

  const std::vector<u32> values = {0x3e800000u, 0x3f000000u, 0x3f400000u,
                                   0x3f800000u};

  std::vector<u32> code;
  code.push_back(EncodeVop1(0x01, 0, 0));
  code.push_back(EncodeVop1(0x01, 1, 1));
  code.push_back(EncodeVop1(0x01, 2, 2));
  code.push_back(EncodeVop1(0x01, 3, 3));
  code.push_back(EncodeExp0(0x00, 0xf));
  code.push_back(EncodeExp1(0, 1, 2, 3));
  AppendEnd(&code);

  GraphicsCase test;
  test.name = "GraphicsDirectSgprPushConstantExport";
  test.fragment_code = code;
  test.expected_pixel = values;
  test.opcodes = {O::V_MOV_B32, O::EXP, O::S_ENDPGM};

  for (size_t i = 0; i < values.size(); i++) {
    test.user_data[i] = values[i];
  }
  test.has_user_data = true;
  test.push_constants = values;
  return test;
}

GraphicsCase GraphicsInlineSrtScalarPromotionExport() {
  using O = ShaderOpcode;

  const std::vector<u32> values = {0x00000000u, 0x00000000u, 0x00000000u,
                                   0x3f800000u};

  std::vector<u32> code;
  AppendVop3(&code, 0x12fu, 0, 0, 1);
  AppendVop3(&code, 0x12fu, 1, 2, 3);
  code.push_back(EncodeExp0(0x00, 0xf, true, true, true));
  code.push_back(EncodeExp1(0, 1, 0, 0));
  AppendEnd(&code);

  GraphicsCase test;
  test.name = "GraphicsInlineSrtScalarPromotionExport";
  test.fragment_code = code;
  test.expected_pixel = values;
  test.opcodes = {O::V_CVT_PKRTZ_F16_F32, O::EXP, O::S_ENDPGM};

  for (size_t i = 0; i < values.size(); i++) {
    test.user_data[i] = values[i];
  }
  test.has_user_data = true;
  test.push_constants = values;
  return test;
}

GraphicsCase GraphicsNullVmExportDiscardsInactiveExec() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x3f000000u);
  AppendVMovLiteral(&code, 1, 0x3f400000u);
  AppendVMovLiteral(&code, 2, 0x3f800000u);
  AppendVMovLiteral(&code, 3, 0x40000000u);
  code.push_back(EncodeExp0(0x00, 0xf, false));
  code.push_back(EncodeExp1(0, 1, 2, 3));
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  code.push_back(EncodeExp0(0x09, 0x0, true, false, true));
  code.push_back(EncodeExp1(0, 0, 0, 0));
  AppendEnd(&code);

  return {"GraphicsNullVmExportDiscardsInactiveExec",
          code,
          {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
          {O::V_MOV_B32, O::EXP, O::S_MOV_B64, O::S_ENDPGM}};
}

GraphicsCase GraphicsMrt0OffVmExportDiscardsInactiveExec() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x3f000000u);
  AppendVMovLiteral(&code, 1, 0x3f400000u);
  AppendVMovLiteral(&code, 2, 0x3f800000u);
  AppendVMovLiteral(&code, 3, 0x40000000u);
  code.push_back(EncodeExp0(0x00, 0xf, false));
  code.push_back(EncodeExp1(0, 1, 2, 3));
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  code.push_back(EncodeExp0(0x00, 0x0, true, true, true));
  code.push_back(EncodeExp1(0, 0, 0, 0));
  AppendEnd(&code);

  return {"GraphicsMrt0OffVmExportDiscardsInactiveExec",
          code,
          {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
          {O::V_MOV_B32, O::EXP, O::S_MOV_B64, O::S_ENDPGM}};
}

GraphicsCase GraphicsFinalVmExportSupersedesEarlierVmMask() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x3f000000u);
  AppendVMovLiteral(&code, 1, 0x3f400000u);
  AppendVMovLiteral(&code, 2, 0x3f800000u);
  AppendVMovLiteral(&code, 3, 0x40000000u);
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  code.push_back(EncodeExp0(0x00, 0xf, false, false, true));
  code.push_back(EncodeExp1(0, 1, 2, 3));
  code.push_back(EncodeSMovB32(126, 193u));
  code.push_back(EncodeSMovB32(127, 193u));
  AppendVMovLiteral(&code, 4, 0x40400000u);
  AppendVMovLiteral(&code, 5, 0x40800000u);
  AppendVMovLiteral(&code, 6, 0x40a00000u);
  AppendVMovLiteral(&code, 7, 0x40c00000u);
  code.push_back(EncodeExp0(0x00, 0xf, true, false, true));
  code.push_back(EncodeExp1(4, 5, 6, 7));
  AppendEnd(&code);

  return {"GraphicsFinalVmExportSupersedesEarlierVmMask",
          code,
          {0x40400000u, 0x40800000u, 0x40a00000u, 0x40c00000u},
          {O::V_MOV_B32, O::S_MOV_B64, O::EXP, O::S_MOV_B32, O::S_ENDPGM}};
}

GraphicsCase GraphicsBranchPathFinalVmExportDiscardsInactiveExec() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  AppendVMovLiteral(&code, 0, 0x3f000000u);
  AppendVMovLiteral(&code, 1, 0x3f400000u);
  AppendVMovLiteral(&code, 2, 0x3f800000u);
  AppendVMovLiteral(&code, 3, 0x40000000u);
  code.push_back(EncodeExp0(0x00, 0xf, false));
  code.push_back(EncodeExp1(0, 1, 2, 3));
  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
  code.push_back(EncodeExp0(0x09, 0x0, true, false, true));
  code.push_back(EncodeExp1(0, 0, 0, 0));
  code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(1)));
  const auto branch_index = code.size();
  code.push_back(0);
  AppendEnd(&code);
  const auto later_export_index = code.size();
  code[branch_index] = EncodeSopp(
      0x05, static_cast<u32>(later_export_index - branch_index - 1u));
  code.push_back(EncodeSMovB32(126, 193u));
  code.push_back(EncodeSMovB32(127, 193u));
  AppendVMovLiteral(&code, 4, 0x40400000u);
  AppendVMovLiteral(&code, 5, 0x40800000u);
  AppendVMovLiteral(&code, 6, 0x40a00000u);
  AppendVMovLiteral(&code, 7, 0x40c00000u);
  code.push_back(EncodeExp0(0x00, 0xf, true, false, true));
  code.push_back(EncodeExp1(4, 5, 6, 7));
  AppendEnd(&code);

  return {"GraphicsBranchPathFinalVmExportDiscardsInactiveExec",
          code,
          {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
          {O::V_MOV_B32, O::EXP, O::S_MOV_B64, O::S_CMP_EQ_U32,
           O::S_CBRANCH_SCC1, O::S_MOV_B32, O::S_ENDPGM}};
}

TestCase MultipleWorkitemsGlobalId() {
  using O = ShaderOpcode;

  std::vector<u32> code = {
      EncodeVop1(0x01, 1, 0),
      EncodeVop2(0x1a, 1, InlineU32(2), 1),
      EncodeVop2(0x25, 1, Vgpr(0), 1),
      EncodeVop2(0x25, 2, InlineU32(64), 1),
      EncodeVop2(0x1a, 3, InlineU32(2), 1),
  };
  AppendBufferStoreDword(&code, 2, 3);
  AppendEnd(&code);

  TestCase test;
  test.name = "MultipleWorkitemsGlobalId";
  test.code = code;
  test.expected = {64, 65, 66, 67, 68, 69, 70, 71};
  test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
  test.compute_info.threads_num[0] = 4;
  test.compute_info.threads_num[1] = 1;
  test.compute_info.threads_num[2] = 1;
  test.compute_info.group_id[0] = true;
  test.compute_info.thread_ids_num = 1;
  test.compute_info.workgroup_register = 0;
  test.has_compute_info = true;
  test.dispatch_x = 2;
  return test;
}

TestCase DispatcherIrreducibleControlFlow() {
  using O = ShaderOpcode;

  std::vector<u32> code;
  code.push_back(EncodeSopp(0x05, 5)); // entry -> B, fallthrough A
  AppendVMovU32(&code, 0, 7);
  AppendStoreVgpr(&code, 0, 0);
  code.push_back(EncodeSopp(0x04, 1)); // A exits while SCC is initially zero
  code.push_back(EncodeSopp(0x02, 0xfffau)); // B -> A
  AppendEnd(&code);

  TestCase test;
  test.name = "DispatcherIrreducibleControlFlow";
  test.code = code;
  test.expected = {7};
  test.opcodes = {O::S_CBRANCH_SCC1, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
                  O::S_CBRANCH_SCC0, O::S_BRANCH,  O::S_ENDPGM};
  return test;
}

#include "ShaderRayTracingGpuTests.inc"

std::vector<TestCase> MakeCases() {
  std::vector<TestCase> cases;
  cases.reserve(128);
  auto AddCase = [&cases](TestCase (*factory)()) {
    cases.push_back(factory());
  };
  cases.push_back(BufferWorkgroupPublication(32));
  cases.push_back(BufferWorkgroupPublication(64));
  cases.push_back(BufferWorkgroupPublication(32, true));
  cases.push_back(BufferWorkgroupPublication(32, false, true));
  cases.push_back(BufferWorkgroupPublication(64, false, true));

  AddCase(IntegerAddSubMul);
  AddCase(BitwiseOps);
  AddCase(Shifts);
  AddCase(ScalarZeroShiftWithRuntimeCount);
  AddCase(ExactPushConstantExtent);
  AddCase(ScalarShiftCountsMaskLowBits);
  AddCase(Rdna2ScalarOpcodes);
  AddCase(ScalarTrapDisabledFallsThroughExactRaw);
  AddCase(ScalarWaitcntDepctrCapturedVmVsrc);
  AddCase(ScalarExtendedArithmetic);
  AddCase(ScalarArithmeticSccCarryBorrowOverflow);
  AddCase(ScalarMinMaxSccComparisonEdges);
  AddCase(ScalarAbsI32UpdatesScc);
  AddCase(ScalarAbsdiffBitset64AndSetprio);
  AddCase(ScalarShiftLeftAddSccCarryEdges);
  AddCase(ScalarCompareOps);
  AddCase(ScalarShiftAddAndMaskOps);
  AddCase(ScalarNotB64UpdatesScc);
  AddCase(ScalarQuadmaskB64);
  AddCase(ScalarFlbitI32B64Gpu);
  AddCase(ScalarFlbitI32B32Gpu);
  AddCase(ScalarSaveExecOps);
  AddCase(ScalarOrn2SaveexecUsesSourceOrNotExec);
  cases.push_back(ScalarOrn2SaveexecB32(32, 32));
  cases.push_back(ScalarOrn2SaveexecB32(64, 64));
  cases.push_back(ScalarOrn2SaveexecB32(32, 4));
  cases.push_back(ScalarSubvectorLoops(32));
  cases.push_back(ScalarSubvectorLoops(64));
  AddCase(ScalarGetpcWritesNextInstructionPc);
  AddCase(ScalarBitfieldPack);
  AddCase(ScalarZeroBitOps);
  AddCase(ScalarBitcmpB64DynamicOperands);
  AddCase(ScalarBitcmpB64IntegerConstants);
  AddCase(ScalarBrevB32PreservesScc);
  for (const u32 bits : {8u, 16u}) {
    cases.push_back(ScalarSextMasks(bits, 32));
    cases.push_back(ScalarSextMasks(bits, 64));
  }
  AddCase(ScalarSextConstantsAndAliases);
  cases.push_back(ScalarBrevB64OperandsAndMasks(32));
  cases.push_back(ScalarBrevB64OperandsAndMasks(64));
  AddCase(ScalarBfeI32CapturedRawSignExtends);
  AddCase(BitfieldExtractWidthPastEndEdges);
  AddCase(Scalar64BitOps);
  AddCase(ScalarDynamic64BitOps);
  cases.push_back(ScalarAshrI64Edges(false));
  cases.push_back(ScalarAshrI64Edges(true));
  AddCase(ScalarAshrI64OperandsAndAliases);
  cases.push_back(ScalarAshrI64Masks(32));
  cases.push_back(ScalarAshrI64Masks(64));
  AddCase(VectorDynamicU64ShiftEdges);
  AddCase(ScalarConditionalMoveB64);
  AddCase(ScalarConditionalMoveB64PreservesMasks);
  AddCase(ScalarAndn2B64SccUsesMaskShadow);
  AddCase(ScalarMaskHighWriteInvalidatesProvenance);
  cases.push_back(ScalarSelectB64PreservesMaskProvenance(32));
  cases.push_back(ScalarSelectB64PreservesMaskProvenance(64));
  AddCase(ScalarWqmB64SelectsSccDomain);
  AddCase(ScalarWqmB64PreservesPartialMasks);
  cases.push_back(ScalarWqmB32Masks(32));
  cases.push_back(ScalarWqmB32Masks(64));
  AddCase(ScalarMaskProvenanceOverlapAndMixedBinary);
  cases.push_back(ScalarB32MaskDataflow(32));
  cases.push_back(ScalarB32MaskDataflow(64));
  AddCase(ScalarLiteral);
  AddCase(VectorMoves);
  AddCase(VectorVop3MoveAppliesFloatSourceModifiers);
  AddCase(VectorIntegerOps);
  AddCase(VectorIntegerExtremaRuntimeEdges);
  AddCase(VectorFfbhI32NativeAndVop3OnGpu);
  AddCase(Vop1SdwaBfrevSourceSelectors);
  AddCase(Vop1SdwaFfbhCapturedScalarLowWordSource);
  AddCase(Vop1SdwaFfblCapturedHighWordSource);
  AddCase(Vop1SdwaNotCapturedByte0Source);
  AddCase(Vop1SdwaNotPreservesHighWordDestination);
  AddCase(Vop1SdwaNotPartialSourcesAndDestinations);
  cases.push_back(Vop2SdwaBitwiseByteDestinations(32));
  cases.push_back(Vop2SdwaBitwiseByteDestinations(64));
  AddCase(Vop1SdwaMovByteDestinations);
  cases.push_back(Vop1SdwaMovByteSources(32));
  cases.push_back(Vop1SdwaMovByteSources(64));
  AddCase(Vop2SdwaSubNcExactByte2Destination);
  AddCase(Vop2SdwaAddNcCapturedHighWordDestination);
  AddCase(Vop2SdwaAshrrevCapturedWord0SignExtends);
  cases.push_back(Vop2SdwaMaxI32CapturedHighWord(32));
  cases.push_back(Vop2SdwaMaxI32CapturedHighWord(64));
  cases.push_back(Vop2SdwaMinI32CapturedByte0(32));
  cases.push_back(Vop2SdwaMinI32CapturedByte0(64));
  cases.push_back(Vop2SdwaMulI24Destinations(32));
  cases.push_back(Vop2SdwaMulI24Destinations(64));
  cases.push_back(Vop2SdwaMulU24Destinations(32));
  cases.push_back(Vop2SdwaMulU24Destinations(64));
  AddCase(Vop2SdwaLshrrevCapturedByte1Source);
  AddCase(Vop2SdwaSubNcPreservesByteAndWordDestinations);
  AddCase(Vop3CvtPkI16I32Captured);
  AddCase(Vop3MulLoU16CapturedAndSelectors);
  AddCase(Vop3MadI16CapturedSelectorsAndSaturation);
  AddCase(Vop3Min3U16CapturedAndSelectors);
  AddCase(Vop3Med3I16Captured);
  AddCase(Vop2SdwaMinU32PreservesWordDestination);
  AddCase(VectorShiftCountsMaskLowBits);
  AddCase(VectorVop3LshrrevB64Captured);
  AddCase(VectorVop3LshlrevB64Captured);
  AddCase(VectorVop3IntegerOps);
  AddCase(VectorBfeI32SignExtendsField);
  AddCase(VectorAlignByteUsesTwoBitByteOffset);
  AddCase(VectorCarryAndBitCountOps);
  AddCase(VectorMbcntUsesThreadMask);
  AddCase(VectorAddcWritesPerLaneCarryOut);
  AddCase(VectorAddcUsesPerLaneCarryIn);
  AddCase(VectorSubCoCiU32CompactAndVop3);
  AddCase(VectorSubrevCoCiU32ExactRawOnGpu);
  AddCase(VectorVop3BSubrevCoCiUsesEncodedMasks);
  AddCase(VectorVop3BCarryOutWritesSgprMask);
  AddCase(VectorVop3BCarryOutUsesEncodedSdst);
  AddCase(VectorVop3BSubCoU32UsesRdna2Opcode310);
  AddCase(VectorMadU64U32UnsignedCarryOut);
  AddCase(VectorLaneAndPackedOps);
  AddCase(Vop2PkFmacF16AccumulatesPackedHalvesIndependently);
  AddCase(Vop2PkFmacF16DppNegatesBothPackedHalves);
  AddCase(Vop3pOpselHiUsesArchitecturalSourceBits);
  AddCase(Vop3pIntegerNegationCapturedAndSelectedHalves);
  AddCase(Vop3pInt16Ops);
  AddCase(CvtPkU8F32PacksSelectedByte);
  AddCase(CvtPkrtzF16F32SubnormalRoundsTowardZero);
  AddCase(CvtPkrtzF16F32SdwaAndOutputModifiers);
  AddCase(PackedMinMaxF16NanAndSignedZeroEdges);
  AddCase(VectorMinMaxF16Ops);
  AddCase(VectorCvtF16U16ByteSource);
  AddCase(VectorCvtU16F16Sdwa);
  AddCase(NativeAndSdwa16BitDestinationWrites);
  AddCase(VectorMinMaxMed3F16Ops);
  AddCase(VectorSpecialF16Ops);
  AddCase(VectorFractF16CapturedAndEdges);
  AddCase(VectorFractF16Modifiers);
  AddCase(VectorCosF16CapturedSdwaAndEdges);
  AddCase(VectorSinF16SdwaAndEdges);
  AddCase(VectorWritelaneIgnoresExecMask);
  AddCase(VectorReadlaneFromInactiveWrittenLane);
  AddCase(VectorLaneWave32RuntimeSelectorWraps);
  AddCase(VectorReadlaneSelectsTwoKeysWithinWave);
  AddCase(VectorPermlanex16);
  AddCase(VectorPermlane16FetchInactiveZero);
  AddCase(VectorPermlane16FetchInactiveFi);
  cases.push_back(VectorDpp8Captured(false));
  for (const u32 wave_size : {32u, 64u})
    for (const bool fi : {false, true})
      cases.push_back(VectorDpp8Captured(true, wave_size, fi));
  AddCase(VectorDppQuadPermuteReverse);
  AddCase(VectorDppRowXmask);
  cases.push_back(VectorDppRowShare(32));
  cases.push_back(VectorDppRowShare(64));
  AddCase(VectorDppBankMaskPreservesDestination);
  AddCase(VectorDppBoundsControlZeroPreservesDestination);
  AddCase(Vop3FmacF32NegatedSourceAccumulates);
  AddCase(VectorMadF32RoundsProduct);
  AddCase(VectorMadF32FlushesDenorms);
  AddCase(Vop3LdexpSourceModifier);
  AddCase(Vop1MoveRelSource);
  AddCase(Vop1MoveRelDestination);
  AddCase(VectorFloatSpecialOps);
  AddCase(CubeIdCapturedNegationAndOutputScale);
  AddCase(MadMixF16LiteralHalfSourceUsesOpsel);
  AddCase(MadMixF16NegHiIsAbsAndNegIsIndependent);
  AddCase(VectorVop3FmaF16UsesRdna2Opcode34b);
  AddCase(FloatInlineConstantF16UsesNumericValue);
  AddCase(Vop3Int16Ops);
  AddCase(VectorFloatControlContractPreservesInfNan);
  AddCase(VectorFloatArithmeticOps);
  AddCase(VectorMinMaxF32NanAndSignedZeroEdges);
  AddCase(VectorMed3F32NanUsesMin3Path);
  AddCase(VectorFloatConversionOps);
  AddCase(VectorFrexpF32Edges);
  AddCase(CvtF32ToIntSaturatesNaNAndOutOfRange);
  AddCase(VectorSpecialF32FlushesDenormalInputs);
  AddCase(VectorRcpIflagF32IntegerReciprocal);
  AddCase(VectorCompareF64Edges);
  AddCase(VectorCompareInteger64Edges);
  cases.push_back(VectorCompareExecWaveMasks(32));
  cases.push_back(VectorCompareExecWaveMasks(64));
  cases.push_back(VectorCompare64WaveMasks(32));
  cases.push_back(VectorCompare64WaveMasks(64));
  cases.push_back(VectorCompare64WaveMasks(32, true));
  cases.push_back(VectorCompare64WaveMasks(64, true));
  AddCase(VectorF64CapturedScreenSpaceShadows);
  AddCase(VectorFractF64CapturedAndEdges);
  AddCase(VectorMinMaxF64CapturedAndEdges);
  AddCase(VectorRoundF64EdgesAndModifiers);
  AddCase(VectorFractF64CapturedChainRuntimeExec);
  AddCase(VectorF64ModesModifiersAndExec);
  AddCase(VectorF64WideningConversions);
  AddCase(VectorSinCosMaxFiniteSpecialCases);
  AddCase(VectorCompareF32ExposureGuard);
  for (const auto mode : {0xc0u, 0xe0u}) {
    cases.push_back(VectorCompareF32DenormalModes(mode));
  }
  AddCase(VectorCompareOps);
  AddCase(VectorVop3CompareEqI64OnGpu);
  AddCase(VectorVop3CompareEqU64OnGpu);
  AddCase(VectorVop3CompareGtU64OnGpu);
  AddCase(VectorVopcCompareLtU64OnGpu);
  AddCase(VectorVop3CompareNeU64OnGpu);
  AddCase(VectorVopcCmpxNeU64CapturedExecMask);
  AddCase(VectorVop3CmpxNeI64CapturedExecMask);
  AddCase(VectorCompareClassF32);
  AddCase(VectorCompareClassF32ImmediateAndDynamicMasks);
  AddCase(VectorVopcSdwaCmpxClassF32CapturedExecMask);
  cases.push_back(VectorCmpClassF16(32));
  cases.push_back(VectorCmpClassF16(64));
  cases.push_back(VectorCmpxClassF16(32));
  cases.push_back(VectorCmpxClassF16(64));
  AddCase(VectorCompareF16Ops);
  AddCase(VectorCompareI16U16Ops);
  AddCase(Wave32VccMasksPreserveOtherHalf);
  AddCase(Vop2SdwaCndmaskSourceModifier);
  AddCase(Vop2SdwaCndmaskCapturedByte3Source);
  AddCase(Vop3CndmaskUsesSgprMaskLaneBits);
  AddCase(Vop3CndmaskAllowsDataSourceModifier);
  AddCase(VectorCompareExecOps);
  AddCase(VectorVopcCmpxOrderedCapturedExecMask);
  AddCase(VectorVop3FloatCompareNegSourceModifier);
  AddCase(VectorVop3CmpxWritesExecMask);
  AddCase(VectorVopcSdwaCmpxWritesExecMask);
  AddCase(VectorVopcCmpxGtU16CapturedSdwaExecMask);
  AddCase(VectorVopcCmpxLtU16CapturedSdwaExecMask);
  AddCase(VectorVopcCmpxLtI16CapturedSdwaExecMask);
  for (u32 wave_size : {32u, 64u}) {
    cases.push_back(VectorVopcCmpxLtI16WaveMasks(wave_size));
  }
  AddCase(VectorVopcCmpxEqU16SdwaCompactVop3ExecMask);
  AddCase(VectorVopcCmpxNeU16SdwaCompactVop3ExecMask);
  AddCase(VectorVopcCmpNgtF16CapturedSdwaAndEdges);
  AddCase(VectorVopcCmpNltF16CapturedSdwaAndEdges);
  AddCase(VectorVopcCmpxNgtF16CapturedSdwaExecMask);
  AddCase(VectorVopcCmpxNleF16CapturedSdwaExecMask);
  AddCase(VectorCompareInvertedMaskSelect);
  AddCase(BranchSelect);
  AddCase(SimpleLoop);
  AddCase(SharedReturnKeepsSelectedValues);
  AddCase(SiblingSharedExitKeepsCapturedConditions);
  AddCase(BranchVccnzUsesWaveMask);
  AddCase([] { return ScalarBranchRestoresInactiveLanes(32, 32); });
  AddCase([] { return ScalarBranchRestoresInactiveLanes(64, 64); });
  AddCase([] { return ScalarBranchRestoresInactiveLanes(32, 4); });
  AddCase([] { return ScalarBranchRestoresInactiveLanes(64, 4); });
  AddCase(BranchVccnzUsesCarryProducedWaveMask);
  AddCase(ScalarMemRealtimeCapturedPlaceholder);
  AddCase(ScalarMemoryLoadVariants);
  AddCase(ScalarBufferOffsetAlignmentAndCarry);
  AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset);
  AddCase(ScalarLoadAlignsComponentsAndMasksAddress);
  AddCase(ScalarLoadAlignsDynamicBase);
  cases.push_back(ScalarBufferFromLoopReadlane(32));
  cases.push_back(ScalarBufferFromLoopReadlane(64));
  AddCase(BufferLoadStore);
  AddCase(BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset);
  AddCase(BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset);
  AddCase(BufferLoadDwordNoAddressFlagsIgnoresVaddr);
  AddCase(BufferLoadDwordIdxenUsesDescriptorStride);
  AddCase(BufferStoreDwordIdxenUsesDescriptorStride);
  AddCase(BufferStoreDwordAppliesHostOffset);
  cases.push_back(BufferOffsetsUsePackedWords(false));
  cases.push_back(BufferOffsetsUsePackedWords(true));
  AddCase(BufferLoadVariants);
  for (const bool high : {false, true}) cases.push_back(BufferLoadShortD16Captured(high));
  for (const u32 component : {4u, 2u, 0u}) {
    cases.push_back(BufferSubwordLoadsAtHostOffset(2, component, false));
  }
  cases.push_back(BufferSubwordLoadsAtHostOffset(2, 4, true));
  cases.push_back(BufferSubwordLoadsAtHostOffset(1, 5, false));
  cases.push_back(BufferSubwordLoadsAtHostOffset(1, 5, true));
  AddCase(BufferSubwordHostOffsetOverflow);
  cases.push_back(BufferSubwordStoresAtHostOffset(2));
  cases.push_back(BufferSubwordStoresAtHostOffset(1));
  AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress);
  AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress);
  AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress);
  AddCase(BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail);
  AddCase(BufferLoadsGpuSelectedDescriptors);
  AddCase(BufferLoadDwordGpuSelectedDescriptors);
  AddCase(BufferLoadDwordx3GpuSelectedDescriptors);
  AddCase(BufferLoadFormatXGpuSelectedDescriptors);
  AddCase(BufferLoadFormatXGpuSelectedDescriptors64);
  AddCase(BufferStoreDwordx4DropsOnlyOutOfBoundsTail);
  AddCase(BufferLoadFormatXyzwRejectsPartialRecord);
  AddCase(BufferStoreFormatXyzwDropsPartialRecord);
  AddCase(BufferLoadFormatXRejectsPartialRecord);
  AddCase(BufferStoreFormatXRejectsPartialRecord);
  AddCase(BufferLoadFormatXyRejectsPartialRecord);
  AddCase(BufferStoreFormatXyRejectsPartialRecord);
  AddCase(BufferStoreVariants);
  AddCase(BufferFormatVariants);
  AddCase(BufferLoadFormatXyzwSnapshotsOverlappingAddress);
  AddCase(BufferLoadFormatXyzwResource32_32FloatAppliesXy01);
  AddCase(BufferLoadFormatXyzwResource323232FloatRepeatsSource);
  AddCase(BufferLoadFormatXyzwResource8_8_8_8UintAppliesWzy1);
  AddCase(BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress);
  AddCase(BufferFormatStoreVariants);
  AddCase(BufferStoreFormatXResource16UintWritesHalfword);
  AddCase(BufferStoreFormatXResource16UintPreservesAdjacentLanes);
  AddCase(BufferStoreFormatXyzwPackedUnormSkinningVectors);
  AddCase(BufferStoreFormatXZeroFillsPackedRecord);
  AddCase(BufferStoreFormatXyzwFloat16ConvertsComponents);
  AddCase(BufferStoreFormatXyzwSnorm16CapturedSkinningVectors);
  AddCase(BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords);
  for (auto &test : BufferLoadFormatD16XCases()) {
    cases.push_back(std::move(test));
  }
  for (auto &test : BufferStoreFormatD16XCases()) {
    cases.push_back(std::move(test));
  }
  for (auto &test : FormattedStoreConversionCases()) {
    cases.push_back(std::move(test));
  }
  AddCase(BufferLoadFormatXResource8UintZeroExtendsByte);
  AddCase(BufferLoadFormatXyResource88UintExtractsBytes);
  AddCase(BufferLoadFormatXyResource8888UnormConvertsFirstTwoComponents);
  AddCase(BufferStoreFormatXyResource88UintWritesBytes);
  AddCase(BufferStoreFormatXyzResource3232UintWritesTwoDwords);
  AddCase(BufferStoreFormatXyzwResource323232UintWritesThreeDwords);
  AddCase(BufferStoreFormatXyResource32UintWritesOneDword);
  AddCase(BufferStoreFormatXyzResource8UintWritesOneByte);
  AddCase(BufferStoreFormatXAddTidUsesLaneIndex);
  AddCase(BufferStoreFormatXDropsOutOfRangeRecord);
  AddCase(TBufferLoadVariants);
  AddCase([] { return TBufferCapturedZeroStrideOob(); });
  AddCase([] { return TBufferCapturedZeroStrideOob(true); });
  AddCase([] { return TBufferCapturedZeroStrideOob(false, true); });
  AddCase(BufferZeroStrideOobFormatsAndWidths);
  AddCase(TBufferLoadFormatXyzwSnapshotsOverlappingAddress);
  AddCase(TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress);
  AddCase(TBufferLoadFormatX8UintZeroExtendsByte);
  AddCase(TBufferLoadFormatX8888UintExtractsFirstByte);
  AddCase(TBufferLoadFormatXIdxenUsesDescriptorStride);
  AddCase(TBufferLoadFormatX16FloatConvertsToFloat);
  AddCase(TBufferStoreFormatX8UintWritesOneByte);
  AddCase(TBufferStoreFormatXSintWritesSubDword);
  AddCase(TBufferLoadFormatXSintSignExtendsSubDword);
  AddCase(TBufferLoadFormatXy1616IntegerComponents);
  AddCase(TBufferStoreFormatXy1616IntegerComponents);
  AddCase(TBufferLoadFormatXyz16161616UintLoadsHalfwords);
  AddCase(TBufferLoadFormatXy1616UnormConvertsToFloat);
  AddCase(TBufferLoadFormatXy88UnormConvertsToFloat);
  AddCase(TBufferLoadFormatXy88SnormConvertsToFloat);
  AddCase(TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents);
  AddCase(TBufferLoadFormatXyzw8888UintExtractsBytes);
  AddCase(TBufferLoadFormatXyzw1010102SnormConvertsToFloat);
  AddCase(TBufferLoadFormatXyz111110FloatUnpacks);
  AddCase(TBufferLoadFormatXyzw3232FloatZerosMissingComponents);
  AddCase(TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents);
  AddCase(TBufferStoreFormatXy88IntegerComponents);
  AddCase(TBufferLoadFormatXy88IntegerComponents);
  AddCase(TBufferStoreVariants);
  cases.push_back(FlatLoadDlcCaptured(32));
  cases.push_back(FlatLoadDlcCaptured(64));
  AddCase(FlatStoreSlcCaptured);
  AddCase(FlatLoadVariants);
  AddCase(FlatSubdwordLoadsApplyByteOffset);
  cases.push_back(GlobalLoadShortD16Captured(32));
  cases.push_back(GlobalLoadShortD16Captured(64));
  AddCase(FlatLoadShortD16AddressSegments);
  AddCase(FlatVirtualAddressRebasesGuestAllocation);
  AddCase(GlobalSignedImmediateRebasesBeforeSaddr);
  AddCase(FlatSegmentIgnoresSaddrAndMasksOffsetMsb);
  AddCase(ScratchIsPrivatePerInvocation);
  cases.push_back(FlatStackApertures(32));
  cases.push_back(FlatStackApertures(64));
  cases.push_back(BvhIntersections(true, true, 1));
  cases.push_back(BvhIntersections(true, true, 0, true));
  for (bool barycentrics : {false, true}) {
    for (bool sorted : {false, true}) {
      cases.push_back(BvhIntersections(barycentrics, sorted));
    }
  }
  AddCase(DsReadWriteVariants);
  for (u32 wave_size : {32u, 64u})
    for (bool gds : {false, true})
      cases.push_back(DsReadWriteFullRegionAddress(wave_size, gds));
  AddCase(DsWriteB16D16HiCapturedUsesHighHalf);
  AddCase(DsWriteB8D16HiWritesByteTwo);
  AddCase(DsReadU16D16CapturedPreservesHighHalf);
  AddCase(DsReadU16D16HiCapturedPreservesLowHalf);
  AddCase(DsAppendConsumeUsesEncodedLdsSelector);
  cases.push_back(DsAppendAllocatesAcrossWaves(32));
  cases.push_back(DsAppendAllocatesAcrossWaves(64));
  AddCase(DsAppendUsesEncodedGdsSelector);
  for (u32 wave_size : {32, 64}) {
    cases.push_back(DsOrderedCountAddressAndExec(wave_size, false));
    cases.push_back(DsOrderedCountAddressAndExec(wave_size, true));
  }
  AddCase(DsAppendConsumeGdsRegionBounds);
  AddCase(DsGdsSubdwordAndAtomicWrites);
  AddCase(DsReadWrite2Variants);
  AddCase(DsWideReadSnapshotsOverlappingAddress);
  AddCase(DsReadWrite2EqualOffsetsUseData0);
  AddCase(DsWideLdsPartialBounds);
  AddCase(DsWideGdsPartialBounds);
  AddCase(DsAtomicNoReturnVariants);
  for (bool gds : {false, true}) {
    cases.push_back(DsMaskedOrBounds(gds));
  }
  for (u32 wave_size : {32, 64}) {
    cases.push_back(DsMaskedOrContention(wave_size));
  }
  AddCase(DsAtomicReturnVariants);
  for (bool add : {false, true}) {
    cases.push_back(DsAtomic64Bounds(add));
    for (u32 wave_size : {32, 64}) {
      cases.push_back(DsAtomic64Contention(add, wave_size));
    }
  }
  AddCase(BufferAtomicAndX2GlcAndExec);
  AddCase(ImageAtomicSignedMinMax<false>);
  AddCase(ImageAtomicSignedMinMax<true>);
  for (bool decrement : {false, true}) {
    for (bool gds : {false, true}) {
      cases.push_back(DsBoundedAtomicBoundaries(decrement, gds));
      for (u32 wave_size : {32u, 64u}) {
        cases.push_back(DsBoundedAtomicContention(decrement, gds, wave_size));
      }
    }
  }
  AddCase(DsMiscVariants);
  AddCase(DsFloatMinMaxIgnoresData1);
  AddCase([] { return DsFloatMinMaxClasses(false); });
  AddCase([] { return DsFloatMinMaxClasses(true); });
  AddCase(DsSwizzleInvalidSourceLaneZero);
  AddCase(DsPermuteCapturedExecOffsetAndWrap);
  AddCase(DsPermuteWave64UsesIndependentHalves);
  AddCase(DsBpermuteCapturedExecOffsetAndWrap);
  AddCase(DsBpermuteWave64UsesIndependentHalves);
  AddCase(Wave64CrossHalfLaneAndLds);
  AddCase(Wave64RawMasksAndScalarBranch);
  AddCase(Wave64PartialMultidimensionalWorkgroup);
  AddCase(Wave64AppendConsumeHighHalf);
  AddCase(BufferAtomicVariants);
  AddCase(BufferAtomicCmpSwapExactRaw);
  AddCase(BufferAtomicGlc0DoesNotReturnOldValue);
  AddCase(BufferAtomicFMinExactRawGlcModes);
  AddCase(BufferAtomicFMinSpecialValues);
  AddCase(BufferAtomicFMinContendedWorkgroup);
  AddCase(BufferAtomicFMaxExactRawGlcModes);
  AddCase(BufferAtomicFMaxSpecialValues);
  AddCase(BufferAtomicFMaxContendedWorkgroup);
  AddCase(ImageLoadVariants);
  AddCase(ImageLoadR32UintUsesIntegerSampledImage);
  AddCase(ImageLoadFmaskUsesNativeSampleMapping);
  AddCase(ImageLoadR32SintUsesSignedSampledImage);
  AddCase(ImageLoadPackedUintUnpacksAndSwizzles);
  AddCase(ImageSamplePackedUintConvertsSampleAndGather);
  AddCase(ImageSamplePacked8<false>);
  AddCase(ImageSamplePacked8<true>);
  AddCase(ImageSamplePacked8<false, true>);
  AddCase(ImageLoadR128IgnoresAdjacentMaskSgprs);
  AddCase(ImageLoad1DUsesScalarCoordinate);
  AddCase(ImageGather2DInstructionWith1DDescriptor);
  AddCase(ImageLoad1DArrayUsesLayerCoordinate);
  AddCase(ImageLoad1DArrayDescriptorUsesSelectedLayer);
  AddCase(ImageLoadMipUsesVaddr2Lod2D);
  AddCase(ImageLoadMipNsaUsesSelectedAddressVgprs);
  AddCase(ImageLoadA16UintCoordsOnGpu);
  AddCase(ImageGetResinfoDmaskWidthHeight);
  AddCase(ImageGetResinfoDmaskMipLevels);
  AddCase(ImageSampleAndGather);
  AddCase(ImageCubeGradientsPreserveDerivatives);
  AddCase(ImageGatherExplicitLod);
  AddCase(ImageGatherLodPerLane<0>);
  AddCase(ImageGatherLodPerLane<1>);
  AddCase(ImageGatherLodPerLane<2>);
  AddCase(ImageGatherLodPerLane<3>);
  AddCase(ImageGatherLodPerLane<4>);
  AddCase(ImageGatherLodPerLane<5>);
  AddCase(ImageGatherLodPerLane<6>);
  AddCase(ImageGatherLodPerLane<7>);
  AddCase(ImageGatherLodPerLane<8>);
  AddCase(ImageD16GatherPacksHalfPairs);
  AddCase(ImageSampleA16SamplerCoordsOnGpu);
  AddCase(ImageSampleOpcodeAliasUsesNormalCoords);
  AddCase(ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu<32>);
  AddCase(ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu<64>);
  AddCase(ImageSampleOffsetsReadFourNeighbors<32>);
  AddCase(ImageSampleOffsetsReadFourNeighbors<64>);
  AddCase(ImageSampleA16CompareBiasRdna2AddressOrder);
  AddCase(ImageGatherCompareOpcodes);
  AddCase(ImageStoreVariants);
  AddCase(ImageD16StoreUnpacksHalfPairs);
  AddCase(ImageStoreMipSelectsPpsa01340Descriptor);
  AddCase(ImageStoreSingleMipBounds<0>);
  AddCase(ImageStoreSingleMipBounds<1>);
  AddCase(ImageStoreSingleMipBounds<2>);
  AddCase(ImageStoreRgbOneUsesInverseSwizzle);
  AddCase(ImageStoreDuplicateSelectorUsesInverseSwizzle);
  AddCase(ImageStoreBgraUsesInverseSwizzle);
  AddCase(ImageStoreYzwxUsesInverseSwizzle);
  AddCase(ImageStoreR32FloatUsesFormatlessStorageImage);
  AddCase(ImageStoreR32SintUsesRawUintView);
  AddCase(ImageStoreR32UintUsesFormatlessStorageImage);
  AddCase(ImageStorePackedUintSaturatesChannels);
  AddCase(ImageStorePackedUintHonorsSparseDmask);
  AddCase(ComputeTgSizeSgprUsesWaveMetadata);
  AddCase(ImageStoreAndAtomicShareTypedBinding);
  AddCase(ImageAtomicCompareSwapGlcAndExec);
  AddCase(ImageAtomicSwapReturnsPreviousTexel);
  AddCase(ImageStoreAndAtomicUseSeparateBindings);
  AddCase(ImageAtomicVariants);
  for (const auto &atomic : ImageAtomicIntegerCases) {
    cases.push_back(ImageAtomicIntegerGlcAndExec(atomic, false));
    cases.push_back(ImageAtomicIntegerGlcAndExec(atomic, true));
  }
  AddCase(ImageAtomicFloatGlcAndExec<true>);
  AddCase(ImageAtomicFloatGlcAndExec<false>);
  AddCase(ImageAtomicFloatSpecialValues<true>);
  AddCase(ImageAtomicFloatSpecialValues<false>);
  AddCase(ImageAtomicFloatContendedWorkgroup<true>);
  AddCase(ImageAtomicFloatContendedWorkgroup<false>);
  AddCase(ImageAtomicGlc0DoesNotReturnOldValue);
  AddCase(MultipleWorkitemsGlobalId);
  AddCase(DispatcherIrreducibleControlFlow);

  return cases;
}

std::vector<GraphicsCase> MakeGraphicsCases() {
  return {
      GraphicsInterpolationExport(),
      GraphicsPositionWExport(),
      GraphicsPackedHalfCentroid(),
      GraphicsPerspectiveSample(false),
      GraphicsPerspectiveSample(true),
      GraphicsPackedHalfInputAlias(false),
      GraphicsPackedHalfInputAlias(true),
      GraphicsSmoothRawInputAlias(),
      GraphicsSmoothFlatInputAlias(false),
      GraphicsSmoothFlatInputAlias(true),
      GraphicsSmoothFlatInputAlias(false, true),
      GraphicsSmoothFlatInputAlias(true, true),
      GraphicsAncillaryLayer(false),
      GraphicsAncillaryLayer(true),
      GraphicsAncillarySampleId(),
      GraphicsFlatInterpolatorExport(),
      GraphicsDsAddtidScratchExport(),
      GraphicsDirectSgprPushConstantExport(),
      GraphicsInlineSrtScalarPromotionExport(),
      GraphicsNullVmExportDiscardsInactiveExec(),
      GraphicsMrt0OffVmExportDiscardsInactiveExec(),
      GraphicsFinalVmExportSupersedesEarlierVmMask(),
      GraphicsBranchPathFinalVmExportDiscardsInactiveExec(),
  };
}

void CheckComputeLdsLimit(VulkanHarness &vulkan) {
  constexpr const char *name = "ComputeLdsLimit";
  std::vector<u32> code;
  AppendVMovU32(&code, 1, 7);
  code.push_back(EncodeDs0(0x0d)); // ds_write_b32 v0, v1
  code.push_back(EncodeDs1(0, 1, 0));
  AppendEnd(&code);
  ShaderMapUserData(reinterpret_cast<uint64_t>(code.data()),
      {.type = Prospero::ShaderBinaryType::kCs,
       .code_size_bytes = static_cast<u32>(code.size() * sizeof(u32))});

  GraphicContext graphics;
  graphics.device = vulkan.Device();
  graphics.physical_device_properties =
      vulkan.RuntimeContext().GetPhysicalDeviceProperties();
  auto &limit = graphics.physical_device_properties.limits.maxComputeSharedMemorySize;
  // Reproduce a 48-KiB host, or a smaller host limit when the test device requires it.
  limit = std::min(limit, 48u * 1024u) & ~511u;
  const auto limit_units = static_cast<uint16_t>(limit / 512u);
  HW::ComputeShaderInfo regs{};
  regs.cs_regs.data_addr = reinterpret_cast<uint64_t>(code.data());
  regs.cs_regs.num_thread_x = 1;
  regs.cs_regs.num_thread_y = 1;
  regs.cs_regs.num_thread_z = 1;
  regs.cs_regs.wave_size = 32;
  PipelineCache cache(graphics);
  ShaderProgram previous;
  uint16_t previous_units = 0;
  for (const auto units : {static_cast<uint16_t>(limit_units - 1u), limit_units,
                           uint16_t{112}, uint16_t{128}, uint16_t{128}}) {
    regs.cs_regs.lds_size = units;
    ShaderComputeInputInfo input{};
    const auto program = cache.GetComputeProgram(regs, {}, input);
    Require(name, "guest allocation and device backing",
            input.lds_size_dwords == units * 128u &&
                input.lds_storage == (units > limit_units),
            "compute LDS was truncated or chose backing that exceeds the device limit");
    Require(name, "pipeline creation",
            cache.GetComputePipeline(input, program).pipeline != nullptr,
            "full-size LDS shader did not create a Vulkan compute pipeline");
    if (previous.id != 0) {
      Require(name, "guest-size cache identity",
              (program.id == previous.id) == (units == previous_units),
              "different LDS sizes shared a shader, or repeating a size recompiled it");
    }
    previous = program;
    previous_units = units;
  }
  std::printf("[gpu]     %-32s ok\n", name);
}

void CheckPs5GameExampleImageClearRuntimeShape() {
  const auto MakeCode = [] {
    std::vector<u32> code;
    AppendVop3(&code, 0x346u, 4, 8, InlineU32(6), Vgpr(0));
    for (u32 i = 0; i < 4; i++) {
      code.push_back(EncodeVop1(0x01u, i, i + 4u));
    }
    code.push_back(EncodeMubuf0(0x07u, 0, true, false));
    code.push_back(EncodeMubuf1(0, 0, 4));
    AppendEnd(&code);
    return code;
  };
  std::array<u32, 8> user_data{};
  user_data[0] = 0x00010000u;
  user_data[1] = 16u << 16u;
  user_data[2] = 64;
  user_data[3] =
      (static_cast<uint32_t>(Prospero::BufferFormat::k32_32_32_32UInt) << 12u) |
      4u | (5u << 3u) | (6u << 6u) | (7u << 9u);
  std::fill(user_data.begin() + 4, user_data.end(), 0xff000000u);
  ShaderComputeInputInfo compute{};
  compute.threads_num[0] = 64;
  compute.threads_num[1] = 1;
  compute.threads_num[2] = 1;
  compute.dispatch_threads_num[0] = 64;
  compute.dispatch_threads_num[1] = 1;
  compute.dispatch_threads_num[2] = 1;
  compute.group_id[0] = true;
  compute.dispatch_thread_dimensions = true;
  compute.wave_size = 64;
  compute.thread_ids_num = 1;
  compute.workgroup_register = 8;
  u32 scalar_clear = 0x40404040u;
  bool clean_scalar = false;

  const auto Compile = [&](const std::vector<u32> &code) {
    ShaderRecompiler::CompileOptions options;
    options.stage = ShaderType::Compute;
    options.wave_size = 64;
    options.user_data_base = 0;
    options.user_data = user_data;
    options.input_info.compute = &compute;
    options.dump_ir = false;
    auto translated = ShaderRecompiler::TranslateProgram(code, options);
    auto resource_plan =
        ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
    ShaderRecompiler::IR::ResourceSnapshot resources;
    ShaderRecompiler::IR::ResourceSpecialization specialization;
    const ShaderRecompiler::IR::SrtRuntime runtime{
        .user_data = options.user_data,
        .shader_base = reinterpret_cast<uint64_t>(code.data()),
        .userdata = &scalar_clear,
        .read_specialization_memory = clean_scalar
            ? +[](void *data, uint64_t address, std::span<uint32_t> words) {
                if (address != reinterpret_cast<uint64_t>(data) || words.size() != 1u) return false;
                words[0] = *static_cast<uint32_t *>(data);
                return true;
              } : nullptr,
    };
    Require("Ps5GameExampleImageClear", "resource materialization",
            ShaderRecompiler::IR::MaterializeResources(
                resource_plan, runtime, resources, specialization),
            "runtime clear resources could not be materialized");
    auto result = ShaderRecompiler::CompileProgram(
        std::move(translated), options, specialization);
    ValidateSpirv("Ps5GameExampleImageClear", result.spirv);
    return resources;
  };

  const auto code = MakeCode();
  auto positive = Compile(code);
  auto runtime_resources = positive;
  compute.stage.resources = &runtime_resources;
  ShaderBufferResource descriptor{};
  u32 packed_clear = 0;
  uint64_t size = 0;
  Require("Ps5GameExampleImageClear", "runtime shape",
          ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
                                   packed_clear, size) &&
              descriptor.Base48() == 0x10000u && size == 64u * 16u &&
              packed_clear == 0xff000000u,
          "exact Prospero runtime binding did not resolve to a complete clear");

  auto narrow_exec = code;
  narrow_exec.insert(narrow_exec.end() - 3,
                     EncodeSop1(0x24u, 10, InlineU32(1))); // S_AND_SAVEEXEC_B64.
  runtime_resources = Compile(narrow_exec);
  Require("Ps5GameExampleImageClear", "guest EXEC narrowing",
          !ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
                                    packed_clear, size),
          "a guest predicate narrowing the store was accepted as a complete clear");

  auto wrong_index = code;
  wrong_index[0] = (wrong_index[0] & ~(0x3ffu << 16u)) | (0x347u << 16u);
  runtime_resources = Compile(wrong_index);
  Require("Ps5GameExampleImageClear", "add-before-shift address",
          !ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
                                    packed_clear, size),
          "V_ADD_LSHL was mistaken for V_LSHL_ADD contiguous coverage");

  user_data[7] ^= 1u;
  runtime_resources = Compile(code);
  Require("Ps5GameExampleImageClear", "non-repeated clear",
          !ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
                                    packed_clear, size),
          "non-uniform uint4 data was replaced with a color clear");
  runtime_resources = positive;
  compute.dispatch_threads_num[0] = 63;
  Require("Ps5GameExampleImageClear", "mismatched requested extent",
          !ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
                                    packed_clear, size),
          "a dispatch exceeding its requested extent was classified as a complete clear");
  runtime_resources.buffers[0].dwords[2] = 65;
  compute.dispatch_threads_num[0] = 65;
  Require("Ps5GameExampleImageClear", "partial final workgroup",
          !ResolveComputeBufferFill(compute, 65, 1, 1, 0x61u, descriptor,
                                    packed_clear, size),
          "a nondivisible thread dispatch was classified as a complete clear");
  runtime_resources = positive;
  compute.dispatch_threads_num[0] = 32;
  Require("Ps5GameExampleImageClear", "partial dispatch",
          !ResolveComputeBufferFill(compute, 32, 1, 1, 0x61u, descriptor,
                                    packed_clear, size),
          "partial buffer coverage was classified as a complete clear");

  // Same descriptor/dispatch as GTA3, with the stored value encoded in the
  // shader.
  user_data = {0x00200000u, 4u << 16u, 64u, 0x14204u};
  compute.workgroup_register = 4;
  compute.dispatch_thread_dimensions = false;
  std::vector<u32> scalar_code;
  AppendVop3(&scalar_code, 0x346u, 0, 4, InlineU32(6), Vgpr(0));
  scalar_code.push_back(EncodeVop1(0x01u, 1, InlineU32(0)));
  scalar_code.push_back(EncodeMubuf0(0x04u, 0, true, false));
  scalar_code.push_back(EncodeMubuf1(1, 0, 0));
  AppendEnd(&scalar_code);
  auto scalar = Compile(scalar_code);
  runtime_resources = scalar;
  Require("Ps5GameExampleImageClear", "inline scalar fill",
          ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
                                   packed_clear, size) &&
              descriptor.Base48() == 0x200000u && size == 256u &&
              packed_clear == 0u,
          "GTA3's actual scalar store was not recognized from IR");
  Require("Ps5GameExampleImageClear", "excess dispatch",
          !ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
                                    packed_clear, size),
          "excess invocation coverage was accepted as a fill");
  auto alias = scalar.buffers.front();
  runtime_resources.buffers.push_back(alias);
  Require("Ps5GameExampleImageClear", "aliased scalar input",
          !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
                                    packed_clear, size),
          "a scalar read aliasing the destination was accepted as uniform");
  runtime_resources = scalar;
  runtime_resources.buffers[0].dwords[3] =
      (static_cast<u32>(Prospero::BufferFormat::k32Float) << 12u) | 0x204u;
  Require("Ps5GameExampleImageClear", "wrong store format",
          !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
                                    packed_clear, size),
          "a format that changes the stored value was accepted as a uint fill");
  // A body change preserving every descriptor and dispatch field must change
  // the result.
  scalar_code[2] = EncodeVop1(0x01u, 1, InlineU32(7));
  runtime_resources = Compile(scalar_code);
  Require("Ps5GameExampleImageClear", "changed store value",
          ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
                                   packed_clear, size) &&
              packed_clear == 7u,
          "clear recognition ignored the shader's actual stored value");
  scalar_code[2] = EncodeVop1(0x01u, 1, Vgpr(0));
  runtime_resources = Compile(scalar_code);
  Require("Ps5GameExampleImageClear", "varying store value",
          !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
                                    packed_clear, size),
          "a varying store with identical resources was treated as a fill");

  const auto scalar_address = reinterpret_cast<uint64_t>(&scalar_clear);
  user_data[4] = static_cast<u32>(scalar_address);
  user_data[5] = static_cast<u32>(scalar_address >> 32u) | (16u << 16u);
  user_data[6] = 1;
  user_data[7] = 0x4dfacu;
  compute.workgroup_register = 8;
  scalar_code.clear();
  AppendVop3(&scalar_code, 0x346u, 0, 8, InlineU32(6), Vgpr(0));
  scalar_code.push_back(EncodeSmem0(0x08u, 106, 2));
  scalar_code.push_back(EncodeSmem1(0, 125));
  scalar_code.push_back(EncodeVop1(0x01u, 1, 106));
  scalar_code.push_back(EncodeMubuf0(0x04u, 0, true, false));
  scalar_code.push_back(EncodeMubuf1(1, 0, 0));
  AppendEnd(&scalar_code);
  clean_scalar = true;
  auto loaded = Compile(scalar_code);
  runtime_resources = loaded;
  Require("Ps5GameExampleImageClear", "scalar-loaded fill",
          ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
                                   packed_clear, size) &&
              packed_clear == scalar_clear && descriptor.Base48() == 0x200000u,
          "GTA3's scalar input was confused with its destination");
  clean_scalar = false;
  runtime_resources = Compile(scalar_code);
  Require("Ps5GameExampleImageClear", "unavailable clean scalar",
          !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
                                    packed_clear, size),
          "a scalar without a clean reader was replaced by a fill");
  std::printf("[host]    %-32s ok\n", "Ps5GameExampleImageClear");
}

void CheckTessellationProgram(const char *name, u32 ls_stride, u32 hs_stride) {
  using namespace ShaderRecompiler;
  const bool multiplied_stride = hs_stride == 112;
  const u32 local_address = multiplied_stride ? 18 : 21;
  const u32 control_point = multiplied_stride ? 30 : 34;
  const u32 relative_patch = multiplied_stride ? 28 : 32;
  const u32 output_address = multiplied_stride ? 31 : 35;
  const u32 input_address = multiplied_stride ? 25 : 29;
  std::array<std::vector<u32>, 3> code;
  auto &local = code[0];
  // The fused LS addresses one control point using its v3 ordinal.
  local.push_back(EncodeVop2(0x0b, local_address, 255u, 3));
  local.push_back(ls_stride);
  local.push_back(EncodeDs0(0x0e, (3u << 8u) | 1u));
  local.push_back(EncodeDs1Ex(0, 5, 2, local_address));
  local.push_back(EncodeSop1(0x20, 0, 6));
  local.push_back(0xffffffffu); // Shader footer is not executable after LS handoff.

  auto &control = code[1];
  AppendVop3(&control, 0x365, 0, 193u, InlineU32(0));
  AppendVop3(&control, 0x366, 0, 193u, Vgpr(0));
  control.push_back(EncodeSopp(0x0a));
  control.push_back(EncodeVopc(0xd1, 249u, 3));
  control.push_back(EncodeVopcSdwa(0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 1));
  AppendVop3(&control, 0x148, control_point, Vgpr(1), InlineU32(8),
             InlineU32(5));
  AppendVop3(&control, 0x148, relative_patch, Vgpr(1), InlineU32(0),
             InlineU32(8));
  if (multiplied_stride) {
    // The later patch writes 112-byte outputs but reads 108-byte LS records.
    // The second multiply overwrites packed v1; only memory consumers identify
    // which of the two independently computed addresses supplies each stride.
    control.push_back(EncodeVop2(0x0b, 0, 255u, control_point));
    control.push_back(hs_stride);
    control.push_back(EncodeVop2(0x0b, 1, 255u, control_point));
    control.push_back(ls_stride);
    AppendVop3(&control, 0x143, output_address, 255u, Vgpr(relative_patch),
               Vgpr(0));
    control.push_back(3 * hs_stride);
    AppendVop3(&control, 0x143, input_address, 255u, Vgpr(relative_patch),
               Vgpr(1));
    control.push_back(3 * ls_stride);
  } else {
    control.push_back(EncodeSopk(0x00, 106, 3 * hs_stride));
    control.push_back(EncodeVop2(0x0b, 2, 249u, 1));
    control.push_back(EncodeVop2Sdwa(106, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1));
    AppendVop3(&control, 0x346, output_address, Vgpr(control_point), InlineU32(7),
               Vgpr(2));
    control.push_back(EncodeVop2(0x0b, input_address, 255u, control_point));
    control.push_back(ls_stride);
  }
  control.push_back(EncodeDs0(0x37, (3u << 8u) | 1u));
  control.push_back(EncodeDs1Ex(10, 0, 0, input_address));
  control.push_back(EncodeMubuf0(0x1d, 28));
  control.push_back(EncodeMubuf1(10, 2, output_address, 2));
  // Keep the control-point-dependent value and the last written dword live.
  control.push_back(EncodeMubuf0(0x1c, hs_stride - 8));
  control.push_back(EncodeMubuf1(0, 2, output_address, 2));
  // The game reuses previously written data VGPRs for CP0-only ring addresses.
  AppendVMovLiteral(&control, 4, 0x42280000u);
  AppendVMovLiteral(&control, 5, 0x422c0000u);
  control.push_back(EncodeVopc(0xd4, InlineU32(1), control_point));
  control.push_back(EncodeVop2(0x1a, 4, InlineU32(4), relative_patch));
  control.push_back(EncodeVop2(0x1a, 6, InlineU32(6), relative_patch));
  control.push_back(EncodeVop2(0x26, 5, 255u, 6));
  control.push_back(0x7fc0);
  for (u32 i = 0; i < 4; i++) {
    AppendVMovLiteral(&control, i, std::bit_cast<u32>(static_cast<float>(i + 1)));
  }
  control.push_back(EncodeMubuf0(0x1e));
  control.push_back(EncodeMubuf1(0, 2, 4, 4));
  for (u32 i = 0; i < 4; i++) {
    control.push_back(EncodeMubuf0(0x1c, i * 16));
    control.push_back(EncodeMubuf1(i, 2, 5, 2));
  }
  AppendEnd(&control);

  auto &evaluation = code[2];
  AppendVop3(&evaluation, 0x169, 24, 255u, Vgpr(7));
  evaluation.push_back(3 * hs_stride);
  evaluation.push_back(EncodeMubuf0(0x0d, 28));
  evaluation.push_back(EncodeMubuf1(10, 2, 24, 4));
  evaluation.push_back(EncodeMubuf0(0x0d, 2 * hs_stride + 28));
  evaluation.push_back(EncodeMubuf1(12, 2, 24, 4));
  evaluation.push_back(EncodeExp0(0x0c, 0xf));
  evaluation.push_back(EncodeExp1(5, 6, 10, 12));
  AppendEnd(&evaluation);

  ShaderTessellationInputInfo tess{.input_control_points = 3,
                                  .output_control_points = 3,
                                  .domain = 1,
                                  .partitioning = 2,
                                  .output_topology = 2};
  AnalyzeTessellationPrograms(local, control, tess);
  Require(name, "decoded interface",
          tess.ls_stride == ls_stride && tess.hs_stride == hs_stride,
          "captured LS and HS address arithmetic must produce distinct strides");

  constexpr std::array stages{ShaderType::Local, ShaderType::TessellationControl,
                              ShaderType::TessellationEvaluation};
  for (u32 stage = 0; stage < stages.size(); stage++) {
    ShaderVertexInputInfo vertex;
    vertex.logical_stage = stages[stage];
    vertex.tess = tess;
    CompileOptions options;
    options.stage = stages[stage];
    options.input_info.vertex = &vertex;
    auto translated = TranslateProgram(code[stage], options);
    std::set<u32> local_offsets, evaluation_offsets, factor_offsets, patch_offsets;
    for (const auto *block : translated.program.blocks) {
      for (const auto &inst : *block) {
        if (inst.GetOpcode() != IR::ValueOpcode::GetTessellationAttribute &&
            inst.GetOpcode() != IR::ValueOpcode::SetTessellationAttribute) {
          continue;
        }
        if (!inst.Arg(1).IsImmediate()) {
          continue;
        }
        switch (static_cast<IR::TessellationAttribute>(inst.Arg(0).U32())) {
        case IR::TessellationAttribute::LocalOutput:
          local_offsets.insert(inst.Arg(1).U32());
          break;
        case IR::TessellationAttribute::EvaluationInput:
          evaluation_offsets.insert(inst.Arg(1).U32());
          break;
        case IR::TessellationAttribute::Factor:
          factor_offsets.insert(inst.Arg(1).U32());
          break;
        case IR::TessellationAttribute::PatchOutput:
          patch_offsets.insert(inst.Arg(1).U32());
          break;
        default:
          break;
        }
      }
    }
    Require(name, "separate DS offsets", stage != 0 || local_offsets == std::set<u32>{4, 12},
            "DS_WRITE2 must preserve both independently encoded offsets");
    Require(name, "offchip component addressing",
            stage != 2 ||
                evaluation_offsets == std::set<u32>{28, 2 * hs_stride + 28},
            "TES control point and component offsets must survive lowering");
    Require(name, "triangle factor layout",
            stage != 1 || factor_offsets == std::set<u32>{0, 4, 8, 12},
            "the four packed guest factors must retain their order");
    Require(name, "predicated patch addresses",
            stage != 1 || patch_offsets == std::set<u32>{0x7fc0, 0x7fd0, 0x7fe0, 0x7ff0},
            "CP0 patch stores must use the address assigned on their active predicate");
    auto result = CompileProgram(std::move(translated), options, {});
    ValidateSpirv(name, result.spirv);
    spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
    std::string assembly;
    Require(name, "disassembly", tools.Disassemble(result.spirv, &assembly),
            "failed to disassemble tessellation shader");
    Require(name, "native local inputs",
            stage != 0 || (assembly.find("VertexIndex") != std::string::npos &&
                           assembly.find("InstanceIndex") != std::string::npos),
            "LS v2/v5 must read native vertex/instance indices");
    Require(name, "control point lane",
            stage != 1 || (assembly.find("InvocationId") != std::string::npos &&
                           assembly.find("SubgroupLocalInvocationId") == std::string::npos),
            "HS lane masks must use control point invocation IDs");
    Require(name, "control point barrier",
            stage != 1 || (assembly.find("OpMemoryModel Logical GLSL450") != std::string::npos &&
                           assembly.find("OpControlBarrier %uint_2 %uint_4 %uint_0") != std::string::npos),
            "HS barriers must match GLSL450 control point synchronization semantics");
    Require(name, "factor builtins",
            stage != 1 || (assembly.find("TessLevelOuter") != std::string::npos &&
                           assembly.find("TessLevelInner") != std::string::npos),
            "triangle factors require outer[4] and inner[2] Vulkan builtins");
    Require(name, "native domain and winding",
            stage != 2 || (assembly.find("TessCoord") != std::string::npos &&
                           assembly.find("Triangles") != std::string::npos &&
                           assembly.find("SpacingFractionalOdd") != std::string::npos &&
                           assembly.find("VertexOrderCw") != std::string::npos),
            "TES must preserve guest triangle domain, partitioning, and winding");
  }
  std::printf("[host]    %-32s ok\n", name);
}

void CheckTessellationPrograms() {
  CheckTessellationProgram("TessellationShiftedStride", 124, 128);
  CheckTessellationProgram("TessellationMultipliedStride", 108, 112);
}

void CheckEmbeddedFetchVertexOffset() {
  const auto MakeFetch = [](std::initializer_list<std::pair<u32, u32>> adds,
                            std::optional<std::pair<u32, u32>> late_add = {},
                            u32 accumulator_vgpr = 0, bool ngg_sad = false,
                            bool local = false) {
    std::vector<u32> code;
    code.push_back(EncodeSMovB32(0, InlineU32(0)));
    code.push_back(EncodeSmem0(0x02u, 20, 4));
    code.push_back(EncodeSmem1(0));
    const auto AppendOffsets = [&]() {
      for (const auto [sgpr, index_vgpr] : adds) {
        if (ngg_sad) {
          AppendVop3(&code, 0x15du, accumulator_vgpr, sgpr, InlineU32(0),
                     Vgpr(index_vgpr));
        } else {
          AppendVop3B(&code, 0x30fu, accumulator_vgpr, 0, sgpr,
                      Vgpr(index_vgpr));
        }
      }
    };
    if (ngg_sad) {
      AppendOffsets();
    }
    code.push_back(EncodeVop2(0x01u, 0, Vgpr(local ? 5 : 8), local ? 2 : 5));
    if (!ngg_sad) {
      AppendOffsets();
    }
    code.push_back(EncodeMubuf0(0x03u, 0, true));
    code.push_back(EncodeMubuf1(9, 5, 0));
    if (late_add.has_value()) {
      AppendVop3B(&code, 0x30fu, 0, 0, late_add->first, Vgpr(late_add->second));
    }
    if (local) {
      AppendVMovU32(&code, 20, 0);
      code.push_back(EncodeDs0(0x0e, 1u << 8u));
      code.push_back(EncodeDs1Ex(0, 10, 9, 20));
      code.push_back(EncodeDs0(0x0e, (3u << 8u) | 2u));
      code.push_back(EncodeDs1Ex(0, 12, 11, 20));
      code.push_back(EncodeSop1(0x20, 0, 6));
      code.push_back(0xffffffffu);
    } else {
      AppendEnd(&code);
    }
    return code;
  };

  const auto Compile = [&](const char *name, const std::vector<u32> &code,
                           u32 slot10, ShaderType stage = ShaderType::Vertex,
                           Prospero::BufferFormat format = Prospero::BufferFormat::k32_32_32_32Float) {
    std::array<u32, 11> user_data{};
    user_data[10] = slot10;
    ShaderVertexInputInfo vertex;
    vertex.logical_stage = stage;
    vertex.tess = {.input_control_points = 3, .output_control_points = 3,
                   .ls_stride = 124, .hs_stride = 128, .domain = 1,
                   .partitioning = 2, .output_topology = 2};
    vertex.fetch_embedded = true;
    vertex.fetch_buffer_reg = 0;
    vertex.fetch_attrib_reg = 2;
    vertex.resources_num = 1;
    vertex.resources[0].fields[3] =
        BufferFormat(format) << 12u | DstSel(4, 5, 6, 7);
    vertex.resources_dst[0].attr_id = 0;
    vertex.resources_dst[0].registers_num = 4;

    ShaderRecompiler::CompileOptions options;
    options.stage = stage;
    options.user_data_base = 8;
    options.user_data = user_data;
    options.input_info.vertex = &vertex;

    auto translated = ShaderRecompiler::TranslateProgram(code, options);
    auto resource_plan =
        ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
    ShaderRecompiler::IR::ResourceSnapshot resources;
    ShaderRecompiler::IR::ResourceSpecialization specialization;
    const ShaderRecompiler::IR::SrtRuntime runtime{
        .user_data = options.user_data,
        .shader_base = reinterpret_cast<uint64_t>(code.data()),
    };
    Require(name, "resource materialization",
            ShaderRecompiler::IR::MaterializeResources(
                resource_plan, runtime, resources, specialization),
            "embedded fetch resources could not be materialized");
    auto result = ShaderRecompiler::CompileProgram(
        std::move(translated), options, specialization);
    Require(name, "fetch rewrite",
            result.program.info.vertex_fetch_components[0] == 4,
            "encoded fetch sequence was not recognized and rewritten");
    return CompiledShader{std::move(result.spirv), std::move(result.program),
                          std::move(resources), {}};
  };

  const auto Resolve = [](const CompiledShader &result,
                          u32 index_offset) {
    ShaderVertexInputInfo vertex;
    vertex.fetch_embedded = true;
    ShaderRecompiler::IR::CompiledShaderInfo program{};
    program.user_data_base = result.program.user_data_base;
    program.info.vertex_offset_sgpr = result.program.info.vertex_offset_sgpr;
    vertex.stage.program = &program;
    vertex.stage.resources = &result.resources;
    return ResolveDrawOffsets(index_offset, vertex).first;
  };

  const auto ResolveInstance = [](const CompiledShader &result) {
    ShaderVertexInputInfo vertex;
    vertex.fetch_embedded = true;
    ShaderRecompiler::IR::CompiledShaderInfo program{};
    program.user_data_base = result.program.user_data_base;
    program.info.instance_offset_sgpr =
        result.program.info.instance_offset_sgpr;
    vertex.stage.program = &program;
    vertex.stage.resources = &result.resources;
    return ResolveDrawOffsets(0, vertex).second;
  };

  for (const auto format : {Prospero::BufferFormat::k32_32_32_32Float,
                            Prospero::BufferFormat::k32_32_32_32UInt}) {
    const auto local = Compile("LocalEmbeddedFetch", MakeFetch({}, {}, 0, false, true),
                               0, ShaderType::Local, format);
    ValidateSpirv("LocalEmbeddedFetch", local.spirv);
    Require("LocalEmbeddedFetch", "live vertex attributes",
            std::ranges::any_of(local.program.info.inputs, [](const auto &input) {
              return input.kind == ShaderRecompiler::IR::StageInputKind::Parameter &&
                     input.location == 0;
            }), "LS indexed fetch must consume native vertex attributes");
    if (format == Prospero::BufferFormat::k32_32_32_32UInt) {
      spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
      std::string assembly;
      Require("LocalEmbeddedFetch", "disassembly", tools.Disassemble(local.spirv, &assembly),
              "failed to disassemble unsigned LS fetch");
      Require("LocalEmbeddedFetch", "unsigned input type",
              assembly.find("OpTypePointer Input %v4uint") != std::string::npos &&
                  assembly.find("OpConvertUToF") == std::string::npos,
              "unsigned LS attributes must preserve their native integer bits");
    }
  }

  const auto valid =
      Compile("EmbeddedFetchVertexOffset", MakeFetch({{18, 0}}), 7);
  Require(
      "EmbeddedFetchVertexOffset", "parse",
      valid.program.info.vertex_offset_sgpr == 18 && Resolve(valid, 0) == 7 &&
          Resolve(valid, 5) == 5,
      "canonical fetch offset or register index-offset precedence is wrong");

  const auto ngg_code = MakeFetch({{18, 5}}, {}, 5, true);
  Require("EmbeddedFetchNggVertexOffset", "encoding",
          ngg_code[3] == 0xd55d0005u && ngg_code[4] == 0x04150012u,
          "test does not encode the PS5 V_SAD_U32 vertex-offset prolog");
  const auto ngg = Compile("EmbeddedFetchNggVertexOffset", ngg_code, 8);
  Require("EmbeddedFetchNggVertexOffset", "parse",
          ngg.program.info.vertex_offset_sgpr == 18 && Resolve(ngg, 0) == 8 &&
              Resolve(ngg, 5) == 5,
          "PS5 NGG vertex-index offset or register index-offset precedence is "
          "wrong");

  const auto ngg_instance =
      Compile("EmbeddedFetchNggInstanceOffset",
              MakeFetch({{18, 8}}, {}, 8, true), 81);
  Require("EmbeddedFetchNggInstanceOffset", "parse",
          ngg_instance.program.info.vertex_offset_sgpr == -1 &&
              ngg_instance.program.info.instance_offset_sgpr == 18 &&
              ResolveInstance(ngg_instance) == 81,
          "PS5 NGG instance-index offset was not detected or resolved");

  const auto instance_conflict = Compile(
      "EmbeddedFetchConflictingInstanceOffset",
      MakeFetch({{17, 8}, {18, 8}}, {}, 8, true), 81);
  const auto instance_malformed =
      Compile("EmbeddedFetchMalformedInstanceOffset",
              MakeFetch({{18, 7}}, {}, 7, true), 81);
  const auto instance_outside =
      Compile("EmbeddedFetchOutsideInstanceOffset",
              MakeFetch({{19, 8}}, {}, 8, true), 81);
  for (const auto *result :
       {&instance_conflict, &instance_malformed, &instance_outside}) {
    Require("EmbeddedFetchNggInstanceOffset", "fail closed",
            result->program.info.instance_offset_sgpr == -1 &&
                ResolveInstance(*result) == 0,
            "conflicting, malformed, or out-of-window instance add was "
            "classified");
  }

  const auto pointer = 0x5b7c5100u;
  const auto late = Compile("EmbeddedFetchLateOffset",
                            MakeFetch({}, std::pair<u32, u32>{18, 0}), pointer);
  const auto conflict = Compile("EmbeddedFetchConflictingOffset",
                                MakeFetch({{17, 0}, {18, 0}}), pointer);
  const auto malformed =
      Compile("EmbeddedFetchMalformedOffset", MakeFetch({{18, 1}}), pointer);
  const auto outside =
      Compile("EmbeddedFetchOutsideOffset", MakeFetch({{19, 0}}), pointer);
  for (const auto *result : {&late, &conflict, &malformed, &outside}) {
    Require("EmbeddedFetchVertexOffset", "fail closed",
            result->program.info.vertex_offset_sgpr == -1 &&
                Resolve(*result, 0) == 0,
            "non-prolog, conflicting, malformed, or out-of-window add was "
            "classified");
  }
  std::printf("[host]    %-32s ok\n", "EmbeddedFetchVertexOffset");
}

[[noreturn]] void RunReverseRenderTargetDeathCase() {
  (void)TextureGetRenderTargetFormat(Prospero::ChannelLayout::k16_16_16_16,
                                     Prospero::ChannelType::kSrgb,
                                     Prospero::ChannelOrder::kStandard);
  std::_Exit(0x7f);
}

#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckRenderTargetFormatContract() {
  const auto r8_uint = TextureGetRenderTargetFormat(
      Prospero::ChannelLayout::k8, Prospero::ChannelType::kUInt,
      Prospero::ChannelOrder::kStandard);
  Require("RenderTargetFormat", "R8 uint",
          r8_uint.format == vk::Format::eR8Uint &&
              r8_uint.bytes_per_element == 1u &&
              r8_uint.export_mapping.IsIdentity(),
          "R8 uint render-target tuple was rejected");

  const auto rgb565 = TextureGetRenderTargetFormat(
      Prospero::ChannelLayout::k5_6_5, Prospero::ChannelType::kUNorm,
      Prospero::ChannelOrder::kStandard);
  Require("RenderTargetFormat", "RGB565 UNorm",
          rgb565.format == vk::Format::eB5G6R5UnormPack16 &&
              rgb565.bytes_per_element == 2u &&
              rgb565.export_mapping.IsIdentity(),
          "RGB565 UNorm render-target tuple was rejected");

  const auto uint_format = TextureGetRenderTargetFormat(
      Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kUInt,
      Prospero::ChannelOrder::kStandard);
  Require("RenderTargetFormat", "RGBA16 uint",
          uint_format.format == vk::Format::eR16G16B16A16Uint &&
              uint_format.bytes_per_element == 8u &&
              uint_format.export_mapping.IsIdentity(),
          "RGBA16 uint render-target tuple was rejected");

  const auto format = TextureGetRenderTargetFormat(
      Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat,
      Prospero::ChannelOrder::kReversed);
  Require("ReverseRenderTarget", "exact format",
          format.format == vk::Format::eR16G16B16A16Sfloat &&
              format.bytes_per_element == 8u &&
              format.export_mapping == Prospero::ColorMappingAbgr,
          "exact reverse RGBA16F render-target tuple was rejected");
  Require("ReverseRenderTarget", "write masks",
          format.export_mapping.ApplyMask(0x1u) == 0x8u &&
              format.export_mapping.ApplyMask(0x2u) == 0x4u &&
              format.export_mapping.ApplyMask(0x4u) == 0x2u &&
              format.export_mapping.ApplyMask(0x8u) == 0x1u &&
              format.export_mapping.ApplyMask(0xfu) == 0xfu,
          "reverse RGBA16F component mask was not mapped exactly once");

  const auto rg32 = TextureGetRenderTargetFormat(
      Prospero::ChannelLayout::k32_32, Prospero::ChannelType::kFloat,
      Prospero::ChannelOrder::kReversed);
  Require(
      "ReverseRenderTarget", "GR32 float",
      rg32.format == vk::Format::eR32G32Sfloat &&
          rg32.bytes_per_element == 8u &&
          rg32.export_mapping == Prospero::ColorMappingGr &&
          rg32.export_mapping.ApplyMask(0x1u) == 0x2u &&
          rg32.export_mapping.ApplyMask(0x2u) == 0x1u &&
          rg32.export_mapping.ApplyMask(0x3u) == 0x3u &&
          rg32.export_mapping.ApplyMask(0xfu) == 0xfu,
      "reverse GR32F render-target export or write-mask mapping is incorrect");

  const auto argb = TextureGetRenderTargetFormat(
      Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat,
      Prospero::ChannelOrder::kAltReversed);
  Require(
      "ReverseRenderTarget", "ARGB float",
      argb.format == vk::Format::eR16G16B16A16Sfloat &&
          argb.export_mapping == Prospero::ColorMappingArgb &&
          argb.export_mapping.ApplyMask(0x1u) == 0x2u &&
          argb.export_mapping.ApplyMask(0x2u) == 0x4u &&
          argb.export_mapping.ApplyMask(0x4u) == 0x8u &&
          argb.export_mapping.ApplyMask(0x8u) == 0x1u,
      "alternate-reversed render-target mapping did not invert its ARGB cycle");

  char path[MAX_PATH]{};
  Require("ReverseRenderTarget", "host",
          GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
          "GetModuleFileName failed");
  std::string command = std::string("\"") + path + "\" --reverse-rt-death";
  std::vector<char> mutable_command(command.begin(), command.end());
  mutable_command.push_back('\0');
  STARTUPINFOA startup{sizeof(startup)};
  PROCESS_INFORMATION process{};
  Require("ReverseRenderTarget", "host",
          CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr,
                         FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup,
                         &process) != 0,
          "CreateProcess failed");
  Require("ReverseRenderTarget", "host",
          WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0,
          "unsupported adjacent render-target tuple timed out");
  DWORD exit_code = 0;
  const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0;
  CloseHandle(process.hThread);
  CloseHandle(process.hProcess);
  Require("ReverseRenderTarget", "hard failure", exited && exit_code == 321,
          "invalid render-target tuple did not retain the fatal guard");
  std::printf("[host]    %-32s ok\n", "RenderTargetFormat");
}
#endif

[[noreturn]] void RunImageViewDeathCase(const char *kind) {
  if (std::strcmp(kind, "sampled-invalid-selector") == 0) {
    (void)SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
                                 vk::Format::eR8G8B8A8Unorm,
                                 DstSel(4, 5, 6, 2));
  } else if (std::strcmp(kind, "sampled-incompatible-format") == 0) {
    (void)SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR16Unorm,
                                 DstSel(4, 0, 0, 1));
  } else if (std::strcmp(kind, "sampled-invalid-high") == 0) {
    (void)SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
                                 vk::Format::eR8G8B8A8Unorm,
                                 DstSel(7, 6, 5, 3));
  } else if (std::strcmp(kind, "storage-incompatible-format") == 0) {
    ValidateStorageColorView(vk::Format::eR8G8B8A8Srgb,
                             vk::Format::eR16G16B16A16Unorm,
                             DstSel(4, 5, 6, 7));
  } else if (std::strcmp(kind, "volume-mip-count") == 0 ||
             std::strcmp(kind, "volume-slice-range") == 0) {
    VulkanHarness vulkan;
    auto &graphics = vulkan.RuntimeContext();
    CommandScheduler scheduler(vulkan.RuntimeRenderer(), graphics);
    ImageInfo volume_info{};
    volume_info.pixel_format = vk::Format::eR8G8B8A8Unorm;
    volume_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm;
    volume_info.type = Prospero::ImageType::kColor3D;
    volume_info.extent = {8, 8, 4};
    volume_info.resources = {2, 1};
    volume_info.pitch = 8;
    volume_info.bytes_per_block = 4;
    volume_info.samples = 1;
    volume_info.tile_mode = Prospero::TileMode::kLinear;
    Libs::Graphics::Image volume(graphics, scheduler, volume_info);
    ImageViewInfo view{};
    view.format = volume_info.pixel_format;
    view.type = vk::ImageViewType::e2DArray;
    view.aspect = vk::ImageAspectFlagBits::eColor;
    view.base_level = 1;
    view.level_count = 1;
    view.layer_count = 1;
    view.usage = vk::ImageUsageFlagBits::eSampled;
    if (std::strcmp(kind, "volume-mip-count") == 0) {
      view.base_level = 0;
      view.level_count = 2;
    } else {
      view.base_layer = 1;
      view.layer_count = 2;
    }
    (void)volume.FindView(view);
  } else {
    ShaderRecompiler::IR::ImageResource resource{};
    resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
    resource.numeric_class = Prospero::TextureNumericClass::Float;
    resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
    resource.written = true;
    if (std::strcmp(kind, "storage-kind") == 0) {
      resource.resource_class =
          ShaderRecompiler::IR::ImageResourceClass::Sampled;
    } else if (std::strcmp(kind, "storage-no-write") == 0) {
      resource.written = false;
    } else if (std::strcmp(kind, "storage-nonuint-atomic") == 0) {
      resource.atomic = true;
    } else if (std::strcmp(kind, "storage-compare") == 0) {
      resource.depth_compare = true;
    } else if (std::strcmp(kind, "storage-dimension") == 0) {
      resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Unknown;
    } else {
      std::_Exit(0x7e);
    }
    ValidateStorageImageResource(resource);
  }
  std::_Exit(0x7f);
}

#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckSampledColorViews() {
  Require("SampledColorViews", "identity",
          SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
                                 vk::Format::eR8G8B8A8Unorm,
                                 DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7),
          "RGBA did not select the identity view");
  uint32_t valid_swizzles = 0;
  for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) {
    bool expected = true;
    for (uint32_t channel = 0; channel < 4; channel++) {
      const auto selector = GetDstSel(swizzle, channel);
      if (selector == 2 || selector == 3) {
        expected = false;
      }
    }
    const bool valid = IsValidImageSwizzle(swizzle);
    const bool supported = IsSupportedSampledColorView(
        vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, swizzle);
    Require("SampledColorViews", "exhaustive read swizzle domain",
            valid == expected && supported == expected,
            "sampled swizzle validator disagreed with the PS5 selector domain");
    valid_swizzles += valid;
  }
  Require("SampledColorViews", "all PS5 read swizzles",
          valid_swizzles == 1296 && !IsValidImageSwizzle(DstSel(4, 5, 6, 2)) &&
              !IsValidImageSwizzle(DstSel(4, 5, 6, 3)) &&
              !IsValidImageSwizzle(0x1000),
          "valid PS5 sampled mappings were rejected or reserved selectors were "
          "admitted");
  const auto arbitrary = DstSel(5, 1, 7, 0);
  const auto components = TextureGetComponentMapping(arbitrary, {});
  Require("SampledColorViews", "generic Vulkan component mapping",
          SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
                                 vk::Format::eR8G8B8A8Unorm,
                                 arbitrary) == arbitrary &&
              components.r == vk::ComponentSwizzle::eG &&
              components.g == vk::ComponentSwizzle::eOne &&
              components.b == vk::ComponentSwizzle::eA &&
              components.a == vk::ComponentSwizzle::eZero,
          "arbitrary valid sampled mapping did not use the generic view path");
  Require("SampledColorViews", "R8 R001",
          SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR8Unorm,
                                 DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1),
          "R8 did not select its R001 view");
  Require("SampledColorViews", "R8 000R",
          SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR8Unorm,
                                 DstSel(0, 0, 0, 4)) == DstSel(0, 0, 0, 4),
          "R8 did not select its 000R component-mapped view");
  Require("SampledColorViews", "mutable R8 uint/unorm 000R",
          SelectSampledColorView(vk::Format::eR8Uint, vk::Format::eR8Unorm,
                                 DstSel(0, 0, 0, 4)) == DstSel(0, 0, 0, 4),
          "compatible R8 integer target sampled view was rejected");
  Require("SampledColorViews", "R16G16 RG01",
          SelectSampledColorView(vk::Format::eR16G16Sfloat,
                                 vk::Format::eR16G16Sfloat,
                                 DstSel(4, 5, 0, 1)) == DstSel(4, 5, 0, 1),
          "R16G16 did not select its RG01 view");
  Require("SampledColorViews", "alpha one",
          SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
                                 vk::Format::eR8G8B8A8Unorm,
                                 DstSel(4, 5, 6, 1)) == DstSel(4, 5, 6, 1),
          "RGB1 did not select the alpha-one view");
  Require("SampledColorViews", "mutable BGRA target",
          SelectSampledColorView(vk::Format::eB8G8R8A8Unorm,
                                 vk::Format::eR8G8B8A8Unorm,
                                 DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
          "BGRA target did not select the exact RGBA/BGRA mutable view");
  Require(
      "SampledColorViews", "mutable sRGB view of UNORM BGRA target",
      SelectSampledColorView(vk::Format::eB8G8R8A8Unorm,
                             vk::Format::eR8G8B8A8Srgb,
                             DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
      "UNORM BGRA target did not select its compatible sRGB RGBA sampled view");
  Require("SampledColorViews", "mutable sRGB BGRA target",
          SelectSampledColorView(vk::Format::eB8G8R8A8Srgb,
                                 vk::Format::eR8G8B8A8Srgb,
                                 DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
          "sRGB BGRA target did not select its matching mutable RGBA view");
  constexpr auto colorspace_swizzle = DstSel(5, 1, 7, 0);
  Require("SampledColorViews", "mutable sRGB/UNORM views",
          SelectSampledColorView(vk::Format::eR8G8B8A8Srgb,
                                 vk::Format::eR8G8B8A8Unorm,
                                 DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) &&
              SelectSampledColorView(vk::Format::eB8G8R8A8Unorm,
                                     vk::Format::eB8G8R8A8Srgb,
                                     colorspace_swizzle) == colorspace_swizzle,
          "same-order mutable sRGB/UNORM sampled views were rejected");
  Require("SampledColorViews", "mutable packed RGB10 view",
          SelectSampledColorView(vk::Format::eA2R10G10B10UnormPack32,
                                 vk::Format::eA2B10G10R10UnormPack32,
                                 DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
          "packed RGB10 target did not select its matching mutable "
          "channel-order view");
  Require(
      "SampledColorViews", "reverse RGBA16F sampled view",
      SelectSampledColorView(vk::Format::eR16G16B16A16Sfloat,
                             vk::Format::eR16G16B16A16Sfloat,
                             DstSel(7, 6, 5, 4)) == DstSel(7, 6, 5, 4),
      "reverse RGBA16F target did not select its reciprocal ABGR sampled view");
  Require("SampledColorViews", "mutable integer-class views",
          SelectSampledColorView(vk::Format::eR16G16B16A16Sfloat,
                                 vk::Format::eR16G16B16A16Uint,
                                 DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) &&
              SelectSampledColorView(
                  vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Uint,
                  DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) &&
              SelectSampledColorView(vk::Format::eR8G8B8A8Uint,
                                     vk::Format::eR8G8B8A8Unorm,
                                     DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7),
          "compatible integer render-target sampled view was rejected");
  Require("SampledColorViews", "invalid depth format",
          !IsSupportedSampledDepthView(vk::Format::eD24UnormS8Uint,
                                       vk::Format::eR8Unorm,
                                       DstSel(4, 4, 4, 4)),
          "incompatible sampled depth format was accepted");
  Require("SampledColorViews", "invalid depth swizzle",
          !IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
                                       vk::Format::eR32Sfloat,
                                       DstSel(4, 5, 6, 7)),
          "incompatible sampled depth swizzle was accepted");
  Require("SampledColorViews", "D32 depth target",
          IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
                                      vk::Format::eR32Sfloat,
                                      DstSel(4, 4, 4, 4)),
          "D32 depth target did not select its depth-aspect view");
  Require("SampledColorViews", "D16 R000 depth target",
          IsSupportedSampledDepthView(
              vk::Format::eD16Unorm, vk::Format::eR16Unorm, DstSel(4, 0, 0, 0)),
          "D16 depth target did not select its R000 depth-aspect view");
  Require("SampledColorViews", "D16 RRR1 depth target",
          IsSupportedSampledDepthView(
              vk::Format::eD16Unorm, vk::Format::eR16Unorm, DstSel(4, 4, 4, 1)),
          "D16 depth target rejected replicated depth with constant alpha");
  Require("SampledColorViews", "D16S8 R001 depth target",
          IsSupportedSampledDepthView(vk::Format::eD16UnormS8Uint,
                                      vk::Format::eR16Unorm,
                                      DstSel(4, 0, 0, 1)),
          "D16S8 depth target did not select its R001 depth-aspect view");
  Require("SampledColorViews", "promoted D24S8 R001 depth target",
          IsSupportedSampledDepthView(vk::Format::eD24UnormS8Uint,
                                      vk::Format::eR16Unorm,
                                      DstSel(4, 0, 0, 1)),
          "D24S8 host fallback did not preserve the guest R16 depth view");
  Require("SampledColorViews", "promoted D32S8 R001 depth target",
          IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
                                      vk::Format::eR16Unorm,
                                      DstSel(4, 0, 0, 1)),
          "D32S8 host fallback did not preserve the guest R16 depth view");
  Require("SampledColorViews", "D32S8 R001 depth target",
          IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
                                      vk::Format::eR32Sfloat,
                                      DstSel(4, 0, 0, 1)),
          "D32S8 depth target did not select its R001 depth-aspect view");
  ShaderRecompiler::IR::ImageResource storage_resource{};
  storage_resource.resource_class =
      ShaderRecompiler::IR::ImageResourceClass::Storage;
  storage_resource.numeric_class = Prospero::TextureNumericClass::Float;
  storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
  storage_resource.written = true;
  Require("SampledColorViews", "storage resource",
          IsSupportedStorageImageResource(storage_resource),
          "exact storage resource contract was rejected");
  storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
  storage_resource.read = true;
  Require("SampledColorViews", "read-write 3D storage resource",
          IsSupportedStorageImageResource(storage_resource),
          "basic read-write 3D storage resource was rejected");
  storage_resource.dimension =
      ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
  storage_resource.read = false;
  Require("SampledColorViews", "write-only 2D-array storage resource",
          IsSupportedStorageImageResource(storage_resource),
          "basic write-only 2D-array storage resource was rejected");
  storage_resource.numeric_class = Prospero::TextureNumericClass::Uint;
  Require("SampledColorViews", "write-only uint 2D-array storage resource",
          IsSupportedStorageImageResource(storage_resource),
          "basic write-only uint 2D-array storage resource was rejected");
  storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
  storage_resource.read = true;
  storage_resource.atomic = true;
  Require("SampledColorViews", "atomic uint 2D storage resource",
          IsSupportedStorageImageResource(storage_resource),
          "atomic uint storage resource was rejected");
  storage_resource.atomic = false;
  storage_resource.mip_mode =
      ShaderRecompiler::IR::ImageMipMode::Dynamic;
  storage_resource.mip_count = 3;
  Require("SampledColorViews", "dynamic storage mip resource",
          IsSupportedStorageImageResource(storage_resource),
          "dynamic storage mip resource was rejected");

  char path[MAX_PATH]{};
  Require("SampledColorViews", "host",
          GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
          "GetModuleFileName failed");
  for (const char *kind :
       {"sampled-invalid-selector", "sampled-incompatible-format",
        "sampled-invalid-high", "storage-incompatible-format", "storage-kind",
        "storage-no-write", "storage-nonuint-atomic", "storage-compare",
        "storage-dimension", "volume-mip-count", "volume-slice-range"}) {
    std::string command =
        std::string("\"") + path + "\" --image-view-death " + kind;
    std::vector<char> mutable_command(command.begin(), command.end());
    mutable_command.push_back('\0');
    STARTUPINFOA startup{sizeof(startup)};
    PROCESS_INFORMATION process{};
    Require("SampledColorViews", "host",
            CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr,
                           FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup,
                           &process) != 0,
            "CreateProcess failed");
    Require("SampledColorViews", "host",
            WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0,
            "unsupported view death case timed out");
    DWORD exit_code = 0;
    const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0;
    CloseHandle(process.hThread);
    CloseHandle(process.hProcess);
    Require("SampledColorViews", "host", exited && exit_code == 321,
            std::string(kind) +
                " component mapping did not report a fatal error");
  }
  std::printf("[host]    %-32s ok\n", "SampledColorRenderTargetViews");
}
#endif

void CheckSampledDepthResource() {
  ShaderRecompiler::IR::ImageResource resource{};
  resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
  resource.numeric_class = Prospero::TextureNumericClass::Float;
  resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
  resource.read = true;
  resource.depth_compare = true;
  Require("SampledDepthResource", "comparison read",
          IsSupportedSampledDepthResource(resource),
          "basic comparison-sampled depth resource was rejected");

  resource.depth_compare = false;
  Require("SampledDepthResource", "ordinary read",
          IsSupportedSampledDepthResource(resource),
          "basic non-comparison depth read was rejected");

  const auto basic = resource;
  resource.read = false;
  Require("SampledDepthResource", "read required",
          !IsSupportedSampledDepthResource(resource),
          "non-reading depth resource was accepted");
  resource = basic;
  resource.written = true;
  Require("SampledDepthResource", "write rejected",
          !IsSupportedSampledDepthResource(resource),
          "writable depth resource was accepted");
  resource = basic;
  resource.atomic = true;
  Require("SampledDepthResource", "atomic rejected",
          !IsSupportedSampledDepthResource(resource),
          "atomic depth resource was accepted");
  resource = basic;
  resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::Dynamic;
  resource.mip_count = 3;
  Require("SampledDepthResource", "dynamic mip read",
          IsSupportedSampledDepthResource(resource),
          "dynamic sampled depth mip resource was rejected");
  resource.depth_compare = true;
  Require("SampledDepthResource", "dynamic comparison mip read",
          IsSupportedSampledDepthResource(resource),
          "dynamic comparison depth mip resource was rejected");
  resource = basic;
  resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
  Require("SampledDepthResource", "singleton array accepted",
          IsSupportedSampledDepthResource(resource),
          "array depth resource was rejected");
  resource = basic;
  resource.numeric_class = Prospero::TextureNumericClass::Uint;
  Require("SampledDepthResource", "integer read",
          IsSupportedSampledDepthResource(resource),
          "read-only uint depth resource was rejected");
  resource.depth_compare = true;
  Require("SampledDepthResource", "integer comparison rejected",
          !IsSupportedSampledDepthResource(resource),
          "uint depth resource accepted comparison sampling");
  Require("SampledDepthResource", "R32 uint depth view",
          IsSupportedSampledDepthFormat(vk::Format::eD32Sfloat,
                                        vk::Format::eR32Uint) &&
              !IsSupportedSampledDepthFormat(vk::Format::eR32Uint,
                                             vk::Format::eR32Uint),
          "R32 uint depth-view compatibility diverged from its backing role");
  std::printf("[host]    %-32s ok\n", "SampledDepthResource");
}

void CheckImageTransitionState(RenderContext &renderer) {
  constexpr const char *name = "ImageTransitionState";
  auto &context = renderer.GetGraphics();
  CommandScheduler scheduler(renderer, context);
  const auto MakeInfo = [](vk::Format format, uint32_t levels,
                           uint32_t layers) {
    ImageInfo info{};
    info.pixel_format = format;
    info.guest_format = Prospero::BufferFormat::k8UNorm;
    info.type = Prospero::ImageType::kColor2D;
    info.extent = {4, 4, 1};
    info.resources = {levels, layers};
    info.pitch = 4;
    info.bytes_per_block = 1;
    info.samples = 1;
    info.tile_mode = Prospero::TileMode::kLinear;
    return info;
  };
  const auto graphics_stage = vk::PipelineStageFlagBits2::eAllGraphics |
                              vk::PipelineStageFlagBits2::eComputeShader;
  constexpr uint64_t copy_capacity = 128ull << 20;
  Require(
      name, "buffered-copy capacity bands",
      ImageTestAccess::CopyRows(32ull << 10, 4096, copy_capacity) == 4096 &&
          ImageTestAccess::CopyRows(32ull << 10, 4097, copy_capacity) == 4096 &&
          ImageTestAccess::CopyRows(copy_capacity + 4, 1, copy_capacity) == 0,
      "buffered image copy does not split at the fixed scratch capacity");

  Image image(context, scheduler, MakeInfo(vk::Format::eR8Unorm, 2, 3));
  auto barriers =
      image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
                        vk::AccessFlagBits2::eShaderRead, graphics_stage, {});
  Require(
      name, "initial full state",
      barriers.size() == 1 &&
          barriers[0].srcStageMask ==
              vk::PipelineStageFlagBits2::eAllCommands &&
          barriers[0].srcAccessMask == vk::AccessFlagBits2::eNone &&
          barriers[0].dstStageMask == graphics_stage &&
          barriers[0].dstAccessMask == vk::AccessFlagBits2::eShaderRead &&
          barriers[0].oldLayout == vk::ImageLayout::eUndefined &&
          barriers[0].newLayout == vk::ImageLayout::eShaderReadOnlyOptimal &&
          barriers[0].subresourceRange.aspectMask ==
              vk::ImageAspectFlagBits::eColor &&
          barriers[0].subresourceRange.levelCount == VK_REMAINING_MIP_LEVELS &&
          barriers[0].subresourceRange.layerCount ==
              VK_REMAINING_ARRAY_LAYERS &&
          image.backing.state.layout ==
              vk::ImageLayout::eShaderReadOnlyOptimal &&
          image.backing.state.access_mask == vk::AccessFlagBits2::eShaderRead &&
          image.backing.state.pl_stage == graphics_stage,
      "initial transition did not record or retain pinned state");
  Require(name, "repeat read elision",
          image
              .GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
                           vk::AccessFlagBits2::eShaderRead, graphics_stage, {})
              .empty(),
          "identical read-only transition emitted a barrier");
  barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
                               vk::AccessFlagBits2::eShaderRead |
                                   vk::AccessFlagBits2::eTransferRead,
                               graphics_stage, {});
  Require(name, "access-only transition",
          barriers.size() == 1 &&
              barriers[0].oldLayout == barriers[0].newLayout &&
              barriers[0].srcAccessMask == vk::AccessFlagBits2::eShaderRead,
          "same-layout access change was lost");

  const auto CheckRepeatedWrite = [&](vk::AccessFlagBits2 access,
                                      const char *label) {
    image.backing.state = {};
    image.backing.subresource_states.clear();
    (void)image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage,
                            {});
    Require(
        name, label,
        image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage, {})
                .size() == 1,
        "pinned repeated-write dependency was elided");
  };
  CheckRepeatedWrite(vk::AccessFlagBits2::eTransferWrite,
                     "repeat transfer write");
  CheckRepeatedWrite(vk::AccessFlagBits2::eShaderWrite, "repeat shader write");
  CheckRepeatedWrite(vk::AccessFlagBits2::eMemoryWrite, "repeat memory write");
  image.backing.state = {};
  image.backing.subresource_states.clear();
  const auto attachment_access = vk::AccessFlagBits2::eColorAttachmentRead |
                                 vk::AccessFlagBits2::eColorAttachmentWrite;
  (void)image.GetBarriers(vk::ImageLayout::eColorAttachmentOptimal,
                          attachment_access, graphics_stage, {});
  Require(name, "pinned attachment-write quirk",
          image
              .GetBarriers(vk::ImageLayout::eColorAttachmentOptimal,
                           attachment_access, graphics_stage, {})
              .empty(),
          "attachment writes diverged from the pinned repeated-write set");

  image.backing.state = {};
  image.backing.subresource_states.clear();
  (void)image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
                          vk::AccessFlagBits2::eShaderRead, graphics_stage, {});
  constexpr ImageSubresourceRange mip1_layer0{1, 1, 0, 1};
  barriers = image.GetBarriers(vk::ImageLayout::eGeneral,
                               vk::AccessFlagBits2::eShaderWrite,
                               graphics_stage, mip1_layer0);
  Require(name, "first partial",
          barriers.size() == 1 &&
              barriers[0].subresourceRange.baseMipLevel == 1 &&
              barriers[0].subresourceRange.baseArrayLayer == 0 &&
              image.backing.subresource_states.size() == 6 &&
              image.backing.subresource_states[3].layout ==
                  vk::ImageLayout::eGeneral &&
              image.backing.subresource_states[3].access_mask ==
                  vk::AccessFlagBits2::eShaderWrite &&
              image.backing.subresource_states[1].layout ==
                  vk::ImageLayout::eShaderReadOnlyOptimal,
          "partial state did not use mip-major indexing");
  constexpr ImageSubresourceRange mip0_layer1{0, 1, 1, 1};
  barriers = image.GetBarriers(
      vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
      vk::PipelineStageFlagBits2::eTransfer, mip0_layer1);
  Require(name, "second partial",
          barriers.size() == 1 &&
              barriers[0].subresourceRange.baseMipLevel == 0 &&
              barriers[0].subresourceRange.baseArrayLayer == 1 &&
              image.backing.subresource_states[1].layout ==
                  vk::ImageLayout::eTransferDstOptimal &&
              image.backing.subresource_states[1].access_mask ==
                  vk::AccessFlagBits2::eTransferWrite &&
              image.backing.subresource_states[3].layout ==
                  vk::ImageLayout::eGeneral,
          "second partial transition changed the wrong cell");
  const ImageSubresourceRange explicit_full{0, 2, 0, 3};
  barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
                               vk::AccessFlagBits2::eShaderRead, graphics_stage,
                               explicit_full);
  bool saw_mip0_layer1 = false;
  bool saw_mip1_layer0 = false;
  for (const auto &barrier : barriers) {
    saw_mip0_layer1 |=
        barrier.subresourceRange.baseMipLevel == 0 &&
        barrier.subresourceRange.baseArrayLayer == 1 &&
        barrier.oldLayout == vk::ImageLayout::eTransferDstOptimal &&
        barrier.srcStageMask == vk::PipelineStageFlagBits2::eTransfer;
    saw_mip1_layer0 |=
        barrier.subresourceRange.baseMipLevel == 1 &&
        barrier.subresourceRange.baseArrayLayer == 0 &&
        barrier.oldLayout == vk::ImageLayout::eGeneral &&
        barrier.srcAccessMask == vk::AccessFlagBits2::eShaderWrite;
  }
  Require(name, "partial normalization",
          barriers.size() == 2 && saw_mip0_layer1 && saw_mip1_layer0 &&
              image.backing.subresource_states.empty() &&
              image.backing.state.layout ==
                  vk::ImageLayout::eShaderReadOnlyOptimal &&
              image.backing.state.access_mask ==
                  vk::AccessFlagBits2::eShaderRead &&
              image.backing.state.pl_stage == graphics_stage,
          "full transition did not reconcile and collapse split state");
  barriers = image.GetBarriers(vk::ImageLayout::eTransferSrcOptimal,
                               vk::AccessFlagBits2::eTransferRead,
                               vk::PipelineStageFlagBits2::eTransfer, {});
  Require(name, "normalized full transition",
          barriers.size() == 1 &&
              barriers[0].subresourceRange.levelCount ==
                  VK_REMAINING_MIP_LEVELS &&
              barriers[0].subresourceRange.layerCount ==
                  VK_REMAINING_ARRAY_LAYERS &&
              barriers[0].srcStageMask == graphics_stage &&
              barriers[0].dstStageMask == vk::PipelineStageFlagBits2::eTransfer,
          "normalized state did not return to one full barrier");

  auto depth_info = MakeInfo(vk::Format::eD32SfloatS8Uint, 1, 1);
  depth_info.guest_format = Prospero::BufferFormat::k32Float;
  depth_info.bytes_per_block = 4;
  depth_info.tile_mode = Prospero::TileMode::kDepth;
  Image depth(context, scheduler, depth_info);
  barriers =
      depth.GetBarriers(vk::ImageLayout::eDepthStencilReadOnlyOptimal,
                        vk::AccessFlagBits2::eShaderRead, graphics_stage, {});
  Require(name, "depth stencil aspects",
          barriers.size() == 1 && barriers[0].subresourceRange.aspectMask ==
                                      (vk::ImageAspectFlagBits::eDepth |
                                       vk::ImageAspectFlagBits::eStencil),
          "depth/stencil transition did not use the full image aspect");

  HW::Context registers{};
  HW::UserConfig user_config{};
  HW::Shader shaders{};
  scheduler.Begin(registers, user_config, shaders);
  image.backing.state = {};
  image.backing.subresource_states.clear();
  image.Transit(vk::ImageLayout::eGeneral,
                vk::AccessFlagBits2::eShaderRead |
                    vk::AccessFlagBits2::eTransferRead,
                {}, scheduler.Current().Handle());
  Require(name, "mixed access stages",
          image.backing.state.pl_stage ==
              (graphics_stage | vk::PipelineStageFlagBits2::eTransfer),
          "mixed shader/transfer access omitted a required pipeline stage");

  auto source_info = depth_info;
  source_info.extent = {2, 2, 1};
  source_info.pitch = 2;
  source_info.stencil = {0x1000, 4};
  auto destination_info = depth_info;
  destination_info.resources.levels = 2;
  destination_info.stencil = {0x2000, 16};
  Image source(context, scheduler, source_info);
  Image destination(context, scheduler, destination_info);
  StreamBuffer upload(context, scheduler, MemoryUsage::Upload, 4096);
  StreamBuffer download(context, scheduler, MemoryUsage::Download, 4096);
  const auto [upload_data, upload_offset] = upload.Map(128, 4);
  Require(name, "depth/stencil upload map", upload_data != nullptr,
          "combined depth/stencil upload allocation failed");
  constexpr std::array<uint32_t, 4> expected_depth{0x3e800000u, 0x3f000000u,
                                                   0x3f400000u, 0x3f800000u};
  constexpr std::array<uint8_t, 4> expected_stencil{0x12, 0x34, 0x56, 0x78};
  std::memcpy(upload_data, expected_depth.data(), sizeof(expected_depth));
  std::memcpy(upload_data + 64, expected_stencil.data(),
              sizeof(expected_stencil));
  upload.Commit();
  std::array<vk::BufferImageCopy, 2> source_copies{};
  source_copies[0].bufferOffset = upload_offset;
  source_copies[0].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0,
                                       1};
  source_copies[0].imageExtent = {2, 2, 1};
  source_copies[1].bufferOffset = upload_offset + 64;
  source_copies[1].imageSubresource = {vk::ImageAspectFlagBits::eStencil, 0, 0,
                                       1};
  source_copies[1].imageExtent = {2, 2, 1};
  source.Upload(source_copies, upload.Handle(), upload_offset, 128);
  destination.CopyMip(source, 1, 0);

  const auto [download_data, download_offset] = download.Map(128, 4);
  Require(name, "depth/stencil download map", download_data != nullptr,
          "combined depth/stencil download allocation failed");
  download.Commit();
  auto destination_copies = source_copies;
  destination_copies[0].bufferOffset = download_offset;
  destination_copies[0].imageSubresource.mipLevel = 1;
  destination_copies[1].bufferOffset = download_offset + 64;
  destination_copies[1].imageSubresource.mipLevel = 1;
  destination.Download(destination_copies, download.Handle(), download_offset,
                       128);

  auto buffered_info = MakeInfo(vk::Format::eR8Unorm, 1, 1);
  buffered_info.extent = {8, 4, 1};
  buffered_info.pitch = 8;
  buffered_info.mip_layout[0] = {0, 32, 8, 4};
  Image buffered_source(context, scheduler, buffered_info);
  Image buffered_destination(context, scheduler, buffered_info);
  constexpr std::array<uint8_t, 32> buffered_expected{
      0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa,
      0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x21, 0x32, 0x43, 0x54, 0x65,
      0x76, 0x87, 0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f};
  const auto [buffered_upload_data, buffered_upload_offset] =
      upload.Map(buffered_expected.size(), 4);
  Require(name, "buffered-copy upload map", buffered_upload_data != nullptr,
          "buffered-copy source allocation failed");
  std::memcpy(buffered_upload_data, buffered_expected.data(),
              buffered_expected.size());
  upload.Commit();
  vk::BufferImageCopy buffered_region{};
  buffered_region.bufferOffset = buffered_upload_offset;
  buffered_region.bufferRowLength = buffered_info.pitch;
  buffered_region.bufferImageHeight = buffered_info.extent.height;
  buffered_region.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1};
  buffered_region.imageExtent = buffered_info.extent;
  buffered_source.Upload(std::span{&buffered_region, 1}, upload.Handle(),
                         buffered_upload_offset, buffered_expected.size());
  Buffer copy_scratch(context, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
                      16);
  StreamBuffer copy_parameters(context, scheduler, MemoryUsage::Stream, 4096);
  TileManager copy_tiler(context, scheduler, copy_parameters);
  buffered_destination.CopyImageWithBuffer(buffered_source, copy_scratch, copy_tiler);
  const auto [buffered_download_data, buffered_download_offset] =
      download.Map(buffered_expected.size(), 4);
  Require(name, "buffered-copy download map", buffered_download_data != nullptr,
          "buffered-copy destination allocation failed");
  download.Commit();
  buffered_region.bufferOffset = buffered_download_offset;
  buffered_destination.Download(std::span{&buffered_region, 1},
                                download.Handle(), buffered_download_offset,
                                buffered_expected.size());
  scheduler.Finish();
  download.Invalidate(download_offset, 128);
  download.Invalidate(buffered_download_offset, buffered_expected.size());
  Require(name, "depth/stencil mip contents",
          std::memcmp(download_data, expected_depth.data(),
                      sizeof(expected_depth)) == 0 &&
              std::memcmp(download_data + 64, expected_stencil.data(),
                          sizeof(expected_stencil)) == 0,
          "combined depth/stencil mip copy lost an aspect");
  Require(name, "buffered-copy chunk contents",
          std::memcmp(buffered_download_data, buffered_expected.data(),
                      buffered_expected.size()) == 0,
          "real buffered image copy lost a row across its scratch boundary");

  std::printf("[host]    %-32s ok\n", name);
}

void CheckDepthTextureEncoding() {
  const ShaderTextureResource disabled_sampler_tweaks{{
      0x05135600u,
      0xc1600000u,
      0x010dc1dfu,
      0x91800924u,
      0x00000000u,
      0x00000000u,
      0x00000000u,
      0x00000000u,
  }};
  Require("DepthTextureEncoding", "disabled sampler tweaks",
          disabled_sampler_tweaks.PerfMod5() == 0 &&
              IsSupportedDepthTextureEncoding(disabled_sampler_tweaks),
          "valid sampler modulation factor zero was rejected");

  const ShaderTextureResource uncompressed_msaa{{
      0x00705d00u,
      0xc1600000u,
      0x010dc1dfu,
      0xe1810924u,
      0x00000000u,
      0x00700010u,
      0x00000000u,
      0x00000000u,
  }};
  Require("DepthTextureEncoding", "uncompressed 2x MSAA depth",
          IsSupportedDepthTextureEncoding(uncompressed_msaa),
          "valid uncompressed MSAA depth descriptor required an HTILE "
          "compatibility flag");

  auto r128_msaa = uncompressed_msaa;
  std::fill(r128_msaa.fields + 4, r128_msaa.fields + 8, 0u);
  Require("DepthTextureEncoding", "R128 uncompressed 2x MSAA depth",
          IsSupportedDepthTextureEncoding(r128_msaa, true),
          "valid R128 MSAA depth descriptor required omitted dwords");

  const ShaderTextureResource compressed_descriptor{{
      0x0104d500u,
      0xc1600000u,
      0x021bc3bfu,
      0x91800924u,
      0x00000000u,
      0x00700000u,
      0x80280000u,
      0x000106d4u,
  }};
  Require("DepthTextureEncoding", "compressed HTILE descriptor",
          IsSupportedDepthTextureEncoding(compressed_descriptor),
          "valid compressed depth descriptor required a prior depth target");

  const ShaderTextureResource disabled_metadata{{
      0x20018100u, 0xc1600000u, 0x021bc1dfu, 0x91800204u,
      0x00000000u, 0x00700000u, 0x00000000u, 0x0020037fu,
  }};
  auto retained_metadata_address = compressed_descriptor;
  retained_metadata_address.fields[6] &= 0xff000000u;
  Require("DepthTextureEncoding", "disabled metadata with retained address",
          IsSupportedDepthTextureEncoding(disabled_metadata) &&
              IsSupportedDepthTextureEncoding(retained_metadata_address),
          "uncompressed depth rejected inactive metadata address bits");

  const ShaderTextureResource first_use_depth{{
      0x0225fc00u, 0x01600000u, 0x00000000u, 0x91800924u,
      0x00000000u, 0x00700000u, 0x80280000u, 0x000225fdu,
  }};
  Require("DepthTextureEncoding", "PPSA30490 first-use 1x1 depth",
          IsSupportedDepthTextureEncoding(first_use_depth),
          "captured comparison texture required tracked HTILE metadata");

  auto missing_metadata = compressed_descriptor;
  missing_metadata.fields[6] &= 0x00ffffffu;
  missing_metadata.fields[7] = 0;
  auto out_of_range_metadata = compressed_descriptor;
  out_of_range_metadata.fields[7] = TRACKER_ADDRESS_SIZE >> 16u;
  auto unaligned_metadata = compressed_descriptor;
  unaligned_metadata.fields[6] ^= 1u << 24u;
  auto dcc_only_control = compressed_descriptor;
  dcc_only_control.fields[6] |= 1u << 22u;
  auto non_msaa_iterate_256 = compressed_descriptor;
  non_msaa_iterate_256.fields[6] |= 1u << 10u;
  Require(
      "DepthTextureEncoding", "invalid HTILE encoding",
      !IsSupportedDepthTextureEncoding(missing_metadata) &&
          !IsSupportedDepthTextureEncoding(out_of_range_metadata) &&
          !IsSupportedDepthTextureEncoding(unaligned_metadata) &&
          !IsSupportedDepthTextureEncoding(dcc_only_control) &&
          !IsSupportedDepthTextureEncoding(non_msaa_iterate_256),
      "compressed sampled depth accepted an unsupported address or control");

  std::printf("[host]    %-32s ok\n", "DepthTextureEncoding");
}

ShaderRecompiler::IR::ImageResource BasicStorageTextureResource() {
  ShaderRecompiler::IR::ImageResource resource{};
  resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
  resource.numeric_class = Prospero::TextureNumericClass::Float;
  resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
  resource.read = true;
  resource.written = true;
  return resource;
}

ShaderTextureResource BasicStorageTextureDescriptor() {
  return {{0x00785d00u, 0x04700000u, 0x00080008u, 0xa1b00facu, 0x00000020u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderRecompiler::IR::ImageResource BasicLinearStorageTextureResource() {
  auto resource = BasicStorageTextureResource();
  resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
  resource.read = false;
  return resource;
}

ShaderTextureResource BasicLinearStorageTextureDescriptor() {
  return {{0x04bcc401u, 0xc3800000u, 0x021bc3bfu, 0x900003acu, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderRecompiler::IR::ImageResource BasicBgraStorageTextureResource() {
  auto resource = BasicStorageTextureResource();
  resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
  resource.read = false;
  return resource;
}

ShaderTextureResource BasicBgraStorageTextureDescriptor() {
  return {{0x007c6500u, 0xc3800000u, 0x010dc1dfu, 0x91b00f2eu, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderTextureResource Ppsa06228R11G11B10StorageTextureDescriptor() {
  return {{0x10c6b500u, 0xc2400000u, 0x010dc1dfu, 0x91b003acu, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderTextureResource Ppsa01530MaxMipStorageTextureDescriptor() {
  return {{0x04a42900u, 0xc3e00000u, 0x000fc00fu, 0x91b0022cu, 0x00000000u,
           0x00700050u, 0x00000000u, 0x00000000u}};
}

ShaderTextureResource Ppsa01340MipRangeStorageTextureDescriptor() {
  return {{0x0294dc00u, 0xc4700000u, 0x00b3c13fu, 0x91b31facu, 0x00000000u,
           0x00700030u, 0xb07b0000u, 0x0002ac3cu}};
}

ShaderTextureResource Ppsa01340OverwideMipStorageTextureDescriptor() {
  return {{0x0346c000u, 0xc4b00000u, 0x007fc07fu, 0x90990facu, 0x00000000u,
           0x00700080u, 0x00000000u, 0x00000000u}};
}

ShaderRecompiler::IR::ImageResource Ppsa01530MaxMipStorageTextureResource() {
  auto resource = BasicBgraStorageTextureResource();
  resource.numeric_class = Prospero::TextureNumericClass::Uint;
  return resource;
}

ShaderTextureResource Ppsa02527R16FloatStorageTextureDescriptor() {
  return {{0x00ce3500u, 0xc0d00000u, 0x010dc1dfu, 0x91b00204u, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderTextureResource Ppsa02527R32FloatStorageTextureDescriptor() {
  return {{0x00cea900u, 0xc1600000u, 0x0086c0efu, 0x91b00204u, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderTextureResource Ppsa02527R8UnormStorageTextureDescriptor() {
  return {{0x00c7d500u, 0xc0100000u, 0x0086c0efu, 0x91b00204u, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderTextureResource BasicYzwxStorageTextureDescriptor() {
  return {{0x00627801u, 0xc4d00000u, 0x0001c001u, 0x900009f5u, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderRecompiler::IR::ImageResource BasicArrayStorageTextureResource() {
  auto resource = BasicLinearStorageTextureResource();
  resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
  return resource;
}

ShaderTextureResource BasicArrayStorageTextureDescriptor() {
  return {{0x20179000u, 0x03800000u, 0x00000000u, 0xd1b00f2eu, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderRecompiler::IR::ImageResource BasicUintArrayStorageTextureResource() {
  auto resource = BasicArrayStorageTextureResource();
  resource.numeric_class = Prospero::TextureNumericClass::Uint;
  return resource;
}

ShaderTextureResource BasicUintArrayStorageTextureDescriptor() {
  return {{0x20179200u, 0x01400000u, 0x00000000u, 0xd1b00204u, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderTextureResource Standard4KBUintArrayStorageTextureDescriptor() {
  return {{0x006c6600u, 0x01400000u, 0x00000000u, 0xd0500204u, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderTextureResource Standard64KBStorageTextureDescriptor() {
  return {{0x011fab00u, 0xc3800000u, 0x0003c003u, 0xd0900facu, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderRecompiler::IR::ImageResource Ppsa14053DepthTileStorageTextureResource() {
  return BasicUintArrayStorageTextureResource();
}

ShaderTextureResource Ppsa14053DepthTileStorageTextureDescriptor() {
  return {{0x20144c00u, 0x00500000u, 0x00000000u, 0xd1800204u, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderTextureResource Ppsa10112D16StorageTextureDescriptor() {
  return {{0x205b9000u, 0xc0700000u, 0x0021803bu, 0xd1800204u, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderRecompiler::IR::ImageResource BasicUintVolumeStorageTextureResource() {
  auto resource = BasicStorageTextureResource();
  resource.numeric_class = Prospero::TextureNumericClass::Uint;
  resource.read = false;
  return resource;
}

ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() {
  return {{0x20180600u, 0xc0b00000u, 0x0003c003u, 0xa0000004u, 0x0000000fu,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

ShaderRecompiler::IR::ImageResource AtomicStorageTextureResource() {
  auto resource = BasicLinearStorageTextureResource();
  resource.numeric_class = Prospero::TextureNumericClass::Uint;
  resource.read = true;
  resource.atomic = true;
  return resource;
}

ShaderTextureResource AtomicStorageTextureDescriptor() {
  return {{0x304bb700u, 0xc1400000u, 0x0000001fu, 0x91b00204u, 0x00000000u,
           0x00700000u, 0x00000000u, 0x00000000u}};
}

[[noreturn]] void RunStorageTextureDescriptorDeathCase(const char *kind) {
  auto resource = BasicStorageTextureResource();
  auto descriptor = BasicStorageTextureDescriptor();
  if (std::strcmp(kind, "linear-rgb1-read") == 0) {
    resource = BasicLinearStorageTextureResource();
    descriptor = BasicLinearStorageTextureDescriptor();
    resource.read = true;
  } else if (std::strcmp(kind, "bgra-read") == 0) {
    resource = BasicBgraStorageTextureResource();
    descriptor = BasicBgraStorageTextureDescriptor();
    resource.read = true;
  } else if (std::strcmp(kind, "r16-float-read") == 0) {
    resource = BasicBgraStorageTextureResource();
    descriptor = Ppsa02527R16FloatStorageTextureDescriptor();
    resource.read = true;
  } else if (std::strcmp(kind, "r8-unorm-read") == 0) {
    resource = BasicBgraStorageTextureResource();
    descriptor = Ppsa02527R8UnormStorageTextureDescriptor();
    resource.read = true;
  } else if (std::strcmp(kind, "yzwx-read") == 0) {
    resource = BasicLinearStorageTextureResource();
    descriptor = BasicYzwxStorageTextureDescriptor();
    resource.read = true;
  } else if (std::strcmp(kind, "reserved-swizzle") == 0) {
    resource = BasicLinearStorageTextureResource();
    descriptor = BasicLinearStorageTextureDescriptor();
    descriptor.fields[3] =
        (descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 2);
  } else if (std::strcmp(kind, "resource") == 0) {
    resource.written = false;
  } else if (std::strcmp(kind, "type") == 0) {
    descriptor.fields[3] =
        (descriptor.fields[3] & 0x0fffffffu) |
        (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
  } else if (std::strcmp(kind, "standard256b-volume") == 0) {
    descriptor.fields[3] =
        (descriptor.fields[3] & ~(0x1fu << 20u)) |
        (static_cast<uint32_t>(Prospero::TileMode::kStandard256B) << 20u);
  } else if (std::strcmp(kind, "inverted-mip-range") == 0) {
    descriptor.fields[3] |= 1u << 12u;
    descriptor.fields[5] |= 1u << 4u;
  } else if (std::strcmp(kind, "swizzle") == 0) {
    descriptor.fields[3] =
        (descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 1);
  } else if (std::strcmp(kind, "array-base-out-of-range") == 0) {
    resource = BasicArrayStorageTextureResource();
    descriptor = BasicArrayStorageTextureDescriptor();
    descriptor.fields[4] |= 1u << 16u;
  } else if (std::strcmp(kind, "reserved") == 0) {
    descriptor.fields[1] |= 1u << 29u;
  } else if (std::strcmp(kind, "uint-format") == 0) {
    descriptor.fields[1] =
        (descriptor.fields[1] & ~0x1ff00000u) |
        (static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8UInt) << 20u);
  } else if (std::strcmp(kind, "uint-resource-float-format") == 0) {
    resource = BasicUintArrayStorageTextureResource();
    descriptor = BasicArrayStorageTextureDescriptor();
  } else if (std::strcmp(kind, "atomic-format") == 0) {
    resource = AtomicStorageTextureResource();
    descriptor = AtomicStorageTextureDescriptor();
    descriptor.fields[1] =
        (descriptor.fields[1] & ~0x1ff00000u) |
        (static_cast<uint32_t>(Prospero::BufferFormat::k8UInt) << 20u);
  } else if (std::strcmp(kind, "depth-tile-read") == 0) {
    resource = Ppsa14053DepthTileStorageTextureResource();
    descriptor = Ppsa14053DepthTileStorageTextureDescriptor();
    resource.read = true;
  } else if (std::strcmp(kind, "depth-tile-extent") == 0) {
    resource = Ppsa14053DepthTileStorageTextureResource();
    descriptor = Ppsa14053DepthTileStorageTextureDescriptor();
    resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
    descriptor.fields[3] =
        (descriptor.fields[3] & 0x0fffffffu) |
        (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
    descriptor.fields[4] |= 1u;
  } else if (std::strcmp(kind, "depth-tile-fmask") == 0) {
    resource = BasicArrayStorageTextureResource();
    descriptor = Ppsa10112D16StorageTextureDescriptor();
    descriptor.fields[1] =
        (descriptor.fields[1] & ~(0x1ffu << 20u)) |
        (static_cast<uint32_t>(Prospero::BufferFormat::kFmask8_S4_F4) << 20u);
  } else {
    std::_Exit(0x7e);
  }
  ValidateStorageTexture(resource, descriptor, 0x10000);
  std::_Exit(0x7f);
}

#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckBasicStorageTextureDescriptor() {
  const auto descriptor = BasicStorageTextureDescriptor();
  Require("BasicStorageTexture", "descriptor",
          descriptor.Base40() == 0x785d0000ull &&
              descriptor.Width5() + 1u == 33 &&
              descriptor.Height5() + 1u == 33 && descriptor.Depth() + 1u == 33,
          "basic 3D storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicStorageTextureResource(), descriptor, 0x10000);

  const ShaderTextureResource extended{{0x204aca00u, 0xc4700000u, 0x000fc00fu,
                                        0xa1b00facu, 0x0000003fu, 0x00700000u,
                                        0x006b0000u, 0x00204b0au}};
  auto extended_resource = BasicStorageTextureResource();
  extended_resource.read = false;
  Require("BasicStorageTexture", "extended descriptor",
          extended.fields[6] != 0 && extended.fields[7] != 0,
          "extended 3D storage descriptor fixture is malformed");
  ValidateStorageTexture(extended_resource, extended, 0x400000);

  const auto linear = BasicLinearStorageTextureDescriptor();
  Require("BasicStorageTexture", "linear descriptor",
          linear.Base40() == 0x4bcc40100ull && linear.Width5() + 1u == 3840 &&
              linear.Height5() + 1u == 2160 && linear.Depth() + 1u == 1 &&
              linear.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
              linear.TileMode() == Prospero::TileMode::kLinear,
          "PPSA07429 linear 2D storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicLinearStorageTextureResource(), linear,
                         0x1fa4000);

  const auto bgra = BasicBgraStorageTextureDescriptor();
  Require("BasicStorageTexture", "BGRA descriptor",
          bgra.Base40() == 0x7c650000ull && bgra.Width5() + 1u == 1920 &&
              bgra.Height5() + 1u == 1080 && bgra.Depth() + 1u == 1 &&
              bgra.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
              bgra.TileMode() == Prospero::TileMode::kRenderTarget &&
              bgra.DstSelXYZW() == DstSel(6, 5, 4, 7),
          "PPSA02604 BGRA 2D storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicBgraStorageTextureResource(), bgra, 0x870000);

  const ShaderTextureResource r128{
      {0x0202e500u, 0xc8200000u, 0x010dc1dfu, 0x91b00facu, 0, 0, 0, 0}};
  auto r128_resource = BasicBgraStorageTextureResource();
  r128_resource.r128 = true;
  Require("BasicStorageTexture", "PPSA01736 R128 descriptor",
          r128.Base40() == 0x202e50000ull && r128.Width5() + 1u == 1920 &&
              r128.Height5() + 1u == 1080 && r128.Depth() + 1u == 1 &&
              r128.Format() == Prospero::BufferFormat::k8_8_8_8Srgb &&
              r128.TileMode() == Prospero::TileMode::kRenderTarget &&
              r128.DstSelXYZW() == DstSel(4, 5, 6, 7),
          "PPSA01736 R128 storage descriptor fixture is malformed");
  ValidateStorageTexture(r128_resource, r128, 0x870000);

  const auto r11g11b10 = Ppsa06228R11G11B10StorageTextureDescriptor();
  Require("BasicStorageTexture", "PPSA06228 R11G11B10 descriptor",
          r11g11b10.Base40() == 0x10c6b50000ull &&
              r11g11b10.Width5() + 1u == 1920 &&
              r11g11b10.Height5() + 1u == 1080 && r11g11b10.Depth() + 1u == 1 &&
              r11g11b10.Format() == Prospero::BufferFormat::k11_11_10Float &&
              r11g11b10.TileMode() == Prospero::TileMode::kRenderTarget &&
              r11g11b10.DstSelXYZW() == DstSel(4, 5, 6, 1),
          "PPSA06228 R11G11B10 storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicBgraStorageTextureResource(), r11g11b10,
                         0x870000);
  ValidateStorageColorView(vk::Format::eB8G8R8A8Unorm,
                           vk::Format::eB10G11R11UfloatPack32,
                           r11g11b10.DstSelXYZW());

  const auto max_mip = Ppsa01530MaxMipStorageTextureDescriptor();
  Require("BasicStorageTexture", "PPSA01530 max-mip descriptor",
          max_mip.Base40() == 0x4a4290000ull && max_mip.Width5() + 1u == 64 &&
              max_mip.Height5() + 1u == 64 && max_mip.Depth() + 1u == 1 &&
              max_mip.BaseLevel() == 0 && max_mip.LastLevel() == 0 &&
              max_mip.MaxMip() == 5 &&
              max_mip.Format() == Prospero::BufferFormat::k32_32UInt &&
              max_mip.TileMode() == Prospero::TileMode::kRenderTarget &&
              max_mip.DstSelXYZW() == DstSel(4, 5, 0, 1),
          "PPSA01530 max-mip storage descriptor fixture is malformed");
  ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), max_mip,
                         0x20000);
  auto mip_one = max_mip;
  mip_one.fields[3] |= (1u << 12u) | (1u << 16u);
  Require("BasicStorageTexture", "PPSA01530 mip-one descriptor",
          mip_one.BaseLevel() == 1 && mip_one.LastLevel() == 1 &&
              mip_one.MaxMip() == 5,
          "PPSA01530 mip-one storage descriptor fixture is malformed");
  ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), mip_one,
                         0x20000);

  const auto mip_range = Ppsa01340MipRangeStorageTextureDescriptor();
  Require("BasicStorageTexture", "PPSA01340 mip-range descriptor",
          mip_range.Base40() == 0x294dc0000ull &&
              mip_range.Width5() + 1u == 1280 &&
              mip_range.Height5() + 1u == 720 && mip_range.Depth() + 1u == 1 &&
              mip_range.BaseLevel() == 1 && mip_range.LastLevel() == 3 &&
              mip_range.MaxMip() == 3 &&
              mip_range.Format() == Prospero::BufferFormat::k16_16_16_16Float &&
              mip_range.TileMode() == Prospero::TileMode::kRenderTarget &&
              mip_range.DstSelXYZW() == DstSel(4, 5, 6, 7),
          "PPSA01340 mip-range storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicBgraStorageTextureResource(), mip_range,
                         0xa30000);

  const auto overwide_mip = Ppsa01340OverwideMipStorageTextureDescriptor();
  Require(
      "BasicStorageTexture", "PPSA01340 over-wide mip view",
      overwide_mip.Base40() == 0x346c00000ull &&
          overwide_mip.Width5() + 1u == 512 &&
          overwide_mip.Height5() + 1u == 512 && overwide_mip.BaseLevel() == 0 &&
          overwide_mip.LastLevel() == 9 && overwide_mip.MaxMip() == 8 &&
          overwide_mip.Format() == Prospero::BufferFormat::k32_32_32_32UInt &&
          overwide_mip.TileMode() == Prospero::TileMode::kStandard64KB,
      "captured over-wide storage mip view is malformed");
  TileSizeAlign overwide_size{};
  TileGetTextureTotalSize(overwide_mip.Format(), overwide_mip.Width5() + 1u,
                          overwide_mip.Height5() + 1u, 1,
                          overwide_mip.MaxMip() + 1u, overwide_mip.TileMode(),
                          false, overwide_size);
  Require("BasicStorageTexture", "PPSA01340 over-wide mip footprint",
          overwide_size.size == 0x560000 && overwide_size.align == 0x10000,
          "captured nine-level Standard64KB footprint changed");
  ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), overwide_mip,
                         overwide_size.size);

  const auto r16_float = Ppsa02527R16FloatStorageTextureDescriptor();
  Require("BasicStorageTexture", "PPSA02527 R16F descriptor",
          r16_float.Base40() == 0xce350000ull &&
              r16_float.Width5() + 1u == 1920 &&
              r16_float.Height5() + 1u == 1080 && r16_float.Depth() + 1u == 1 &&
              r16_float.Format() == Prospero::BufferFormat::k16Float &&
              r16_float.TileMode() == Prospero::TileMode::kRenderTarget &&
              r16_float.DstSelXYZW() == DstSel(4, 0, 0, 1),
          "PPSA02527 R16F 2D storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicBgraStorageTextureResource(), r16_float,
                         0x480000);

  const auto r32_float = Ppsa02527R32FloatStorageTextureDescriptor();
  Require("BasicStorageTexture", "PPSA02527 R32F descriptor",
          r32_float.Base40() == 0xcea90000ull &&
              r32_float.Width5() + 1u == 960 &&
              r32_float.Height5() + 1u == 540 && r32_float.Depth() + 1u == 1 &&
              r32_float.Format() == Prospero::BufferFormat::k32Float &&
              r32_float.TileMode() == Prospero::TileMode::kRenderTarget &&
              r32_float.DstSelXYZW() == DstSel(4, 0, 0, 1),
          "PPSA02527 R32F 2D storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicBgraStorageTextureResource(), r32_float,
                         0x280000);

  const auto r8_unorm = Ppsa02527R8UnormStorageTextureDescriptor();
  Require("BasicStorageTexture", "PPSA02527 R8 UNORM descriptor",
          r8_unorm.Base40() == 0xc7d50000ull && r8_unorm.Width5() + 1u == 960 &&
              r8_unorm.Height5() + 1u == 540 && r8_unorm.Depth() + 1u == 1 &&
              r8_unorm.Format() == Prospero::BufferFormat::k8UNorm &&
              r8_unorm.TileMode() == Prospero::TileMode::kRenderTarget &&
              r8_unorm.DstSelXYZW() == DstSel(4, 0, 0, 1),
          "PPSA02527 R8 UNORM 2D storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicBgraStorageTextureResource(), r8_unorm, 0xc0000);

  const auto yzwx = BasicYzwxStorageTextureDescriptor();
  Require("BasicStorageTexture", "YZWX descriptor",
          yzwx.Base40() == 0x62780100ull && yzwx.Width5() + 1u == 8 &&
              yzwx.Height5() + 1u == 8 && yzwx.Depth() + 1u == 1 &&
              yzwx.Format() == Prospero::BufferFormat::k32_32_32_32Float &&
              yzwx.TileMode() == Prospero::TileMode::kLinear &&
              yzwx.DstSelXYZW() == DstSel(5, 6, 7, 4),
          "PPSA04181 linear YZWX storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicLinearStorageTextureResource(), yzwx, 0x800);
  auto all_swizzles = yzwx;
  all_swizzles.fields[1] =
      (all_swizzles.fields[1] & ~0x1ff00000u) |
      (static_cast<uint32_t>(Prospero::BufferFormat::k16_16_16_16Float) << 20u);
  uint32_t valid_storage_swizzles = 0;
  for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) {
    if (!IsValidImageSwizzle(swizzle)) {
      continue;
    }
    all_swizzles.fields[3] = (all_swizzles.fields[3] & ~0xfffu) | swizzle;
    ValidateStorageTexture(BasicLinearStorageTextureResource(), all_swizzles,
                           0x800);
    ValidateStorageColorView(vk::Format::eR16G16B16A16Sfloat,
                             vk::Format::eR16G16B16A16Sfloat, swizzle);
    ValidateStorageColorView(vk::Format::eR8G8B8A8Srgb,
                             vk::Format::eR8G8B8A8Unorm, swizzle);
    ValidateStorageColorView(vk::Format::eB8G8R8A8Srgb,
                             vk::Format::eR8G8B8A8Unorm, swizzle);
    valid_storage_swizzles++;
  }
  Require("BasicStorageTexture", "all write swizzles",
          valid_storage_swizzles == 1296,
          "valid write-only storage image mappings were rejected");

  const ShaderTextureResource cube{{0x025bca00u, 0xc4700000u, 0x003fc03fu,
                                    0xb1b00facu, 0x00000005u, 0x00700080u,
                                    0x00000000u, 0x00000000u}};
  auto cube_resource = BasicArrayStorageTextureResource();
  cube_resource.cube = true;
  Require("BasicStorageTexture", "PPSA07429 cube descriptor",
          cube.Base40() == 0x25bca0000ull && cube.Width5() + 1u == 256 &&
              cube.Height5() + 1u == 256 && cube.Depth() + 1u == 6 &&
              cube.BaseArray5() == 0 && cube.MaxMip() == 8 &&
              cube.Type() == Prospero::ImageType::kCube &&
              cube.Format() == Prospero::BufferFormat::k16_16_16_16Float &&
              cube.TileMode() == Prospero::TileMode::kRenderTarget &&
              cube.DstSelXYZW() == DstSel(4, 5, 6, 7),
          "PPSA07429 cube storage descriptor fixture is malformed");
  ValidateStorageTexture(cube_resource, cube, 0x420000);

  const auto array = BasicArrayStorageTextureDescriptor();
  Require("BasicStorageTexture", "2D-array descriptor",
          array.Base40() == 0x2017900000ull && array.Width5() + 1u == 1 &&
              array.Height5() + 1u == 1 && array.Depth() + 1u == 1 &&
              array.BaseArray5() == 0 &&
              array.Type() == Prospero::ImageType::kColor2DArray &&
              array.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
              array.TileMode() == Prospero::TileMode::kRenderTarget &&
              array.DstSelXYZW() == DstSel(6, 5, 4, 7),
          "PPSA21268 2D-array storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicArrayStorageTextureResource(), array, 0x10000);
  const ShaderTextureResource mip_array{{0x20268d00u, 0xc4700000u, 0x001fc01fu,
                                         0xd1b11facu, 0x00000000u, 0x00700070u,
                                         0x00000000u, 0x00000000u}};
  Require("BasicStorageTexture", "PPSA14457 mip-one 2D-array descriptor",
          mip_array.BaseLevel() == 1 && mip_array.LastLevel() == 1 &&
              mip_array.MaxMip() == 7 &&
              mip_array.Type() == Prospero::ImageType::kColor2DArray &&
              mip_array.Depth() == 0 && mip_array.BaseArray5() == 0,
          "PPSA14457 mip-one 2D-array storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicArrayStorageTextureResource(), mip_array,
                         0x30000);

  const auto uint_array = BasicUintArrayStorageTextureDescriptor();
  Require("BasicStorageTexture", "uint 2D-array descriptor",
          uint_array.Base40() == 0x2017920000ull &&
              uint_array.Width5() + 1u == 1 && uint_array.Height5() + 1u == 1 &&
              uint_array.Depth() + 1u == 1 && uint_array.BaseArray5() == 0 &&
              uint_array.Type() == Prospero::ImageType::kColor2DArray &&
              uint_array.Format() == Prospero::BufferFormat::k32UInt &&
              uint_array.TileMode() == Prospero::TileMode::kRenderTarget &&
              uint_array.DstSelXYZW() == DstSel(4, 0, 0, 1),
          "PPSA21268 uint 2D-array storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicUintArrayStorageTextureResource(), uint_array,
                         0x10000);

  const ShaderTextureResource standard256b{{
      0x204e4900u,
      0x43c00000u,
      0x00004000u,
      0x90100facu,
      0x00000000u,
      0x00700000u,
      0x00000000u,
      0x00000000u,
  }};
  auto standard256b_resource = BasicBgraStorageTextureResource();
  standard256b_resource.numeric_class = Prospero::TextureNumericClass::Uint;
  TileSizeAlign standard256b_size{};
  TileGetTextureTotalSize(standard256b.Format(), standard256b.Width5() + 1u,
                          standard256b.Height5() + 1u,
                          standard256b.Depth() + 1u, 1, standard256b.TileMode(),
                          false, standard256b_size);
  Require("BasicStorageTexture", "Standard256B uint 2D descriptor",
          standard256b.Width5() + 1u == 2 && standard256b.Height5() + 1u == 2 &&
              standard256b.Depth() + 1u == 1 &&
              standard256b.Type() == Prospero::ImageType::kColor2D &&
              standard256b.Format() == Prospero::BufferFormat::k8_8_8_8UInt &&
              standard256b.TileMode() == Prospero::TileMode::kStandard256B &&
              standard256b.DstSelXYZW() == DstSel(4, 5, 6, 7) &&
              standard256b_size.size == 0x100 &&
              standard256b_size.align == 0x100,
          "captured PPSA08511 Standard256B storage descriptor is malformed");
  ValidateStorageTexture(standard256b_resource, standard256b,
                         standard256b_size.size);

  const auto standard4kb_array = Standard4KBUintArrayStorageTextureDescriptor();
  TileSizeAlign standard4kb_size{};
  TileGetTextureTotalSize(standard4kb_array.Format(), 1, 1, 1, 1,
                          standard4kb_array.TileMode(), false,
                          standard4kb_size);
  Require(
      "BasicStorageTexture", "Standard4KB uint 2D-array descriptor",
      standard4kb_array.Type() == Prospero::ImageType::kColor2DArray &&
          standard4kb_array.Format() == Prospero::BufferFormat::k32UInt &&
          standard4kb_array.TileMode() == Prospero::TileMode::kStandard4KB &&
          standard4kb_array.DstSelXYZW() == DstSel(4, 0, 0, 1) &&
          standard4kb_size.size == 0x1000 && standard4kb_size.align == 0x1000,
      "captured Standard4KB uint 2D-array descriptor is malformed");
  ValidateStorageTexture(BasicUintArrayStorageTextureResource(),
                         standard4kb_array, standard4kb_size.size);

  auto based_standard4kb_array = standard4kb_array;
  based_standard4kb_array.fields[0] = 0x006c6800u;
  based_standard4kb_array.fields[4] = 0x00010001u;
  TileSizeAlign based_standard4kb_size{};
  TileGetTextureTotalSize(based_standard4kb_array.Format(), 1, 1,
                          based_standard4kb_array.Depth() + 1u, 1,
                          based_standard4kb_array.TileMode(), false,
                          based_standard4kb_size);
  Require("BasicStorageTexture", "based Standard4KB array view",
          based_standard4kb_array.Base40() == 0x6c680000ull &&
              based_standard4kb_array.BaseArray5() == 1 &&
              based_standard4kb_array.Depth() == 1 &&
              based_standard4kb_size.size == 0x2000 &&
              based_standard4kb_size.align == 0x1000,
          "captured based Standard4KB array view is malformed");
  ValidateStorageTexture(BasicUintArrayStorageTextureResource(),
                         based_standard4kb_array, based_standard4kb_size.size);

  const auto standard64kb = Standard64KBStorageTextureDescriptor();
  TileSizeAlign standard64kb_size{};
  TileGetTextureTotalSize(standard64kb.Format(), standard64kb.Width5() + 1u,
                          standard64kb.Height5() + 1u,
                          standard64kb.Depth() + 1u, 1, standard64kb.TileMode(),
                          false, standard64kb_size);
  Require("BasicStorageTexture", "Standard64KB 2D descriptor",
          standard64kb.Type() == Prospero::ImageType::kColor2DArray &&
              standard64kb.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
              standard64kb.TileMode() == Prospero::TileMode::kStandard64KB &&
              standard64kb.DstSelXYZW() == DstSel(4, 5, 6, 7) &&
              standard64kb_size.size == 0x10000 &&
              standard64kb_size.align == 0x10000,
          "captured Standard64KB storage descriptor is malformed");
  ValidateStorageTexture(BasicBgraStorageTextureResource(), standard64kb,
                         standard64kb_size.size);

  const auto uint_volume = BasicUintVolumeStorageTextureDescriptor();
  Require("BasicStorageTexture", "uint 3D descriptor",
          uint_volume.Base40() == 0x2018060000ull &&
              uint_volume.Width5() + 1u == 16 &&
              uint_volume.Height5() + 1u == 16 &&
              uint_volume.Depth() + 1u == 16 && uint_volume.BaseArray5() == 0 &&
              uint_volume.Type() == Prospero::ImageType::kColor3D &&
              uint_volume.Format() == Prospero::BufferFormat::k16UInt &&
              uint_volume.TileMode() == Prospero::TileMode::kLinear &&
              uint_volume.DstSelXYZW() == DstSel(4, 0, 0, 0),
          "PPSA21268 uint 3D storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicUintVolumeStorageTextureResource(), uint_volume,
                         0x10000);

  const ShaderTextureResource tiled_uint_volume{
      {0x1ac0e530u, 0xc0b00000u, 0x0003c003u, 0xa0500004u, 0x0000000fu,
       0x00700000u, 0x00000000u, 0x00000000u}};
  Require("BasicStorageTexture", "tiled uint 3D descriptor",
          tiled_uint_volume.Width5() + 1u == 16 &&
              tiled_uint_volume.Height5() + 1u == 16 &&
              tiled_uint_volume.Depth() + 1u == 16 &&
              tiled_uint_volume.Type() == Prospero::ImageType::kColor3D &&
              tiled_uint_volume.TileMode() == Prospero::TileMode::kStandard4KB,
          "captured tiled uint 3D storage descriptor fixture is malformed");
  ValidateStorageTexture(BasicUintVolumeStorageTextureResource(),
                         tiled_uint_volume, 0x2000);

  const auto depth_tile = Ppsa14053DepthTileStorageTextureDescriptor();
  Require("BasicStorageTexture", "PPSA14053 depth-tile descriptor",
          depth_tile.Base40() == 0x20144c0000ull &&
              depth_tile.Width5() + 1u == 1 && depth_tile.Height5() + 1u == 1 &&
              depth_tile.Depth() + 1u == 1 && depth_tile.BaseArray5() == 0 &&
              depth_tile.Type() == Prospero::ImageType::kColor2DArray &&
              depth_tile.Format() == Prospero::BufferFormat::k8UInt &&
              depth_tile.TileMode() == Prospero::TileMode::kDepth &&
              depth_tile.DstSelXYZW() == DstSel(4, 0, 0, 1),
          "PPSA14053 write-only depth-tile storage descriptor fixture is "
          "malformed");
  ValidateStorageTexture(Ppsa14053DepthTileStorageTextureResource(), depth_tile,
                         0x10000);
  const auto d16_depth_tile = Ppsa10112D16StorageTextureDescriptor();
  Require("BasicStorageTexture", "PPSA10112 D16 depth-tile descriptor",
          d16_depth_tile.Base40() == 0x205b900000ull &&
              d16_depth_tile.Width5() + 1u == 240 &&
              d16_depth_tile.Height5() + 1u == 135 &&
              d16_depth_tile.Depth() + 1u == 1 &&
              d16_depth_tile.Type() == Prospero::ImageType::kColor2DArray &&
              d16_depth_tile.Format() == Prospero::BufferFormat::k16UNorm &&
              d16_depth_tile.TileMode() == Prospero::TileMode::kDepth &&
              d16_depth_tile.DstSelXYZW() == DstSel(4, 0, 0, 1),
          "PPSA10112 writable D16 depth-plane descriptor fixture is malformed");
  const auto d16_pitch =
      TileGetTexturePitch(d16_depth_tile.Format(), d16_depth_tile.Width5() + 1u,
                          d16_depth_tile.TileMode());
  TileSizeAlign d16_size{};
  TileGetTextureTotalSize(d16_depth_tile.Format(), d16_depth_tile.Width5() + 1u,
                          d16_depth_tile.Height5() + 1u,
                          d16_depth_tile.Depth() + 1u, 1,
                          d16_depth_tile.TileMode(), false, d16_size);
  Require("BasicStorageTexture", "PPSA10112 D16 depth-tile footprint",
          d16_pitch == 256 && d16_size.size == 0x20000 &&
              d16_size.align == 0x10000,
          "PPSA10112 writable D16 depth-plane footprint is incorrect");
  ValidateStorageTexture(BasicArrayStorageTextureResource(), d16_depth_tile,
                         d16_size.size);
  auto depth_tile_r32 = depth_tile;
  depth_tile_r32.fields[1] =
      (depth_tile_r32.fields[1] & ~(0x1ffu << 20u)) |
      (static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u);
  depth_tile_r32.fields[3] =
      (depth_tile_r32.fields[3] & ~(0xfu << 28u)) |
      (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
  auto depth_tile_r32_resource = Ppsa14053DepthTileStorageTextureResource();
  depth_tile_r32_resource.dimension =
      ShaderRecompiler::Decoder::ImageDimension::Dim2D;
  ValidateStorageTexture(depth_tile_r32_resource, depth_tile_r32, 0x10000);
  Require("BasicStorageTexture", "R32_UINT replicated write mapping",
          IsValidImageSwizzle(DstSel(4, 4, 4, 4)),
          "single-channel replicated destination selection was rejected");

  const auto atomic = AtomicStorageTextureDescriptor();
  Require("BasicStorageTexture", "atomic R32_UINT descriptor",
          atomic.Width5() + 1u == 128 && atomic.Height5() + 1u == 1 &&
              atomic.Depth() + 1u == 1 &&
              atomic.Type() == Prospero::ImageType::kColor2D &&
              atomic.Format() == Prospero::BufferFormat::k32UInt &&
              atomic.DstSelXYZW() == DstSel(4, 0, 0, 1),
          "PPSA22102 image-atomic descriptor fixture is malformed");
  ValidateStorageTexture(AtomicStorageTextureResource(), atomic, 0x10000);

  char path[MAX_PATH]{};
  Require("BasicStorageTexture", "host",
          GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
          "GetModuleFileName failed");
  for (const char *kind : {"resource",
                           "type",
                           "standard256b-volume",
                           "inverted-mip-range",
                           "swizzle",
                           "linear-rgb1-read",
                           "bgra-read",
                           "r16-float-read",
                           "r8-unorm-read",
                           "yzwx-read",
                           "reserved-swizzle",
                           "array-base-out-of-range",
                           "reserved",
                           "uint-format",
                           "uint-resource-float-format",
                           "atomic-format",
                           "depth-tile-read",
                           "depth-tile-extent",
                           "depth-tile-fmask"}) {
    std::string command = std::string("\"") + path +
                          "\" --storage-texture-descriptor-death " + kind;
    std::vector<char> mutable_command(command.begin(), command.end());
    mutable_command.push_back('\0');
    STARTUPINFOA startup{sizeof(startup)};
    PROCESS_INFORMATION process{};
    Require("BasicStorageTexture", "host",
            CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr,
                           FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup,
                           &process) != 0,
            "CreateProcess failed");
    Require("BasicStorageTexture", "host",
            WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0,
            "descriptor death case timed out");
    DWORD exit_code = 0;
    const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0;
    CloseHandle(process.hThread);
    CloseHandle(process.hProcess);
    Require("BasicStorageTexture", "host", exited && exit_code == 321,
            std::string(kind) +
                " storage descriptor did not report a fatal error");
  }
  std::printf("[host]    %-32s ok\n", "BasicStorageTextureDescriptor");
}
#endif

void CheckStorageTextureLinearUploadLayout() {
  constexpr auto format = Prospero::BufferFormat::k8_8_8_8UNorm;
  constexpr uint32_t width = 3840;
  constexpr uint32_t height = 2160;
  constexpr uint32_t depth = 1;
  constexpr auto tile = Prospero::TileMode::kLinear;
  const auto pitch = TileGetTexturePitch(format, width, tile);
  TileSizeAlign total{};
  TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total);
  const auto layout =
      MakeTilingImage(format, width, height, 1, depth, tile, total.size, false);
  const auto regions = layout.BufferCopies();
  Require("StorageTextureLinearUpload", "layout",
          pitch == width && total.size == 0x1fa4000 && total.align == 256 &&
              layout.tile_mode == tile && layout.pitch == width &&
              layout.linear_slice_stride == total.size && regions.size() == 1 &&
              regions[0].bufferOffset == 0 &&
              regions[0].imageExtent.width == width &&
              regions[0].imageExtent.height == height &&
              regions[0].bufferRowLength == width,
          "linear RGBA8 storage upload lost Prospero pitch or allocation size");
  std::printf("[host]    %-32s ok\n", "StorageTextureLinearUpload");
}

void CheckStorageTextureDepthTileUploadLayout() {
  constexpr auto format = Prospero::BufferFormat::k8UInt;
  constexpr uint32_t width = 1;
  constexpr uint32_t height = 1;
  constexpr uint32_t depth = 1;
  constexpr auto tile = Prospero::TileMode::kDepth;
  const auto pitch = TileGetTexturePitch(format, width, tile);
  TileSizeAlign slice{};
  TileSizeAlign total{};
  TileSizeOffset level{};
  TilePaddedSize padded{};
  TileGetTextureSize(format, width, height, 1, tile, &slice, &level, &padded);
  TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total);
  const auto layout =
      MakeTilingImage(format, width, height, 1, depth, tile, total.size, false);
  const auto regions = layout.BufferCopies();
  Require("StorageTextureDepthTileUpload", "PPSA14053 layout",
          pitch == 256 && padded.width == 256 && padded.height == 256 &&
              slice.size == 0x10000 && slice.align == 0x10000 &&
              level.size == slice.size && level.offset == 0 &&
              total.size == slice.size && total.align == slice.align &&
              layout.tile_mode == tile &&
              layout.tiling.block.family == TileBlockFamily::Depth64KB &&
              layout.pitch == pitch && layout.linear_slice_stride == 0 &&
              layout.tiled_slice_stride == total.size &&
              layout.mip_layout[0].linear_size == pitch &&
              layout.mip_layout[0].size == total.size && regions.size() == 1 &&
              regions[0].bufferOffset == 0 &&
              regions[0].imageExtent.width == width &&
              regions[0].imageExtent.height == height &&
              regions[0].bufferRowLength == pitch,
          "1x1 R8_UINT depth tile lost its 64 KiB source footprint");
  std::printf("[host]    %-32s ok\n", "StorageTextureDepthTileUpload");
}

void CheckImageSamplerSpecialization() {
  ShaderSamplerResource filtered_sampler{};
  filtered_sampler.fields[2] =
      (1u << 20u) | (3u << 22u) | (2u << 24u) | (2u << 26u);
  filtered_sampler.SetPointFiltering();
  Require("ImageSamplerSpecialization", "forced point sampler",
          filtered_sampler.XyMagFilter() == 0u &&
              filtered_sampler.XyMinFilter() == 0u &&
              filtered_sampler.ZFilter() == 1u &&
              filtered_sampler.MipFilter() == 1u,
          "point-only image format retained a filtered sampler mode");

  ShaderSamplerResource base_mip_sampler{};
  base_mip_sampler.fields[2] = (3u << 20u) | (1u << 22u) | (2u << 24u);
  base_mip_sampler.SetPointFiltering();
  Require("ImageSamplerSpecialization", "base-mip point sampler",
          base_mip_sampler.XyMagFilter() == 0u &&
              base_mip_sampler.XyMinFilter() == 0u &&
              base_mip_sampler.ZFilter() == 1u &&
              base_mip_sampler.MipFilter() == 0u,
          "forcing point filtering enabled mipmapping on a base-mip sampler");

  ShaderRecompiler::IR::Program mixed_sampler_program;
  mixed_sampler_program.resource_tracking_complete = true;
  ShaderRecompiler::IR::ImageResource sampled_image;
  sampled_image.resource_class =
      ShaderRecompiler::IR::ImageResourceClass::Sampled;
  sampled_image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
  sampled_image.read = true;
  mixed_sampler_program.info.images = {sampled_image, sampled_image,
                                       sampled_image, sampled_image};
  mixed_sampler_program.info.samplers.push_back({0u, 4u});
  mixed_sampler_program.info.sampled_pairs = {
      {0u, 0u, 8u}, {1u, 0u, 12u}, {2u, 0u, 16u}, {3u, 0u, 20u}};
  ShaderRecompiler::IR::MemoryInfo signed_memory;
  signed_memory.kind = ShaderRecompiler::IR::ResourceKind::Image;
  signed_memory.resource = 2u;
  signed_memory.sampler = 0u;
  mixed_sampler_program.memory_info.push_back(signed_memory);
  mixed_sampler_program.block_storage.push_back(
      std::make_unique<ShaderRecompiler::IR::Block>());
  auto *mixed_sampler_block = mixed_sampler_program.block_storage.back().get();
  mixed_sampler_program.blocks.push_back(mixed_sampler_block);
  using ShaderRecompiler::IR::Value;
  using ShaderRecompiler::IR::ValueOpcode;
  auto &image_handle = mixed_sampler_block->AppendNewInst(
      ValueOpcode::GetImageResource,
      {Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
       Value(0u), Value(0u)});
  auto &sampler_handle = mixed_sampler_block->AppendNewInst(
      ValueOpcode::GetSamplerResource,
      {Value(0u), Value(0u), Value(0u), Value(0u)});
  auto &image_address = mixed_sampler_block->AppendNewInst(
      ValueOpcode::MakeImageAddress,
      {Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
       Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
       Value(0u)});
  const ShaderRecompiler::IR::MemoryFlags signed_memory_flags{0u, 16u};
  uint64_t signed_memory_flag_bits = 0;
  std::memcpy(&signed_memory_flag_bits, &signed_memory_flags,
              sizeof(signed_memory_flags));
  mixed_sampler_block->AppendNewInst(
      ValueOpcode::ImageSampleRaw,
      {Value(&image_handle), Value(&sampler_handle), Value(&image_address), Value(true)},
      signed_memory_flag_bits);

  ShaderRecompiler::IR::DescriptorValue native_image_descriptor{};
  native_image_descriptor.dword_count = 8;
  native_image_descriptor.dwords[0] = 0x1000u;
  native_image_descriptor.dwords[1] =
      static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u;
  native_image_descriptor.dwords[2] = 3u | (3u << 14u);
  native_image_descriptor.dwords[3] =
      DstSel(4, 5, 6, 7) |
      (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
  auto packed_image_descriptor = native_image_descriptor;
  packed_image_descriptor.dwords[0] = 0x2000u;
  packed_image_descriptor.dwords[1] =
      static_cast<uint32_t>(Prospero::BufferFormat::k11_11_10UInt) << 20u;
  auto signed_image_descriptor = native_image_descriptor;
  signed_image_descriptor.dwords[0] = 0x3000u;
  signed_image_descriptor.dwords[1] =
      static_cast<uint32_t>(Prospero::BufferFormat::k32SInt) << 20u;
  auto unsigned_image_descriptor = native_image_descriptor;
  unsigned_image_descriptor.dwords[0] = 0x4000u;
  unsigned_image_descriptor.dwords[1] =
      static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u;

  ShaderRecompiler::IR::DescriptorValue sampler_descriptor{};
  sampler_descriptor.dword_count = 4;
  const std::array descriptors{native_image_descriptor, packed_image_descriptor,
                               signed_image_descriptor, unsigned_image_descriptor,
                               sampler_descriptor};
  mixed_sampler_program.descriptor_sources.resize(descriptors.size());
  for (u32 source = 0; source < descriptors.size(); source++) {
    auto &destination = mixed_sampler_program.descriptor_sources[source];
    destination.dword_count = descriptors[source].dword_count;
    for (u32 dword = 0; dword < destination.dword_count; dword++) {
      destination.dwords[dword] = Value(descriptors[source].dwords[dword]);
    }
  }
  for (u32 image = 0; image < mixed_sampler_program.info.images.size(); image++) {
    mixed_sampler_program.info.images[image].source = image;
  }
  mixed_sampler_program.info.samplers[0].source = 4;
  mixed_sampler_program.srt_plan_complete = true;
  auto mixed_sampler_plan =
      ShaderRecompiler::IR::ExtractResourcePlan(mixed_sampler_program);
  ShaderRecompiler::IR::ResourceSnapshot mixed_sampler_snapshot;
  ShaderRecompiler::IR::ResourceSpecialization mixed_sampler_specialization;
  Require("ImageSamplerSpecialization", "resource materialization",
          ShaderRecompiler::IR::MaterializeResources(
              mixed_sampler_plan, {}, mixed_sampler_snapshot,
              mixed_sampler_specialization),
          "mixed sampler descriptors could not be materialized");
  ShaderRecompiler::IR::ApplyResourceSpecialization(
      mixed_sampler_program, mixed_sampler_specialization);
  Require("ImageSamplerSpecialization", "mixed sampler variant",
          mixed_sampler_program.info.samplers.size() == 2u &&
              !mixed_sampler_program.info.samplers[0].force_point_filtering &&
              !mixed_sampler_program.info.samplers[0].integer_border &&
              mixed_sampler_program.info.samplers[1].force_point_filtering &&
              mixed_sampler_program.info.samplers[1].integer_border &&
              mixed_sampler_program.info.sampled_pairs[0].sampler == 0u &&
              mixed_sampler_program.info.sampled_pairs[1].sampler == 1u &&
              mixed_sampler_program.info.sampled_pairs[2].sampler == 1u &&
              mixed_sampler_program.info.sampled_pairs[3].sampler == 1u &&
              mixed_sampler_program.memory_info[0].sampler == 1u &&
              mixed_sampler_snapshot.samplers.size() == 1u &&
              std::ranges::all_of(mixed_sampler_program.info.samplers,
                  [](const auto& sampler) {
                    return sampler.source == 4u && sampler.snapshot_index == 0u;
                  }),
          "a shared float/unsigned/signed sampler lost its border or filtering "
          "class in the compiled bindings, instruction or descriptor snapshot");

  for (const bool scalar : {false, true}) {
    using namespace ShaderRecompiler::IR;
    const char *name = scalar ? "ScalarMaterialImageDomain" : "GpuMaterialImageDomain";
    constexpr u32 count = 70, table_base = 0x1000, material_base = 0x2000;
    std::vector<u32> memory(4096);
    for (u32 i = 0; i < count; ++i) {
      auto descriptor = native_image_descriptor;
      descriptor.dwords[0] += i * 0x100u;
      std::copy(descriptor.dwords.begin(), descriptor.dwords.end(),
                memory.begin() + table_base / 4u + i * 8u);
      memory[material_base / 4u + i] = (i << 4u) | 3u;
    }
    memory[material_base / 4u + count] = (7u << 4u) | 15u; // Duplicate after shift.
    memory[material_base / 4u + count + 1u] = 0x80000010u; // key<<5 wraps to entry1.
    memory[material_base / 4u + count + 2u] = count << 4u; // Scalar table OOB is null.
    std::array<u32, 64> user_data{};
    const std::array<u32, 8> buffers{table_base, 32u << 16u, count, 0x5204u,
                                    material_base, 4u << 16u, count + 3u, 0x5204u};
    std::copy(buffers.begin(), buffers.end(), user_data.begin());
    const auto output = MakeStructuredStorageBufferData(
        0u, 64u, false, static_cast<u32>(Prospero::BufferFormat::k32UInt));
    std::copy_n(output.begin(), 4, user_data.begin() + 48);
    user_data[48] = 0x3000u;
    if (scalar) user_data[30] = 1u;
    std::vector<u32> code{EncodeVopc(0xc5, InlineU32(0), 0),
                          EncodeSop1(0x24, 24, 106), EncodeSopp(0x08, 0)};
    const auto append_scalar_key = [&](bool numeric_use) {
      code.insert(code.end(), {EncodeVop1(0x02, 20, Vgpr(0)),
                               EncodeSop2(0x00, 20, 20, 21),
                               EncodeSop2(0x26, 20, 20, InlineU32(3)),
                               EncodeSop2(0x1e, 20, 20, InlineU32(2)),
                               EncodeSop2(0x00, 20, 20, InlineU32(8)),
                               EncodeSmem0(0x08, 20, 2), EncodeSmem1(0, 20)});
      if (numeric_use) code.push_back(EncodeVop1(0x01, 17, 20));
      code.push_back(EncodeSop2(0x20, 20, 20, InlineU32(4)));
    };
    if (scalar) {
      append_scalar_key(true);
    } else {
      code.insert(code.end(), {EncodeSop1(0x04, 26, 126), EncodeVop1(0x01, 16, InlineU32(1))});
      // An earlier descriptor waterfall separates the EXECZ witness from the material load.
      const auto previous_loop = code.size();
      code.insert(code.end(), {EncodeSop1(0x04, 28, 126),
                               EncodeVop1(0x02, 22, Vgpr(0)), EncodeVopc(0xd2, 22, 0),
                               EncodeVop1(0x01, 16, InlineU32(0)),
                               EncodeSop1(0x04, 126, 28), EncodeVopc(0xd5, InlineU32(0), 16)});
      code.push_back(EncodeSopp(0x09, static_cast<int16_t>(previous_loop - code.size() - 1u)));
      code.insert(code.end(), {EncodeSop1(0x04, 126, 26),
                               EncodeMubuf0(0x0c, 0, true, false), EncodeMubuf1(15, 1, 0),
                               EncodeVop2(0x16, 15, InlineU32(4), 15),
                               EncodeVop1(0x01, 16, InlineU32(1))});
    }
    const auto image_loop = code.size();
    if (!scalar)
      code.insert(code.end(), {EncodeSop1(0x04, 28, 126),
                               EncodeVop1(0x02, 20, Vgpr(15)), EncodeVopc(0xd2, 20, 15)});
    code.insert(code.end(), {EncodeSop2(0x1e, 20, 20, InlineU32(5)),
                             EncodeSmem0(0x0b, 12), EncodeSmem1(0, 20)});
    const auto table_read_end = code.size();
    code.insert(code.end(), {EncodeMimg0(0x24, 1), EncodeMimg1(4, 0, 3, 2)});
    AppendBufferStoreDword(&code, 4, 0);
    if (!scalar) {
      code.insert(code.end(), {EncodeVop1(0x01, 16, InlineU32(0)),
                               EncodeSop1(0x04, 126, 28), EncodeVopc(0xd5, InlineU32(0), 16)});
      code.push_back(EncodeSopp(0x09, static_cast<int16_t>(image_loop - code.size() - 1u)));
    } else {
      // Crossed exits introduce stale s16:19 Phi arms before the later s16:23 samples.
      code.push_back(EncodeVop1(0x01, 5, InlineU32(0)));
      code.push_back(EncodeSopc(0x06, 30, InlineU32(0)));
      const auto skip_load = code.size();
      code.push_back(EncodeSopp(0x05, 0));
      append_scalar_key(false);
      code.insert(code.end(), {EncodeSop2(0x1e, 20, 20, InlineU32(5)),
                               EncodeSmem0(0x0b, 16), EncodeSmem1(0, 20),
                               EncodeSopc(0x06, 31, InlineU32(0))});
      const auto later_sample = code.size();
      code.push_back(EncodeSopp(0x05, 0));
      code.insert(code.end(), {EncodeMimg0(0x24, 1), EncodeMimg1(4, 0, 4, 2)});
      const auto early_exit = code.size();
      code.push_back(EncodeSopp(0x02, 0));
      code[later_sample] = EncodeSopp(0x05, static_cast<int16_t>(code.size() - later_sample - 1u));
      code.insert(code.end(), {EncodeMimg0(0x24, 1), EncodeMimg1(4, 0, 4, 2),
                               EncodeMimg0(0x24, 1), EncodeMimg1(5, 0, 4, 2)});
      const auto leave_samples = code.size();
      code.push_back(EncodeSopp(0x02, 0));
      code[skip_load] = EncodeSopp(0x05, static_cast<int16_t>(code.size() - skip_load - 1u));
      code.push_back(EncodeVop1(0x01, 4, InlineU32(0)));
      code[early_exit] = EncodeSopp(0x02, static_cast<int16_t>(code.size() - early_exit - 1u));
      code[leave_samples] = EncodeSopp(0x02, static_cast<int16_t>(code.size() - leave_samples - 1u));
      AppendBufferStoreDword(&code, 4, 0);
      AppendBufferStoreDword(&code, 5, 0);
      AppendBufferStoreDword(&code, 17, 0);
    }
    code[2] = EncodeSopp(0x08, static_cast<int16_t>(code.size() - 3u));
    AppendEnd(&code);
    ShaderComputeInputInfo compute{};
    compute.wave_size = 32;
    compute.threads_num[0] = 32;
    compute.thread_ids_num = 1;
    ShaderRecompiler::CompileOptions options;
    options.stage = ShaderType::Compute;
    options.wave_size = 32;
    options.user_data = user_data;
    options.input_info.compute = &compute;
    auto translated = ShaderRecompiler::TranslateProgram(code, options);
    auto plan = ExtractResourcePlan(translated.program);
    ResourceSnapshot snapshot;
    ResourceSpecialization specialization;
    const SrtRuntime runtime{.user_data = user_data, .read_memory = ReadTestMemory,
                             .userdata = &memory, .read_specialization_memory = ReadTestMemory};
    Require(name, "bounded native material plan",
            plan.capture_specialization_reads && !translated.program.has_address_writes &&
                MaterializeResources(plan, runtime, snapshot, specialization) &&
                snapshot.images.size() == count + 1u,
            "GPU-selected material descriptors did not expand beyond native root capacity");
    const auto offset = specialization.images[0].indirect_mapping_offset;
    const auto ordinal = [&](u32 byte_offset) {
      const auto word = byte_offset >> 2u;
      return word < snapshot.flattened_srt[offset]
                 ? snapshot.flattened_srt[offset + 1u + word] : 0u;
    };
    Require(name, "dense table map and coherent snapshot",
            specialization.images[0].indirect_search_iterations == 0u &&
                snapshot.flattened_srt[offset] == (count - 1u) * 8u + 1u &&
                ordinal(32u) != 0u &&
                ordinal(count * 32u) == 0u && snapshot.images[0].dwords[0] == 0u &&
                snapshot.specialization_reads ==
                    std::vector<std::pair<uint64_t, uint64_t>>{{table_base, count * 32u}},
            "image table lost its dense offsets, explicit null or single coherent read");
    const auto previous = specialization;
    memory[table_base / 4u + 8u] += 0x10000u;
    Require(name, "runtime descriptor refresh",
            MaterializeResources(plan, runtime, snapshot, specialization) &&
                specialization == previous && snapshot.images[ordinal(32u)].dwords[0] == 0x11100u,
            "descriptor addresses were frozen or entered the shader specialization");
    auto compiled = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization);
    ValidateSpirv(name, compiled.spirv);
    if (!scalar) {
      user_data[4] += 1u;
      Require(name, "GPU material reads remain on GPU",
              MaterializeResources(plan, runtime, snapshot, specialization) &&
                  specialization == previous && snapshot.specialization_reads ==
                      std::vector<std::pair<uint64_t, uint64_t>>{{table_base, count * 32u}},
              "GPU material addressing reentered host descriptor selection");
      user_data[4] -= 1u;
    }
    auto dirty = runtime;
    dirty.read_specialization_memory = +[](void*, uint64_t, std::span<u32>) { return false; };
    Require(name, "unreadable image table rejection",
            !MaterializeResources(plan, dirty, snapshot, specialization),
            "GPU-selected texture materialization bypassed strict read provenance");
    if (scalar) {
      code.insert(code.begin() + table_read_end, EncodeVop1(0x01, 18, 12));
      code.pop_back();
      AppendBufferStoreDword(&code, 18, 0);
      code[2] = EncodeSopp(0x08, static_cast<int16_t>(code.size() - 3u));
      AppendEnd(&code);
    } else {
      code[2] = EncodeSopp(0x00, 0); // GPU selectors do not need a host nonempty-EXEC witness.
    }
    auto retained = ShaderRecompiler::TranslateProgram(code, options);
    auto retained_plan = ExtractResourcePlan(retained.program);
    Require(name, "retained descriptor consumers",
            MaterializeResources(retained_plan, runtime, snapshot, specialization),
            "retained GPU descriptor reads could not use the shared image table");
    compiled = ShaderRecompiler::CompileProgram(std::move(retained), options, specialization);
    ValidateSpirv(name, compiled.spirv);
  }

  std::printf("[host]    %-32s ok\n", "ImageSpecializationPipelineId");
}

void CheckResourcePlanHandoff() {
  constexpr const char *name = "ResourcePlanHandoff";
  using namespace ShaderRecompiler::IR;
  for (const bool numeric_read : {false, true}) {
    std::vector<u32> code{EncodeSmem0(0x02, 8), EncodeSmem1(0, 125),
                          EncodeSmem0(0x08, 12, 4), EncodeSmem1(0, 125)};
    if (numeric_read) {
      code.insert(code.end(), {EncodeSmem0(0x00, 13), EncodeSmem1(16, 125)});
      AppendVop3(&code, 0x125u, 0, 12, 13); // v_add_nc_u32 v0, s12, s13
    } else {
      code.push_back(EncodeVop1(0x01, 0, 12));
    }
    code.insert(code.end(), {EncodeMubuf0(0x1c, 0, false, false),
                             EncodeMubuf1(0, 2, 0)});
    AppendEnd(&code);

    std::array<u32, 2> user_data{};
    std::vector<u32> memory{0x1000u, 0u, 16u, 0x30000000u | DstSel(4, 5, 6, 7), 7u};
    ShaderComputeInputInfo compute{};
    ShaderRecompiler::CompileOptions options;
    options.stage = ShaderType::Compute;
    options.user_data = user_data;
    options.input_info.compute = &compute;
    auto translated = ShaderRecompiler::TranslateProgram(code, options);
    auto plan = ExtractResourcePlan(translated.program);
    ResourceSnapshot snapshot;
    ResourceSpecialization specialization;
    const SrtRuntime runtime{.user_data = user_data,
                             .read_memory = ReadTestMemory,
                             .userdata = &memory,
                             .read_specialization_memory = ReadTestMemory};
    Require(name, "initial descriptors",
            MaterializeResources(plan, runtime, snapshot, specialization) &&
                snapshot.buffers.size() == 1 && snapshot.buffers[0].dwords[0] == 0x1000u,
            "SRT buffer descriptor was not materialized");
    auto compiled = ShaderRecompiler::CompileProgram(std::move(translated), options,
                                                     specialization);
    ValidateSpirv(name, compiled.spirv);
    const auto &bindings = compiled.program.bindings;
    Require(name, "GPU data requirements",
            compiled.program.info.user_data_registers.empty() && bindings.descriptor_counts[0] == 1 &&
                (bindings.descriptor_counts[static_cast<size_t>(DescriptorBindingKind::FlattenedSrt)] != 0) ==
                    numeric_read,
            "descriptor-only shader retained SRT uploads or live scalar data was removed");

    memory[0] = 0x2000u;
    memory[4] = 13u;
    Require(name, "independent host plan",
            MaterializeResources(plan, runtime, snapshot, specialization) &&
                snapshot.buffers[0].dwords[0] == 0x2000u &&
                (!numeric_read || snapshot.flattened_srt.back() == 13u),
            "compiled shader cleanup invalidated or froze the host resource plan");
  }
  std::printf("[host]    %-32s ok\n", name);
}

void CheckNativeImageDescriptorTypes() {
  using ShaderRecompiler::IR::DescriptorBindingKind;
  using ShaderRecompiler::IR::FirstImageBinding;
  using ShaderRecompiler::IR::FirstStorageImageBinding;

  for (uint32_t value = FirstImageBinding;
       value < static_cast<uint32_t>(DescriptorBindingKind::Samplers);
       value++) {
    const auto expected = value < FirstStorageImageBinding
                              ? vk::DescriptorType::eSampledImage
                              : vk::DescriptorType::eStorageImage;
    Require("NativeImageDescriptorTypes", "image binding",
            NativeDescriptorType(static_cast<DescriptorBindingKind>(value)) ==
                expected,
            "generated image binding has the wrong Vulkan descriptor type");
  }
  Require("NativeImageDescriptorTypes", "non-image bindings",
          NativeDescriptorType(DescriptorBindingKind::Samplers) ==
                  vk::DescriptorType::eSampler &&
              NativeDescriptorType(DescriptorBindingKind::Buffers) ==
                  vk::DescriptorType::eStorageBuffer &&
              NativeDescriptorType(DescriptorBindingKind::Gds) ==
                  vk::DescriptorType::eStorageBuffer &&
              NativeDescriptorType(DescriptorBindingKind::BdaPagetable) ==
                  vk::DescriptorType::eStorageBuffer &&
              NativeDescriptorType(DescriptorBindingKind::FaultBuffer) ==
                  vk::DescriptorType::eStorageBuffer &&
              NativeDescriptorType(DescriptorBindingKind::FlattenedSrt) ==
                  vk::DescriptorType::eStorageBuffer &&
              NativeDescriptorType(DescriptorBindingKind::ShaderData) ==
                  vk::DescriptorType::eStorageBuffer,
          "non-image binding has the wrong Vulkan descriptor type");
  std::printf("[host]    %-32s ok\n", "NativeImageDescriptorTypes");
}

#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
void CheckShaderRecompilerFatalContracts() {
  ExpectFatal("InvalidDescriptorBindingRejection", [] {
    (void)NativeDescriptorType(
        ShaderRecompiler::IR::DescriptorBindingKind::Count);
  });

  ExpectFatal("WritableFlatStoreRejection", [] {
    const auto test = FlatStoreVariants();
    (void)CompileCase(test);
  });

  ExpectFatal("GdsOrB64Rejection", [] {
    TestCase test;
    test.name = "GdsOrB64Rejection";
    test.code = {EncodeDs0(0x4a, 0, true), EncodeDs1(0, 3, 7)};
    AppendEnd(&test.code);
    test.opcodes = {ShaderOpcode::DS_OR_B64, ShaderOpcode::S_ENDPGM};
    (void)CompileCase(test);
  });

  {
    ShaderRecompiler::IR::Program program;
    program.resource_tracking_complete = true;

    ShaderRecompiler::IR::ImageResource image;
    image.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
    image.numeric_class = Prospero::TextureNumericClass::Uint;
    image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
    image.read = true;
    image.written = true;
    image.atomic = true;
    program.info.images.push_back(image);

    ShaderRecompiler::IR::DescriptorValue descriptor{};
    descriptor.dword_count = 8;
    descriptor.dwords[0] = 0x2000u;
    descriptor.dwords[1] =
        static_cast<uint32_t>(Prospero::BufferFormat::k11_11_10UInt) << 20u;
    descriptor.dwords[2] = 3u | (3u << 14u);
    descriptor.dwords[3] =
        DstSel(4, 5, 6, 7) |
        (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);

    image.source = 0;
    program.info.images[0].source = 0;
    program.descriptor_sources.resize(1);
    program.descriptor_sources[0].dword_count = descriptor.dword_count;
    for (u32 dword = 0; dword < descriptor.dword_count; dword++) {
      program.descriptor_sources[0].dwords[dword] =
          ShaderRecompiler::IR::Value(descriptor.dwords[dword]);
    }
    program.srt_plan_complete = true;
    auto plan = ShaderRecompiler::IR::ExtractResourcePlan(program);
    ShaderRecompiler::IR::ResourceSnapshot snapshot;
    ShaderRecompiler::IR::ResourceSpecialization specialization;
    Require("PackedAtomicImageRejection", "resource materialization",
            !ShaderRecompiler::IR::MaterializeResources(
                plan, {}, snapshot, specialization),
            "packed atomic image descriptor was accepted");
  }
}
#endif

void CheckStorageTextureVolumeUploadLayout() {
  constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float;
  constexpr uint32_t width = 33;
  constexpr uint32_t height = 33;
  constexpr uint32_t depth = 33;
  const auto pitch =
      TileGetTexturePitch(format, width, Prospero::TileMode::kRenderTarget);
  TileSizeAlign total{};
  TileGetTextureTotalSize(format, width, height, depth, 1,
                          Prospero::TileMode::kRenderTarget, true, total);
  const auto layout =
      MakeTilingImage(format, width, height, 1, depth,
                      Prospero::TileMode::kRenderTarget, total.size, true);
  const auto regions = layout.BufferCopies();
  Require("StorageTextureVolumeUpload", "layout",
          pitch == 128 && total.size == 0x210000 &&
              layout.linear_slice_stride == 0 &&
              layout.tiled_slice_stride == 0x10000 &&
              layout.mip_layout[0].linear_size == 0x2208 &&
              layout.mip_layout[0].size == 0x10000 && regions.size() == 1 &&
              regions[0].imageExtent.depth == depth,
          "3D render-target upload did not preserve its compact linear layout");

  Require("StorageTextureVolumeUpload", "GPU records",
          layout.resources.levels == 1 && layout.MipExtent(0).depth == depth,
          "3D render-target canonical mip record lost its depth");
  for (const uint32_t z : {0u, 1u, depth - 1u}) {
    const auto &mip = layout.mip_layout[0];
    Require("StorageTextureVolumeUpload", "slice offsets",
            mip.linear_offset + z * mip.linear_size == uint64_t{z} * 0x2208 &&
                mip.offset + z * layout.tiled_slice_stride ==
                    uint64_t{z} * 0x10000 &&
                mip.surface_z + z == z && mip.linear_pitch == width,
            "volume slice lost its linear stride, block slice, or Z swizzle");
  }
  std::printf("[host]    %-32s ok\n", "StorageTextureVolumeUpload");
}

void CheckStorageTextureVolumeMipRegions() {
  constexpr auto format = Prospero::BufferFormat::k8_8_8_8UNorm;
  constexpr uint32_t width = 8;
  constexpr uint32_t height = 4;
  constexpr uint32_t depth = 5;
  constexpr uint32_t levels = 3;
  constexpr auto tile = Prospero::TileMode::kLinear;
  TileSizeAlign total{};
  TileGetTextureTotalSize(format, width, height, depth, levels, tile, true,
                          total);
  const auto layout = MakeTilingImage(format, width, height, levels, depth,
                                      tile, total.size, true);
  const auto copies = layout.BufferCopies();

  bool valid = copies.size() == levels;
  for (uint32_t level = 0; level < levels && valid; ++level) {
    const auto &copy = copies[level];
    const auto &mip = layout.mip_layout[level];
    const auto active = layout.MipExtent(level);
    valid &= copy.imageSubresource.mipLevel == level &&
             copy.imageOffset.z == 0 &&
             copy.imageExtent.width == active.width &&
             copy.imageExtent.height == active.height &&
             copy.imageExtent.depth == active.depth &&
             copy.bufferOffset == mip.linear_offset &&
             copy.bufferRowLength == mip.linear_pitch &&
             uint64_t{copy.bufferRowLength} * copy.bufferImageHeight *
                     sizeof(uint32_t) ==
                 layout.linear_slice_stride;
  }
  const auto saved_mips = layout.mip_layout;
  const auto prefix = layout.BufferCopies(1024, 1);
  const auto again = layout.BufferCopies();
  valid &= prefix.size() == 1 &&
           prefix[0].bufferOffset == 1024 + layout.mip_layout[0].linear_offset &&
           copies[0].bufferOffset == 768 && copies[1].bufferOffset == 256 &&
           copies[2].bufferOffset == 0 && again == copies &&
           layout.mip_layout == saved_mips;
  Require("StorageTextureVolumeMipRegions", "per-mip depth", valid,
          "direction-neutral image copies did not shrink depth or preserve "
          "Vulkan Z coordinates");
  std::printf("[host]    %-32s ok\n", "StorageTextureVolumeMipRegions");
}
void CheckStandard64RenderTargetTileRoundTrip() {
  constexpr auto format = Prospero::BufferFormat::k32Float;
  constexpr auto tile = Prospero::TileMode::kStandard64KB;

  constexpr uint32_t observed_width = 3840;
  constexpr uint32_t observed_height = 2160;
  const auto observed_pitch = TileGetTexturePitch(format, observed_width, tile);
  TileSizeAlign observed{};
  TileGetTextureSize(format, observed_width, observed_height, 1, tile,
                     &observed, nullptr, nullptr);
  Require("Standard64RenderTarget", "observed layout",
          observed_pitch == 3840 && observed.size == 0x1fe0000 &&
              observed.align == 0x10000,
          "PPSA02721 Standard64KB render-target footprint changed");

  constexpr uint32_t width = 257;
  constexpr uint32_t height = 131;
  const auto pitch = TileGetTexturePitch(format, width, tile);
  TileSizeAlign storage{};
  TileGetTextureSize(format, width, height, 1, tile, &storage, nullptr,
                     nullptr);
  Require("Standard64RenderTarget", "partial layout",
          pitch == 384 && storage.size == 0x60000 && storage.align == 0x10000,
          "partial Standard64KB footprint was not padded in 128x128 blocks");

  std::printf("[host]    %-32s ok\n", "Standard64RenderTarget");
}

#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckStorageTextureGpuOwnedRebindState() {
  constexpr uintptr_t base = 0x0000000200200000ull;
  constexpr uint64_t size = 0x10000;
  const auto guest_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
      base, size, Common::VirtualMemory::Mode::ReadWrite,
      "storage_texture_gpu_owned_rebind", true);
  auto *memory = reinterpret_cast<uint8_t *>(guest_memory);
  Require("StorageTextureGpuOwnedRebind", "allocation", guest_memory == base,
          "fixed guest-owner allocation failed");
  PageManager page_manager;
  MemoryTracker tracker(page_manager);
  tracker.ForEachUploadRange(
      base, size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
  uint64_t readable = 0;
  uint64_t mapped = 0;
  MEMORY_BASIC_INFORMATION protection{};
  Require(
      "StorageTextureGpuOwnedRebind", "owned",
      tracker.IsRegionGpuModified(base, size) &&
          (!HostMemoryQueryReadable(base, size, readable) || readable < size) &&
          HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, mapped) &&
          mapped == size &&
          VirtualQuery(memory, &protection, sizeof(protection)) != 0 &&
          protection.Protect == PAGE_NOACCESS,
      "GPU-owned storage pages remained host-readable or lost tracker "
      "identity");

  tracker.UnmarkRegionAsGpuModified(base, size);
  readable = 0;
  Require("StorageTextureGpuOwnedRebind", "clean readback",
          !tracker.IsRegionGpuModified(base, size) &&
              !tracker.IsRegionCpuModified(base, size) &&
              HostMemoryQueryReadable(base, size, readable) && readable == size,
          "clean storage readback did not publish readable coherent backing");
  tracker.MarkRegionAsGpuModified(base, size);
  readable = 0;
  Require(
      "StorageTextureGpuOwnedRebind", "clean reclaim",
      tracker.IsRegionGpuModified(base, size) &&
          !tracker.IsRegionCpuModified(base, size) &&
          (!HostMemoryQueryReadable(base, size, readable) || readable < size),
      "clean storage rebind did not reclaim GPU ownership without an upload");

  tracker.UnmarkRegionAsGpuModified(base, size);
  tracker.MarkRegionAsCpuModified(base, size);
  uint32_t dirty_ranges = 0;
  bool upload_called = false;
  tracker.ForEachUploadRange(
      base, size, true, [&](uint64_t, uint64_t) noexcept { dirty_ranges++; },
      [&]() noexcept { upload_called = true; });
  readable = 0;
  Require(
      "StorageTextureGpuOwnedRebind", "dirty refresh",
      dirty_ranges == 1 && upload_called &&
          tracker.IsRegionGpuModified(base, size) &&
          !tracker.IsRegionCpuModified(base, size) &&
          (!HostMemoryQueryReadable(base, size, readable) || readable < size),
      "CPU-dirty storage rebind did not refresh once and reclaim GPU "
      "ownership");

  tracker.UnmarkRegionAsGpuModified(base, size);
  tracker.UntrackMemory(base, size);
  Require("StorageTextureGpuOwnedRebind", "free",
          Libs::LibKernel::Memory::FreeGuestMemory(base, size),
          "guest-owner free failed");
  std::printf("[host]    %-32s ok\n", "StorageTextureGpuOwnedRebind");
}
#endif

void CheckNativeMsaaState() {
  Require("NativeMsaaState", "sample encoding",
          render_sample_count(0) == 1 && render_sample_count(1) == 2 &&
              render_sample_count(2) == 4 && render_sample_count(3) == 8 &&
              render_sample_count(4) == 0,
          "PS5 sample encodings were not mapped exactly");
  Require("NativeMsaaState", "Vulkan sample mapping",
          vulkan_sample_count(1) == vk::SampleCountFlagBits::e1 &&
              vulkan_sample_count(2) == vk::SampleCountFlagBits::e2 &&
              vulkan_sample_count(4) == vk::SampleCountFlagBits::e4 &&
              vulkan_sample_count(8) == vk::SampleCountFlagBits::e8 &&
              vulkan_sample_count(3) == vk::SampleCountFlagBits{},
          "native sample counts were not mapped exactly to Vulkan");

  TileSizeAlign color{};
  const auto color_pitch = TileGetRenderTargetPitch(1920, 8, 3);
  Require("NativeMsaaState", "8x color footprint",
          color_pitch == 1920 &&
              TileGetRenderTargetSize(1920, 1080, color_pitch, 8, color, 3) &&
              color.align == 0x10000 && color.size == 0x07f80000,
          "8x R16G16B16A16 color footprint was not preserved");

  TileSizeAlign depth{};
  TileSizeAlign stencil{};
  TileSizeAlign htile{};
  Require("NativeMsaaState", "8x depth/stencil footprint",
          TileGetDepthPitch(1920, 4, 3) == 1920 &&
              TileGetDepthSize(1920, 1080, 0, Prospero::DepthFormat::kZ32F,
                               Prospero::StencilFormat::k8UInt, true, stencil,
                               htile, depth, 3) &&
              depth.align == 0x10000 && depth.size == 0x03fc0000 &&
              stencil.align == 0x10000 && stencil.size == 0x010e0000 &&
              htile.align == 0x8000 && htile.size == 0x00030000,
          "8x depth/stencil or fragment-independent HTile footprint regressed");
  std::printf("[host]    %-32s ok\n", "NativeMsaaState");
}

void CheckDepthHtileStencilCompatibility() {
  Require("DepthHtileStencilCompatibility", "inactive stencil plane",
          depth_htile_stencil_acceleration_compatible(false, true, false),
          "inactive Hi-Stencil state was treated as an attachment");
  Require("DepthHtileStencilCompatibility", "disabled acceleration",
          depth_htile_stencil_acceleration_compatible(true, false, true),
          "disabled Hi-Stencil state was rejected");
  Require("DepthHtileStencilCompatibility", "PS5 stencil plus HTile",
          depth_htile_stencil_acceleration_compatible(true, true, false),
          "valid PS5 Hi-Stencil attachment was rejected");
  Require("DepthHtileStencilCompatibility", "missing HTile metadata",
          !depth_htile_stencil_acceleration_compatible(true, false, false),
          "active Hi-Stencil without HTile metadata was admitted");
  std::printf("[host]    %-32s ok\n", "DepthHtileStencilCompatibility");
}

void CheckPs5DepthRegisterDecoding() {
  constexpr uint32_t captured_z = 0x22900803u;
  constexpr uint32_t captured_stencil = 0x00100801u;
  const auto z = HW::DepthZInfo::Decode(captured_z);
  const auto stencil = HW::DepthStencilInfo::Decode(captured_stencil);
  Require(
      "Ps5DepthRegisterDecoding", "captured texture-compatible state",
      z.format == Prospero::DepthFormat::kZ32F && z.num_samples == 0 &&
          z.htile_acceleration && !z.expclear_enabled &&
          !z.partially_resident && z.max_mip_level == 0 &&
          z.z_compare_base == Prospero::ZCompareBase::kZMin &&
          z.texture_compatibility ==
              Prospero::TextureCompatiblePlaneCompression::kEnable &&
          z.HasValidTextureCompatibility() &&
          stencil.format == Prospero::StencilFormat::k8UInt &&
          !stencil.htile_stencil_disabled && !stencil.expclear_enabled &&
          !stencil.partially_resident &&
          stencil.texture_compatibility ==
              Prospero::TextureCompatibleStencil::kEnable &&
          stencil.HasValidTextureCompatibility(),
      "valid PS5 depth/stencil aggregate encodings were not decoded exactly");

  const auto malformed_z = HW::DepthZInfo::Decode(0x00000803u);
  const auto malformed_stencil = HW::DepthStencilInfo::Decode(0x00100001u);
  Require("Ps5DepthRegisterDecoding", "malformed aggregate state",
          !malformed_z.HasValidTextureCompatibility() &&
              !malformed_stencil.HasValidTextureCompatibility(),
          "partial texture-compatible aggregate encodings were accepted");
  std::printf("[host]    %-32s ok\n", "Ps5DepthRegisterDecoding");
}

void CheckDepthAttachmentWrites() {
  RenderDepthInfo target{};
  target.desc.view_info.format = vk::Format::eD32SfloatS8Uint;
  Require("DepthAttachmentWrites", "disabled depth test",
          !target.AttachmentWriteAspects(),
          "disabled depth testing claimed a depth write");

  target.depth_test_enable = true;
  target.depth_write_enable = true;
  target.depth_compare_op = vk::CompareOp::eLess;
  Require("DepthAttachmentWrites", "depth write",
          target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth,
          "write-enabled depth testing did not claim depth");

  target.depth_test_enable = false;
  target.depth_write_enable = false;
  target.depth_load_clear_enable = true;
  Require("DepthAttachmentWrites", "depth clear",
          target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth,
          "depth attachment clear did not claim depth");

  target.depth_load_clear_enable = false;
  target.stencil_test_enable = true;
  target.stencil_front = {vk::StencilOp::eKeep, vk::StencilOp::eKeep,
                          vk::StencilOp::eKeep, vk::CompareOp::eAlways, 0xff, 0xff, 0};
  target.stencil_back = target.stencil_front;
  Require("DepthAttachmentWrites", "stencil keep",
          !target.AttachmentWriteAspects(),
          "KEEP-only stencil state claimed a stencil write");

  target.stencil_front.failOp = vk::StencilOp::eZero;
  Require("DepthAttachmentWrites", "unreachable stencil fail",
          !target.AttachmentWriteAspects(),
          "ALWAYS comparison claimed an unreachable fail operation");

  target.stencil_front.failOp = vk::StencilOp::eKeep;
  target.stencil_front.passOp = vk::StencilOp::eReplace;
  Require("DepthAttachmentWrites", "stencil pass write",
          target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil,
          "write-capable stencil pass did not claim stencil");

  target.stencil_front.writeMask = 0;
  Require("DepthAttachmentWrites", "back-face keep",
          !target.AttachmentWriteAspects(),
          "masked front write or KEEP-only back face claimed stencil");

  target.stencil_test_enable = false;
  target.stencil_clear_enable = true;
  Require("DepthAttachmentWrites", "stencil clear",
          target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil,
          "stencil attachment clear did not claim stencil");

  target.desc.view_info.format = vk::Format::eD32Sfloat;
  Require("DepthAttachmentWrites", "missing stencil aspect",
          !target.AttachmentWriteAspects(),
          "depth-only format claimed a stencil write");
  std::printf("[host]    %-32s ok\n", "DepthAttachmentWrites");
}

void CheckDepthFeedbackAspects() {
  RenderDepthInfo target{};
  target.desc.view_info.format = vk::Format::eD32SfloatS8Uint;
  target.desc.view_info.base_level = 1;
  target.desc.view_info.level_count = 2;
  target.desc.view_info.base_layer = 3;
  target.desc.view_info.layer_count = 2;
  target.depth_test_enable = true;
  target.depth_write_enable = true;
  target.depth_compare_op = vk::CompareOp::eAlways;
  target.stencil_test_enable = true;
  target.stencil_front = {vk::StencilOp::eKeep, vk::StencilOp::eReplace,
                          vk::StencilOp::eKeep, vk::CompareOp::eAlways, 0xff, 0xff, 0};
  target.stencil_back = target.stencil_front;

  auto sampled = target.desc.view_info;
  const auto feedback = [&](const ImageViewInfo& view) {
    return DepthFeedbackAspects(target.AttachmentWriteAspects(),
                                target.desc.view_info, view);
  };
  sampled.aspect = vk::ImageAspectFlagBits::eDepth;
  Require("DepthFeedbackAspects", "sampled writable depth",
          feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
          "overlapping depth sampling did not request depth feedback");
  sampled.aspect = vk::ImageAspectFlagBits::eStencil;
  Require("DepthFeedbackAspects", "sampled writable stencil",
          feedback(sampled) == vk::ImageAspectFlagBits::eStencil,
          "overlapping stencil sampling did not request stencil feedback");
  sampled.aspect = vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
  Require("DepthFeedbackAspects", "both writable aspects",
          feedback(sampled) == sampled.aspect,
          "combined depth/stencil sampling lost a writable aspect");

  target.depth_write_enable = false;
  Require("DepthFeedbackAspects", "read-only depth",
          feedback(sampled) == vk::ImageAspectFlagBits::eStencil,
          "read-only depth was included in stencil feedback");
  target.depth_write_enable = true;
  target.stencil_test_enable = false;
  Require("DepthFeedbackAspects", "read-only stencil",
          feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
          "read-only stencil was included in depth feedback");

  sampled.aspect = vk::ImageAspectFlagBits::eStencil;
  Require("DepthFeedbackAspects", "sampled read-only aspect",
          !feedback(sampled),
          "sampling a read-only stencil requested feedback for depth writes");
  sampled.aspect = vk::ImageAspectFlagBits::eDepth;
  sampled.base_level = 3;
  Require("DepthFeedbackAspects", "disjoint mip",
          !feedback(sampled),
          "disjoint mip ranges requested depth feedback");
  sampled.base_level = 2;
  sampled.base_layer = 5;
  Require("DepthFeedbackAspects", "disjoint layer",
          !feedback(sampled),
          "disjoint layer ranges requested depth feedback");
  sampled.base_layer = 4;
  Require("DepthFeedbackAspects", "overlapping range boundary",
          feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
          "overlapping mip and layer boundaries missed depth feedback");
  target.depth_write_enable = false;
  target.depth_load_clear_enable = true;
  Require("DepthFeedbackAspects", "depth clear only",
          feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
          "a sampled depth attachment clear missed feedback");
  target.depth_load_clear_enable = false;
  target.stencil_clear_enable = true;
  sampled.aspect = vk::ImageAspectFlagBits::eStencil;
  Require("DepthFeedbackAspects", "stencil clear only",
          feedback(sampled) == vk::ImageAspectFlagBits::eStencil,
          "a sampled stencil attachment clear missed feedback");
  std::printf("[host]    %-32s ok\n", "DepthFeedbackAspects");
}

void CheckDynamicRenderingState() {
  RenderState first{};
  first.width = 64;
  first.height = 32;
  first.num_layers = 2;
  first.num_color_attachments = 1;
  first.color_attachments[0].image_layout =
      vk::ImageLayout::eColorAttachmentOptimal;
  first.color_attachments[0].clear_value = {1, 2, 3, 4};
  first.color_attachments[0].is_clear = true;
  auto changed = first;
  Require("DynamicRenderingState", "identical state", first == changed,
          "fieldwise RenderState identity rejected an identical scope");
  changed.color_attachments[0].is_clear = false;
  Require("DynamicRenderingState", "clear identity", first != changed,
          "clear/load state did not participate in rendering identity");
  changed = first;
  changed.num_layers = 1;
  Require("DynamicRenderingState", "layer identity", first != changed,
          "layer count did not participate in rendering identity");
  changed = first;
  changed.color_attachments[0].image_layout = vk::ImageLayout::eGeneral;
  Require("DynamicRenderingState", "layout identity", first != changed,
          "attachment layout did not participate in rendering identity");

  PipelineRenderingState rgba{};
  rgba.color_count = 1;
  rgba.color_formats[0] = vk::Format::eR8G8B8A8Unorm;
  auto uint_color = rgba;
  uint_color.color_formats[0] = vk::Format::eR8G8B8A8Uint;
  Require("DynamicRenderingState", "pipeline color format identity",
          rgba != uint_color,
          "distinct dynamic-rendering formats can alias one pipeline key");
  auto depth = rgba;
  depth.depth_format = vk::Format::eD32SfloatS8Uint;
  depth.stencil_format = vk::Format::eD32SfloatS8Uint;
  Require("DynamicRenderingState", "pipeline depth/stencil identity",
          rgba != depth,
          "depth/stencil formats did not participate in pipeline identity");

  RenderDepthInfo attachment{};
  attachment.desc.view_info.format = vk::Format::eD32SfloatS8Uint;
  Require("DynamicRenderingState", "read-only depth/stencil layout",
          depth_attachment_layout(attachment) ==
              vk::ImageLayout::eDepthStencilReadOnlyOptimal,
          "fully read-only depth/stencil used a writable layout");
  attachment.depth_load_clear_enable = true;
  Require("DynamicRenderingState", "deferred depth clear layout",
          depth_attachment_layout(attachment) ==
              vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal,
          "a depth load clear used a read-only attachment layout");
  attachment.depth_test_enable = true;
  attachment.depth_write_enable = true;
  Require("DynamicRenderingState", "depth-write stencil-read layout",
          depth_attachment_layout(attachment) ==
              vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal,
          "depth-only writes did not retain read-only stencil");
  attachment.depth_load_clear_enable = false;
  attachment.depth_write_enable = false;
  attachment.stencil_clear_enable = true;
  Require("DynamicRenderingState", "depth-read stencil-write layout",
          depth_attachment_layout(attachment) ==
              vk::ImageLayout::eDepthReadOnlyStencilAttachmentOptimal,
          "stencil-only writes did not retain read-only depth");
  attachment.depth_write_enable = true;
  Require("DynamicRenderingState", "writable depth/stencil layout",
          depth_attachment_layout(attachment) ==
              vk::ImageLayout::eDepthStencilAttachmentOptimal,
          "combined depth/stencil writes did not use the writable layout");
  std::printf("[host]    %-32s ok\n", "DynamicRenderingState");
}

void CheckDepthTargetFootprints() {
  TileSizeAlign stencil{};
  TileSizeAlign htile{};
  TileSizeAlign depth{};
  struct AttachmentFormatCase {
    const char *name;
    Prospero::DepthFormat depth_format;
    Prospero::StencilFormat stencil_format;
    vk::Format expected;
  };
  constexpr AttachmentFormatCase attachment_cases[] = {
      {"Z16", Prospero::DepthFormat::kZ16, Prospero::StencilFormat::kInvalid,
       vk::Format::eD16Unorm},
      {"Z16S8", Prospero::DepthFormat::kZ16, Prospero::StencilFormat::k8UInt,
       vk::Format::eD16UnormS8Uint},
      {"Z32", Prospero::DepthFormat::kZ32F, Prospero::StencilFormat::kInvalid,
       vk::Format::eD32Sfloat},
      {"Z32S8", Prospero::DepthFormat::kZ32F, Prospero::StencilFormat::k8UInt,
       vk::Format::eD32SfloatS8Uint},
      {"invalid depth", static_cast<Prospero::DepthFormat>(2u),
       Prospero::StencilFormat::k8UInt, vk::Format::eUndefined},
      {"invalid stencil", Prospero::DepthFormat::kZ16,
       static_cast<Prospero::StencilFormat>(2u), vk::Format::eUndefined},
  };
  for (const auto &test : attachment_cases) {
    Require("DepthTargetFootprints", test.name,
            DepthAttachmentFormat(test.depth_format, test.stencil_format) ==
                test.expected,
            "PS5 depth/stencil attachment mapping changed");
  }
  Require("DepthTargetFootprints", "1920x1080 Z16S8 without HTile",
          TileGetDepthSize(1920, 1080, 0, Prospero::DepthFormat::kZ16,
                           Prospero::StencilFormat::k8UInt, false, stencil,
                           htile, depth) &&
              depth.size == 0x480000 && depth.align == 0x10000 &&
              stencil.size == 0x280000 && stencil.align == 0x10000 &&
              htile.size == 0,
          "captured Z16S8 footprint disagrees with Prospero block rules");
  struct TargetFormatCase {
    const char *name;
    vk::Format host_format;
    Prospero::BufferFormat guest_format;
    uint32_t bytes_per_element;
    bool stencil;
    bool supported;
    bool readback;
  };
  constexpr TargetFormatCase target_cases[] = {
      {"D16", vk::Format::eD16Unorm, Prospero::BufferFormat::k16UNorm, 2, false,
       true, true},
      {"D16S8", vk::Format::eD16UnormS8Uint, Prospero::BufferFormat::k16UNorm,
       2, true, true, true},
      {"D16 via D24S8", vk::Format::eD24UnormS8Uint,
       Prospero::BufferFormat::k16UNorm, 2, true, true, true},
      {"D16 via D32S8", vk::Format::eD32SfloatS8Uint,
       Prospero::BufferFormat::k16UNorm, 2, true, true, true},
      {"D32", vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float, 4,
       false, true, true},
      {"D32S8", vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k32Float,
       4, true, true, true},
      {"D16 plus stencil mismatch", vk::Format::eD16Unorm,
       Prospero::BufferFormat::k16UNorm, 2, true, false, false},
      {"fallback without stencil", vk::Format::eD24UnormS8Uint,
       Prospero::BufferFormat::k16UNorm, 2, false, false, false},
      {"D32 guest via D24", vk::Format::eD24UnormS8Uint,
       Prospero::BufferFormat::k32Float, 4, true, false, false},
      {"D16 byte mismatch", vk::Format::eD16Unorm,
       Prospero::BufferFormat::k16UNorm, 4, false, false, false},
  };
  for (const auto &test : target_cases) {
    ImageInfo target{};
    target.pixel_format = test.host_format;
    target.guest_format = test.guest_format;
    target.bytes_per_block = test.bytes_per_element;
    target.stencil = test.stencil ? GuestRange{0x10000, 0x10000} : GuestRange{};
    Require("DepthTargetFootprints", test.name,
            IsSupportedDepthTargetFormat(target) == test.supported &&
                IsSupportedDepthPlaneReadback(target) == test.readback,
            "host/guest depth format or exact readback policy changed");
  }
  ImageInfo compressed_stencil{};
  compressed_stencil.pixel_format = vk::Format::eD32SfloatS8Uint;
  compressed_stencil.guest_format = Prospero::BufferFormat::k32Float;
  compressed_stencil.bytes_per_block = 4;
  compressed_stencil.stencil = {0x10000, 0x10000};
  compressed_stencil.metadata.stencil_compressed = true;
  Require("DepthTargetFootprints", "compressed stencil depth-plane readback",
          IsSupportedDepthTargetFormat(compressed_stencil) &&
              IsSupportedDepthPlaneReadback(compressed_stencil),
          "stencil metadata incorrectly blocked independent depth-plane "
          "readback");
  Require("DepthTargetFootprints", "lexicographic subresource comparison",
          ImageSubresources{2, 1} > ImageSubresources{1, 4} &&
              ImageSubresources{1, 4} < ImageSubresources{2, 1},
          "subresource ordering diverged from required overlap semantics");
  struct TransferPlaneCase {
    const char *name;
    vk::Format attachment;
    vk::Format transfer;
    uint32_t bytes;
  };
  constexpr TransferPlaneCase transfer_cases[] = {
      {"D16S8 transfer", vk::Format::eD16UnormS8Uint, vk::Format::eD16Unorm, 2},
      {"D24S8 transfer", vk::Format::eD24UnormS8Uint,
       vk::Format::eX8D24UnormPack32, 4},
      {"D32S8 transfer", vk::Format::eD32SfloatS8Uint, vk::Format::eD32Sfloat,
       4},
      {"invalid transfer", vk::Format::eR16Unorm, vk::Format::eUndefined, 0},
  };
  for (const auto &test : transfer_cases) {
    Require("DepthTargetFootprints", test.name,
            DepthAspectTransferFormat(test.attachment) == test.transfer &&
                DepthAspectTransferBytes(test.attachment) == test.bytes,
            "combined depth transfer-plane layout changed");
  }
  struct PromotionCase {
    const char *name;
    uint16_t source;
    uint32_t d24;
    uint32_t d32;
  };
  constexpr PromotionCase promotion_cases[] = {
      {"zero", 0, 0, 0},
      {"midpoint", 0x8000, 0x00800080, 0x3f000080},
      {"maximum", 0xffff, 0x00ffffff, 0x3f800000},
  };
  for (const auto &test : promotion_cases) {
    Require("DepthTargetFootprints", test.name,
            EncodeD16AsD24(test.source) == test.d24 &&
                EncodeD16AsD32(test.source) == test.d32,
            "D16 host promotion changed the represented depth value");
  }
  Require("DepthTargetFootprints", "640x360 Z32S8 without HTile",
          TileGetDepthSize(640, 360, 0, Prospero::DepthFormat::kZ32F,
                           Prospero::StencilFormat::k8UInt, false, stencil,
                           htile, depth),
          "valid non-HTile depth/stencil footprint was rejected");
  Require(
      "DepthTargetFootprints", "640x360 Prospero block sizes",
      depth.size == 0xf0000 && depth.align == 0x10000 &&
          stencil.size == 0x60000 && stencil.align == 0x10000 &&
          htile.size == 0 && htile.align == 0,
      "non-HTile depth/stencil footprint disagrees with Prospero block rules");

  const auto depth_pitch = TileGetTexturePitch(Prospero::BufferFormat::k32Float,
                                               640, Prospero::TileMode::kDepth);
  TileSizeAlign texture_depth{};
  TileGetTextureTotalSize(Prospero::BufferFormat::k32Float, 640, 360, 1, 1,
                          Prospero::TileMode::kDepth, false, texture_depth);
  Require("DepthTargetFootprints", "640x360 generic depth tile",
          depth_pitch == 640 && texture_depth.size == 0xf0000 &&
              texture_depth.align == 0x10000,
          "generic depth texture sizing bypassed 64 KiB block padding");

  Require(
      "DepthTargetFootprints", "960x540 Z32S8 with HTile",
      TileGetDepthSize(960, 540, 0, Prospero::DepthFormat::kZ32F,
                       Prospero::StencilFormat::k8UInt, true, stencil, htile,
                       depth) &&
          depth.size == 0x280000 && depth.align == 0x10000 &&
          stencil.size == 0xc0000 && stencil.align == 0x10000 &&
          htile.size == 0x10000 && htile.align == 0x8000,
      "generic Prospero HTile block calculation rejected the title footprint");
  Require("DepthTargetFootprints", "known HTile extent",
          TileGetDepthSize(1280, 720, 0, Prospero::DepthFormat::kZ32F,
                           Prospero::StencilFormat::k8UInt, true, stencil,
                           htile, depth) &&
              depth.size == 0x3c0000 && stencil.size == 0xf0000 &&
              htile.size == 0x20000,
          "validated 1280x720 HTile footprint regressed");
  Require("DepthTargetFootprints", "PPSA06228 3840x2160 Z32S8 with HTile",
          TileGetDepthSize(3840, 2160, 0, Prospero::DepthFormat::kZ32F,
                           Prospero::StencilFormat::k8UInt, true, stencil,
                           htile, depth) &&
              depth.size == 0x1fe0000 && depth.align == 0x10000 &&
              stencil.size == 0x870000 && stencil.align == 0x10000 &&
              htile.size == 0xa0000 && htile.align == 0x8000,
          "captured 3840x2160 depth/stencil/HTile footprint disagrees with "
          "Prospero rules");
  Require("DepthTargetFootprints", "invalid depth format",
          !TileGetDepthSize(960, 540, 0, static_cast<Prospero::DepthFormat>(2u),
                            Prospero::StencilFormat::k8UInt, true, stencil,
                            htile, depth),
          "unsupported depth format was silently admitted");
  Require("DepthTargetFootprints", "invalid stencil format",
          !TileGetDepthSize(960, 540, 0, Prospero::DepthFormat::kZ32F,
                            static_cast<Prospero::StencilFormat>(2u), true,
                            stencil, htile, depth),
          "unsupported stencil format was silently admitted");
  Require("DepthTargetFootprints", "invalid extent",
          !TileGetDepthSize(0, 540, 0, Prospero::DepthFormat::kZ32F,
                            Prospero::StencilFormat::k8UInt, true, stencil,
                            htile, depth) &&
              !TileGetDepthSize(16385, 540, 0, Prospero::DepthFormat::kZ32F,
                                Prospero::StencilFormat::k8UInt, true, stencil,
                                htile, depth),
          "invalid HTile extent was silently admitted");
  std::printf("[host]    %-32s ok\n", "DepthTargetFootprints");
}

struct SlotLifetimeProbe {
  explicit SlotLifetimeProbe(bool *destroyed) : destroyed(destroyed) {
    if (destroyed == nullptr || *destroyed) {
      EXIT("fence-lifetime probe has invalid construction state\n");
    }
  }
  ~SlotLifetimeProbe() { *destroyed = true; }

  bool *destroyed = nullptr;
};

void CheckSlotVectorLifetime() {
  bool destroyed = false;
  Common::SlotVector<SlotLifetimeProbe> slots;
  const auto first = slots.insert(&destroyed);
  Require("SlotVectorLifetime", "inserted",
          slots.is_allocated(first) && slots.try_get(first) != nullptr &&
              !destroyed,
          "slot insertion did not expose its cache-owned object");
  slots.erase(first);
  const bool first_destroyed = destroyed;
  destroyed = false;
  const auto second = slots.insert(&destroyed);
  Require("SlotVectorLifetime", "generation",
          first_destroyed && !destroyed && first.index == second.index &&
              first != second && !slots.is_allocated(first) &&
              slots.is_allocated(second),
          "reused slot accepted a stale cache resource id");
  std::printf("[host]    %-32s ok\n", "SlotVectorLifetime");
}

void CheckEmbeddedFetchLaneSpill() {
  std::vector<u32> code;
  code.push_back(EncodeSMovB32(0, InlineU32(0)));
  code.push_back(EncodeSmem0(0x02u, 20, 4));
  code.push_back(EncodeSmem1(0)); // s_load_dwordx4 s[20:23], s[8:9]
  AppendVop3(&code, 0x361u, 6, 20, InlineU32(0)); // v_writelane_b32 v6, s20, 0
  code.push_back(EncodeSMovB32(20, InlineU32(0)));
  AppendVop3(&code, 0x360u, 20, Vgpr(6),
             InlineU32(0)); // v_readlane_b32 s20, v6, 0
  code.push_back(EncodeVop2(0x01u, 0, Vgpr(8), 5));
  code.push_back(EncodeMubuf0(0x03u, 0, true));
  code.push_back(EncodeMubuf1(9, 5, 0));
  AppendEnd(&code);

  std::array<u32, 11> user_data{};
  ShaderVertexInputInfo vertex;
  vertex.fetch_embedded = true;
  vertex.fetch_buffer_reg = 0;
  vertex.resources_num = 1;
  vertex.resources[0].fields[3] =
      BufferFormat(Prospero::BufferFormat::k32_32_32_32Float) << 12u | DstSel(4, 5, 6, 7);
  vertex.resources_dst[0].attr_id = 0;
  vertex.resources_dst[0].registers_num = 4;

  ShaderRecompiler::CompileOptions options;
  options.stage = ShaderType::Vertex;
  options.user_data_base = 8;
  options.user_data = user_data;
  options.input_info.vertex = &vertex;

  auto translated = ShaderRecompiler::TranslateProgram(code, options);
  auto resource_plan =
      ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
  ShaderRecompiler::IR::ResourceSnapshot resources;
  ShaderRecompiler::IR::ResourceSpecialization specialization;
  const ShaderRecompiler::IR::SrtRuntime runtime{
      .user_data = options.user_data,
      .shader_base = reinterpret_cast<uint64_t>(code.data()),
  };
  Require("EmbeddedFetchLaneSpill", "resource materialization",
          ShaderRecompiler::IR::MaterializeResources(
              resource_plan, runtime, resources, specialization),
          "lane-spilled fetch resources could not be materialized");
  auto result = ShaderRecompiler::CompileProgram(
      std::move(translated), options, specialization);
  Require("EmbeddedFetchLaneSpill", "fetch rewrite",
          result.program.info.vertex_fetch_components[0] == 4,
          "lane-spilled fetch descriptor was not recognized and rewritten");
  std::printf("[host]    %-32s ok\n", "EmbeddedFetchLaneSpill");
}

void CheckReferenceClockScale() {
  uint64_t value = 0;
  Require("ReferenceClockScale", "zero",
          Sync::ScaleReferenceClock(0, 3000000000ull, value) && value == 0,
          "zero host tick did not produce a zero GPU clock");
  Require("ReferenceClockScale", "fractional second",
          Sync::ScaleReferenceClock(1500000000ull, 3000000000ull, value) &&
              value == 50000000ull,
          "host half-second did not scale to 50,000,000 ticks");
  Require("ReferenceClockScale", "whole and fractional",
          Sync::ScaleReferenceClock(3750000000ull, 3000000000ull, value) &&
              value == 125000000ull,
          "host 1.25 seconds did not scale to 125,000,000 ticks");
  Require("ReferenceClockScale", "monotonic floor",
          Sync::ScaleReferenceClock(3750000001ull, 3000000000ull, value) &&
              value == 125000000ull,
          "sub-reference-tick increment did not use a monotonic floor");
  Require("ReferenceClockScale", "guards",
          !Sync::ScaleReferenceClock(1, 0, value) &&
              !Sync::ScaleReferenceClock(UINT64_MAX, 1, value),
          "invalid frequency or overflow was accepted");
  std::printf("[host]    %-32s ok\n", "ReferenceClockScale");
}

void CheckClipControlDepthClipState() {
  HW::ClipControl clip;
  Require("ClipControlDepthClipState", "default", clip.IsZClipEnabled(),
          "default paired Z clipping was not enabled");

  clip.min_z_clip_disable = true;
  Require("ClipControlDepthClipState", "asymmetric near",
          !clip.IsZClipEnabled(),
          "near-plane disable did not disable paired host Z clipping");

  clip.min_z_clip_disable = false;
  clip.max_z_clip_disable = true;
  Require("ClipControlDepthClipState", "asymmetric far", !clip.IsZClipEnabled(),
          "far-plane disable did not disable paired host Z clipping");

  clip.min_z_clip_disable = true;
  Require("ClipControlDepthClipState", "both disabled", !clip.IsZClipEnabled(),
          "paired Z-clip disable was not represented");
  std::printf("[host]    %-32s ok\n", "ClipControlDepthClipState");
}

void CheckVulkan13FeatureRequirements() {
  const auto features = WindowContext::RequiredVulkan13Features();
  Require(
      "Vulkan13FeatureRequirements", "production requirements",
      features.sType == vk::StructureType::ePhysicalDeviceVulkan13Features &&
          features.pNext == nullptr && features.dynamicRendering == VK_TRUE &&
          features.synchronization2 == VK_TRUE,
      "production device creation did not require dynamic rendering and "
      "synchronization2 from one feature declaration");
  std::printf("[host]    %-32s ok\n", "Vulkan13FeatureRequirements");
}

void CheckPm4AcquireMemNoOp(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  const std::array<uint32_t, 6> standard_payload{0x00400000u, 1u, 0u,
                                                 0u,          0u, 10u};
  const std::array<uint32_t, 7> custom_payload{
      0xdeadbeefu, 0xffffffffu, 0x80000000u, 0x13579bdfu,
      0x2468ace0u, 0xaaaaaaaau, 0x55555555u};
  const auto standard_before = standard_payload;
  const auto custom_before = custom_payload;
  Require("Pm4AcquireMemNoOp", "recognized packets",
          CpOpAcquireMem(processor, 0xC0055800u, standard_payload.data(), 0,
                         0) == 6 &&
              CpOpAcquireMem(processor, 0xc0061050u, custom_payload.data(), 0,
                             0) == 7 &&
              standard_payload == standard_before &&
              custom_payload == custom_before,
          "ACQUIRE_MEM did not consume both pinned packet forms as "
          "side-effect-free "
          "no-ops");
  std::printf("[host]    %-32s ok\n", "Pm4AcquireMemNoOp");
}

void CheckPm4SyntheticOcclusionCounterDump(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  constexpr uint64_t untouched = 0x1122334455667788ull;
  constexpr uint64_t ready_bit = 1ull << 63u;
  alignas(16) std::array<std::array<uint64_t, 2>, 16> results;
  for (auto &pair : results) {
    pair.fill(untouched);
  }

  const auto dump = [&](uint64_t *address) {
    const auto raw_address = reinterpret_cast<uint64_t>(address);
    std::array<uint32_t, 4> packet{
        KYTY_PM4(4, Pm4::IT_EVENT_WRITE, Pm4::R_ZERO), 0x00000139u,
        static_cast<uint32_t>(raw_address),
        static_cast<uint32_t>(raw_address >> 32u)};
    Pm4Execution execution;
    return processor.Process(execution, packet);
  };

  const auto begin_result = dump(&results[0][0]);
  bool begin_written = begin_result == Pm4ProcessResult::Complete;
  for (const auto &pair : results) {
    begin_written &= pair[0] == ready_bit && pair[1] == untouched;
  }

  const auto end_result = dump(&results[0][1]);
  bool end_written = end_result == Pm4ProcessResult::Complete;
  uint64_t visible_samples = 0;
  for (const auto &pair : results) {
    end_written &= pair[0] == ready_bit && pair[1] == (ready_bit | 1u);
    visible_samples += pair[1] - pair[0];
  }

  Require("Pm4SyntheticOcclusionCounterDump", "always-visible result",
          begin_written && end_written && visible_samples == results.size(),
          "EVENT_WRITE did not publish valid nonzero begin/end counters to "
          "every PS5 DB");
  std::printf("[host]    %-32s ok\n", "Pm4SyntheticOcclusionCounterDump");
}

void CheckPm4Predication(RenderContext &renderer) {
  constexpr const char *name = "Pm4Predication";
  constexpr uint64_t ready = 1ull << 63u;
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  processor.BufferInit();
  alignas(16) std::array<uint64_t, 32> query{};
  alignas(16) uint64_t boolean = 0;
  uint32_t predicated = 0;
  uint32_t unconditional = 0;
  const auto commands = [&](uint32_t op, uint32_t condition, uint32_t wait,
                            const void *source) {
    const auto address = reinterpret_cast<uint64_t>(source);
    const auto predicated_address = reinterpret_cast<uint64_t>(&predicated);
    const auto unconditional_address = reinterpret_cast<uint64_t>(&unconditional);
    return std::array<uint32_t, 14>{
        // Captured packet form: c0022000 00010100 <query low> <query high>.
        0xc0022000u, (op << 16u) | (wait << 12u) | (condition << 8u),
        static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
        KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0) | 1u, 0,
        static_cast<uint32_t>(predicated_address),
        static_cast<uint32_t>(predicated_address >> 32u), 11,
        KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0), 0,
        static_cast<uint32_t>(unconditional_address),
        static_cast<uint32_t>(unconditional_address >> 32u), 22};
  };
  const auto check = [&](const char *stage, uint32_t op, uint32_t condition,
                         uint32_t wait, const void *source, bool skip) {
    predicated = unconditional = 0;
    const auto packet = commands(op, condition, wait, source);
    const auto tick = renderer.GetCommandScheduler().CurrentTick();
    Pm4Execution execution;
    Require(name, stage,
            processor.Process(execution, packet) == Pm4ProcessResult::Complete &&
                processor.ShouldSkipPredicatedPackets() == skip &&
                predicated == (skip ? 0u : 11u) && unconditional == 22,
            "predicate polarity, tagged packet execution, or packet consumption is wrong");
    Require(name, stage, renderer.GetCommandScheduler().CurrentTick() == tick,
            "predication unnecessarily submitted and drained the host queue");
  };
  struct Case {
    uint64_t delta;
    uint32_t condition;
    bool skip;
  };
  constexpr Case cases[]{{1, 1, false}, {1, 0, true},
                         {0, 1, true}, {0, 0, false}};
  for (const auto &test : cases) {
    query.fill(ready | 0x344u);
    for (size_t db = 0; db < 16; db++) {
      query[db * 2 + 1] += test.delta;
    }
    boolean = test.delta;
    for (uint32_t wait : {0u, 1u}) {
      check("ready Z-pass", 1, test.condition, wait, query.data(), test.skip);
      check("bool preservation", 3, test.condition, wait, &boolean, test.skip);
    }
  }
  query.fill(ready | 0x344u);
  query[31]++;
  check("only last DB visible", 1, 1, 0, query.data(), false);
  check("inverse last DB visibility", 1, 0, 0, query.data(), true);
  check("clear", 0, 0, 0, nullptr, false);

  // Every begin AND end counter must be ready; no-wait must discard old skip state.
  for (size_t missing = 0; missing < query.size(); missing++) {
    query.fill(ready | 0x344u);
    query[missing] &= ~ready;
    for (uint32_t condition : {0u, 1u}) {
      boolean = 0;
      processor.SetPredication(1, 3, 0, &boolean, 0);
      check("unavailable no-wait", 1, condition, 1, query.data(), false);
    }
  }

  // Retry the same packet when its last DB's begin or end arrives later.
  for (size_t missing : {30u, 31u}) {
    query.fill(ready | 0x344u);
    query[31]++;
    query[missing] &= ~ready;
    boolean = 0;
    processor.SetPredication(1, 3, 0, &boolean, 0);
    predicated = unconditional = 0;
    const auto packet = commands(1, 1, 0, query.data());
    Pm4Execution execution;
    for (int retry = 0; retry < 2; retry++) {
      Require(name, "pending wait suspends",
              processor.Process(execution, packet) == Pm4ProcessResult::Blocked &&
                  !execution.MadeProgress() && predicated == 0 && unconditional == 0 &&
                  processor.ShouldSkipPredicatedPackets(),
              "pending query advanced its packet or changed the prior predicate");
    }
    query[missing] |= ready;
    Require(name, "ready wait resumes",
            processor.Process(execution, packet) == Pm4ProcessResult::Complete &&
                execution.MadeProgress() && predicated == 11 && unconditional == 22 &&
                !processor.ShouldSkipPredicatedPackets(),
            "ready query did not reevaluate and execute the suspended packet's suffix");
  }
  processor.BufferWait();
  std::printf("[host]    %-32s ok\n", name);
}

void CheckPm4StencilInfoValueLane(RenderContext &renderer) {
  CommandProcessor processor(renderer, 0);
  constexpr std::array<uint32_t, 2> payload{0x00100801u, 0x28000000u};
  const auto consumed = HwCtxSetStencilInfo(
      processor, 0xC0016900u, Pm4::DB_STENCIL_INFO, payload.data(), 1);
  const auto &stencil = processor.GetCtx().GetDepthStencilInfo();
  Require("Pm4StencilInfoValueLane", "standalone register value",
          consumed == 1 && stencil.format == Prospero::StencilFormat::k8UInt &&
              stencil.texture_compatibility ==
                  Prospero::TextureCompatibleStencil::kEnable &&
              !stencil.expclear_enabled && !stencil.htile_stencil_disabled,
          "standalone DB_STENCIL_INFO did not decode its sole payload value");
  std::printf("[host]    %-32s ok\n", "Pm4StencilInfoValueLane");
}

void CheckPm4NativeTargetGeometryRegisters(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor packet(renderer, 0);
  CommandProcessor indirect(renderer, 0);

  const auto write_context = [](CommandProcessor &processor, uint32_t offset,
                                uint32_t value) {
    std::array<uint32_t, 3> command{
        KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO), offset, value};
    Pm4Execution execution;
    return processor.Process(execution, command) == Pm4ProcessResult::Complete;
  };

  constexpr uint32_t color_slot = 7;
  constexpr uint32_t width_minus_one = 1919;
  constexpr uint32_t height_minus_one = 1079;
  constexpr uint32_t depth_minus_one = 5;
  constexpr uint32_t max_mip = 3;
  constexpr uint32_t dimension_3d = 2;
  constexpr uint32_t attrib2_value =
      height_minus_one | (width_minus_one << 14u) | (max_mip << 28u);
  constexpr uint32_t attrib3_value =
      depth_minus_one |
      (static_cast<uint32_t>(Prospero::TileMode::kStandard64KB) << 14u) |
      (dimension_3d << 24u) | (1u << 26u) | (1u << 30u);

  const bool color_packets_complete =
      write_context(packet, Pm4::CB_COLOR7_ATTRIB2, attrib2_value) &&
      write_context(packet, Pm4::CB_COLOR7_ATTRIB3, attrib3_value);
  g_hw_ctx_indirect_func[Pm4::CB_COLOR7_ATTRIB2](
      indirect, Pm4::CB_COLOR7_ATTRIB2, attrib2_value);
  g_hw_ctx_indirect_func[Pm4::CB_COLOR7_ATTRIB3](
      indirect, Pm4::CB_COLOR7_ATTRIB3, attrib3_value);

  const auto color_matches = [](const HW::RenderTarget &target) {
    return target.attrib2.width == width_minus_one &&
           target.attrib2.height == height_minus_one &&
           target.attrib2.num_mip_levels == max_mip &&
           target.attrib3.depth == depth_minus_one &&
           target.attrib3.tile_mode == Prospero::TileMode::kStandard64KB &&
           target.attrib3.dimension == dimension_3d &&
           target.attrib3.metadata_pipe_aligned;
  };
  Require("Pm4NativeTargetGeometry", "sparse color geometry",
          color_packets_complete &&
              color_matches(packet.GetCtx().GetRenderTarget(color_slot)) &&
              color_matches(indirect.GetCtx().GetRenderTarget(color_slot)) &&
              !color_matches(packet.GetCtx().GetRenderTarget(0)) &&
              !color_matches(indirect.GetCtx().GetRenderTarget(0)),
          "ATTRIB2/ATTRIB3 did not provide native color extent and layout");

  constexpr uint32_t depth_size_value =
      width_minus_one | (height_minus_one << 16u);
  const bool depth_packet_complete =
      write_context(packet, Pm4::DB_DEPTH_SIZE_XY, depth_size_value);
  g_hw_ctx_indirect_func[Pm4::DB_DEPTH_SIZE_XY](indirect, Pm4::DB_DEPTH_SIZE_XY,
                                                depth_size_value);
  const auto depth_matches = [](const HW::DepthRenderTarget &target) {
    return target.size.valid && target.size.x_max == width_minus_one &&
           target.size.y_max == height_minus_one;
  };
  Require("Pm4NativeTargetGeometry", "native depth geometry",
          depth_packet_complete &&
              depth_matches(packet.GetCtx().GetDepthRenderTarget()) &&
              depth_matches(indirect.GetCtx().GetDepthRenderTarget()),
          "DB_DEPTH_SIZE_XY did not provide the native depth extent");

  bool legacy_slots_are_unhandled = true;
  for (uint32_t slot = 0; slot < 8; slot++) {
    const auto stride = slot * 15u;
    const std::array legacy_color_slots{
        Pm4::CB_COLOR0_BASE + stride + 1u,
        Pm4::CB_COLOR0_BASE + stride + 2u,
        Pm4::CB_COLOR0_CMASK + stride + 1u,
        Pm4::CB_COLOR0_FMASK + stride + 1u,
    };
    for (const auto offset : legacy_color_slots) {
      legacy_slots_are_unhandled &= g_hw_ctx_func[offset] == nullptr &&
                                    g_hw_ctx_indirect_func[offset] == nullptr;
    }
  }
  // These offsets are the removed GCN DB_DEPTH_INFO, DB_DEPTH_SIZE, and
  // DB_DEPTH_SLICE holes. Keep them numeric so deleting their legacy PM4 names
  // does not weaken this dispatch-table regression check.
  constexpr std::array<uint32_t, 3> legacy_depth_slots{0x000fu, 0x0016u,
                                                       0x0017u};
  for (const auto offset : legacy_depth_slots) {
    legacy_slots_are_unhandled &= g_hw_ctx_func[offset] == nullptr &&
                                  g_hw_ctx_indirect_func[offset] == nullptr;
  }
  // Removed GCN PA_SC_NGG_MODE_CNTL.
  legacy_slots_are_unhandled &= g_hw_ctx_func[0x0314u] == nullptr &&
                                g_hw_ctx_indirect_func[0x0314u] == nullptr;

  // Removed GCN shader resource/checksum/queue registers. Numeric offsets keep
  // this check independent of the deleted legacy names.
  constexpr std::array<uint32_t, 17> legacy_shader_slots{
      0x000u, 0x001u, 0x030u, 0x0b0u, 0x0bcu, 0x130u,
      0x14au, 0x14bu, 0x20eu, 0x20fu, 0x210u, 0x211u, 0x216u, 0x217u,
      0x219u, 0x21au, 0x27du,
  };
  for (const auto offset : legacy_shader_slots) {
    legacy_slots_are_unhandled &= g_hw_sh_func[offset] == nullptr &&
                                  g_hw_sh_indirect_func[offset] == nullptr;
  }
  // Restoring shader state after Toolkit rendering still emits the compiler's
  // GS-front allocation metadata at 0xca.
  for (uint32_t offset = 0x0cbu; offset <= 0x0ebu; offset++) {
    legacy_slots_are_unhandled &= g_hw_sh_func[offset] == nullptr &&
                                  g_hw_sh_indirect_func[offset] == nullptr;
  }

  std::array<uint32_t, 3> index_size_packet{
      KYTY_PM4(3, Pm4::IT_SET_UCONFIG_REG_INDEX, Pm4::R_ZERO),
      0x20000000u | Pm4::VGT_INDEX_TYPE,
      0x441u,
  };
  Pm4Execution index_size_execution;
  const bool native_index_size_packet_complete =
      packet.Process(index_size_execution, index_size_packet) ==
          Pm4ProcessResult::Complete &&
      g_hw_uc_func[Pm4::VGT_INDEX_TYPE] != nullptr &&
      g_hw_uc_indirect_func[Pm4::VGT_INDEX_TYPE] != nullptr &&
      g_hw_uc_func[Pm4::IA_MULTI_VGT_PARAM] != nullptr &&
      g_hw_uc_indirect_func[Pm4::IA_MULTI_VGT_PARAM] != nullptr;

  const bool native_ps_address_slots_are_handled =
      g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS] != nullptr &&
      g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_PS] != nullptr &&
      g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS] != nullptr &&
      g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_PS] != nullptr;
  const bool native_pace_slots_are_handled =
      g_hw_sh_indirect_func[Pm4::SPI_SHADER_PACE_ID_PS] != nullptr &&
      g_hw_sh_indirect_func[Pm4::SPI_SHADER_PACE_ID_GS] != nullptr &&
      g_hw_sh_indirect_func[Pm4::COMPUTE_PACE_ID] != nullptr;
  Require("Pm4NativeTargetGeometry", "legacy register holes",
          legacy_slots_are_unhandled && native_pace_slots_are_handled &&
              native_ps_address_slots_are_handled && native_index_size_packet_complete,
          "a removed GCN register has a handler or a native packet is not handled");
  std::printf("[host]    %-32s ok\n", "Pm4NativeTargetGeometry");
}

void CheckPm4IndirectShaderRegisters(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  std::array<uint32_t, 24> registers{
      Pm4::SPI_SHADER_PGM_RSRC4_GS, 0x0badc0deu,
      Pm4::SPI_SHADER_PGM_CHKSUM_HS, 0x12345678u,
      Pm4::SPI_SHADER_PGM_RSRC4_HS, 0x87654321u,
      Pm4::SPI_SHADER_PGM_LO_ES, 0x050025e5u,
      Pm4::SPI_SHADER_PGM_HI_ES, 0u,
      Pm4::SPI_SHADER_PGM_RSRC1_GS, 0x60000002u,
      Pm4::SPI_SHADER_PGM_RSRC2_GS, 0x00030008u,
      0x000000cau, 0x03000002u,
      Pm4::SPI_SHADER_PGM_LO_PS, 0x05000104u,
      Pm4::SPI_SHADER_PGM_HI_PS, 0u,
      Pm4::SPI_SHADER_PGM_RSRC1_PS, 0x022c018du,
      Pm4::SPI_SHADER_PGM_RSRC2_PS, 0x0000003cu,
  };
  const auto address = reinterpret_cast<uint64_t>(registers.data());
  std::array<uint32_t, 5> command{
      KYTY_PM4(5, Pm4::IT_SET_SH_REG_INDIRECT, Pm4::R_ZERO),
      static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
      0x80000000u, static_cast<uint32_t>(registers.size() / 2u),
  };
  Pm4Execution execution;
  const bool handlers_present =
      g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_GS] != nullptr &&
      g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_CHKSUM_HS] != nullptr &&
      g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_HS] != nullptr;
  Require("Pm4IndirectShaderRegisters", "indirect shader register stream",
          handlers_present &&
              processor.Process(execution, command) == Pm4ProcessResult::Complete,
          "shader registers were rejected by SET_SH_REG_INDIRECT");
  const auto &vs = processor.GetShCtx().GetVs();
  const auto &ps = processor.GetShCtx().GetPs().ps_regs;
  Require("Pm4IndirectShaderRegisters", "native state around front metadata",
          vs.es_regs.data_addr == 0x50025e500ull &&
              vs.gs_regs.rsrc1.vgprs == 2u &&
              vs.gs_regs.rsrc1.gs_vgpr_component_count == 3u &&
              vs.gs_regs.rsrc2.es_vgpr_component_count == 3u &&
              vs.gs_regs.rsrc2.user_sgpr == 4u &&
              ps.data_addr == 0x500010400ull && ps.rsrc1.vgprs == 13u &&
              ps.rsrc2.user_sgpr == 30u,
          "GS-front metadata changed native GS state or prevented later PS writes");
  std::printf("[host]    %-32s ok\n", "Pm4IndirectShaderRegisters");
}

void CheckPm4IndirectUconfigRegisters(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  std::array<uint32_t, 2> registers{
      Pm4::UC_PARAMETER_OVERSUBSCRIPTION, 0x12345678u,
  };
  const auto address = reinterpret_cast<uint64_t>(registers.data());
  std::array<uint32_t, 5> command{
      KYTY_PM4(5, Pm4::IT_SET_UCONFIG_REG_INDIRECT, Pm4::R_ZERO),
      static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
      0x80000000u, 1u,
  };
  Pm4Execution execution;
  const bool handler_present =
      g_hw_uc_indirect_func[Pm4::UC_PARAMETER_OVERSUBSCRIPTION] != nullptr;
  Require("Pm4IndirectUconfigRegisters", "indirect UCONFIG register stream",
          handler_present &&
              processor.Process(execution, command) == Pm4ProcessResult::Complete,
          "UCONFIG register was rejected by SET_UCONFIG_REG_INDIRECT");
  std::printf("[host]    %-32s ok\n", "Pm4IndirectUconfigRegisters");
}

void CheckPm4DirectShaderRegisterFallback(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  constexpr uint64_t shader_address = 0x0000123456789a00ull;
  std::array<uint32_t, 4> packet{
      KYTY_PM4(4, Pm4::IT_SET_SH_REG, Pm4::R_ZERO),
      Pm4::SPI_SHADER_PGM_LO_ES,
      static_cast<uint32_t>(shader_address >> 8u),
      static_cast<uint32_t>(shader_address >> 40u),
  };
  Pm4Execution execution;
  Require("Pm4DirectShaderRegisterFallback", "ES program base",
          processor.Process(execution, packet) == Pm4ProcessResult::Complete &&
              processor.GetShCtx().GetVs().es_regs.data_addr == shader_address,
          "direct SET_SH_REG did not route its ES base pair through the "
          "register handlers");
  std::printf("[host]    %-32s ok\n", "Pm4DirectShaderRegisterFallback");
}

void CheckPm4GuardBandRegisterRanges(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  processor.GetCtx().SetGuardBands(2.0f, 3.0f, 4.0f, 5.0f);

  std::array<uint32_t, 3> single{
      KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
      Pm4::PA_CL_GB_VERT_CLIP_ADJ,
      std::bit_cast<uint32_t>(6.0f),
  };
  Pm4Execution single_execution;
  const auto single_result = processor.Process(single_execution, single);
  const auto &single_viewport = processor.GetCtx().GetScreenViewport();
  Require("Pm4GuardBandRegisterRanges", "single register",
          single_result == Pm4ProcessResult::Complete &&
              single_viewport.guard_band_horz_clip == 2.0f &&
              single_viewport.guard_band_vert_clip == 6.0f &&
              single_viewport.guard_band_horz_discard == 4.0f &&
              single_viewport.guard_band_vert_discard == 5.0f,
          "single guard-band write changed the wrong context state");

  std::array<uint32_t, 6> range{
      KYTY_PM4(6, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
      Pm4::PA_CL_GB_VERT_CLIP_ADJ,
      std::bit_cast<uint32_t>(7.0f),
      std::bit_cast<uint32_t>(8.0f),
      std::bit_cast<uint32_t>(9.0f),
      std::bit_cast<uint32_t>(10.0f),
  };
  Pm4Execution range_execution;
  const auto range_result = processor.Process(range_execution, range);
  const auto &range_viewport = processor.GetCtx().GetScreenViewport();
  Require("Pm4GuardBandRegisterRanges", "full range",
          range_result == Pm4ProcessResult::Complete &&
              range_viewport.guard_band_horz_clip == 9.0f &&
              range_viewport.guard_band_vert_clip == 7.0f &&
              range_viewport.guard_band_horz_discard == 10.0f &&
              range_viewport.guard_band_vert_discard == 8.0f,
          "full guard-band range did not preserve hardware register order");
  std::printf("[host]    %-32s ok\n", "Pm4GuardBandRegisterRanges");
}

void CheckPm4BlendColorRegisterRanges(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  processor.GetCtx().SetBlendColor({1.0f, 2.0f, 3.0f, 4.0f});

  constexpr std::array<std::pair<uint32_t, float>, 4> singleton_values{{
      {Pm4::CB_BLEND_RED, 5.0f},
      {Pm4::CB_BLEND_BLUE, 7.0f},
      {Pm4::CB_BLEND_GREEN, 6.0f},
      {Pm4::CB_BLEND_ALPHA, 8.0f},
  }};
  bool singletons_complete = true;
  for (const auto &[offset, value] : singleton_values) {
    std::array<uint32_t, 3> packet{
        KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
        offset,
        std::bit_cast<uint32_t>(value),
    };
    Pm4Execution execution;
    singletons_complete &=
        processor.Process(execution, packet) == Pm4ProcessResult::Complete;
  }
  const auto &singletons = processor.GetCtx().GetBlendColor();
  Require("Pm4BlendColorRegisterRanges", "single registers",
          singletons_complete && singletons.red == 5.0f &&
              singletons.green == 6.0f && singletons.blue == 7.0f &&
              singletons.alpha == 8.0f,
          "single blend-color writes did not follow native PS5 register order");

  std::array<uint32_t, 4> partial{
      KYTY_PM4(4, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
      Pm4::CB_BLEND_BLUE,
      std::bit_cast<uint32_t>(9.0f),
      std::bit_cast<uint32_t>(10.0f),
  };
  Pm4Execution partial_execution;
  const auto partial_result = processor.Process(partial_execution, partial);
  const auto &partial_color = processor.GetCtx().GetBlendColor();
  Require("Pm4BlendColorRegisterRanges", "partial range",
          partial_result == Pm4ProcessResult::Complete &&
              partial_color.red == 5.0f && partial_color.green == 10.0f &&
              partial_color.blue == 9.0f && partial_color.alpha == 8.0f,
          "partial blend-color range changed the wrong channels");

  std::array<uint32_t, 6> full{
      KYTY_PM4(6, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
      Pm4::CB_BLEND_RED,
      std::bit_cast<uint32_t>(11.0f),
      std::bit_cast<uint32_t>(13.0f),
      std::bit_cast<uint32_t>(12.0f),
      std::bit_cast<uint32_t>(14.0f),
  };
  Pm4Execution full_execution;
  const auto full_result = processor.Process(full_execution, full);
  const auto &full_color = processor.GetCtx().GetBlendColor();
  Require("Pm4BlendColorRegisterRanges", "full range",
          full_result == Pm4ProcessResult::Complete &&
              full_color.red == 11.0f && full_color.green == 12.0f &&
              full_color.blue == 13.0f && full_color.alpha == 14.0f,
          "full blend-color range did not preserve native PS5 register order");
  std::printf("[host]    %-32s ok\n", "Pm4BlendColorRegisterRanges");
}

void CheckPm4PolygonOffsetRegisters(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  const auto &defaults = processor.GetCtx().GetPolyOffset();
  Require("Pm4PolygonOffset", "format defaults",
          defaults.neg_num_db_bits == -23 && defaults.db_is_float_fmt,
          "polygon offset Z format did not default to D32F");

  constexpr std::array<uint32_t, 6> payload{0x000000f0u,
                                            std::bit_cast<uint32_t>(2.5f),
                                            std::bit_cast<uint32_t>(16.0f),
                                            std::bit_cast<uint32_t>(-3.25f),
                                            std::bit_cast<uint32_t>(32.0f),
                                            std::bit_cast<uint32_t>(4.5f)};
  const auto consumed = HwCtxSetPolyOffsetRegisters(
      processor, KYTY_PM4(8, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
      Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL, payload.data(), 0);
  const auto &direct = processor.GetCtx().GetPolyOffset();
  Require("Pm4PolygonOffset", "direct registers",
          consumed == payload.size() && direct.neg_num_db_bits == -16 &&
              !direct.db_is_float_fmt && direct.clamp == 2.5f &&
              direct.front_scale == 16.0f && direct.front_offset == -3.25f &&
              direct.back_scale == 32.0f && direct.back_offset == 4.5f,
          "direct polygon offset register packet was decoded incorrectly");

  g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL](
      processor, Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL, 0x000001e9u);
  g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE](
      processor, Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE,
      std::bit_cast<uint32_t>(24.0f));
  const auto &indirect = processor.GetCtx().GetPolyOffset();
  Require("Pm4PolygonOffset", "indirect registers",
          indirect.neg_num_db_bits == -23 && indirect.db_is_float_fmt &&
              indirect.front_scale == 24.0f && indirect.front_offset == -3.25f,
          "indirect polygon offset register write was decoded incorrectly");
  std::printf("[host]    %-32s ok\n", "Pm4PolygonOffset");
}

void CheckPm4DepthControlHighBits(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);

  std::array<uint32_t, 3> direct{
      KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
      Pm4::DB_DEPTH_CONTROL,
      0x80000072u,
  };
  Pm4Execution direct_execution;
  const bool direct_complete =
      processor.Process(direct_execution, direct) == Pm4ProcessResult::Complete;
  const auto &direct_control = processor.GetCtx().GetDepthControl();
  const bool direct_decoded =
      !direct_control.stencil_enable && direct_control.z_enable &&
      !direct_control.z_write_enable && !direct_control.depth_bounds_enable &&
      direct_control.zfunc == 7u && !direct_control.backface_enable &&
      direct_control.stencilfunc == 0u && direct_control.stencilfunc_bf == 0u;

  std::array<uint32_t, 2> registers{
      Pm4::DB_DEPTH_CONTROL,
      0x4070078du,
  };
  const auto address = reinterpret_cast<uint64_t>(registers.data());
  std::array<uint32_t, 5> indirect{
      KYTY_PM4(5, Pm4::IT_SET_CONTEXT_REG_INDIRECT, Pm4::R_ZERO),
      static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
      0x80000000u, 1u,
  };
  Pm4Execution indirect_execution;
  const bool indirect_complete =
      processor.Process(indirect_execution, indirect) ==
      Pm4ProcessResult::Complete;
  const auto &indirect_control = processor.GetCtx().GetDepthControl();
  const bool indirect_decoded =
      indirect_control.stencil_enable && !indirect_control.z_enable &&
      indirect_control.z_write_enable && indirect_control.depth_bounds_enable &&
      indirect_control.zfunc == 0u && indirect_control.backface_enable &&
      indirect_control.stencilfunc == 7u &&
      indirect_control.stencilfunc_bf == 7u;

  Require("Pm4DepthControlHighBits", "preserved high bits",
          direct_complete && direct_decoded && indirect_complete &&
              indirect_decoded,
          "DB_DEPTH_CONTROL high bits rejected or changed decoded PS5 state");
  std::printf("[host]    %-32s ok\n", "Pm4DepthControlHighBits");
}

void CheckPm4DepthRenderOverride(RenderContext &renderer) {
  constexpr const char *name = "Pm4DepthRenderOverride";
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  struct Case {
    uint32_t value;
    HW::DepthRenderOverride expected;
  };
  constexpr std::array cases{Case{0x28000000u, {true, true, false, false}},
                             Case{0x50000000u, {false, false, true, true}},
                             Case{0x18000000u, {false, true, false, true}},
                             Case{0x60000000u, {true, false, true, false}},
                             Case{0xf8000000u, {true, true, true, true}},
                             Case{0x82001a0fu, {false, false, false, false}}};
  for (const bool indirect : {false, true}) {
    for (const auto &test : cases) {
      std::array<uint32_t, 2> registers{Pm4::DB_RENDER_OVERRIDE, test.value};
      const auto address = reinterpret_cast<uint64_t>(registers.data());
      const std::array<uint32_t, 3> direct_packet{
          KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
          Pm4::DB_RENDER_OVERRIDE, test.value};
      const std::array<uint32_t, 5> indirect_packet{
          KYTY_PM4(5, Pm4::IT_SET_CONTEXT_REG_INDIRECT, Pm4::R_ZERO),
          static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
          0x80000000u, 1u};
      Pm4Execution execution;
      const auto result = processor.Process(
          execution, indirect ? std::span<const uint32_t>{indirect_packet}
                              : std::span<const uint32_t>{direct_packet});
      const auto &actual = processor.GetCtx().GetDepthRenderOverride();
      Require(
          name, indirect ? "indirect register" : "direct register",
          result == Pm4ProcessResult::Complete &&
              actual.force_z_valid == test.expected.force_z_valid &&
              actual.force_z_dirty == test.expected.force_z_dirty &&
              actual.force_stencil_valid == test.expected.force_stencil_valid &&
              actual.force_stencil_dirty == test.expected.force_stencil_dirty,
          "DB_RENDER_OVERRIDE lost, exchanged, or retained force flags");
    }
  }
  std::printf("[host]    %-32s ok\n", name);
}

void CheckShaderFusion() {
  struct Case {
    bool gs;
    uint32_t front1, front2, back1, back2, fused1, fused2, older2;
  };
  // Expected register results for both shader-fusion variants.
  constexpr Case cases[]{
      {true, 0x600c0007, 0x00030010, 0x600c0007, 0x008b0000,
       0x600c0007, 0x008b0010, 0x008b0010}, // Captured volume ES + GS.
      {true, 0x20ac0003, 0xf805004a, 0x40ec0007, 0x028b003e,
       0x40ec0007, 0x1a8f000a, 0xfa8f000a},
      {false, 0x20ac0003, 0xf805004a, 0x10ec0007, 0x028b003e,
       0x20ec0007, 0x1a8b000a, 0xfa8b000a},
  };
  ShaderUserData user_data{};
  ShaderSpecialRegs specials{};
  for (const auto &c : cases) {
    const auto checksum = c.gs ? Pm4::SPI_SHADER_PGM_CHKSUM_GS : Pm4::SPI_SHADER_PGM_CHKSUM_HS;
    const auto rsrc1 = c.gs ? Pm4::SPI_SHADER_PGM_RSRC1_GS : Pm4::SPI_SHADER_PGM_RSRC1_HS;
    const auto rsrc2 = c.gs ? Pm4::SPI_SHADER_PGM_RSRC2_GS : Pm4::SPI_SHADER_PGM_RSRC2_HS;
    const auto lo = c.gs ? Pm4::SPI_SHADER_PGM_LO_ES : Pm4::SPI_SHADER_PGM_LO_LS;
    std::array<ShaderRegister, 6> front_regs{{
        {checksum, 0x1234}, {checksum, 0x5678}, {rsrc1, c.front1},
        {rsrc2, c.front2}, {lo, 0}, {lo + 1u, 0}}};
    const std::array<ShaderRegister, 6> back_regs{{
        {checksum, 0}, {checksum, 0}, {rsrc1, c.back1},
        {rsrc2, c.back2}, {lo, 0}, {lo + 1u, 0xabcdef00}}};
    Shader front{};
    front.user_data = &user_data;
    front.code = reinterpret_cast<const void *>(0x12580e91c00ull);
    front.sh_registers = front_regs.data();
    front.specials = &specials;
    front.type = static_cast<uint8_t>(c.gs ? Prospero::ShaderBinaryType::kGsFront
                                         : Prospero::ShaderBinaryType::kHsFront);
    front.num_sh_registers = front_regs.size();
    for (const bool older : {false, true}) {
      for (const bool in_place : {false, true}) {
        auto registers = back_regs;
        std::array<ShaderRegister, 6> scratch{};
        Shader back = front;
        back.sh_registers = registers.data();
        back.type = static_cast<uint8_t>(c.gs ? Prospero::ShaderBinaryType::kGsBack
                                            : Prospero::ShaderBinaryType::kHsBack);
        Shader fused{};
        const auto fuse = older ? Gen5::AgcUnknownNApJjpKNBl4 : Gen5::AgcUnknownFuseShaderHalves;
        const auto result = fuse(&fused, &front, &back, in_place ? nullptr : scratch.data());
        const auto *regs = fused.sh_registers;
        Require("ShaderFusion", "native fused registers",
                result == 0 && regs == (in_place ? registers.data() : scratch.data()) &&
                    regs[0].value == 0x1234 && regs[1].value == 0x5678 &&
                    regs[2].value == c.fused1 && regs[3].value == (older ? c.older2 : c.fused2) &&
                    regs[4].value == 0x2580e91c && regs[5].value == 0xabcdef01 &&
                    fused.type == static_cast<uint8_t>(c.gs ? Prospero::ShaderBinaryType::kGs
                                                            : Prospero::ShaderBinaryType::kHs) &&
                    fused.user_data == (older ? &user_data : nullptr) &&
                    (in_place || std::memcmp(registers.data(), back_regs.data(), sizeof(registers)) == 0),
                "shader fusion lost front resources or changed the export's allocation/metadata behavior");
      }
    }
  }
  std::printf("[host]    %-32s ok\n", "ShaderFusion");
}

void CheckAgcSystemTable(RenderContext &renderer) {
  constexpr const char *name = "AgcSystemTable";
  constexpr uint64_t driver_base = 0x0fe0000000ull;
  constexpr uint64_t table_address = driver_base + 0x40000;
  constexpr std::array<uint32_t, 16> native_validator{
      0xbeeb03ff, 0x00000021, 0xb96a1818, 0xbf06106a,
      0xbf800000, 0x8590807e, 0xb96a0a18, 0xbf066a80,
      0xbf800000, 0x85ea807e, 0x88ea106a, 0xbf870003,
      0xbefc03ff, 0x8a6ca000, 0xbf920009, 0xbefd210e};
  LibKernel::Memory::InstallGpuResources(&renderer);
  Require(name, "driver module startup", Gen5Driver::Initialize() == 0,
          "driver module startup failed before guest initialization");
  const auto direct_memory_size = LibKernel::Memory::KernelGetDirectMemorySize();
  int64_t available_offset = -1;
  size_t available_size = 0;
  Require(name, "remaining direct pool",
          LibKernel::Memory::KernelAvailableDirectMemorySize(
              0, direct_memory_size, 0, &available_offset, &available_size) == 0 &&
              available_size == direct_memory_size - 0x200000,
          "driver startup did not reserve its backing in the reported direct pool");
  int64_t guest_allocation = -1;
  Require(name, "guest consumes remaining direct pool",
          LibKernel::Memory::KernelAllocateDirectMemory(
              0, direct_memory_size, available_size, 0x4000, 0, &guest_allocation) == 0,
          "guest could not allocate all remaining direct memory");
  uint32_t state = 0xabcdef01u;
  Require(name, "driver startup", Gen5::AgcInit(&state, 13) == 0 && state == 0xabcdef01u,
          "Agc startup required free guest memory or changed its ignored public state pointer");
  LibKernel::Memory::VirtualQueryInfo mapping{};
  Require(name, "shared driver backing",
          LibKernel::Memory::KernelVirtualQuery(reinterpret_cast<void *>(driver_base), 0,
                                                 &mapping, sizeof(mapping)) == 0 &&
              mapping.start == driver_base && mapping.end == driver_base + 0x200000 &&
              mapping.memory_type == 12 && mapping.protection == 0x33 && mapping.is_direct &&
              std::strcmp(mapping.name, "SceAgcDriver") == 0 &&
              renderer.IsMapped(driver_base, 0x200000),
          "driver startup bypassed shared DMEM metadata or GPU range registration");
  ShaderBufferResource descriptor{};
  std::array<uint64_t, 16> functions{};
  std::array<uint32_t, 16> code{};
  Require(name, "ordered-count function backing",
          LibKernel::Memory::TryReadBacking(table_address, &descriptor, sizeof(descriptor)) &&
              descriptor.Base48() == table_address + 0x60 && descriptor.Stride() == 16 &&
              descriptor.NumRecords() == 8 && descriptor.fields[3] == 0x5204 &&
              LibKernel::Memory::TryReadBacking(descriptor.Base48(), functions.data(),
                                                 sizeof(functions)) &&
              functions[4] != 0 && (functions[4] & 0xffu) == 0 && functions[5] == 0 &&
              LibKernel::Memory::TryReadBacking(functions[4], code.data(), sizeof(code)) &&
              code == native_validator,
          "system descriptor did not reach the queue validator and its user data");
  auto *instrumentation = reinterpret_cast<uint64_t *>(table_address + 0x18);
  Require(name, "default instrumentation", *instrumentation == 0,
          "unused vertex validation was enabled at startup");
  *instrumentation = 0x12340000u;
  Require(name, "repeat startup", Gen5Driver::Initialize() == 0 &&
              Gen5::AgcInit(nullptr, 13) == 0 &&
              *instrumentation == 0x12340000u,
          "repeat Agc initialization reset existing system state");
  *instrumentation = 0;
  Require(name, "release guest allocation",
          LibKernel::Memory::KernelCheckedReleaseDirectMemory(guest_allocation, available_size) == 0,
          "guest allocation release failed");
  LibKernel::Memory::InstallGpuResources(nullptr);
  std::printf("[host]    %-32s ok\n", name);
}

struct CommandBufferLayout {
  using Callback = KYTY_SYSV_ABI bool (*)(Gen5::CommandBuffer *, uint32_t,
                                          void *);

  uint32_t *bottom;
  uint32_t *top;
  uint32_t *cursor_up;
  uint32_t *cursor_down;
  Callback callback;
  void *user_data;
  uint32_t reserved_dw;
};
static_assert(offsetof(CommandBufferLayout, cursor_up) == 0x10);
static_assert(offsetof(CommandBufferLayout, reserved_dw) == 0x30);

void CheckPm4WaitPackets(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  constexpr auto packet_mismatch = static_cast<int>(0x8a6c000cu);
  const auto execute = [&](uint32_t *packet, uint32_t size_dw) {
    Pm4Execution execution;
    return processor.Process(execution, {packet, size_dw}) ==
           Pm4ProcessResult::Complete;
  };

  alignas(8) uint32_t label32 = 0x11223344u;
  std::array<uint32_t, 16> packet32{};
  CommandBufferLayout dcb32{packet32.data(),
                            packet32.data() + packet32.size(),
                            packet32.data(),
                            packet32.data() + packet32.size(),
                            nullptr,
                            nullptr,
                            0};
  const auto address32 = reinterpret_cast<uint64_t>(&label32);
  auto *emitted32 =
      Gen5::AgcDcbWaitRegMem(reinterpret_cast<Gen5::CommandBuffer *>(&dcb32), 0,
                             3, 0, 0, &label32, label32, UINT32_MAX, 400);
  Require("Pm4WaitPackets", "native 32-bit packet",
          Gen5::AgcDcbWaitOnAddressGetSize(0) == 56 &&
              emitted32 == packet32.data() &&
              dcb32.cursor_up == packet32.data() + 14 &&
              packet32[0] == 0xc0027904u && packet32[1] == 0x342u &&
              packet32[2] ==
                  (0xc8010000u |
                   (static_cast<uint32_t>(address32 >> 32u) & 0xffffu)) &&
              packet32[3] == static_cast<uint32_t>(address32) &&
              packet32[4] == 0xc0053c00u && packet32[5] == 0x13u &&
              packet32[6] == (static_cast<uint32_t>(address32) & ~0x3u) &&
              packet32[7] ==
                  (static_cast<uint32_t>(address32 >> 32u) & 0x3ffffu) &&
              packet32[8] == label32 && packet32[9] == UINT32_MAX &&
              packet32[10] == 25u && packet32[11] == 0xc0017904u &&
              packet32[12] == 0x342u && packet32[13] == 0xc8000000u &&
              execute(packet32.data(), 14),
          "32-bit wait packet differs from the expected stream");

  alignas(8) uint32_t patched_label32 = 0xaabbccddu;
  const auto patched_address32 = reinterpret_cast<uint64_t>(&patched_label32);
  packet32[6] |= 0x2u;
  packet32[7] |= 0xa5a40000u;
  Require(
      "Pm4WaitPackets", "native 32-bit patch",
      Gen5::AgcWaitRegMemPatchAddress(packet32.data(), &patched_label32) == 0 &&
          Gen5::AgcWaitRegMemPatchReference(packet32.data(), patched_label32) ==
              0 &&
          packet32[2] ==
              (0xc8010000u |
               (static_cast<uint32_t>(patched_address32 >> 32u) & 0xffffu)) &&
          packet32[3] == static_cast<uint32_t>(patched_address32) &&
          packet32[6] ==
              ((static_cast<uint32_t>(patched_address32) & ~0x3u) | 0x2u) &&
          packet32[7] ==
              ((static_cast<uint32_t>(patched_address32 >> 32u) & 0x3ffffu) |
               0xa5a40000u) &&
          packet32[8] == patched_label32 && execute(packet32.data(), 14),
      "32-bit native wait patch did not update metadata and hardware packet");

  alignas(8) uint64_t label64 = 0x1122334455667788ull;
  std::array<uint32_t, 16> packet64{};
  CommandBufferLayout dcb64{packet64.data(),
                            packet64.data() + packet64.size(),
                            packet64.data(),
                            packet64.data() + packet64.size(),
                            nullptr,
                            nullptr,
                            0};
  const auto address64 = reinterpret_cast<uint64_t>(&label64);
  auto *emitted64 =
      Gen5::AgcDcbWaitRegMem(reinterpret_cast<Gen5::CommandBuffer *>(&dcb64), 1,
                             3, 0, 0, &label64, label64, UINT64_MAX, 400);
  Require("Pm4WaitPackets", "native 64-bit packet",
          Gen5::AgcDcbWaitOnAddressGetSize(1) == 64 &&
              Gen5::AgcDcbWaitOnAddressGetSize(2) == 0 &&
              emitted64 == packet64.data() &&
              dcb64.cursor_up == packet64.data() + packet64.size() &&
              packet64[0] == 0xc0027904u && packet64[1] == 0x342u &&
              packet64[2] ==
                  (0xc8020000u |
                   (static_cast<uint32_t>(address64 >> 32u) & 0xffffu)) &&
              packet64[3] == static_cast<uint32_t>(address64) &&
              packet64[4] == 0xc0079300u && packet64[5] == 0x13u &&
              packet64[6] == (static_cast<uint32_t>(address64) & ~0x7u) &&
              packet64[7] ==
                  (static_cast<uint32_t>(address64 >> 32u) & 0x3ffffu) &&
              packet64[8] == static_cast<uint32_t>(label64) &&
              packet64[9] == static_cast<uint32_t>(label64 >> 32u) &&
              packet64[10] == UINT32_MAX && packet64[11] == UINT32_MAX &&
              packet64[12] == 25u && packet64[13] == 0xc0017904u &&
              packet64[14] == 0x342u && packet64[15] == 0xc8000000u &&
              execute(packet64.data(), packet64.size()),
          "64-bit wait packet differs from the expected stream");

  alignas(8) uint64_t patched_label64 = 0x11223344aabbccddull;
  const auto patched_address64 = reinterpret_cast<uint64_t>(&patched_label64);
  packet64[6] |= 0x5u;
  packet64[7] |= 0x5a580000u;
  Require(
      "Pm4WaitPackets", "native 64-bit patch",
      Gen5::AgcWaitRegMemPatchAddress(packet64.data(), &patched_label64) == 0 &&
          Gen5::AgcWaitRegMemPatchReference(packet64.data(),
                                            0xdeadbeefaabbccddull) == 0 &&
          packet64[2] ==
              (0xc8020000u |
               (static_cast<uint32_t>(patched_address64 >> 32u) & 0xffffu)) &&
          packet64[3] == static_cast<uint32_t>(patched_address64) &&
          packet64[6] ==
              ((static_cast<uint32_t>(patched_address64) & ~0x7u) | 0x5u) &&
          packet64[7] ==
              ((static_cast<uint32_t>(patched_address64 >> 32u) & 0x3ffffu) |
               0x5a580000u) &&
          packet64[8] == 0xaabbccddu && packet64[9] == 0x11223344u &&
          execute(packet64.data(), packet64.size()),
      "64-bit native wait patch did not preserve the native high reference "
      "DWORD");

  std::array<uint32_t, 16> invalid{};
  const uint32_t short_payload = 0x342u;
  Require(
      "Pm4WaitPackets", "invalid packet",
      Gen5::AgcWaitRegMemPatchAddress(invalid.data(), &label32) ==
              packet_mismatch &&
          Gen5::AgcWaitRegMemPatchReference(invalid.data(), label32) ==
              packet_mismatch &&
          !Gen5::AgcIsInternalDataPacket(
              KYTY_PM4(2, Pm4::IT_SET_UCONFIG_REG, 1), &short_payload),
      "wait patch accepted a packet without the metadata prefix");
  std::printf("[host]    %-32s ok\n", "Pm4WaitPackets");
}

void CheckPm4DrawIndirectMultiPacket(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  const auto execute = [&](uint32_t *packet, uint32_t size_dw) {
    Pm4Execution execution;
    return processor.Process(execution, {packet, size_dw}) ==
           Pm4ProcessResult::Complete;
  };

  constexpr uint64_t modifier = 0x0000000069380b1dull;
  constexpr auto packet_mismatch = static_cast<int>(0x8a6c000cu);
  const auto *count_address = reinterpret_cast<const volatile void *>(
      static_cast<uintptr_t>(0x123456789abcdef3ull));
  constexpr std::array<uint32_t, 16> expected{
      0xc0017904u, 0x00000342u, 0xc6000008u, 0xc0082c00u,
      0x11223344u, 0x00000111u, 0x00000113u, 0xc8000115u,
      0x55667788u, 0x9abcdef0u, 0x12345678u, 0xaabbccddu,
      0x00000022u, 0xc0017904u, 0x00000342u, 0xc6000000u};

  std::array<uint32_t, 16> packet{};
  CommandBufferLayout dcb{packet.data(),
                          packet.data() + packet.size(),
                          packet.data(),
                          packet.data() + packet.size(),
                          nullptr,
                          nullptr,
                          0};
  auto *emitted = Gen5::AgcDcbDrawIndirectMulti(
      reinterpret_cast<Gen5::CommandBuffer *>(&dcb), 0x11223344u, 1u,
      0x55667788u, count_address, 0xaabbccddu, modifier);

  const uint32_t invalid_payload[]{0x342u, 0xc6000010u};
  Require("Pm4DrawIndirectMulti", "native packet",
          emitted == packet.data() &&
              dcb.cursor_up == packet.data() + packet.size() &&
              packet == expected && execute(packet.data(), 3u) &&
              execute(packet.data() + 13u, 3u) &&
              Gen5::AgcWaitRegMemPatchAddress(packet.data(), count_address) ==
                  packet_mismatch &&
              Gen5::AgcWaitRegMemPatchReference(packet.data(), 0x12345678u) ==
                  packet_mismatch &&
              !Gen5::AgcIsInternalDataPacket(0xc0017904u, invalid_payload),
          "indirect multi-draw packet differs from the expected stream");

  constexpr uint32_t data_offset = 0x5c08;
  constexpr uint32_t stride = 32;
  std::array<uint32_t, data_offset / 4 + stride / 4 + 5> arguments{};
  const std::array<uint32_t, 5> first{0, 1, 3, 0xfffffffeu, 9};
  const std::array<uint32_t, 5> second{0, 2, 5, 7, 11};
  std::copy(first.begin(), first.end(), arguments.begin() + data_offset / 4);
  std::copy(second.begin(), second.end(), arguments.begin() + (data_offset + stride) / 4);
  std::array<uint32_t, 8> indices{};
  processor.BufferInit();
  processor.SetDrawIndirectArgsBaseAddress(reinterpret_cast<uint64_t>(arguments.data()));
  processor.SetIndexBaseAddress(reinterpret_cast<uint64_t>(indices.data()));
  processor.SetIndexType(0);
  std::array<uint32_t, 5> captured{0xc0032500u, data_offset, 0x94, 0x95, 0};
  Require("Pm4DrawIndirectMulti", "captured indexed single execution",
          execute(captured.data(), captured.size()),
          "indexed indirect draw rejected native source-select zero");
  // Zero primitive counts still traverse CP argument decoding and renderer entry,
  // while avoiding unrelated shader and attachment requirements.
  for (const bool indexed : {false, true}) {
    for (const bool multi : {false, true}) {
      for (const uint64_t draw_modifier : {0ull, 0x100ull, 1ull << 32u, (1ull << 32u) | 0x100ull}) {
        packet.fill(0);
        dcb.cursor_up = packet.data();
        auto *buffer = reinterpret_cast<Gen5::CommandBuffer *>(&dcb);
        alignas(4) uint32_t draw_count = 3;
        if (multi) {
          emitted = indexed ? Gen5::AgcDcbDrawIndexIndirectMulti(buffer, data_offset, 1, 2,
                                  &draw_count, stride, draw_modifier)
                            : Gen5::AgcDcbDrawIndirectMulti(buffer, data_offset, 1, 2,
                                  &draw_count, stride, draw_modifier);
        } else {
          emitted = indexed ? Gen5::AgcDcbDrawIndexIndirect(buffer, data_offset, draw_modifier)
                            : Gen5::AgcDcbDrawIndirect(buffer, data_offset, draw_modifier);
        }
        const uint32_t begin = multi && !indexed ? 3 : 0;
        const uint32_t words = multi ? (indexed ? 10 : 16) : 5;
        const uint32_t opcode = multi ? (indexed ? 0xc0083800u : 0xc0082c00u)
                                      : (indexed ? 0xc0032500u : 0xc0032400u);
        const uint32_t initiator = (indexed ? 0 : 2) |
            (draw_modifier == 0x100 ? 0x20 : 0);
        Require("Pm4DrawIndirectMulti", "indexed/auto initiator emission",
                emitted == packet.data() && dcb.cursor_up == packet.data() + words &&
                    packet[begin] == opcode && packet[begin + 1] == data_offset &&
                    packet[begin + (multi ? 9 : 4)] == initiator,
                "indirect opcode, source select, or modifier bits differ from the native packet");
        Require("Pm4DrawIndirectMulti", "single/multi argument execution",
                execute(packet.data(), words),
                "indirect draw failed shared argument decoding or padded/count-limited traversal");
        if (multi) {
          draw_count = 0;
          Require("Pm4DrawIndirectMulti", "zero indirect count execution",
                  execute(packet.data(), words), "zero indirect draw count did not complete");
        }
      }
    }
  }
  renderer.GetCommandScheduler().Finish();
  processor.BufferInit();
  auto &shaders = processor.GetShCtx();
  const auto &gs = shaders.GetVs().gs_user_sgpr.value;
  const auto &hs = shaders.GetVs().hs_user_sgpr.value;
  const auto check_offsets = [&](auto &commands, const auto &sgprs,
                                 std::initializer_list<uint32_t> expected_offsets,
                                 const char *label) {
    Require("Pm4DrawIndirectMulti", label,
            execute(commands.data(), commands.size()) &&
                std::equal(expected_offsets.begin(), expected_offsets.end(), sgprs + 9),
            "indirect draw did not write the expected user SGPR offsets");
  };
  // These locations match the captured NGG prolog: user SGPR9/10 become s17/s18.
  shaders.SetGsUserSgpr(9, 17821, HW::UserSgprType::Unknown);
  shaders.SetGsUserSgpr(10, 0x898f3f24u, HW::UserSgprType::Unknown);
  arguments = {0, 1};
  std::array<uint32_t, 5> indexed_packet{
      KYTY_PM4(5, Pm4::IT_DRAW_INDEX_INDIRECT, 0), 0,
      Pm4::SPI_SHADER_USER_DATA_GS_0 + 9,
      Pm4::SPI_SHADER_USER_DATA_GS_0 + 10, 0};
  check_offsets(indexed_packet, gs, {0, 0}, "clear stale offsets");
  arguments = {0, 1, 5, static_cast<uint32_t>(-17), 3};
  indexed_packet[2] |= (Pm4::SPI_SHADER_USER_DATA_GS_0 + 11) << 16u;
  // Native sceAgcDcbDrawIndexIndirect (0x5a00) emits first-index enable in word3 bit28.
  indexed_packet[3] |= 1u << 28u;
  check_offsets(indexed_packet, gs, {static_cast<uint32_t>(-17), 3, 5}, "indexed offsets");
  indexed_packet[3] ^= (1u << 28u) | (1u << 27u);
  arguments[2] = 6;
  check_offsets(indexed_packet, gs, {static_cast<uint32_t>(-17), 3, 5},
                "index destination disabled");

  arguments = {0, 1, 31, 5};
  std::array<uint32_t, 5> auto_packet{
      KYTY_PM4(5, Pm4::IT_DRAW_INDIRECT, 0), 0,
      Pm4::SPI_SHADER_USER_DATA_HS_0 + 9,
      Pm4::SPI_SHADER_USER_DATA_HS_0 + 10, 2};
  check_offsets(auto_packet, hs, {31, 5}, "non-indexed offsets");
  Require("Pm4DrawIndirectMulti", "shader stage", gs[9] == static_cast<uint32_t>(-17),
          "non-indexed draw patched the wrong shader stage");

  arguments = {0, 1, 6, 41, 7, 0, 1, 8, 43, 9, 0, 1, 10, 45, 11};
  uint32_t count = 2;
  const auto multi_count_address = reinterpret_cast<uint64_t>(&count);
  // Native sceAgcDcbDrawIndexIndirectMulti (0x5800) puts this enable in word4 bit28.
  std::array<uint32_t, 10> multi_packet{
      KYTY_PM4(10, Pm4::IT_DRAW_INDEX_INDIRECT_MULTI, 0), 0,
      indexed_packet[2], Pm4::SPI_SHADER_USER_DATA_GS_0 + 10,
      (1u << 30u) | (1u << 28u) | Pm4::SH_NOP, 3,
      static_cast<uint32_t>(multi_count_address),
      static_cast<uint32_t>(multi_count_address >> 32u), 20, 0};
  check_offsets(multi_packet, gs, {43, 9, 8}, "counted multi-draw offsets");
  count = 0;
  check_offsets(multi_packet, gs, {43, 9, 8}, "empty multi-draw");
  count = 3;
  multi_packet[5] = 1;
  multi_packet[3] |= 1u << 28u;
  multi_packet[4] &= ~(1u << 28u);
  check_offsets(multi_packet, gs, {41, 7, 8}, "multi-draw enable word and count clamp");

  arguments = {0, 1, 51, 13, 0, 1, 53, 15};
  std::array<uint32_t, 10> auto_multi_packet{
      KYTY_PM4(10, Pm4::IT_DRAW_INDIRECT_MULTI, 0), 0,
      auto_packet[2], auto_packet[3], Pm4::SH_NOP, 2, 0, 0, 16, 2};
  check_offsets(auto_multi_packet, hs, {53, 15}, "non-indexed multi-draw offsets");
  arguments = {0, 1, 99, 101};
  auto_packet[2] = Pm4::SH_NOP;
  check_offsets(auto_packet, hs, {53, 101}, "independent destinations");
  auto_packet[3] = Pm4::SH_NOP;
  arguments[3] = 103;
  check_offsets(auto_packet, hs, {53, 101}, "disabled destinations");
  processor.BufferFlush();
  std::printf("[host]    %-32s ok\n", "Pm4DrawIndirectMulti");
}

void CheckPm4ContextStateOperations(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  constexpr auto primitive_type = static_cast<Prospero::PrimitiveType>(0x35u);

  HW::RenderControl original_control{};
  original_control.depth_clear_enable = true;
  original_control.stencil_clear_enable = true;
  processor.GetCtx().SetRenderControl(original_control);
  processor.GetCtx().SetRenderTargetMask(0x1234abcd);
  processor.GetUcfg().SetPrimitiveType(primitive_type);
  g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_PS](
      processor, Pm4::SPI_SHADER_PGM_LO_PS, 0x00123456);
  const auto shader_address = processor.GetShCtx().GetPs().ps_regs.data_addr;
  const auto invoke = [&](ContextStateOperation operation) {
    std::array<uint32_t, 32> packet{};
    std::array<uint32_t, 6> segment_sizes{};
    CommandBufferLayout dcb{packet.data(),
                            packet.data() + packet.size(),
                            packet.data(),
                            packet.data() + packet.size(),
                            nullptr,
                            nullptr,
                            0};
    const auto operation_value = static_cast<uint32_t>(operation);
    size_t segment_count = 0;
    switch (operation) {
    case ContextStateOperation::Clear:
      segment_sizes = {5};
      segment_count = 1;
      break;
    case ContextStateOperation::Push:
      segment_sizes = {5, 8, 9, 3, 2};
      segment_count = 5;
      break;
    case ContextStateOperation::Pop:
      segment_sizes = {3, 5, 8, 9, 2};
      segment_count = 5;
      break;
    case ContextStateOperation::PushClear:
      segment_sizes = {5, 8, 9, 3, 2, 5};
      segment_count = 6;
      break;
    }
    const auto size_dw = static_cast<size_t>(
        Gen5::AgcDcbContextStateOpGetSize(operation_value) / 4);
    auto *emitted = Gen5::AgcDcbContextStateOp(
        reinterpret_cast<Gen5::CommandBuffer *>(&dcb), operation_value);
    bool packet_matches = true;
    size_t segment_offset = 0;
    for (size_t i = 0; i < segment_count; i++) {
      const auto segment_size = segment_sizes[i];
      const auto selector = (i == 0 ? Pm4::R_CONTEXT_STATE : Pm4::R_ZERO);
      packet_matches &= packet[segment_offset] ==
                        KYTY_PM4(segment_size, Pm4::IT_NOP, selector);
      for (size_t j = 1; j < segment_size; j++) {
        packet_matches &= packet[segment_offset + j] ==
                          (i == 0 && j == 1 ? operation_value : 0u);
      }
      segment_offset += segment_size;
    }
    Require("Pm4ContextState", "HLE packet",
            emitted == packet.data() &&
                dcb.cursor_up == packet.data() + size_dw &&
                segment_offset == size_dw && packet_matches,
            "context-state HLE packet does not match its real allocation size");
    Pm4Execution execution;
    return processor.Process(execution, {packet.data(), size_dw});
  };

  Require("Pm4ContextState", "push-clear",
          invoke(ContextStateOperation::PushClear) ==
                  Pm4ProcessResult::Complete &&
              !processor.GetCtx().GetRenderControl().depth_clear_enable &&
              !processor.GetCtx().GetRenderControl().stencil_clear_enable &&
              processor.GetCtx().GetRenderTargetMask() ==
                  HW::Context{}.GetRenderTargetMask() &&
              processor.GetUcfg().GetPrimType() == primitive_type &&
              processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address,
          "push-clear did not save and clear only Cx state");

  processor.GetCtx().SetRenderTargetMask(0x55aa55aa);
  Require("Pm4ContextState", "pop",
          invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
              processor.GetCtx().GetRenderControl().depth_clear_enable &&
              processor.GetCtx().GetRenderControl().stencil_clear_enable &&
              processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
          "pop did not restore the complete saved Cx state");

  Require("Pm4ContextState", "push",
          invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete &&
              processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
          "push changed live Cx state");
  processor.GetCtx().SetRenderTargetMask(0x01020304);
  Require("Pm4ContextState", "push-pop restore",
          invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
              processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
          "ordinary push/pop did not restore Cx state");

  Require("Pm4ContextState", "clear",
          invoke(ContextStateOperation::Clear) == Pm4ProcessResult::Complete &&
              !processor.GetCtx().GetRenderControl().depth_clear_enable &&
              processor.GetCtx().GetRenderTargetMask() ==
                  HW::Context{}.GetRenderTargetMask() &&
              processor.GetUcfg().GetPrimType() == primitive_type &&
              processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address,
          "clear reset state outside the Cx register domain");

  processor.GetCtx().SetRenderTargetMask(0x0badc0de);
  Require("Pm4ContextState", "push before clear-state packet",
          invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete,
          "push before CLEAR_STATE failed");
  processor.GetCtx().SetRenderTargetMask(0xfeedface);
  std::array<uint32_t, 2> clear_state_packet{0xc0001200, 0};
  Pm4Execution clear_state_execution;
  Require("Pm4ContextState", "clear-state packet",
          processor.Process(clear_state_execution, clear_state_packet) ==
                  Pm4ProcessResult::Complete &&
              processor.GetCtx().GetRenderTargetMask() ==
                  HW::Context{}.GetRenderTargetMask(),
          "CLEAR_STATE did not clear the current Cx state");
  Require("Pm4ContextState", "pop after clear-state packet",
          invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
              processor.GetCtx().GetRenderTargetMask() == 0x0badc0de,
          "CLEAR_STATE discarded the pushed Cx state");

  // CLEAR_STATE restores the viewport, scissor, and guard-band register defaults.
  processor.GetCtx().SetScreenScissor(1, 2, 3, 4);
  processor.GetCtx().SetGuardBands(2.0f, 3.0f, 4.0f, 5.0f);
  Pm4Execution scissor_reset;
  Require("Pm4ContextState", "scissor reset packet",
          processor.Process(scissor_reset, clear_state_packet) == Pm4ProcessResult::Complete,
          "CLEAR_STATE could not reset the rasterization state");
  const auto &viewport = processor.GetCtx().GetScreenViewport();
  const auto defaults = calc_final_scissor(viewport, {}, {20000, 20000}, 0);
  Require("Pm4ContextState", "Rasterization defaults",
          defaults.left == 0 && defaults.top == 0 && defaults.right == 16384 &&
              defaults.bottom == 16384 && viewport.guard_band_horz_clip == 1.0f &&
              viewport.guard_band_vert_clip == 1.0f &&
              viewport.guard_band_horz_discard == 1.0f &&
              viewport.guard_band_vert_discard == 1.0f &&
              std::ranges::all_of(viewport.viewports, [](const auto &slot) {
                return slot.xscale == 1.0f && slot.yscale == 1.0f && slot.zscale == 1.0f &&
                       slot.viewport_scissor_right == 16384 &&
                       slot.viewport_scissor_bottom == 16384 &&
                       !slot.viewport_scissor_window_offset_enable;
              }),
          "CLEAR_STATE did not restore scissor and guard-band defaults");
  const std::array scissor_registers{
      std::pair{Pm4::PA_SC_SCREEN_SCISSOR_TL, Pm4::PA_SC_SCREEN_SCISSOR_BR},
      std::pair{Pm4::PA_SC_WINDOW_SCISSOR_TL, Pm4::PA_SC_WINDOW_SCISSOR_BR},
      std::pair{Pm4::PA_SC_GENERIC_SCISSOR_TL, Pm4::PA_SC_GENERIC_SCISSOR_BR},
      std::pair{Pm4::PA_SC_VPORT_SCISSOR_0_TL, Pm4::PA_SC_VPORT_SCISSOR_0_BR}};
  for (const auto &[tl, br] : scissor_registers) {
    processor.GetCtx().Reset();
    g_hw_ctx_indirect_func[tl](processor, tl, 0);
    g_hw_ctx_indirect_func[br](processor, br, 0);
    HW::ScanModeControl scan{};
    scan.vport_scissor_enable = tl == Pm4::PA_SC_VPORT_SCISSOR_0_TL;
    const auto empty = calc_final_scissor(processor.GetCtx().GetScreenViewport(),
                                         scan, {3840, 2160}, 0);
    Require("Pm4ContextState", "explicit empty scissor",
            empty.right == empty.left && empty.bottom == empty.top,
            "an explicitly empty scissor was expanded or ignored");
  }

  processor.GetCtx().SetRenderTargetMask(0xabcdef01);
  Require("Pm4ContextState", "push before processor reset",
          invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete,
          "push before reset failed");
  processor.Reset();
  processor.GetCtx().SetRenderTargetMask(0x76543210);
  Require("Pm4ContextState", "processor reset discards push",
          invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete &&
              invoke(ContextStateOperation::Pop) ==
                  Pm4ProcessResult::Complete &&
              processor.GetCtx().GetRenderTargetMask() == 0x76543210,
          "processor reset retained an invalid saved Cx state");

  Require("Pm4ContextState", "HLE packet size",
          Gen5::AgcDcbContextStateOpGetSize(0) == 20 &&
              Gen5::AgcDcbContextStateOpGetSize(1) == 108 &&
              Gen5::AgcDcbContextStateOpGetSize(2) == 108 &&
              Gen5::AgcDcbContextStateOpGetSize(3) == 128 &&
              Gen5::AgcDcbContextStateOpGetSize(4) == 0,
          "context-state HLE sizes do not match the expected packet layout");
  std::printf("[host]    %-32s ok\n", "Pm4ContextState");
}

void CheckPm4IndirectControlFlow(RenderContext &renderer) {
  constexpr const char *name = "Pm4IndirectControlFlow";
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  uint32_t selected = 0;
  uint32_t returned = 0;
  const auto address = [](const void *value) {
    return reinterpret_cast<uint64_t>(value);
  };
  const auto write = [&](uint32_t *destination, uint32_t value) {
    return std::array<uint32_t, 5>{
        KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0), 0,
        static_cast<uint32_t>(address(destination)),
        static_cast<uint32_t>(address(destination) >> 32u), value};
  };
  const auto indirect = [&](std::span<const uint32_t> target, bool chain) {
    uint32_t control = 0x0f200000u | static_cast<uint32_t>(target.size());
    if (chain) {
      control |= 1u << 20u;
    }
    return std::array<uint32_t, 4>{
        KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0),
        static_cast<uint32_t>(address(target.data())),
        static_cast<uint32_t>(address(target.data()) >> 32u), control};
  };
  // A one-word NOP must leave the following completion write reachable.
  std::array<uint32_t, 7> nop_commands{};
  CommandBufferLayout nop_dcb{nop_commands.data(), nop_commands.data() + nop_commands.size(),
      nop_commands.data(), nop_commands.data() + nop_commands.size(), nullptr, nullptr, 0};
  auto* nop_cb = reinterpret_cast<Gen5::CommandBuffer*>(&nop_dcb);
  Require(name, "zero-sized NOP", Gen5::AgcCbNop(nop_cb, 0) == nullptr &&
              nop_dcb.cursor_up == nop_commands.data() && nop_commands[0] == 0,
          "zero-sized reservation changed the command buffer");
  Require(name, "empty command list NOP", Gen5::AgcCbNop(nop_cb, 1) == nop_commands.data() &&
              nop_dcb.cursor_up == nop_commands.data() + 1 && nop_commands[0] == 0xffff1000u &&
              Gen5::AgcCbNopGetSize(1) == 4 && Gen5::AgcGetPacketSize(nop_commands.data()) == 1,
          "native NOP(1) did not reserve one DWORD");
  const auto completion = write(&selected, 7);
  std::copy(completion.begin(), completion.end(), nop_commands.begin() + 1);
  nop_commands.back() = 0xffff1000u;
  for (uint32_t predication : {1u, 0u}) {
    Gen5::AgcSetRangePredication(nop_commands.data(), nop_commands.data() + nop_commands.size(), predication);
    Require(name, "NOP range predication", nop_commands[0] == (0xffff1000u | predication) &&
                nop_commands.back() == (0xffff1000u | predication) &&
                nop_commands[1] == (completion[0] | predication) && nop_commands[5] == 7,
            "range predication skipped a packet or modified a payload");
    Pm4Execution execution;
    selected = 0;
    Require(name, "NOP and completion execution",
            processor.Process(execution, nop_commands) == Pm4ProcessResult::Complete && selected == 7,
            "one-DWORD NOP prevented the completion write or overread the stream end");
  }
  Common::File dump;
  Require(name, "NOP dump file", dump.CreateInMem(), "could not create PM4 dump");
  Pm4::DumpPm4PacketStream(&dump, nop_commands.data(), 0, nop_commands.size());
  std::array<uint32_t, 3> payload{0, 0x12345678u, 0x9abcdef0u};
  CommandBufferLayout payload_dcb{payload.data(), payload.data() + payload.size(),
      payload.data(), payload.data() + payload.size(), nullptr, nullptr, 0};
  Require(name, "NOP payload reservation",
          Gen5::AgcCbNop(reinterpret_cast<Gen5::CommandBuffer*>(&payload_dcb), 3) == payload.data() &&
              payload == std::array<uint32_t, 3>{0xc0011000u, 0x12345678u, 0x9abcdef0u},
          "NOP emitter overwrote the caller's reserved payload");
  alignas(8) uint64_t released = 0;
  std::array<uint32_t, 20> disabled_commands{};
  const auto disabled_write = write(&returned, 99);
  std::copy(disabled_write.begin(), disabled_write.end(), disabled_commands.begin());
  disabled_commands[5] = KYTY_PM4(2, Pm4::IT_NUM_INSTANCES, 0);
  disabled_commands[6] = 0x68750123u;
  CommandBufferLayout release_dcb{disabled_commands.data() + 7, disabled_commands.data() + 15,
      disabled_commands.data() + 7, disabled_commands.data() + 15, nullptr, nullptr, 0};
  auto* release = Gen5::AgcCbReleaseMem(
      reinterpret_cast<Gen5::CommandBuffer*>(&release_dcb), 0x28, 0, 0, 0,
      reinterpret_cast<const volatile Gen5::Label*>(&released), 2, 77, 0, 0, 0, 0);
  const auto enabled_write = write(&selected, 88);
  std::copy(enabled_write.begin(), enabled_write.end(), disabled_commands.begin() + 15);
  const auto unpatched_commands = disabled_commands;
  Require(name, "SetNop packets",
          release == disabled_commands.data() + 7 &&
              Gen5::AgcSetNop(disabled_commands.data()) == 0 &&
              Gen5::AgcSetNop(disabled_commands.data() + 5) == 0 && Gen5::AgcSetNop(release) == 0 &&
              disabled_commands[0] == ((disabled_write[0] & 0xffff00ffu) | (Pm4::IT_NOP << 8u)) &&
              disabled_commands[5] == KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO) &&
              disabled_commands[6] == 0x123u &&
              release[0] == KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_ZERO) &&
              unpatched_commands[7] == KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM) &&
              Gen5::AgcGetPacketSize(disabled_commands.data()) == disabled_write.size() &&
              Gen5::AgcGetPacketSize(disabled_commands.data() + 5) == 2 &&
              Gen5::AgcGetPacketSize(release) == 8 &&
              std::equal(disabled_commands.begin() + 1, disabled_commands.begin() + 5,
                         unpatched_commands.begin() + 1) &&
              std::equal(disabled_commands.begin() + 8, disabled_commands.end(),
                         unpatched_commands.begin() + 8),
          "SetNop changed a packet length, kept its operation or marker, or touched its payload");
  selected = returned = 0;
  Pm4Execution disabled_execution;
  Require(name, "SetNop execution",
          processor.Process(disabled_execution, disabled_commands) == Pm4ProcessResult::Complete &&
              returned == 0 && released == 0 && selected == 88,
          "a disabled packet still executed or hid the packet after it");
  const auto then_commands = write(&selected, 11);
  const auto else_commands = write(&selected, 33);
  const auto branch_suffix = write(&selected, 44);
  const auto caller_suffix = write(&returned, 22);
  alignas(8) uint64_t condition = 0;
  std::array<uint32_t, 19> branch{
      KYTY_PM4(14, Pm4::IT_INDIRECT_BUFFER, 0), 0,
      static_cast<uint32_t>(address(&condition)),
      static_cast<uint32_t>(address(&condition) >> 32u), UINT32_MAX,
      UINT32_MAX, 1, 0,
      static_cast<uint32_t>(address(then_commands.data())),
      static_cast<uint32_t>(address(then_commands.data()) >> 32u),
      static_cast<uint32_t>(then_commands.size()),
      static_cast<uint32_t>(address(else_commands.data())),
      static_cast<uint32_t>(address(else_commands.data()) >> 32u),
      static_cast<uint32_t>(else_commands.size())};
  std::copy(branch_suffix.begin(), branch_suffix.end(), branch.begin() + 14);
  std::array<uint32_t, 9> caller{};
  const auto call_branch = indirect(branch, false);
  std::copy(call_branch.begin(), call_branch.end(), caller.begin());
  std::copy(caller_suffix.begin(), caller_suffix.end(), caller.begin() + 4);
  for (uint32_t mode : {1u, 2u}) {
    branch[1] = mode | (3u << 8u);
    for (uint64_t value : {0ull, 1ull}) {
      condition = value;
      selected = returned = 0;
      uint32_t expected = 44;
      if (value == 1) {
        expected = 11;
      } else if (mode == 2) {
        expected = 33;
      }
      Pm4Execution execution;
      Require(name, "conditional fetcher replacement",
              processor.Process(execution, caller) == Pm4ProcessResult::Complete &&
                  selected == expected && returned == 22,
              "taken branch resumed its discarded suffix or lost the caller's return");
    }
  }

  std::array<uint32_t, 5> cx_packet{};
  CommandBufferLayout cx_dcb{cx_packet.data(), cx_packet.data() + cx_packet.size(),
                            cx_packet.data(), cx_packet.data() + cx_packet.size(),
                            nullptr, nullptr, 0};
  const ShaderRegister cx_register{Pm4::CB_TARGET_MASK, 0x76543210u};
  Require(name, "indirect context packet size",
          Gen5::AgcDcbSetCxRegistersIndirectGetSize() == sizeof(cx_packet) &&
              Gen5::AgcDcbSetCxRegistersIndirect(
                  reinterpret_cast<Gen5::CommandBuffer *>(&cx_dcb), &cx_register, 1) ==
                  cx_packet.data() && cx_dcb.cursor_up == cx_dcb.top,
          "indirect Cx size did not match the emitted command buffer");

  std::array<uint32_t, 14> patched_branch{};
  CommandBufferLayout branch_dcb{
      patched_branch.data(), patched_branch.data() + patched_branch.size(),
      patched_branch.data(), patched_branch.data() + patched_branch.size(),
      nullptr, nullptr, 0};
  Require(name, "branch packet size",
          Gen5::AgcCbBranchGetSize() == sizeof(patched_branch) &&
              Gen5::AgcCbBranch(reinterpret_cast<Gen5::CommandBuffer *>(&branch_dcb),
                                1, 0, &condition, 0, 0, 0, nullptr, 0,
                                3, else_commands.data(), else_commands.size()) ==
                  patched_branch.data() && branch_dcb.cursor_up == branch_dcb.top,
          "branch size did not match the emitted command buffer");
  patched_branch[8] = 3;
  patched_branch[10] = 0xc5a00000u;
  const auto unpatched = patched_branch;
  Require(name, "patched indirect context branch",
          Gen5::AgcBranchPatchSetThenTarget(patched_branch.data(), 2,
                                            cx_packet.data(), cx_packet.size()) == 0 &&
              (patched_branch[8] & 3u) == 3u &&
              patched_branch[10] == 0xe5a00005u &&
              std::equal(patched_branch.begin(), patched_branch.begin() + 8,
                         unpatched.begin()) &&
              std::equal(patched_branch.begin() + 11, patched_branch.end(),
                         unpatched.begin() + 11),
          "branch target patch lost reserved fields or changed the condition/else target");
  Pm4Execution patched_execution;
  Require(name, "patched branch execution",
          processor.Process(patched_execution, patched_branch) == Pm4ProcessResult::Complete &&
              processor.GetCtx().GetRenderTargetMask() == cx_register.value,
          "patched branch did not fetch and apply its indirect context registers");
  patched_branch[0] = KYTY_PM4(14, Pm4::IT_NOP, 0);
  const auto mismatched = patched_branch;
  Require(name, "branch packet mismatch",
          Gen5::AgcBranchPatchSetThenTarget(patched_branch.data(), 0,
                                            then_commands.data(), then_commands.size()) ==
                  static_cast<int>(0x8a6c000cu) && patched_branch == mismatched,
          "branch patch accepted or changed a different packet type");

  // This stream exceeds native recursion capacity, while chains need one fetcher cursor.
  std::vector<std::array<uint32_t, 4>> links(65536);
  std::span<const uint32_t> target = then_commands;
  for (auto &link : links) {
    link = indirect(target, true);
    target = link;
  }
  const auto call_chain = indirect(target, false);
  std::copy(call_chain.begin(), call_chain.end(), caller.begin());
  selected = returned = 0;
  Pm4Execution execution;
  Require(name, "long chain and call return",
          processor.Process(execution, caller) == Pm4ProcessResult::Complete &&
              selected == 11 && returned == 22,
          "long chain did not complete and return to the original caller");
  std::printf("[host]    %-32s ok\n", name);
}

void CheckPm4WaitResume(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);

  uint32_t label = 0;
  uint32_t prefix = 0;
  uint32_t child_observation = UINT32_MAX;
  uint32_t suffix = 0;
  const auto address = [](const void *value) {
    return reinterpret_cast<uint64_t>(value);
  };

  std::array<uint32_t, 12> child{};
  child[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
  child[1] = 0;
  child[2] = static_cast<uint32_t>(address(&child_observation));
  child[3] = static_cast<uint32_t>(address(&child_observation) >> 32u);
  child[4] = 0;
  child[5] = KYTY_PM4(7, Pm4::IT_WAIT_REG_MEM, 0);
  child[6] = 0x10u | 3u;
  child[7] = static_cast<uint32_t>(address(&label));
  child[8] = static_cast<uint32_t>(address(&label) >> 32u);
  child[9] = 1;
  child[10] = UINT32_MAX;

  std::array<uint32_t, 4> nested{};
  nested[0] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
  nested[1] = static_cast<uint32_t>(address(child.data()));
  nested[2] = static_cast<uint32_t>(address(child.data()) >> 32u);
  nested[3] = 0x0f300000u | static_cast<uint32_t>(child.size());

  std::array<uint32_t, 14> commands{};
  commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
  commands[1] = 0;
  commands[2] = static_cast<uint32_t>(address(&prefix));
  commands[3] = static_cast<uint32_t>(address(&prefix) >> 32u);
  commands[4] = 11;
  commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
  commands[6] = static_cast<uint32_t>(address(nested.data()));
  commands[7] = static_cast<uint32_t>(address(nested.data()) >> 32u);
  commands[8] = 0x0f200000u | static_cast<uint32_t>(nested.size());
  commands[9] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
  commands[10] = 0;
  commands[11] = static_cast<uint32_t>(address(&suffix));
  commands[12] = static_cast<uint32_t>(address(&suffix) >> 32u);
  commands[13] = 22;

  Pm4Execution execution;
  Require("Pm4WaitResume", "suspend",
          processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
              prefix == 11 && child_observation == 0 && suffix == 0,
          "blocked indirect wait did not preserve its command position");
  Require("Pm4WaitResume", "still blocked",
          processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
              !execution.MadeProgress() && suffix == 0,
          "blocked wait advanced or replayed its caller");

  label = 1;
  child[4] = 1;
  Require(
      "Pm4WaitResume", "resume",
      processor.Process(execution, commands) == Pm4ProcessResult::Complete &&
          prefix == 11 && child_observation == 0 && suffix == 22,
      "resumed indirect wait replayed a child or did not finish its parent");
  std::printf("[host]    %-32s ok\n", "Pm4WaitResume");
}

void CheckPm4RewindResume(RenderContext &renderer) {
  constexpr const char *name = "Pm4RewindResume";
  Loader::SymbolDatabase symbols;
  Libs::InitAgcDriver_1(&symbols);
  const auto *emit_symbol = symbols.FindByNid("DwICrVxerkY", Loader::SymbolType::Func);
  const auto *patch_symbol = symbols.FindByNid("eWaWyFegzgQ", Loader::SymbolType::Func);
  Require(name, "async exports",
          emit_symbol != nullptr && patch_symbol != nullptr &&
              emit_symbol->vaddr == reinterpret_cast<uint64_t>(&Gen5::AgcAcbRewind) &&
              patch_symbol->vaddr ==
                  reinterpret_cast<uint64_t>(&Gen5::AgcRewindPatchSetRewindState),
          "async rewind imports did not resolve to the emitter and state patch");
  const auto async_patch = reinterpret_cast<decltype(&Gen5::AgcRewindPatchSetRewindState)>(
      patch_symbol->vaddr);

  constexpr uint32_t sentinel = 0x12345678u;
  std::array<uint32_t, 4> packet{sentinel, sentinel, sentinel, sentinel};
  CommandBufferLayout acb{packet.data(), packet.data() + packet.size(),
                          packet.data() + 1, packet.data() + 3, nullptr, nullptr, 0};
  auto *buffer = reinterpret_cast<Gen5::CommandBuffer *>(&acb);
  for (const uint8_t state : {0, 1}) {
    for (const uint8_t offload : {0, 1}) {
      acb.cursor_up = packet.data() + 1;
      Require(name, "async packet",
              Gen5::AgcAcbRewind(buffer, state, offload) == packet.data() + 1 &&
                  acb.cursor_up == packet.data() + 3 &&
                  packet[0] == sentinel && packet[3] == sentinel &&
                  packet[1] == 0xc0005900u &&
                  packet[2] == ((uint32_t{state} << 31u) | (uint32_t{offload} << 24u)),
              "async rewind packet or two-DWORD allocation differs from the native command");
    }
  }
  const auto before = packet;
  acb.cursor_up = packet.data() + 1;
  acb.reserved_dw = 1;
  Require(name, "bounded allocation",
          Gen5::AgcAcbRewind(buffer, 1, 1) == nullptr &&
              Gen5::AgcAcbRewind(nullptr, 1, 1) == nullptr &&
              acb.cursor_up == packet.data() + 1 && packet == before,
          "failed rewind allocation changed the cursor or command storage");

  // Native patchers inspect only the opcode, preserving the header and low 31 bits.
  std::array<uint32_t, 2> command{0xa5ff591fu, 0x0155aa55u};
  Require(name, "patch preserves fields",
          async_patch(command.data(), 1) == 0 && command[0] == 0xa5ff591fu &&
              command[1] == 0x8155aa55u && async_patch(command.data(), 0) == 0 &&
              command[0] == 0xa5ff591fu && command[1] == 0x0155aa55u,
          "rewind patch changed the offload bit, header, or unrelated payload bits");
  command[0] = 0xc0001000u;
  const auto invalid = command;
  Require(name, "wrong opcode",
          async_patch(command.data(), 1) == static_cast<int>(0x8a6c000cu) &&
              command == invalid,
          "rewind patch accepted or modified a packet with the wrong opcode");

  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  for (const bool async : {false, true}) {
    for (const uint8_t offload : {0, 1}) {
      if (!async && offload != 0) {
        continue;
      }
      uint32_t suffix = 0;
      const auto address = reinterpret_cast<uint64_t>(&suffix);
      std::array<uint32_t, 7> commands{
          0, 0, KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0), 0,
          static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u), 33};
      CommandBufferLayout cb{commands.data(), commands.data() + 2,
                             commands.data(), commands.data() + 2, nullptr, nullptr, 0};
      auto *command_buffer = reinterpret_cast<Gen5::CommandBuffer *>(&cb);
      const auto *emitted = async ? Gen5::AgcAcbRewind(command_buffer, 0, offload)
                                 : Gen5::AgcDcbRewind(command_buffer, 0);
      const auto patch = async ? async_patch : &Gen5::AgcRewindPatchSetRewindState;
      Pm4Execution execution;
      Require(name, "pending",
              emitted == commands.data() &&
                  processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
                  suffix == 0,
              "pending rewind did not preserve its command position");
      Require(name, "still pending",
              processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
                  !execution.MadeProgress() && suffix == 0,
              "pending rewind advanced before its state was patched");
      Require(name, "patch and resume",
              patch(commands.data(), 1) == 0 &&
                  commands[1] == (0x80000000u | (uint32_t{offload} << 24u)) &&
                  processor.Process(execution, commands) == Pm4ProcessResult::Complete &&
                  suffix == 33,
              "patched rewind lost the offload bit or failed to resume");
    }
  }
  std::printf("[host]    %-32s ok\n", name);
}

void CheckPm4CeCompletion(RenderContext &renderer) {
  GraphicsInitJmpTables();
  CommandProcessor processor(renderer, 0);
  uint32_t suffix = 0;
  const auto address = reinterpret_cast<uint64_t>(&suffix);

  std::array<uint32_t, 9> commands{};
  commands[0] = 0xc0008600u;
  commands[1] = 1;
  commands[2] = 0xc0008500u;
  commands[3] = 0;
  commands[4] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
  commands[5] = 0;
  commands[6] = static_cast<uint32_t>(address);
  commands[7] = static_cast<uint32_t>(address >> 32u);
  commands[8] = 33;

  processor.ResetDeCe();
  Pm4Execution execution;
  Require("Pm4CeCompletion", "wait",
          processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
              suffix == 0,
          "DE did not wait for an active CE stream");

  std::array<uint32_t, 2> ce_increment{0xc0008400u, 1};
  Pm4Execution ce_execution;
  Require("Pm4CeCompletion", "CE increment",
          processor.Process(ce_execution, ce_increment) ==
              Pm4ProcessResult::Complete,
          "CE counter increment did not complete");
  Require(
      "Pm4CeCompletion", "counter gate",
      processor.Process(execution, commands) == Pm4ProcessResult::Complete &&
          suffix == 33,
      "DE did not resume after CE advanced or failed to increment its counter");

  suffix = 0;
  Pm4Execution balanced_execution;
  Require("Pm4CeCompletion", "balanced wait",
          processor.Process(balanced_execution, commands) ==
                  Pm4ProcessResult::Blocked &&
              suffix == 0,
          "balanced CE/DE counters did not gate the next DE packet");
  processor.SetCeComplete(true);
  Require("Pm4CeCompletion", "stream complete",
          processor.Process(balanced_execution, commands) ==
                  Pm4ProcessResult::Complete &&
              suffix == 33,
          "DE remained blocked after the CE stream completed");
  std::printf("[host]    %-32s ok\n", "Pm4CeCompletion");
}

} // namespace
} // namespace Libs::Graphics

int main(int argc, char **argv) {
  using namespace Libs::Graphics;

  std::setvbuf(stdout, nullptr, _IONBF, 0);
  EnsureConfigInitialized();
  CheckLeastRecentlyUsedCacheOrdering();
  if (argc == 2 && std::strcmp(argv[1], "--thread-dimensions-only") == 0) {
    VulkanHarness vulkan;
    CheckComputeThreadDimensions(vulkan);
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--lds-limit-only") == 0) {
    VulkanHarness vulkan;
    CheckComputeLdsLimit(vulkan);
    for (u32 wave_size : {32u, 64u})
      for (bool gds : {false, true})
        RunCase(&vulkan, DsReadWriteFullRegionAddress(wave_size, gds));
    RunCase(&vulkan, DsWideLdsPartialBounds());
    RunCase(&vulkan, DsAtomic64Bounds(false));
    RunCase(&vulkan, DsAtomic64Bounds(true));
    for (auto test : {DsWideLdsPartialBounds(), DsAtomic64Bounds(false), DsAtomic64Bounds(true)}) {
      test.compute_info.lds_storage = true;
      std::erase(test.required_spirv, "WorkgroupMemoryExplicitLayoutKHR");
      test.required_spirv.push_back("StorageBuffer");
      RunCase(&vulkan, test);
    }
    vulkan.CheckNativeIndirectDispatch();
    return 0;
  }
  if (argc == 2 && (std::strcmp(argv[1], "--integer64-compare-only") == 0 ||
                    std::strcmp(argv[1], "--cmp-i64-only") == 0)) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorCompareInteger64Edges());
    RunCase(&vulkan, VectorCompareExecWaveMasks(32));
    RunCase(&vulkan, VectorCompareExecWaveMasks(64));
    RunCase(&vulkan, VectorCompare64WaveMasks(32, true));
    RunCase(&vulkan, VectorCompare64WaveMasks(64, true));
    RunCase(&vulkan, VectorVop3CompareEqI64OnGpu());
    RunCase(&vulkan, VectorVop3CompareEqU64OnGpu());
    RunCase(&vulkan, VectorVop3CompareGtU64OnGpu());
    RunCase(&vulkan, VectorVopcCompareLtU64OnGpu());
    RunCase(&vulkan, VectorVop3CompareNeU64OnGpu());
    RunCase(&vulkan, VectorVopcCmpxNeU64CapturedExecMask());
    RunCase(&vulkan, VectorVop3CmpxNeI64CapturedExecMask());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--new-opcodes-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorCompareInteger64Edges());
    RunCase(&vulkan, VectorCompare64WaveMasks(32, true));
    RunCase(&vulkan, VectorCompare64WaveMasks(64, true));
    RunCase(&vulkan, VectorCompareExecWaveMasks(32));
    RunCase(&vulkan, VectorCompareExecWaveMasks(64));
    RunCase(&vulkan, VectorCompareF64Edges());
    RunCase(&vulkan, VectorF64ModesModifiersAndExec());
    RunCase(&vulkan, VectorCompareF16Ops());
    RunCase(&vulkan, ScalarBrevB64OperandsAndMasks(32));
    RunCase(&vulkan, ScalarBrevB64OperandsAndMasks(64));
    for (bool add : {false, true}) {
      RunCase(&vulkan, DsAtomic64Bounds(add));
      for (u32 wave_size : {32, 64}) {
        RunCase(&vulkan, DsAtomic64Contention(add, wave_size));
      }
    }
    RunCase(&vulkan, BufferAtomicAndX2GlcAndExec());
    RunCase(&vulkan, ImageAtomicSignedMinMax<false>());
    RunCase(&vulkan, ImageAtomicSignedMinMax<true>());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--s-memrealtime-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ScalarMemRealtimeCapturedPlaceholder());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--packed-integer-neg-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop3pIntegerNegationCapturedAndSelectedHalves());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--dpp-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorDpp8Captured(false));
    for (const u32 wave_size : {32u, 64u})
      for (const bool fi : {false, true})
        RunCase(&vulkan, VectorDpp8Captured(true, wave_size, fi));
    RunCase(&vulkan, VectorDppQuadPermuteReverse());
    RunCase(&vulkan, VectorDppRowXmask());
    RunCase(&vulkan, VectorDppRowShare(32));
    RunCase(&vulkan, VectorDppRowShare(64));
    RunCase(&vulkan, VectorDppBankMaskPreservesDestination());
    RunCase(&vulkan, VectorDppBoundsControlZeroPreservesDestination());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--cmpx-o-f32-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorVopcCmpxOrderedCapturedExecMask());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--fp32-compare-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorCompareF32ExposureGuard());
    for (const auto mode : {0xc0u, 0xe0u}) {
      RunCase(&vulkan, VectorCompareF32DenormalModes(mode));
    }
    RunCase(&vulkan, VectorCompareClassF32());
    RunCase(&vulkan, VectorCompareClassF32ImmediateAndDynamicMasks());
    RunCase(&vulkan, VectorCompareF16Ops());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--int16-alu-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorCompareI16U16Ops());
    RunCase(&vulkan, Vop3Int16Ops());
    RunCase(&vulkan, Vop3pInt16Ops());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--scalar-zero-bit-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ScalarZeroBitOps());
    RunCase(&vulkan, ScalarBitfieldPack());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--buffer-format-store-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, BufferStoreFormatXyzwPackedUnormSkinningVectors());
    RunCase(&vulkan, BufferStoreFormatXZeroFillsPackedRecord());
    RunCase(&vulkan, BufferStoreFormatXyzwFloat16ConvertsComponents());
    RunCase(&vulkan, BufferStoreFormatXyzwSnorm16CapturedSkinningVectors());
    RunCase(&vulkan, BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords());
    for (const auto &test : BufferLoadFormatD16XCases()) {
      RunCase(&vulkan, test);
    }
    for (const auto &test : BufferStoreFormatD16XCases()) {
      RunCase(&vulkan, test);
    }
    for (const auto &test : FormattedStoreConversionCases()) {
      RunCase(&vulkan, test);
    }
    RunCase(&vulkan, BufferStoreFormatXResource16UintWritesHalfword());
    RunCase(&vulkan, BufferStoreFormatXResource16UintPreservesAdjacentLanes());
    RunCase(&vulkan, BufferStoreFormatXyResource88UintWritesBytes());
    RunCase(&vulkan, BufferStoreFormatXyzwDropsPartialRecord());
    RunCase(&vulkan, BufferStoreFormatXRejectsPartialRecord());
    RunCase(&vulkan, BufferStoreFormatXyRejectsPartialRecord());
    RunCase(&vulkan, BufferFormatStoreVariants());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--buffer-float16-store-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, BufferStoreFormatXyzwFloat16ConvertsComponents());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--s-ashr-i64-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ScalarAshrI64Edges(false));
    RunCase(&vulkan, ScalarAshrI64Edges(true));
    RunCase(&vulkan, ScalarAshrI64OperandsAndAliases());
    RunCase(&vulkan, ScalarAshrI64Masks(32));
    RunCase(&vulkan, ScalarAshrI64Masks(64));
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--tessellation-only") == 0) {
    CheckTessellationPrograms();
    CheckEmbeddedFetchVertexOffset();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--cmpx-eq-u16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorVopcCmpxEqU16SdwaCompactVop3ExecMask());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--cmpx-ne-u16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorVopcCmpxNeU16SdwaCompactVop3ExecMask());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--cmpx-lt-i16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorVopcCmpxLtI16CapturedSdwaExecMask());
    RunCase(&vulkan, VectorVopcCmpxLtI16WaveMasks(32));
    RunCase(&vulkan, VectorVopcCmpxLtI16WaveMasks(64));
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--cmpx-lt-u16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorVopcCmpxLtU16CapturedSdwaExecMask());
    RunCase(&vulkan, VectorVopcCmpxGtU16CapturedSdwaExecMask());
    RunCase(&vulkan, VectorVopcSdwaCmpxWritesExecMask());
    RunCase(&vulkan, VectorVop3CmpxWritesExecMask());
    RunCase(&vulkan, VectorVopcSdwaCmpxClassF32CapturedExecMask());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--class-f16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorCmpClassF16(32));
    RunCase(&vulkan, VectorCmpClassF16(64));
    RunCase(&vulkan, VectorCmpxClassF16(32));
    RunCase(&vulkan, VectorCmpxClassF16(64));
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--cmpx-class-f16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorCmpxClassF16(32));
    RunCase(&vulkan, VectorCmpxClassF16(64));
    RunCase(&vulkan, VectorCompareClassF32());
    RunCase(&vulkan, VectorVopcSdwaCmpxClassF32CapturedExecMask());
    RunCase(&vulkan, FloatInlineConstantF16UsesNumericValue());
    RunCase(&vulkan, VectorCompareF16Ops());
    RunCase(&vulkan, VectorMinMaxF16Ops());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--cmp-eq-f64-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorCompareF64Edges());
    RunCase(&vulkan, VectorCompare64WaveMasks(32));
    RunCase(&vulkan, VectorCompare64WaveMasks(64));
    RunCase(&vulkan, VectorF64CapturedScreenSpaceShadows());
    RunCase(&vulkan, VectorFractF64CapturedAndEdges());
    RunCase(&vulkan, VectorMinMaxF64CapturedAndEdges());
    RunCase(&vulkan, VectorRoundF64EdgesAndModifiers());
    RunCase(&vulkan, VectorFractF64CapturedChainRuntimeExec());
    RunCase(&vulkan, VectorF64ModesModifiersAndExec());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--cmp-ngt-f16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorVopcCmpNgtF16CapturedSdwaAndEdges());
    RunCase(&vulkan, VectorVopcCmpxNgtF16CapturedSdwaExecMask());
    RunCase(&vulkan, VectorVopcCmpxNleF16CapturedSdwaExecMask());
    RunCase(&vulkan, VectorCompareF16Ops());
    RunCase(&vulkan, Wave32VccMasksPreserveOtherHalf());
    RunCase(&vulkan, VectorVop3FloatCompareNegSourceModifier());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--cmp-nlt-f16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorVopcCmpNltF16CapturedSdwaAndEdges());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--fract-f16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorFractF16CapturedAndEdges());
    RunCase(&vulkan, VectorFractF16Modifiers());
    RunCase(&vulkan, NativeAndSdwa16BitDestinationWrites());
    RunCase(&vulkan, VectorSpecialF16Ops());
    RunCase(&vulkan, VectorCosF16CapturedSdwaAndEdges());
    RunCase(&vulkan, VectorSinF16SdwaAndEdges());
    RunCase(&vulkan, VectorFloatConversionOps());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--scaled-texture-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ImageSamplePacked8<false>());
    RunCase(&vulkan, ImageSamplePacked8<true>());
    RunCase(&vulkan, ImageSamplePacked8<false, true>());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--packed-texture-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckPackedTextureComponents();
    vulkan.CheckBgra16Readback();
    RunCase(&vulkan, ImageLoadPackedUintUnpacksAndSwizzles());
    RunCase(&vulkan, ImageSamplePackedUintConvertsSampleAndGather());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--system-dialog-only") == 0) {
    CheckErrorDialogLifecycle();
    CheckSigninDialogLifecycle();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--fmask-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ImageLoadFmaskUsesNativeSampleMapping());
    RunCase(&vulkan, ImageLoadR32UintUsesIntegerSampledImage());
    return 0;
  }
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
  if (argc == 2 && std::strcmp(argv[1], "--shader-fatal-only") == 0) {
    CheckShaderRecompilerFatalContracts();
    return 0;
  }
#endif
  if (argc == 2 && std::strcmp(argv[1], "--ray-tracing-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, FlatStackApertures(32));
    RunCase(&vulkan, FlatStackApertures(64));
    RunCase(&vulkan, BvhIntersections(true, true, 1));
    RunCase(&vulkan, BvhIntersections(true, true, 0, true));
    for (bool barycentrics : {false, true}) {
      for (bool sorted : {false, true}) {
        RunCase(&vulkan, BvhIntersections(barycentrics, sorted));
      }
    }
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--scalar-sext-only") == 0) {
    VulkanHarness vulkan;
    for (const u32 bits : {8u, 16u}) {
      RunCase(&vulkan, ScalarSextMasks(bits, 32));
      RunCase(&vulkan, ScalarSextMasks(bits, 64));
    }
    RunCase(&vulkan, ScalarSextConstantsAndAliases());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--flat-d16-only") == 0) {
    VulkanHarness vulkan;
    for (const bool high : {false, true}) RunCase(&vulkan, BufferLoadShortD16Captured(high));
    RunCase(&vulkan, FlatStoreSlcCaptured());
    RunCase(&vulkan, FlatLoadDlcCaptured(32));
    RunCase(&vulkan, FlatLoadDlcCaptured(64));
    RunCase(&vulkan, GlobalLoadShortD16Captured(32));
    RunCase(&vulkan, GlobalLoadShortD16Captured(64));
    RunCase(&vulkan, FlatLoadShortD16AddressSegments());
    RunCase(&vulkan, FlatSubdwordLoadsApplyByteOffset());
    RunCase(&vulkan, FlatVirtualAddressRebasesGuestAllocation());
    RunCase(&vulkan, GlobalSignedImmediateRebasesBeforeSaddr());
    RunCase(&vulkan, FlatSegmentIgnoresSaddrAndMasksOffsetMsb());
    RunCase(&vulkan, ScratchIsPrivatePerInvocation());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--indirect-buffer-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ScalarBufferFromLoopReadlane(32));
    RunCase(&vulkan, ScalarBufferFromLoopReadlane(64));
    CheckBoundedFormattedBufferCopy(vulkan);
    CheckIndirectBufferStore(vulkan);
    CheckRuntimeBufferRecords(vulkan);
    RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors());
    RunCase(&vulkan, BufferLoadDwordGpuSelectedDescriptors());
    RunCase(&vulkan, BufferLoadDwordx3GpuSelectedDescriptors());
    RunCase(&vulkan, BufferLoadFormatXGpuSelectedDescriptors());
    RunCase(&vulkan, BufferLoadFormatXGpuSelectedDescriptors64());
    RunCase(&vulkan, BufferLoadFormatXRejectsPartialRecord());
    RunCase(&vulkan, BufferLoadFormatXyRejectsPartialRecord());
    RunCase(&vulkan, BufferLoadDwordx4SnapshotsOverlappingAddress());
    RunCase(&vulkan, BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail());
    RunCase(&vulkan, BufferLoadDwordIdxenUsesDescriptorStride());
    RunCase(&vulkan, BufferStoreFormatXAddTidUsesLaneIndex());
    RunCase(&vulkan, FlatVirtualAddressRebasesGuestAllocation());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--image-sample-offset-only") == 0) {
    VulkanHarness vulkan;
    CheckDynamicSampleOffsets(vulkan, 32u);
    CheckDynamicSampleOffsets(vulkan, 64u);
    RunCase(&vulkan, ImageSampleOffsetsReadFourNeighbors<32>());
    RunCase(&vulkan, ImageSampleOffsetsReadFourNeighbors<64>());
    RunCase(&vulkan, ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu<32>());
    RunCase(&vulkan, ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu<64>());
    RunCase(&vulkan, ImageSampleAndGather());
    RunCase(nullptr, ImageGatherCompareOpcodes());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--image-gather-lod-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ImageGatherExplicitLod());
    RunCase(&vulkan, ImageGatherLodPerLane<0>());
    RunCase(&vulkan, ImageGatherLodPerLane<1>());
    RunCase(&vulkan, ImageGatherLodPerLane<2>());
    RunCase(&vulkan, ImageGatherLodPerLane<3>());
    RunCase(&vulkan, ImageGatherLodPerLane<4>());
    RunCase(&vulkan, ImageGatherLodPerLane<5>());
    RunCase(&vulkan, ImageGatherLodPerLane<6>());
    RunCase(&vulkan, ImageGatherLodPerLane<7>());
    RunCase(&vulkan, ImageGatherLodPerLane<8>());
    RunCase(&vulkan, ImageStoreSingleMipBounds<0>());
    RunCase(&vulkan, ImageStoreSingleMipBounds<1>());
    RunCase(&vulkan, ImageStoreSingleMipBounds<2>());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--wave64-only") == 0) {
    CheckWave64WholeWaveResults();
    VulkanHarness vulkan;
    RunCase(&vulkan, Wave32VccMasksPreserveOtherHalf());
    RunCase(&vulkan, DsPermuteWave64UsesIndependentHalves());
    RunCase(&vulkan, DsBpermuteWave64UsesIndependentHalves());
    RunCase(&vulkan, Wave64CrossHalfLaneAndLds());
    RunCase(&vulkan, Wave64RawMasksAndScalarBranch());
    RunCase(&vulkan, Wave64PartialMultidimensionalWorkgroup());
    RunCase(&vulkan, Wave64AppendConsumeHighHalf());
    RunCase(&vulkan, VectorAddcWritesPerLaneCarryOut());
    RunCase(&vulkan, VectorVop3BCarryOutWritesSgprMask());
    RunCase(&vulkan, VectorVop3BCarryOutUsesEncodedSdst());
    RunCase(&vulkan, DispatcherIrreducibleControlFlow());
    RunCase(&vulkan, ScalarSaveExecOps());
    RunCase(&vulkan, ScalarOrn2SaveexecUsesSourceOrNotExec());
    RunCase(&vulkan, ScalarOrn2SaveexecB32(32, 32));
    RunCase(&vulkan, ScalarOrn2SaveexecB32(64, 64));
    RunCase(&vulkan, ScalarOrn2SaveexecB32(32, 4));
    RunCase(&vulkan, ScalarSubvectorLoops(32));
    RunCase(&vulkan, ScalarSubvectorLoops(64));
    RunCase(&vulkan, ScalarNotB64UpdatesScc());
    RunCase(&vulkan, ScalarSelectB64PreservesMaskProvenance(32));
    RunCase(&vulkan, ScalarSelectB64PreservesMaskProvenance(64));
    RunCase(&vulkan, ScalarConditionalMoveB64());
    RunCase(&vulkan, ScalarConditionalMoveB64PreservesMasks());
    RunCase(&vulkan, ScalarWqmB64SelectsSccDomain());
    RunCase(&vulkan, ScalarWqmB64PreservesPartialMasks());
    RunCase(&vulkan, ScalarWqmB32Masks(32));
    RunCase(&vulkan, ScalarWqmB32Masks(64));
    RunCase(&vulkan, ScalarMaskProvenanceOverlapAndMixedBinary());
    RunCase(&vulkan, ScalarB32MaskDataflow(32));
    RunCase(&vulkan, ScalarB32MaskDataflow(64));
    RunCase(&vulkan, ScratchIsPrivatePerInvocation());
    RunCase(&vulkan, Vop1MoveRelDestination());
    RunCase(&vulkan, BranchVccnzUsesWaveMask());
    RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(32, 32));
    RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(64, 64));
    RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(32, 4));
    RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(64, 4));
    RunCase(&vulkan, BranchVccnzUsesCarryProducedWaveMask());
    RunCase(&vulkan, ImageSampleAndGather());
    RunCase(&vulkan, ImageGatherExplicitLod());
    RunCase(&vulkan, SharedReturnKeepsSelectedValues());
    RunCase(&vulkan, SiblingSharedExitKeepsCapturedConditions());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--position-w-only") == 0) {
    VulkanHarness vulkan;
    RunGraphicsCase(&vulkan, GraphicsPositionWExport());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--centroid-only") == 0) {
    VulkanHarness vulkan;
    RunGraphicsCase(&vulkan, GraphicsPackedHalfCentroid());
    RunGraphicsCase(&vulkan, GraphicsPerspectiveSample(false));
    RunGraphicsCase(&vulkan, GraphicsPerspectiveSample(true));
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--pixel-alias-only") == 0) {
    CheckPixelParameterAliases();
    CheckRectListShaders();
    VulkanHarness vulkan;
    RunGraphicsCase(&vulkan, GraphicsPackedHalfCentroid());
    RunGraphicsCase(&vulkan, GraphicsPackedHalfInputAlias(false));
    RunGraphicsCase(&vulkan, GraphicsPackedHalfInputAlias(true));
    RunGraphicsCase(&vulkan, GraphicsSmoothRawInputAlias());
    RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(false));
    RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(true));
    RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(false, true));
    RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(true, true));
    RunGraphicsCase(&vulkan, GraphicsFlatInterpolatorExport());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--clip-control-only") == 0) {
    CheckClipControlDepthClipState();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--reference-clock-only") == 0) {
    CheckReferenceClockScale();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--scheduler-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckSchedulerTimeline();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--host-image-allocation-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckHostImageAllocation();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--occlusion-dump-only") == 0) {
    VulkanHarness vulkan;
    CheckPm4SyntheticOcclusionCounterDump(vulkan.RuntimeRenderer());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--predication-only") == 0) {
    VulkanHarness vulkan;
    CheckPm4Predication(vulkan.RuntimeRenderer());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--descriptor-heap-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckDescriptorHeapLargeSet();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--push-constant-bank-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckGraphicsPushConstantBank();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--shader-data-storage-only") == 0) {
    CheckResourcePlanHandoff();
    VulkanHarness vulkan;
    RunCase(&vulkan, BufferOffsetsUsePackedWords(false));
    RunCase(&vulkan, BufferOffsetsUsePackedWords(true));
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--buffer-publication-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, BufferWorkgroupPublication(32));
    RunCase(&vulkan, BufferWorkgroupPublication(64));
    RunCase(&vulkan, BufferWorkgroupPublication(32, true));
    RunCase(&vulkan, BufferWorkgroupPublication(32, false, true));
    RunCase(&vulkan, BufferWorkgroupPublication(64, false, true));
    RunCase(&vulkan, BufferAtomicVariants());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--buffer-cmpswap-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, BufferAtomicCmpSwapExactRaw());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--image-cmpswap-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ImageAtomicCompareSwapGlcAndExec());
    RunCase(&vulkan, ImageAtomicSwapReturnsPreviousTexel());
    RunCase(&vulkan, ImageAtomicVariants());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--image-atomic64-only") == 0) {
    VulkanHarness vulkan;
    for (const auto &atomic : ImageAtomicIntegerCases) {
      RunCase(&vulkan, ImageAtomicIntegerGlcAndExec(atomic, false));
      RunCase(&vulkan, ImageAtomicIntegerGlcAndExec(atomic, true));
    }
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--ds-ordered-count-only") == 0) {
    VulkanHarness vulkan;
    for (u32 wave_size : {32, 64}) {
      RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, false));
      RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, true));
    }
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--ds-atomics-only") == 0) {
    VulkanHarness vulkan;
    for (const auto &test : MakeCases()) {
      if (std::string_view(test.name).starts_with("Ds") ||
          std::string_view(test.name).starts_with("BufferAtomic")) {
        RunCase(&vulkan, test);
      }
    }
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckGpuMappedRangeLifecycle();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--stream-buffer-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckStreamBufferRing();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--image-layout-only") == 0) {
    CheckStorageTextureLinearUploadLayout();
    CheckStorageTextureDepthTileUploadLayout();
    CheckStorageTextureVolumeUploadLayout();
    CheckStorageTextureVolumeMipRegions();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--gpu-tiler-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckGpuTilerCpuParity();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckGpuCommandLane();
    vulkan.CheckScalarLeafCall();
    CheckAgcSystemTable(vulkan.RuntimeRenderer());
    CheckPm4IndirectControlFlow(vulkan.RuntimeRenderer());
    CheckPm4WaitResume(vulkan.RuntimeRenderer());
    CheckPm4RewindResume(vulkan.RuntimeRenderer());
    CheckPm4CeCompletion(vulkan.RuntimeRenderer());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--suspend-point-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckGpuSuspendPoint();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--native64-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ScalarDynamic64BitOps());
    RunCase(&vulkan, VectorDynamicU64ShiftEdges());
    RunCase(&vulkan, ScalarAshrI64Edges(false));
    RunCase(&vulkan, ScalarAshrI64Edges(true));
    RunCase(&vulkan, VectorCompareInteger64Edges());
    RunCase(&vulkan, VectorMadU64U32UnsignedCarryOut());
    RunCase(&vulkan, ScalarWqmB64SelectsSccDomain());
    RunCase(&vulkan, ScalarWqmB64PreservesPartialMasks());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--alignbyte-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorAlignByteUsesTwoBitByteOffset());
    RunCase(&vulkan, VectorVop3IntegerOps());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--mad-f32-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorMadF32RoundsProduct());
    RunCase(&vulkan, VectorMadF32FlushesDenorms());
    RunCase(&vulkan, Vop3FmacF32NegatedSourceAccumulates());
    RunCase(&vulkan, VectorFloatArithmeticOps());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--zero-shift-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ScalarZeroShiftWithRuntimeCount());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--sdwa-bitwise-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop2SdwaBitwiseByteDestinations(32));
    RunCase(&vulkan, Vop2SdwaBitwiseByteDestinations(64));
    RunCase(&vulkan, Vop1SdwaNotPartialSourcesAndDestinations());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--sdwa-mov-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop1SdwaMovByteDestinations());
    RunCase(&vulkan, Vop1SdwaMovByteSources(32));
    RunCase(&vulkan, Vop1SdwaMovByteSources(64));
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--sdwa-ffbh-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop1SdwaBfrevSourceSelectors());
    RunCase(&vulkan, Vop1SdwaFfbhCapturedScalarLowWordSource());
    RunCase(&vulkan, VectorFfbhI32NativeAndVop3OnGpu());
    RunCase(&vulkan, Vop1SdwaFfblCapturedHighWordSource());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--cubeid-omod-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, CubeIdCapturedNegationAndOutputScale());
    RunCase(&vulkan, VectorFloatSpecialOps());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--zero-stride-oob-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, TBufferCapturedZeroStrideOob());
    RunCase(&vulkan, TBufferCapturedZeroStrideOob(true));
    RunCase(&vulkan, TBufferCapturedZeroStrideOob(false, true));
    RunCase(&vulkan, BufferZeroStrideOobFormatsAndWidths());
    RunCase(&vulkan, TBufferLoadFormatXIdxenUsesDescriptorStride());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--sdwa-ashr-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop2SdwaAshrrevCapturedWord0SignExtends());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--sdwa-max-i32-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop2SdwaMaxI32CapturedHighWord(32));
    RunCase(&vulkan, Vop2SdwaMaxI32CapturedHighWord(64));
    RunCase(&vulkan, VectorIntegerOps());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--sdwa-addc-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorAddcWritesPerLaneCarryOut());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--sdwa-mul-i24-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop2SdwaMulI24Destinations(32));
    RunCase(&vulkan, Vop2SdwaMulI24Destinations(64));
    RunCase(&vulkan, VectorIntegerOps());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--sdwa-mul-u24-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop2SdwaMulU24Destinations(32));
    RunCase(&vulkan, Vop2SdwaMulU24Destinations(64));
    RunCase(&vulkan, Vop2SdwaMulI24Destinations(32));
    RunCase(&vulkan, Vop2SdwaMulI24Destinations(64));
    RunCase(&vulkan, VectorIntegerOps());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--cvt-pk-i16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop3CvtPkI16I32Captured());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--med3-i16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop3Min3U16CapturedAndSelectors());
    RunCase(&vulkan, Vop3Med3I16Captured());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--mul-lo-u16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop3MulLoU16CapturedAndSelectors());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--mad-i16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop3MadI16CapturedSelectorsAndSaturation());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--waitcnt-depctr-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ScalarWaitcntDepctrCapturedVmVsrc());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--sff1-b64-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, ScalarBitfieldPack());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--context-state-only") == 0) {
    VulkanHarness vulkan;
    CheckShaderFusion();
    CheckPm4NativeTargetGeometryRegisters(vulkan.RuntimeRenderer());
    CheckPm4IndirectShaderRegisters(vulkan.RuntimeRenderer());
    CheckPm4IndirectUconfigRegisters(vulkan.RuntimeRenderer());
    CheckPm4BlendColorRegisterRanges(vulkan.RuntimeRenderer());
    CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer());
    CheckPm4DepthControlHighBits(vulkan.RuntimeRenderer());
    CheckPm4DepthRenderOverride(vulkan.RuntimeRenderer());
    CheckPm4ContextStateOperations(vulkan.RuntimeRenderer());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--pm4-wait-only") == 0) {
    VulkanHarness vulkan;
    CheckPm4WaitPackets(vulkan.RuntimeRenderer());
    CheckPm4WaitResume(vulkan.RuntimeRenderer());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--pm4-draw-multi-only") == 0) {
    VulkanHarness vulkan;
    CheckPm4DrawIndirectMultiPacket(vulkan.RuntimeRenderer());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--rewind-only") == 0) {
    VulkanHarness vulkan;
    CheckPm4RewindResume(vulkan.RuntimeRenderer());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--image-overlap-only") == 0) {
    CheckDepthAttachmentWrites();
    CheckDepthFeedbackAspects();
    CheckDynamicRenderingState();
    VulkanHarness vulkan;
    vulkan.CheckRasterization(false);
    vulkan.CheckRenderExecutorColorDiscovery();
    vulkan.CheckRenderExecutorColor1DArrayDiscovery();
    vulkan.CheckRenderExecutorColorVolumeDiscovery();
    vulkan.CheckRenderExecutorColorMetadataClear();
    vulkan.CheckSampledDccClear();
    vulkan.CheckRenderExecutorColorDepthTileDiscovery();
    vulkan.CheckRenderExecutorStencilBindingDiscovery();
    vulkan.CheckUnifiedTextureCacheFlow();
    vulkan.CheckBgra16Readback();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--draw-offset-only") == 0) {
    CheckEmbeddedFetchVertexOffset();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--depth-feedback-only") == 0) {
    CheckDepthAttachmentWrites();
    CheckDepthFeedbackAspects();
    CheckDynamicRenderingState();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--polygon-mode-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckRasterization(false);
    vulkan.CheckRasterization(false, Prospero::BufferFormat::k11_11_10Float);
    vulkan.CheckRasterization(false, Prospero::BufferFormat::k10_10_10_2UScaled);
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--htile-clear-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckUnifiedTextureCacheFlow();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--dcc-clear-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckRenderExecutorColorMetadataClear();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--compute-meta-clear-only") == 0) {
    CheckDynamicRenderingState();
    VulkanHarness vulkan;
    vulkan.CheckComputeMetaClearClassification();
    vulkan.CheckNativeIndirectDispatch();
    vulkan.CheckRenderExecutorColorVolumeDiscovery();
    vulkan.CheckRenderExecutorColorMetadataClear();
    vulkan.CheckSampledDccClear();
    vulkan.CheckRenderExecutorStencilBindingDiscovery();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--native-indirect-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckNativeIndirectDispatch();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--layered-image-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckUnifiedImageViewCache();
    vulkan.CheckCubeFaceStorageExpansion();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--image-view-cache-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckUnifiedImageViewCache();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--storage-sampled-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckUnifiedImageViewCache();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--depth-readback-only") == 0) {
    CheckDepthTargetFootprints();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--bda-page-table-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckBdaPageTableUploads();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckUnifiedTextureCacheFlow();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-gc-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckBufferCacheDirtyGarbageCollection();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--sampler-border-only") == 0) {
    CheckImageSamplerSpecialization();
    VulkanHarness vulkan;
    vulkan.CheckSamplerBorderColors();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--sampled-depth-resource-only") == 0) {
    VulkanHarness vulkan;
    CheckSampledDepthResource();
    CheckDepthTextureEncoding();
    vulkan.CheckComparisonDepthTexture();
    vulkan.CheckRasterization(true);
    RunCase(nullptr, ImageSampleA16CompareBiasRdna2AddressOrder());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--indirect-image-only") == 0) {
    CheckImageSamplerSpecialization();
    VulkanHarness vulkan;
    CheckCountedRowImageSelector(vulkan);
    CheckIndirectImageKeySwitch(vulkan);
    CheckFiniteInlineSamplerPhi(vulkan);
    RunCase(&vulkan, ImageCubeGradientsPreserveDerivatives());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--readlane-key-guard-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, VectorReadlaneSelectsTwoKeysWithinWave());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--rt1d-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckRenderExecutorColorDiscovery();
    vulkan.CheckRenderExecutorColor1DArrayDiscovery();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--image-transition-only") == 0) {
    VulkanHarness vulkan;
    CheckImageTransitionState(vulkan.RuntimeRenderer());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--image-rebind-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckRenderExecutorStencilBindingDiscovery();
    return 0;
  }
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
  if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-death") == 0) {
    RunReverseRenderTargetDeathCase();
  }
  if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-only") == 0) {
    CheckRenderTargetFormatContract();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--standard64-rt-only") == 0) {
    CheckStandard64RenderTargetTileRoundTrip();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--standard-tile-rt-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckRenderExecutorColorStandardTileDiscovery();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--image-view-only") == 0) {
    VulkanHarness vulkan;
    CheckSampledColorViews();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--storage-bgra-only") == 0) {
    CheckSampledColorViews();
    CheckBasicStorageTextureDescriptor();
    VulkanHarness vulkan;
    vulkan.CheckUnifiedImageViewCache();
    RunCase(&vulkan, ImageStoreBgraUsesInverseSwizzle());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--storage-mip-host-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckRenderExecutorStencilBindingDiscovery();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--storage-mip-only") == 0) {
    RunCase(nullptr, ImageStoreMipSelectsPpsa01340Descriptor());
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--storage-yzwx-only") == 0) {
    CheckSampledColorViews();
    CheckBasicStorageTextureDescriptor();
    CheckStorageTextureGpuOwnedRebindState();
    VulkanHarness vulkan;
    RunCase(&vulkan, ImageStoreYzwxUsesInverseSwizzle());
    return 0;
  }
  if (argc == 3 && std::strcmp(argv[1], "--image-view-death") == 0) {
    RunImageViewDeathCase(argv[2]);
  }
  if (argc == 3 &&
      std::strcmp(argv[1], "--storage-texture-descriptor-death") == 0) {
    RunStorageTextureDescriptorDeathCase(argv[2]);
  }
  if (argc != 1) {
    std::fprintf(stderr, "unknown test selector: %s\n", argv[1]);
    return 2;
  }
  VulkanHarness vulkan;
  // Keep compute and host checks running when rasterization features are missing.
  const bool rasterization = vulkan.RasterizationSupported();
  bool skipped_device_checks = false;
  CheckRenderTargetFormatContract();
  CheckSampledColorViews();
  CheckImageTransitionState(vulkan.RuntimeRenderer());
  CheckSampledDepthResource();
  CheckDepthTextureEncoding();
  vulkan.CheckSamplerBorderColors();
  if (rasterization) {
    vulkan.CheckComparisonDepthTexture();
    vulkan.CheckRasterization(true);
  } else {
    skipped_device_checks = true;
  }
  CheckBasicStorageTextureDescriptor();
  CheckStorageTextureLinearUploadLayout();
  CheckStorageTextureDepthTileUploadLayout();
  CheckStandard64RenderTargetTileRoundTrip();
  CheckStorageTextureVolumeUploadLayout();
  CheckStorageTextureVolumeMipRegions();
  CheckStorageTextureGpuOwnedRebindState();
  CheckNativeMsaaState();
  CheckPs5DepthRegisterDecoding();
  CheckDepthHtileStencilCompatibility();
  CheckDepthAttachmentWrites();
  CheckDepthFeedbackAspects();
  CheckDynamicRenderingState();
  CheckDepthTargetFootprints();
  CheckSlotVectorLifetime();
#else
  if (argc != 1) {
    std::fprintf(stderr, "unknown test selector: %s\n", argv[1]);
    return 2;
  }
  CheckShaderRecompilerFatalContracts();
  CheckDepthFeedbackAspects();
  VulkanHarness vulkan;
  const bool rasterization = vulkan.RasterizationSupported();
  bool skipped_device_checks = false;
#endif
  CheckImageSamplerSpecialization();
  CheckResourcePlanHandoff();
  CheckNativeImageDescriptorTypes();
  CheckClipControlDepthClipState();
  CheckReferenceClockScale();
  CheckErrorDialogLifecycle();
  CheckSigninDialogLifecycle();
  CheckVulkan13FeatureRequirements();
  CheckPm4AcquireMemNoOp(vulkan.RuntimeRenderer());
  CheckPm4SyntheticOcclusionCounterDump(vulkan.RuntimeRenderer());
  CheckPm4Predication(vulkan.RuntimeRenderer());
  CheckPm4StencilInfoValueLane(vulkan.RuntimeRenderer());
  CheckPm4NativeTargetGeometryRegisters(vulkan.RuntimeRenderer());
  CheckPm4IndirectShaderRegisters(vulkan.RuntimeRenderer());
  CheckPm4IndirectUconfigRegisters(vulkan.RuntimeRenderer());
  CheckPm4DirectShaderRegisterFallback(vulkan.RuntimeRenderer());
  CheckPm4GuardBandRegisterRanges(vulkan.RuntimeRenderer());
  CheckPm4BlendColorRegisterRanges(vulkan.RuntimeRenderer());
  CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer());
  CheckPm4DepthControlHighBits(vulkan.RuntimeRenderer());
  CheckPm4DepthRenderOverride(vulkan.RuntimeRenderer());
  CheckShaderFusion();
  CheckAgcSystemTable(vulkan.RuntimeRenderer());
  CheckPm4WaitPackets(vulkan.RuntimeRenderer());
  CheckPm4DrawIndirectMultiPacket(vulkan.RuntimeRenderer());
  CheckPm4ContextStateOperations(vulkan.RuntimeRenderer());
  CheckPm4IndirectControlFlow(vulkan.RuntimeRenderer());
  CheckPm4WaitResume(vulkan.RuntimeRenderer());
  CheckPm4RewindResume(vulkan.RuntimeRenderer());
  CheckPm4CeCompletion(vulkan.RuntimeRenderer());
  CheckEmbeddedFetchVertexOffset();
  CheckEmbeddedFetchLaneSpill();
  CheckTessellationPrograms();
  CheckPixelParameterAliases();
  CheckRectListShaders();
  CheckBoundedFormattedBufferCopy(vulkan);
  CheckIndirectBufferStore(vulkan);
  CheckRuntimeBufferRecords(vulkan);
  CheckComputeThreadDimensions(vulkan);
  CheckCountedRowImageSelector(vulkan);
  CheckIndirectImageKeySwitch(vulkan);
  CheckDynamicSampleOffsets(vulkan, 32u);
  CheckDynamicSampleOffsets(vulkan, 64u);
  CheckFiniteInlineSamplerPhi(vulkan);
  CheckWave64WholeWaveResults();
  CheckPs5GameExampleImageClearRuntimeShape();
  vulkan.CheckSchedulerTimeline();
  vulkan.CheckHostImageAllocation();
  vulkan.CheckDescriptorHeapLargeSet();
  vulkan.CheckGpuMappedRangeLifecycle();
  vulkan.CheckStreamBufferRing();
  vulkan.CheckGpuTilerCpuParity();
  vulkan.CheckNativeIndirectDispatch();
  CheckComputeLdsLimit(vulkan);
  vulkan.CheckBdaPageTableUploads();
  vulkan.CheckUnifiedTextureCacheFlow();
  vulkan.CheckUnifiedImageViewCache();
  vulkan.CheckPackedTextureComponents();
  vulkan.CheckCubeFaceStorageExpansion();
  if (rasterization) {
    vulkan.CheckGraphicsPushConstantBank();
    vulkan.CheckRenderExecutorColorDiscovery();
    vulkan.CheckRenderExecutorColor1DArrayDiscovery();
    vulkan.CheckRenderExecutorColorVolumeDiscovery();
    vulkan.CheckRenderExecutorColorMetadataClear();
    vulkan.CheckSampledDccClear();
    vulkan.CheckRenderExecutorColorStandardTileDiscovery();
    vulkan.CheckRenderExecutorColorDepthTileDiscovery();
    vulkan.CheckRenderExecutorStencilBindingDiscovery();
    vulkan.CheckBgra16Readback();
    vulkan.CheckRasterization(false);
    vulkan.CheckRasterization(false, Prospero::BufferFormat::k11_11_10Float);
    vulkan.CheckRasterization(false, Prospero::BufferFormat::k10_10_10_2UScaled);
    vulkan.CheckBufferCacheDirtyGarbageCollection();
  } else {
    skipped_device_checks = true;
  }
  const auto tests = MakeCases();
  const auto graphics_tests = MakeGraphicsCases();
  CheckOpcodeCoverage(tests, graphics_tests);
  for (const auto &test : tests) {
    RunCase(&vulkan, test);
  }
  for (const auto &test : graphics_tests) {
    RunGraphicsCase(&vulkan, test);
  }
  vulkan.CheckGpuSuspendPoint();
  vulkan.CheckGpuCommandLane();
  vulkan.CheckScalarLeafCall();
  if (skipped_device_checks) {
    std::printf(
        "ShaderRecompilerComputeTests: device rasterization checks skipped, this "
        "device does not support the production rasterization features\n");
  }
  if (vulkan.SkippedCaseCount() > 0) {
    std::printf("ShaderRecompilerComputeTests: %u graphics cases skipped\n",
                vulkan.SkippedCaseCount());
  }
  std::printf("ShaderRecompilerComputeTests: all cases passed\n");
  return 0;
}