shifathosn
/
KytyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
KytyPS5/tests/ShaderRecompilerComputeTest...

42989 lines
1.9 MiB
Raw Blame History

#include "common/assert.h"
#include "common/emulatorConfig.h"
#include "common/hostException.h"
#include "common/logging/log.h"
#include "common/subsystems.h"
#include "common/threads.h"
#include "gpu_test_shaders/gpu_test_ms_depth_spv.h"
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/command_processor/pm4Dispatch.h"
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/guest_gpu/gpu_format.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/guest_gpu/hardwareContext.h"
#include "graphics/guest_gpu/pm4.h"
#include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/pageManager.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "graphics/host_gpu/renderer/colorRenderTarget.h"
#include "graphics/host_gpu/renderer/depthRenderTarget.h"
#include "graphics/host_gpu/renderer/image/blitHelper.h"
#include "graphics/host_gpu/renderer/image/image.h"
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "graphics/host_gpu/renderer/image/textureCommon.h"
#include "graphics/host_gpu/renderer/image/tiler.h"
#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderDraw.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
#include "graphics/host_gpu/renderer/sync.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include "graphics/presentation/window/windowInternal.h"
#include "graphics/shader/recompiler/ShaderRecompiler.h"
#include "graphics/shader/recompiler/Tessellation.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
#include "graphics/shader/rectListShader.h"
#include "graphics/shader/shader.h"
#include "graphics/shader/shaderCompiler.h"
#include "kernel/eventQueue.h"
#include "kernel/memory.h"
#include "libs/agc.h"
#include "libs/dialog.h"
#include "libs/errno.h"
#include "loader/symbolDatabase.h"
#include "spirv-tools/libspirv.hpp"
#if __has_include("graphics/host_gpu/renderer/renderTargetBarriers.h")
#error "legacy render-target barrier API must remain deleted"
#endif
#if __has_include("graphics/host_gpu/transfer.h")
#error "legacy transfer facade must remain deleted"
#endif
#if __has_include("graphics/host_gpu/renderer/framebufferCache.h")
#error "classic framebuffer/render-pass cache must remain deleted"
#endif
#include <algorithm>
#include <array>
#include <atomic>
#include <bit>
#include <chrono>
#include <cinttypes>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <initializer_list>
#include <limits>
#include <memory>
#include <mutex>
#include <numeric>
#include <optional>
#include <semaphore>
#include <set>
#include <span>
#include <sstream>
#include <string>
#include <string_view>
#include <vector>
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
#include <cerrno>
#include <sys/wait.h>
#include <unistd.h>
#endif
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#undef min
#undef max
#endif
namespace Libs {
void InitAgcDriver_1(Loader::SymbolDatabase *symbols);
}
namespace Libs::Graphics {
template <typename Cache>
concept HasGetDownloadBuffer =
requires(Cache &cache) { cache.GetDownloadBuffer(uint64_t{1}); };
static_assert(!HasGetDownloadBuffer<BufferCache>);
static_assert(std::same_as<decltype(&BufferCache::GetBuffer),
Buffer &(BufferCache::*)(BufferId)>);
static_assert(std::same_as<decltype(&TextureCache::GetImage),
Image &(TextureCache::*)(ImageId)>);
template <typename Cache>
concept HasSynchronizeImageToBuffer = requires(Cache &cache) {
cache.SynchronizeImageToBuffer(uint64_t{1}, uint64_t{1});
};
template <typename Cache>
concept HasObtainBufferForImageCopy = requires(Cache &cache) {
cache.ObtainBufferForImageCopy(uint64_t{1}, uint64_t{1});
};
template <typename Cache>
concept HasObtainBufferForImageWrite = requires(Cache &cache) {
cache.ObtainBufferForImageWrite(uint64_t{1}, uint64_t{1});
};
template <typename Cache>
concept HasDiscardGpuDirtyBytes = requires(Cache &cache) {
cache.DiscardGpuDirtyBytes(uint64_t{1}, uint64_t{1});
};
static_assert(!HasSynchronizeImageToBuffer<TextureCache>);
static_assert(!HasObtainBufferForImageCopy<BufferCache>);
static_assert(!HasObtainBufferForImageWrite<BufferCache>);
static_assert(!HasDiscardGpuDirtyBytes<BufferCache>);
template <typename Backing>
concept HasLegacyImageLayout = requires(Backing &backing) { backing.layout; };
static_assert(!HasLegacyImageLayout<VulkanImage>);
template <typename Manager>
concept HasPolicyDetileImage = requires { &Manager::DetileImage; };
template <typename Manager>
concept HasPolicyTileImageBacking = requires { &Manager::TileImageBacking; };
static_assert(!HasPolicyDetileImage<TileManager>);
static_assert(!HasPolicyTileImageBacking<TileManager>);
static_assert(BlitHelper::ColorToMsDepthLayout ==
vk::ImageLayout::eDepthStencilAttachmentOptimal);
struct BufferCacheTestAccess {
static_assert(std::same_as<decltype(BufferCache::m_slot_buffers),
Common::SlotVector<Buffer>>);
static void SetGarbageCollectionThresholds(BufferCache &cache,
uint64_t trigger,
uint64_t critical) {
cache.m_trigger_gc_memory = trigger;
cache.m_critical_gc_memory = critical;
}
static StreamBuffer &DownloadBuffer(BufferCache &cache) {
return cache.m_download_buffer;
}
static void SetUnusedStagingBufferSize(BufferCache &cache, uint64_t size) {
// Called immediately after cache construction, before any staging reservation.
std::destroy_at(&cache.m_staging_buffer);
std::construct_at(&cache.m_staging_buffer, cache.m_graphics, cache.m_scheduler,
MemoryUsage::Upload, size);
}
static bool SynchronizeBufferFromImage(BufferCache &cache, Buffer &buffer,
uint64_t address, uint64_t size) {
return cache.SynchronizeBufferFromImage(buffer, address, size);
}
static BufferId PageOwner(const BufferCache &cache, uint64_t address) {
const auto *owner = cache.m_page_table.Find(
address >> BufferCache::PageTable::kPageBits);
return owner == nullptr ? BufferId{} : *owner;
}
static bool IsBufferAllocated(const BufferCache &cache, BufferId id) {
return cache.m_slot_buffers.is_allocated(id);
}
};
struct StreamBufferTestAccess {
static bool NormalizeReservation(bool coherent, uint64_t atom, uint64_t &size,
uint64_t &alignment) {
return StreamBuffer::NormalizeReservation(coherent, atom, size, alignment);
}
};
struct ImageTestAccess {
static uint32_t CopyRows(uint64_t row_size, uint32_t rows,
uint64_t capacity) {
return Image::CopyRows(row_size, rows, capacity);
}
};
struct TileManagerTestAccess {
static uint32_t ConversionRows(uint64_t offset, uint64_t row_stride,
uint64_t active, uint32_t remaining,
uint64_t alignment, uint64_t max_range,
uint32_t max_groups) {
return TileManager::ConversionRows(offset, row_stride, active, remaining,
alignment, max_range, max_groups);
}
};
struct TextureCacheTestAccess {
static_assert(std::same_as<decltype(TextureCache::m_slot_images),
Common::SlotVector<Image>>);
static_assert(TextureCache::ImagePageTable::kPageBits == 20);
static_assert(TextureCache::ImagePageTable::kAddressSpaceBits == 44);
static_assert(TextureCache::ImagePageTable::kFirstLevelBits == 14);
static std::unique_lock<TrackingSpinLock> Lock(TextureCache &cache) {
return std::unique_lock(cache.m_lock);
}
static void ClearImage(TextureCache &cache, CommandBuffer &command, ImageId id,
const vk::ImageSubresourceRange &range,
const vk::ClearValue &clear) {
auto lock = Lock(cache);
cache.ClearImage(command, id, cache.GetImage(id).backing.format, range, clear);
}
static void ConfigureGarbageCollection(TextureCache &cache,
std::span<const ImageId> oldest,
uint64_t tick, uint64_t pressure) {
cache.m_trigger_gc_memory = 0;
cache.m_pressure_gc_memory = pressure;
cache.m_critical_gc_memory = UINT64_MAX;
cache.m_gc_tick = tick;
std::vector<ImageId> live;
cache.m_lru_cache = {};
cache.m_slot_images.ForEach([&](ImageId id, Image &image) {
if (image.registered) {
image.tick_accessed_last = cache.m_scheduler.CurrentTick();
live.push_back(id);
}
});
for (const auto id : oldest) {
const auto owner = cache.m_slot_images.try_get(id);
if (owner != nullptr && owner->registered) {
owner->tick_accessed_last = 0;
owner->lru_id = cache.m_lru_cache.Insert(id, 0);
}
}
for (const auto id : live) {
if (std::ranges::find(oldest, id) == oldest.end()) {
cache.m_slot_images[id].lru_id = cache.m_lru_cache.Insert(id, tick);
}
}
}
static bool Contains(const TextureCache &cache, ImageId id) {
const auto owner = cache.m_slot_images.try_get(id);
return owner != nullptr && owner->registered;
}
static std::vector<ImageId> FindImages(TextureCache &cache, uint64_t address,
uint64_t size, bool page_overlap) {
std::lock_guard lock(cache.m_lock);
const auto found = cache.FindImagesInRegion(address, size, page_overlap);
std::vector<ImageId> result;
result.reserve(found.size());
for (const auto id : found) {
result.push_back(id);
}
return result;
}
static size_t PageOwnerCount(TextureCache &cache, uint64_t address) {
std::lock_guard lock(cache.m_lock);
const auto *owners = cache.m_image_page_table.Find(static_cast<size_t>(
address >> TextureCache::ImagePageTable::kPageBits));
return owners == nullptr ? 0 : owners->size();
}
static size_t OwnedPageCount(TextureCache &cache, uint64_t address,
uint64_t size, ImageId id) {
std::lock_guard lock(cache.m_lock);
TextureCache::ImagePageTable::PageRange pages{};
if (!TextureCache::ImagePageTable::TryGetPageRange(address, size, pages)) {
return 0;
}
size_t count = 0;
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
const auto *owners = cache.m_image_page_table.Find(page);
count += owners != nullptr && owners->Contains(id) ? 1 : 0;
}
return count;
}
static void AddPageOwner(TextureCache &cache, uint64_t address, ImageId id) {
std::lock_guard lock(cache.m_lock);
cache
.m_image_page_table[static_cast<size_t>(
address >> TextureCache::ImagePageTable::kPageBits)]
.push_back(id);
}
static bool RemovePageOwner(TextureCache &cache, uint64_t address,
ImageId id) {
std::lock_guard lock(cache.m_lock);
auto *owners = cache.m_image_page_table.Find(static_cast<size_t>(
address >> TextureCache::ImagePageTable::kPageBits));
return owners != nullptr && owners->Erase(id);
}
static void SetQueryEpoch(TextureCache &cache, uint32_t epoch) {
std::lock_guard lock(cache.m_lock);
cache.m_image_query_epoch = epoch;
}
static uint32_t QueryEpoch(TextureCache &cache) {
std::lock_guard lock(cache.m_lock);
return cache.m_image_query_epoch;
}
static ImageId InsertImage(TextureCache &cache, const ImageInfo &info) {
std::lock_guard lock(cache.m_lock);
return cache.InsertImage(info);
}
static void DeleteImage(TextureCache &cache, ImageId id) {
std::lock_guard lock(cache.m_lock);
cache.DeleteImage(id);
}
static const Image *Owner(const TextureCache &cache, ImageId id) {
return cache.m_slot_images.try_get(id);
}
static bool PendingDownload(const TextureCache &cache, ImageId id) {
return cache.m_download_images.contains(id);
}
static size_t NullImageCount(const TextureCache &cache) {
return cache.m_null_images.size();
}
static void TrackDownload(TextureCache &cache, ImageId id) {
std::lock_guard lock(cache.m_lock);
cache.TrackImageDownload(id, cache.m_slot_images[id]);
}
static void AssociateStencil(TextureCache &cache, ImageId depth,
GuestRange stencil) {
std::lock_guard lock(cache.m_lock);
(void)cache.AssociateStencil(depth, stencil);
}
static void SetLinearReadback(TextureCache &cache, bool enabled) {
cache.m_readback_linear_images = enabled;
}
static std::pair<uint8_t *, uint64_t>
MapDownload(TextureCache &cache, uint64_t size, uint64_t alignment) {
auto &download =
cache.m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download);
auto mapping = download.Map(size, std::max<uint64_t>(alignment, 4));
if (mapping.first != nullptr) {
download.Commit();
}
return mapping;
}
static bool TryDownload(TextureCache &cache, ImageId id) {
return cache.DownloadImageMemory(id);
}
static void RegisterHtileMeta(TextureCache &cache, uint64_t address) {
std::lock_guard lock(cache.m_lock);
auto &metadata = cache.m_surface_metas[address];
metadata.type = TextureCache::MetaDataInfo::Type::HTile;
metadata.clear_mask = 0;
}
static TileManager &Tiler(TextureCache &cache) { return cache.m_tiler; }
};
struct RenderExecutorTestAccess {
static void DrawIndex(RenderExecutor &executor, CommandBuffer &command,
const DrawIndexArgs &args) {
executor.DrawIndex(0, command, args);
}
static void DrawAuto(RenderExecutor &executor, CommandBuffer &command,
const DrawAutoArgs &args) {
executor.DrawAuto(0, command, args);
}
static bool TryConsumeComputeImageClear(RenderExecutor &executor,
const ShaderComputeInputInfo &input, CommandBuffer &command,
uint32_t x, uint32_t y, uint32_t z, uint32_t mode) {
return executor.TryConsumeComputeImageClear(input, command, x, y, z, mode);
}
static bool TryConsumeComputeMetaClear(RenderExecutor &executor,
const ShaderComputeInputInfo &input,
const CommandBuffer &buffer) {
return executor.TryConsumeComputeMetaClear(input, buffer);
}
static TextureBinding
ResolveTexture(RenderExecutor &executor,
const ShaderRecompiler::IR::ImageResource &resource,
const ShaderRecompiler::IR::DescriptorValue &value) {
return executor.ResolveTexture(resource, value);
}
static auto PrepareGraphicsBindings(RenderExecutor &executor,
const ShaderStageRuntime &vertex,
const ShaderStageRuntime &pixel,
bool pixel_active) {
RenderExecutor::GraphicsBindings result;
executor.PrepareBindings(vertex, result.vertex[0]);
std::array<PreparedBindings *, 2> stages{&result.vertex[0], nullptr};
if (pixel_active) {
result.pixel.emplace();
executor.PrepareBindings(pixel, *result.pixel);
stages[1] = &*result.pixel;
}
executor.PrepareGraphicsBindings(
std::span{stages.data(), pixel_active ? 2u : 1u}, {});
return result;
}
static void PrepareGraphicsBindings(RenderExecutor &executor,
std::span<PreparedBindings *const> stages,
std::span<RenderColorInfo> colors) {
executor.PrepareGraphicsBindings(stages, colors);
}
static PipelineCache::Pipeline
CreateDescriptorPipeline(RenderExecutor &executor,
std::span<PreparedBindings *const> stages) {
std::vector<vk::DescriptorSetLayoutBinding> layout_bindings;
bool compute = false;
for (const auto *prepared : stages) {
const auto &program = *prepared->runtime->program;
compute |= program.stage == ShaderType::Compute;
const auto shader_stage = program.stage == ShaderType::Vertex
? vk::ShaderStageFlagBits::eVertex
: program.stage == ShaderType::Pixel
? vk::ShaderStageFlagBits::eFragment
: vk::ShaderStageFlagBits::eCompute;
for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) {
const auto index = std::countr_zero(mask);
const auto kind = static_cast<ShaderRecompiler::IR::DescriptorBindingKind>(index);
layout_bindings.push_back(
{ShaderRecompiler::IR::NativeBinding(program.stage, kind),
NativeDescriptorType(kind), program.bindings.descriptor_counts[index], shader_stage});
}
}
vk::DescriptorSetLayoutCreateInfo descriptor_info{};
descriptor_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
descriptor_info.flags =
vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR;
descriptor_info.bindingCount =
static_cast<uint32_t>(layout_bindings.size());
descriptor_info.pBindings = layout_bindings.data();
PipelineCache::Pipeline pipeline{};
auto &device = executor.m_context.GetGraphics().device;
EXIT_IF(device.createDescriptorSetLayout(&descriptor_info, nullptr,
&pipeline.descriptor_set_layout) !=
vk::Result::eSuccess);
vk::PipelineLayoutCreateInfo pipeline_info{};
pipeline_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
pipeline_info.setLayoutCount = 1;
pipeline_info.pSetLayouts = &pipeline.descriptor_set_layout;
const vk::PushConstantRange push_range {
compute ? vk::ShaderStageFlags {vk::ShaderStageFlagBits::eCompute}
: vk::ShaderStageFlags {vk::ShaderStageFlagBits::eVertex |
vk::ShaderStageFlagBits::eFragment},
0, ShaderRecompiler::IR::NativePushConstantSize};
pipeline_info.pushConstantRangeCount = 1;
pipeline_info.pPushConstantRanges = &push_range;
EXIT_IF(device.createPipelineLayout(&pipeline_info, nullptr,
&pipeline.pipeline_layout) !=
vk::Result::eSuccess);
pipeline.uses_push_descriptors = true;
return pipeline;
}
static PipelineCache::Pipeline CommitBindings(RenderExecutor &executor,
CommandBuffer &buffer,
PreparedBindings &bindings) {
std::array<PreparedBindings *, 1> stages{&bindings};
auto pipeline = CreateDescriptorPipeline(executor, stages);
const auto bind_point = bindings.runtime->program->stage == ShaderType::Compute
? vk::PipelineBindPoint::eCompute
: vk::PipelineBindPoint::eGraphics;
executor.CommitBindings(buffer, bind_point, pipeline, stages);
return pipeline;
}
static PipelineCache::Pipeline CommitBindings(RenderExecutor &executor,
CommandBuffer &buffer,
PreparedBindings &vertex,
PreparedBindings &pixel) {
std::array<PreparedBindings *, 2> stages{&vertex, &pixel};
auto pipeline = CreateDescriptorPipeline(executor, stages);
executor.CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline,
stages);
return pipeline;
}
static void CommitBindings(RenderExecutor &executor, CommandBuffer &buffer,
const PipelineCache::Pipeline &pipeline,
PreparedBindings &vertex, PreparedBindings &pixel) {
std::array<PreparedBindings *, 2> stages{&vertex, &pixel};
executor.CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline,
stages);
}
static void DestroyDescriptorPipelines(
RenderExecutor &executor,
std::span<const PipelineCache::Pipeline> pipelines) {
auto &device = executor.m_context.GetGraphics().device;
for (const auto &pipeline : pipelines) {
device.destroyPipelineLayout(pipeline.pipeline_layout, nullptr);
device.destroyDescriptorSetLayout(pipeline.descriptor_set_layout,
nullptr);
}
}
static void ResolveRenderDepthTarget(RenderExecutor &executor,
CommandBuffer &buffer,
RenderDepthInfo &depth) {
executor.ResolveRenderDepthTarget(buffer, depth);
}
static bool DepthStencilCopy(RenderExecutor &executor, CommandBuffer &buffer) {
return executor.DepthStencilCopy(buffer);
}
static void ResolveRenderColorTarget(RenderExecutor &executor,
CommandBuffer &buffer,
RenderColorInfo &color, uint32_t slot) {
executor.ResolveRenderColorTarget(buffer, color, 0, slot);
}
static void BindRenderTarget(RenderExecutor &executor, ImageId id) {
executor.BindRenderTarget(id);
}
static RenderState AcquireRenderTargets(RenderExecutor &executor,
CommandBuffer &buffer,
RenderColorInfo *colors,
uint32_t color_count,
RenderDepthInfo &depth,
std::span<PreparedBindings *const> stages = {},
vk::ImageAspectFlags *feedback_out = nullptr) {
vk::ImageAspectFlags feedback;
auto state = executor.AcquireRenderTargets(buffer, colors, color_count, depth,
feedback, stages);
if (feedback_out != nullptr) {
*feedback_out = feedback;
}
return state;
}
static void ResetBindings(RenderExecutor &executor) {
executor.ResetBindings();
}
static bool BoundImagesInOrder(const RenderExecutor &executor, ImageId first,
ImageId second) {
return executor.m_bound_images.size() == 2 &&
executor.m_bound_images[0] == first &&
executor.m_bound_images[1] == second;
}
};
namespace {
using u32 = uint32_t;
using BindingType = TextureCache::BindingType;
using ImageDesc = TextureCache::ImageDesc;
using ShaderOpcode = ShaderRecompiler::Decoder::Opcode;
constexpr u32 InlineU32(u32 value) { return 128u + value; }
constexpr u32 Vgpr(u32 reg) { return 256u + reg; }
constexpr u32 EncodeSMovB32(u32 dst, u32 src) {
return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | (0x03u << 8u) |
(src & 0xffu);
}
constexpr u32 EncodeSop1(u32 opcode, u32 dst, u32 src) {
return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) |
((opcode & 0xffu) << 8u) | (src & 0xffu);
}
constexpr u32 EncodeSop2(u32 opcode, u32 dst, u32 src0, u32 src1) {
return 0x80000000u | ((opcode & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) |
((src1 & 0xffu) << 8u) | (src0 & 0xffu);
}
constexpr u32 EncodeSopc(u32 opcode, u32 src0, u32 src1) {
return 0x80000000u | (0x7eu << 23u) | ((opcode & 0x7fu) << 16u) |
((src1 & 0xffu) << 8u) | (src0 & 0xffu);
}
constexpr u32 EncodeSopp(u32 opcode, u32 simm = 0) {
return 0x80000000u | (0x7fu << 23u) | ((opcode & 0x7fu) << 16u) |
(simm & 0xffffu);
}
constexpr u32 EncodeSopk(u32 opcode, u32 dst, u32 imm) {
return 0x80000000u | (((opcode + 0x60u) & 0x7fu) << 23u) |
((dst & 0x7fu) << 16u) | (imm & 0xffffu);
}
constexpr u32 EncodeVop1(u32 opcode, u32 dst, u32 src0) {
return (0x3fu << 25u) | ((dst & 0xffu) << 17u) | ((opcode & 0xffu) << 9u) |
(src0 & 0x1ffu);
}
constexpr u32 EncodeVop1Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0,
u32 src0_sel = 6, u32 src0_sext = 0,
u32 src0_neg = 0, u32 src0_abs = 0, u32 s0 = 0,
u32 clamp = 0, u32 omod = 0) {
return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) |
((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) |
((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) |
((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
((s0 & 0x1u) << 23u);
}
constexpr u32 EncodeVop1Dpp(u32 src0, u32 dpp_ctrl = 0, u32 row_mask = 0xf,
u32 bank_mask = 0xf) {
return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) |
((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u);
}
constexpr u32 EncodeVop2(u32 opcode, u32 dst, u32 src0, u32 src1) {
return ((opcode & 0x3fu) << 25u) | ((dst & 0xffu) << 17u) |
((src1 & 0xffu) << 9u) | (src0 & 0x1ffu);
}
constexpr u32 EncodeVop2Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0,
u32 src0_sel = 6, u32 src1_sel = 6,
u32 src0_sext = 0, u32 src1_sext = 0,
u32 src0_neg = 0, u32 src0_abs = 0,
u32 src1_neg = 0, u32 src1_abs = 0, u32 s0 = 0,
u32 s1 = 0, u32 clamp = 0, u32 omod = 0) {
return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) |
((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) |
((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) |
((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) |
((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) |
((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u);
}
constexpr u32 EncodeVop2Dpp(u32 src0, u32 dpp_ctrl = 0, u32 row_mask = 0xf,
u32 bank_mask = 0xf, u32 src0_neg = 0,
u32 src0_abs = 0, u32 src1_neg = 0,
u32 src1_abs = 0) {
return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) |
((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
((src1_neg & 0x1u) << 22u) | ((src1_abs & 0x1u) << 23u) |
((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u);
}
constexpr u32 EncodeVop3Word0(u32 opcode, u32 dst, u32 abs = 0, u32 op_sel = 0,
bool clamp = false) {
return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((abs & 0x7u) << 8u) |
((op_sel & 0xfu) << 11u) | (clamp ? (1u << 15u) : 0u) | (dst & 0xffu);
}
constexpr u32 EncodeVop3BWord0(u32 opcode, u32 vdst, u32 sdst) {
return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((sdst & 0x7fu) << 8u) |
(vdst & 0xffu);
}
constexpr u32 EncodeVop3Word1(u32 src0, u32 src1, u32 src2 = 0, u32 omod = 0,
u32 neg = 0) {
return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) |
((omod & 0x3u) << 27u) | ((neg & 0x7u) << 29u);
}
constexpr u32 EncodeVop3pWord0(u32 opcode, u32 dst, u32 op_sel_hi = 0,
u32 op_sel = 0, u32 neg_hi = 0,
bool clamp = false) {
return (0x33u << 26u) | ((opcode & 0x7fu) << 16u) | ((neg_hi & 0x7u) << 8u) |
((op_sel & 0x7u) << 11u) | (((op_sel_hi >> 2u) & 0x1u) << 14u) |
(clamp ? (1u << 15u) : 0u) | (dst & 0xffu);
}
constexpr u32 EncodeVop3pWord1(u32 src0, u32 src1, u32 src2 = 0,
u32 op_sel_hi = 0, u32 neg = 0) {
return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) |
((op_sel_hi & 0x3u) << 27u) | ((neg & 0x7u) << 29u);
}
constexpr u32 EncodeVopc(u32 opcode, u32 src0, u32 src1) {
return (0x3eu << 25u) | ((opcode & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) |
(src0 & 0x1ffu);
}
constexpr u32 EncodeVopcSdwa(u32 src0, u32 sdst = 0, u32 sd = 0,
u32 src0_sel = 6, u32 src1_sel = 6,
u32 src0_sext = 0, u32 src1_sext = 0,
u32 src0_neg = 0, u32 src0_abs = 0,
u32 src1_neg = 0, u32 src1_abs = 0, u32 s0 = 0,
u32 s1 = 0) {
return (src0 & 0xffu) | ((sdst & 0x7fu) << 8u) | ((sd & 0x1u) << 15u) |
((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) |
((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) |
((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) |
((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u);
}
constexpr u32 EncodeMubuf0(u32 opcode, u32 offset = 0, bool idxen = false,
bool offen = true, bool glc = false) {
return (0x38u << 26u) | ((opcode & 0xffu) << 18u) |
(offen ? (1u << 12u) : 0u) | (idxen ? (1u << 13u) : 0u) |
(glc ? (1u << 14u) : 0u) | (offset & 0xfffu);
}
constexpr u32 EncodeMubuf1(u32 vdata, u32 srsrc, u32 vaddr, u32 soffset = 128) {
return ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) |
((vdata & 0xffu) << 8u) | (vaddr & 0xffu);
}
constexpr u32 EncodeMtbuf0(u32 opcode, u32 dfmt, u32 nfmt, u32 offset = 0,
bool idxen = false, bool offen = true) {
return (0x3au << 26u) | (offset & 0xfffu) | (offen ? (1u << 12u) : 0u) |
(idxen ? (1u << 13u) : 0u) | ((opcode & 0x7u) << 16u) |
((dfmt & 0xfu) << 19u) | ((nfmt & 0x7u) << 23u);
}
constexpr u32 EncodeMtbuf1(u32 opcode, u32 vdata, u32 srsrc, u32 vaddr,
u32 soffset = 128) {
return (((opcode >> 3u) & 1u) << 21u) | ((soffset & 0xffu) << 24u) |
((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | (vaddr & 0xffu);
}
constexpr u32 EncodeSmem0(u32 opcode, u32 dst, u32 sbase = 0) {
return (0x3du << 26u) | ((opcode & 0xffu) << 18u) | ((dst & 0x7fu) << 6u) |
(sbase & 0x3fu);
}
constexpr u32 EncodeSmem1(u32 offset, u32 soffset = 0) {
return (offset & 0x1fffffu) | ((soffset & 0x7fu) << 25u);
}
constexpr u32 EncodeMimg0(u32 opcode, u32 dmask, u32 nsa_dwords = 0,
bool glc = false, u32 dim = 1, bool r128 = false) {
return (0x3cu << 26u) | ((opcode >> 7u) & 0x1u) |
((nsa_dwords & 0x3u) << 1u) | ((dim & 0x7u) << 3u) |
((dmask & 0xfu) << 8u) | (glc ? (1u << 13u) : 0u) |
(r128 ? (1u << 15u) : 0u) | ((opcode & 0x7fu) << 18u);
}
constexpr u32 EncodeMimg1(u32 vdata, u32 vaddr, u32 srsrc = 0, u32 ssamp = 0,
bool a16 = false, bool d16 = false) {
return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) |
((vdata & 0xffu) << 8u) | (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u) |
(d16 ? (1u << 31u) : 0u);
}
constexpr u32 EncodeVintrp(u32 opcode, u32 dst, u32 attr, u32 chan, u32 src) {
return (0x32u << 26u) | ((dst & 0xffu) << 18u) | ((opcode & 0x3u) << 16u) |
((attr & 0x3fu) << 10u) | ((chan & 0x3u) << 8u) | (src & 0xffu);
}
constexpr u32 EncodeExp0(u32 target, u32 en, bool done = true,
bool compr = false, bool vm = false) {
return (0x3eu << 26u) | ((target & 0x3fu) << 4u) | (en & 0xfu) |
(compr ? (1u << 10u) : 0u) | (done ? (1u << 11u) : 0u) |
(vm ? (1u << 12u) : 0u);
}
constexpr u32 EncodeExp1(u32 src0, u32 src1, u32 src2, u32 src3) {
return (src0 & 0xffu) | ((src1 & 0xffu) << 8u) | ((src2 & 0xffu) << 16u) |
((src3 & 0xffu) << 24u);
}
constexpr u32 EncodeFlat0(u32 opcode, u32 segment, u32 offset = 0) {
return (0x37u << 26u) | ((opcode & 0x7fu) << 18u) |
((segment & 0x3u) << 14u) | (offset & 0xfffu);
}
constexpr u32 EncodeFlat1(u32 vdst, u32 saddr, u32 data, u32 addr) {
return ((vdst & 0xffu) << 24u) | ((saddr & 0x7fu) << 16u) |
((data & 0xffu) << 8u) | (addr & 0xffu);
}
constexpr u32 EncodeDs0(u32 opcode, u32 offset = 0, bool gds = false) {
return (0x36u << 26u) | ((opcode & 0xffu) << 18u) | (gds ? (1u << 17u) : 0u) |
(offset & 0xffffu);
}
constexpr u32 EncodeDs1Ex(u32 vdst, u32 data1, u32 data0, u32 addr) {
return ((vdst & 0xffu) << 24u) | ((data1 & 0xffu) << 16u) |
((data0 & 0xffu) << 8u) | (addr & 0xffu);
}
constexpr u32 EncodeDs1(u32 vdst, u32 data0, u32 addr) {
return EncodeDs1Ex(vdst, 0, data0, addr);
}
void AppendSMovLiteral(std::vector<u32> *code, u32 dst, u32 literal) {
code->push_back(EncodeSMovB32(dst, 255u));
code->push_back(literal);
}
void AppendVMovLiteral(std::vector<u32> *code, u32 dst, u32 literal) {
code->push_back(EncodeVop1(0x01, dst, 255u));
code->push_back(literal);
}
void AppendVMovU32(std::vector<u32> *code, u32 dst, u32 value) {
if (value <= 64u) {
code->push_back(EncodeVop1(0x01, dst, InlineU32(value)));
return;
}
AppendVMovLiteral(code, dst, value);
}
void AppendVop3(std::vector<u32> *code, u32 opcode, u32 dst, u32 src0, u32 src1,
u32 src2 = 0, u32 abs = 0, u32 op_sel = 0, bool clamp = false,
u32 omod = 0, u32 neg = 0) {
code->push_back(EncodeVop3Word0(opcode, dst, abs, op_sel, clamp));
code->push_back(EncodeVop3Word1(src0, src1, src2, omod, neg));
}
void AppendVop3B(std::vector<u32> *code, u32 opcode, u32 vdst, u32 sdst,
u32 src0, u32 src1, u32 src2 = 0) {
code->push_back(EncodeVop3BWord0(opcode, vdst, sdst));
code->push_back(EncodeVop3Word1(src0, src1, src2));
}
void AppendVop3p(std::vector<u32> *code, u32 opcode, u32 dst, u32 src0,
u32 src1, u32 src2 = 0, u32 op_sel_hi = 0, u32 op_sel = 0,
u32 neg_hi = 0, u32 neg = 0) {
code->push_back(EncodeVop3pWord0(opcode, dst, op_sel_hi, op_sel, neg_hi));
code->push_back(EncodeVop3pWord1(src0, src1, src2, op_sel_hi, neg));
}
void AppendBufferLoadDword(std::vector<u32> *code, u32 dst_vgpr,
u32 address_vgpr) {
code->push_back(EncodeMubuf0(0x0cu));
code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr));
}
void AppendBufferLoadOpcode(std::vector<u32> *code, u32 opcode, u32 dst_vgpr,
u32 address_vgpr) {
code->push_back(EncodeMubuf0(opcode));
code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr));
}
void AppendBufferStoreDword(std::vector<u32> *code, u32 value_vgpr,
u32 address_vgpr) {
code->push_back(EncodeMubuf0(0x1cu));
code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr));
}
void AppendBufferStoreOpcode(std::vector<u32> *code, u32 opcode, u32 value_vgpr,
u32 address_vgpr, bool glc = false) {
code->push_back(EncodeMubuf0(opcode, 0, false, true, glc));
code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr));
}
void AppendTBufferLoadOpcode(std::vector<u32> *code, u32 opcode, u32 dst_vgpr,
u32 address_vgpr) {
code->push_back(EncodeMtbuf0(opcode, 14, 7));
code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr));
}
constexpr u32 BufferFormat(Prospero::BufferFormat format) {
return static_cast<uint32_t>(format);
}
void AppendTBufferLoadFormatOpcode(std::vector<u32> *code, u32 opcode,
u32 dst_vgpr, u32 address_vgpr,
Prospero::BufferFormat format) {
const auto value = BufferFormat(format);
code->push_back(EncodeMtbuf0(opcode, value & 0xfu, (value >> 4u) & 0x7u));
code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr));
}
void AppendTBufferStoreOpcode(std::vector<u32> *code, u32 opcode,
u32 value_vgpr, u32 address_vgpr) {
code->push_back(EncodeMtbuf0(opcode, 14, 7));
code->push_back(EncodeMtbuf1(opcode, value_vgpr, 0, address_vgpr));
}
void AppendSmemLoadOpcode(std::vector<u32> *code, u32 opcode, u32 dst_sgpr,
u32 byte_offset) {
code->push_back(EncodeSmem0(opcode, dst_sgpr));
code->push_back(EncodeSmem1(byte_offset));
}
void AppendStoreVgpr(std::vector<u32> *code, u32 value_vgpr, u32 dword_index) {
AppendVMovU32(code, 31, dword_index * 4u);
AppendBufferStoreDword(code, value_vgpr, 31);
}
void AppendStoreVgprAtLaneDwordOffset(std::vector<u32> *code, u32 value_vgpr,
u32 lane_vgpr, u32 dword_offset) {
if (dword_offset == 0u) {
code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), lane_vgpr));
} else {
AppendVMovU32(code, 31, dword_offset);
code->push_back(EncodeVop2(0x25, 31, Vgpr(lane_vgpr), 31));
code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), 31));
}
AppendBufferStoreDword(code, value_vgpr, 31);
}
void AppendStoreSgpr(std::vector<u32> *code, u32 value_sgpr, u32 dword_index) {
code->push_back(EncodeVop1(0x01, 30, value_sgpr));
AppendStoreVgpr(code, 30, dword_index);
}
void AppendStoreSgprAtLaneDwordOffset(std::vector<u32> *code, u32 value_sgpr,
u32 lane_vgpr, u32 dword_offset) {
code->push_back(EncodeVop1(0x01, 30, value_sgpr));
AppendStoreVgprAtLaneDwordOffset(code, 30, lane_vgpr, dword_offset);
}
void AppendStoreSgprPair(std::vector<u32> *code, u32 value_sgpr,
u32 first_dword_index) {
AppendStoreSgpr(code, value_sgpr, first_dword_index);
AppendStoreSgpr(code, value_sgpr + 1u, first_dword_index + 1u);
}
void AppendEnd(std::vector<u32> *code) { code->push_back(0xbf810000u); }
std::string Hex(u32 value) {
char buffer[32] = {};
std::snprintf(buffer, sizeof(buffer), "0x%08" PRIx32, value);
return buffer;
}
[[noreturn]] void Fail(const char *shader_name, const char *stage,
const std::string &message) {
std::fprintf(stderr, "ShaderRecompilerComputeTests: %s failed at %s: %s\n",
shader_name, stage, message.c_str());
std::abort();
}
void Require(const char *shader_name, const char *stage, bool value,
const std::string &message);
void EnsureConfigInitialized() {
static bool config_initialized = false;
if (!config_initialized) {
static Common::Subsystems subsystems;
Common::InitializeThreads();
subsystems.Initialize<Config::Lifecycle>();
Config::ConfigOptions options;
options.printf_direction = Config::LogDirection::Silent;
Config::Load(options);
subsystems.Initialize<Log::Lifecycle>();
subsystems.Initialize<Libs::LibKernel::Memory::Lifecycle>();
ShaderInit();
config_initialized = true;
}
}
void RequireVk(const char *shader_name, const char *stage, vk::Result result,
const char *action) {
if (result != vk::Result::eSuccess) {
Fail(shader_name, stage,
std::string(action) + " returned " + vk::to_string(result));
}
}
void Require(const char *shader_name, const char *stage, bool value,
const std::string &message) {
if (!value) {
Fail(shader_name, stage, message);
}
}
void CheckErrorDialogLifecycle() {
namespace ErrorDialog = Libs::Dialog::ErrorDialog;
namespace SystemDialog = Libs::Dialog::SystemDialog;
struct Param {
int32_t size = 16;
int32_t error_code = static_cast<int32_t>(0x80550006u);
int32_t user_id = 1;
int32_t reserved = 0;
} param;
static_assert(sizeof(Param) == 16);
constexpr int invalid_state = static_cast<int>(0x80ed0005u);
constexpr int invalid_param = static_cast<int>(0x80ed0003u);
static std::vector<SystemDialog::VisualState> notifications;
notifications.clear();
SystemDialog::SetVisibilityCallback([] {
const auto visual = SystemDialog::GetVisualState();
SystemDialog::HostSnapshot snapshot{};
Require("ErrorDialog", "callback reentry",
SystemDialog::GetHostSnapshot(&snapshot) == visual.active &&
(ErrorDialog::ErrorDialogGetStatus() == 2) == visual.active,
"visibility callback observed inconsistent dialog state");
notifications.push_back(visual);
});
const auto initial_revision = SystemDialog::GetVisualState().revision;
const auto check_state = [&](const char *stage, int status,
size_t visibility_changes) {
const auto visual = SystemDialog::GetVisualState();
SystemDialog::HostSnapshot snapshot{};
Require("ErrorDialog", stage,
ErrorDialog::ErrorDialogGetStatus() == status &&
ErrorDialog::ErrorDialogUpdateStatus() == status &&
visual.active == (status == 2) && !visual.background &&
SystemDialog::GetHostSnapshot(&snapshot) == visual.active &&
visual.revision == initial_revision + visibility_changes &&
notifications.size() == visibility_changes &&
(notifications.empty() ||
(notifications.back().active == visual.active &&
notifications.back().revision == visual.revision)),
"status, host visibility, or visibility notification changed "
"unexpectedly");
};
check_state("uninitialized", 0, 0);
Require("ErrorDialog", "uninitialized operations",
ErrorDialog::ErrorDialogTerminate() ==
static_cast<int>(0x80ed0001u) &&
ErrorDialog::ErrorDialogClose() == invalid_state,
"uninitialized dialog accepted termination or close");
Require("ErrorDialog", "initialize",
ErrorDialog::ErrorDialogInitialize() == 0 &&
ErrorDialog::ErrorDialogInitialize() ==
static_cast<int>(0x80ed0002u),
"initialization did not reject a second initialization");
Param invalid = param;
invalid.size--;
Require("ErrorDialog", "invalid open",
ErrorDialog::ErrorDialogOpen(nullptr) == invalid_param &&
ErrorDialog::ErrorDialogOpen(&invalid) == invalid_param &&
ErrorDialog::ErrorDialogClose() == invalid_state,
"invalid parameters or close were accepted before opening");
check_state("failed open preserves initialized state", 1, 0);
Require("ErrorDialog", "open", ErrorDialog::ErrorDialogOpen(&param) == 0,
"valid error dialog did not open");
SystemDialog::HostSnapshot first{};
Require("ErrorDialog", "copied error code",
SystemDialog::GetHostSnapshot(&first) &&
first.error_code == param.error_code,
"host dialog did not expose the guest error code");
param.error_code = static_cast<int32_t>(0x8055000au);
for (int poll = 0; poll < 8; ++poll) {
check_state("polling waits for acknowledgement", 2, 1);
}
Require(
"ErrorDialog", "running failures",
ErrorDialog::ErrorDialogOpen(&param) == invalid_state &&
!SystemDialog::HostClose(first.generation + 1),
"an invalid open or stale acknowledgement replaced the active dialog");
SystemDialog::HostSnapshot current{};
Require("ErrorDialog", "failed open preserves active dialog",
SystemDialog::GetHostSnapshot(&current) &&
current.generation == first.generation &&
current.error_code == first.error_code,
"active error code or generation changed after a failed open");
check_state("failed operations preserve running state", 2, 1);
Require("ErrorDialog", "acknowledge",
SystemDialog::HostClose(first.generation),
"current host acknowledgement did not finish the dialog");
check_state("acknowledged", 3, 2);
Require("ErrorDialog", "finished operations",
!SystemDialog::HostClose(first.generation) &&
ErrorDialog::ErrorDialogClose() == invalid_state &&
ErrorDialog::ErrorDialogOpen(&invalid) == invalid_param,
"finished dialog accepted duplicate completion or an invalid open");
check_state("failed operations preserve finished state", 3, 2);
Require("ErrorDialog", "reopen and stale acknowledgement",
ErrorDialog::ErrorDialogOpen(&param) == 0 &&
SystemDialog::GetHostSnapshot(&current) &&
current.generation != first.generation &&
current.error_code == param.error_code &&
!SystemDialog::HostClose(first.generation),
"reopened dialog accepted acknowledgement from its predecessor");
check_state("reopened", 2, 3);
Require("ErrorDialog", "guest close", ErrorDialog::ErrorDialogClose() == 0,
"guest close did not finish the running dialog");
check_state("closed", 3, 4);
Require("ErrorDialog", "terminate running dialog",
ErrorDialog::ErrorDialogOpen(&param) == 0 &&
SystemDialog::GetHostSnapshot(&current) &&
ErrorDialog::ErrorDialogTerminate() == 0 &&
!SystemDialog::HostClose(current.generation),
"termination failed to remove the active dialog");
check_state("terminated", 0, 6);
Require("ErrorDialog", "reinitialize and stale acknowledgement",
ErrorDialog::ErrorDialogInitialize() == 0 &&
ErrorDialog::ErrorDialogOpen(&param) == 0 &&
!SystemDialog::HostClose(current.generation),
"reinitialization reused an old dialog generation");
check_state("reinitialized", 2, 7);
Require("ErrorDialog", "final termination",
ErrorDialog::ErrorDialogTerminate() == 0 &&
ErrorDialog::ErrorDialogTerminate() ==
static_cast<int>(0x80ed0001u),
"termination did not restore the uninitialized state");
check_state("final state", 0, 8);
SystemDialog::SetVisibilityCallback(nullptr);
Require("ErrorDialog", "lifecycle without a visibility listener",
ErrorDialog::ErrorDialogInitialize() == 0 &&
ErrorDialog::ErrorDialogOpen(&param) == 0 &&
ErrorDialog::ErrorDialogClose() == 0 &&
ErrorDialog::ErrorDialogTerminate() == 0 &&
ErrorDialog::ErrorDialogGetStatus() == 0 &&
!SystemDialog::GetVisualState().active &&
SystemDialog::GetVisualState().revision ==
initial_revision + 10 &&
notifications.size() == 8,
"guest lifecycle depended on a registered host visibility listener");
std::printf("[host] %-32s ok\n", "ErrorDialogLifecycle");
}
void CheckSigninDialogLifecycle() {
namespace Signin = Libs::Dialog::SigninDialog;
namespace Error = Libs::Dialog::ErrorDialog;
namespace Host = Libs::Dialog::SystemDialog;
struct Param {
int32_t size = 16;
int32_t user_id = Config::GetUserId();
int32_t reserved[2] = {};
} param;
struct Result {
int32_t result = -1;
int32_t reserved[3] = {-1, -1, -1};
} result;
constexpr int not_initialized = static_cast<int>(0x81350001u);
constexpr int already_initialized = static_cast<int>(0x81350002u);
constexpr int invalid_param = static_cast<int>(0x81350003u);
constexpr int invalid_state = static_cast<int>(0x81350005u);
Require("SigninDialog", "uninitialized",
Signin::SigninDialogOpen(&param) == not_initialized &&
Signin::SigninDialogClose() == not_initialized &&
Signin::SigninDialogGetResult(&result) == not_initialized,
"uninitialized operations returned the wrong error");
Require("SigninDialog", "initialize and close",
Signin::SigninDialogInitialize() == 0 &&
Signin::SigninDialogClose() == 0 &&
Signin::SigninDialogUpdateStatus() == 3,
"close must also succeed from INITIALIZED");
Param invalid = param;
invalid.reserved[0] = 1;
Require("SigninDialog", "parameter validation",
Signin::SigninDialogOpen(nullptr) == invalid_param &&
Signin::SigninDialogOpen(&invalid) == invalid_param,
"invalid parameters were accepted");
invalid = param;
invalid.user_id = -1;
Require("SigninDialog", "user validation",
Signin::SigninDialogOpen(&invalid) == static_cast<int>(0x81350007u),
"an invalid user was accepted");
Host::SetVisibilityCallback([] {
const auto visual = Host::GetVisualState();
Require("SigninDialog", "callback reentry",
visual.background == (Signin::SigninDialogGetStatus() == 2),
"foreground state did not follow the dialog lifetime");
});
Require("SigninDialog", "open", Signin::SigninDialogOpen(&param) == 0,
"valid sign-in dialog did not open");
Host::HostSnapshot first{};
Require(
"SigninDialog", "visible and backgrounded",
Host::GetHostSnapshot(&first) && first.kind == Host::Kind::Signin &&
Host::GetVisualState().active && Host::GetVisualState().background,
"sign-in dialog was not visible or did not background the application");
for (int poll = 0; poll < 8; ++poll) {
Require(
"SigninDialog", "polling waits for the user",
Signin::SigninDialogUpdateStatus() == 2 &&
Signin::SigninDialogGetStatus() == 2 &&
Signin::SigninDialogInitialize() == already_initialized &&
Signin::SigninDialogGetResult(&result) == invalid_state,
"polling silently finished sign-in or lost the active initialization");
}
Require(
"SigninDialog", "user cancellation",
Host::HostClose(first.generation) &&
Signin::SigninDialogGetResult(&result) == 0 && result.result == 1 &&
result.reserved[0] == 0 && result.reserved[1] == 0 &&
result.reserved[2] == 0 && !Host::GetVisualState().background &&
!Host::GetVisualState().active && Signin::SigninDialogClose() == 0,
"user cancellation did not finish sign-in and restore the foreground");
const int32_t error_param[4] = {16, static_cast<int32_t>(0x80550006u), 1, 0};
Host::HostSnapshot error{};
Require("SigninDialog", "shared presentation generations",
Error::ErrorDialogInitialize() == 0 &&
Error::ErrorDialogOpen(error_param) == 0 &&
Host::GetHostSnapshot(&error) &&
error.kind == Host::Kind::Error &&
error.generation != first.generation &&
!Host::HostClose(first.generation),
"an old sign-in action affected an error dialog");
Host::HostSnapshot overlay{};
Require("SigninDialog", "overlapping dialogs",
Signin::SigninDialogOpen(&param) == 0 &&
Host::GetHostSnapshot(&overlay) &&
overlay.kind == Host::Kind::Signin &&
!Host::HostClose(error.generation) &&
Host::HostClose(overlay.generation) &&
Host::GetHostSnapshot(&overlay) &&
overlay.kind == Host::Kind::Error &&
overlay.generation == error.generation &&
!Host::GetVisualState().background &&
Host::HostClose(error.generation) &&
Error::ErrorDialogTerminate() == 0,
"closing sign-in did not restore the pending error dialog");
Host::HostSnapshot reopened{};
Require(
"SigninDialog", "reopen and abort",
Signin::SigninDialogOpen(&param) == 0 &&
Host::GetHostSnapshot(&reopened) &&
!Host::HostClose(first.generation) &&
!Host::HostClose(error.generation) &&
Signin::SigninDialogTerminate() == 0 &&
!Host::HostClose(reopened.generation) &&
!Host::GetVisualState().active &&
!Host::GetVisualState().background &&
Signin::SigninDialogTerminate() == not_initialized,
"abort did not remove the dialog or a stale host action was accepted");
Host::SetVisibilityCallback(nullptr);
std::printf("[host] %-32s ok\n", "SigninDialogLifecycle");
}
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
template <typename Function>
void ExpectFatal(const char *name, Function function) {
const pid_t pid = ::fork();
Require(name, "fatal subprocess", pid >= 0,
"fork failed while starting fatal shader test");
if (pid == 0) {
function();
::_exit(0);
}
int status = 0;
pid_t waited = -1;
do {
waited = ::waitpid(pid, &status, 0);
} while (waited < 0 && errno == EINTR);
Require(name, "fatal subprocess", waited == pid,
"waitpid failed while collecting fatal shader test");
Require(name, "fatal exit",
WIFEXITED(status) && WEXITSTATUS(status) == (321 & 0xff),
"fatal shader path did not terminate with status 321");
std::printf("[host] %-32s ok\n", name);
}
#endif
void CheckLeastRecentlyUsedCacheOrdering() {
Common::LeastRecentlyUsedCache<uint32_t, uint64_t> cache;
const auto first = cache.Insert(1, 1);
const auto middle = cache.Insert(2, 2);
const auto last = cache.Insert(3, 3);
(void)first;
(void)last;
cache.Touch(middle, 4);
std::vector<uint32_t> visited;
cache.ForEachItemBelow(4, [&](uint32_t value) {
visited.push_back(value);
return visited.size() > 3;
});
Require("LeastRecentlyUsedCache", "non-tail touch traversal",
visited == std::vector<uint32_t>{1, 3, 2},
"touching a non-tail item left a cycle or changed LRU order");
std::printf("[host] %-32s ok\n", "LeastRecentlyUsedCache");
}
struct BdaMapping {
uint64_t guest_base = 0;
u32 backing_offset = 0;
};
struct TestCase {
const char *name = "";
std::vector<u32> code;
std::vector<u32> initial;
std::vector<u32> expected;
std::vector<ShaderOpcode> opcodes;
u32 image_width = 4;
u32 image_height = 4;
std::vector<u32> sampled_image_rgba;
std::vector<std::vector<u32>> sampled_image_rgba_mips;
vk::Format sampled_image_format = vk::Format::eR32G32B32A32Sfloat;
vk::Filter sampler_filter = vk::Filter::eNearest;
u32 sampled_image_dwords_per_pixel = 4;
vk::ImageType sampled_image_type = vk::ImageType::e2D;
vk::ImageViewType sampled_image_view_type = vk::ImageViewType::e2D;
u32 sampled_image_layers = 1;
u32 sampled_image_view_base_mip = 0;
u32 sampled_image_view_base_layer = 0;
u32 sampled_image_view_layers = 0;
std::vector<u32> storage_image_rgba;
std::vector<u32> expected_storage_image_rgba;
vk::Format storage_image_format = vk::Format::eR32G32B32A32Sfloat;
u32 storage_image_dwords_per_pixel = 4;
std::vector<u32> storage_image_r32ui;
std::vector<u32> expected_storage_image_r32ui;
std::vector<std::string> required_spirv;
std::vector<std::string> forbidden_spirv;
ShaderComputeInputInfo compute_info = [] {
ShaderComputeInputInfo info{};
info.lds_size_dwords = 1024;
return info;
}();
bool has_compute_info = false;
u32 dispatch_x = 1;
u32 dispatch_y = 1;
u32 dispatch_z = 1;
std::array<u32, 64> user_data{};
bool has_user_data = false;
u32 image_descriptor_swizzle = DstSel(4, 5, 6, 7);
bool compile_only = false;
size_t storage_buffer_range_bytes = 0;
std::vector<u32> storage_buffer_offsets;
std::vector<BdaMapping> bda_mappings;
bool expand_shader_data_storage = false;
bool expected_force_point_sampler = false;
float expected_float_tolerance = 0.0f;
std::vector<u32> gds_initial;
std::vector<u32> expected_gds;
std::vector<std::pair<std::string, size_t>> decoded_counts;
std::vector<std::pair<std::string, size_t>> ir_counts;
u32 expected_mip_descriptors = 0;
std::optional<std::vector<u32>> expected_buffer_resources;
};
struct GraphicsCase {
const char *name = "";
std::vector<u32> fragment_code;
std::vector<u32> expected_pixel;
std::vector<ShaderOpcode> opcodes;
std::array<u32, 64> user_data{};
bool has_user_data = false;
std::vector<u32> push_constants;
std::vector<u32> pixel_interpolator_settings;
bool pixel_no_perspective = false;
bool provoking_vtx_last = false;
std::vector<u32> vertices;
bool pixel_ancillary = false;
bool pixel_front_face = false;
u32 layers = 1;
vk::SampleCountFlagBits samples = vk::SampleCountFlagBits::e1;
bool pixel_position_w = false;
float vertex_clip_w = 1.0f;
bool pixel_depth_export = false;
u32 pixel_perspective_sample_vgpr = UINT32_MAX;
u32 pixel_perspective_centroid_vgpr = UINT32_MAX;
u32 pixel_custom_interpolation_mask = 0;
};
struct CompiledShader {
std::vector<u32> spirv;
ShaderRecompiler::IR::Program program;
ShaderRecompiler::IR::ResourceSnapshot resources;
std::vector<u32> packed_user_data;
};
std::array<u32, 64> MakeNativeUserData(const std::array<u32, 64> *source) {
std::array<u32, 64> data{};
data[3] = 3u << 28u;
data[50] = 1u << 20u;
data[51] = 3u << 28u;
if (source != nullptr) {
data = *source;
}
return data;
}
bool ReadTestMemory(void *userdata, uint64_t address, std::span<u32> values) {
const auto *data = static_cast<const std::vector<u32> *>(userdata);
if (data == nullptr || address % 4u != 0 || values.size() > data->size() ||
address / 4u > data->size() - values.size()) {
return false;
}
std::copy_n(data->begin() + address / 4u, values.size(), values.begin());
return true;
}
void ValidateSpirv(const char *shader_name, const std::vector<u32> &spirv) {
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string messages;
tools.SetMessageConsumer([&messages](spv_message_level_t, const char *,
const spv_position_t &position,
const char *message) {
char buffer[1024] = {};
std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line,
position.column, message);
messages += buffer;
});
if (!tools.Validate(spirv)) {
Fail(shader_name, "SPIR-V validation", messages);
}
}
size_t CountText(const std::string &text, const std::string &needle) {
size_t count = 0;
for (size_t offset = 0;
(offset = text.find(needle, offset)) != std::string::npos;
offset += needle.size()) {
count++;
}
return count;
}
void CheckPixelParameterAliases() {
constexpr const char *name = "PixelParameterAliases";
ShaderPixelInputInfo pixel{};
pixel.input_num = 2;
const std::array<uint32_t, 2> pair = {0, 1};
for (const auto flat : {0u, 0x400u}) {
pixel.interpolator_settings[0] = flat | 3u;
pixel.interpolator_settings[1] = flat | 3u;
Require(name, "same-mode nonidentity alias",
ShaderPixelParameterLocation(pixel, pair, 0) == 3u &&
ShaderPixelParameterLocation(pixel, pair, 1) == 3u,
"aliases of one export with the same interpolation mode must "
"share that physical location");
}
pixel.interpolator_settings[0] = 3u;
pixel.interpolator_settings[1] = 0x423u;
pixel.custom_interpolation_mask = 2u;
Require(name, "smooth/custom alias",
ShaderPixelParameterLocation(pixel, pair, 0) == 3u &&
ShaderPixelParameterLocation(pixel, pair, 1) == 3u,
"custom interpolation must retain its shared vertex export");
pixel.input_num = 3;
pixel.custom_interpolation_mask = 0;
pixel.interpolator_settings[0] = 0x400u;
pixel.interpolator_settings[1] = 0u;
pixel.interpolator_settings[2] = 1u;
const std::array<uint32_t, 3> active = {0, 1, 2};
Require(name, "ordinary mixed aliases",
ShaderPixelParameterLocation(pixel, active, 0) == 0u &&
ShaderPixelParameterLocation(pixel, active, 1) == 0u &&
ShaderPixelParameterLocation(pixel, active, 2) == 1u,
"flat/smooth aliases must retain the real vertex export location");
std::vector<uint32_t> ordinary_key, rectangle_key, provoking_key;
BuildStageStaticKey(pixel, ordinary_key);
pixel.parameter_mode = ShaderPixelParameterMode::LastVertex;
BuildStageStaticKey(pixel, provoking_key);
pixel.parameter_mode = ShaderPixelParameterMode::Rectangle;
BuildStageStaticKey(pixel, rectangle_key);
Require(name, "interpolation specialization",
ordinary_key != rectangle_key && ordinary_key != provoking_key,
"rectangle and provoking-vertex modes must participate in the shader key");
Require(name, "reserved physical locations",
ShaderPixelParameterLocation(pixel, active, 0) == 0u &&
ShaderPixelParameterLocation(pixel, active, 1) == 2u &&
ShaderPixelParameterLocation(pixel, active, 2) == 1u,
"a synthesized opposite-mode alias must not occupy another "
"vertex export's physical location");
std::printf("[host] %-32s ok\n", name);
}
void CheckRectListShaders() {
constexpr const char *name = "RectListShaders";
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.info.inputs.push_back(
{ShaderRecompiler::IR::StageInputKind::Parameter, 0, 4, "in_param_0"});
program.info.inputs.push_back(
{ShaderRecompiler::IR::StageInputKind::Parameter, 1, 4, "in_param_1"});
ShaderVertexInputInfo vertex{};
ShaderRecompiler::IR::CompiledShaderInfo vertex_program{.param_export_mask = 1u};
ShaderRecompiler::IR::ResourceSnapshot empty_snapshot;
vertex.stage.program = &vertex_program;
vertex.stage.resources = &empty_snapshot;
ShaderPixelInputInfo pixel{};
pixel.parameter_mode = ShaderPixelParameterMode::Rectangle;
pixel.input_num = 2;
pixel.interpolator_settings[0] = 0x400u;
pixel.interpolator_settings[1] = 0;
pixel.stage.program = &program;
pixel.stage.resources = &empty_snapshot;
std::vector<uint32_t> perspective_specialization;
BuildStageStaticKey(pixel, perspective_specialization);
pixel.ps_no_perspective = true;
std::vector<uint32_t> no_perspective_specialization;
BuildStageStaticKey(pixel, no_perspective_specialization);
pixel.ps_no_perspective = false;
Require(name, "pipeline identity",
perspective_specialization != no_perspective_specialization,
"pixel interpolation mode must participate in the shader and "
"pipeline key");
const std::array<uint32_t, 2> active_inputs = {0, 1};
Require(name, "duplicate mapping",
ShaderPixelParameterLocation(pixel, active_inputs, 0) == 0 &&
ShaderPixelParameterLocation(pixel, active_inputs, 1) == 1,
"duplicate pixel mappings must receive distinct effective output "
"locations");
const auto shaders = BuildRectListShaders(vertex, &pixel);
Require(name, "SPIR-V version",
shaders.control.size() > 1 && shaders.control[1] == 0x00010500u &&
shaders.evaluation.size() > 1 &&
shaders.evaluation[1] == 0x00010500u,
"vector selection requires SPIR-V 1.5");
ValidateSpirv(name, shaders.control);
ValidateSpirv(name, shaders.evaluation);
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string control_text;
std::string evaluation_text;
Require(name, "control disassembly",
tools.Disassemble(shaders.control, &control_text),
"failed to disassemble rectangle-list tessellation control shader");
Require(
name, "evaluation disassembly",
tools.Disassemble(shaders.evaluation, &evaluation_text),
"failed to disassemble rectangle-list tessellation evaluation shader");
Require(name, "control execution mode",
control_text.find("TessellationControl") != std::string::npos &&
control_text.find("OutputVertices 4") != std::string::npos,
"rectangle-list control shader must produce four control points");
Require(name, "evaluation execution modes",
evaluation_text.find("TessellationEvaluation") != std::string::npos &&
evaluation_text.find("Quads") != std::string::npos &&
evaluation_text.find("SpacingEqual") != std::string::npos &&
evaluation_text.find("VertexOrderCw") != std::string::npos,
"rectangle-list evaluation shader has the wrong patch modes");
Require(name, "no geometry stage",
control_text.find("Geometry") == std::string::npos &&
evaluation_text.find("Geometry") == std::string::npos,
"rectangle-list expansion must not use geometry shaders");
Require(name, "flat broadcast",
CountText(control_text, "OpVectorTimesScalar") == 6 &&
CountText(control_text, "OpSelect %v4float") == 2,
"flat parameters must use guest vertex zero instead of reconstructed "
"values");
Require(name, "remapped interface",
CountText(control_text, " Location 0") == 2 &&
CountText(control_text, " Location 1") == 1 &&
CountText(evaluation_text, " Location 0") == 2 &&
CountText(evaluation_text, " Location 1") == 2,
"duplicate pixel mappings must share one vertex input and keep "
"distinct patch outputs");
for (const auto flat : {0u, 0x400u}) {
pixel.interpolator_settings[0] = flat;
pixel.interpolator_settings[1] = flat;
const auto shared = BuildRectListShaders(vertex, &pixel);
ValidateSpirv(name, shared.control);
ValidateSpirv(name, shared.evaluation);
Require(name, "same-mode alias disassembly",
tools.Disassemble(shared.control, &control_text) &&
tools.Disassemble(shared.evaluation, &evaluation_text),
"failed to disassemble shared rectangle-list parameters");
Require(name, "same-mode interface deduplication",
CountText(control_text, " Location 0") == 2 &&
CountText(evaluation_text, " Location 0") == 2 &&
control_text.find(" Location 1") == std::string::npos &&
evaluation_text.find(" Location 1") == std::string::npos,
"same-mode aliases must produce one input and one output at "
"the shared location in each rectangle-list stage");
}
const auto position_only = BuildRectListShaders(vertex, nullptr);
ValidateSpirv(name, position_only.control);
ValidateSpirv(name, position_only.evaluation);
}
void CheckSpirvText(const TestCase &test, const std::vector<u32> &spirv) {
if (test.required_spirv.empty() && test.forbidden_spirv.empty()) {
return;
}
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string text;
if (!tools.Disassemble(spirv, &text)) {
Fail(test.name, "SPIR-V disassembly",
"failed to disassemble emitted SPIR-V");
}
for (const auto &required : test.required_spirv) {
if (text.find(required) == std::string::npos) {
Fail(test.name, "SPIR-V disassembly",
std::string("missing required text: ") + required);
}
}
for (const auto &forbidden : test.forbidden_spirv) {
if (text.find(forbidden) != std::string::npos) {
Fail(test.name, "SPIR-V disassembly",
std::string("found forbidden text: ") + forbidden);
}
}
}
CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
auto user_data =
MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr);
const auto uses_image =
std::any_of(test.opcodes.begin(), test.opcodes.end(), [](auto op) {
return op >= ShaderOpcode::IMAGE_GET_RESINFO &&
op <= ShaderOpcode::IMAGE_GATHER4H;
});
if (uses_image && ((user_data[3] >> 28u) & 0xfu) == 0) {
user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u;
}
if (uses_image) {
user_data[3] = (user_data[3] & ~0xfffu) | test.image_descriptor_swizzle;
}
if (!test.has_user_data) {
user_data[2] = static_cast<u32>(test.initial.size() * sizeof(u32));
}
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Compute;
options.dump_ir = true;
auto compute_info = test.compute_info;
compute_info.host_subgroup_size = host_subgroup_size;
options.input_info.compute = &compute_info;
options.user_data = user_data;
if (test.has_compute_info) {
options.wave_size = test.compute_info.wave_size;
}
auto translated = ShaderRecompiler::TranslateProgram(test.code, options);
auto resource_plan =
ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot resources;
ShaderRecompiler::IR::ResourceSpecialization specialization;
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = options.user_data,
.shader_base = reinterpret_cast<uint64_t>(test.code.data()),
.read_memory = ReadTestMemory,
.userdata = const_cast<std::vector<u32> *>(&test.initial),
.read_specialization_memory = ReadTestMemory,
};
Require(test.name, "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
resource_plan, runtime, resources, specialization),
"translated resources could not be materialized");
auto result = ShaderRecompiler::CompileProgram(
std::move(translated), options, specialization);
for (const auto &[text, expected] : test.decoded_counts) {
const auto actual = CountText(result.decoded_dump, text);
Require(test.name, "decoded RDNA2", actual == expected,
text + " count=" + std::to_string(actual) +
", expected=" + std::to_string(expected));
}
for (const auto &[text, expected] : test.ir_counts) {
const auto actual = CountText(result.ir_dump, text);
Require(test.name, "typed IR", actual == expected,
text + " count=" + std::to_string(actual) +
", expected=" + std::to_string(expected));
}
if (test.expected_mip_descriptors != 0) {
const auto image = std::ranges::find_if(
result.program.info.images, [](const auto &resource) {
return resource.mip_mode ==
ShaderRecompiler::IR::ImageMipMode::Dynamic;
});
Require(test.name, "dynamic mip specialization",
image != result.program.info.images.end() &&
image->mip_count == test.expected_mip_descriptors,
"runtime descriptor range did not specialize the mip count");
const auto kind = ShaderRecompiler::IR::DescriptorBindingForImage(*image);
Require(test.name, "dynamic mip binding",
kind && image->descriptor_index != UINT32_MAX &&
image->descriptor_index + image->mip_count <=
result.program.bindings.descriptor_counts[static_cast<size_t>(*kind)],
"dynamic image did not receive one descriptor per mip");
}
if (test.expand_shader_data_storage) {
auto &block = *result.program.blocks.front();
for (u32 reg = 0; reg < 33; reg++) {
auto &user_data = block.AppendNewInst(
ShaderRecompiler::IR::ValueOpcode::GetUserData,
{ShaderRecompiler::IR::Value(
static_cast<ShaderRecompiler::IR::ScalarReg>(reg))});
block.AppendNewInst(ShaderRecompiler::IR::ValueOpcode::ReferenceU32,
{ShaderRecompiler::IR::Value(&user_data)});
}
result.program.shader_info_complete = false;
ShaderRecompiler::IR::CollectShaderInfo(result.program, options.input_info);
result.spirv = ShaderRecompiler::Spirv::EmitProgram(result.program, options.input_info);
const auto shader_data = result.program.bindings.descriptor_counts[
static_cast<size_t>(ShaderRecompiler::IR::DescriptorBindingKind::ShaderData)];
Require(test.name, "binding layout",
result.program.info.user_data_registers.size() == 33 &&
!result.program.bindings.UsesPushData() &&
shader_data != 0,
"oversized shader data did not use its storage fallback");
}
Require(test.name, "SPIR-V emit", !result.spirv.empty(),
"recompiler returned empty SPIR-V");
ValidateSpirv(test.name, result.spirv);
CheckSpirvText(test, result.spirv);
if (test.expected_buffer_resources) {
std::vector<u32> resources;
for (u32 i = 0; i < result.program.info.buffers.size(); ++i) {
if (result.program.info.buffers[i].descriptor_index != UINT32_MAX) resources.push_back(i);
}
Require(test.name, "live buffer bindings",
resources == *test.expected_buffer_resources,
"dead buffer changed the surviving descriptor order");
}
std::vector<u32> packed_user_data;
for (const auto reg : result.program.info.user_data_registers) {
packed_user_data.push_back(
resources.user_data[reg - result.program.user_data_base]);
}
packed_user_data.resize(result.program.bindings.ShaderDataDwords());
for (u32 resource = 0; resource < result.program.info.buffers.size(); ++resource) {
const auto index = result.program.info.buffers[resource].descriptor_index;
if (index == UINT32_MAX) continue;
const auto offset = resource < test.storage_buffer_offsets.size()
? test.storage_buffer_offsets[resource] : 0u;
Require(test.name, "shader data", offset < 256,
"storage buffer offset is not representable");
packed_user_data[result.program.bindings.memory_offset_dword + index / 4u] |=
offset << ((index % 4u) * 8u);
}
return {std::move(result.spirv), std::move(result.program),
std::move(resources), std::move(packed_user_data)};
}
std::array<u32, 64> MakeStructuredStorageBufferData(u32 stride_bytes,
u32 num_records,
bool add_tid = false,
u32 format = 0) {
std::array<u32, 64> data{};
data[1] = (stride_bytes & 0x3fffu) << 16u;
data[2] = num_records;
data[3] = DstSel(4, 5, 6, 7) | (1u << 24u) |
(stride_bytes == 0 ? 3u << 28u : 0u);
if (add_tid) {
data[3] |= 1u << 23u;
}
data[3] |= (format & 0x7fu) << 12u;
return data;
}
std::array<u32, 64> MakeStorageTextureData(Prospero::BufferFormat format) {
std::array<u32, 64> data{};
data[0] = 0x1000u;
data[1] = (static_cast<uint32_t>(format) & 0x1ffu) << 20u;
data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u;
return data;
}
std::string StorageUint2DImageBindingName(bool atomic) {
ShaderRecompiler::IR::ImageResource image{};
image.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
image.numeric_class = Prospero::TextureNumericClass::Uint;
image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
image.atomic = atomic;
const auto binding = ShaderRecompiler::IR::DescriptorBindingForImage(image);
Require("StorageUint2DImageBindingName", "descriptor binding", binding.has_value(),
"storage image has no descriptor binding");
return "image_" + std::to_string(static_cast<uint32_t>(*binding));
}
CompiledShader CompileFragmentCase(const GraphicsCase &test) {
const auto user_data =
MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr);
ShaderPixelInputInfo pixel_info{};
pixel_info.input_num =
test.pixel_interpolator_settings.empty()
? 1u
: static_cast<u32>(test.pixel_interpolator_settings.size());
pixel_info.ps_no_perspective = test.pixel_no_perspective;
pixel_info.parameter_mode = test.provoking_vtx_last ? ShaderPixelParameterMode::LastVertex
: ShaderPixelParameterMode::FirstVertex;
pixel_info.ps_ancillary = test.pixel_ancillary;
pixel_info.ps_front_face = test.pixel_front_face;
pixel_info.ps_pos_w = test.pixel_position_w;
pixel_info.ps_depth_export_enable = test.pixel_depth_export;
pixel_info.ps_system_input_base = 2;
pixel_info.ps_perspective_sample_vgpr = test.pixel_perspective_sample_vgpr;
pixel_info.ps_perspective_centroid_vgpr = test.pixel_perspective_centroid_vgpr;
pixel_info.custom_interpolation_mask = test.pixel_custom_interpolation_mask;
for (u32 i = 0; i < std::size(pixel_info.interpolator_settings); i++) {
pixel_info.interpolator_settings[i] = i;
}
for (u32 i = 0; i < test.pixel_interpolator_settings.size() &&
i < std::size(pixel_info.interpolator_settings);
i++) {
pixel_info.interpolator_settings[i] = test.pixel_interpolator_settings[i];
}
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Pixel;
options.dump_ir = false;
options.input_info.pixel = &pixel_info;
options.user_data = user_data;
auto translated =
ShaderRecompiler::TranslateProgram(test.fragment_code, options);
auto resource_plan =
ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot resources;
ShaderRecompiler::IR::ResourceSpecialization specialization;
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = options.user_data,
.shader_base = reinterpret_cast<uint64_t>(test.fragment_code.data()),
};
Require(test.name, "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
resource_plan, runtime, resources, specialization),
"translated resources could not be materialized");
auto result = ShaderRecompiler::CompileProgram(
std::move(translated), options, specialization);
Require(test.name, "SPIR-V emit", !result.spirv.empty(),
"recompiler returned empty SPIR-V");
ValidateSpirv(test.name, result.spirv);
std::vector<u32> packed_user_data;
for (const auto reg : result.program.info.user_data_registers) {
packed_user_data.push_back(
resources.user_data[reg - result.program.user_data_base]);
}
packed_user_data.resize(result.program.bindings.ShaderDataDwords());
return {std::move(result.spirv), std::move(result.program),
std::move(resources), std::move(packed_user_data)};
}
std::array<u32, 64> MakeSampledTextureData(Prospero::BufferFormat format) {
return MakeStorageTextureData(format);
}
namespace TestSpv {
std::vector<u32> MakePassthroughVertexSpirv(bool layered, float clip_w = 1.0f) {
using ShaderRecompiler::Spirv::Builder;
Builder b;
const auto void_type = b.Type(spv::OpTypeVoid);
const auto uint_type = b.Type(spv::OpTypeInt, 32, 0);
const auto int_type = b.Type(spv::OpTypeInt, 32, 1);
const auto float_type = b.Type(spv::OpTypeFloat, 32);
const auto vec2_type = b.Type(spv::OpTypeVector, float_type, 2);
const auto vec4_type = b.Type(spv::OpTypeVector, float_type, 4);
const auto per_vertex_type =
b.DecoratedType(spv::OpTypeStruct, {{spv::OpDecorate, {spv::DecorationBlock}},
{spv::OpMemberDecorate,
{0, spv::DecorationBuiltIn, spv::BuiltInPosition}}}, vec4_type);
const auto ptr_input_vec2 =
b.Type(spv::OpTypePointer, spv::StorageClassInput, vec2_type);
const auto ptr_input_vec4 =
b.Type(spv::OpTypePointer, spv::StorageClassInput, vec4_type);
const auto ptr_output_vec4 =
b.Type(spv::OpTypePointer, spv::StorageClassOutput, vec4_type);
const auto ptr_output_per_vertex =
b.Type(spv::OpTypePointer, spv::StorageClassOutput, per_vertex_type);
const auto func_type = b.Type(spv::OpTypeFunction, void_type);
const auto const_u32_0 = b.Constant(spv::OpConstant, uint_type, 0);
const auto const_f32_0 =
b.Constant(spv::OpConstant, float_type, 0x00000000u);
const auto const_f32_w =
b.Constant(spv::OpConstant, float_type, std::bit_cast<u32>(clip_w));
const auto in_pos =
b.DefineGlobalVariable(ptr_input_vec2, spv::StorageClassInput);
const auto in_color =
b.DefineGlobalVariable(ptr_input_vec4, spv::StorageClassInput);
const auto out_color =
b.DefineGlobalVariable(ptr_output_vec4, spv::StorageClassOutput);
const auto per_vertex =
b.DefineGlobalVariable(ptr_output_per_vertex, spv::StorageClassOutput);
u32 instance = 0;
u32 layer = 0;
std::vector<u32> interfaces = {in_pos, in_color, per_vertex, out_color};
if (layered) {
instance = b.DefineGlobalVariable(
b.Type(spv::OpTypePointer, spv::StorageClassInput, int_type),
spv::StorageClassInput);
layer = b.DefineGlobalVariable(
b.Type(spv::OpTypePointer, spv::StorageClassOutput, int_type),
spv::StorageClassOutput);
b.AddAnnotation(spv::OpDecorate, instance, spv::DecorationBuiltIn,
spv::BuiltInInstanceIndex);
b.AddAnnotation(spv::OpDecorate, layer, spv::DecorationBuiltIn, spv::BuiltInLayer);
b.RequireVersion(0x00010500u);
b.RequireCapability(spv::CapabilityShaderLayer);
interfaces.insert(interfaces.end(), {instance, layer});
}
const auto main = b.AllocateId();
const auto label = b.AllocateId();
const auto pos2 = b.AllocateId();
const auto color4 = b.AllocateId();
const auto pos_x = b.AllocateId();
const auto pos_y = b.AllocateId();
const auto position = b.AllocateId();
const auto position_ptr = b.AllocateId();
b.RequireCapability(spv::CapabilityShader);
b.AddMemoryModel(spv::AddressingModelLogical, spv::MemoryModelGLSL450);
b.AddEntryPoint(spv::ExecutionModelVertex, main, "main", interfaces);
b.AddAnnotation(spv::OpDecorate, in_pos, spv::DecorationLocation, 0);
b.AddAnnotation(spv::OpDecorate, in_color, spv::DecorationLocation, 1);
b.AddAnnotation(spv::OpDecorate, out_color, spv::DecorationLocation, 0);
b.AddFunction(spv::OpFunction, void_type, main, spv::FunctionControlMaskNone,
func_type);
b.AddFunction(spv::OpLabel, label);
b.AddFunction(spv::OpLoad, vec2_type, pos2, in_pos);
b.AddFunction(spv::OpLoad, vec4_type, color4, in_color);
b.AddFunction(spv::OpCompositeExtract, float_type, pos_x, pos2, 0);
b.AddFunction(spv::OpCompositeExtract, float_type, pos_y, pos2, 1);
b.AddFunction(spv::OpCompositeConstruct, vec4_type, position, pos_x, pos_y,
const_f32_0, const_f32_w);
b.AddFunction(spv::OpAccessChain, ptr_output_vec4, position_ptr, per_vertex,
const_u32_0);
b.AddFunction(spv::OpStore, position_ptr, position);
b.AddFunction(spv::OpStore, out_color, color4);
if (layered) {
const auto index = b.AllocateId();
b.AddFunction(spv::OpLoad, int_type, index, instance);
b.AddFunction(spv::OpStore, layer, index);
}
b.AddFunction(spv::OpReturn);
b.AddFunction(spv::OpFunctionEnd);
return b.Build();
}
} // namespace TestSpv
class VulkanHarness {
public:
VulkanHarness() { Init(); }
~VulkanHarness() { Destroy(); }
VulkanHarness(const VulkanHarness &) = delete;
VulkanHarness &operator=(const VulkanHarness &) = delete;
struct Buffer {
vk::Buffer buffer = nullptr;
vk::DeviceMemory memory = nullptr;
vk::DeviceAddress device_address = 0;
vk::DeviceSize size = 0;
bool coherent = false;
};
struct Image {
vk::Image image = nullptr;
vk::DeviceMemory memory = nullptr;
vk::ImageView view = nullptr;
vk::Format format = vk::Format::eUndefined;
vk::ImageLayout layout = vk::ImageLayout::eUndefined;
u32 width = 0;
u32 height = 0;
u32 layers = 1;
u32 mip_levels = 1;
u32 dwords_per_pixel = 0;
};
[[nodiscard]] vk::Device Device() const { return m_device; }
[[nodiscard]] u32 SubgroupSize() const {
vk::PhysicalDeviceSubgroupProperties subgroup{};
vk::PhysicalDeviceProperties2 properties{};
properties.pNext = &subgroup;
m_physical_device.getProperties2(&properties);
return subgroup.subgroupSize;
}
// A consumer GPU may lack the production rasterization features the fragment
// cases need. The compute cases still run on such a device, so the harness reports
// the capability instead of failing the whole binary before the first case.
[[nodiscard]] bool RasterizationSupported() const {
return m_rasterization_supported;
}
void SkipRasterizationCases(u32 count) { m_skipped_cases += count; }
[[nodiscard]] u32 SkippedCaseCount() const { return m_skipped_cases; }
[[nodiscard]] GraphicContext &RuntimeContext() {
EnsureRuntimeContext();
return m_runtime_context;
}
[[nodiscard]] RenderContext &RuntimeRenderer() {
EnsureRuntimeContext();
return Renderer();
}
void CheckHostImageAllocation() {
constexpr const char *name = "HostImageAllocation";
auto &graphics = RuntimeContext();
VulkanImage image;
vk::ImageCreateInfo create{};
create.flags = vk::ImageCreateFlagBits::eMutableFormat;
create.imageType = vk::ImageType::e2D;
create.format = vk::Format::eR8G8B8A8Unorm;
create.extent = {16, 8, 1};
create.mipLevels = 3;
create.arrayLayers = 4;
create.samples = vk::SampleCountFlagBits::e1;
create.tiling = vk::ImageTiling::eOptimal;
create.usage = vk::ImageUsageFlagBits::eTransferSrc |
vk::ImageUsageFlagBits::eTransferDst |
vk::ImageUsageFlagBits::eSampled;
create.initialLayout = vk::ImageLayout::eUndefined;
for (uint32_t pass = 0; pass < 2; ++pass) {
Require(name, "create", graphics.CreateImage(create, image),
"could not allocate native image");
Require(name, "allocation metadata",
image.image != nullptr && image.allocation != nullptr &&
image.format == create.format &&
image.image_type == create.imageType &&
image.extent == create.extent &&
image.layers == create.arrayLayers &&
image.mip_levels == create.mipLevels &&
image.samples == static_cast<uint32_t>(create.samples) &&
image.usage == create.usage && image.flags == create.flags,
"native image metadata did not follow its creation descriptor");
Require(name, "fresh synchronization state",
image.state.layout == create.initialLayout &&
image.state.access_mask == vk::AccessFlags2{} &&
image.state.pl_stage == vk::PipelineStageFlagBits2::eAllCommands &&
image.subresource_states.empty(),
"new allocation retained synchronization state from an old image");
VkMemoryPropertyFlags properties{};
vmaGetAllocationMemoryProperties(graphics.allocator, image.allocation,
&properties);
Require(name, "allocation policy",
(properties & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0,
"native image lost its device-local memory requirement");
// A reused presentation image may still describe its previous recording.
// No commands use this fixture; seed that bookkeeping before retirement.
image.state = {vk::PipelineStageFlagBits2::eTransfer,
vk::AccessFlagBits2::eTransferWrite,
vk::ImageLayout::eTransferDstOptimal};
image.subresource_states.assign(image.layers * image.mip_levels, image.state);
graphics.DeleteImage(image);
Require(name, "retirement",
image.image == nullptr && image.allocation == nullptr,
"retired native image retained an allocation handle");
create.flags = vk::ImageCreateFlagBits::e2DArrayCompatible;
create.imageType = vk::ImageType::e3D;
create.format = vk::Format::eR32Uint;
create.extent = {8, 4, 2};
create.mipLevels = 2;
create.arrayLayers = 1;
create.usage = vk::ImageUsageFlagBits::eTransferSrc |
vk::ImageUsageFlagBits::eTransferDst;
}
std::printf("[host] %-32s ok\n", name);
}
void CheckDescriptorHeapLargeSet() {
EnsureRuntimeContext();
std::array<vk::DescriptorSetLayoutBinding, 3> bindings{};
constexpr std::array stages{
vk::ShaderStageFlagBits::eVertex,
vk::ShaderStageFlagBits::eFragment,
vk::ShaderStageFlagBits::eCompute,
};
for (uint32_t i = 0; i < bindings.size(); i++) {
bindings[i].binding = i;
bindings[i].descriptorType = vk::DescriptorType::eSampler;
bindings[i].descriptorCount = 16;
bindings[i].stageFlags = stages[i];
}
vk::DescriptorSetLayoutCreateInfo create{};
create.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
create.bindingCount = static_cast<uint32_t>(bindings.size());
create.pBindings = bindings.data();
vk::DescriptorSetLayout layout = nullptr;
RequireVk("DescriptorHeapLargeSet", "layout",
m_device.createDescriptorSetLayout(&create, nullptr, &layout),
"vkCreateDescriptorSetLayout");
vk::PipelineLayoutCreateInfo pipeline_create{};
pipeline_create.sType = vk::StructureType::ePipelineLayoutCreateInfo;
pipeline_create.setLayoutCount = 1;
pipeline_create.pSetLayouts = &layout;
vk::PipelineLayout pipeline_layout = nullptr;
RequireVk("DescriptorHeapLargeSet", "pipeline layout",
m_device.createPipelineLayout(&pipeline_create, nullptr,
&pipeline_layout),
"vkCreatePipelineLayout");
{
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
const auto commit = [&] {
const auto set = context.GetDescriptorHeap().Commit(layout);
Require("DescriptorHeapLargeSet", "allocation", set != nullptr,
"ordinary descriptor heap could not allocate an oversized sampler set");
scheduler.Current().Handle().bindDescriptorSets(
vk::PipelineBindPoint::eCompute, pipeline_layout, 0, 1, &set, 0,
nullptr);
};
for (uint32_t i = 0; i < 22; i++) {
commit();
}
scheduler.Finish();
for (uint32_t i = 0; i < 21; i++) {
commit();
}
}
m_device.destroyPipelineLayout(pipeline_layout, nullptr);
m_device.destroyDescriptorSetLayout(layout, nullptr);
std::printf("[host] %-32s ok\n", "DescriptorHeapLargeSet");
}
void CheckGraphicsPushConstantBank() {
constexpr const char *name = "GraphicsPushConstantStages";
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
ShaderRecompiler::IR::CompiledShaderInfo vertex_program{};
vertex_program.stage = ShaderType::Vertex;
vertex_program.bindings.push_data_start_dword = 2;
vertex_program.info.user_data_registers = {0, 1};
vertex_program.bindings.memory_offset_dword = 2;
ShaderRecompiler::IR::CompiledShaderInfo pixel_program{};
pixel_program.stage = ShaderType::Pixel;
pixel_program.bindings.push_data_start_dword = 0;
pixel_program.info.user_data_registers = {0, 1};
pixel_program.bindings.memory_offset_dword = 2;
ShaderRecompiler::IR::ResourceSnapshot vertex_snapshot;
vertex_snapshot.user_data = {0x11111111u, 0x22222222u};
ShaderStageRuntime vertex_runtime{&vertex_program, &vertex_snapshot};
ShaderRecompiler::IR::ResourceSnapshot pixel_snapshot;
pixel_snapshot.user_data = {0x33333333u, 0x44444444u};
ShaderStageRuntime pixel_runtime{&pixel_program, &pixel_snapshot};
PreparedBindings vertex;
PreparedBindings pixel;
auto &executor = context.GetRenderExecutor();
executor.PrepareBindings(vertex_runtime, vertex);
executor.PrepareBindings(pixel_runtime, pixel);
const auto *vertex_storage = vertex.shader_data.data();
const auto *pixel_storage = pixel.shader_data.data();
vertex.buffer_sources.push_back({});
vertex.buffers.push_back({});
vertex.gds.offset = 1;
vertex.flattened_srt.offset = 1;
vertex.shader_data_buffer.offset = 1;
vertex_snapshot.user_data = {0x55555555u, 0x66666666u};
pixel_snapshot.user_data = {0x77777777u, 0x88888888u};
executor.PrepareBindings(vertex_runtime, vertex);
executor.PrepareBindings(pixel_runtime, pixel);
std::array<PreparedBindings *, 2> stages{&vertex, &pixel};
RenderExecutorTestAccess::PrepareGraphicsBindings(executor, stages, {});
Require(name, "prepared binding scratch",
vertex.shader_data.data() == vertex_storage &&
pixel.shader_data.data() == pixel_storage &&
vertex.buffer_sources.empty() && vertex.buffers.empty() &&
vertex.gds.offset == 0 && vertex.flattened_srt.offset == 0 &&
vertex.shader_data_buffer.offset == 0,
"repeated shader binding preparation allocated or retained stale state");
const auto pipeline = RenderExecutorTestAccess::CommitBindings(
context.GetRenderExecutor(), scheduler.Current(), vertex, pixel);
Require(name, "shared push data",
vertex.shader_data ==
std::vector<uint32_t>{0x55555555u, 0x66666666u} &&
pixel.shader_data ==
std::vector<uint32_t>{0x77777777u, 0x88888888u},
"graphics stages did not commit their shared push data");
scheduler.Finish();
RenderExecutorTestAccess::DestroyDescriptorPipelines(
context.GetRenderExecutor(), std::span {&pipeline, 1u});
std::printf("[host] %-32s ok\n", name);
}
void CheckSchedulerTimeline() {
EnsureRuntimeContext();
CommandScheduler scheduler(Renderer(), m_runtime_context);
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto *command = &scheduler.Current();
const auto original_handle = command->Handle();
HW::Context next_registers{};
HW::UserConfig next_user_config{};
HW::Shader next_shaders{};
scheduler.Begin(next_registers, next_user_config, next_shaders);
Require("SchedulerTimeline", "guest context rebind",
&scheduler.Current() == command &&
command->Handle() == original_handle &&
&command->GetRegisters() == &next_registers &&
&command->GetUserConfig() == &next_user_config &&
&command->GetShaders() == &next_shaders,
"rebinding guest state replaced the active command recording");
scheduler.FlushAndWait();
Require("SchedulerTimeline", "guest context after recording restart",
scheduler.Active() && &scheduler.Current() == command &&
&command->GetRegisters() == &next_registers &&
&command->GetUserConfig() == &next_user_config &&
&command->GetShaders() == &next_shaders,
"restarting command recording lost the current guest context");
scheduler.Begin(registers, user_config, shaders);
const auto first_tick = scheduler.CurrentTick();
uint32_t completed = 0;
scheduler.DeferOperation([owned = std::make_unique<uint32_t>(1),
&completed] { completed = *owned; });
scheduler.Flush();
scheduler.Wait(first_tick);
Require("SchedulerTimeline", "wait boundary",
scheduler.CurrentTick() == first_tick + 1 &&
scheduler.IsFree(first_tick) && completed == 0,
"timeline wait released a resource in the middle of an operation");
scheduler.PopPendingOperations();
Require("SchedulerTimeline", "first tick", completed == 1,
"operation boundary did not release its deferred operation");
const auto second_tick = scheduler.CurrentTick();
std::atomic<bool> priority_in_callback{false};
scheduler.DeferPriorityOperation([owned = std::make_unique<uint32_t>(2),
&completed, &priority_in_callback] {
priority_in_callback = CommandScheduler::InDeferredOperation();
completed = *owned;
});
scheduler.Finish();
scheduler.DrainPriorityOperations();
Require("SchedulerTimeline", "finish",
scheduler.CurrentTick() == second_tick + 1 &&
scheduler.IsFree(second_tick) && completed == 2 &&
priority_in_callback.load() &&
!CommandScheduler::InDeferredOperation(),
"scheduler completion drain lost its tick or callback context");
const auto implicit_flush_tick = scheduler.CurrentTick();
scheduler.DeferOperation([owned = std::make_unique<uint32_t>(3),
&completed] { completed = *owned; });
scheduler.Wait(implicit_flush_tick);
Require("SchedulerTimeline", "implicit flush",
scheduler.CurrentTick() == implicit_flush_tick + 1 &&
scheduler.IsFree(implicit_flush_tick) && completed == 2,
"waiting for the current tick released a deferred resource");
scheduler.PopPendingOperations();
Require("SchedulerTimeline", "implicit boundary", completed == 3,
"operation boundary did not release the implicit tick callback");
vk::SemaphoreTypeCreateInfo timeline_type{};
timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo;
timeline_type.semaphoreType = vk::SemaphoreType::eTimeline;
vk::SemaphoreCreateInfo timeline_create{};
timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo;
timeline_create.pNext = &timeline_type;
vk::Semaphore external_timeline = nullptr;
Require("SchedulerTimeline", "external timeline create",
m_runtime_context.device.createSemaphore(&timeline_create, nullptr,
&external_timeline) ==
vk::Result::eSuccess &&
external_timeline != nullptr,
"failed to create the external timeline semaphore");
constexpr uint64_t external_wait_value = 9;
const auto external_wait_tick = scheduler.CurrentTick();
SubmitInfo external_wait;
external_wait.AddWait(external_timeline, external_wait_value);
constexpr size_t blocked_submission_count = 6;
std::array<vk::CommandBuffer, blocked_submission_count> blocked_handles{};
for (size_t i = 0; i < blocked_submission_count; ++i) {
blocked_handles[i] = scheduler.Current().Handle();
if (i == 0) {
scheduler.Flush(external_wait);
} else {
scheduler.Flush();
}
}
bool distinct_handles = true;
for (size_t i = 0; i < blocked_handles.size(); ++i) {
for (size_t j = i + 1; j < blocked_handles.size(); ++j) {
distinct_handles &= blocked_handles[i] != blocked_handles[j];
}
}
const auto last_blocked_tick = scheduler.CurrentTick() - 1;
Require("SchedulerTimeline", "external timeline overflow",
distinct_handles &&
last_blocked_tick ==
external_wait_tick + blocked_submission_count - 1 &&
!scheduler.IsFree(last_blocked_tick),
"blocked submissions did not grow the command-buffer pool");
vk::SemaphoreSignalInfo signal_info{};
signal_info.sType = vk::StructureType::eSemaphoreSignalInfo;
signal_info.semaphore = external_timeline;
signal_info.value = external_wait_value;
Require("SchedulerTimeline", "external timeline signal",
m_runtime_context.device.signalSemaphore(&signal_info) ==
vk::Result::eSuccess,
"failed to signal the external timeline semaphore");
scheduler.Wait(last_blocked_tick);
m_runtime_context.device.destroySemaphore(external_timeline, nullptr);
scheduler.Flush();
scheduler.Flush();
Require(
"SchedulerTimeline", "timeline pool reuse",
std::ranges::find(blocked_handles, scheduler.Current().Handle()) !=
blocked_handles.end(),
"completed command buffers were not reused after timeline progress");
scheduler.Shutdown();
CommandScheduler draining(Renderer(), m_runtime_context);
HW::Context drain_registers{};
HW::UserConfig drain_user_config{};
HW::Shader drain_shaders{};
draining.Begin(drain_registers, drain_user_config, drain_shaders);
uint32_t reentrant = 0;
draining.DeferOperation([&draining, &reentrant] {
reentrant++;
draining.DeferOperation([&reentrant] { reentrant++; });
});
std::binary_semaphore priority_entered{0};
std::binary_semaphore release_priority{0};
draining.DeferPriorityOperation(
[&draining, &priority_entered, &release_priority] {
priority_entered.release();
draining.Shutdown();
release_priority.acquire();
});
std::jthread shutdown_thread([&draining] { draining.Shutdown(); });
priority_entered.acquire();
std::jthread second_shutdown_thread([&draining] { draining.Shutdown(); });
std::atomic<bool> concurrent_completed{false};
std::jthread concurrent_defer([&draining, &concurrent_completed] {
draining.DeferOperation(
[&concurrent_completed] { concurrent_completed = true; });
});
release_priority.release();
shutdown_thread.join();
second_shutdown_thread.join();
concurrent_defer.join();
Require("SchedulerTimeline", "shutdown drain",
reentrant == 2 && concurrent_completed.load(),
"shutdown lost reentrant or concurrent deferred work");
std::printf("[host] %-32s ok\n", "SchedulerTimeline");
}
void CheckGpuMappedRangeLifecycle() {
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.InitializeGpu(nullptr);
auto &resources = context;
constexpr uint64_t base = 0x0000000200000000ull;
constexpr uint64_t page = 0x4000;
const auto cpu_only_tick = scheduler.CurrentTick();
resources.MapMemory(base, page * 4);
resources.MapMemory(base + page * 2, page * 4);
Require("GpuMappedRangeLifecycle", "union",
resources.IsMapped(base, page * 6) &&
!resources.IsMapped(base, page * 7),
"overlapping maps did not form one interval union");
resources.UnmapMemory(base + page * 2, page * 2);
Require("GpuMappedRangeLifecycle", "subtract",
resources.IsMapped(base, page * 2) &&
resources.IsMapped(base + page * 4, page * 2) &&
!resources.IsMapped(base, page * 6),
"partial unmap did not punch the expected interval hole");
resources.UnmapMemory(base + page * 2, page * 2);
Require("GpuMappedRangeLifecycle", "idempotent unmap",
resources.IsMapped(base, page * 2) &&
resources.IsMapped(base + page * 4, page * 2),
"unmapping an absent interval changed neighboring mappings");
resources.UnmapMemory(base, page * 6);
Require("GpuMappedRangeLifecycle", "clear",
!resources.IsMapped(base, page * 6),
"full unmap did not clear the interval union");
constexpr uint64_t old_prt = base + page * 8;
constexpr uint64_t new_prt = base + page * 16;
resources.MapMemory(old_prt, page * 4);
resources.UnmapMemory(old_prt, page * 4);
resources.MapMemory(new_prt, page * 6);
Require("GpuMappedRangeLifecycle", "PRT replacement",
!resources.IsMapped(old_prt, page * 4) &&
resources.IsMapped(new_prt, page * 6) &&
scheduler.CurrentTick() == cpu_only_tick,
"CPU-only unmaps submitted GPU work or lost full PRT coverage");
scheduler.Finish();
context.ShutdownGpu();
std::printf("[host] %-32s ok\n", "GpuMappedRangeLifecycle");
}
void CheckStreamBufferRing() {
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &cache = resources.GetBufferCache();
constexpr std::array<std::pair<MemoryUsage, uint64_t>, 4> utilities{{
{MemoryUsage::Upload, 512ull << 20},
{MemoryUsage::Stream, 64ull << 20},
{MemoryUsage::Download, 64ull << 20},
{MemoryUsage::DeviceLocal, 128ull << 20},
}};
std::array<vk::Buffer, utilities.size()> handles{};
for (size_t i = 0; i < utilities.size(); i++) {
auto &utility = cache.GetUtilityBuffer(utilities[i].first);
handles[i] = utility.Handle();
Require("StreamBufferRing", "utility ownership",
utility.Usage() == utilities[i].first &&
utility.Size() == utilities[i].second &&
utility.Handle() != nullptr,
"BufferCache utility buffer does not match the fixed layout");
}
for (size_t i = 0; i < handles.size(); i++) {
for (size_t j = i + 1; j < handles.size(); j++) {
Require("StreamBufferRing", "utility identity",
handles[i] != handles[j],
"utility usages alias the same native buffer");
}
}
auto *fixed_download = &cache.GetUtilityBuffer(MemoryUsage::Download);
const auto fixed_handle = fixed_download->Handle();
const auto [oversized_download, oversized_offset] =
fixed_download->Map(fixed_download->Size() + 4, 64, false);
const auto [download_probe, download_probe_offset] =
fixed_download->Map(64, 64, false);
Require("StreamBufferRing", "fixed download utility",
oversized_download == nullptr && oversized_offset == 0 &&
download_probe != nullptr && download_probe_offset == 0 &&
&cache.GetUtilityBuffer(MemoryUsage::Download) ==
fixed_download &&
fixed_download->Size() == (64ull << 20) &&
fixed_download->Handle() == fixed_handle,
"oversized download replaced or corrupted the fixed shared ring");
fixed_download->Commit();
const std::array<uint8_t, 16> overflow_data{
0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5,
0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5};
auto &upload_stream = cache.GetUtilityBuffer(MemoryUsage::Stream);
const auto stream_offset =
upload_stream.Copy(overflow_data.data(), overflow_data.size(), 16);
Require("StreamBufferRing", "cache-owned stream upload",
upload_stream.Handle() != nullptr &&
stream_offset < upload_stream.Size() &&
std::memcmp(upload_stream.Mapped().data() + stream_offset,
overflow_data.data(), overflow_data.size()) == 0,
"cache-owned stream upload lost bytes");
const auto atom =
m_runtime_context.physical_device_properties.limits.nonCoherentAtomSize;
uint64_t policy_size = 17;
uint64_t policy_alignment = 12;
uint64_t coherent_size = policy_size;
uint64_t coherent_alignment = policy_alignment;
Require("StreamBufferRing", "non-coherent reservation policy",
StreamBufferTestAccess::NormalizeReservation(
false, 256, policy_size, policy_alignment) &&
policy_size == 256 && policy_alignment == 768 &&
StreamBufferTestAccess::NormalizeReservation(
true, 256, coherent_size, coherent_alignment) &&
coherent_size == 17 && coherent_alignment == 12,
"non-coherent reservations are not isolated at the atom/texel LCM");
const auto small_ring_size = std::max<uint64_t>(64, atom * 2);
StreamBuffer stream(m_runtime_context, scheduler, MemoryUsage::Upload,
small_ring_size);
const auto first_size =
stream.IsCoherent() ? small_ring_size - 16 : small_ring_size - atom + 1;
constexpr uint64_t blocked_size = 24;
const auto [first, first_offset] = stream.Map(first_size, 16, false);
Require("StreamBufferRing", "first allocation",
first != nullptr && first_offset == 0,
"initial stream allocation failed");
std::memset(first, 0x5a, static_cast<size_t>(first_size));
stream.Commit();
const auto [oversize, oversize_offset] =
stream.Map(small_ring_size + 1, 1, false);
Require("StreamBufferRing", "oversize",
oversize == nullptr && oversize_offset == 0,
"oversize allocation was accepted");
const auto [blocked, blocked_offset] = stream.Map(blocked_size, 1, false);
const auto [blocked_again, blocked_again_offset] =
stream.Map(blocked_size, 1, false);
Require("StreamBufferRing", "transactional failure",
blocked == nullptr && blocked_offset == 0 &&
blocked_again == nullptr && blocked_again_offset == 0,
"failed wrap mutated stream state");
scheduler.Flush();
const auto [wrapped, wrapped_offset] = stream.Map(blocked_size, 1);
Require("StreamBufferRing", "watched wrap",
wrapped != nullptr && wrapped_offset == 0,
"completed watch did not release the wrapped allocation");
stream.Commit();
const auto download_alignment = atom / std::gcd<uint64_t>(atom, 12) * 12;
const auto download_ring_size = std::max<uint64_t>(
64, download_alignment + std::max<uint64_t>(atom, 16));
StreamBuffer download(m_runtime_context, scheduler, MemoryUsage::Download,
download_ring_size);
const auto [download_data, download_offset] = download.Map(16, 8);
Require("StreamBufferRing", "download allocation",
download_data != nullptr && download_offset == 0,
"download stream is not host visible");
download.Commit();
scheduler.Flush();
const auto [second_download, second_download_offset] = download.Map(16, 12);
Require("StreamBufferRing", "download atom isolation",
second_download != nullptr &&
(download.IsCoherent() ||
(download_offset % atom == 0 &&
second_download_offset % atom == 0 &&
second_download_offset >= download_offset + atom)) &&
second_download_offset % 12 == 0,
"download reservations share a non-coherent atom or lost caller "
"alignment");
download.Commit();
scheduler.Finish();
download.Invalidate(download_offset, 16);
std::printf("[host] %-32s ok\n", "StreamBufferRing");
}
void CheckGpuSuspendPoint() {
constexpr const char *name = "GpuSuspendPoint";
EnsureRuntimeContext();
auto &context = Renderer();
context.InitializeGpu(nullptr);
auto &gpu = context.GetGpu();
auto &scheduler = context.GetCommandScheduler();
CommandProcessor setup(context, 0);
vk::SemaphoreTypeCreateInfo timeline_type{};
timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo;
timeline_type.semaphoreType = vk::SemaphoreType::eTimeline;
vk::SemaphoreCreateInfo timeline_create{};
timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo;
timeline_create.pNext = &timeline_type;
vk::Semaphore gate = nullptr;
Require(name, "timeline create",
m_runtime_context.device.createSemaphore(
&timeline_create, nullptr, &gate) == vk::Result::eSuccess,
"failed to create the native completion gate");
const auto signal = [&](uint64_t value) {
vk::SemaphoreSignalInfo info{};
info.sType = vk::StructureType::eSemaphoreSignalInfo;
info.semaphore = gate;
info.value = value;
Require(name, "timeline signal",
m_runtime_context.device.signalSemaphore(&info) ==
vk::Result::eSuccess,
"failed to release native GPU work");
};
const auto finish = [&] {
gpu.WaitForIdle();
gpu.SendCommandSync([&] {
scheduler.Finish();
scheduler.WaitPriorityOperations(scheduler.CurrentTick() - 1);
});
};
const auto write = [](uint32_t *packet, uint32_t *destination,
uint32_t value, bool predicated = false) {
const auto address = reinterpret_cast<uint64_t>(destination);
packet[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0) | uint32_t{predicated};
packet[1] = 0;
packet[2] = static_cast<uint32_t>(address);
packet[3] = static_cast<uint32_t>(address >> 32u);
packet[4] = value;
};
gpu.SendCommandSync([&] {
setup.BufferInit();
SubmitInfo blocked;
blocked.AddWait(gate, 1);
scheduler.Flush(blocked);
});
std::binary_semaphore first_returned{0};
std::jthread first([&] {
gpu.SuspendPoint();
first_returned.release();
});
const bool first_nonblocking =
first_returned.try_acquire_for(std::chrono::seconds(2));
if (!first_nonblocking) {
signal(1);
}
first.join();
Require(name, "first call", first_nonblocking,
"the first suspend point waited for native GPU completion");
// Let the marker reach Vulkan while its completion remains gated.
std::binary_semaphore marker_parsed{0};
std::jthread parser([&] {
gpu.WaitForIdle();
marker_parsed.release();
});
const bool parser_nonblocking =
marker_parsed.try_acquire_for(std::chrono::seconds(2));
if (!parser_nonblocking) {
signal(1);
}
parser.join();
Require(name, "marker insertion", parser_nonblocking,
"inserting a suspend point blocked the GPU command thread");
std::binary_semaphore second_entered{0};
std::binary_semaphore second_returned{0};
std::jthread second([&] {
second_entered.release();
gpu.SuspendPoint();
second_returned.release();
});
second_entered.acquire();
const bool second_overtook_completion =
second_returned.try_acquire_for(std::chrono::milliseconds(100));
uint32_t compute_value = 0;
std::array<uint32_t, 5> compute_commands{};
write(compute_commands.data(), &compute_value, 33);
bool host_lane_responded = false;
std::binary_semaphore compute_progress{0};
std::jthread compute([&] {
gpu.SubmitCompute(0x20, compute_commands);
gpu.SendCommandSync([&] { host_lane_responded = true; });
gpu.WaitForIdle();
compute_progress.release();
});
const bool compute_did_not_wait =
compute_progress.try_acquire_for(std::chrono::seconds(2));
signal(1);
second.join();
compute.join();
finish();
Require(name, "one outstanding point", !second_overtook_completion,
"a second suspend point returned before the first native completion");
Require(name, "independent queue progress",
compute_did_not_wait && host_lane_responded && compute_value == 33,
"waiting for a previous suspend point blocked compute submissions "
"or the host command lane");
Require(name, "frame count", gpu.GetFrameNum() == 2,
"suspend-point insertion did not advance the frame counter once");
// A marker must stay behind a stalled graphics stream, then clear its
// predicate before the next stream. Async compute releases that stream.
alignas(16) uint64_t predicate = 0;
uint32_t label = 0;
uint32_t prefix = 0;
uint32_t skipped = 0;
uint32_t after_reset = 0;
const auto predicate_address = reinterpret_cast<uint64_t>(&predicate);
const auto label_address = reinterpret_cast<uint64_t>(&label);
std::array<uint32_t, 21> before{};
before[0] = KYTY_PM4(4, Pm4::IT_SET_PREDICATION, 0);
before[1] = (3u << 16u) | (1u << 8u);
before[2] = static_cast<uint32_t>(predicate_address);
before[3] = static_cast<uint32_t>(predicate_address >> 32u);
write(before.data() + 4, &skipped, 11, true);
write(before.data() + 9, &prefix, 22);
before[14] = KYTY_PM4(7, Pm4::IT_WAIT_REG_MEM, 0);
before[15] = 0x10u | 3u;
before[16] = static_cast<uint32_t>(label_address);
before[17] = static_cast<uint32_t>(label_address >> 32u);
before[18] = 1;
before[19] = UINT32_MAX;
std::array<uint32_t, 5> after{};
write(after.data(), &after_reset, 44, true);
std::binary_semaphore lane_entered{0};
std::binary_semaphore release_lane{0};
gpu.SendCommand([&] {
lane_entered.release();
release_lane.acquire();
});
lane_entered.acquire();
gpu.Submit(before, {});
std::binary_semaphore queued_returned{0};
std::jthread queued([&] {
gpu.SuspendPoint();
queued_returned.release();
});
const bool queued_nonblocking =
queued_returned.try_acquire_for(std::chrono::seconds(2));
if (!queued_nonblocking) {
release_lane.release();
gpu.SendCommandSync([&] { label = 1; });
queued.join();
finish();
Require(name, "queued insertion", false,
"suspend-point insertion drained pending guest graphics");
}
queued.join();
gpu.Submit(after, {});
release_lane.release();
bool prefix_reached = false;
bool graphics_ordered = false;
const auto deadline = std::chrono::steady_clock::now() +
std::chrono::seconds(2);
do {
gpu.SendCommandSync([&] {
prefix_reached = prefix == 22;
graphics_ordered = skipped == 0 && after_reset == 0;
});
std::this_thread::yield();
} while (!prefix_reached && std::chrono::steady_clock::now() < deadline);
std::array<uint32_t, 5> release_graphics{};
write(release_graphics.data(), &label, 1);
gpu.SubmitCompute(0x20, release_graphics);
finish();
Require(name, "graphics FIFO",
prefix_reached && graphics_ordered && skipped == 0,
"suspend point or later graphics overtook a stalled graphics stream");
Require(name, "graphics reset", after_reset == 44,
"suspend point preserved the previous frame's predicate");
// Delay the completion callback beyond the owner's shutdown when possible.
// Its credit must remain alive independently of the GuestGpu object.
std::binary_semaphore priority_entered{0};
std::binary_semaphore release_priority{0};
gpu.SendCommandSync([&] {
scheduler.DeferPriorityOperation([&] {
priority_entered.release();
release_priority.acquire();
});
SubmitInfo blocked;
blocked.AddWait(gate, 2);
scheduler.Flush(blocked);
});
gpu.SuspendPoint();
gpu.WaitForIdle();
std::binary_semaphore shutdown_returned{0};
std::jthread shutdown([&] {
context.ShutdownGpu();
shutdown_returned.release();
});
const bool shutdown_overtook_native =
shutdown_returned.try_acquire_for(std::chrono::milliseconds(100));
signal(2);
priority_entered.acquire();
if (!shutdown_overtook_native) {
(void)shutdown_returned.try_acquire_for(std::chrono::seconds(2));
}
release_priority.release();
shutdown.join();
scheduler.DrainPriorityOperations();
m_runtime_context.device.destroySemaphore(gate, nullptr);
Require(name, "shutdown native completion", !shutdown_overtook_native,
"shutdown returned while the suspend point was still executing");
std::printf("[host] %-32s ok\n", name);
}
void CheckGpuCommandLane() {
EnsureRuntimeContext();
auto &context = Renderer();
context.InitializeGpu(nullptr);
auto &gpu = context.GetGpu();
const auto caller_thread = std::this_thread::get_id();
std::thread::id gpu_thread;
std::thread::id nested_thread;
uint32_t order = 0;
gpu.SendCommand([&order] { order = 1; });
gpu.SendCommandSync([&] {
gpu_thread = std::this_thread::get_id();
Require("GpuCommandLane", "FIFO", order == 1,
"synchronous command overtook an older host command");
Require("GpuCommandLane", "GPU context", GuestGpu::IsGpuThread(),
"host command did not run on the GPU thread");
gpu.SendCommandSync(
[&nested_thread] { nested_thread = std::this_thread::get_id(); });
order = 2;
});
Require(
"GpuCommandLane", "dispatch thread",
order == 2 && gpu_thread != caller_thread &&
nested_thread == gpu_thread,
"host command did not run on the GPU thread or nested sync deadlocked");
{
const auto make_release_mem =
[](uint32_t data_sel, uint32_t interrupt_selector, void *destination,
uint64_t value, uint32_t action = 0x28u,
uint32_t gcr_cntl = 1u << 9u) {
std::array<uint32_t, 8> packet{};
const auto address = reinterpret_cast<uint64_t>(destination);
const auto event_index = (action >= 0x2fu ? 6u : 5u);
packet[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM);
packet[1] = action | (event_index << 8u) | (gcr_cntl << 12u);
packet[2] = (interrupt_selector << 24u) | (data_sel << 29u);
packet[3] = static_cast<uint32_t>(address);
packet[4] = static_cast<uint32_t>(address >> 32u);
packet[5] = static_cast<uint32_t>(value);
packet[6] = static_cast<uint32_t>(value >> 32u);
packet[7] = 0x123u;
return packet;
};
vk::SemaphoreTypeCreateInfo timeline_type{};
timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo;
timeline_type.semaphoreType = vk::SemaphoreType::eTimeline;
vk::SemaphoreCreateInfo timeline_create{};
timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo;
timeline_create.pNext = &timeline_type;
vk::Semaphore blocked_timeline = nullptr;
Require("GpuCommandLane", "blocked timeline create",
m_runtime_context.device.createSemaphore(
&timeline_create, nullptr, &blocked_timeline) ==
vk::Result::eSuccess &&
blocked_timeline != nullptr,
"failed to create the blocked GPU timeline");
constexpr uint64_t release_tick = 1;
auto &gpu_scheduler = context.GetCommandScheduler();
auto processor = std::make_unique<CommandProcessor>(context, 0);
gpu.SendCommandSync([&] {
processor->BufferInit();
SubmitInfo blocked_submit;
blocked_submit.AddWait(blocked_timeline, release_tick);
gpu_scheduler.Flush(blocked_submit);
});
std::binary_semaphore parser_complete{0};
alignas(uint64_t) uint64_t interrupt_only_gds_label = UINT64_MAX;
alignas(uint64_t) uint64_t cb_db_release_label = UINT64_MAX;
bool parser_kept_nonblocking_boundaries = false;
bool host_resource_retired = false;
bool cpu_unmap_did_not_submit = false;
std::jthread no_gpu_wait([&] {
gpu.SendCommandSync([&] {
processor->BufferInit();
const auto tick_before = gpu_scheduler.CurrentTick();
gpu_scheduler.DeferOperation([&] { host_resource_retired = true; });
constexpr uint64_t cpu_only_address = 0x0000000200400000ull;
context.MapMemory(cpu_only_address, 0x4000);
context.UnmapMemory(cpu_only_address, 0x4000);
cpu_unmap_did_not_submit =
gpu_scheduler.CurrentTick() == tick_before &&
!host_resource_retired &&
!context.IsMapped(cpu_only_address, 0x4000);
processor->TriggerEvent(0x16u, 0u);
auto packet = make_release_mem(0, 0, nullptr, 0);
Pm4Execution execution;
const auto result = processor->Process(execution, packet);
processor->IncrementDe();
const bool cache_and_counter_did_not_submit =
result == Pm4ProcessResult::Complete &&
gpu_scheduler.CurrentTick() == tick_before;
const auto cb_db_tick = gpu_scheduler.CurrentTick();
auto cb_db_release =
make_release_mem(2, 0, &cb_db_release_label, 0, 0x14u, 0);
Pm4Execution cb_db_execution;
const auto cb_db_result =
processor->Process(cb_db_execution, cb_db_release);
const bool cb_db_release_did_not_submit =
cb_db_result == Pm4ProcessResult::Complete &&
cb_db_release_label == 0 &&
gpu_scheduler.CurrentTick() == cb_db_tick;
for (const auto gcr : {0u, 1u << 9u}) {
const auto before = Sync::ReadReferenceClock();
auto timestamp =
make_release_mem(3, 0, &cb_db_release_label, UINT64_MAX, 0x14u, gcr);
Pm4Execution timestamp_execution;
const auto timestamp_result =
processor->Process(timestamp_execution, timestamp);
Require("GpuCommandLane", "timestamp write",
timestamp_result == Pm4ProcessResult::Complete &&
cb_db_release_label >= before &&
cb_db_release_label <= Sync::ReadReferenceClock() &&
gpu_scheduler.CurrentTick() == cb_db_tick,
"timestamp write failed, returned an invalid time, or "
"submitted extra GPU work");
}
auto gds_interrupt_only =
make_release_mem(5, 1, &interrupt_only_gds_label, 1ull << 16u);
Pm4Execution gds_interrupt_execution;
const auto interrupt_tick = gpu_scheduler.CurrentTick();
const auto interrupt_result =
processor->Process(gds_interrupt_execution, gds_interrupt_only);
parser_kept_nonblocking_boundaries =
cache_and_counter_did_not_submit &&
cb_db_release_did_not_submit &&
interrupt_result == Pm4ProcessResult::Complete &&
gpu_scheduler.CurrentTick() == interrupt_tick + 1 &&
interrupt_only_gds_label == UINT64_MAX;
});
parser_complete.release();
});
const bool parser_did_not_wait_gpu =
parser_complete.try_acquire_for(std::chrono::seconds(2));
vk::SemaphoreSignalInfo signal_info{};
signal_info.sType = vk::StructureType::eSemaphoreSignalInfo;
signal_info.semaphore = blocked_timeline;
signal_info.value = release_tick;
Require("GpuCommandLane", "blocked timeline signal",
m_runtime_context.device.signalSemaphore(&signal_info) ==
vk::Result::eSuccess,
"failed to release the blocked GPU timeline");
no_gpu_wait.join();
gpu.SendCommandSync([&] { gpu_scheduler.Finish(); });
m_runtime_context.device.destroySemaphore(blocked_timeline, nullptr);
Require("GpuCommandLane", "CPU-only unmap with pending host retirement",
parser_did_not_wait_gpu && cpu_unmap_did_not_submit &&
host_resource_retired,
"CPU-only unmap submitted or waited for unrelated native work");
Require("GpuCommandLane", "nonblocking GPU packets",
parser_did_not_wait_gpu && parser_kept_nonblocking_boundaries,
"a cache event, GL2 writeback, DE counter, or interrupt boundary "
"used the wrong "
"submission behavior");
alignas(uint64_t) uint64_t release_label = 0;
alignas(uint64_t) uint64_t gds_label = UINT64_MAX;
bool release_mem_submission_counts = false;
gpu.SendCommandSync([&] {
processor->BufferInit();
const auto clock_before = Sync::ReadReferenceClock();
processor->WriteAtEndOfPipe64(0, 0, 0x04, 0x38, 5, 4,
&release_label, UINT64_MAX, 2);
Require("GpuCommandLane", "EOP reference clock with interrupt",
release_label >= clock_before &&
release_label <= Sync::ReadReferenceClock(),
"clock write with writeback and interrupt lost its data");
for (const auto interrupt : {0u, 3u, 1u, 2u}) {
release_label = 0;
auto immediate = make_release_mem(1, interrupt, &release_label, 0x11223344u);
Pm4Execution immediate_execution;
const auto immediate_tick = gpu_scheduler.CurrentTick();
const auto immediate_result =
processor->Process(immediate_execution, immediate);
Require("GpuCommandLane", "32-bit release boundary",
immediate_result == Pm4ProcessResult::Complete &&
gpu_scheduler.CurrentTick() ==
immediate_tick + (interrupt == 1 || interrupt == 2) &&
release_label == (interrupt == 1 ? 0 : 0x11223344u),
"label-only release submitted work or interrupt release lost its boundary");
}
auto gds = make_release_mem(5, 0, &gds_label, 1ull << 16u);
Pm4Execution gds_execution;
const auto gds_tick = gpu_scheduler.CurrentTick();
const auto gds_result = processor->Process(gds_execution, gds);
const bool gds_waited_once =
gpu_scheduler.CurrentTick() == gds_tick + 1;
auto interrupt_only = make_release_mem(0, 4, nullptr, 0);
Pm4Execution interrupt_execution;
const auto interrupt_tick = gpu_scheduler.CurrentTick();
const auto interrupt_result =
processor->Process(interrupt_execution, interrupt_only);
const bool interrupt_split_once =
gpu_scheduler.CurrentTick() == interrupt_tick + 1;
auto gds_interrupt = make_release_mem(5, 2, &gds_label, 1ull << 16u);
Pm4Execution gds_interrupt_execution;
const auto gds_interrupt_tick = gpu_scheduler.CurrentTick();
const auto gds_interrupt_result =
processor->Process(gds_interrupt_execution, gds_interrupt);
const bool gds_interrupt_waited_once =
gpu_scheduler.CurrentTick() == gds_interrupt_tick + 1;
release_mem_submission_counts =
gds_result == Pm4ProcessResult::Complete &&
gds_waited_once && interrupt_result == Pm4ProcessResult::Complete &&
interrupt_split_once &&
gds_interrupt_result == Pm4ProcessResult::Complete &&
gds_interrupt_waited_once;
});
gpu.SendCommandSync([&] {
gpu_scheduler.Finish();
gpu_scheduler.WaitPriorityOperations(gpu_scheduler.CurrentTick() - 1);
});
Require("GpuCommandLane", "RELEASE_MEM submission counts",
release_mem_submission_counts &&
static_cast<uint32_t>(release_label) == 0x11223344u &&
static_cast<uint32_t>(gds_label) == 0,
"RELEASE_MEM lost its required split/readback or retained a "
"redundant GPU wait");
}
alignas(uint32_t) uint32_t packet_marker_a = 0;
alignas(uint32_t) uint32_t packet_marker_b = 0;
const auto write_packet = [](uint32_t *packet, uint32_t *destination,
uint32_t value) {
const auto destination_address = reinterpret_cast<uint64_t>(destination);
packet[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
packet[1] = 0;
packet[2] = static_cast<uint32_t>(destination_address);
packet[3] = static_cast<uint32_t>(destination_address >> 32u);
packet[4] = value;
};
uint32_t live_graphics_value = 0;
std::array<uint32_t, 5> live_graphics_commands{};
write_packet(live_graphics_commands.data(), &live_graphics_value, 11);
std::binary_semaphore graphics_gate_entered{0};
std::binary_semaphore graphics_gate_release{0};
gpu.SendCommand([&] {
graphics_gate_entered.release();
graphics_gate_release.acquire();
});
gpu.Submit(live_graphics_commands, {});
graphics_gate_entered.acquire();
live_graphics_commands[4] = 22;
graphics_gate_release.release();
gpu.WaitForIdle();
Require("GpuCommandLane", "borrowed graphics commands",
live_graphics_value == 22,
"graphics submission executed a copied PM4 stream");
uint32_t live_compute_value = 0;
std::array<uint32_t, 5> live_compute_commands{};
write_packet(live_compute_commands.data(), &live_compute_value, 33);
std::binary_semaphore compute_gate_entered{0};
std::binary_semaphore compute_gate_release{0};
gpu.SendCommand([&] {
compute_gate_entered.release();
compute_gate_release.acquire();
});
gpu.SubmitCompute(0x20, live_compute_commands);
compute_gate_entered.acquire();
live_compute_commands[4] = 44;
compute_gate_release.release();
gpu.WaitForIdle();
Require("GpuCommandLane", "borrowed compute commands",
live_compute_value == 44,
"compute submission executed a copied PM4 stream");
LibKernel::EventQueue::KernelEqueue interrupt_queue =
LibKernel::EventQueue::KERNEL_EQUEUE_INVALID;
uint32_t graphics_interrupt_udata = 0;
uint32_t compute_interrupt_udata = 0;
uint32_t other_compute_interrupt_udata = 0;
const bool interrupt_queue_ready =
LibKernel::EventQueue::KernelCreateEqueue(&interrupt_queue,
"pm4-interrupts") == 0 &&
Sync::AddEqEvent(context, interrupt_queue, 0,
&graphics_interrupt_udata) == 0 &&
Sync::AddEqEvent(context, interrupt_queue, 0x20,
&compute_interrupt_udata) == 0 &&
Sync::AddEqEvent(context, interrupt_queue, 0x21,
&other_compute_interrupt_udata) == 0;
Require("GpuCommandLane", "interrupt queue", interrupt_queue_ready,
"failed to register the PM4 interrupt events");
const auto make_interrupt_packet =
[](uint32_t data_selector, uint32_t interrupt_selector,
void *destination, uint64_t value, uint32_t context_id) {
std::array<uint32_t, 8> packet{};
const auto address = reinterpret_cast<uint64_t>(destination);
packet[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM);
packet[1] = 0x28u | (5u << 8u);
packet[2] = (interrupt_selector << 24u) | (data_selector << 29u);
packet[3] = static_cast<uint32_t>(address);
packet[4] = static_cast<uint32_t>(address >> 32u);
packet[5] = static_cast<uint32_t>(value);
packet[6] = static_cast<uint32_t>(value >> 32u);
packet[7] = context_id & 0x07ffffffu;
return packet;
};
const auto finish_gpu = [&] {
gpu.SendCommandSync([&] {
auto &scheduler = context.GetCommandScheduler();
scheduler.Finish();
scheduler.WaitPriorityOperations(scheduler.CurrentTick() - 1);
});
};
const auto wait_for_interrupt =
[&](LibKernel::EventQueue::KernelEvent &event, int &count) {
LibKernel::KernelUseconds timeout = 0;
return LibKernel::EventQueue::KernelWaitEqueue(
interrupt_queue, &event, 1, &count, &timeout);
};
uint32_t no_interrupt_graphics_value = 0;
std::array<uint32_t, 5> no_interrupt_graphics_commands{};
write_packet(no_interrupt_graphics_commands.data(),
&no_interrupt_graphics_value, 55);
gpu.Submit(no_interrupt_graphics_commands, {});
gpu.WaitForIdle();
finish_gpu();
LibKernel::EventQueue::KernelEvent interrupt_event{};
int interrupt_count = 0;
const auto no_graphics_interrupt_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
Require("GpuCommandLane", "no implicit graphics interrupt",
no_graphics_interrupt_wait == LibKernel::KERNEL_ERROR_ETIMEDOUT &&
interrupt_count == 0 && no_interrupt_graphics_value == 55,
"graphics submission completion synthesized an interrupt");
auto live_interrupt_commands =
make_interrupt_packet(0, 1, nullptr, 0, 0x234u);
std::binary_semaphore interrupt_gate_entered{0};
std::binary_semaphore interrupt_gate_release{0};
gpu.SendCommand([&] {
interrupt_gate_entered.release();
interrupt_gate_release.acquire();
});
gpu.SubmitCompute(0x20, live_interrupt_commands);
interrupt_gate_entered.acquire();
live_interrupt_commands[2] = 0;
interrupt_gate_release.release();
gpu.WaitForIdle();
finish_gpu();
interrupt_count = 0;
const auto no_interrupt_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
Require("GpuCommandLane", "no implicit completion interrupt",
no_interrupt_wait == LibKernel::KERNEL_ERROR_ETIMEDOUT &&
interrupt_count == 0,
"submission completion synthesized an interrupt absent from the "
"executed PM4 stream");
uint32_t graphics_interrupt_label = 0xa5a5a5a5u;
auto graphics_interrupt_commands = make_interrupt_packet(
1, 1, &graphics_interrupt_label, 0x11223344u, 0);
gpu.Submit(graphics_interrupt_commands, {});
gpu.WaitForIdle();
finish_gpu();
interrupt_count = 0;
const auto graphics_interrupt_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
Require("GpuCommandLane", "graphics interrupt routing",
graphics_interrupt_wait == 0 && interrupt_count == 1 &&
interrupt_event.ident == 0 && interrupt_event.data == 0 &&
interrupt_event.udata == &graphics_interrupt_udata &&
graphics_interrupt_label == 0xa5a5a5a5u,
"graphics kOnly interrupt was misrouted or wrote its label");
uint32_t compute_interrupt_label = 0x5a5a5a5au;
auto compute_interrupt_commands = make_interrupt_packet(
1, 0, &compute_interrupt_label, 0x55667788u, 0x456u);
std::binary_semaphore compute_interrupt_gate_entered{0};
std::binary_semaphore compute_interrupt_gate_release{0};
gpu.SendCommand([&] {
compute_interrupt_gate_entered.release();
compute_interrupt_gate_release.acquire();
});
gpu.SubmitCompute(0x20, compute_interrupt_commands);
compute_interrupt_gate_entered.acquire();
compute_interrupt_commands[2] |= 1u << 24u;
compute_interrupt_gate_release.release();
gpu.WaitForIdle();
finish_gpu();
interrupt_count = 0;
const auto compute_interrupt_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
const bool compute_interrupt_matches =
compute_interrupt_wait == 0 && interrupt_count == 1 &&
interrupt_event.ident == 0x20 && interrupt_event.data == 0x456u &&
interrupt_event.udata == &compute_interrupt_udata &&
compute_interrupt_label == 0x55667788u;
uint64_t compute_clock_label = 0;
auto compute_clock_commands = make_interrupt_packet(
3, 1, &compute_clock_label, 0, 0x567u);
gpu.SubmitCompute(0x20, compute_clock_commands);
gpu.WaitForIdle();
finish_gpu();
interrupt_count = 0;
const auto compute_clock_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
const bool compute_clock_matches =
compute_clock_wait == 0 && interrupt_count == 1 &&
interrupt_event.ident == 0x20 && interrupt_event.data == 0x567u &&
interrupt_event.udata == &compute_interrupt_udata &&
compute_clock_label != 0;
constexpr uint64_t compute_done_value = 0x1234567887654321ull;
uint64_t compute_done_label = 0;
auto compute_done_commands = make_interrupt_packet(
2, 2, &compute_done_label, compute_done_value, 0x678u);
compute_done_commands[1] = 0x62fu; // CS_DONE, shader-done index, no GCR action.
gpu.SubmitCompute(0x20, compute_done_commands);
gpu.WaitForIdle();
finish_gpu();
interrupt_count = 0;
const auto compute_done_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
Require("GpuCommandLane", "CS_DONE write-confirm interrupt",
compute_done_wait == 0 && interrupt_count == 1 &&
interrupt_event.ident == 0x20 && interrupt_event.data == 0x678u &&
interrupt_event.udata == &compute_interrupt_udata &&
compute_done_label == compute_done_value,
"CS_DONE lost its full 64-bit label write or write-confirm interrupt");
interrupt_count = 0;
const auto extra_interrupt_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
const auto delete_other_compute_event =
Sync::DeleteEqEvent(context, interrupt_queue, 0x21);
LibKernel::EventQueue::KernelDeleteEqueue(interrupt_queue);
context.TriggerInterrupt(0, 0);
context.TriggerInterrupt(0x20, 0);
context.TriggerInterrupt(0x21, 0);
Require("GpuCommandLane", "compute interrupt routing",
compute_interrupt_matches && compute_clock_matches &&
extra_interrupt_wait == LibKernel::KERNEL_ERROR_ETIMEDOUT &&
interrupt_count == 0 && delete_other_compute_event == 0,
"compute kOnly interrupt was misrouted, broadcast, or lost a "
"label write");
std::array<uint32_t, 1024> polling_loop{};
for (size_t i = 0; i < polling_loop.size() - 4; i += 2) {
polling_loop[i] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
polling_loop[i + 1] = 0;
}
const auto loop_address = reinterpret_cast<uint64_t>(polling_loop.data());
polling_loop[1020] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
polling_loop[1021] = static_cast<uint32_t>(loop_address);
polling_loop[1022] = static_cast<uint32_t>(loop_address >> 32u);
polling_loop[1023] =
0x0f200000u | static_cast<uint32_t>(polling_loop.size());
std::array<uint32_t, 14> polling_commands{};
write_packet(polling_commands.data(), &packet_marker_a, 11);
polling_commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
polling_commands[6] = static_cast<uint32_t>(loop_address);
polling_commands[7] = static_cast<uint32_t>(loop_address >> 32u);
polling_commands[8] =
0x0f200000u | static_cast<uint32_t>(polling_loop.size());
write_packet(polling_commands.data() + 9, &packet_marker_b, 22);
std::binary_semaphore stream_gate_entered{0};
std::binary_semaphore stream_gate_release{0};
gpu.SendCommand([&] {
stream_gate_entered.release();
stream_gate_release.acquire();
});
gpu.Submit(polling_commands, {});
stream_gate_entered.acquire();
stream_gate_release.release();
uint32_t packet_marker_a_at_callback = UINT32_MAX;
uint32_t packet_marker_b_at_callback = UINT32_MAX;
for (uint32_t attempt = 0; attempt < 8 && packet_marker_a_at_callback != 11;
attempt++) {
gpu.SendCommandSync([&] {
if (packet_marker_a == 11 && packet_marker_b == 0) {
packet_marker_a_at_callback = packet_marker_a;
packet_marker_b_at_callback = packet_marker_b;
polling_commands[13] = 33;
polling_loop[1020] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
polling_loop[1021] = 0;
polling_loop[1022] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
polling_loop[1023] = 0;
}
});
std::this_thread::yield();
}
gpu.WaitForIdle();
Require("GpuCommandLane", "packet-boundary command polling",
packet_marker_a_at_callback == 11 &&
packet_marker_b_at_callback == 0 && packet_marker_a == 11 &&
packet_marker_b == 33,
"host command waited for an entire PM4 stream or resumed a copied "
"stream");
uint32_t label = 0;
uint32_t prefix = 0;
uint32_t suffix = 0;
const auto address = [](const void *value) {
return reinterpret_cast<uint64_t>(value);
};
std::array<uint32_t, 17> commands{};
commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
commands[1] = 0;
commands[2] = static_cast<uint32_t>(address(&prefix));
commands[3] = static_cast<uint32_t>(address(&prefix) >> 32u);
commands[4] = 11;
commands[5] = KYTY_PM4(7, Pm4::IT_WAIT_REG_MEM, 0);
commands[6] = 0x10u | 3u;
commands[7] = static_cast<uint32_t>(address(&label));
commands[8] = static_cast<uint32_t>(address(&label) >> 32u);
commands[9] = 1;
commands[10] = UINT32_MAX;
commands[12] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
commands[13] = 0;
commands[14] = static_cast<uint32_t>(address(&suffix));
commands[15] = static_cast<uint32_t>(address(&suffix) >> 32u);
commands[16] = 22;
gpu.Submit(commands, {});
std::binary_semaphore ordered_started{0};
std::atomic<bool> ordered_finished{false};
uint32_t ordered_suffix = 0;
std::jthread ordered([&] {
ordered_started.release();
gpu.WaitForIdle();
ordered_suffix = suffix;
ordered_finished = true;
});
ordered_started.acquire();
gpu.SendCommandSync([&] {
Require("GpuCommandLane", "guest queue idle fence", !ordered_finished.load(),
"idle fence returned before a queued submission");
label = 1;
});
ordered.join();
Require("GpuCommandLane", "ordered completion",
prefix == 11 && suffix == 22 && ordered_suffix == 22 &&
ordered_finished.load(),
"idle fence did not drain prior PM4 work");
auto &resources = context;
constexpr uint64_t empty_unmap_base = 0x0000000200400000ull;
constexpr uint64_t empty_unmap_size = 0x4000;
resources.MapMemory(empty_unmap_base, empty_unmap_size);
label = 0;
prefix = 0;
suffix = 0;
gpu.Submit(commands, {});
std::binary_semaphore unmap_complete{0};
std::jthread unmap_thread([&] {
resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
unmap_complete.release();
});
const bool unmap_returned =
unmap_complete.try_acquire_for(std::chrono::seconds(2));
gpu.SendCommandSync([&] { label = 1; });
if (!unmap_returned) {
unmap_complete.acquire();
}
unmap_thread.join();
gpu.WaitForIdle();
Require("GpuCommandLane", "unmap queue progress",
unmap_returned &&
!resources.IsMapped(empty_unmap_base, empty_unmap_size) &&
prefix == 11 && suffix == 22,
"an unrelated unmap waited for a blocked PM4 submission");
auto &scheduler = context.GetCommandScheduler();
std::atomic<bool> completion_published{false};
gpu.SendCommandSync([&] {
scheduler.DeferPriorityOperation([&] { completion_published = true; });
});
resources.MapMemory(empty_unmap_base, empty_unmap_size);
resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
Require("GpuCommandLane", "unmap native completion",
completion_published.load() &&
!resources.IsMapped(empty_unmap_base, empty_unmap_size),
"unmap returned before an earlier native guest-memory callback");
std::binary_semaphore priority_entered{0};
std::binary_semaphore release_priority{0};
gpu.SendCommandSync([&] {
scheduler.DeferPriorityOperation([&] {
priority_entered.release();
release_priority.acquire();
});
scheduler.Flush();
});
priority_entered.acquire();
resources.MapMemory(empty_unmap_base, empty_unmap_size);
std::binary_semaphore priority_unmap_entered{0};
std::binary_semaphore priority_unmap_complete{0};
std::jthread priority_unmap_thread([&] {
gpu.SendCommandSync([&] {
priority_unmap_entered.release();
resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
});
priority_unmap_complete.release();
});
priority_unmap_entered.acquire();
const bool unmap_overtook_priority =
priority_unmap_complete.try_acquire_for(std::chrono::seconds(1));
release_priority.release();
if (!unmap_overtook_priority) {
priority_unmap_complete.acquire();
}
priority_unmap_thread.join();
Require("GpuCommandLane", "unmap priority ordering",
!unmap_overtook_priority &&
!resources.IsMapped(empty_unmap_base, empty_unmap_size),
"unmap returned before an earlier guest-memory callback");
constexpr uintptr_t fault_base = 0x0000000200500000ull;
constexpr uint64_t fault_size = 0x10000;
int64_t fault_direct_offset = -1;
Require("GpuCommandLane", "processor fault direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
fault_size, fault_size, 0, &fault_direct_offset) == 0,
"processor-fault direct-memory allocation failed");
void *fault_memory = reinterpret_cast<void *>(fault_base);
Require("GpuCommandLane", "processor fault direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&fault_memory, fault_size, 0x3, 0x10, fault_direct_offset,
fault_size) == 0,
"processor-fault fixed direct-memory mapping failed");
Require("GpuCommandLane", "processor fault allocation",
fault_memory == reinterpret_cast<void *>(fault_base),
"fixed processor-fault allocation failed");
resources.MapMemory(fault_base, fault_size);
constexpr uint64_t immediate_dst = fault_base + 0x1000;
constexpr uint64_t immediate_memory_dst = fault_base + 0x2000;
constexpr uint64_t memory_src = fault_base + 0x4000;
constexpr uint64_t memory_copy_dst = fault_base + 0x6000;
constexpr uint64_t memory_dst = fault_base + 0x8000;
constexpr uint64_t l2_copy_dst = fault_base + 0xa000;
constexpr uint64_t nowhere_dst = fault_base + 0xc000;
constexpr uint64_t clean_cached_fill = fault_base + 0x3000;
constexpr uint64_t clean_cached_copy = fault_base + 0x3008;
constexpr uint64_t clean_cached_source = fault_base + 0x3010;
constexpr uint64_t clean_cached_readback = fault_base + 0xb000;
constexpr std::array<uint32_t, 2> source_words{0x11223344u, 0x55667788u};
constexpr std::array<uint32_t, 2> nowhere_words{0x89abcdefu, 0x01234567u};
constexpr std::array<uint32_t, 2> clean_copy_words{0x13579bdfu,
0x2468ace0u};
std::memcpy(reinterpret_cast<void *>(memory_src), source_words.data(),
sizeof(source_words));
std::memcpy(reinterpret_cast<void *>(nowhere_dst), nowhere_words.data(),
sizeof(nowhere_words));
std::memcpy(reinterpret_cast<void *>(clean_cached_source),
clean_copy_words.data(), sizeof(clean_copy_words));
std::array<uint32_t, 56> dma_commands{};
size_t dma_cursor = 0;
const auto append_dma = [&](uint8_t dst_sel, uint64_t dst, uint8_t src_sel,
uint64_t src, uint32_t bytes,
uint32_t command_flags = 0) {
auto *packet = dma_commands.data() + dma_cursor;
dma_cursor += 7;
packet[0] = KYTY_PM4(7, Pm4::IT_DMA_DATA, 0);
packet[1] = (static_cast<uint32_t>(dst_sel) << 20u) |
(static_cast<uint32_t>(src_sel) << 29u);
packet[2] = static_cast<uint32_t>(src);
packet[3] = static_cast<uint32_t>(src >> 32u);
packet[4] = static_cast<uint32_t>(dst);
packet[5] = static_cast<uint32_t>(dst >> 32u);
packet[6] = bytes | command_flags;
};
constexpr uint64_t gds_immediate_offset = 0;
constexpr uint64_t gds_memory_offset = 0x10;
constexpr uint32_t immediate_value = 0xa1b2c3d4u;
append_dma(1, gds_immediate_offset, 2, immediate_value,
sizeof(source_words));
append_dma(0, immediate_dst, 1, gds_immediate_offset, sizeof(source_words));
append_dma(1, gds_memory_offset, 0, memory_src, sizeof(source_words));
append_dma(0, memory_dst, 1, gds_memory_offset, sizeof(source_words));
append_dma(0, immediate_memory_dst, 2, immediate_value,
sizeof(source_words));
append_dma(0, memory_copy_dst, 0, memory_src, sizeof(source_words));
append_dma(3, l2_copy_dst, 3, memory_src, sizeof(source_words));
append_dma(2, nowhere_dst, 3, nowhere_dst, sizeof(nowhere_words),
1u << 31u);
Require("GpuCommandLane", "DMA_DATA packet assembly",
dma_cursor == dma_commands.size(),
"DMA_DATA packet stream has the wrong size");
gpu.Submit(dma_commands, {});
gpu.WaitForIdle();
constexpr uint32_t clean_fill_value = 0xdecafbad;
gpu.SendCommandSync([&] {
auto &buffer_cache = resources.GetBufferCache();
Require("GpuCommandLane", "clean cached setup",
buffer_cache.IsRegionRegistered(clean_cached_fill,
sizeof(source_words)) &&
!buffer_cache.HasGpuDirtyBytes(nowhere_dst,
sizeof(nowhere_words)) &&
buffer_cache.HasGpuDirtyBytes(immediate_dst,
sizeof(source_words)),
"DMA prefetch modified ownership or did not establish a dirty "
"cached page");
buffer_cache.FillBuffer(clean_cached_fill, sizeof(source_words),
clean_fill_value, false);
buffer_cache.CopyBuffer(clean_cached_copy, clean_cached_source,
sizeof(clean_copy_words), false, false);
Require("GpuCommandLane", "clean cached DMA ownership",
!buffer_cache.HasGpuDirtyBytes(clean_cached_fill,
sizeof(source_words)) &&
!buffer_cache.HasGpuDirtyBytes(clean_cached_copy,
sizeof(clean_copy_words)) &&
buffer_cache.HasGpuDirtyBytes(immediate_dst,
sizeof(source_words)),
"clean cached DMA acquired ownership or released a byte-disjoint "
"dirty sibling");
constexpr uint64_t gds_clean_mirror = 0x20;
buffer_cache.CopyBuffer(gds_clean_mirror, clean_cached_fill,
2 * sizeof(source_words), true, false);
buffer_cache.CopyBuffer(clean_cached_readback, gds_clean_mirror,
2 * sizeof(source_words), false, true);
});
Require("GpuCommandLane", "DMA_DATA immediate GDS readback",
resources.HandleFault(PageFaultAccess::Read, immediate_dst),
"immediate-to-GDS-to-memory copy did not publish GPU bytes");
Require("GpuCommandLane", "DMA_DATA memory GDS readback",
resources.HandleFault(PageFaultAccess::Read, memory_dst),
"memory-to-GDS-to-memory copy did not publish GPU bytes");
Require("GpuCommandLane", "DMA_DATA immediate memory readback",
resources.HandleFault(PageFaultAccess::Read, immediate_memory_dst),
"immediate-to-memory copy did not publish GPU bytes");
Require("GpuCommandLane", "DMA_DATA memory copy readback",
resources.HandleFault(PageFaultAccess::Read, memory_copy_dst),
"memory-to-memory copy did not publish GPU bytes");
Require("GpuCommandLane", "DMA_DATA L2 readback",
resources.HandleFault(PageFaultAccess::Read, l2_copy_dst),
"MemoryUsingL2 copy did not publish GPU bytes");
Require("GpuCommandLane", "clean cached mirror readback",
resources.HandleFault(PageFaultAccess::Read, clean_cached_readback),
"clean host DMA was not reflected in the cached buffer");
std::array<uint32_t, 2> immediate_words{};
std::array<uint32_t, 2> copied_words{};
std::array<uint32_t, 2> immediate_memory_words{};
std::array<uint32_t, 2> memory_copy_words{};
std::array<uint32_t, 2> l2_copy_words{};
std::array<uint32_t, 4> clean_cached_words{};
std::memcpy(immediate_words.data(),
reinterpret_cast<const void *>(immediate_dst),
sizeof(immediate_words));
std::memcpy(copied_words.data(), reinterpret_cast<const void *>(memory_dst),
sizeof(copied_words));
std::memcpy(immediate_memory_words.data(),
reinterpret_cast<const void *>(immediate_memory_dst),
sizeof(immediate_memory_words));
std::memcpy(memory_copy_words.data(),
reinterpret_cast<const void *>(memory_copy_dst),
sizeof(memory_copy_words));
std::memcpy(l2_copy_words.data(),
reinterpret_cast<const void *>(l2_copy_dst),
sizeof(l2_copy_words));
std::memcpy(clean_cached_words.data(),
reinterpret_cast<const void *>(clean_cached_readback),
sizeof(clean_cached_words));
Require("GpuCommandLane", "DMA_DATA immediate GDS contents",
immediate_words ==
std::array<uint32_t, 2>{immediate_value, immediate_value},
"immediate-to-GDS-to-memory bytes do not match");
Require("GpuCommandLane", "DMA_DATA memory GDS contents",
copied_words == source_words,
"memory-to-GDS-to-memory bytes do not match");
Require("GpuCommandLane", "DMA_DATA immediate memory contents",
immediate_memory_words ==
std::array<uint32_t, 2>{immediate_value, immediate_value},
"immediate-to-memory bytes do not match");
Require("GpuCommandLane", "DMA_DATA memory copy contents",
memory_copy_words == source_words,
"memory-to-memory bytes do not match");
Require("GpuCommandLane", "DMA_DATA L2 contents",
l2_copy_words == source_words, "MemoryUsingL2 bytes do not match");
Require("GpuCommandLane", "DMA_DATA nowhere contents",
std::memcmp(reinterpret_cast<const void *>(nowhere_dst),
nowhere_words.data(), sizeof(nowhere_words)) == 0,
"Nowhere destination modified guest memory");
Require("GpuCommandLane", "clean cached DMA contents",
clean_cached_words == std::array<uint32_t, 4>{clean_fill_value,
clean_fill_value,
clean_copy_words[0],
clean_copy_words[1]},
"host-memory fill/copy did not update the clean cached mirror");
gpu.SendCommandSync([&] {
Require("GpuCommandLane", "GPU memory invalidation",
resources.InvalidateMemory(fault_base, sizeof(uint32_t)),
"GPU memory invalidation did not find its mapped range");
});
auto &buffer_cache = resources.GetBufferCache();
Require("GpuCommandLane", "clean tracked fault setup",
buffer_cache.IsRegionRegistered(immediate_dst, sizeof(uint32_t)) &&
!buffer_cache.HasGpuDirtyBytes(immediate_dst, sizeof(uint32_t)),
"clean write-fault test address is not backed by a clean cached "
"Buffer");
constexpr uint64_t dirty_fault_address = fault_base + 0xd000;
constexpr uint32_t dirty_fault_value = 0x5a17c0deu;
constexpr uint32_t dirty_fault_stale = 0x0ddba11u;
std::memcpy(reinterpret_cast<void *>(dirty_fault_address),
&dirty_fault_stale, sizeof(dirty_fault_stale));
gpu.SendCommandSync([&] {
auto dirty = buffer_cache.ObtainBuffer(
dirty_fault_address, sizeof(dirty_fault_value), true, false);
Require("GpuCommandLane", "dirty tracked fault setup",
dirty.first != nullptr,
"dirty write-fault test could not create a cached Buffer");
buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value),
dirty_fault_value, false);
});
Require("GpuCommandLane", "dirty tracked fault ownership",
buffer_cache.HasGpuDirtyBytes(dirty_fault_address,
sizeof(dirty_fault_value)),
"dirty write-fault test address did not acquire GPU ownership");
std::binary_semaphore block_entered{0};
std::binary_semaphore block_release{0};
std::binary_semaphore clean_fault_complete{0};
bool clean_fault_handled = false;
gpu.SendCommand([&] {
block_entered.release();
block_release.acquire();
});
block_entered.acquire();
std::jthread clean_fault_thread([&] {
clean_fault_handled =
resources.HandleFault(PageFaultAccess::Write, immediate_dst);
clean_fault_complete.release();
});
const bool clean_fault_was_direct =
clean_fault_complete.try_acquire_for(std::chrono::seconds(1));
if (!clean_fault_was_direct) {
block_release.release();
clean_fault_complete.acquire();
}
clean_fault_thread.join();
Require("GpuCommandLane", "clean fault direct path",
clean_fault_was_direct && clean_fault_handled,
"a clean CPU write fault waited for the blocked GPU command lane");
std::binary_semaphore dirty_fault_complete{0};
bool dirty_fault_handled = false;
std::jthread dirty_fault_thread([&] {
dirty_fault_handled =
resources.HandleFault(PageFaultAccess::Write, dirty_fault_address);
dirty_fault_complete.release();
});
const bool dirty_fault_bypassed_lane =
dirty_fault_complete.try_acquire_for(std::chrono::milliseconds(200));
block_release.release();
if (!dirty_fault_bypassed_lane) {
dirty_fault_complete.acquire();
}
dirty_fault_thread.join();
uint32_t dirty_fault_backing = 0;
std::memcpy(&dirty_fault_backing,
reinterpret_cast<const void *>(dirty_fault_address),
sizeof(dirty_fault_backing));
Require("GpuCommandLane", "dirty fault synchronization",
!dirty_fault_bypassed_lane && dirty_fault_handled &&
dirty_fault_backing == dirty_fault_value &&
!buffer_cache.HasGpuDirtyBytes(dirty_fault_address,
sizeof(dirty_fault_value)),
"a GPU-dirty write fault bypassed synchronization or lost the "
"published bytes");
constexpr uint64_t empty_copy_fault_address = fault_base + 0xe000;
constexpr uint32_t empty_copy_fault_value = 0x6b28d1efu;
gpu.SendCommandSync([&] {
auto dirty = buffer_cache.ObtainBuffer(empty_copy_fault_address,
sizeof(empty_copy_fault_value),
true, false);
Require("GpuCommandLane", "empty-copy write setup",
dirty.first != nullptr,
"empty-copy write test could not create a cached Buffer");
buffer_cache.FillBuffer(empty_copy_fault_address,
sizeof(empty_copy_fault_value),
empty_copy_fault_value, false);
buffer_cache.ReadMemory(empty_copy_fault_address,
sizeof(empty_copy_fault_value));
const bool gpu_owned = buffer_cache.IsRegionGpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
const bool cpu_owned = buffer_cache.IsRegionCpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
Require("GpuCommandLane", "empty-copy read ownership",
!gpu_owned && !cpu_owned,
"readback did not leave clean ownership for the queued write "
"invalidation");
buffer_cache.ReadMemory(empty_copy_fault_address,
sizeof(empty_copy_fault_value), true);
});
uint32_t empty_copy_fault_backing = 0;
std::memcpy(&empty_copy_fault_backing,
reinterpret_cast<const void *>(empty_copy_fault_address),
sizeof(empty_copy_fault_backing));
const bool empty_copy_cpu_owned = buffer_cache.IsRegionCpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
const bool empty_copy_gpu_owned = buffer_cache.IsRegionGpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
const bool empty_copy_dirty = buffer_cache.HasGpuDirtyBytes(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
Require("GpuCommandLane", "empty-copy write ownership",
empty_copy_fault_backing == empty_copy_fault_value &&
empty_copy_cpu_owned && !empty_copy_gpu_owned &&
!empty_copy_dirty,
"write invalidation without a remaining copy did not establish CPU "
"ownership");
resources.UnmapMemory(fault_base, fault_size);
Require("GpuCommandLane", "processor fault unmap",
Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0,
"processor-fault direct-memory mapping release failed");
Require("GpuCommandLane", "processor fault free",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
fault_direct_offset, fault_size) == 0,
"processor-fault direct-memory allocation release failed");
std::binary_semaphore command_entered{0};
std::binary_semaphore release_command{0};
std::atomic<bool> shutdown_complete{false};
bool shutdown_owner_visible = false;
resources.MapMemory(empty_unmap_base, empty_unmap_size);
gpu.SendCommand([&] {
command_entered.release();
release_command.acquire();
shutdown_owner_visible = &context.GetGpu() == &gpu;
});
command_entered.acquire();
std::jthread shutdown_thread([&] {
context.ShutdownGpu();
shutdown_complete = true;
});
while (!gpu.IsStopping()) {
std::this_thread::yield();
}
Require("GpuCommandLane", "owned shutdown", !shutdown_complete.load(),
"GPU owner returned before an in-flight command completed");
release_command.release();
shutdown_thread.join();
Require(
"GpuCommandLane", "owned shutdown completion",
shutdown_complete.load() && shutdown_owner_visible,
"GPU owner did not remain available while draining queued work");
resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
Require("GpuCommandLane", "unmap after shutdown",
!resources.IsMapped(empty_unmap_base, empty_unmap_size),
"renderer retained mapped coverage after the GPU command lane stopped");
context.ShutdownGpu();
std::printf("[host] %-32s ok\n", "GpuCommandLane");
}
void CheckUnifiedImageViewCache() {
EnsureRuntimeContext();
constexpr const char *name = "UnifiedImageViewCache";
CommandScheduler scheduler(Renderer(), m_runtime_context);
ImageInfo color_info{};
color_info.pixel_format = vk::Format::eR8G8B8A8Unorm;
color_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm;
color_info.type = Prospero::ImageType::kColor2D;
color_info.extent = {8, 8, 1};
color_info.resources = {2, 2};
color_info.pitch = 8;
color_info.bytes_per_block = 4;
color_info.samples = 1;
color_info.tile_mode = Prospero::TileMode::kLinear;
color_info.mip_layout[0] = {0, 512, 8, 8};
color_info.mip_layout[1] = {512, 128, 4, 4};
Libs::Graphics::Image color(m_runtime_context, scheduler, color_info);
const vk::ComponentMapping identity{};
const vk::ComponentMapping bgra{
vk::ComponentSwizzle::eB, vk::ComponentSwizzle::eG,
vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eA};
ImageViewInfo sampled{};
sampled.format = color_info.pixel_format;
sampled.type = vk::ImageViewType::e2D;
sampled.aspect = vk::ImageAspectFlagBits::eColor;
sampled.base_level = 0;
sampled.level_count = 1;
sampled.base_layer = 0;
sampled.layer_count = 1;
sampled.mapping = identity;
sampled.usage = vk::ImageUsageFlagBits::eSampled;
const auto first = color.FindView(sampled);
const auto first_again = color.FindView(sampled);
auto swizzled_info = sampled;
swizzled_info.mapping = bgra;
const auto swizzled = color.FindView(swizzled_info);
auto mip_info = sampled;
mip_info.base_level = 1;
const auto mip = color.FindView(mip_info);
auto layer_info = sampled;
layer_info.base_layer = 1;
const auto layer = color.FindView(layer_info);
auto array_info = sampled;
array_info.type = vk::ImageViewType::e2DArray;
array_info.layer_count = 2;
const auto array = color.FindView(array_info);
auto reinterpreted_info = sampled;
reinterpreted_info.format = vk::Format::eR8G8B8A8Uint;
const auto reinterpreted = color.FindView(reinterpreted_info);
auto storage_info = sampled;
storage_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto storage = color.FindView(storage_info);
auto attachment_info = sampled;
attachment_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto attachment = color.FindView(attachment_info);
Require(name, "color views",
first != nullptr && first_again == first && swizzled != nullptr &&
swizzled != first && mip != nullptr && mip != first &&
layer != nullptr && layer != first && array != nullptr &&
array != first && reinterpreted != nullptr &&
reinterpreted != first && storage != nullptr &&
storage != first && attachment == first &&
color.views.size() == 7,
"dynamic view identity omitted mapping, format, mip, layer, type, "
"or storage usage");
auto video_out_info = color_info;
video_out_info.pixel_format = vk::Format::eA2R10G10B10UnormPack32;
video_out_info.guest_format = Prospero::BufferFormat::k10_10_10_2UNorm;
Libs::Graphics::Image video_out(m_runtime_context, scheduler,
video_out_info);
auto video_out_view_info = sampled;
video_out_view_info.format = video_out_info.pixel_format;
video_out_view_info.usage = vk::ImageUsageFlagBits::eTransferSrc;
const auto video_out_view = video_out.FindView(video_out_view_info);
auto storage_alias_info = video_out_view_info;
storage_alias_info.format = vk::Format::eA2B10G10R10UnormPack32;
storage_alias_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto storage_alias_view = video_out.FindView(storage_alias_info);
Require(
name, "video-out storage alias usage",
ImageViewOps::FormatsCompatible(video_out_info.pixel_format,
storage_alias_info.format) &&
static_cast<bool>(video_out.backing.usage &
vk::ImageUsageFlagBits::eStorage) &&
video_out_view != nullptr && storage_alias_view != nullptr,
"compatible video-out backing was created without storage usage");
ImageInfo depth_info{};
depth_info.pixel_format = vk::Format::eD32Sfloat;
depth_info.guest_format = Prospero::BufferFormat::k32Float;
depth_info.type = Prospero::ImageType::kColor2D;
depth_info.extent = {8, 8, 1};
depth_info.resources = {1, 2};
depth_info.pitch = 8;
depth_info.bytes_per_block = 4;
depth_info.samples = 1;
depth_info.tile_mode = Prospero::TileMode::kDepth;
depth_info.mip_layout[0] = {0, 512, 8, 8};
Libs::Graphics::Image depth(m_runtime_context, scheduler, depth_info);
ImageViewInfo depth_sampled{};
depth_sampled.format = depth_info.pixel_format;
depth_sampled.type = vk::ImageViewType::e2D;
depth_sampled.aspect = vk::ImageAspectFlagBits::eDepth;
depth_sampled.base_level = 0;
depth_sampled.level_count = 1;
depth_sampled.base_layer = 0;
depth_sampled.layer_count = 1;
depth_sampled.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eR};
depth_sampled.usage = vk::ImageUsageFlagBits::eSampled;
const auto depth_first = depth.FindView(depth_sampled);
auto depth_alias_info = depth_sampled;
depth_alias_info.format = vk::Format::eR32Uint;
const auto depth_alias = depth.FindView(depth_alias_info);
const auto depth_again = depth.FindView(depth_sampled);
auto depth_array_info = depth_sampled;
depth_array_info.type = vk::ImageViewType::e2DArray;
depth_array_info.layer_count = 2;
const auto depth_array = depth.FindView(depth_array_info);
auto depth_attachment_info = depth_array_info;
depth_attachment_info.mapping = {};
depth_attachment_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto depth_attachment = depth.FindView(depth_attachment_info);
Require(name, "depth views",
depth_first != nullptr && depth_again == depth_first &&
depth_alias == depth_first && depth_array != nullptr &&
depth_array != depth_first && depth_attachment != nullptr &&
depth_attachment != depth_array &&
depth.views.size() == 3,
"unified depth view cache lost sampled/attachment identity");
auto depth_stencil_info = depth_info;
depth_stencil_info.pixel_format = vk::Format::eD24UnormS8Uint;
depth_stencil_info.guest_format = Prospero::BufferFormat::k16UNorm;
depth_stencil_info.bytes_per_block = 2;
Libs::Graphics::Image depth_stencil(m_runtime_context, scheduler,
depth_stencil_info);
auto depth_only_info = depth_sampled;
depth_only_info.format = depth_stencil_info.pixel_format;
const auto depth_only = depth_stencil.FindView(depth_only_info);
auto combined_info = depth_only_info;
combined_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
combined_info.mapping = {};
combined_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto combined = depth_stencil.FindView(combined_info);
Require(name, "depth/stencil aspects",
depth_only != nullptr && combined != nullptr &&
depth_only != combined &&
depth_stencil.views.size() == 2 &&
depth_stencil.views[0].info.aspect ==
vk::ImageAspectFlagBits::eDepth &&
depth_stencil.views[1].info.aspect ==
(vk::ImageAspectFlagBits::eDepth |
vk::ImageAspectFlagBits::eStencil),
"explicit depth-only and combined aspects shared one image view");
Require(name, "role-free backing",
color.backing.image_type == vk::ImageType::e2D &&
color.backing.layers == 2 &&
depth.backing.image_type == vk::ImageType::e2D &&
ImageViewOps::FormatsCompatible(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Uint) &&
!ImageViewOps::FormatsCompatible(vk::Format::eD32Sfloat,
vk::Format::eR32Sfloat),
"role-free image backing or host format compatibility diverged");
auto line_info = color_info;
line_info.type = Prospero::ImageType::kColor1D;
line_info.extent = {8, 1, 1};
line_info.resources = {2, 2};
line_info.mip_layout[0] = {0, 64, 8, 1};
line_info.mip_layout[1] = {64, 16, 4, 1};
Libs::Graphics::Image line(m_runtime_context, scheduler, line_info);
auto line_view_info = sampled;
line_view_info.type = vk::ImageViewType::e1D;
const auto line_view = line.FindView(line_view_info);
auto line_array_info = line_view_info;
line_array_info.type = vk::ImageViewType::e1DArray;
line_array_info.layer_count = 2;
const auto line_array = line.FindView(line_array_info);
Require(name, "1D views",
line_view != nullptr && line_array != nullptr &&
line_array != line_view &&
line.backing.image_type == vk::ImageType::e1D &&
line.backing.layers == 2,
"1D and 1D-array views did not use a first-class 1D backing");
auto volume_info = color_info;
volume_info.type = Prospero::ImageType::kColor3D;
volume_info.extent = {8, 8, 4};
volume_info.resources = {2, 1};
Libs::Graphics::Image volume(m_runtime_context, scheduler, volume_info);
auto slice_info = sampled;
slice_info.type = vk::ImageViewType::e2D;
slice_info.base_level = 1;
slice_info.base_layer = 1;
const auto slice = volume.FindView(slice_info);
auto slice_array_info = slice_info;
slice_array_info.type = vk::ImageViewType::e2DArray;
slice_array_info.base_layer = 0;
slice_array_info.layer_count = 2;
const auto slice_array = volume.FindView(slice_array_info);
Require(name, "3D slice views",
slice != nullptr && slice_array != nullptr &&
slice_array != slice &&
static_cast<bool>(volume.backing.flags &
vk::ImageCreateFlagBits::e2DArrayCompatible),
"2D slice views of a compatible 3D backing were rejected");
std::printf("[host] %-32s ok\n", name);
}
void CheckBufferCacheDirtyGarbageCollection() {
constexpr const char *name = "BufferCacheDirtyGarbageCollection";
constexpr uintptr_t base = 0x0000000200700000ull;
constexpr uint64_t allocation_size = 0x5000000;
constexpr uint64_t allocation_alignment = 0x10000;
constexpr uint64_t first_offset = 0x100;
constexpr uint64_t second_offset = 0x200;
constexpr uint64_t clean_offset = 0x300;
constexpr uint64_t unmap_offset = 0x4100;
constexpr uint64_t partial_unmap_offset = 0x8100;
constexpr uint64_t partial_unmap_survivor_offset = 0xc100;
constexpr uint64_t ring_fault_first_offset = 0x2100;
constexpr uint64_t ring_fault_second_offset = 0x2110;
constexpr uint32_t first_value = 0x10203040u;
constexpr uint32_t second_value = 0x50607080u;
constexpr uint32_t first_stale = 0x0badf00du;
constexpr uint32_t second_stale = 0xdeadbeefu;
constexpr uint32_t clean_value = 0xaabbccddu;
constexpr uint32_t unmap_value = 0x91a2b3c4u;
constexpr uint32_t remap_value = 0xd5e6f708u;
constexpr uint32_t partial_unmap_value = 0x62738495u;
constexpr uint32_t partial_unmap_survivor_value = 0xa6b7c8d9u;
constexpr uint32_t ring_fault_first_value = 0x0a1b2c3du;
constexpr uint32_t ring_fault_second_value = 0x4e5f6071u;
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.InitializeGpu(nullptr);
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"dirty-GC direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"dirty-GC fixed direct-memory mapping failed");
auto *memory = static_cast<uint8_t *>(mapped);
std::memcpy(memory + first_offset, &first_stale, sizeof(first_stale));
std::memcpy(memory + second_offset, &second_stale, sizeof(second_stale));
std::memcpy(memory + clean_offset, &clean_value, sizeof(clean_value));
{
auto &resources = context;
auto &cache = resources.GetBufferCache();
resources.MapMemory(base, allocation_size);
const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) {
auto allocation = cache.ObtainBuffer(address, size, true, false);
Require(name, "dirty allocation", allocation.first != nullptr,
"dirty-GC buffer allocation failed");
};
const auto ReadNativeValue =
[&](const Libs::Graphics::Buffer &buffer, uint64_t offset) {
auto readback =
CreateHostBuffer(name, sizeof(uint32_t),
vk::BufferUsageFlagBits::eTransferDst, {0});
const vk::BufferCopy copy{offset, 0, sizeof(uint32_t)};
scheduler.Current().Handle().copyBuffer(buffer.Handle(), readback.buffer,
1, &copy);
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = readback.buffer;
barrier.size = readback.size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
nullptr);
scheduler.Finish();
const auto value = ReadBuffer(name, readback, 1)[0];
DestroyBuffer(&readback);
return value;
};
constexpr uint64_t stream_pages = base + 0x18000;
constexpr uint64_t stream_address = stream_pages + 0x4000 - sizeof(uint32_t);
const std::array<uint32_t, 2> stream_values{0x13579bdfu, 0x2468ace0u};
std::memcpy(reinterpret_cast<void *>(stream_address), stream_values.data(),
sizeof(stream_values));
Require(name, "GPU-readable stream source",
Libs::LibKernel::Memory::KernelMprotect(
reinterpret_cast<void *>(stream_pages), 0x8000, 0x10) == 0 &&
cache.IsRegionCpuModified(stream_address, sizeof(stream_values)) &&
!cache.IsRegionGpuModified(stream_address, sizeof(stream_values)),
"cross-page stream source is not CPU-dirty and GPU-clean");
const auto stream_tick = scheduler.CurrentTick();
const auto [streamed, streamed_offset] =
cache.ObtainBuffer(stream_address, sizeof(stream_values), false);
Require(name, "cross-page read-only stream upload",
streamed == &cache.GetUtilityBuffer(MemoryUsage::Stream) &&
scheduler.CurrentTick() == stream_tick &&
std::memcmp(streamed->Mapped().data() + streamed_offset,
stream_values.data(), sizeof(stream_values)) == 0,
"GPU-readable guest bytes were not streamed without submission");
Require(name, "stream source protection restore",
Libs::LibKernel::Memory::KernelMprotect(
reinterpret_cast<void *>(stream_pages), 0x8000, 0x3) == 0,
"stream source protection was not restored");
Require(name, "streamed GPU bytes",
ReadNativeValue(*streamed, streamed_offset) == stream_values[0] &&
ReadNativeValue(*streamed, streamed_offset + sizeof(uint32_t)) == stream_values[1],
"stream commit did not publish both cross-page words to the GPU");
constexpr uint64_t large_copy_source_offset = 0x40000;
constexpr uint64_t large_copy_destination_offset = 0x80000;
constexpr uint64_t large_copy_size = 96 * 1024;
std::vector<uint8_t> large_copy_source(large_copy_size);
for (uint64_t index = 0; index < large_copy_size; ++index) {
large_copy_source[index] =
static_cast<uint8_t>((index * 37 + index / 251) & 0xffu);
}
std::memcpy(memory + large_copy_source_offset, large_copy_source.data(),
large_copy_size);
std::memset(memory + large_copy_destination_offset, 0, large_copy_size);
cache.CopyBuffer(base + large_copy_destination_offset,
base + large_copy_source_offset, large_copy_size, false,
false);
Require(name, "large host copy",
std::memcmp(memory + large_copy_destination_offset,
large_copy_source.data(), large_copy_size) == 0,
"chunked host copy lost bytes across its scratch boundary");
constexpr uint64_t index_offset = 0x180000;
constexpr uint64_t index_page = BufferCache::CACHING_PAGESIZE;
constexpr uint64_t index_span = 3 * index_page;
const uint64_t index_begin = base + index_offset;
Require(name, "registered-range empty lookup",
!cache.IsRegionRegistered(index_begin, index_span) &&
!BufferCacheTestAccess::PageOwner(cache, index_begin),
"empty Buffer cache reported a registered range");
const auto index_left =
cache.FindBuffer(index_begin + 0x100, sizeof(uint32_t));
const auto index_right = cache.FindBuffer(
index_begin + 2 * index_page + 0x100, sizeof(uint32_t));
Require(name, "registered-range owners",
index_left && index_right &&
cache.FindBuffer(index_begin + 0x200, sizeof(uint32_t)) ==
index_left &&
BufferCacheTestAccess::PageOwner(cache, index_begin) ==
index_left &&
!BufferCacheTestAccess::PageOwner(cache,
index_begin + index_page) &&
BufferCacheTestAccess::PageOwner(
cache, index_begin + 2 * index_page) == index_right,
"failed to create disjoint Buffer index owners");
Require(
name, "registered-range boundaries",
cache.IsRegionRegistered(index_begin, 1) &&
cache.IsRegionRegistered(index_begin + index_page - 1, 1) &&
!cache.IsRegionRegistered(index_begin - 1, 1) &&
!cache.IsRegionRegistered(index_begin + index_page, index_page) &&
cache.IsRegionRegistered(index_begin + index_page,
index_page + 1) &&
!cache.IsRegionRegistered(index_begin + index_span, 1) &&
cache.IsRegionRegistered(index_begin - 1, index_span + 2),
"indexed lookup mishandled a half-open boundary, gap, or broad "
"overlap");
const auto clean_unmap_tick = scheduler.CurrentTick();
resources.UnmapMemory(index_begin, index_page);
Require(name, "clean cached buffer unmap drain",
scheduler.CurrentTick() == clean_unmap_tick + 1 &&
!resources.IsMapped(index_begin, index_page),
"unmapping a clean cached buffer did not drain native work");
resources.MapMemory(index_begin, index_page);
const auto index_bridge =
cache.FindBuffer(index_begin + index_page - 1, index_page + 2);
Require(name, "registered-range merge",
index_bridge && cache.IsRegionRegistered(index_begin + index_page,
index_page) &&
BufferCacheTestAccess::PageOwner(cache, index_begin) ==
index_bridge &&
BufferCacheTestAccess::PageOwner(
cache, index_begin + index_page) == index_bridge &&
BufferCacheTestAccess::PageOwner(
cache, index_begin + 2 * index_page) == index_bridge,
"bridging acquisition did not publish its merged Buffer range");
scheduler.Finish();
constexpr uint64_t recycled_offset = index_offset + 4 * index_page;
const auto recycled = cache.FindBuffer(base + recycled_offset + 0x100,
sizeof(uint32_t));
Require(name, "slot generation reuse",
(recycled.index == index_left.index && recycled != index_left) ||
(recycled.index == index_right.index &&
recycled != index_right),
"retired Buffer slot was not reused with a new generation");
const auto [resolved_left, resolved_left_offset] = cache.ObtainBuffer(
index_begin + 0x100, sizeof(uint32_t), true, false, index_left);
const auto [resolved_right, resolved_right_offset] =
cache.ObtainBuffer(index_begin + 2 * index_page + 0x100,
sizeof(uint32_t), true, false, index_right);
Require(name, "two-pass stale id resolution",
resolved_left == &cache.GetBuffer(index_bridge) &&
resolved_right == resolved_left &&
resolved_left_offset != resolved_right_offset,
"saved descriptor IDs did not resolve through the merged owner");
constexpr uint64_t stream_page = BufferCache::CACHING_PAGESIZE;
constexpr uint64_t stream_leap = stream_page * 128;
for (const bool forward : {true, false}) {
const char *stream_name =
forward ? "ForwardBufferStream" : "BackwardBufferStream";
const uint64_t stream_begin = base + (forward ? 0x1800000 : 0x2000000);
const uint64_t caught_address = forward
? stream_begin + stream_leap / 2
: stream_begin - stream_leap / 2;
const uint64_t untouched_address = forward
? stream_begin - stream_leap / 2
: stream_begin + stream_leap / 2;
const auto caught = cache.FindBuffer(caught_address, stream_page);
const auto untouched = cache.FindBuffer(untouched_address, stream_page);
auto grown = cache.FindBuffer(stream_begin, stream_page * 2);
constexpr uint32_t stream_value = 0x6a3b92d1u;
for (const auto address : {stream_begin, caught_address}) {
MarkGpuWrite(address, sizeof(stream_value));
cache.FillBuffer(address, sizeof(stream_value), stream_value, false);
}
uint64_t requested_begin = stream_begin;
uint64_t requested_end = stream_begin + stream_page * 2;
for (uint32_t step = 1; step <= 18; ++step) {
const auto address = (forward ? requested_end : requested_begin) - 4;
grown = cache.FindBuffer(address, 8);
if (forward) {
requested_end += stream_page;
} else {
requested_begin -= stream_page;
}
if (step < 18) {
const auto &owner = cache.GetBuffer(grown);
Require(stream_name, "growth before threshold",
owner.CpuAddress() == requested_begin &&
owner.Size() == requested_end - requested_begin &&
owner.StreamScore() == static_cast<int>(step),
"sequential requests grew beyond their consumed pages too "
"early");
}
}
const auto &grown_buffer = cache.GetBuffer(grown);
Require(stream_name, "growth reserve",
grown_buffer.CpuAddress() ==
(forward ? requested_begin
: requested_begin - stream_leap) &&
grown_buffer.Size() ==
requested_end - requested_begin + stream_leap &&
grown_buffer.StreamScore() == 0 &&
cache.GetBuffer(caught).is_deleted &&
!cache.GetBuffer(untouched).is_deleted &&
BufferCacheTestAccess::PageOwner(cache, caught_address) ==
grown &&
BufferCacheTestAccess::PageOwner(
cache, untouched_address) == untouched,
"streaming did not reserve the next pages or retain correct "
"neighboring owners");
for (uint32_t step = 0; step < 128; ++step) {
const auto address = forward
? requested_end + step * stream_page - 4
: requested_begin - step * stream_page - 4;
Require(stream_name, "reuse reserved pages",
cache.FindBuffer(address, 8) == grown,
"continued sequential requests replaced the buffer within "
"its growth reserve");
}
for (const auto address : {stream_begin, caught_address}) {
Require(
stream_name, "preserve GPU contents",
ReadNativeValue(grown_buffer, grown_buffer.Offset(address)) ==
stream_value,
"stream growth lost GPU-written contents from a merged owner");
}
}
resources.UnmapMemory(index_begin, index_span);
resources.UnmapMemory(base + recycled_offset, index_page);
Require(name, "invalidation retains owners",
!resources.IsMapped(index_begin, index_span) &&
!resources.IsMapped(base + recycled_offset, index_page) &&
cache.IsRegionRegistered(index_begin, index_span) &&
BufferCacheTestAccess::PageOwner(cache, index_begin) ==
index_bridge &&
BufferCacheTestAccess::PageOwner(
cache, index_begin + index_page) == index_bridge &&
BufferCacheTestAccess::PageOwner(
cache, index_begin + 2 * index_page) == index_bridge &&
BufferCacheTestAccess::PageOwner(
cache, base + recycled_offset) == recycled &&
cache.IsRegionCpuModified(index_begin, index_span) &&
cache.IsRegionCpuModified(base + recycled_offset,
index_page),
"CPU invalidation retired a reusable Buffer owner");
resources.MapMemory(index_begin, index_span);
resources.MapMemory(base + recycled_offset, index_page);
MarkGpuWrite(base + first_offset, sizeof(first_value));
MarkGpuWrite(base + second_offset, sizeof(second_value));
cache.FillBuffer(base + first_offset, sizeof(first_value), first_value,
false);
cache.FillBuffer(base + second_offset, sizeof(second_value), second_value,
false);
auto &download = BufferCacheTestAccess::DownloadBuffer(cache);
auto *fixed_download = &download;
const auto fixed_download_handle = download.Handle();
auto [ring_tail, ring_tail_offset] =
download.Map(download.Size() - sizeof(uint32_t), 4);
Require(name, "fault-ring tail reservation",
ring_tail != nullptr && ring_tail_offset == 0,
"failed to position the reusable download ring near wrap");
download.Commit();
MarkGpuWrite(base + ring_fault_first_offset,
sizeof(ring_fault_first_value));
MarkGpuWrite(base + ring_fault_second_offset,
sizeof(ring_fault_second_value));
cache.FillBuffer(base + ring_fault_first_offset,
sizeof(ring_fault_first_value), ring_fault_first_value,
false);
cache.FillBuffer(base + ring_fault_second_offset,
sizeof(ring_fault_second_value), ring_fault_second_value,
false);
Require(name, "fault-ring wrapped batch",
resources.HandleFault(PageFaultAccess::Read,
base + ring_fault_first_offset),
"fault readback could not wrap a live download-ring tick");
// Both disjoint writes on the faulting page share one wrapped batch.
uint64_t expected_packing_offset = 2 * 64;
uint64_t expected_packing_alignment = 1;
if (!download.IsCoherent()) {
Require(name, "fault-ring atom policy",
StreamBufferTestAccess::NormalizeReservation(
false,
m_runtime_context.physical_device_properties.limits
.nonCoherentAtomSize,
expected_packing_offset, expected_packing_alignment),
"failed to calculate non-coherent fault-ring stride");
}
auto [packing_probe, packing_offset] = download.Map(1, 1, false);
Require(name, "adjacent download reservation stride",
packing_probe != nullptr &&
packing_offset == expected_packing_offset,
"adjacent fault downloads did not reserve an atom-safe stride");
download.Commit();
uint32_t first_page_backing = 0;
uint32_t second_page_backing = 0;
uint32_t ring_fault_first_backing = 0;
uint32_t ring_fault_second_backing = 0;
Libs::LibKernel::Memory::TryReadBacking(base + first_offset,
&first_page_backing, sizeof(first_page_backing));
Libs::LibKernel::Memory::TryReadBacking(base + second_offset,
&second_page_backing, sizeof(second_page_backing));
std::memcpy(&ring_fault_first_backing, memory + ring_fault_first_offset,
sizeof(ring_fault_first_backing));
std::memcpy(&ring_fault_second_backing, memory + ring_fault_second_offset,
sizeof(ring_fault_second_backing));
Require(name, "fault-ring wrapped contents",
first_page_backing == first_stale &&
second_page_backing == second_stale &&
cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) &&
cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)) &&
ring_fault_first_backing == ring_fault_first_value &&
ring_fault_second_backing == ring_fault_second_value &&
&BufferCacheTestAccess::DownloadBuffer(cache) ==
fixed_download &&
download.Handle() == fixed_download_handle &&
download.Size() == (64ull << 20),
"wrapped fault batch lost its page contents or published an unrelated page");
cache.ReadMemory(base + first_offset, sizeof(first_value));
Libs::LibKernel::Memory::TryReadBacking(base + first_offset,
&first_page_backing, sizeof(first_page_backing));
Libs::LibKernel::Memory::TryReadBacking(base + second_offset,
&second_page_backing, sizeof(second_page_backing));
Require(name, "deferred neighboring page read",
first_page_backing == first_value && second_page_backing == second_value &&
!cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) &&
!cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)),
"the earlier page did not publish its dirty ranges on its own read");
constexpr uint64_t page_size = TRACKER_PAGE_SIZE;
constexpr uint64_t page_owner_offset = 0x240000;
constexpr uint64_t page_owner_size = 0xc0000;
constexpr uint64_t page_fault_offset = page_owner_offset + 0x100;
constexpr uint64_t page_inside_offset =
page_owner_offset + page_size - sizeof(uint32_t);
constexpr uint64_t page_outside_offset =
page_owner_offset + page_size;
constexpr uint32_t page_stale = 0x13579bdfu;
constexpr uint32_t page_value = 0x2468ace0u;
constexpr std::array page_offsets{
page_fault_offset, page_inside_offset, page_outside_offset};
static_assert((base + page_owner_offset) % page_size == 0);
for (const auto offset : page_offsets) {
Libs::LibKernel::Memory::WriteBacking(base + offset, &page_stale,
sizeof(page_stale));
}
(void)cache.FindBuffer(base + page_owner_offset, page_owner_size);
for (const auto offset : page_offsets) {
MarkGpuWrite(base + offset, sizeof(page_value));
cache.FillBuffer(base + offset, sizeof(page_value), page_value,
false);
}
const auto page_publication_tick = scheduler.CurrentTick();
std::binary_semaphore page_publication_entered{0};
std::binary_semaphore release_page_publication{0};
std::atomic<bool> page_readback_returned{false};
scheduler.DeferPriorityOperation([&] {
page_publication_entered.release();
release_page_publication.acquire();
Libs::LibKernel::Memory::WriteBacking(
base + page_inside_offset, &page_stale, sizeof(page_stale));
});
std::jthread release_page_callback([&] {
page_publication_entered.acquire();
const auto deadline = std::chrono::steady_clock::now() +
std::chrono::milliseconds(100);
while (!page_readback_returned.load() &&
std::chrono::steady_clock::now() < deadline) {
std::this_thread::yield();
}
release_page_publication.release();
});
cache.ReadMemory(base + page_fault_offset, sizeof(page_value));
page_readback_returned = true;
release_page_callback.join();
scheduler.WaitPriorityOperations(page_publication_tick);
uint32_t page_inside_backing = 0;
uint32_t page_outside_backing = 0;
Libs::LibKernel::Memory::TryReadBacking(base + page_inside_offset,
&page_inside_backing,
sizeof(page_inside_backing));
Libs::LibKernel::Memory::TryReadBacking(base + page_outside_offset,
&page_outside_backing,
sizeof(page_outside_backing));
Require(name, "page publication order and boundary",
page_inside_backing == page_value &&
page_outside_backing == page_stale &&
!cache.HasGpuDirtyBytes(base + page_inside_offset,
sizeof(page_value)) &&
cache.HasGpuDirtyBytes(base + page_outside_offset,
sizeof(page_value)) &&
!cache.IsRegionGpuModified(base + page_inside_offset,
sizeof(page_value)) &&
cache.IsRegionGpuModified(base + page_outside_offset,
sizeof(page_value)),
"readback did not publish after the older overlapping callback "
"or retained ownership of the wrong page");
cache.ReadMemory(base + page_outside_offset, sizeof(page_value));
Libs::LibKernel::Memory::TryReadBacking(base + page_outside_offset,
&page_outside_backing, sizeof(page_outside_backing));
Require(name, "next page publication",
page_outside_backing == page_value &&
!cache.HasGpuDirtyBytes(base + page_outside_offset,
sizeof(page_value)) &&
!cache.IsRegionGpuModified(base + page_outside_offset,
sizeof(page_value)),
"the next page did not publish and release ownership on its own read");
Libs::LibKernel::Memory::WriteBacking(base + first_offset, &first_stale,
sizeof(first_stale));
Libs::LibKernel::Memory::WriteBacking(base + second_offset, &second_stale,
sizeof(second_stale));
MarkGpuWrite(base + first_offset, sizeof(first_value));
MarkGpuWrite(base + second_offset, sizeof(second_value));
cache.FillBuffer(base + first_offset, sizeof(first_value), first_value,
false);
cache.FillBuffer(base + second_offset, sizeof(second_value), second_value,
false);
for (uint32_t tick = 0; tick < 160; tick++) {
cache.RunGarbageCollector();
}
Require(name, "age before pressure",
cache.IsRegionRegistered(base, allocation_size),
"buffer was reclaimed without memory pressure");
BufferCacheTestAccess::SetGarbageCollectionThresholds(
cache, 0, std::numeric_limits<uint64_t>::max());
const auto gc_submission_tick = scheduler.CurrentTick();
cache.RunGarbageCollector();
uint32_t first_before_completion = 0;
uint32_t second_before_completion = 0;
Libs::LibKernel::Memory::TryReadBacking(base + first_offset,
&first_before_completion,
sizeof(first_before_completion));
Libs::LibKernel::Memory::TryReadBacking(base + second_offset,
&second_before_completion,
sizeof(second_before_completion));
Require(name, "normal dirty retention",
first_before_completion == first_stale &&
second_before_completion == second_stale &&
cache.IsRegionRegistered(base, allocation_size) &&
cache.IsRegionGpuModified(base + first_offset,
sizeof(first_value)) &&
cache.IsRegionGpuModified(base + second_offset,
sizeof(second_value)) &&
cache.HasGpuDirtyBytes(base + first_offset,
sizeof(first_value)) &&
cache.HasGpuDirtyBytes(base + second_offset,
sizeof(second_value)) &&
scheduler.CurrentTick() == gc_submission_tick,
"normal GC retired or synchronized a dirty Buffer");
BufferCacheTestAccess::SetGarbageCollectionThresholds(cache, 0, 0);
const auto older_publication_tick = scheduler.CurrentTick();
std::binary_semaphore older_publication_entered{0};
std::binary_semaphore release_older_publication{0};
std::atomic<bool> gc_returned{false};
scheduler.DeferPriorityOperation([&] {
older_publication_entered.release();
release_older_publication.acquire();
Libs::LibKernel::Memory::WriteBacking(
base + first_offset, &first_stale, sizeof(first_stale));
});
std::jthread release_publication([&] {
older_publication_entered.acquire();
const auto deadline = std::chrono::steady_clock::now() +
std::chrono::milliseconds(100);
while (!gc_returned.load() &&
std::chrono::steady_clock::now() < deadline) {
std::this_thread::yield();
}
release_older_publication.release();
});
cache.RunGarbageCollector();
gc_returned = true;
release_publication.join();
scheduler.WaitPriorityOperations(older_publication_tick);
uint32_t first_backing = 0;
uint32_t second_backing = 0;
uint32_t clean_backing = 0;
std::memcpy(&first_backing, memory + first_offset, sizeof(first_backing));
std::memcpy(&second_backing, memory + second_offset,
sizeof(second_backing));
std::memcpy(&clean_backing, memory + clean_offset, sizeof(clean_backing));
Require(name, "downloaded contents",
first_backing == first_value && second_backing == second_value &&
clean_backing == clean_value &&
!cache.IsRegionRegistered(base, allocation_size) &&
!cache.IsRegionGpuModified(base + first_offset,
sizeof(first_value)) &&
!cache.IsRegionGpuModified(base + second_offset,
sizeof(second_value)) &&
!cache.HasGpuDirtyBytes(base + first_offset,
sizeof(first_value)) &&
!cache.HasGpuDirtyBytes(base + second_offset,
sizeof(second_value)) &&
scheduler.CurrentTick() > gc_submission_tick,
"critical GC did not publish exact ranges before clearing "
"Buffer ownership");
constexpr uint64_t starvation_offset = 0x100000;
constexpr uint64_t starvation_stride = 2 * BufferCache::CACHING_PAGESIZE;
constexpr uint32_t starvation_count = 33;
constexpr uint32_t starvation_value = 0x31415926u;
BufferCacheTestAccess::SetGarbageCollectionThresholds(
cache, std::numeric_limits<uint64_t>::max(),
std::numeric_limits<uint64_t>::max());
for (uint32_t index = 0; index < starvation_count; index++) {
const auto address = base + starvation_offset + index * starvation_stride;
(void)cache.ObtainBuffer(address, sizeof(starvation_value), true);
cache.FillBuffer(address, sizeof(starvation_value), starvation_value,
false);
}
for (uint32_t tick = 0; tick < 160; tick++) {
cache.RunGarbageCollector();
}
constexpr uint64_t starvation_clean_offset =
starvation_offset + starvation_count * starvation_stride;
const auto starvation_clean = cache.FindBuffer(
base + starvation_clean_offset, sizeof(starvation_value));
Require(name, "normal-GC clean candidate",
static_cast<bool>(starvation_clean),
"failed to create the clean starvation candidate");
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
BufferCacheTestAccess::SetGarbageCollectionThresholds(
cache, 0, std::numeric_limits<uint64_t>::max());
cache.RunGarbageCollector();
Require(
name, "normal-GC dirty bypass",
cache.IsRegionRegistered(base + starvation_offset,
sizeof(starvation_value)) &&
cache.IsRegionRegistered(
base + starvation_offset +
(starvation_count - 1) * starvation_stride,
sizeof(starvation_value)) &&
!cache.IsRegionRegistered(base + starvation_clean_offset,
sizeof(starvation_value)),
"old dirty owners hid an eligible clean Buffer from normal GC");
BufferCacheTestAccess::SetGarbageCollectionThresholds(cache, 0, 0);
const auto starvation_retired =
BufferCacheTestAccess::PageOwner(cache, base + starvation_offset);
cache.RunGarbageCollector();
Require(name, "critical-GC starvation cleanup",
!cache.IsRegionRegistered(base + starvation_offset,
sizeof(starvation_value)) &&
!cache.IsRegionRegistered(
base + starvation_offset +
(starvation_count - 1) * starvation_stride,
sizeof(starvation_value)) &&
!BufferCacheTestAccess::IsBufferAllocated(
cache, starvation_retired),
"critical GC did not immediately free the skipped dirty owners");
constexpr uint64_t lookup_only_offset = 0x218000;
constexpr uint64_t obtained_offset = 0x220000;
constexpr uint64_t residency_size =
2 * BufferCache::CACHING_PAGESIZE;
BufferCacheTestAccess::SetGarbageCollectionThresholds(
cache, std::numeric_limits<uint64_t>::max(),
std::numeric_limits<uint64_t>::max());
const auto lookup_only = cache.FindBuffer(
base + lookup_only_offset, residency_size);
const auto obtained =
cache.FindBuffer(base + obtained_offset, residency_size);
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
Require(name, "lookup-only owner identity",
cache.FindBuffer(base + lookup_only_offset, residency_size) ==
lookup_only,
"lookup-only discovery replaced its valid Buffer owner");
const auto [obtained_buffer, obtained_buffer_offset] = cache.ObtainBuffer(
base + obtained_offset, residency_size, false, false, obtained);
BufferCacheTestAccess::SetGarbageCollectionThresholds(
cache, 0, std::numeric_limits<uint64_t>::max());
cache.RunGarbageCollector();
Require(name, "lookup versus acquisition residency",
!cache.IsRegionRegistered(base + lookup_only_offset,
residency_size) &&
!BufferCacheTestAccess::PageOwner(
cache, base + lookup_only_offset) &&
obtained_buffer == &cache.GetBuffer(obtained) &&
obtained_buffer_offset == 0 &&
cache.IsRegionRegistered(base + obtained_offset,
residency_size) &&
BufferCacheTestAccess::PageOwner(
cache, base + obtained_offset) == obtained,
"lookup refreshed residency or actual acquisition did not");
BufferCacheTestAccess::SetGarbageCollectionThresholds(cache, 0, 0);
auto unmap_allocation = cache.ObtainBuffer(
base + unmap_offset, sizeof(unmap_value), true, false);
Require(name, "direct-invalidation allocation",
unmap_allocation.first != nullptr,
"direct-invalidation buffer allocation failed");
cache.FillBuffer(base + unmap_offset, sizeof(unmap_value), unmap_value,
false);
resources.UnmapMemory(base + 0x4000, 0x4000);
uint32_t unmap_backing = 0;
std::memcpy(&unmap_backing, memory + unmap_offset, sizeof(unmap_backing));
Require(name, "direct-invalidation contents",
unmap_backing == unmap_value &&
!resources.IsMapped(base + 0x4000, 0x4000) &&
cache.IsRegionRegistered(base + 0x4000, 0x4000) &&
cache.IsRegionCpuModified(base + 0x4000, 0x4000) &&
!cache.IsRegionGpuModified(base + 0x4000, 0x4000) &&
!cache.HasGpuDirtyBytes(base + 0x4000, 0x4000),
"dirty invalidation did not publish bytes and retain its owner");
std::memcpy(memory + unmap_offset, &remap_value, sizeof(remap_value));
resources.MapMemory(base + 0x4000, 0x4000);
const auto remapped =
cache.ObtainBuffer(base + unmap_offset, sizeof(remap_value), true);
Require(name, "same-address remap allocation",
remapped.first == unmap_allocation.first,
"same-address remap did not synchronize the retained Buffer");
Require(name, "same-address remap contents",
ReadNativeValue(*remapped.first, remapped.second) == remap_value,
"same-address remap reused stale native Buffer bytes");
auto partial_unmap_allocation =
cache.ObtainBuffer(base + 0x8000, 0x8000, true, false);
Require(name, "partial-invalidation allocation",
partial_unmap_allocation.first != nullptr,
"cross-page cached buffer allocation failed");
cache.FillBuffer(base + partial_unmap_offset, sizeof(partial_unmap_value),
partial_unmap_value, false);
cache.FillBuffer(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value),
partial_unmap_survivor_value, false);
resources.UnmapMemory(base + 0x8000, 0x4000);
Require(
name, "partial-invalidation keeps disjoint bytes dirty",
resources.IsMapped(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
!cache.IsRegionCpuModified(
base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
cache.IsRegionGpuModified(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
cache.HasGpuDirtyBytes(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
!cache.HasGpuDirtyBytes(base + partial_unmap_offset,
sizeof(partial_unmap_value)),
"partial invalidation read back an unrelated mapped page");
Require(name, "partial-invalidation survivor read fault",
resources.HandleFault(PageFaultAccess::Read,
base + partial_unmap_survivor_offset),
"CPU read did not publish the retained GPU-dirty page");
uint32_t partial_unmap_survivor_backing = 0;
std::memcpy(&partial_unmap_survivor_backing,
memory + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_backing));
Require(
name, "partial-invalidation ownership",
!resources.IsMapped(base + 0x8000, 0x4000) &&
cache.IsRegionRegistered(base + 0x8000, 0x8000) &&
cache.IsRegionCpuModified(base + 0x8000, 0x4000) &&
!cache.IsRegionCpuModified(
base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
!cache.HasGpuDirtyBytes(base + partial_unmap_offset,
sizeof(partial_unmap_value)) &&
!cache.HasGpuDirtyBytes(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
partial_unmap_survivor_backing == partial_unmap_survivor_value,
"partial invalidation or subsequent CPU read mishandled ownership or "
"backing");
resources.MapMemory(base + 0x8000, 0x4000);
auto survivor =
cache.ObtainBuffer(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value), false);
Require(name, "partial-invalidation survivor", survivor.first != nullptr,
"cached-buffer remainder was not retained");
Require(name, "partial-invalidation survivor contents",
ReadNativeValue(*survivor.first, survivor.second) ==
partial_unmap_survivor_value,
"partial invalidation lost disjoint native bytes");
for (const uint64_t large_size : {64ull << 20, 68ull << 20}) {
constexpr uint64_t large_offset = 0x10000;
const uint32_t large_value =
large_size == (64ull << 20) ? 0x5aa55aa5u : 0xa55aa55au;
constexpr uint32_t large_stale = 0x12345678u;
std::memcpy(memory + large_offset, &large_stale, sizeof(large_stale));
std::memcpy(memory + large_offset + large_size - sizeof(large_stale),
&large_stale, sizeof(large_stale));
auto large_allocation =
cache.ObtainBuffer(base + large_offset, large_size, true, false);
Require(name, "large dirty allocation",
large_allocation.first != nullptr,
"failed to allocate the large dirty native buffer");
cache.FillBuffer(base + large_offset, large_size, large_value, false);
const auto large_submission_tick = scheduler.CurrentTick();
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
// The capacity-sized case can wrap the ring; the oversized case must
// keep its separate staging allocation alive through publication.
Require(name, "large synchronized retirement",
!cache.IsRegionRegistered(base + large_offset, large_size) &&
scheduler.CurrentTick() > large_submission_tick &&
scheduler.CurrentTick() <= large_submission_tick + 2,
"large dirty Buffer exceeded the GC synchronization budget");
uint32_t large_before_completion = 0;
Libs::LibKernel::Memory::TryReadBacking(base + large_offset,
&large_before_completion,
sizeof(large_before_completion));
Require(name, "large backing publication",
large_before_completion == large_value,
"large Buffer GC retired ownership before publication");
const auto large_image_source =
cache.ObtainBufferForImage(base + large_offset, sizeof(large_value));
Require(name, "fixed download after Buffer retirement",
large_image_source.first != nullptr &&
&BufferCacheTestAccess::DownloadBuffer(cache) ==
fixed_download &&
fixed_download->Handle() == fixed_download_handle &&
fixed_download->Size() == (64ull << 20),
"image acquisition replaced the shared Buffer download stream");
std::vector<uint32_t> large_published(large_size / sizeof(uint32_t));
Require(name, "large Buffer publication contents",
Libs::LibKernel::Memory::TryReadBacking(
base + large_offset, large_published.data(), large_size) &&
std::ranges::all_of(large_published, [large_value](uint32_t value) {
return value == large_value;
}),
"large Buffer GC did not publish its complete transfer");
}
constexpr uint64_t grouped_first_offset = 0x10000;
constexpr uint64_t grouped_second_offset = 0x2300000;
constexpr uint64_t grouped_owner_size = 33ull * 1024 * 1024;
constexpr uint32_t grouped_first_value = 0x1122aabbu;
constexpr uint32_t grouped_second_value = 0x3344ccddu;
constexpr uint32_t grouped_stale = 0;
static_assert(grouped_owner_size < (64ull << 20) &&
grouped_owner_size * 2 > (64ull << 20));
Libs::LibKernel::Memory::WriteBacking(
base + grouped_first_offset, &grouped_stale, sizeof(grouped_stale));
Libs::LibKernel::Memory::WriteBacking(
base + grouped_second_offset, &grouped_stale, sizeof(grouped_stale));
auto grouped_first = cache.ObtainBuffer(base + grouped_first_offset,
grouped_owner_size, true, false);
auto grouped_second = cache.ObtainBuffer(base + grouped_second_offset,
grouped_owner_size, true, false);
Require(name, "per-owner GC allocation",
grouped_first.first != nullptr &&
grouped_second.first != nullptr &&
grouped_first.first != grouped_second.first,
"disjoint GC candidates merged into one Buffer owner");
cache.FillBuffer(base + grouped_first_offset, grouped_owner_size,
grouped_first_value, false);
cache.FillBuffer(base + grouped_second_offset, grouped_owner_size,
grouped_second_value, false);
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
Require(name, "per-owner fixed-ring retirement",
!cache.IsRegionRegistered(base + grouped_first_offset,
grouped_owner_size) &&
!cache.IsRegionRegistered(base + grouped_second_offset,
grouped_owner_size) &&
&BufferCacheTestAccess::DownloadBuffer(cache) ==
fixed_download &&
fixed_download->Handle() == fixed_download_handle,
"GC aggregated disjoint retirees or replaced the download ring");
uint32_t grouped_first_backing = 0;
uint32_t grouped_second_backing = 0;
Libs::LibKernel::Memory::TryReadBacking(base + grouped_first_offset,
&grouped_first_backing,
sizeof(grouped_first_backing));
Libs::LibKernel::Memory::TryReadBacking(base + grouped_second_offset,
&grouped_second_backing,
sizeof(grouped_second_backing));
Require(name, "per-owner fixed-ring contents",
grouped_first_backing == grouped_first_value &&
grouped_second_backing == grouped_second_value,
"per-owner GC transfers lost data while wrapping the fixed ring");
constexpr uint64_t sparse_offset = 0x2180000;
constexpr uint64_t sparse_owner_stride = 2 * BufferCache::CACHING_PAGESIZE;
constexpr std::array<uint64_t, 3> sparse_starts{1, 7, 13};
constexpr std::array<uint64_t, 3> sparse_sizes{3, 2, 1};
constexpr std::array<uint32_t, 2> sparse_values{0x41372b19u, 0x957f6953u};
std::array<uint8_t, 16> sparse_clean{};
sparse_clean.fill(0xcdu);
const auto FillSparseOwner = [&](uint32_t owner_index) {
const auto address = base + sparse_offset +
owner_index * sparse_owner_stride;
Libs::LibKernel::Memory::WriteBacking(address, sparse_clean.data(),
sparse_clean.size());
Libs::Graphics::Buffer *owner = nullptr;
for (size_t index = 0; index < sparse_starts.size(); ++index) {
const auto allocation = cache.ObtainBuffer(
address + sparse_starts[index], sparse_sizes[index], true, false);
Require(name, "sparse byte-range owner",
allocation.first != nullptr &&
(owner == nullptr || owner == allocation.first),
"disjoint byte ranges did not resolve to one cached owner");
owner = allocation.first;
}
// Deliberately change native clean neighbors too: only the exact dirty
// bytes may be published to guest memory.
owner->Fill(owner->Offset(address), sparse_clean.size(),
sparse_values[owner_index]);
};
const auto CheckSparseOwner = [&](uint32_t owner_index) {
const auto address = base + sparse_offset +
owner_index * sparse_owner_stride;
auto expected = sparse_clean;
std::array<uint8_t, sizeof(uint32_t)> value_bytes{};
std::memcpy(value_bytes.data(), &sparse_values[owner_index],
value_bytes.size());
for (size_t index = 0; index < sparse_starts.size(); ++index) {
for (uint64_t byte = sparse_starts[index];
byte < sparse_starts[index] + sparse_sizes[index]; ++byte) {
expected[byte] = value_bytes[byte % value_bytes.size()];
}
}
std::array<uint8_t, 16> published{};
Require(name, "sparse byte-range publication",
Libs::LibKernel::Memory::TryReadBacking(
address, published.data(), published.size()) &&
published == expected &&
!cache.HasGpuDirtyBytes(address, published.size()) &&
!cache.IsRegionGpuModified(address, published.size()),
"odd-byte readback lost dirty bytes or overwrote clean neighbors");
};
FillSparseOwner(0);
bool sparse_callback_ran = false;
bool sparse_callback_clean = false;
scheduler.DeferOperation([&] {
sparse_callback_clean =
!cache.HasGpuDirtyBytes(base + sparse_offset, sparse_clean.size()) &&
!cache.IsRegionGpuModified(base + sparse_offset, sparse_clean.size());
sparse_callback_ran = true;
});
const auto sparse_read_tick = scheduler.CurrentTick();
cache.ReadMemory(base + sparse_offset + sparse_starts[0], sparse_sizes[0]);
Require(name, "sparse byte-range submission",
scheduler.CurrentTick() == sparse_read_tick + 1,
"sparse source ranges were downloaded in separate submissions");
scheduler.PopPendingOperations();
Require(name, "readback callback ownership consistency",
sparse_callback_ran && sparse_callback_clean,
"readback drained a deferred callback before clearing page ownership");
CheckSparseOwner(0);
FillSparseOwner(0);
FillSparseOwner(1);
Require(name, "sparse disjoint GC owners",
BufferCacheTestAccess::PageOwner(cache, base + sparse_offset) !=
BufferCacheTestAccess::PageOwner(
cache, base + sparse_offset + sparse_owner_stride),
"sparse GC fixtures merged into one source owner");
const auto sparse_gc_tick = scheduler.CurrentTick();
for (uint32_t tick = 0; tick <= 160; ++tick) {
cache.RunGarbageCollector();
}
Require(name, "sparse multi-owner GC submission",
scheduler.CurrentTick() == sparse_gc_tick + 1 &&
!cache.IsRegionRegistered(base + sparse_offset,
sparse_owner_stride + 16),
"sparse GC owners did not share one completion drain");
CheckSparseOwner(0);
CheckSparseOwner(1);
constexpr uint64_t disjoint_owner_offset = 0x2140000;
constexpr uint64_t disjoint_owner_size = 0x8000;
constexpr uint64_t disjoint_dirty_offset = disjoint_owner_offset + 0x100;
constexpr uint64_t disjoint_new_owner_offset =
disjoint_owner_offset + 0x5000;
constexpr uint32_t disjoint_value = 0x89abcdefu;
constexpr uint32_t disjoint_stale = 0x76543210u;
std::memcpy(memory + disjoint_dirty_offset, &disjoint_stale,
sizeof(disjoint_stale));
auto wide_owner = cache.ObtainBuffer(base + disjoint_owner_offset,
disjoint_owner_size, false);
Require(name, "disjoint retirement owner", wide_owner.first != nullptr,
"failed to create the clean cross-page retirement owner");
(void)cache.ObtainBuffer(base + disjoint_dirty_offset,
sizeof(disjoint_value), true, false);
cache.FillBuffer(base + disjoint_dirty_offset, sizeof(disjoint_value),
disjoint_value, false);
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
Require(name, "disjoint synchronized retirement",
!cache.IsRegionRegistered(base + disjoint_owner_offset,
disjoint_owner_size),
"cross-page owner survived pressured collection");
uint32_t disjoint_before_unmap = 0;
Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset,
&disjoint_before_unmap,
sizeof(disjoint_before_unmap));
Require(
name, "disjoint retirement publication",
disjoint_before_unmap == disjoint_value,
"whole-owner Buffer retired before publishing its dirty bytes");
resources.UnmapMemory(base + disjoint_owner_offset + 0x4000, 0x4000);
uint32_t disjoint_after_unmap = 0;
Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset,
&disjoint_after_unmap,
sizeof(disjoint_after_unmap));
Require(name, "disjoint synchronized invalidation",
disjoint_after_unmap == disjoint_value &&
!cache.HasGpuDirtyBytes(base + disjoint_dirty_offset,
sizeof(disjoint_value)),
"disjoint invalidation lost the completed whole-owner publication");
resources.MapMemory(base + disjoint_owner_offset + 0x4000, 0x4000);
auto disjoint_new_owner =
cache.ObtainBuffer(base + disjoint_new_owner_offset,
sizeof(disjoint_value), true, false);
Require(name, "disjoint post-publication reacquire",
disjoint_new_owner.first != nullptr,
"disjoint acquisition failed after whole-owner publication");
uint32_t disjoint_backing = 0;
Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset,
&disjoint_backing,
sizeof(disjoint_backing));
Require(name, "disjoint retirement contents",
disjoint_backing == disjoint_value,
"old retirement callback lost dirty bytes or retained ownership "
"after a disjoint reacquire");
cache.ReadMemory(base + disjoint_new_owner_offset,
sizeof(disjoint_value));
constexpr uint64_t reacquire_owner_offset = 0x2200000;
constexpr uint64_t reacquire_owner_size = 0x8000;
constexpr uint64_t reacquire_dirty_offset =
reacquire_owner_offset + 0x100;
constexpr uint64_t reacquire_disjoint_offset =
reacquire_owner_offset + 0x5000;
constexpr uint32_t reacquire_value = 0xc001d00du;
constexpr uint32_t reacquire_stale = 0x0badf00du;
std::memcpy(memory + reacquire_dirty_offset, &reacquire_stale,
sizeof(reacquire_stale));
auto reacquire_owner = cache.ObtainBuffer(base + reacquire_owner_offset,
reacquire_owner_size, false);
Require(name, "reacquire retirement owner",
reacquire_owner.first != nullptr,
"failed to create a fresh cross-page retirement owner");
(void)cache.ObtainBuffer(base + reacquire_dirty_offset,
sizeof(reacquire_value), true, false);
cache.FillBuffer(base + reacquire_dirty_offset, sizeof(reacquire_value),
reacquire_value, false);
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
Require(name, "reacquire synchronized retirement",
!cache.IsRegionRegistered(base + reacquire_owner_offset,
reacquire_owner_size),
"fresh cross-page owner survived pressured collection");
uint32_t reacquire_before = 0;
Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset,
&reacquire_before,
sizeof(reacquire_before));
Require(name, "reacquire publication completed",
reacquire_before == reacquire_value,
"fresh whole-owner ownership retired before publication");
auto reacquired =
cache.ObtainBuffer(base + reacquire_disjoint_offset,
sizeof(reacquire_value), true, false);
Require(name, "independent disjoint post-publication reacquire",
reacquired.first != nullptr,
"clean disjoint-half acquisition failed after publication");
uint32_t reacquire_after = 0;
Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset,
&reacquire_after,
sizeof(reacquire_after));
Require(name, "independent disjoint reacquire contents",
reacquire_after == reacquire_value &&
!cache.HasGpuDirtyBytes(base + reacquire_dirty_offset,
sizeof(reacquire_value)),
"disjoint-half acquisition failed to publish the retired "
"owner's dirty prefix");
cache.ReadMemory(base + reacquire_disjoint_offset,
sizeof(reacquire_value));
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
context.ShutdownGpu();
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"dirty-GC direct-memory mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"dirty-GC direct-memory allocation release failed");
std::printf("[host] %-32s ok\n", name);
}
void CheckComputeMetaClearClassification() {
constexpr const char *name = "ComputeMetaClearClassification";
constexpr uint64_t read_only_meta = 0x0000000204201f00ull;
constexpr uint64_t read_write_meta = 0x0000000204202000ull;
constexpr uint64_t write_only_meta = 0x0000000204202100ull;
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto &texture_cache = context.GetTextureCache();
TextureCacheTestAccess::RegisterHtileMeta(texture_cache, read_only_meta);
TextureCacheTestAccess::RegisterHtileMeta(texture_cache, read_write_meta);
TextureCacheTestAccess::RegisterHtileMeta(texture_cache, write_only_meta);
// An empty dispatch must not inspect even an unreadable shader address.
shaders.SetCsShader({.data_addr = 1});
constexpr std::array<std::array<uint32_t, 3>, 3> empty_dispatches{{
{0, 1, 1}, {1, 0, 1}, {1, 1, 0}}};
for (const auto mode : {0x41u, 0x61u}) {
for (const auto &groups : empty_dispatches) {
context.GetRenderExecutor().DispatchDirect(
0, scheduler.Current(), groups[0], groups[1], groups[2], mode);
}
}
Require(name, "HTile fixture",
texture_cache.IsMeta(read_only_meta) &&
texture_cache.IsMeta(read_write_meta) &&
texture_cache.IsMeta(write_only_meta) &&
!texture_cache.IsMetaCleared(read_only_meta, 0) &&
!texture_cache.IsMetaCleared(read_write_meta, 0) &&
!texture_cache.IsMetaCleared(write_only_meta, 0),
"test HTile entries were not registered in their initial state");
const auto MakeInput = [](uint64_t address, bool read, bool written,
ShaderRecompiler::IR::CompiledShaderInfo &program,
ShaderRecompiler::IR::ResourceSnapshot &snapshot) {
program.stage = ShaderType::Compute;
ShaderRecompiler::IR::BufferResource resource{};
resource.read = read;
resource.written = written;
resource.formatted = true;
program.info.buffers.push_back(resource);
ShaderBufferResource descriptor{};
descriptor.UpdateAddress48(address);
descriptor.fields[1] |= 16u << 16u;
descriptor.fields[2] = 8;
descriptor.fields[3] =
DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32_32_32_32UInt)
<< 12u);
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 4;
std::copy_n(descriptor.fields, value.dword_count, value.dwords.begin());
ShaderComputeInputInfo input{};
input.stage.program = &program;
snapshot.buffers.push_back(value);
input.stage.resources = &snapshot;
return input;
};
auto &executor = context.GetRenderExecutor();
auto &command = scheduler.Current();
ShaderRecompiler::IR::CompiledShaderInfo read_only_program{};
ShaderRecompiler::IR::ResourceSnapshot read_only_snapshot;
const auto read_only_input =
MakeInput(read_only_meta, true, false, read_only_program,
read_only_snapshot);
const bool read_only_consumed =
RenderExecutorTestAccess::TryConsumeComputeMetaClear(
executor, read_only_input, command);
Require(name, "read-only dispatch preserved",
!read_only_consumed &&
!texture_cache.IsMetaCleared(read_only_meta, 0),
"a metadata read-only dispatch changed logical clear state");
ShaderRecompiler::IR::CompiledShaderInfo read_write_program{};
ShaderRecompiler::IR::ResourceSnapshot read_write_snapshot;
const auto read_write_input =
MakeInput(read_write_meta, true, true, read_write_program,
read_write_snapshot);
const bool read_write_consumed =
RenderExecutorTestAccess::TryConsumeComputeMetaClear(
executor, read_write_input, command);
Require(name, "read/write dispatch preserved",
!read_write_consumed &&
!texture_cache.IsMetaCleared(read_write_meta, 0),
"a metadata read/modify/write dispatch was replaced by a clear");
ShaderRecompiler::IR::CompiledShaderInfo write_only_program{};
ShaderRecompiler::IR::ResourceSnapshot write_only_snapshot;
const auto write_only_input =
MakeInput(write_only_meta, false, true, write_only_program,
write_only_snapshot);
const bool write_only_consumed =
RenderExecutorTestAccess::TryConsumeComputeMetaClear(
executor, write_only_input, command);
Require(name, "write-only fill consumed",
write_only_consumed &&
texture_cache.IsMetaCleared(write_only_meta, 0),
"a metadata write-only fill was not consumed as a clear");
scheduler.Finish();
std::printf("[host] %-32s ok\n", name);
}
void CheckBdaPageTableUploads() {
constexpr const char *name = "BdaPageTableUploads";
constexpr uintptr_t base = 0x0000000207b00000ull;
constexpr uint64_t page = BufferCache::CACHING_PAGESIZE;
constexpr std::array<uint64_t, 3> page_counts{2, 521, 525};
constexpr uint64_t result_entries = page_counts[0] + page_counts[1] + page_counts[2];
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto &cache = context.GetBufferCache();
// A small real ring exercises chunking and waits for same-command-buffer reuse.
BufferCacheTestAccess::SetUnusedStagingBufferSize(cache, 4096);
auto &table = *cache.GetBdaPageTableBuffer();
const auto table_offset = BufferCache::PageIndex(base) * sizeof(vk::DeviceAddress);
auto readback = CreateHostBuffer(name, result_entries * sizeof(vk::DeviceAddress),
vk::BufferUsageFlagBits::eTransferDst, {});
std::vector<uint64_t> expected;
expected.reserve(result_entries);
for (const auto pages : page_counts) {
// The destination has a preceding GPU write and, after the first iteration,
// a preceding read. Registration must preserve both ordering requirements.
table.Fill(table_offset, pages * sizeof(vk::DeviceAddress), 0xa5a5a5a5u);
const auto previous_tick = scheduler.CurrentTick();
const auto id = cache.FindBuffer(base, pages * page);
const auto &buffer = cache.GetBuffer(id);
const auto destination_offset = expected.size() * sizeof(vk::DeviceAddress);
for (uint64_t index = 0; index < pages; ++index) {
expected.push_back(buffer.BufferDeviceAddress() + index * page);
}
const vk::BufferCopy copy{table_offset, destination_offset,
pages * sizeof(vk::DeviceAddress)};
scheduler.Current().Handle().copyBuffer(table.Handle(), readback.buffer, 1, &copy);
Require(name, "chunked registration",
buffer.CpuAddress() == base && buffer.Size() == pages * page &&
(pages == page_counts.front() || scheduler.CurrentTick() > previous_tick),
"BDA registration failed to replace its owner or wait for staging-ring reuse");
}
vk::BufferMemoryBarrier barrier{};
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = readback.buffer;
barrier.size = readback.size;
scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr);
scheduler.Finish();
const auto words = ReadBuffer(name, readback, result_entries * 2);
for (size_t index = 0; index < expected.size(); ++index) {
const auto actual = uint64_t{words[index * 2]} | (uint64_t{words[index * 2 + 1]} << 32);
Require(name, "GPU page-table snapshot", actual == expected[index],
"staged BDA updates were stale, overwritten before their read, or incompletely flushed");
}
DestroyBuffer(&readback);
std::printf("[gpu] %-32s ok\n", name);
}
void CheckUnifiedTextureCacheFlow() {
constexpr const char *name = "UnifiedTextureCacheFlow";
constexpr uintptr_t base = 0x0000000200600000ull;
constexpr uint64_t allocation_size = 0x3000000;
constexpr uint64_t allocation_alignment = 0x200000;
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.InitializeGpu(nullptr);
auto &gpu = context.GetGpu();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"cache test direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"fixed cache direct-memory mapping failed");
auto *memory = static_cast<uint8_t *>(mapped);
const uint32_t initial = 0x44332211u;
std::memcpy(memory, &initial, sizeof(initial));
{
auto &resources = context;
LibKernel::Memory::InstallGpuResources(&resources);
auto &texture_cache = resources.GetTextureCache();
const auto [narrow_download, narrow_download_offset] =
TextureCacheTestAccess::MapDownload(texture_cache, 4, 4);
const auto [wide_download, wide_download_offset] =
TextureCacheTestAccess::MapDownload(texture_cache, 1, 16);
Require(name, "image download alignment",
narrow_download != nullptr && narrow_download_offset % 4 == 0 &&
wide_download != nullptr && wide_download_offset % 16 == 0,
"wide/block image readback was not aligned to its texel block");
resources.MapMemory(base, allocation_size);
ImageDesc sampled{};
sampled.type = BindingType::Texture;
sampled.info.data = {base, sizeof(initial)};
sampled.info.pixel_format = vk::Format::eR8G8B8A8Srgb;
sampled.info.guest_format = Prospero::BufferFormat::k8_8_8_8Srgb;
sampled.info.type = Prospero::ImageType::kColor2D;
sampled.info.extent = {1, 1, 1};
sampled.info.resources = {1, 1};
sampled.info.pitch = 1;
sampled.info.bytes_per_block = 4;
sampled.info.samples = 1;
sampled.info.tile_mode = Prospero::TileMode::kLinear;
sampled.info.mip_layout[0] = {0, 4, 1, 1};
sampled.view_info.format = sampled.info.pixel_format;
sampled.view_info.type = vk::ImageViewType::e2D;
sampled.view_info.aspect = vk::ImageAspectFlagBits::eColor;
sampled.view_info.usage = vk::ImageUsageFlagBits::eSampled;
auto &command = scheduler.Current();
auto first_desc = sampled;
const auto first = texture_cache.FindImage(first_desc);
auto repeated_desc = sampled;
const auto repeated = texture_cache.FindImage(repeated_desc);
auto compatible_desc = sampled;
compatible_desc.info.pixel_format = vk::Format::eR8G8B8A8Uint;
compatible_desc.info.guest_format = Prospero::BufferFormat::k8_8_8_8UInt;
compatible_desc.view_info.format = compatible_desc.info.pixel_format;
const auto compatible = texture_cache.FindImage(compatible_desc);
Require(name, "normalized FindImage",
first && repeated == first && compatible == first &&
texture_cache.GetImage(first).info.pixel_format ==
vk::Format::eR8G8B8A8Srgb,
"registered compatible backing did not reuse one ImageId");
const auto IsOnlyImage = [](const std::vector<ImageId> &ids,
ImageId expected) {
return ids.size() == 1 && ids.front() == expected;
};
const auto exact_byte_miss =
TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, false);
const auto touched_page_hit =
TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, true);
const auto next_page_miss = TextureCacheTestAccess::FindImages(
texture_cache, base + 0x1000, 1, true);
Require(name, "exact and 4-KiB page filtering",
exact_byte_miss.empty() && IsOnlyImage(touched_page_hit, first) &&
next_page_miss.empty(),
"coarse candidates did not preserve exact-byte and touched-page "
"semantics");
const auto MakeOwnershipInfo = [](uint64_t address, uint64_t size) {
ImageInfo info{};
info.data = {address, size};
info.extent = {1, 1, 1};
info.resources = {1, 1};
info.samples = 1;
return info;
};
const auto spanning_info = MakeOwnershipInfo(base + 0x1ff000, 0x2000);
const auto spanning =
TextureCacheTestAccess::InsertImage(texture_cache, spanning_info);
const auto spanning_results = TextureCacheTestAccess::FindImages(
texture_cache, spanning_info.data.address, spanning_info.data.size,
false);
Require(name, "one-MiB cross-page deduplication",
spanning && IsOnlyImage(spanning_results, spanning) &&
TextureCacheTestAccess::PageOwnerCount(
texture_cache, spanning_info.data.address) == 1 &&
TextureCacheTestAccess::PageOwnerCount(
texture_cache, spanning_info.data.End() - 1) == 1,
"an image spanning two coarse pages was missing or returned more "
"than once");
TextureCacheTestAccess::DeleteImage(texture_cache, spanning);
Require(name, "cross-page owner cleanup",
TextureCacheTestAccess::PageOwnerCount(
texture_cache, spanning_info.data.address) == 0 &&
TextureCacheTestAccess::PageOwnerCount(
texture_cache, spanning_info.data.End() - 1) == 0 &&
TextureCacheTestAccess::FindImages(
texture_cache, spanning_info.data.address,
spanning_info.data.size, false)
.empty(),
"cross-page unregister left a stale coarse-page membership");
const auto shared_info = MakeOwnershipInfo(base + 0x500100, 0x100);
const auto shared_first =
TextureCacheTestAccess::InsertImage(texture_cache, shared_info);
const auto shared_second =
TextureCacheTestAccess::InsertImage(texture_cache, shared_info);
const auto shared_results = TextureCacheTestAccess::FindImages(
texture_cache, shared_info.data.address, shared_info.data.size,
false);
Require(name, "shared coarse-page registration",
shared_results.size() == 2 &&
std::find(shared_results.begin(), shared_results.end(),
shared_first) != shared_results.end() &&
std::find(shared_results.begin(), shared_results.end(),
shared_second) != shared_results.end(),
"two registered owners were not retained in one coarse page");
TextureCacheTestAccess::DeleteImage(texture_cache, shared_first);
const auto shared_survivor = TextureCacheTestAccess::FindImages(
texture_cache, shared_info.data.address, shared_info.data.size,
false);
Require(name, "shared coarse-page unregister",
IsOnlyImage(shared_survivor, shared_second) &&
TextureCacheTestAccess::PageOwnerCount(
texture_cache, shared_info.data.address) == 1,
"unregistering one owner removed its coarse-page neighbor");
TextureCacheTestAccess::DeleteImage(texture_cache, shared_second);
Require(name, "final shared coarse-page unregister",
TextureCacheTestAccess::PageOwnerCount(
texture_cache, shared_info.data.address) == 0,
"the final shared owner remained registered");
constexpr uint64_t large_owner_size = 64ull * 1024 * 1024;
const auto large_info =
MakeOwnershipInfo(base + 0x800000, large_owner_size);
const auto large_owner =
TextureCacheTestAccess::InsertImage(texture_cache, large_info);
Require(name, "production one-MiB registration granularity",
TextureCacheTestAccess::OwnedPageCount(
texture_cache, large_info.data.address, large_info.data.size,
large_owner) == 64,
"64 MiB image registration did not create exactly 64 coarse "
"memberships");
TextureCacheTestAccess::DeleteImage(texture_cache, large_owner);
Require(name, "production one-MiB unregister granularity",
TextureCacheTestAccess::OwnedPageCount(
texture_cache, large_info.data.address, large_info.data.size,
large_owner) == 0,
"large image unregister left coarse memberships behind");
const ImageId stale{first.index, first.generation + 1};
TextureCacheTestAccess::AddPageOwner(texture_cache, base, stale);
const auto stale_filtered = TextureCacheTestAccess::FindImages(
texture_cache, base, sizeof(initial), false);
Require(name, "stale page owner filtering",
IsOnlyImage(stale_filtered, first) &&
TextureCacheTestAccess::RemovePageOwner(texture_cache, base,
stale),
"a stale generation escaped the direct page-owner lookup");
TextureCacheTestAccess::SetQueryEpoch(texture_cache, UINT32_MAX);
const auto wrap_results = TextureCacheTestAccess::FindImages(
texture_cache, base, sizeof(initial), false);
Require(name, "page-owner query epoch wrap",
IsOnlyImage(wrap_results, first) &&
TextureCacheTestAccess::QueryEpoch(texture_cache) == 1,
"query deduplication failed while wrapping its epoch");
auto &image = texture_cache.GetImage(first);
constexpr uint32_t final_sampled_value = 0x88776655u;
Require(name, "sampled write between discovery and acquisition",
resources.HandleFault(PageFaultAccess::Write,
sampled.info.data.address),
"sampled image did not accept a CPU update after discovery");
std::memcpy(memory, &final_sampled_value, sizeof(final_sampled_value));
Require(name, "sampled final-acquisition precondition",
image.IsCpuDirty(),
"the between-phase CPU write did not dirty the discovered image");
const auto first_view = texture_cache.FindTexture(first, first_desc);
const auto repeated_view =
texture_cache.FindTexture(repeated, repeated_desc);
auto mapped_view_info = sampled.view_info;
mapped_view_info.mapping.r = vk::ComponentSwizzle::eB;
mapped_view_info.mapping.b = vk::ComponentSwizzle::eR;
const auto mapped_view = image.FindView(mapped_view_info);
Require(
name, "dynamic views",
first_view != nullptr && repeated_view == first_view &&
mapped_view != nullptr && mapped_view != first_view &&
image.views.size() == 2 && !image.usage.texture &&
!image.IsGpuModified() && !image.IsCpuDirty() &&
!TextureCacheTestAccess::PendingDownload(texture_cache, first),
"TextureCache::FindTexture did not preserve sampled view "
"identity, final refresh, and cache-only ownership");
constexpr uint64_t resource_replacement_offset = 0x700000;
const std::array<uint32_t, 3> resource_replacement_data{
0x01020304u, 0x11121314u, 0x21222324u};
std::memcpy(memory + resource_replacement_offset,
resource_replacement_data.data(),
sizeof(resource_replacement_data));
auto underspecified = sampled;
underspecified.info.data = {base + resource_replacement_offset,
sizeof(resource_replacement_data)};
underspecified.info.pixel_format = vk::Format::eR32Uint;
underspecified.info.guest_format = Prospero::BufferFormat::k32UInt;
underspecified.info.extent = {2, 1, 1};
underspecified.info.resources = {1, 1};
underspecified.info.pitch = 2;
underspecified.info.mip_layout[0] = {0, 8, 2, 1};
underspecified.info.mip_layout[1] = {8, 4, 1, 1};
underspecified.view_info.format = underspecified.info.pixel_format;
underspecified.view_info.level_count = 1;
const auto underspecified_id = texture_cache.FindImage(underspecified);
(void)texture_cache.FindTexture(underspecified_id, underspecified);
texture_cache.MarkGpuWritten(underspecified_id);
auto complete = underspecified;
complete.info.resources = {2, 1};
complete.view_info.level_count = 2;
const auto complete_id = texture_cache.FindImage(complete);
const auto complete_owner =
TextureCacheTestAccess::Owner(texture_cache, complete_id);
Require(name, "insufficient-resource replacement",
complete_id && complete_id != underspecified_id &&
!TextureCacheTestAccess::Contains(texture_cache,
underspecified_id) &&
complete_owner != nullptr && complete_owner->registered &&
complete_owner->info.resources == complete.info.resources &&
!complete_owner->IsGpuModified() &&
complete_owner->IsDefinitelyCpuDirty(),
"an exact backing with insufficient resources was expanded "
"instead of being discarded and recreated");
ImageInfo chain = sampled.info;
chain.data = {0x2394c1000, 0x384000};
chain.extent = {64, 64, 1};
chain.resources = {7, 150};
chain.pitch = 64;
chain.tile_mode = Prospero::TileMode::kStandard4KB;
chain.mip_layout[0] = {0x2000, 0x258000, 64, 64};
chain.mip_layout[1] = {0x1000, 0x96000, 32, 32};
for (uint32_t level = 2; level < chain.resources.levels; ++level) {
chain.mip_layout[level] = {0, 0x96000, 32, 32};
}
ImageInfo subresource = chain;
subresource.data = {0x2394c3000, 0x4000};
subresource.resources = {1, 1};
subresource.mip_layout = {};
subresource.mip_layout[0] = {0, 0x4000, 64, 64};
const auto mip = subresource.MipOf(chain);
Require(name, "overlap resolution",
mip == 0 && subresource.SliceOf(chain, mip) == 0 &&
ImageRangeOverlaps(chain.data, subresource.data) &&
ImagePageRangesOverlap(chain.data, subresource.data),
"captured PS5 atlas mip/slice overlap did not resolve");
auto exact_desc = compatible_desc;
const auto exact = texture_cache.FindImage(exact_desc, true);
Require(name, "exact-format coexistence",
exact && exact != first &&
TextureCacheTestAccess::Contains(texture_cache, first) &&
texture_cache.GetImage(first).info.pixel_format ==
sampled.info.pixel_format &&
texture_cache.GetImage(exact).info.pixel_format ==
compatible_desc.info.pixel_format,
"exact-format lookup replaced or synchronized its compatible "
"cache record");
auto compatible_after_exact_desc = compatible_desc;
const auto compatible_after_exact =
texture_cache.FindImage(compatible_after_exact_desc);
Require(name, "last compatible format winner",
compatible_after_exact == exact,
"non-exact lookup did not retain the last compatible "
"registration");
auto null_desc = sampled;
null_desc.info.data = {};
auto null_repeat_desc = null_desc;
const auto null_image = texture_cache.FindImage(null_desc);
const auto null_repeat = texture_cache.FindImage(null_repeat_desc);
Require(name, "null image",
null_image && null_repeat == null_image && null_image != exact,
"typed null-image lookup was not stable");
constexpr uint64_t extent_alias_offset = 0x2400000;
auto narrow_target = sampled;
narrow_target.type = BindingType::RenderTarget;
narrow_target.info.data = {base + extent_alias_offset, 0x10000};
narrow_target.info.pixel_format = vk::Format::eR16G16B16A16Sfloat;
narrow_target.info.guest_format =
Prospero::BufferFormat::k16_16_16_16Float;
narrow_target.info.extent = {1, 1, 1};
narrow_target.info.pitch = 128;
narrow_target.info.bytes_per_block = 8;
narrow_target.info.tile_mode = Prospero::TileMode::kRenderTarget;
narrow_target.info.mip_layout[0] = {0, 0x10000, 128, 1};
narrow_target.view_info.format = narrow_target.info.pixel_format;
narrow_target.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto narrow_target_id = texture_cache.FindImage(narrow_target);
const auto narrow_target_view =
texture_cache.FindRenderTarget(narrow_target_id, narrow_target);
auto wide_target = narrow_target;
wide_target.info.extent.width = 9;
const auto wide_target_id = texture_cache.FindImage(wide_target);
const auto wide_target_view =
texture_cache.FindRenderTarget(wide_target_id, wide_target);
const auto wide_owner =
TextureCacheTestAccess::Owner(texture_cache, wide_target_id);
Require(name, "equal-allocation render-target growth",
narrow_target_view != nullptr && wide_target_view != nullptr &&
wide_target_id != narrow_target_id &&
!TextureCacheTestAccess::Contains(texture_cache,
narrow_target_id) &&
wide_owner != nullptr &&
wide_owner->info.extent == vk::Extent3D{9, 1, 1} &&
wide_owner->backing.extent == vk::Extent3D{9, 1, 1},
"a 9x1 render target reused its equal-size 1x1 native backing");
auto narrow_again = narrow_target;
const auto narrow_again_id = texture_cache.FindImage(narrow_again);
Require(name, "larger render-target backing reuse",
narrow_again_id == wide_target_id &&
texture_cache.GetImage(narrow_again_id).backing.extent ==
vk::Extent3D{9, 1, 1},
"a smaller render area replaced its compatible larger backing");
const auto HostReadBarrier = [&](vk::Buffer buffer, uint64_t size,
vk::PipelineStageFlags source_stage,
vk::AccessFlags source_access) {
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = source_access;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = buffer;
barrier.offset = 0;
barrier.size = size;
scheduler.Current().Handle().pipelineBarrier(
source_stage, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
&barrier, 0, nullptr);
};
const auto TransferReadBarrier = [&](vk::Buffer buffer, uint64_t size) {
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = buffer;
barrier.offset = 0;
barrier.size = size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr, 1, &barrier,
0, nullptr);
};
constexpr uint64_t mip_layout_offset = 0x27f0000;
std::vector<u32> mip_guest(0x2000 / sizeof(u32));
std::iota(mip_guest.begin(), mip_guest.end(), 0x51000000u);
ImageId previous{};
for (const uint32_t width : {32u, 64u, 32u}) {
if (previous) {
Require(name, "mip-layout CPU overwrite",
resources.HandleFault(PageFaultAccess::Write,
base + mip_layout_offset),
"mip-layout guest overwrite did not invalidate the old texture");
}
std::memcpy(memory + mip_layout_offset, mip_guest.data(),
mip_guest.size() * sizeof(u32));
TileSurfaceLayout surface{};
Require(name, "mip-tail layout",
TileGetTiledTextureLayout(
{.format = Prospero::BufferFormat::kBc3Srgb,
.tile_mode = Prospero::TileMode::kStandard4KB,
.width = width, .height = 64, .levels = 7}, surface) &&
surface.first_tail_level == (width == 32 ? 0u : 1u) &&
surface.total_size == (width == 32 ? 0x1000u : 0x2000u),
"BC3 width change did not move mip zero into or out of the tail");
ImageDesc desc{};
desc.type = BindingType::Texture;
desc.info.data = {base + mip_layout_offset, surface.total_size};
desc.info.pixel_format = vk::Format::eBc3SrgbBlock;
desc.info.guest_format = Prospero::BufferFormat::kBc3Srgb;
desc.info.extent = {width, 64, 1};
desc.info.resources = {7, 1};
desc.info.pitch = 64;
desc.info.bytes_per_block = 16;
desc.info.tile_mode = Prospero::TileMode::kStandard4KB;
desc.view_info.format = desc.info.pixel_format;
desc.view_info.type = vk::ImageViewType::e2D;
desc.view_info.level_count = 7;
desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
std::vector<vk::BufferImageCopy> copies;
std::vector<u32> expected;
for (uint32_t level = 0; level < 7; level++) {
const auto &layout = surface.mips[level];
desc.info.mip_layout[level] = {
layout.offset, layout.size, layout.padded_width,
layout.padded_height};
vk::BufferImageCopy copy{};
copy.bufferOffset = expected.size() * sizeof(u32);
copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, level, 0, 1};
copy.imageExtent = {std::max(width >> level, 1u),
std::max(64u >> level, 1u), 1};
copies.push_back(copy);
for (uint32_t y = 0; y < layout.height; y++) {
for (uint32_t x = 0; x < layout.width; x++) {
uint32_t offset = 0;
Require(name, "mip-tail CPU address",
TileGetBlockOffset(
surface.texture.block, x + layout.tail_x,
y + layout.tail_y, 0, offset),
"BC3 mip-tail block address could not be computed");
const auto word = (layout.offset + offset) / sizeof(u32);
expected.insert(expected.end(), mip_guest.begin() + word,
mip_guest.begin() + word + 4);
}
}
}
const auto id = texture_cache.FindImage(desc);
Require(name, "distinct mip-layout backing", id && id != previous,
"different guest mip layouts reused the same native image");
(void)texture_cache.FindTexture(id, desc);
auto readback = CreateHostBuffer(
name, expected.size() * sizeof(u32),
vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(expected.size()));
texture_cache.GetImage(id).Download(copies, readback.buffer, 0,
readback.size);
HostReadBarrier(readback.buffer, readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "mip-layout GPU contents",
ReadBuffer(name, readback, expected.size()) == expected,
"cached native copying contaminated the requested guest mip layout");
DestroyBuffer(&readback);
if (width == 64) {
// Leave only the larger backing so shrinking must reject its layout.
TextureCacheTestAccess::DeleteImage(texture_cache, previous);
}
previous = id;
}
// The game reuses a raw render target allocation for a larger BC5 mip
// chain. Equal block extents do not make their guest tiling compatible.
constexpr uint64_t bc5_alias_offset = 0x1800000;
TileSurfaceLayout bc5_surface{};
Require(name, "BC5 alias layout",
TileGetTiledTextureLayout(
{.format = Prospero::BufferFormat::kBc5UNorm,
.tile_mode = Prospero::TileMode::kStandard64KB,
.width = 1024, .height = 1024, .levels = 11}, bc5_surface) &&
bc5_surface.total_size == 0x160000,
"captured BC5 allocation no longer has its 0x160000-byte footprint");
std::memset(memory + bc5_alias_offset, 0, bc5_surface.total_size);
auto raw_alias = MakeLinearDesc(
base + bc5_alias_offset, 0x100000, vk::Format::eR32G32B32A32Sfloat,
Prospero::BufferFormat::k32_32_32_32Float, Prospero::ImageType::kColor2D,
{256, 256, 1}, 1, 16, 1);
raw_alias.type = BindingType::RenderTarget;
raw_alias.info.tile_mode = Prospero::TileMode::kRenderTarget;
raw_alias.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto raw_alias_id = texture_cache.FindImage(raw_alias);
(void)texture_cache.FindRenderTarget(raw_alias_id, raw_alias);
vk::ClearValue raw_alias_clear{};
std::fill_n(raw_alias_clear.color.float32.data(), 4, 1.0f);
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), raw_alias_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, raw_alias_clear);
Require(name, "BC5 alias CPU overwrite",
resources.InvalidateMemory(base + bc5_alias_offset,
bc5_surface.total_size),
"new BC5 data could not invalidate the old render target");
std::vector<u32> bc5_guest(bc5_surface.total_size / sizeof(u32));
std::iota(bc5_guest.begin(), bc5_guest.end(), 0x41000000u);
std::memcpy(memory + bc5_alias_offset, bc5_guest.data(),
bc5_surface.total_size);
auto bc5_alias = MakeLinearDesc(
base + bc5_alias_offset, bc5_surface.total_size, vk::Format::eBc5UnormBlock,
Prospero::BufferFormat::kBc5UNorm, Prospero::ImageType::kColor2D,
{1024, 1024, 1}, 1, 16, 1);
bc5_alias.info.resources.levels = 11;
bc5_alias.info.tile_mode = Prospero::TileMode::kStandard64KB;
bc5_alias.view_info.level_count = 11;
std::vector<vk::BufferImageCopy> bc5_probes;
std::vector<u32> bc5_expected;
for (u32 level = 0; level < 11; ++level) {
const auto &layout = bc5_surface.mips[level];
bc5_alias.info.mip_layout[level] = {
layout.offset, layout.size, layout.padded_width, layout.padded_height};
u32 offset = 0;
Require(name, "BC5 alias block address",
TileGetBlockOffset(bc5_surface.texture.block, layout.tail_x,
layout.tail_y, 0, offset),
"BC5 mip origin could not be addressed in guest memory");
vk::BufferImageCopy copy{};
copy.bufferOffset = bc5_expected.size() * sizeof(u32);
copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, level, 0, 1};
copy.imageExtent = {std::min(1024u >> level, 4u),
std::min(1024u >> level, 4u), 1};
bc5_probes.push_back(copy);
const auto word = (layout.offset + offset) / sizeof(u32);
bc5_expected.insert(bc5_expected.end(), bc5_guest.begin() + word,
bc5_guest.begin() + word + 4);
}
const auto bc5_alias_id = texture_cache.FindImage(bc5_alias);
Require(name, "BC5 alias owner separation",
bc5_alias_id != raw_alias_id &&
TextureCacheTestAccess::Contains(texture_cache, raw_alias_id) &&
!texture_cache.GetImage(bc5_alias_id).usage.render_target &&
texture_cache.GetImage(bc5_alias_id).info.tile_mode ==
Prospero::TileMode::kStandard64KB,
"BC5 image expanded the unrelated raw render target or inherited its usage");
(void)texture_cache.FindTexture(bc5_alias_id, bc5_alias);
auto bc5_readback = CreateHostBuffer(
name, bc5_expected.size() * sizeof(u32),
vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>(bc5_expected.size()));
texture_cache.GetImage(bc5_alias_id).Download(
bc5_probes, bc5_readback.buffer, 0, bc5_readback.size);
HostReadBarrier(bc5_readback.buffer, bc5_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "BC5 alias mip contents",
ReadBuffer(name, bc5_readback, bc5_expected.size()) == bc5_expected,
"raw render-target copying overwrote the BC5 guest mip layout");
DestroyBuffer(&bc5_readback);
{
constexpr uint64_t tiled_array_offset = 0x2c00000;
constexpr uint32_t tiled_levels = 5, tiled_layers = 6;
TileSurfaceLayout surface{};
Require(name, "tiled array alias layout",
TileGetTiledTextureLayout(
{Prospero::BufferFormat::k32UInt, Prospero::TileMode::kStandard64KB,
TileSurfaceDimension::Dim2D, 256, 256, 1, tiled_levels, tiled_layers},
surface) && surface.total_size == 0x240000 &&
surface.block_slice_size == 0x60000 &&
surface.mips[0].offset + surface.mips[0].size * tiled_layers == 0x1a0000,
"fixture lost the gap between an array mip span and its full backing");
auto tiled_array = MakeLinearDesc(
base + tiled_array_offset, surface.total_size, vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{256, 256, 1}, tiled_layers, sizeof(uint32_t), 1);
tiled_array.info.tile_mode = Prospero::TileMode::kStandard64KB;
tiled_array.info.resources.levels = tiled_levels;
tiled_array.view_info.level_count = tiled_levels;
for (uint32_t level = 0; level < tiled_levels; ++level) {
const auto &mip = surface.mips[level];
tiled_array.info.mip_layout[level] = {
mip.offset, mip.size * tiled_layers, mip.padded_width, mip.padded_height};
}
std::memset(memory + tiled_array_offset, 0, surface.total_size);
const auto tiled_array_id = texture_cache.FindImage(tiled_array);
(void)texture_cache.FindTexture(tiled_array_id, tiled_array);
constexpr uint32_t all_mips = 0x12345678u, last_mip = 0x89abcdefu;
vk::ClearValue clear{};
clear.color.uint32[0] = all_mips;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), tiled_array_id,
{vk::ImageAspectFlagBits::eColor, 0, tiled_levels, 0, tiled_layers}, clear);
clear.color.uint32[0] = last_mip;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), tiled_array_id,
{vk::ImageAspectFlagBits::eColor, tiled_levels - 1, 1, tiled_layers - 1, 1}, clear);
Libs::Graphics::Buffer full_array(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
tiled_array.info.data.address, AllFlags, surface.total_size);
Libs::Graphics::Buffer partial_array(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
tiled_array.info.data.address, AllFlags, 0x1a0000);
Require(name, "tiled array alias containment",
BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), full_array,
tiled_array.info.data.address, surface.total_size) &&
!BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), partial_array,
tiled_array.info.data.address, 0x1a0000),
"full array synchronization failed or accepted an incomplete slice chain");
auto readback = CreateHostBuffer(name, 2 * sizeof(uint32_t),
vk::BufferUsageFlagBits::eTransferDst, {0, 0});
TransferReadBarrier(full_array.Handle(), surface.total_size);
for (uint32_t probe = 0; probe < 2; ++probe) {
const auto &mip = surface.mips[probe == 0 ? 0 : tiled_levels - 1];
uint32_t within_block = 0;
Require(name, "tiled array probe address",
TileGetBlockOffset(surface.texture.block, mip.tail_x, mip.tail_y,
0, within_block),
"last-layer mip origin has no guest address");
const vk::BufferCopy copy{
(tiled_layers - 1) * surface.block_slice_size + mip.offset + within_block,
probe * sizeof(uint32_t), sizeof(uint32_t)};
command.Handle().copyBuffer(full_array.Handle(), readback.buffer, 1, &copy);
}
HostReadBarrier(readback.buffer, readback.size, vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "tiled array last-layer contents",
ReadBuffer(name, readback, 2) == std::vector<u32>{all_mips, last_mip},
"array synchronization lost GPU contents beyond the first mip span");
DestroyBuffer(&readback);
}
// A formatted Buffer read must use the private
// Exercise the cache-native image-copy path. Use a request larger
// than the stream shortcut and poison guest backing
// after upload so stale CPU staging cannot accidentally
// satisfy the content check.
constexpr uint64_t mip_prefix_offset = 0x2740000;
constexpr auto mip_format = Prospero::BufferFormat::k32UInt;
constexpr auto mip_tile = Prospero::TileMode::kLinear;
constexpr uint32_t mip_width = 4097;
constexpr uint32_t mip_height = 1;
constexpr uint32_t mip_levels = 1;
const uint32_t mip_pitch =
TileGetTexturePitch(mip_format, mip_width, mip_tile);
TileSizeAlign mip_total{};
std::array<TileSizeOffset, 16> mip_sizes{};
std::array<TilePaddedSize, 16> mip_padded{};
TileGetTextureSize(mip_format, mip_width, mip_height, mip_levels,
mip_tile, &mip_total, mip_sizes.data(),
mip_padded.data());
const uint64_t mip_prefix_size = mip_sizes[0].offset + mip_sizes[0].size;
const uint64_t mip_guest_size = mip_total.size + 256;
Require(name, "formatted mip fixture",
mip_sizes[0].offset == 0 &&
mip_prefix_size > BufferCache::CACHING_PAGESIZE &&
mip_prefix_size < mip_guest_size &&
mip_guest_size % sizeof(uint32_t) == 0,
"single-mip fixture did not expose a cache-sized fitting backing "
"prefix");
std::vector<uint32_t> mip_native(mip_guest_size / sizeof(uint32_t));
std::iota(mip_native.begin(), mip_native.end(), 0x61000000u);
std::memcpy(memory + mip_prefix_offset, mip_native.data(),
mip_guest_size);
auto mip_desc = MakeLinearDesc(
base + mip_prefix_offset, mip_guest_size, vk::Format::eR32Uint,
mip_format, Prospero::ImageType::kColor2D, {mip_width, mip_height, 1},
1, sizeof(uint32_t), 1);
mip_desc.info.resources.levels = mip_levels;
mip_desc.info.pitch = mip_pitch;
mip_desc.view_info.level_count = mip_levels;
for (uint32_t level = 0; level < mip_levels; level++) {
mip_desc.info.mip_layout[level] = {
mip_sizes[level].offset, mip_sizes[level].size,
mip_padded[level].width, mip_padded[level].height};
}
const auto mip_image = texture_cache.FindImage(mip_desc);
(void)texture_cache.FindTexture(mip_image, mip_desc);
texture_cache.MarkGpuWritten(mip_image);
std::vector<uint32_t> mip_stale(mip_native.size(), 0xdeadbeefu);
Libs::LibKernel::Memory::WriteBacking(base + mip_prefix_offset,
mip_stale.data(), mip_guest_size);
Libs::Graphics::Buffer mip_insufficient(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
mip_desc.info.data.address, AllFlags, mip_prefix_size - 1);
Libs::Graphics::Buffer mip_prefix(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
mip_desc.info.data.address, AllFlags, mip_prefix_size);
Require(name, "formatted mip containment",
!BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), mip_insufficient,
mip_desc.info.data.address, mip_prefix_size - 1) &&
BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), mip_prefix,
mip_desc.info.data.address, mip_prefix_size) &&
texture_cache.GetImage(mip_image).IsGpuModified(),
"image synchronization accepted a partial first "
"mip, rejected a fitting mip "
"prefix, or transferred ownership");
auto &buffer_cache = resources.GetBufferCache();
std::pair<Libs::Graphics::Buffer *, uint64_t> mip_formatted;
bool formatted_owner_was_absent = false;
gpu.SendCommandSync([&] {
formatted_owner_was_absent = !buffer_cache.IsRegionRegistered(
mip_desc.info.data.address, mip_desc.info.data.size);
mip_formatted = buffer_cache.ObtainBuffer(
mip_desc.info.data.address, mip_desc.info.data.size, false, true);
});
Require(name, "formatted cache fixture", formatted_owner_was_absent,
"formatted image already had a cached Buffer owner");
Require(name, "formatted Buffer path",
mip_formatted.first != nullptr &&
texture_cache.GetImage(mip_image).IsGpuModified(),
"formatted read bypassed the cached image-copy "
"path or transferred ownership");
auto mip_prefix_readback = CreateHostBuffer(
name, mip_prefix_size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(mip_prefix_size / sizeof(uint32_t), 0));
auto mip_formatted_readback = CreateHostBuffer(
name, mip_guest_size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(mip_guest_size / sizeof(uint32_t), 0));
TransferReadBarrier(mip_prefix.Handle(), mip_prefix_size);
const vk::BufferCopy mip_prefix_copy{0, 0, mip_prefix_size};
command.Handle().copyBuffer(
mip_prefix.Handle(), mip_prefix_readback.buffer, 1, &mip_prefix_copy);
TransferReadBarrier(mip_formatted.first->Handle(), mip_guest_size);
const vk::BufferCopy mip_formatted_copy{mip_formatted.second, 0,
mip_guest_size};
command.Handle().copyBuffer(mip_formatted.first->Handle(),
mip_formatted_readback.buffer, 1,
&mip_formatted_copy);
HostReadBarrier(mip_prefix_readback.buffer, mip_prefix_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
HostReadBarrier(mip_formatted_readback.buffer,
mip_formatted_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
const std::array<uint32_t, 2> volume_values{0x10203040u, 0x50607080u};
TileSizeAlign volume_size{};
TileGetTextureTotalSize(Prospero::BufferFormat::k32UInt, 1, 1, 2, 1,
Prospero::TileMode::kLinear, true, volume_size);
const auto volume_pitch = TileGetTexturePitch(
Prospero::BufferFormat::k32UInt, 1, Prospero::TileMode::kLinear);
const auto volume_slice_words = volume_size.size / 2 / sizeof(uint32_t);
std::vector<uint32_t> volume_guest(volume_size.size / sizeof(uint32_t));
volume_guest[0] = volume_values[0];
volume_guest[volume_slice_words] = volume_values[1];
std::memcpy(memory + 0x1000, volume_guest.data(), volume_size.size);
auto array_desc =
MakeLinearDesc(base + 0x1000, volume_size.size,
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1);
array_desc.info.pitch = volume_pitch;
array_desc.info.mip_layout[0].pitch = volume_pitch;
const auto array_image = texture_cache.FindImage(array_desc);
(void)texture_cache.FindTexture(array_image, array_desc);
texture_cache.MarkGpuWritten(array_image);
auto volume_desc = array_desc;
volume_desc.info.type = Prospero::ImageType::kColor3D;
volume_desc.info.extent = {1, 1, 2};
volume_desc.info.resources.layers = 1;
volume_desc.view_info.type = vk::ImageViewType::e3D;
volume_desc.view_info.layer_count = 1;
const auto volume_image = texture_cache.FindImage(volume_desc);
Require(name, "2D-array/3D overlap copy",
volume_image && volume_image != array_image &&
texture_cache.GetImage(volume_image).backing.image_type ==
vk::ImageType::e3D &&
texture_cache.GetImage(volume_image).IsGpuModified(),
"array-to-volume expansion lost slices or GPU "
"ownership");
constexpr uint64_t unique_volume_offset = 0x27b0000;
std::memcpy(memory + unique_volume_offset, volume_guest.data(),
volume_size.size);
auto unique_volume_desc = volume_desc;
unique_volume_desc.info.data.address = base + unique_volume_offset;
const auto unique_volume_image =
texture_cache.FindImage(unique_volume_desc);
(void)texture_cache.FindTexture(unique_volume_image, unique_volume_desc);
texture_cache.MarkGpuWritten(unique_volume_image);
Libs::Graphics::Buffer partial_volume(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
unique_volume_desc.info.data.address, AllFlags,
unique_volume_desc.info.data.size - sizeof(uint32_t));
Libs::Graphics::Buffer full_volume(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
unique_volume_desc.info.data.address, AllFlags,
unique_volume_desc.info.data.size);
Require(name, "formatted volume containment",
!BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), partial_volume,
unique_volume_desc.info.data.address,
unique_volume_desc.info.data.size - sizeof(uint32_t)) &&
BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), full_volume,
unique_volume_desc.info.data.address,
unique_volume_desc.info.data.size) &&
texture_cache.GetImage(unique_volume_image).IsGpuModified(),
"partial 3D synchronization was accepted, "
"full-volume synchronization was "
"rejected, or ownership changed");
auto volume_readback = CreateHostBuffer(
name, volume_size.size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(volume_guest.size(), 0));
TransferReadBarrier(full_volume.Handle(), volume_size.size);
const vk::BufferCopy volume_copy{0, 0, volume_size.size};
command.Handle().copyBuffer(full_volume.Handle(), volume_readback.buffer,
1, &volume_copy);
HostReadBarrier(volume_readback.buffer, volume_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
constexpr uint64_t depth_containment_offset = 0x27c0000;
constexpr std::array<float, 3> depth_containment_values{0.25f, 0.75f,
0.5f};
std::memcpy(memory + depth_containment_offset,
depth_containment_values.data(),
sizeof(depth_containment_values));
auto depth_containment = MakeLinearDesc(
base + depth_containment_offset, sizeof(depth_containment_values),
vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float,
Prospero::ImageType::kColor2D, {2, 1, 1}, 1, sizeof(float), 1);
depth_containment.type = BindingType::DepthTarget;
depth_containment.info.resources.levels = 2;
depth_containment.info.mip_layout[0] = {0, 2 * sizeof(float), 2, 1};
depth_containment.info.mip_layout[1] = {2 * sizeof(float), sizeof(float),
1, 1};
depth_containment.view_info.format = vk::Format::eD32Sfloat;
depth_containment.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
depth_containment.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
depth_containment.view_info.level_count = 2;
const auto depth_containment_image =
texture_cache.FindImage(depth_containment);
(void)texture_cache.FindDepthTarget(depth_containment_image,
depth_containment);
texture_cache.MarkGpuWritten(depth_containment_image);
Libs::Graphics::Buffer partial_depth(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
depth_containment.info.data.address, AllFlags,
depth_containment.info.mip_layout[0].size);
Require(
name, "formatted depth containment",
!BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), partial_depth,
depth_containment.info.data.address,
depth_containment.info.mip_layout[0].size) &&
texture_cache.GetImage(depth_containment_image).IsGpuModified(),
"partial depth synchronization was accepted or "
"transferred ownership");
constexpr uint64_t raw_d16_offset = 0x27d0000;
constexpr std::array<uint16_t, 2> raw_d16_values{0x2000u, 0xc000u};
std::memcpy(memory + raw_d16_offset, raw_d16_values.data(),
sizeof(raw_d16_values));
auto raw_d16 = MakeLinearDesc(
base + raw_d16_offset, sizeof(raw_d16_values), vk::Format::eD16Unorm,
Prospero::BufferFormat::k16UNorm, Prospero::ImageType::kColor2D,
{2, 1, 1}, 1, sizeof(uint16_t), 1);
raw_d16.type = BindingType::DepthTarget;
raw_d16.view_info.format = vk::Format::eD16Unorm;
raw_d16.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
raw_d16.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto raw_d16_image = texture_cache.FindImage(raw_d16);
(void)texture_cache.FindDepthTarget(raw_d16_image, raw_d16);
auto raw_d16_uint = raw_d16;
raw_d16_uint.type = BindingType::Texture;
raw_d16_uint.info.pixel_format = vk::Format::eR16Uint;
raw_d16_uint.info.guest_format = Prospero::BufferFormat::k16UInt;
raw_d16_uint.view_info.format = vk::Format::eR16Uint;
raw_d16_uint.view_info.aspect = vk::ImageAspectFlagBits::eColor;
raw_d16_uint.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto raw_d16_uint_image = texture_cache.FindImage(raw_d16_uint);
Require(
name, "raw D16 uint texture backing",
raw_d16_uint_image && raw_d16_uint_image != raw_d16_image &&
!TextureCacheTestAccess::Contains(texture_cache, raw_d16_image) &&
texture_cache.GetImage(raw_d16_uint_image).backing.format ==
vk::Format::eR16Uint,
"D16 uint sampling did not recreate a true integer color backing");
Require(name, "raw D16 uint texture view",
texture_cache.FindTexture(raw_d16_uint_image, raw_d16_uint) !=
nullptr,
"D16 uint sampling did not create an integer sampled view");
constexpr uint64_t layered_raw_d16_offset = raw_d16_offset + 0x1000;
constexpr std::array<uint16_t, 12> layered_raw_d16_values{
0x0102u, 0x0304u, 0x1112u, 0x1314u, 0x2122u, 0x2324u,
0x3132u, 0x3334u, 0x4142u, 0x4344u, 0x5152u, 0x5354u};
std::memcpy(memory + layered_raw_d16_offset,
layered_raw_d16_values.data(),
sizeof(layered_raw_d16_values));
auto layered_d16_color = MakeLinearDesc(
base + layered_raw_d16_offset, sizeof(layered_raw_d16_values),
vk::Format::eR16Unorm, Prospero::BufferFormat::k16UNorm,
Prospero::ImageType::kColor2D, {2, 1, 1}, 6, sizeof(uint16_t), 1);
const auto layered_d16_color_image =
texture_cache.FindImage(layered_d16_color);
(void)texture_cache.FindTexture(layered_d16_color_image,
layered_d16_color);
auto &layered_d16_color_native =
texture_cache.GetImage(layered_d16_color_image);
layered_d16_color_native.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {},
scheduler.Current().Handle());
vk::ClearColorValue layered_raw_d16_clear_value{};
layered_raw_d16_clear_value.float32[0] = 1.0f;
const vk::ImageSubresourceRange layered_raw_d16_clear_range{
vk::ImageAspectFlagBits::eColor, 0, 1, 0, 6};
scheduler.Current().Handle().clearColorImage(
layered_d16_color_native.backing.image,
vk::ImageLayout::eTransferDstOptimal, &layered_raw_d16_clear_value, 1,
&layered_raw_d16_clear_range);
texture_cache.MarkGpuWritten(layered_d16_color_image);
auto layered_raw_d16_depth = layered_d16_color;
layered_raw_d16_depth.type = BindingType::DepthTarget;
layered_raw_d16_depth.info.data.size = sizeof(uint16_t) * 2 * 5;
layered_raw_d16_depth.info.pixel_format = vk::Format::eD16Unorm;
layered_raw_d16_depth.info.guest_format =
Prospero::BufferFormat::k16UNorm;
layered_raw_d16_depth.info.resources.layers = 5;
layered_raw_d16_depth.info.mip_layout[0].size =
layered_raw_d16_depth.info.data.size;
layered_raw_d16_depth.view_info.format = vk::Format::eD16Unorm;
layered_raw_d16_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
layered_raw_d16_depth.view_info.layer_count = 5;
layered_raw_d16_depth.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto layered_raw_d16_depth_image =
texture_cache.FindImage(layered_raw_d16_depth);
const auto &layered_depth_owner =
texture_cache.GetImage(layered_raw_d16_depth_image);
Require(name, "layered raw D16 owner layout",
layered_raw_d16_depth_image != layered_d16_color_image &&
!TextureCacheTestAccess::Contains(texture_cache,
layered_d16_color_image) &&
layered_depth_owner.info.resources.layers == 6 &&
layered_depth_owner.info.data.size ==
sizeof(layered_raw_d16_values) &&
layered_depth_owner.info.mip_layout[0].size ==
sizeof(layered_raw_d16_values),
"a partial depth view detached the retained six-layer owner from "
"its full guest range");
(void)texture_cache.FindDepthTarget(layered_raw_d16_depth_image,
layered_raw_d16_depth);
auto reacquired_layered_raw_d16_uint = layered_d16_color;
reacquired_layered_raw_d16_uint.info.pixel_format = vk::Format::eR16Uint;
reacquired_layered_raw_d16_uint.info.guest_format =
Prospero::BufferFormat::k16UInt;
reacquired_layered_raw_d16_uint.view_info.format = vk::Format::eR16Uint;
const auto reacquired_layered_raw_d16_uint_image =
texture_cache.FindImage(reacquired_layered_raw_d16_uint);
const auto reacquired_layered_raw_d16_uint_view =
texture_cache.FindTexture(reacquired_layered_raw_d16_uint_image,
reacquired_layered_raw_d16_uint);
auto layered_raw_d16_readback = CreateHostBuffer(
name, sizeof(layered_raw_d16_values),
vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>(6, 0));
auto &reacquired_layered_raw_d16_native =
texture_cache.GetImage(reacquired_layered_raw_d16_uint_image);
reacquired_layered_raw_d16_native.Transit(
vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {}, scheduler.Current().Handle());
vk::BufferImageCopy layered_raw_d16_copy{};
layered_raw_d16_copy.bufferRowLength = 2;
layered_raw_d16_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor,
0, 0, 6};
layered_raw_d16_copy.imageExtent = {2, 1, 1};
scheduler.Current().Handle().copyImageToBuffer(
reacquired_layered_raw_d16_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal, layered_raw_d16_readback.buffer,
1, &layered_raw_d16_copy);
HostReadBarrier(layered_raw_d16_readback.buffer,
layered_raw_d16_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto layered_raw_d16_words =
ReadBuffer(name, layered_raw_d16_readback, 6);
std::array<uint16_t, 12> layered_raw_d16_after{};
std::memcpy(layered_raw_d16_after.data(), layered_raw_d16_words.data(),
sizeof(layered_raw_d16_after));
std::array<uint16_t, 12> layered_raw_d16_expected{};
layered_raw_d16_expected.fill(UINT16_MAX);
Require(
name, "layered raw D16 round trip",
reacquired_layered_raw_d16_uint_image &&
reacquired_layered_raw_d16_uint_image !=
layered_raw_d16_depth_image &&
!TextureCacheTestAccess::Contains(texture_cache,
layered_raw_d16_depth_image) &&
reacquired_layered_raw_d16_native.backing.format ==
vk::Format::eR16Uint &&
reacquired_layered_raw_d16_uint_view != nullptr &&
layered_raw_d16_after == layered_raw_d16_expected,
"the partial depth view lost a layer or failed to restore raw UINT "
"sampling");
DestroyBuffer(&layered_raw_d16_readback);
// RE2 converts a prefix of a 32-layer HTile depth array to R32F storage.
constexpr uint32_t htile_layers = 32;
constexpr uint64_t htile_slice_size = 0x10000;
auto htile_depth = MakeLinearDesc(
base + 0x2800000, htile_layers * htile_slice_size,
vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float,
Prospero::ImageType::kColor2D, {128, 128, 1}, htile_layers, 4, 1);
htile_depth.type = BindingType::DepthTarget;
htile_depth.info.tile_mode = Prospero::TileMode::kDepth;
htile_depth.info.metadata.kind = ImageMetadataKind::Htile;
htile_depth.info.metadata.range = {base + 0x2a00000, htile_layers * 0x8000};
htile_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
htile_depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto htile_depth_image = texture_cache.FindImage(htile_depth);
vk::ClearValue htile_clear{};
htile_clear.depthStencil.depth = 0.25f;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), htile_depth_image,
{vk::ImageAspectFlagBits::eDepth, 0, 1, 0, htile_layers}, htile_clear);
auto htile_storage = htile_depth;
htile_storage.type = BindingType::Storage;
htile_storage.info.pixel_format = vk::Format::eR32Sfloat;
htile_storage.info.metadata = {};
htile_storage.view_info.format = vk::Format::eR32Sfloat;
htile_storage.view_info.aspect = vk::ImageAspectFlagBits::eColor;
htile_storage.view_info.usage = vk::ImageUsageFlagBits::eStorage;
ImageId htile_storage_image{};
for (uint32_t layers : {1u, 2u}) {
htile_storage.info.data.size = layers * htile_slice_size;
htile_storage.info.resources.layers = layers;
htile_storage.info.mip_layout[0].size = htile_storage.info.data.size;
htile_storage.view_info.layer_count = layers;
const auto id = texture_cache.FindImage(htile_storage);
const auto &owner = texture_cache.GetImage(id);
Require(name, "HTile depth storage prefix layout",
id != htile_depth_image &&
(!htile_storage_image || id == htile_storage_image) &&
owner.info.data == htile_depth.info.data &&
owner.info.resources == htile_depth.info.resources &&
owner.info.mip_layout == htile_depth.info.mip_layout &&
owner.backing.layers == htile_layers &&
!owner.info.HasMetadata(),
"partial storage conversion separated array layers from their guest layout");
Require(name, "HTile depth storage prefix view",
texture_cache.FindTexture(id, htile_storage) != nullptr,
"converted depth prefix has no storage view");
htile_storage_image = id;
}
auto htile_readback = CreateHostBuffer(
name, htile_layers * sizeof(float), vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(htile_layers));
vk::BufferImageCopy htile_copy{};
htile_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, htile_layers};
htile_copy.imageExtent = {1, 1, 1};
texture_cache.GetImage(htile_storage_image).Download(
std::span<const vk::BufferImageCopy>(&htile_copy, 1), htile_readback.buffer,
0, htile_readback.size);
HostReadBarrier(htile_readback.buffer, htile_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "HTile depth storage array contents",
ReadBuffer(name, htile_readback, htile_layers) ==
std::vector<u32>(htile_layers, std::bit_cast<u32>(0.25f)),
"partial storage conversion lost native depth data outside its view");
DestroyBuffer(&htile_readback);
auto layered_stencil_desc = MakeLinearDesc(
base + 0x27d4000, 6 * 4 * sizeof(float),
vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k32Float,
Prospero::ImageType::kColor2D, {4, 1, 1}, 6, 4, 1);
layered_stencil_desc.type = BindingType::DepthTarget;
// Four live stencil bytes per slice, with allocation padding.
constexpr uint32_t stencil_slice_size = 16;
layered_stencil_desc.info.stencil = {base + 0x27d6000, 6 * stencil_slice_size};
layered_stencil_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
layered_stencil_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto layered_stencil_id = texture_cache.FindImage(layered_stencil_desc);
auto &layered_stencil_native = texture_cache.GetImage(layered_stencil_id);
layered_stencil_native.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {}, scheduler.Current().Handle());
const vk::ClearDepthStencilValue layered_stencil_clear{0.75f, 0x91};
const vk::ImageSubresourceRange layered_stencil_range{
layered_stencil_desc.view_info.aspect, 0, 1, 0, 6};
scheduler.Current().Handle().clearDepthStencilImage(
layered_stencil_native.backing.image,
vk::ImageLayout::eTransferDstOptimal, &layered_stencil_clear, 1,
&layered_stencil_range);
texture_cache.MarkGpuWritten(layered_stencil_id);
std::array<uint32_t, 6> stencil_expected;
stencil_expected.fill(0x91919191u);
constexpr std::array<std::array<uint32_t, 3>, 4> stencil_cases{{
{0, 1, 0x35353535u}, {0, 3, 0x57575757u},
{2, 1, 0x79797979u}, {0, 1, 0xa5a5a5a5u}}};
for (uint32_t pass = 0; pass < stencil_cases.size(); pass++) {
const auto [first, layers, stencil_word] = stencil_cases[pass];
auto partial_stencil_desc = layered_stencil_desc;
partial_stencil_desc.info.data.address += first * 4 * sizeof(float);
partial_stencil_desc.info.data.size = layers * 4 * sizeof(float);
partial_stencil_desc.info.stencil.address += first * stencil_slice_size;
partial_stencil_desc.info.stencil.size = layers * stencil_slice_size;
partial_stencil_desc.info.resources.layers = layers;
partial_stencil_desc.info.mip_layout[0].size =
partial_stencil_desc.info.data.size;
partial_stencil_desc.view_info.type = layers == 1
? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
partial_stencil_desc.view_info.layer_count = layers;
const auto stencil = partial_stencil_desc.info.stencil;
if (pass != 3) {
(void)resources.GetBufferCache().ObtainBuffer(stencil.address, stencil.size, true);
resources.GetBufferCache().FillBuffer(
stencil.address, stencil.size, stencil_word, false);
const auto partial_stencil_id = texture_cache.FindImage(partial_stencil_desc);
Require(name, "partial stencil retains depth owner",
partial_stencil_id == layered_stencil_id &&
texture_cache.GetImage(partial_stencil_id).info.resources.layers == 6 &&
partial_stencil_desc.view_info.base_layer == first &&
texture_cache.FindDepthTarget(partial_stencil_id,
partial_stencil_desc) != nullptr,
"a partial stencil view replaced its six-layer depth owner");
} else {
// The old prefix association must survive the owner's shifted binding.
Require(name, "partial stencil buffer clear",
texture_cache.ClearImageFromBuffer(scheduler.Current(),
stencil.address, stencil.size, stencil_word),
"the existing partial stencil association could not be cleared");
const auto rebound = texture_cache.FindImage(partial_stencil_desc);
Require(name, "rebind cleared stencil association",
rebound == layered_stencil_id &&
texture_cache.FindDepthTarget(rebound, partial_stencil_desc) != nullptr,
"the cleared stencil association lost its depth owner");
}
std::fill_n(stencil_expected.begin() + first, layers, stencil_word);
auto readback = CreateHostBuffer(name, 30 * sizeof(uint32_t),
vk::BufferUsageFlagBits::eTransferDst, {});
std::vector<vk::BufferImageCopy> copies(2);
copies[0].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0, 6};
copies[1].bufferOffset = 24 * sizeof(float);
copies[1].imageSubresource = {vk::ImageAspectFlagBits::eStencil, 0, 0, 6};
for (auto &copy : copies) copy.imageExtent = {4, 1, 1};
texture_cache.GetImage(layered_stencil_id).Download(
copies, readback.buffer, 0, readback.size);
HostReadBarrier(readback.buffer, readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto words = ReadBuffer(name, readback, 30);
for (uint32_t layer = 0; layer < 6; layer++) {
for (uint32_t pixel = 0; pixel < 4; pixel++) {
Require(name, "partial stencil preserves depth",
words[layer * 4 + pixel] == std::bit_cast<uint32_t>(0.75f),
"stencil upload changed the retained depth plane");
}
Require(name, "partial stencil preserves other layers",
words[24 + layer] == stencil_expected[layer],
"stencil upload used the retained depth allocation's layer count");
}
DestroyBuffer(&readback);
}
auto native_array_info = array_desc.info;
native_array_info.data = {};
auto native_volume_info = volume_desc.info;
native_volume_info.data = {};
Libs::Graphics::Image native_array(m_runtime_context, scheduler,
native_array_info);
Libs::Graphics::Image native_volume(m_runtime_context, scheduler,
native_volume_info);
native_volume.CopyImage(native_array);
native_array.CopyImage(native_volume);
Require(name, "Image-owned 2D-array/3D copy state",
native_volume.backing.state.layout ==
vk::ImageLayout::eTransferSrcOptimal &&
native_volume.backing.state.access_mask ==
vk::AccessFlagBits2::eTransferRead &&
native_array.backing.state.layout ==
vk::ImageLayout::eGeneral &&
native_array.backing.state.access_mask ==
(vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eTransferRead),
"Image::CopyImage did not retain pinned "
"source/destination states");
ImageInfo resolve_source_info{};
resolve_source_info.pixel_format = vk::Format::eR8G8B8A8Unorm;
resolve_source_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm;
resolve_source_info.type = Prospero::ImageType::kColor2D;
resolve_source_info.extent = {4, 4, 1};
resolve_source_info.resources = {1, 1};
resolve_source_info.pitch = 4;
resolve_source_info.bytes_per_block = 4;
resolve_source_info.samples = 2;
resolve_source_info.tile_mode = Prospero::TileMode::kRenderTarget;
ImageInfo resolve_destination_info = resolve_source_info;
resolve_destination_info.type = Prospero::ImageType::kColor2D;
resolve_destination_info.samples = 1;
Libs::Graphics::Image resolve_source(m_runtime_context, scheduler,
resolve_source_info);
Libs::Graphics::Image resolve_destination(m_runtime_context, scheduler,
resolve_destination_info);
resolve_destination.Resolve(resolve_source, {}, {});
Require(
name, "Image-owned MSAA resolve",
resolve_source.backing.state.layout ==
vk::ImageLayout::eTransferSrcOptimal &&
resolve_destination.backing.state.layout ==
vk::ImageLayout::eTransferDstOptimal,
"Image::Resolve did not issue the MSAA-to-single-sample transfer");
Libs::Graphics::Image copy_destination(m_runtime_context, scheduler,
resolve_destination_info);
copy_destination.Resolve(resolve_destination, {}, {});
Require(name, "Image-owned single-sample resolve copy",
resolve_destination.backing.state.layout ==
vk::ImageLayout::eTransferSrcOptimal &&
copy_destination.backing.state.layout ==
vk::ImageLayout::eTransferDstOptimal,
"Image::Resolve did not retain the single-sample copy path");
const std::array<uint16_t, 2> multisample_source{0x4000u, 0xc000u};
std::memcpy(memory + 0x2000, &multisample_source,
sizeof(multisample_source));
auto color_desc = MakeLinearDesc(
base + 0x2000, sizeof(multisample_source), vk::Format::eR16G16Unorm,
Prospero::BufferFormat::k16_16UNorm, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
const auto color_image = texture_cache.FindImage(color_desc);
(void)texture_cache.FindTexture(color_image, color_desc);
texture_cache.MarkGpuWritten(color_image);
const std::array<uint16_t, 2> refreshed_multisample_source{0x2000u,
0xe000u};
Require(name, "unequal-sample source CPU write",
resources.HandleFault(PageFaultAccess::Write,
color_desc.info.data.address),
"color-to-MS-depth source did not accept a CPU update");
std::memcpy(memory + 0x2000, refreshed_multisample_source.data(),
sizeof(refreshed_multisample_source));
constexpr uint64_t ms_stencil_offset = 0x80000;
constexpr uint64_t ms_stencil_size = 0x10000;
auto ms_stencil_owner = resources.GetBufferCache().ObtainBuffer(
base + ms_stencil_offset, ms_stencil_size, false);
Require(name, "MS stencil buffer allocation",
ms_stencil_owner.first != nullptr,
"failed to create an unequal-sample stencil source");
resources.GetBufferCache().FillBuffer(
base + ms_stencil_offset, ms_stencil_size, 0x41414141u, false);
auto ms_depth_desc = color_desc;
ms_depth_desc.type = BindingType::DepthTarget;
ms_depth_desc.info.stencil = {base + ms_stencil_offset, ms_stencil_size};
ms_depth_desc.info.pixel_format = vk::Format::eD24UnormS8Uint;
ms_depth_desc.info.guest_format = Prospero::BufferFormat::k16UNorm;
ms_depth_desc.info.bytes_per_block = 2;
ms_depth_desc.info.samples = 2;
ms_depth_desc.info.type = Prospero::ImageType::kColor2D;
ms_depth_desc.view_info.format = vk::Format::eD24UnormS8Uint;
ms_depth_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
ms_depth_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto ms_depth_image = texture_cache.FindImage(ms_depth_desc);
constexpr uint64_t ms_htile_address = base + 0x2600000;
constexpr uint64_t ms_htile_size = 0x10000;
ms_depth_desc.info.metadata.range = {ms_htile_address, ms_htile_size};
ms_depth_desc.info.metadata.kind = ImageMetadataKind::Htile;
const auto ms_depth_with_htile = texture_cache.FindImage(ms_depth_desc);
const auto ms_depth_view =
texture_cache.FindDepthTarget(ms_depth_with_htile, ms_depth_desc);
auto sampled_ms_depth = ms_depth_desc;
sampled_ms_depth.type = BindingType::Texture;
sampled_ms_depth.info.metadata = {};
sampled_ms_depth.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto sampled_ms_depth_image =
texture_cache.FindImage(sampled_ms_depth);
const auto ms_stencil_alias = texture_cache.FindImageFromRange(
base + ms_stencil_offset, ms_stencil_size, false);
Require(
name, "unequal-sample depth overlap",
ms_depth_image && ms_depth_with_htile == ms_depth_image &&
sampled_ms_depth_image == ms_depth_image &&
ms_depth_image != color_image && ms_depth_view != nullptr &&
texture_cache.GetImage(ms_depth_image).backing.samples == 2 &&
texture_cache.GetImage(ms_depth_image).backing.state.layout ==
vk::ImageLayout::eDepthStencilAttachmentOptimal &&
texture_cache.GetImage(ms_depth_image).IsGpuModified() &&
texture_cache.GetImage(ms_depth_image).info.htile_clear_mask ==
0 &&
texture_cache.GetImage(ms_depth_image).info.metadata.range ==
ms_depth_desc.info.metadata.range &&
texture_cache.IsMeta(ms_htile_address) &&
!texture_cache.IsMetaCleared(ms_htile_address, 0) &&
!resources.GetBufferCache().HasGpuDirtyBytes(
base + ms_stencil_offset, ms_stencil_size) &&
ms_stencil_alias &&
texture_cache.GetImage(ms_stencil_alias).depth_id == ms_depth_image &&
texture_cache.GetImage(ms_stencil_alias).backing.image == nullptr,
"unequal-sample overlap did not run the color-to-MS-depth pass "
"without manufacturing stencil ownership");
auto &oversized_ms = texture_cache.GetImage(ms_depth_image);
const auto ms_data_size = oversized_ms.info.data.size;
oversized_ms.info.data.size = (32ull << 20) + 4;
const bool oversized_ms_readback =
!TextureCacheTestAccess::TryDownload(texture_cache, ms_depth_image);
oversized_ms.info.data.size = ms_data_size;
Require(name, "oversized multisample download rejection",
oversized_ms_readback && oversized_ms.IsGpuModified() &&
!oversized_ms.IsBufferModified() &&
!resources.GetBufferCache().HasGpuDirtyBytes(
ms_depth_desc.info.data.address, ms_data_size),
"an unsupported multisample source reserved download storage or "
"published Buffer ownership");
const std::array<uint16_t, 4> multisample_source_4x{0x0000u, 0x4000u,
0x8000u, 0xffffu};
std::memcpy(memory + 0x7000, multisample_source_4x.data(),
sizeof(multisample_source_4x));
auto color_desc_4x =
MakeLinearDesc(base + 0x7000, sizeof(multisample_source_4x),
vk::Format::eR16G16B16A16Unorm,
Prospero::BufferFormat::k16_16_16_16UNorm,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1);
const auto color_image_4x = texture_cache.FindImage(color_desc_4x);
auto ms_depth_desc_4x = color_desc_4x;
ms_depth_desc_4x.type = BindingType::DepthTarget;
ms_depth_desc_4x.info.pixel_format = vk::Format::eD16Unorm;
ms_depth_desc_4x.info.guest_format = Prospero::BufferFormat::k16UNorm;
ms_depth_desc_4x.info.bytes_per_block = 2;
ms_depth_desc_4x.info.samples = 4;
ms_depth_desc_4x.info.type = Prospero::ImageType::kColor2D;
ms_depth_desc_4x.view_info.format = vk::Format::eD16Unorm;
ms_depth_desc_4x.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
ms_depth_desc_4x.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto ms_depth_image_4x = texture_cache.FindImage(ms_depth_desc_4x);
Require(name, "four-sample depth overlap",
ms_depth_image_4x && ms_depth_image_4x != color_image_4x &&
texture_cache.GetImage(ms_depth_image_4x).backing.samples ==
4,
"four-sample color packing did not produce a depth image");
const auto ms_depth_backing =
texture_cache.GetImage(ms_depth_image).backing.image;
const auto first_stencil_association = texture_cache.FindImageFromRange(
base + ms_stencil_offset, ms_stencil_size, false);
Require(
name, "stencil association shape",
first_stencil_association &&
first_stencil_association != ms_depth_image &&
texture_cache.GetImage(first_stencil_association)
.info.pixel_format == vk::Format::eUndefined &&
texture_cache.GetImage(first_stencil_association).backing.image ==
nullptr &&
texture_cache.GetImage(first_stencil_association).depth_id ==
ms_depth_image,
"stencil address did not create a lightweight depth association");
Require(
name, "stencil association download rejection",
!TextureCacheTestAccess::TryDownload(texture_cache,
first_stencil_association),
"a lightweight stencil association entered image download planning");
auto exact_ms_depth_alias = ms_depth_desc;
exact_ms_depth_alias.type = BindingType::Texture;
exact_ms_depth_alias.info.stencil = {};
exact_ms_depth_alias.info.pixel_format = vk::Format::eR8G8B8A8Unorm;
exact_ms_depth_alias.info.guest_format =
Prospero::BufferFormat::k8_8_8_8UNorm;
exact_ms_depth_alias.info.bytes_per_block = 4;
exact_ms_depth_alias.view_info.format = vk::Format::eR8G8B8A8Unorm;
exact_ms_depth_alias.view_info.aspect = vk::ImageAspectFlagBits::eColor;
exact_ms_depth_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto exact_ms_depth_image =
texture_cache.FindImage(exact_ms_depth_alias, true);
Require(
name, "exact multisample depth/stencil coexistence",
exact_ms_depth_image && exact_ms_depth_image != ms_depth_image &&
TextureCacheTestAccess::Contains(texture_cache, ms_depth_image) &&
texture_cache.GetImage(ms_depth_image).backing.image ==
ms_depth_backing &&
texture_cache.GetImage(first_stencil_association).depth_id ==
ms_depth_image,
"exact multisample depth/stencil lookup replaced the old image "
"or stole its stencil association");
constexpr uint64_t second_stencil_offset = 0x90000;
auto stencil_storage_desc = MakeLinearDesc(
base + second_stencil_offset, ms_stencil_size, vk::Format::eR8Uint,
Prospero::BufferFormat::k8UInt, Prospero::ImageType::kColor2D,
ms_depth_desc.info.extent, 1, 1, 1);
stencil_storage_desc.type = BindingType::Storage;
stencil_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto stencil_storage_image =
texture_cache.FindImage(stencil_storage_desc);
(void)texture_cache.FindTexture(stencil_storage_image,
stencil_storage_desc);
auto switched_ms_depth = ms_depth_desc;
switched_ms_depth.info.stencil = {base + second_stencil_offset,
ms_stencil_size};
const auto switched_ms_depth_id =
texture_cache.FindImage(switched_ms_depth);
TextureCacheTestAccess::AssociateStencil(
texture_cache, switched_ms_depth_id, switched_ms_depth.info.stencil);
const auto second_stencil_association = texture_cache.FindImageFromRange(
base + second_stencil_offset, ms_stencil_size, false);
Require(name, "stencil association switch",
switched_ms_depth_id == ms_depth_image &&
texture_cache.GetImage(switched_ms_depth_id).backing.image ==
ms_depth_backing &&
texture_cache.FindImageFromRange(base + ms_stencil_offset,
ms_stencil_size, false) ==
first_stencil_association &&
texture_cache.GetImage(first_stencil_association).depth_id ==
ms_depth_image &&
second_stencil_association &&
second_stencil_association == stencil_storage_image &&
texture_cache.GetImage(second_stencil_association).IsTracked() &&
texture_cache.GetImage(second_stencil_association).depth_id ==
ms_depth_image,
"changing stencil address recreated depth or discarded an "
"existing lightweight association");
Require(name, "CPU write to tracked stencil association",
resources.HandleFault(PageFaultAccess::Write,
base + second_stencil_offset) &&
!texture_cache.GetImage(second_stencil_association).IsTracked() &&
texture_cache.GetImage(second_stencil_association).IsCpuDirty() &&
texture_cache.GetImage(second_stencil_association).depth_id ==
ms_depth_image,
"CPU invalidation left a reused stencil image write-protected");
memory[second_stencil_offset] = 0x5a;
constexpr uint64_t added_stencil_depth_offset = 0x60000;
constexpr uint64_t added_stencil_offset = 0x70000;
constexpr uint64_t added_stencil_size = 0x10000;
constexpr float added_stencil_depth_value = 0.75f;
std::memcpy(memory + added_stencil_depth_offset,
&added_stencil_depth_value,
sizeof(added_stencil_depth_value));
auto depth_only_source = MakeLinearDesc(
base + added_stencil_depth_offset, 0x10000, vk::Format::eD32Sfloat,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
depth_only_source.type = BindingType::DepthTarget;
depth_only_source.view_info.format = vk::Format::eD32Sfloat;
depth_only_source.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
depth_only_source.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto depth_only_source_image =
texture_cache.FindImage(depth_only_source);
auto combined_destination = depth_only_source;
combined_destination.info.stencil = {base + added_stencil_offset,
added_stencil_size};
combined_destination.info.pixel_format = vk::Format::eD32SfloatS8Uint;
combined_destination.view_info.format = vk::Format::eD32SfloatS8Uint;
combined_destination.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
const auto combined_destination_image =
texture_cache.FindImage(combined_destination);
TextureCacheTestAccess::AssociateStencil(
texture_cache, combined_destination_image,
combined_destination.info.stencil);
const auto combined_stencil_association =
texture_cache.FindImageFromRange(base + added_stencil_offset,
added_stencil_size, false);
Require(
name, "depth-only to combined association",
combined_destination_image != depth_only_source_image &&
combined_stencil_association &&
texture_cache.GetImage(combined_stencil_association).depth_id ==
combined_destination_image &&
texture_cache.GetImage(combined_stencil_association)
.backing.image == nullptr,
"stencil addition did not recreate depth and register its "
"lightweight association");
const uint32_t mirror_value = 0xa1b2c3d4u;
std::memcpy(memory + 0x5000, &mirror_value, sizeof(mirror_value));
auto mirror_desc =
MakeLinearDesc(base + 0x5000, sizeof(mirror_value),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto mirror_image = texture_cache.FindImage(mirror_desc);
(void)texture_cache.FindTexture(mirror_image, mirror_desc);
texture_cache.MarkGpuWritten(mirror_image);
constexpr uint32_t mirror_cpu_value = 0x0f1e2d3cu;
Require(name, "CPU-dirty formatted mirror fault",
resources.HandleFault(PageFaultAccess::Write,
mirror_desc.info.data.address),
"GPU image did not accept a CPU write before Buffer mirroring");
std::memcpy(memory + 0x5000, &mirror_cpu_value, sizeof(mirror_cpu_value));
auto mirror_binding = resources.GetBufferCache().ObtainBuffer(
base + 0x5000, sizeof(mirror_value), false, true);
Require(name, "CPU-dirty formatted mirror source",
mirror_binding.first != nullptr &&
!texture_cache.GetImage(mirror_image).IsBufferModified(),
"formatted mirror did not expose a readable Buffer source");
auto mirror_cpu_readback = CreateHostBuffer(
name, sizeof(mirror_cpu_value), vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>{0});
const vk::BufferCopy mirror_cpu_copy{mirror_binding.second, 0,
sizeof(mirror_cpu_value)};
command.Handle().copyBuffer(mirror_binding.first->Handle(),
mirror_cpu_readback.buffer, 1,
&mirror_cpu_copy);
HostReadBarrier(mirror_cpu_readback.buffer, mirror_cpu_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
auto mirror_refresh_desc = mirror_desc;
const auto mirror_refresh = texture_cache.FindImage(mirror_refresh_desc);
(void)texture_cache.FindTexture(mirror_refresh, mirror_refresh_desc);
Require(name, "buffer-to-image ownership",
mirror_refresh == mirror_image &&
!texture_cache.GetImage(mirror_refresh).IsBufferModified(),
"buffer-backed refresh incorrectly transferred dirty ownership "
"to the image");
texture_cache.MarkGpuWritten(mirror_refresh);
// A write through a separate buffer must not replace untouched image
// texels with stale backing bytes. Exact-base writes still refresh it.
constexpr uint64_t buffer_alias_offset = 0x1a00000;
constexpr uint32_t buffer_alias_image_value = 0x13579bdfu;
constexpr uint32_t buffer_alias_write_value = 0x2468ace0u;
std::memset(memory + buffer_alias_offset, 0, 4 * sizeof(uint32_t));
auto buffer_alias_desc = MakeLinearDesc(
base + buffer_alias_offset, 4 * sizeof(uint32_t), vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{4, 1, 1}, 1, 4, 1);
const auto buffer_alias_image = texture_cache.FindImage(buffer_alias_desc);
for (const uint32_t write_offset : {0u, 12u}) {
vk::ClearValue painted{};
painted.color.uint32 = std::array{buffer_alias_image_value, 0u, 0u, 0u};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), buffer_alias_image,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
const auto write_address = buffer_alias_desc.info.data.address + write_offset;
auto writable = resources.GetBufferCache().ObtainBuffer(
write_address, sizeof(uint32_t), true, false);
Require(name, "image alias buffer allocation", writable.first != nullptr,
"failed to allocate the image alias write buffer");
writable.first->Fill(writable.second, sizeof(uint32_t), buffer_alias_write_value);
texture_cache.InvalidateMemoryFromGPU(write_address, sizeof(uint32_t));
(void)texture_cache.FindTexture(buffer_alias_image, buffer_alias_desc);
const bool exact_base = write_offset == 0;
const uint32_t read_texels = exact_base ? 1 : 3;
Require(name, exact_base ? "exact-base buffer write refresh"
: "overlapping buffer preserves untouched texels",
ReadCachedTexel(name, context, buffer_alias_image, {},
{read_texels, 1, 1}) ==
std::vector<u32>(read_texels, exact_base ? buffer_alias_write_value
: buffer_alias_image_value),
exact_base ? "exact-base buffer write did not refresh the native image"
: "overlapping buffer write replaced untouched GPU image texels");
}
constexpr uint64_t exact_buffer_offset = 0x90000;
constexpr uint32_t exact_buffer_value = 0x3f234567u;
std::memcpy(memory + exact_buffer_offset, &exact_buffer_value,
sizeof(exact_buffer_value));
auto exact_buffer_desc =
MakeLinearDesc(base + exact_buffer_offset, sizeof(exact_buffer_value),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto exact_buffer_image =
texture_cache.FindImage(exact_buffer_desc);
(void)texture_cache.FindTexture(exact_buffer_image, exact_buffer_desc);
texture_cache.MarkGpuWritten(exact_buffer_image);
auto exact_buffer_binding = resources.GetBufferCache().ObtainBuffer(
exact_buffer_desc.info.data.address, exact_buffer_desc.info.data.size,
false, true);
Require(
name, "exact replacement Buffer synchronization",
exact_buffer_binding.first != nullptr &&
!texture_cache.GetImage(exact_buffer_image).IsBufferModified() &&
texture_cache.GetImage(exact_buffer_image).IsGpuModified(),
"exact replacement copy transferred cache ownership");
auto exact_float_desc = exact_buffer_desc;
exact_float_desc.info.pixel_format = vk::Format::eR32Sfloat;
exact_float_desc.info.guest_format = Prospero::BufferFormat::k32Float;
exact_float_desc.view_info.format = vk::Format::eR32Sfloat;
const auto exact_float_image =
texture_cache.FindImage(exact_float_desc, true);
(void)texture_cache.FindTexture(exact_float_image, exact_float_desc);
Require(
name, "Buffer-superseded exact coexistence",
exact_float_image != exact_buffer_image &&
TextureCacheTestAccess::Contains(texture_cache,
exact_buffer_image) &&
texture_cache.GetImage(exact_buffer_image).IsGpuModified() &&
!texture_cache.GetImage(exact_float_image).IsBufferModified() &&
!texture_cache.GetImage(exact_float_image).IsGpuModified(),
"exact-format lookup retired its old record or transferred Buffer "
"ownership");
auto exact_buffer_readback = CreateHostBuffer(
name, sizeof(exact_buffer_value),
vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>{0});
auto &exact_float_native = texture_cache.GetImage(exact_float_image);
exact_float_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
command.Handle());
vk::BufferImageCopy exact_float_copy{};
exact_float_copy.imageSubresource.aspectMask =
vk::ImageAspectFlagBits::eColor;
exact_float_copy.imageSubresource.layerCount = 1;
exact_float_copy.imageExtent = {1, 1, 1};
command.Handle().copyImageToBuffer(exact_float_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal,
exact_buffer_readback.buffer, 1,
&exact_float_copy);
HostReadBarrier(exact_buffer_readback.buffer, exact_buffer_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
const uint32_t clear_source = 0x01010101u;
std::memcpy(memory + 0x6000, &clear_source, sizeof(clear_source));
auto clear_desc = MakeLinearDesc(
base + 0x6000, sizeof(clear_source), vk::Format::eR8G8B8A8Unorm,
Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
const auto clear_image = texture_cache.FindImage(clear_desc);
auto buffer_write = resources.GetBufferCache().ObtainBuffer(
base + 0x6000, sizeof(clear_source), true, true);
texture_cache.InvalidateMemoryFromGPU(base + 0x6000,
sizeof(clear_source));
Require(name, "buffer-write ownership",
buffer_write.first != nullptr &&
texture_cache.GetImage(clear_image).IsBufferModified(),
"formatted buffer write did not supersede the image");
Require(name, "buffer-owned full image clear",
texture_cache.ClearImageFromBuffer(
command, base + 0x6000, sizeof(clear_source), 0xaabbccddu) &&
!texture_cache.GetImage(clear_image).IsBufferModified() &&
texture_cache.GetImage(clear_image).IsGpuModified(),
"full image clear retained dual buffer/image GPU ownership");
texture_cache.MarkGpuWritten(clear_image);
auto clear_read_desc = clear_desc;
Require(name, "post-clear image reuse",
texture_cache.FindImage(clear_read_desc) == clear_image,
"cleared image could not be read or written after ownership "
"transfer");
auto stencil_clear_desc = MakeLinearDesc(
base + 0x27e0000, 0x1000, vk::Format::eD32SfloatS8Uint,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{13, 11, 1}, 2, 4, 1);
stencil_clear_desc.info.stencil = {base + 0x27e2000, 0x1000};
stencil_clear_desc.info.resources.levels = 2;
stencil_clear_desc.info.mip_layout[0] = {0, 0x800, 13, 11};
stencil_clear_desc.info.mip_layout[1] = {0x800, 0x800, 6, 5};
stencil_clear_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
const auto stencil_clear_id = texture_cache.FindImage(stencil_clear_desc);
auto &stencil_clear_image = texture_cache.GetImage(stencil_clear_id);
stencil_clear_image.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {}, command.Handle());
const vk::ClearDepthStencilValue initial_depth_stencil{0.75f, 0x91};
const vk::ImageSubresourceRange all_depth_stencil{
stencil_clear_desc.view_info.aspect, 0, 2, 0, 2};
command.Handle().clearDepthStencilImage(stencil_clear_image.backing.image,
vk::ImageLayout::eTransferDstOptimal, &initial_depth_stencil, 1,
&all_depth_stencil);
texture_cache.MarkGpuWritten(stencil_clear_id);
vk::ClearValue stencil_value{};
stencil_value.depthStencil = vk::ClearDepthStencilValue{0.f, 0x35};
TextureCacheTestAccess::ClearImage(texture_cache, command, stencil_clear_id,
{vk::ImageAspectFlagBits::eStencil, 1, 1, 1, 1}, stencil_value);
// Read every mip/layer of both aspects to prove that the stencil clear is local.
auto stencil_readback = CreateHostBuffer(name, 0x4000,
vk::BufferUsageFlagBits::eTransferDst, {});
std::vector<vk::BufferImageCopy> stencil_copies;
for (uint32_t mip = 0; mip < 2; mip++) {
for (uint32_t layer = 0; layer < 2; layer++) {
for (const auto aspect : {vk::ImageAspectFlagBits::eDepth,
vk::ImageAspectFlagBits::eStencil}) {
vk::BufferImageCopy copy{};
copy.bufferOffset = stencil_copies.size() * 0x800;
copy.imageSubresource = {aspect, mip, layer, 1};
copy.imageExtent = {13u >> mip, 11u >> mip, 1};
stencil_copies.push_back(copy);
}
}
}
stencil_clear_image.Download(stencil_copies, stencil_readback.buffer, 0,
stencil_readback.size);
HostReadBarrier(stencil_readback.buffer, stencil_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto stencil_words = ReadBuffer(name, stencil_readback, 0x1000);
const auto *stencil_bytes = reinterpret_cast<const uint8_t *>(stencil_words.data());
for (const auto &copy : stencil_copies) {
const auto &sub = copy.imageSubresource;
const auto count = copy.imageExtent.width * copy.imageExtent.height;
const auto *bytes = stencil_bytes + copy.bufferOffset;
const bool depth = sub.aspectMask == vk::ImageAspectFlagBits::eDepth;
uint32_t expected = 0x91u;
if (depth) {
expected = std::bit_cast<uint32_t>(0.75f);
} else if (sub.mipLevel == 1 && sub.baseArrayLayer == 1) {
expected = 0x35u;
}
for (uint32_t pixel = 0; pixel < count; pixel++) {
uint32_t actual = bytes[pixel];
if (depth) std::memcpy(&actual, bytes + pixel * 4, 4);
Require(name, "stencil clear preserves depth and neighbors", actual == expected,
"selected stencil clear changed depth, another mip, or another layer");
}
}
DestroyBuffer(&stencil_readback);
constexpr uint64_t partial_image_offset = 0xa000;
constexpr uint64_t partial_buffer_offset = 0xa010;
constexpr uint64_t partial_clean_offset = 0xa020;
constexpr uint32_t partial_image_value = 0x31415926u;
constexpr uint32_t partial_buffer_value = 0x27182818u;
constexpr uint32_t partial_clean_value = 0xabcdef01u;
std::memcpy(memory + partial_clean_offset, &partial_clean_value,
sizeof(partial_clean_value));
auto partial_write = resources.GetBufferCache().ObtainBuffer(
base + partial_image_offset, sizeof(partial_image_value), true);
Require(name, "partial-page buffer allocation",
partial_write.first != nullptr,
"partial-page GPU write did not create a native buffer");
resources.GetBufferCache().FillBuffer(base + partial_image_offset,
sizeof(partial_image_value),
partial_image_value, false);
resources.GetBufferCache().FillBuffer(base + partial_buffer_offset,
sizeof(partial_buffer_value),
partial_buffer_value, false);
auto partial_desc = MakeLinearDesc(
base + partial_image_offset, sizeof(partial_image_value),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto partial_image = texture_cache.FindImage(partial_desc);
Require(name, "partial-page image upload",
!texture_cache.GetImage(partial_image).IsGpuModified(),
"image upload incorrectly consumed buffer dirty ownership");
auto partial_image_mirror = resources.GetBufferCache().ObtainBuffer(
base + partial_image_offset, sizeof(partial_image_value), false,
true);
Require(name, "partial-page image mirror",
partial_image_mirror.first != nullptr &&
!texture_cache.GetImage(partial_image).IsGpuModified(),
"same-page image upload lost its buffer source");
const auto partial_clean_source =
resources.GetBufferCache().ObtainBufferForImage(
base + partial_clean_offset, sizeof(partial_clean_value));
Require(name, "partial-page clean source",
partial_clean_source.first != nullptr,
"clean same-page bytes inherited unrelated buffer ownership");
Require(name, "partial-page remaining fault",
resources.HandleFault(PageFaultAccess::Read,
base + partial_buffer_offset),
"same-page buffer ownership was lost after image transfer");
uint32_t partial_buffer_backing = 0;
std::memcpy(&partial_buffer_backing, memory + partial_buffer_offset,
sizeof(partial_buffer_backing));
uint32_t partial_image_backing = 0;
std::memcpy(&partial_image_backing, memory + partial_image_offset,
sizeof(partial_image_backing));
Require(name, "partial-page readback values",
partial_image_backing == partial_image_value &&
partial_buffer_backing == partial_buffer_value,
"same-page image/buffer readback published incorrect bytes");
constexpr uint32_t partial_cpu_refresh_value = 0x13579bdfu;
Require(name, "partial-page CPU write fault",
resources.HandleFault(PageFaultAccess::Write,
base + partial_clean_offset),
"same-page CPU write did not invalidate cached ownership");
std::memcpy(memory + partial_clean_offset, &partial_cpu_refresh_value,
sizeof(partial_cpu_refresh_value));
constexpr std::array<uint32_t, 2> partial_neighbors{0x2468ace0u,
0xfedcba98u};
std::memcpy(memory + partial_clean_offset - sizeof(uint32_t),
&partial_neighbors[0], sizeof(uint32_t));
std::memcpy(memory + partial_clean_offset + sizeof(uint32_t),
&partial_neighbors[1], sizeof(uint32_t));
std::vector<u32> partial_cpu_refresh_expected(TRACKER_PAGE_SIZE / sizeof(u32));
std::memcpy(partial_cpu_refresh_expected.data(), memory + partial_image_offset,
TRACKER_PAGE_SIZE);
const auto partial_cpu_refresh_source =
resources.GetBufferCache().ObtainBufferForImage(
base + partial_clean_offset, sizeof(partial_cpu_refresh_value));
Require(name, "partial-page CPU refresh source",
partial_cpu_refresh_source.first != nullptr,
"CPU-dirty same-page bytes did not resolve through the cached "
"buffer");
auto partial_cpu_refresh_readback =
CreateHostBuffer(name, TRACKER_PAGE_SIZE,
vk::BufferUsageFlagBits::eTransferDst, {0});
const vk::BufferCopy partial_cpu_refresh_copy{
partial_cpu_refresh_source.first->Offset(base + partial_image_offset), 0,
TRACKER_PAGE_SIZE};
command.Handle().copyBuffer(partial_cpu_refresh_source.first->Handle(),
partial_cpu_refresh_readback.buffer, 1,
&partial_cpu_refresh_copy);
vk::BufferMemoryBarrier partial_cpu_refresh_barrier{};
partial_cpu_refresh_barrier.sType =
vk::StructureType::eBufferMemoryBarrier;
partial_cpu_refresh_barrier.srcAccessMask =
vk::AccessFlagBits::eTransferWrite;
partial_cpu_refresh_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
partial_cpu_refresh_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
partial_cpu_refresh_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
partial_cpu_refresh_barrier.buffer = partial_cpu_refresh_readback.buffer;
partial_cpu_refresh_barrier.size = partial_cpu_refresh_readback.size;
command.Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0,
nullptr, 1, &partial_cpu_refresh_barrier,
0, nullptr);
const uint32_t fault_a = 0x01020304u;
const uint32_t fault_b = 0x11121314u;
std::memcpy(memory + 0x8000, &fault_a, sizeof(fault_a));
std::memcpy(memory + 0x8010, &fault_b, sizeof(fault_b));
auto fault_a_desc =
MakeLinearDesc(base + 0x8000, sizeof(fault_a), vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
auto fault_b_desc = fault_a_desc;
fault_b_desc.info.data.address = base + 0x8010;
const auto fault_a_image = texture_cache.FindImage(fault_a_desc);
const auto fault_b_image = texture_cache.FindImage(fault_b_desc);
(void)texture_cache.FindTexture(fault_a_image, fault_a_desc);
(void)texture_cache.FindTexture(fault_b_image, fault_b_desc);
texture_cache.MarkGpuWritten(fault_a_image);
texture_cache.MarkGpuWritten(fault_b_image);
Require(name, "per-image watcher installation",
texture_cache.GetImage(fault_a_image).IsTracked() &&
texture_cache.GetImage(fault_b_image).IsTracked(),
"same-page images did not install independent write watchers");
constexpr uint64_t padding_fault_offset = 0x8080;
uint32_t write_only_read_a = 0;
uint32_t write_only_read_b = 0;
std::memcpy(&write_only_read_a, memory + 0x8000,
sizeof(write_only_read_a));
std::memcpy(&write_only_read_b, memory + 0x8010,
sizeof(write_only_read_b));
Require(name, "write-only image read policy",
write_only_read_a == fault_a && write_only_read_b == fault_b &&
texture_cache.GetImage(fault_a_image).IsGpuModified() &&
texture_cache.GetImage(fault_b_image).IsGpuModified(),
"TextureCache claimed a CPU read fault instead of preserving "
"GPU image ownership");
Require(name, "same-page image write invalidation",
resources.HandleFault(PageFaultAccess::Write,
base + padding_fault_offset) &&
texture_cache.GetImage(fault_a_image).IsGpuModified() &&
texture_cache.GetImage(fault_b_image).IsGpuModified() &&
!texture_cache.GetImage(fault_a_image).IsTracked() &&
!texture_cache.GetImage(fault_b_image).IsTracked() &&
texture_cache.GetImage(fault_a_image).IsMaybeCpuDirty() &&
texture_cache.GetImage(fault_b_image).IsMaybeCpuDirty() &&
!texture_cache.GetImage(fault_a_image).IsDefinitelyCpuDirty() &&
!texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty(),
"a byte-disjoint CPU write discarded authoritative images");
const auto retracked_a = texture_cache.FindImage(fault_a_desc);
const auto retracked_b = texture_cache.FindImage(fault_b_desc);
(void)texture_cache.FindTexture(retracked_a, fault_a_desc);
(void)texture_cache.FindTexture(retracked_b, fault_b_desc);
auto fault_a_mirror = resources.GetBufferCache().ObtainBuffer(
base + 0x8000, sizeof(fault_a), false, true);
Require(name, "same-page image re-track",
retracked_a == fault_a_image && retracked_b == fault_b_image &&
texture_cache.GetImage(fault_a_image).IsTracked() &&
texture_cache.GetImage(fault_b_image).IsTracked() &&
!texture_cache.GetImage(fault_a_image).IsCpuDirty() &&
!texture_cache.GetImage(fault_b_image).IsCpuDirty() &&
fault_a_mirror.first != nullptr &&
texture_cache.GetImage(fault_a_image).IsTracked() &&
texture_cache.GetImage(fault_b_image).IsTracked() &&
texture_cache.GetImage(fault_a_image).IsGpuModified() &&
texture_cache.GetImage(fault_b_image).IsGpuModified(),
"copying one same-page image lost an authoritative owner");
Require(name, "same-page survivor write fault",
resources.HandleFault(PageFaultAccess::Write, base + 0x8010) &&
texture_cache.GetImage(fault_b_image).IsGpuModified() &&
texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty(),
"the surviving image was not protected after its alias retired");
constexpr uint32_t fault_b_cpu = 0xa5a6a7a8u;
std::memcpy(memory + 0x8010, &fault_b_cpu, sizeof(fault_b_cpu));
const auto refreshed_b = texture_cache.FindImage(fault_b_desc);
(void)texture_cache.FindTexture(refreshed_b, fault_b_desc);
Require(
name, "same-page survivor refresh",
refreshed_b == fault_b_image &&
!texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty() &&
texture_cache.GetImage(fault_b_image).IsGpuModified(),
"the surviving image could not reconcile its CPU write");
constexpr uint64_t publish_image_offset = 0x26000;
constexpr uint64_t publish_buffer_offset = 0x26010;
constexpr uint32_t publish_image_value = 0x31415926u;
constexpr uint32_t publish_buffer_value = 0x27182818u;
std::memcpy(memory + publish_image_offset, &publish_image_value,
sizeof(publish_image_value));
auto publish_image_desc = MakeLinearDesc(
base + publish_image_offset, sizeof(publish_image_value),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto publish_image = texture_cache.FindImage(publish_image_desc);
(void)texture_cache.FindTexture(publish_image, publish_image_desc);
texture_cache.MarkGpuWritten(publish_image);
resources.GetBufferCache().FillBuffer(base + publish_buffer_offset,
sizeof(publish_buffer_value),
publish_buffer_value, false);
auto publish_replacement_desc = publish_image_desc;
publish_replacement_desc.info.pixel_format = vk::Format::eR32Sfloat;
publish_replacement_desc.info.guest_format =
Prospero::BufferFormat::k32Float;
publish_replacement_desc.view_info.format = vk::Format::eR32Sfloat;
const auto publish_replacement =
texture_cache.FindImage(publish_replacement_desc, true);
Require(name, "exact-disjoint image replacement",
publish_replacement && publish_replacement != publish_image,
"exact-format lookup did not replace the image");
Require(name, "exact-disjoint image publication",
!resources.GetBufferCache().HasGpuDirtyBytes(
base + publish_buffer_offset, sizeof(publish_buffer_value)),
"clean neighboring buffer unexpectedly became GPU-owned");
uint32_t published_image_backing = 0;
uint32_t published_buffer_backing = 0;
std::memcpy(&published_image_backing, memory + publish_image_offset,
sizeof(published_image_backing));
std::memcpy(&published_buffer_backing, memory + publish_buffer_offset,
sizeof(published_buffer_backing));
Require(name, "exact-disjoint publication contents",
published_image_backing == publish_image_value &&
published_buffer_backing == publish_buffer_value,
"image publication or neighboring buffer readback changed bytes");
for (const auto [offset, samples] :
std::array<std::pair<uint64_t, uint32_t>, 2>{
{{0x10000, 2}, {0x11000, 4}}}) {
std::memset(memory + offset, 0, samples * sizeof(uint16_t));
auto standalone_ms = MakeLinearDesc(
base + offset, samples * sizeof(uint16_t), vk::Format::eD16Unorm,
Prospero::BufferFormat::k16UNorm, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 2, samples);
standalone_ms.type = BindingType::DepthTarget;
standalone_ms.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
standalone_ms.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto image = texture_cache.FindImage(standalone_ms);
(void)texture_cache.FindDepthTarget(image, standalone_ms);
texture_cache.MarkGpuWritten(image);
uint16_t standalone_cpu_read = 1;
std::memcpy(&standalone_cpu_read, memory + offset,
sizeof(standalone_cpu_read));
Require(
name, "standalone MSAA ownership",
texture_cache.GetImage(image).IsGpuModified() &&
standalone_cpu_read == 0 &&
texture_cache.GetImage(image).IsGpuModified() &&
resources.HandleFault(PageFaultAccess::Write, base + offset) &&
texture_cache.GetImage(image).IsGpuModified() &&
texture_cache.GetImage(image).IsDefinitelyCpuDirty(),
"fresh multisample target violated write-only ownership");
}
// A one-layer guest descriptor can alias layer 1 of a cached native array.
constexpr uint64_t alias_dcc_address = base + 0x2120000;
auto dcc_array = MakeLinearDesc(
base + 0x2100000, 0x20000, vk::Format::eR16G16B16A16Sfloat,
Prospero::BufferFormat::k16_16_16_16Float, Prospero::ImageType::kColor2D,
{128, 64, 1}, 2, 8, 1);
dcc_array.type = BindingType::RenderTarget;
dcc_array.info.tile_mode = Prospero::TileMode::kRenderTarget;
dcc_array.info.pitch = TileGetRenderTargetPitch(128, 8);
dcc_array.info.mip_layout[0].pitch = dcc_array.info.pitch;
dcc_array.info.metadata.kind = ImageMetadataKind::Dcc;
dcc_array.info.metadata.range = {alias_dcc_address, 0x2000};
dcc_array.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
WriteMetadata(context, alias_dcc_address, 0x2000, UINT32_MAX);
WriteMetadata(context, alias_dcc_address + 0x2000, 0x1000, 0x5a5a5a5au);
const auto dcc_array_id = texture_cache.FindImage(dcc_array);
vk::ClearValue array_paint{};
array_paint.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), dcc_array_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 2}, array_paint);
auto dcc_alias = dcc_array;
dcc_alias.info.data = {dcc_array.info.data.address + 0x10000, 0x10000};
dcc_alias.info.resources.layers = 1;
dcc_alias.info.mip_layout[0].size = 0x10000;
dcc_alias.info.metadata.range = {alias_dcc_address + 0x1000, 0x1000};
dcc_alias.view_info.type = vk::ImageViewType::e2D;
dcc_alias.view_info.layer_count = 1;
WriteMetadata(context, dcc_alias.info.metadata.range.address, 0x1000, 0x40404040u);
Require(name, "DCC alias host layer",
texture_cache.FindImage(dcc_alias) == dcc_array_id &&
dcc_alias.view_info.base_layer == 1 &&
ReadCachedTexel(name, context, dcc_array_id) ==
std::vector<u32>{0x00003c00u, 0x3c003c00u} &&
ReadCachedTexel(name, context, dcc_array_id, {}, {1, 1, 1}, 1) ==
std::vector<u32>{0, 0x3c000000u},
"guest metadata layer 0 did not clear only remapped native layer 1");
resources.GetBufferCache().ReadMemory(alias_dcc_address, 0x3000, false);
std::array<uint8_t, 0x3000> alias_metadata{};
Require(name, "DCC alias metadata bounds",
LibKernel::Memory::TryReadBacking(alias_dcc_address, alias_metadata.data(),
alias_metadata.size()) &&
std::all_of(alias_metadata.begin(), alias_metadata.begin() + 0x2000,
[](uint8_t value) { return value == 0xff; }) &&
std::all_of(alias_metadata.begin() + 0x2000, alias_metadata.end(),
[](uint8_t value) { return value == 0x5a; }),
"remapped host layer changed metadata outside the guest alias allocation");
RenderColorInfo retired_alias{};
retired_alias.desc = dcc_alias;
retired_alias.image_id = dcc_array_id;
retired_alias.guest_mip_level = 0;
retired_alias.guest_array_layer = 0;
texture_cache.UnmapMemory(dcc_array.info.data.address, dcc_array.info.data.size);
WriteMetadata(context, dcc_alias.info.metadata.range.address, 0x1000, 0x40404040u);
RenderExecutorTestAccess::PrepareGraphicsBindings(
context.GetRenderExecutor(), std::span<PreparedBindings *const>{},
std::span{&retired_alias, 1u});
RenderDepthInfo alias_depth{};
const auto alias_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
context.GetRenderExecutor(), scheduler.Current(), &retired_alias, 1, alias_depth);
Require(name, "retired DCC alias native coordinates",
retired_alias.image_id != dcc_array_id &&
retired_alias.desc.view_info.base_level == 0 &&
retired_alias.desc.view_info.base_layer == 0 &&
retired_alias.desc.info.resources.layers == 1 &&
alias_rendering.color_attachments[0].image_view != nullptr &&
ReadCachedTexel(name, context, retired_alias.image_id) ==
std::vector<u32>{0, 0x3c000000u},
"rediscovery treated a retired host layer as the native DCC slice");
resources.GetBufferCache().ReadMemory(alias_dcc_address, 0x3000);
Require(name, "retired DCC alias metadata bounds",
LibKernel::Memory::TryReadBacking(alias_dcc_address, alias_metadata.data(),
alias_metadata.size()) &&
std::all_of(alias_metadata.begin(), alias_metadata.begin() + 0x2000,
[](uint8_t byte) { return byte == 0xff; }) &&
std::all_of(alias_metadata.begin() + 0x2000, alias_metadata.end(),
[](uint8_t byte) { return byte == 0x5a; }),
"retired alias rediscovery overwrote metadata outside its native slice");
RenderExecutorTestAccess::ResetBindings(context.GetRenderExecutor());
constexpr uint64_t metadata_data_a = 0x12000;
constexpr uint64_t metadata_data_b = 0x12100;
constexpr uint64_t metadata_a = 0x13000;
constexpr uint64_t metadata_b = 0x13100;
auto MakeMetadataDepth = [&](uint64_t data, uint64_t metadata) {
auto desc = MakeLinearDesc(
base + data, sizeof(uint32_t), vk::Format::eD32Sfloat,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
desc.type = BindingType::DepthTarget;
desc.info.metadata.kind = ImageMetadataKind::Htile;
desc.info.metadata.range = {base + metadata, 0x80};
desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
return desc;
};
auto metadata_depth_a = MakeMetadataDepth(metadata_data_a, metadata_a);
auto metadata_depth_b = MakeMetadataDepth(metadata_data_b, metadata_b);
metadata_depth_a.info.htile_clear_mask = 0;
const auto metadata_depth_a_id =
texture_cache.FindImage(metadata_depth_a);
const auto metadata_depth_b_id =
texture_cache.FindImage(metadata_depth_b);
WriteMetadata(context, base + metadata_a, 0x80, 0);
Require(
name, "unbound metadata separation",
!texture_cache.IsMeta(base + metadata_a),
"unbound metadata bytes were exposed as registered HTile");
const auto metadata_depth_a_view =
texture_cache.FindDepthTarget(metadata_depth_a_id, metadata_depth_a);
const auto metadata_depth_b_view =
texture_cache.FindDepthTarget(metadata_depth_b_id, metadata_depth_b);
Require(name, "shared-page metadata state",
metadata_depth_a_view != nullptr &&
metadata_depth_b_view != nullptr &&
!texture_cache.IsMetaCleared(base + metadata_a, 0) &&
texture_cache.ClearMeta(base + metadata_a) &&
texture_cache.ClearMeta(base + metadata_b),
"shared-page metadata did not retain logical clear state");
auto metadata_alias = MakeLinearDesc(
base + metadata_a, sizeof(uint32_t), vk::Format::eR32Sfloat,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
metadata_alias.type = BindingType::RenderTarget;
metadata_alias.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto metadata_alias_id =
texture_cache.FindImage(metadata_alias, true);
const auto metadata_alias_view =
texture_cache.FindRenderTarget(metadata_alias_id, metadata_alias);
Require(
name, "exact alias metadata coexistence",
metadata_alias_view != nullptr &&
texture_cache.GetImage(metadata_depth_a_id).IsGpuModified() &&
texture_cache.GetImage(metadata_depth_b_id).IsGpuModified() &&
texture_cache.IsMeta(base + metadata_a) &&
texture_cache.IsMetaCleared(base + metadata_a, 0) &&
texture_cache.IsMeta(base + metadata_b) &&
texture_cache.IsMetaCleared(base + metadata_b, 0),
"image discovery retired aliased metadata or its live depth image");
const bool touched_metadata = texture_cache.TouchMeta(base + metadata_a, 0, false);
WriteMetadata(context, base + metadata_a, 0x80, 0);
Require(
name, "registered DCC/HTile separation",
touched_metadata && !texture_cache.IsMetaCleared(base + metadata_a, 0),
"a PS5 DCC fill changed registered HTile state");
Require(
name, "metadata first-touch state",
texture_cache.TouchMeta(base + metadata_b, 0, true) &&
texture_cache.IsMetaCleared(base + metadata_b, 0) &&
texture_cache.TouchMeta(base + metadata_b, 0, false) &&
!texture_cache.IsMetaCleared(base + metadata_b, 0),
"per-slice metadata update incorrectly required prior GPU ownership");
resources.GetBufferCache().FillBuffer(base + metadata_b, sizeof(uint32_t),
0, false);
Require(
name, "metadata buffer fill state",
texture_cache.IsMetaCleared(base + metadata_b, 0),
"BufferCache fill did not publish an exact-address metadata clear");
// R-Type Final 3 reuses an old color allocation inside a new stencil plane,
// with the same metadata address changing from DCC to HTile.
auto reused_depth = MakeMetadataDepth(0x12300, 0x28000);
reused_depth.info.pixel_format = vk::Format::eD32SfloatS8Uint;
reused_depth.info.stencil = {base + 0x12200, 0x80};
reused_depth.info.htile_clear_mask = 0;
reused_depth.view_info.format = vk::Format::eD32SfloatS8Uint;
reused_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth |
vk::ImageAspectFlagBits::eStencil;
auto reused_color = MakeLinearDesc(
base + 0x12240, sizeof(uint32_t), vk::Format::eR8G8B8A8Unorm,
Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
reused_color.type = BindingType::RenderTarget;
reused_color.info.metadata.kind = ImageMetadataKind::Dcc;
reused_color.info.metadata.range = {base + 0x28000, 0x1000};
reused_color.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto reused_color_id = texture_cache.FindImage(reused_color);
(void)texture_cache.FindRenderTarget(reused_color_id, reused_color);
// Repeated CPU stores must remain visible through ordinary buffer ownership.
for (uint32_t repeat = 0; repeat < 2; repeat++) {
vk::ClearValue painted{};
painted.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), reused_color_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
std::memset(memory + 0x28000, 0, 0x1000);
Require(name, "CPU metadata rediscovery",
texture_cache.FindImage(reused_color) == reused_color_id,
"metadata writes replaced the live expanded image");
(void)texture_cache.FindRenderTarget(reused_color_id, reused_color);
Require(name, "repeated CPU metadata clear",
ReadCachedTexel(name, context, reused_color_id) == std::vector<u32>{0},
"a same-value CPU metadata write lost its clear after rendering");
}
WriteMetadata(context, base + 0x28000, 0x1000, 0);
const auto reused_depth_id = texture_cache.FindImage(reused_depth);
(void)texture_cache.FindDepthTarget(reused_depth_id, reused_depth);
Require(
name, "DCC allocation reused as HTile",
TextureCacheTestAccess::Contains(texture_cache, reused_color_id) &&
!texture_cache.IsMetaCleared(base + 0x28000, 0) &&
texture_cache.ClearMeta(base + 0x28000) &&
texture_cache.TouchMeta(base + 0x28000, 0, false),
"depth binding retained incompatible DCC clear state");
texture_cache.UnmapMemory(reused_color.info.data.address,
reused_color.info.data.size);
(void)texture_cache.FindDepthTarget(reused_depth_id, reused_depth);
Require(
name, "reused metadata owner retirement",
!TextureCacheTestAccess::Contains(texture_cache, reused_color_id) &&
texture_cache.IsMeta(base + 0x28000) &&
!texture_cache.IsMetaCleared(base + 0x28000, 0) &&
texture_cache.IsMetaCleared(base + 0x28000, 1),
"retiring the old DCC image erased the live HTile slice state");
texture_cache.UnmapMemory(reused_depth.info.data.address,
reused_depth.info.data.size);
Require(name, "reused metadata final retirement",
!texture_cache.IsMeta(base + 0x28000),
"retiring the HTile image left its metadata registered");
constexpr uint64_t partial_unmap_image_offset = 0x2700000;
auto partial_unmap_image =
MakeLinearDesc(base + partial_unmap_image_offset, 0x2000,
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {2048, 1, 1}, 1, 4, 1);
const auto partial_unmap_image_id =
texture_cache.FindImage(partial_unmap_image);
const auto image_address = partial_unmap_image.info.data.address;
Require(name, "registered image boundaries",
texture_cache.IsRegionRegistered(image_address, 1) &&
texture_cache.IsRegionRegistered(image_address + 0x1fff, 1) &&
!texture_cache.IsRegionRegistered(image_address - 1, 1) &&
!texture_cache.IsRegionRegistered(image_address + 0x2000, 1) &&
texture_cache.IsRegionRegistered(image_address - 1, 0x2002),
"image lookup mishandled a partial overlap or adjacent range");
scheduler.Finish();
scheduler.DrainPriorityOperations();
const auto image_unmap_tick = scheduler.CurrentTick();
resources.UnmapMemory(image_address + 0x1000, 0x1000);
Require(name, "partial image unmap tracking",
partial_unmap_image_id &&
scheduler.CurrentTick() == image_unmap_tick + 1 &&
!texture_cache.IsRegionRegistered(image_address, 0x2000) &&
!texture_cache.FindImageFromRange(image_address, 0x2000, false),
"partial image unmap skipped its drain or retained the image");
constexpr uint64_t unformatted_alias_offset = 0x2500000;
auto unformatted_alias =
MakeLinearDesc(base + unformatted_alias_offset, sizeof(uint32_t),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto unformatted_alias_image =
texture_cache.FindImage(unformatted_alias);
(void)texture_cache.FindTexture(unformatted_alias_image,
unformatted_alias);
texture_cache.MarkGpuWritten(unformatted_alias_image);
auto unformatted_alias_buffer = resources.GetBufferCache().ObtainBuffer(
unformatted_alias.info.data.address, unformatted_alias.info.data.size,
false);
Require(
name, "unformatted buffer/image alias",
unformatted_alias_buffer.first != nullptr &&
texture_cache.GetImage(unformatted_alias_image).IsGpuModified() &&
!texture_cache.GetImage(unformatted_alias_image)
.IsBufferModified(),
"read-only unformatted buffer alias did not follow ordinary "
"BufferCache acquisition");
constexpr uint64_t layered_offset = 0x14000;
constexpr uint64_t layered_guest_size = 0x800;
const std::array<float, 10> layered_values{0.0f, 0.125f, 0.25f, 0.375f,
0.5f, 0.625f, 0.75f, 0.875f,
1.0f, 0.0625f};
std::memset(memory + layered_offset, 0, layered_guest_size);
const auto layered_layout = TextureCalcUploadLayout(
Prospero::BufferFormat::k32Float, 2, 2, 2, 2,
Prospero::TileMode::kLinear, layered_guest_size, false,
"UnifiedTextureCacheFlow");
const auto layered_upload_regions =
TextureBuildImageCopies(layered_layout);
Require(name, "layered guest layout", layered_upload_regions.size() == 4,
"unexpected layered/mipped upload-region count");
for (const auto &region : layered_upload_regions) {
for (uint32_t y = 0; y < region.imageExtent.height; y++) {
for (uint32_t x = 0; x < region.imageExtent.width; x++) {
const uint32_t logical =
region.imageSubresource.mipLevel == 0
? region.imageSubresource.baseArrayLayer * 4 + y * 2 + x
: 8 + region.imageSubresource.baseArrayLayer;
const uint64_t byte_offset =
region.bufferOffset +
(static_cast<uint64_t>(y) * region.bufferRowLength + x) *
sizeof(float);
Require(name, "layered guest layout bounds",
byte_offset + sizeof(float) <= layered_guest_size,
"layered guest texel lies outside its backing range");
std::memcpy(memory + layered_offset + byte_offset,
&layered_values[logical], sizeof(float));
}
}
}
auto layered_color = MakeLinearDesc(
base + layered_offset, layered_guest_size, vk::Format::eR32Sfloat,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{2, 2, 1}, 2, 4, 1);
layered_color.info.resources.levels = 2;
layered_color.info.mip_layout[0] = {0, 32, 2, 2};
layered_color.info.mip_layout[1] = {32, 8, 1, 1};
layered_color.view_info.level_count = 2;
const auto layered_color_image = texture_cache.FindImage(layered_color);
(void)texture_cache.FindTexture(layered_color_image, layered_color);
texture_cache.MarkGpuWritten(layered_color_image);
auto layered_depth = layered_color;
layered_depth.type = BindingType::DepthTarget;
layered_depth.info.resources = {1, 4};
layered_depth.info.mip_layout = {};
layered_depth.info.mip_layout[0] = {0, layered_guest_size, 2, 2};
layered_depth.view_info.level_count = 1;
layered_depth.view_info.layer_count = 4;
layered_depth.info.pixel_format = vk::Format::eD32Sfloat;
layered_depth.view_info.format = vk::Format::eD32Sfloat;
layered_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
layered_depth.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto layered_depth_image = texture_cache.FindImage(layered_depth);
const auto &layered_native = texture_cache.GetImage(layered_depth_image);
Require(name, "layered mipped depth alias",
layered_depth_image != layered_color_image &&
layered_native.backing.layers == 4 &&
layered_native.backing.mip_levels == 1 &&
layered_native.info.resources ==
layered_depth.info.resources &&
layered_native.info.mip_layout == layered_depth.info.mip_layout &&
layered_native.IsGpuModified(),
"depth/color conversion mixed the source mip count with the requested layout");
const volatile auto layered_guest_byte =
*reinterpret_cast<const volatile uint8_t *>(memory + layered_offset);
(void)layered_guest_byte;
Require(name, "layered depth write-only read policy",
texture_cache.GetImage(layered_depth_image).IsGpuModified(),
"a CPU read invalidated layered depth GPU ownership");
constexpr uint64_t replacement_offset = 0x18000;
constexpr uint64_t replacement_size = 0x2000;
constexpr uint32_t replacement_pixel = 0xcafebabeu;
constexpr uint32_t replacement_tail = 0x0badf00du;
std::memset(memory + replacement_offset, 0, replacement_size);
std::memcpy(memory + replacement_offset, &replacement_pixel,
sizeof(replacement_pixel));
std::memcpy(memory + replacement_offset + replacement_size -
sizeof(replacement_tail),
&replacement_tail, sizeof(replacement_tail));
auto replacement_source =
MakeLinearDesc(base + replacement_offset, sizeof(uint32_t),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto replacement_source_image =
texture_cache.FindImage(replacement_source);
(void)texture_cache.FindTexture(replacement_source_image,
replacement_source);
texture_cache.MarkGpuWritten(replacement_source_image);
auto replacement_desc =
MakeLinearDesc(base + replacement_offset, replacement_size,
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1);
const auto replacement = texture_cache.FindImage(replacement_desc);
const bool replacement_changed = replacement != replacement_source_image;
const bool replacement_gpu_before =
texture_cache.GetImage(replacement).IsGpuModified();
const bool replacement_fault = resources.HandleFault(
PageFaultAccess::Write, base + replacement_offset + 0x1000);
const bool replacement_gpu_after =
texture_cache.GetImage(replacement).IsGpuModified();
uint32_t replacement_pixel_after = 0;
uint32_t replacement_tail_after = 0;
std::memcpy(&replacement_pixel_after, memory + replacement_offset,
sizeof(replacement_pixel_after));
std::memcpy(&replacement_tail_after,
memory + replacement_offset + replacement_size -
sizeof(replacement_tail_after),
sizeof(replacement_tail_after));
Require(name, "replacement tracking",
replacement_changed && replacement_gpu_before &&
replacement_fault && replacement_gpu_after &&
texture_cache.GetImage(replacement).IsDefinitelyCpuDirty() &&
replacement_pixel_after == replacement_pixel &&
replacement_tail_after == replacement_tail,
"replacement write invalidation lost native authority or guest "
"backing");
const uint32_t compressed_value = 0x55667788u;
std::memcpy(memory + 0xc000, &compressed_value, sizeof(compressed_value));
auto compressed_desc = MakeLinearDesc(
base + 0xc000, sizeof(compressed_value), vk::Format::eR8G8B8A8Srgb,
Prospero::BufferFormat::k8_8_8_8Srgb, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
compressed_desc.type = BindingType::RenderTarget;
compressed_desc.info.metadata.kind = ImageMetadataKind::Dcc;
compressed_desc.info.metadata.range = {base + 0xd000, 0};
compressed_desc.info.metadata.compression =
VideoOutCompression::Dcc256_256_0;
compressed_desc.view_info.usage =
vk::ImageUsageFlagBits::eColorAttachment;
const auto compressed_image = texture_cache.FindImage(compressed_desc);
(void)texture_cache.FindRenderTarget(compressed_image, compressed_desc);
uint32_t compressed_cpu_read = 0;
std::memcpy(&compressed_cpu_read, memory + 0xc000,
sizeof(compressed_cpu_read));
Require(name, "compressed write-only read policy",
compressed_cpu_read == compressed_value &&
texture_cache.GetImage(compressed_image).IsGpuModified(),
"compressed image incorrectly claimed a CPU read fault");
Require(name, "compressed download rejection",
!TextureCacheTestAccess::TryDownload(texture_cache,
compressed_image) &&
texture_cache.GetImage(compressed_image).IsGpuModified() &&
!texture_cache.GetImage(compressed_image).IsBufferModified(),
"a compressed image escaped the unified download guard");
auto compressed_video_desc = compressed_desc;
compressed_video_desc.type = BindingType::VideoOut;
compressed_video_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto compressed_video =
texture_cache.FindImage(compressed_video_desc);
const auto &compressed_video_image =
texture_cache.GetImage(compressed_video);
Require(
name, "cache-only video-out discovery",
compressed_video == compressed_image &&
!compressed_video_image.binding.is_bound &&
!compressed_video_image.binding.is_target &&
!compressed_video_image.binding.needs_rebind &&
!compressed_video_image.usage.video_out,
"FindImage claimed caller-owned video-out usage or binding state");
constexpr uint64_t mixed_source_offset = 0x20000;
constexpr uint64_t mixed_source_size = 0x1004;
constexpr uint32_t mixed_source_width = 1025;
constexpr uint32_t mixed_cpu_value = 0x1234abcdu;
constexpr uint32_t mixed_gpu_value = 0x9876fedcu;
auto mixed_owner = resources.GetBufferCache().ObtainBuffer(
base + mixed_source_offset, mixed_source_size, true);
Require(name, "mixed-page source allocation",
mixed_owner.first != nullptr,
"mixed CPU/GPU image source did not create a containing buffer");
Require(name, "mixed-page CPU write fault",
resources.HandleFault(PageFaultAccess::Write,
base + mixed_source_offset),
"mixed image source could not dirty its first page");
std::memcpy(memory + mixed_source_offset, &mixed_cpu_value,
sizeof(mixed_cpu_value));
resources.GetBufferCache().FillBuffer(base + mixed_source_offset + 0x1000,
sizeof(mixed_gpu_value),
mixed_gpu_value, false);
auto mixed_source_desc = MakeLinearDesc(
base + mixed_source_offset, mixed_source_size, vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{mixed_source_width, 1, 1}, 1, 4, 1);
const auto mixed_source_image =
texture_cache.FindImage(mixed_source_desc);
(void)texture_cache.FindTexture(mixed_source_image, mixed_source_desc);
Require(name, "mixed-page image upload",
!texture_cache.GetImage(mixed_source_image).IsGpuModified(),
"clean Buffer source manufactured GPU image ownership");
auto mixed_source_readback = CreateHostBuffer(
name, mixed_source_size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(mixed_source_size / sizeof(u32), 0));
auto &mixed_source_native = texture_cache.GetImage(mixed_source_image);
mixed_source_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
command.Handle());
vk::BufferImageCopy mixed_source_copy{};
mixed_source_copy.bufferRowLength = mixed_source_width;
mixed_source_copy.imageSubresource.aspectMask =
vk::ImageAspectFlagBits::eColor;
mixed_source_copy.imageSubresource.layerCount = 1;
mixed_source_copy.imageExtent = {mixed_source_width, 1, 1};
command.Handle().copyImageToBuffer(mixed_source_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal,
mixed_source_readback.buffer, 1,
&mixed_source_copy);
HostReadBarrier(mixed_source_readback.buffer, mixed_source_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
constexpr uint64_t bridged_gpu_offset = 0x101000;
constexpr uint64_t bridged_cpu_offset = 0x105000;
constexpr uint64_t bridged_texel_size = 16;
constexpr uint64_t bridged_source_size =
bridged_cpu_offset - bridged_gpu_offset + bridged_texel_size;
constexpr uint32_t bridged_source_width =
bridged_source_size / bridged_texel_size;
constexpr uint32_t bridged_gpu_value = 0x10293847u;
constexpr uint32_t bridged_cpu_value = 0x56473829u;
constexpr uint32_t bridged_gpu_guest_value = 0xa5a5a5a5u;
std::array<uint32_t, bridged_texel_size / sizeof(uint32_t)>
bridged_cpu_values{};
bridged_cpu_values.fill(bridged_cpu_value);
std::memcpy(memory + bridged_cpu_offset, bridged_cpu_values.data(),
sizeof(bridged_cpu_values));
auto bridged_cpu_owner = resources.GetBufferCache().ObtainBuffer(
base + bridged_cpu_offset, bridged_texel_size, true);
Require(name, "bridged CPU owner allocation",
bridged_cpu_owner.first != nullptr,
"bridged image source did not create its CPU-side cache owner");
resources.GetBufferCache().FillBuffer(base + bridged_cpu_offset,
bridged_texel_size, 0, false);
Require(
name, "bridged CPU write fault",
resources.HandleFault(PageFaultAccess::Write,
base + bridged_cpu_offset),
"bridged image source could not transfer its second owner to CPU");
std::memcpy(memory + bridged_cpu_offset, bridged_cpu_values.data(),
sizeof(bridged_cpu_values));
std::array<uint32_t, bridged_texel_size / sizeof(uint32_t)>
bridged_gpu_guest_values{};
bridged_gpu_guest_values.fill(bridged_gpu_guest_value);
std::memcpy(memory + bridged_gpu_offset, bridged_gpu_guest_values.data(),
sizeof(bridged_gpu_guest_values));
auto bridged_gpu_owner = resources.GetBufferCache().ObtainBuffer(
base + bridged_gpu_offset, bridged_texel_size, true);
Require(name, "bridged GPU owner allocation",
bridged_gpu_owner.first != nullptr &&
bridged_gpu_owner.first != bridged_cpu_owner.first,
"bridged image source did not retain two disjoint cache owners");
resources.GetBufferCache().FillBuffer(base + bridged_gpu_offset,
bridged_texel_size,
bridged_gpu_value, false);
Require(name, "bridged ownership split",
resources.GetBufferCache().HasGpuDirtyBytes(
base + bridged_gpu_offset, bridged_texel_size) &&
!resources.GetBufferCache().HasGpuDirtyBytes(
base + bridged_cpu_offset, bridged_texel_size) &&
resources.GetBufferCache().IsRegionCpuModified(
base + bridged_cpu_offset, bridged_texel_size),
"bridged image source did not retain distinct GPU and CPU "
"ownership domains");
auto bridged_source_desc = MakeLinearDesc(
base + bridged_gpu_offset, bridged_source_size,
vk::Format::eR32G32B32A32Uint,
Prospero::BufferFormat::k32_32_32_32UInt,
Prospero::ImageType::kColor2D, {bridged_source_width, 1, 1}, 1,
bridged_texel_size, 1);
const auto bridged_source_image =
texture_cache.FindImage(bridged_source_desc);
(void)texture_cache.FindTexture(bridged_source_image,
bridged_source_desc);
auto bridged_source_readback = CreateHostBuffer(
name, bridged_source_size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(bridged_source_size / sizeof(u32), 0));
auto &bridged_source_native =
texture_cache.GetImage(bridged_source_image);
bridged_source_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
scheduler.Current().Handle());
vk::BufferImageCopy bridged_source_copy{};
bridged_source_copy.bufferRowLength = bridged_source_width;
bridged_source_copy.imageSubresource.aspectMask =
vk::ImageAspectFlagBits::eColor;
bridged_source_copy.imageSubresource.layerCount = 1;
bridged_source_copy.imageExtent = {bridged_source_width, 1, 1};
scheduler.Current().Handle().copyImageToBuffer(
bridged_source_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal, bridged_source_readback.buffer,
1, &bridged_source_copy);
HostReadBarrier(bridged_source_readback.buffer,
bridged_source_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
constexpr uint64_t byte_mirror_page_offset = 0x24000;
constexpr uint64_t byte_mirror_offset = byte_mirror_page_offset + 1;
const std::array<uint8_t, 4> byte_mirror_guest{0xacu, 0x5au, 0xbdu,
0xceu};
std::memcpy(memory + byte_mirror_page_offset, byte_mirror_guest.data(),
byte_mirror_guest.size());
auto byte_mirror_desc =
MakeLinearDesc(base + byte_mirror_offset, 1, vk::Format::eR8Unorm,
Prospero::BufferFormat::k8UNorm,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 1, 1);
const auto byte_mirror_image = texture_cache.FindImage(byte_mirror_desc);
(void)texture_cache.FindTexture(byte_mirror_image, byte_mirror_desc);
texture_cache.MarkGpuWritten(byte_mirror_image);
auto byte_mirror = resources.GetBufferCache().ObtainBuffer(
base + byte_mirror_offset, 1, false, true);
Require(
name, "byte image mirror",
byte_mirror.first != nullptr &&
!texture_cache.GetImage(byte_mirror_image).IsBufferModified() &&
texture_cache.GetImage(byte_mirror_image).IsGpuModified(),
"one-byte image copy transferred cache ownership");
const std::array<uint16_t, 4> bgra16_guest{0x3c00u, 0x4000u, 0x4200u,
0x4400u};
std::memcpy(memory + 0xb000, bgra16_guest.data(), sizeof(bgra16_guest));
auto bgra16_desc = MakeLinearDesc(
base + 0xb000, sizeof(bgra16_guest), vk::Format::eR16G16B16A16Sfloat,
Prospero::BufferFormat::k16_16_16_16Float,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1);
bgra16_desc.type = BindingType::RenderTarget;
bgra16_desc.info.bgra16 = true;
bgra16_desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto bgra16_image = texture_cache.FindImage(bgra16_desc);
(void)texture_cache.FindRenderTarget(bgra16_image, bgra16_desc);
auto bgra16_readback =
CreateHostBuffer(name, sizeof(bgra16_guest),
vk::BufferUsageFlagBits::eTransferDst, {0, 0});
auto &bgra16_native = texture_cache.GetImage(bgra16_image);
bgra16_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
command.Handle());
vk::BufferImageCopy bgra16_copy{};
bgra16_copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
bgra16_copy.imageSubresource.layerCount = 1;
bgra16_copy.imageExtent = {1, 1, 1};
command.Handle().copyImageToBuffer(
bgra16_native.backing.image, vk::ImageLayout::eTransferSrcOptimal,
bgra16_readback.buffer, 1, &bgra16_copy);
HostReadBarrier(bgra16_readback.buffer, bgra16_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
auto layered_readback = CreateHostBuffer(
name, 8 * sizeof(float), vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(8, 0));
auto &layered_depth_native = texture_cache.GetImage(layered_depth_image);
layered_depth_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
command.Handle());
vk::BufferImageCopy layered_copy{};
layered_copy.imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0, 2};
layered_copy.imageExtent = {2, 2, 1};
command.Handle().copyImageToBuffer(
layered_depth_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal, layered_readback.buffer,
1, &layered_copy);
HostReadBarrier(layered_readback.buffer, layered_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
vk::ShaderModuleCreateInfo module_info{};
module_info.sType = vk::StructureType::eShaderModuleCreateInfo;
module_info.codeSize = sizeof(GPU_TEST_MS_DEPTH_SPV);
module_info.pCode = GPU_TEST_MS_DEPTH_SPV;
vk::ShaderModule ms_depth_module = nullptr;
RequireVk(
name, "MS depth content",
m_device.createShaderModule(&module_info, nullptr, &ms_depth_module),
"vkCreateShaderModule");
std::array<vk::DescriptorSetLayoutBinding, 2> observer_bindings{};
observer_bindings[0].binding = 0;
observer_bindings[0].descriptorType = vk::DescriptorType::eSampledImage;
observer_bindings[0].descriptorCount = 1;
observer_bindings[0].stageFlags = vk::ShaderStageFlagBits::eCompute;
observer_bindings[1].binding = 1;
observer_bindings[1].descriptorType = vk::DescriptorType::eStorageBuffer;
observer_bindings[1].descriptorCount = 1;
observer_bindings[1].stageFlags = vk::ShaderStageFlagBits::eCompute;
vk::DescriptorSetLayoutCreateInfo observer_layout_info{};
observer_layout_info.sType =
vk::StructureType::eDescriptorSetLayoutCreateInfo;
observer_layout_info.bindingCount =
static_cast<uint32_t>(observer_bindings.size());
observer_layout_info.pBindings = observer_bindings.data();
vk::DescriptorSetLayout observer_set_layout = nullptr;
RequireVk(name, "MS depth content",
m_device.createDescriptorSetLayout(
&observer_layout_info, nullptr, &observer_set_layout),
"vkCreateDescriptorSetLayout");
const vk::PushConstantRange observer_push{
vk::ShaderStageFlagBits::eCompute, 0, sizeof(uint32_t)};
vk::PipelineLayoutCreateInfo observer_pipeline_layout_info{};
observer_pipeline_layout_info.sType =
vk::StructureType::ePipelineLayoutCreateInfo;
observer_pipeline_layout_info.setLayoutCount = 1;
observer_pipeline_layout_info.pSetLayouts = &observer_set_layout;
observer_pipeline_layout_info.pushConstantRangeCount = 1;
observer_pipeline_layout_info.pPushConstantRanges = &observer_push;
vk::PipelineLayout observer_pipeline_layout = nullptr;
RequireVk(name, "MS depth content",
m_device.createPipelineLayout(&observer_pipeline_layout_info,
nullptr,
&observer_pipeline_layout),
"vkCreatePipelineLayout");
vk::PipelineShaderStageCreateInfo observer_stage{};
observer_stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
observer_stage.stage = vk::ShaderStageFlagBits::eCompute;
observer_stage.module = ms_depth_module;
observer_stage.pName = "main";
vk::ComputePipelineCreateInfo observer_pipeline_info{};
observer_pipeline_info.sType =
vk::StructureType::eComputePipelineCreateInfo;
observer_pipeline_info.stage = observer_stage;
observer_pipeline_info.layout = observer_pipeline_layout;
vk::Pipeline observer_depth_pipeline = nullptr;
RequireVk(
name, "MS depth content",
m_device.createComputePipelines(nullptr, 1, &observer_pipeline_info,
nullptr, &observer_depth_pipeline),
"vkCreateComputePipelines");
const std::array<vk::DescriptorPoolSize, 2> observer_pool_sizes{{
{vk::DescriptorType::eSampledImage, 2},
{vk::DescriptorType::eStorageBuffer, 2},
}};
vk::DescriptorPoolCreateInfo observer_pool_info{};
observer_pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo;
observer_pool_info.maxSets = 2;
observer_pool_info.poolSizeCount =
static_cast<uint32_t>(observer_pool_sizes.size());
observer_pool_info.pPoolSizes = observer_pool_sizes.data();
vk::DescriptorPool observer_pool = nullptr;
RequireVk(name, "MS depth content",
m_device.createDescriptorPool(&observer_pool_info, nullptr,
&observer_pool),
"vkCreateDescriptorPool");
const std::array<vk::DescriptorSetLayout, 2> observer_layouts{
observer_set_layout, observer_set_layout};
std::array<vk::DescriptorSet, 2> observer_sets{};
vk::DescriptorSetAllocateInfo observer_allocate{};
observer_allocate.sType = vk::StructureType::eDescriptorSetAllocateInfo;
observer_allocate.descriptorPool = observer_pool;
observer_allocate.descriptorSetCount =
static_cast<uint32_t>(observer_sets.size());
observer_allocate.pSetLayouts = observer_layouts.data();
RequireVk(name, "MS depth content",
m_device.allocateDescriptorSets(&observer_allocate,
observer_sets.data()),
"vkAllocateDescriptorSets");
std::array<Buffer, 2> ms_observer_outputs{
CreateHostBuffer(name, 4 * sizeof(u32),
vk::BufferUsageFlagBits::eStorageBuffer,
std::vector<u32>(4, 0)),
CreateHostBuffer(name, 4 * sizeof(u32),
vk::BufferUsageFlagBits::eStorageBuffer,
std::vector<u32>(4, 0))};
const std::array<ImageId, 2> observed_images{ms_depth_image,
ms_depth_image_4x};
const std::array<uint32_t, 2> observed_samples{2, 4};
for (uint32_t index = 0; index < observed_images.size(); index++) {
auto &observed = texture_cache.GetImage(observed_images[index]);
observed.Transit(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, {},
command.Handle());
ImageViewInfo view_info{};
view_info.format = observed.backing.format;
view_info.type = vk::ImageViewType::e2D;
view_info.aspect = vk::ImageAspectFlagBits::eDepth;
view_info.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eR};
view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto view = observed.FindView(view_info);
const vk::DescriptorImageInfo image_info{
nullptr, view, vk::ImageLayout::eShaderReadOnlyOptimal};
const vk::DescriptorBufferInfo buffer_info{
ms_observer_outputs[index].buffer, 0,
ms_observer_outputs[index].size};
std::array<vk::WriteDescriptorSet, 2> writes{};
writes[0].sType = vk::StructureType::eWriteDescriptorSet;
writes[0].dstSet = observer_sets[index];
writes[0].dstBinding = 0;
writes[0].descriptorCount = 1;
writes[0].descriptorType = vk::DescriptorType::eSampledImage;
writes[0].pImageInfo = &image_info;
writes[1].sType = vk::StructureType::eWriteDescriptorSet;
writes[1].dstSet = observer_sets[index];
writes[1].dstBinding = 1;
writes[1].descriptorCount = 1;
writes[1].descriptorType = vk::DescriptorType::eStorageBuffer;
writes[1].pBufferInfo = &buffer_info;
m_device.updateDescriptorSets(static_cast<uint32_t>(writes.size()),
writes.data(), 0, nullptr);
command.Handle().bindPipeline(vk::PipelineBindPoint::eCompute,
observer_depth_pipeline);
command.Handle().bindDescriptorSets(vk::PipelineBindPoint::eCompute,
observer_pipeline_layout, 0, 1,
&observer_sets[index], 0, nullptr);
command.Handle().pushConstants(
observer_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0,
sizeof(observed_samples[index]), &observed_samples[index]);
command.Handle().dispatch(1, 1, 1);
HostReadBarrier(ms_observer_outputs[index].buffer,
ms_observer_outputs[index].size,
vk::PipelineStageFlagBits::eComputeShader,
vk::AccessFlagBits::eShaderWrite);
}
scheduler.Finish();
const auto mip_prefix_words = ReadBuffer(
name, mip_prefix_readback, mip_prefix_size / sizeof(uint32_t));
const auto volume_words =
ReadBuffer(name, volume_readback, volume_guest.size());
Require(name, "formatted volume content",
volume_words[0] == volume_values[0] &&
volume_words[volume_slice_words] == volume_values[1],
"volume synchronization lost a padded depth slice");
DestroyBuffer(&volume_readback);
const auto mip_formatted_words = ReadBuffer(
name, mip_formatted_readback, mip_guest_size / sizeof(uint32_t));
const auto mip0_word = mip_sizes[0].offset / sizeof(uint32_t);
Require(name, "formatted mip prefix content",
mip0_word < mip_prefix_words.size() &&
mip0_word < mip_native.size() &&
mip_prefix_words[mip0_word] == mip_native[mip0_word],
"fitting mip-prefix synchronization copied "
"stale guest backing");
Require(name, "formatted Buffer content",
mip0_word < mip_formatted_words.size() &&
mip0_word < mip_native.size() &&
mip_formatted_words[mip0_word] == mip_native[mip0_word],
"formatted Buffer read bypassed authoritative "
"native image mip data");
Require(name, "CPU-dirty formatted mirror content",
ReadBuffer(name, mirror_cpu_readback, 1) ==
std::vector<u32>{mirror_cpu_value},
"formatted Buffer mirror published stale "
"native image bytes");
Require(name, "Buffer-superseded exact content",
ReadBuffer(name, exact_buffer_readback, 1) ==
std::vector<u32>{exact_buffer_value},
"exact-format recreation initialized from "
"stale guest bytes");
Require(name, "partial-page CPU refresh content",
ReadBuffer(name, partial_cpu_refresh_readback,
partial_cpu_refresh_expected.size()) ==
partial_cpu_refresh_expected,
"cached buffer uploaded bytes outside the exact "
"staged guest range");
const auto mixed_source_words = ReadBuffer(
name, mixed_source_readback, mixed_source_size / sizeof(u32));
Require(name, "mixed-page image content",
mixed_source_words.front() == mixed_cpu_value &&
mixed_source_words[0x1000 / sizeof(u32)] == mixed_gpu_value,
"mixed CPU/GPU source upload lost one "
"ownership domain");
const auto bridged_source_words = ReadBuffer(
name, bridged_source_readback, bridged_source_size / sizeof(u32));
Require(name, "bridged-owner image content",
bridged_source_words.front() == bridged_gpu_value &&
bridged_source_words.back() == bridged_cpu_value,
"image upload bridging separate cache owners lost a GPU or CPU "
"ownership domain");
Require(name, "BGRA16 content",
ReadBuffer(name, bgra16_readback, 2) ==
std::vector<u32>{0x40004200u, 0x44003c00u},
"GPU BGRA16 swap did not publish RGBA half-word order");
scheduler.Finish();
const auto layered_words = ReadBuffer(name, layered_readback, 8);
const std::array<float, 8> layered_expected{
layered_values[0], layered_values[1], layered_values[2],
layered_values[3], layered_values[4], layered_values[5],
layered_values[6], layered_values[7]};
bool layered_content = layered_words.size() == layered_expected.size();
for (uint32_t index = 0;
layered_content && index < layered_expected.size(); index++) {
layered_content &=
layered_words[index] == std::bit_cast<u32>(layered_expected[index]);
}
Require(name, "layered depth common-mip content", layered_content,
"depth/color conversion changed a shared array-layer value");
const std::array<std::array<float, 4>, 2> expected_ms_depth{{
{0x2000 / 65535.0f, 0xe000 / 65535.0f, 0.0f, 0.0f},
{0.0f, 0x4000 / 65535.0f, 0x8000 / 65535.0f, 1.0f},
}};
for (uint32_t image = 0; image < expected_ms_depth.size(); image++) {
const auto words = ReadBuffer(name, ms_observer_outputs[image], 4);
bool content = words.size() == 4;
for (uint32_t sample = 0; content && sample < observed_samples[image];
sample++) {
const float actual = std::bit_cast<float>(words[sample]);
content &= std::abs(actual - expected_ms_depth[image][sample]) <=
1.5f / 65535.0f;
}
Require(name,
image == 0 ? "two-sample depth content"
: "four-sample depth content",
content,
"color-to-multisample-depth changed a packed sample value");
}
auto unrelated_stencil_alias = MakeLinearDesc(
base + second_stencil_offset, ms_stencil_size, vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{static_cast<uint32_t>(ms_stencil_size / sizeof(uint32_t)), 1, 1}, 1,
4, 1);
const auto unrelated_stencil_image =
texture_cache.FindImage(unrelated_stencil_alias);
Require(name, "stencil association filtering",
unrelated_stencil_image &&
unrelated_stencil_image != ms_depth_image &&
!texture_cache.GetImage(unrelated_stencil_image).depth_id,
"an unrelated color alias was hijacked by the stencil "
"association record");
texture_cache.UnmapMemory(base + second_stencil_offset, ms_stencil_size);
Require(name, "stencil proxy unmap",
!texture_cache.FindImageFromRange(base + second_stencil_offset,
ms_stencil_size, false) &&
texture_cache.GetImage(ms_depth_image).backing.image ==
ms_depth_backing,
"stencil unmap removed the depth backing or retained its proxy");
const auto reassociated_depth =
texture_cache.FindImage(switched_ms_depth);
TextureCacheTestAccess::AssociateStencil(
texture_cache, reassociated_depth, switched_ms_depth.info.stencil);
const auto reassociated_proxy = texture_cache.FindImageFromRange(
base + second_stencil_offset, ms_stencil_size, false);
Require(name, "stencil proxy reassociation",
reassociated_depth == ms_depth_image && reassociated_proxy &&
texture_cache.GetImage(reassociated_proxy).depth_id ==
ms_depth_image,
"stencil proxy did not re-associate with the surviving depth");
texture_cache.UnmapMemory(ms_depth_desc.info.data.address,
ms_depth_desc.info.data.size);
Require(name, "depth unmap proxy retirement",
!texture_cache.FindImageFromRange(base + second_stencil_offset,
ms_stencil_size, false),
"unmapping depth retained its re-associated stencil proxy");
auto reused_stencil_color = MakeLinearDesc(
base + 0x2200000, 0x40000, vk::Format::eR8Unorm,
Prospero::BufferFormat::k8UNorm, Prospero::ImageType::kColor2D,
{480, 408, 1}, 1, 1, 1);
const auto reused_stencil_color_id =
texture_cache.FindImage(reused_stencil_color);
auto reused_stencil_depth = MakeLinearDesc(
base + 0x2240000, 0x10000, vk::Format::eD32SfloatS8Uint,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{128, 128, 1}, 1, 4, 1);
reused_stencil_depth.type = BindingType::DepthTarget;
reused_stencil_depth.info.stencil = {reused_stencil_color.info.data.address,
0x10000};
reused_stencil_depth.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
reused_stencil_depth.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto reused_stencil_depth_id =
texture_cache.FindImage(reused_stencil_depth);
TextureCacheTestAccess::AssociateStencil(
texture_cache, reused_stencil_depth_id,
reused_stencil_depth.info.stencil);
const auto reused_stencil_proxy = texture_cache.FindImageFromRange(
reused_stencil_depth.info.stencil.address,
reused_stencil_depth.info.stencil.size, false);
Require(name, "stencil address reuse preserves color",
reused_stencil_proxy &&
reused_stencil_proxy != reused_stencil_color_id &&
!texture_cache.GetImage(reused_stencil_color_id).depth_id &&
texture_cache.GetImage(reused_stencil_proxy).depth_id ==
reused_stencil_depth_id &&
texture_cache.GetImage(reused_stencil_proxy).info.data ==
reused_stencil_depth.info.stencil &&
texture_cache.GetImage(reused_stencil_proxy).info.extent ==
reused_stencil_depth.info.extent,
"a smaller stencil plane took ownership of an existing R8 image");
vk::ClearValue reused_stencil_color_clear{};
reused_stencil_color_clear.color.float32 =
std::array{1.0f, 0.0f, 0.0f, 0.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), reused_stencil_color_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1},
reused_stencil_color_clear);
Require(name, "reused stencil plane clear",
texture_cache.ClearImageFromBuffer(
scheduler.Current(), reused_stencil_depth.info.stencil.address,
reused_stencil_depth.info.stencil.size, 0x5a5a5a5au) &&
texture_cache.GetImage(reused_stencil_depth_id).IsGpuModified(),
"the separate stencil association did not resolve to depth");
Require(name, "color clear after stencil address reuse",
ReadCachedTexel(name, context, reused_stencil_color_id, {},
{4, 1, 1}) == std::vector<u32>{0xffffffffu},
"stencil address reuse changed the native R8 clear contents");
texture_cache.UnmapMemory(reused_stencil_depth.info.data.address,
reused_stencil_depth.info.data.size);
texture_cache.UnmapMemory(reused_stencil_color.info.data.address,
reused_stencil_color.info.data.size);
constexpr uint64_t exact_image_offset = 0x334100;
constexpr uint64_t dirty_sibling_offset = 0x334200;
constexpr uint32_t dirty_sibling_value = 0xc001d00du;
auto dirty_sibling = resources.GetBufferCache().ObtainBuffer(
base + dirty_sibling_offset, sizeof(dirty_sibling_value), true);
Require(name, "same-page dirty sibling allocation",
dirty_sibling.first != nullptr,
"failed to create the disjoint same-page Buffer owner");
resources.GetBufferCache().FillBuffer(base + dirty_sibling_offset,
sizeof(dirty_sibling_value),
dirty_sibling_value, false);
auto exact_image_desc =
MakeLinearDesc(base + exact_image_offset, sizeof(uint32_t),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto exact_image = texture_cache.FindImage(exact_image_desc);
(void)texture_cache.FindTexture(exact_image, exact_image_desc);
texture_cache.MarkGpuWritten(exact_image);
Require(name, "same-page exact alias ownership",
resources.GetBufferCache().HasGpuDirtyBytes(
base + dirty_sibling_offset, sizeof(dirty_sibling_value)) &&
!resources.GetBufferCache().HasGpuDirtyBytes(
base + exact_image_offset, sizeof(uint32_t)) &&
texture_cache.GetImage(exact_image).IsGpuModified(),
"image ownership discarded or conflicted with disjoint dirty "
"Buffer bytes on the same tracker page");
auto exact_buffer = resources.GetBufferCache().ObtainBuffer(
base + exact_image_offset, sizeof(uint32_t), true);
Require(name, "overlapping buffer ownership allocation",
exact_buffer.first != nullptr,
"failed to create the overlapping dirty Buffer owner");
exact_buffer.first->Fill(exact_buffer.second, sizeof(uint32_t),
dirty_sibling_value);
texture_cache.InvalidateMemoryFromGPU(base + exact_image_offset,
sizeof(uint32_t));
Libs::Graphics::Buffer exact_download(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
base + exact_image_offset, AllFlags, sizeof(uint32_t));
Require(name, "overlapping buffer ownership rejection",
!texture_cache.FindImageFromRange(base + exact_image_offset,
sizeof(uint32_t)) &&
!TextureCacheTestAccess::TryDownload(texture_cache,
exact_image) &&
!BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), exact_download,
base + exact_image_offset, sizeof(uint32_t)),
"dirty Buffer bytes did not block the stale image source");
resources.GetBufferCache().ReadMemory(base + exact_image_offset,
sizeof(uint32_t));
resources.GetBufferCache().ReadMemory(base + dirty_sibling_offset,
sizeof(dirty_sibling_value));
constexpr std::array<uint64_t, 2> gc_image_offsets{0x330000, 0x332000};
constexpr std::array<uint32_t, 2> gc_image_values{0x76543210u,
0x89abcdefu};
constexpr std::array<uint32_t, 2> gc_stale_values{0x10293847u,
0x56473829u};
for (size_t index = 0; index < gc_image_offsets.size(); index++) {
std::memcpy(memory + gc_image_offsets[index], &gc_image_values[index],
sizeof(uint32_t));
}
auto gc_image_desc_a =
MakeLinearDesc(base + gc_image_offsets[0], sizeof(uint32_t),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
auto gc_image_desc_b = gc_image_desc_a;
gc_image_desc_b.info.data.address = base + gc_image_offsets[1];
auto clean_buffer_alias = resources.GetBufferCache().ObtainBuffer(
gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size,
true);
Require(name, "clean pre-image buffer alias",
clean_buffer_alias.first != nullptr,
"failed to create the clean cached Buffer alias");
resources.GetBufferCache().ReadMemory(gc_image_desc_a.info.data.address,
gc_image_desc_a.info.data.size);
const std::array gc_images{texture_cache.FindImage(gc_image_desc_a),
texture_cache.FindImage(gc_image_desc_b)};
Require(
name, "non-GPU image range validity",
!texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address,
gc_image_desc_a.info.data.size),
"FindImageFromRange accepted an image without current GPU "
"contents");
(void)texture_cache.FindTexture(gc_images[0], gc_image_desc_a);
(void)texture_cache.FindTexture(gc_images[1], gc_image_desc_b);
for (const auto image : gc_images) {
texture_cache.MarkGpuWritten(image);
}
Require(name, "GPU image range validity",
texture_cache.FindImageFromRange(
gc_image_desc_a.info.data.address,
gc_image_desc_a.info.data.size) == gc_images[0],
"FindImageFromRange rejected a clean GPU-current image");
auto &gc_native = texture_cache.GetImage(gc_images[0]);
texture_cache.InvalidateMemory(gc_image_desc_a.info.data.address,
gc_image_desc_a.info.data.size);
Require(
name, "CPU-dirty image range validity",
!texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address,
gc_image_desc_a.info.data.size) &&
!TextureCacheTestAccess::TryDownload(texture_cache, gc_images[0]) &&
gc_native.IsDefinitelyCpuDirty() && gc_native.IsGpuModified() &&
!gc_native.IsTracked(),
"CPU writes did not supersede native contents while retaining GPU history");
texture_cache.MarkGpuWritten(gc_images[0]);
Require(name, "GPU image reacquisition",
gc_native.SafeToDownload(),
"a new GPU write did not reclaim image authority");
for (size_t index = 0; index < gc_image_offsets.size(); index++) {
Libs::LibKernel::Memory::WriteBacking(base + gc_image_offsets[index],
&gc_stale_values[index],
sizeof(uint32_t));
}
TextureCacheTestAccess::ConfigureGarbageCollection(
texture_cache, gc_images, 17, UINT64_MAX);
texture_cache.RunGarbageCollector();
Require(name, "downloadable image pre-pressure retention",
std::ranges::all_of(gc_images,
[&](ImageId image) {
return TextureCacheTestAccess::Contains(
texture_cache, image);
}),
"GC retired safely downloadable GPU images before pressure");
const auto gc_batch_tick = scheduler.CurrentTick();
TextureCacheTestAccess::ConfigureGarbageCollection(texture_cache,
gc_images, 81, 0);
texture_cache.RunGarbageCollector();
std::array<uint32_t, 2> gc_before_completion{};
for (size_t index = 0; index < gc_image_offsets.size(); index++) {
Libs::LibKernel::Memory::TryReadBacking(base + gc_image_offsets[index],
&gc_before_completion[index],
sizeof(uint32_t));
}
Require(name, "batched image pressure retirement",
std::ranges::none_of(gc_images,
[&](ImageId image) {
return TextureCacheTestAccess::Contains(
texture_cache, image);
}) &&
scheduler.CurrentTick() == gc_batch_tick &&
gc_before_completion == gc_stale_values,
"GC submitted per image or published a readback before GPU "
"completion");
scheduler.Finish();
scheduler.DrainPriorityOperations();
auto refreshed_buffer_alias = resources.GetBufferCache().ObtainBuffer(
gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size,
false);
Require(name, "post-publication Buffer reacquire",
refreshed_buffer_alias.first != nullptr,
"Buffer lookup failed after the retired image publication");
std::array<uint32_t, 2> gc_after_completion{};
for (size_t index = 0; index < gc_image_offsets.size(); index++) {
Libs::LibKernel::Memory::TryReadBacking(base + gc_image_offsets[index],
&gc_after_completion[index],
sizeof(uint32_t));
}
Require(name, "batched image readback publication",
scheduler.CurrentTick() == gc_batch_tick + 1 &&
gc_after_completion == gc_image_values,
"one submission did not publish both deferred image readbacks");
Require(name, "refreshed post-image buffer alias",
refreshed_buffer_alias.first != nullptr,
"failed to reacquire the cached Buffer alias after image "
"publication");
auto alias_readback = CreateHostBuffer(
name, sizeof(uint32_t), vk::BufferUsageFlagBits::eTransferDst, {0});
const vk::BufferCopy alias_copy{refreshed_buffer_alias.second, 0,
sizeof(uint32_t)};
scheduler.Current().Handle().copyBuffer(
refreshed_buffer_alias.first->Handle(), alias_readback.buffer, 1,
&alias_copy);
HostReadBarrier(alias_readback.buffer, alias_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "post-image Buffer alias content",
ReadBuffer(name, alias_readback, 1) ==
std::vector<u32>{gc_image_values[0]},
"a clean cached Buffer alias survived image ownership with "
"stale native bytes");
DestroyBuffer(&alias_readback);
constexpr uint64_t submit_readback_offset = 0x336000;
constexpr uint32_t submit_readback_value = 0x13579bdfu;
constexpr uint32_t submit_readback_stale = 0x2468ace0u;
auto submit_readback_desc =
MakeLinearDesc(base + submit_readback_offset, sizeof(uint32_t),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
submit_readback_desc.type = BindingType::Storage;
submit_readback_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
TextureCacheTestAccess::SetLinearReadback(texture_cache, true);
const auto submit_readback_image =
texture_cache.FindImage(submit_readback_desc);
auto submit_buffer = resources.GetBufferCache().ObtainBuffer(
submit_readback_desc.info.data.address,
submit_readback_desc.info.data.size, true);
Require(name, "linear submit dirty buffer allocation",
submit_buffer.first != nullptr,
"failed to create the older GPU Buffer contents");
submit_buffer.first->Fill(submit_buffer.second, sizeof(uint32_t),
submit_readback_stale);
texture_cache.InvalidateMemoryFromGPU(
submit_readback_desc.info.data.address,
submit_readback_desc.info.data.size);
(void)texture_cache.FindTexture(submit_readback_image,
submit_readback_desc);
Require(name, "linear submit image clear",
texture_cache.ClearImageFromBuffer(
command, submit_readback_desc.info.data.address,
submit_readback_desc.info.data.size, submit_readback_value),
"failed to create GPU-current linear storage contents");
Require(name, "older buffer permits current image readback",
resources.GetBufferCache().HasGpuDirtyBytes(
submit_readback_desc.info.data.address,
submit_readback_desc.info.data.size) &&
texture_cache.FindImageFromRange(
submit_readback_desc.info.data.address,
submit_readback_desc.info.data.size) ==
submit_readback_image,
"historical GPU Buffer dirtiness rejected newer image contents");
TextureCacheTestAccess::TrackDownload(texture_cache,
submit_readback_image);
Libs::LibKernel::Memory::WriteBacking(
submit_readback_desc.info.data.address, &submit_readback_stale,
sizeof(submit_readback_stale));
const auto submit_readback_tick = scheduler.CurrentTick();
texture_cache.ProcessDownloadImages();
TextureCacheTestAccess::SetLinearReadback(texture_cache, false);
uint32_t submit_before_completion = 0;
Libs::LibKernel::Memory::TryReadBacking(
submit_readback_desc.info.data.address, &submit_before_completion,
sizeof(submit_before_completion));
Require(name, "linear submit deferred publication",
submit_before_completion == submit_readback_stale &&
scheduler.CurrentTick() == submit_readback_tick,
"submit-time linear readback published synchronously");
scheduler.Finish();
scheduler.DrainPriorityOperations();
uint32_t submit_after_completion = 0;
Libs::LibKernel::Memory::TryReadBacking(
submit_readback_desc.info.data.address, &submit_after_completion,
sizeof(submit_after_completion));
Require(name, "linear submit readback reuse",
submit_after_completion == submit_readback_value &&
TextureCacheTestAccess::Contains(texture_cache,
submit_readback_image) &&
texture_cache.GetImage(submit_readback_image).IsGpuModified(),
"submit-time readback did not publish or retained no reusable "
"GPU-current image");
constexpr uint64_t linear_depth_offset = 0x33a000;
constexpr uint64_t linear_stencil_offset = 0x33b000;
constexpr uint32_t linear_depth_width = 3;
constexpr uint32_t linear_depth_height = 2;
constexpr uint32_t linear_depth_pitch = 4;
constexpr uint32_t linear_depth_layers = 2;
constexpr uint32_t linear_depth_words =
linear_depth_pitch * linear_depth_height * linear_depth_layers;
constexpr uint32_t linear_depth_clear = 0x3f400000u;
std::array<uint32_t, linear_depth_words> linear_depth_guest{};
for (uint32_t index = 0; index < linear_depth_words; index++) {
linear_depth_guest[index] = 0x51000000u + index;
}
constexpr std::array<uint8_t, 8> linear_stencil_guest{
0x91, 0x82, 0x73, 0x64, 0x55, 0x46, 0x37, 0x28};
std::memcpy(memory + linear_depth_offset, linear_depth_guest.data(),
sizeof(linear_depth_guest));
std::memcpy(memory + linear_stencil_offset, linear_stencil_guest.data(),
sizeof(linear_stencil_guest));
auto linear_depth_desc = MakeLinearDesc(
base + linear_depth_offset, sizeof(linear_depth_guest),
vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k32Float,
Prospero::ImageType::kColor2D,
{linear_depth_width, linear_depth_height, 1}, linear_depth_layers, 4,
1);
linear_depth_desc.type = BindingType::DepthTarget;
linear_depth_desc.info.pitch = linear_depth_pitch;
linear_depth_desc.info.stencil = {base + linear_stencil_offset,
sizeof(linear_stencil_guest)};
linear_depth_desc.info.mip_layout[0] = {0, sizeof(linear_depth_guest),
linear_depth_pitch,
linear_depth_height};
linear_depth_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
linear_depth_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto linear_depth_image =
texture_cache.FindImage(linear_depth_desc);
Require(name, "linear depth native clear",
texture_cache.ClearImageFromBuffer(
command, linear_depth_desc.info.data.address,
linear_depth_desc.info.data.size, linear_depth_clear),
"failed to create GPU-current padded linear depth contents");
TextureCacheTestAccess::SetLinearReadback(texture_cache, true);
TextureCacheTestAccess::TrackDownload(texture_cache, linear_depth_image);
const auto linear_depth_tick = scheduler.CurrentTick();
texture_cache.ProcessDownloadImages();
TextureCacheTestAccess::SetLinearReadback(texture_cache, false);
std::array<uint32_t, linear_depth_words> linear_depth_before{};
std::memcpy(linear_depth_before.data(), memory + linear_depth_offset,
sizeof(linear_depth_before));
Require(name, "linear depth deferred publication",
linear_depth_before == linear_depth_guest &&
scheduler.CurrentTick() == linear_depth_tick,
"linear depth readback published before GPU completion");
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::array<uint32_t, linear_depth_words> linear_depth_after{};
std::array<uint8_t, linear_stencil_guest.size()> linear_stencil_after{};
std::memcpy(linear_depth_after.data(), memory + linear_depth_offset,
sizeof(linear_depth_after));
std::memcpy(linear_stencil_after.data(), memory + linear_stencil_offset,
sizeof(linear_stencil_after));
bool linear_depth_matches = true;
uint32_t linear_depth_mismatch = UINT32_MAX;
for (uint32_t layer = 0; layer < linear_depth_layers; layer++) {
for (uint32_t y = 0; y < linear_depth_height; y++) {
for (uint32_t x = 0; x < linear_depth_pitch; x++) {
const auto index =
layer * linear_depth_pitch * linear_depth_height +
y * linear_depth_pitch + x;
const auto expected = x < linear_depth_width
? linear_depth_clear
: linear_depth_guest[index];
if (linear_depth_after[index] != expected &&
linear_depth_mismatch == UINT32_MAX) {
linear_depth_mismatch = index;
}
linear_depth_matches &= linear_depth_after[index] == expected;
}
}
}
Require(name, "linear depth native contents",
linear_depth_matches &&
linear_stencil_after == linear_stencil_guest &&
TextureCacheTestAccess::Contains(texture_cache,
linear_depth_image) &&
texture_cache.GetImage(linear_depth_image).IsGpuModified(),
fmt::format(
"linear depth mismatch={} actual=0x{:08x} expected=0x{:08x} "
"stencil={} contains={} gpu={}",
linear_depth_mismatch,
linear_depth_mismatch == UINT32_MAX
? 0
: linear_depth_after[linear_depth_mismatch],
linear_depth_mismatch == UINT32_MAX
? 0
: (linear_depth_mismatch % linear_depth_pitch <
linear_depth_width
? linear_depth_clear
: linear_depth_guest[linear_depth_mismatch]),
linear_stencil_after == linear_stencil_guest,
TextureCacheTestAccess::Contains(texture_cache,
linear_depth_image),
texture_cache.GetImage(linear_depth_image).IsGpuModified())
.c_str());
constexpr uint64_t tiled_depth_offset = 0x380000;
constexpr uint32_t tiled_depth_width = 3;
constexpr uint32_t tiled_depth_height = 2;
constexpr uint32_t tiled_depth_pitch = 4;
constexpr uint32_t tiled_depth_layers = 2;
constexpr uint64_t tiled_depth_slice = 0x10000;
constexpr uint64_t tiled_depth_size =
tiled_depth_slice * tiled_depth_layers;
constexpr uint32_t tiled_depth_clear = 0x3f200000u;
constexpr uint32_t tiled_depth_stale = 0xdeadbeefu;
std::vector<uint32_t> tiled_depth_guest(
tiled_depth_size / sizeof(uint32_t), tiled_depth_stale);
std::memcpy(memory + tiled_depth_offset, tiled_depth_guest.data(),
tiled_depth_size);
auto tiled_depth_desc = MakeLinearDesc(
base + tiled_depth_offset, tiled_depth_size, vk::Format::eD32Sfloat,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{tiled_depth_width, tiled_depth_height, 1}, tiled_depth_layers, 4, 1);
tiled_depth_desc.type = BindingType::DepthTarget;
tiled_depth_desc.info.pitch = tiled_depth_pitch;
tiled_depth_desc.info.tile_mode = Prospero::TileMode::kDepth;
tiled_depth_desc.info.mip_layout[0] = {
0, tiled_depth_size, tiled_depth_pitch, tiled_depth_height};
tiled_depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
tiled_depth_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto tiled_depth_image = texture_cache.FindImage(tiled_depth_desc);
Require(name, "tiled depth native clear",
texture_cache.ClearImageFromBuffer(
command, tiled_depth_desc.info.data.address,
tiled_depth_desc.info.data.size, tiled_depth_clear),
"failed to create GPU-current layered tiled depth contents");
const auto [tiled_prefix, tiled_prefix_offset] =
TextureCacheTestAccess::MapDownload(texture_cache, 64, 64);
Require(name, "tiled depth download prefix",
tiled_prefix != nullptr && tiled_prefix_offset != 0,
"failed to force a nonzero reusable-download offset");
const auto tiled_depth_tick = scheduler.CurrentTick();
Require(
name, "tiled depth download queue",
TextureCacheTestAccess::TryDownload(texture_cache, tiled_depth_image),
"layered tiled depth readback was rejected");
std::vector<uint32_t> tiled_depth_before(tiled_depth_guest.size());
std::memcpy(tiled_depth_before.data(), memory + tiled_depth_offset,
tiled_depth_size);
Require(name, "tiled depth deferred publication",
tiled_depth_before == tiled_depth_guest &&
scheduler.CurrentTick() == tiled_depth_tick,
"tiled depth readback published before GPU completion");
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::vector<uint32_t> tiled_depth_after(tiled_depth_guest.size());
std::memcpy(tiled_depth_after.data(), memory + tiled_depth_offset,
tiled_depth_size);
const auto active_tiled_words =
tiled_depth_width * tiled_depth_height * tiled_depth_layers;
const auto clear_tiled_words =
std::ranges::count(tiled_depth_after, tiled_depth_clear);
const auto stale_tiled_words =
std::ranges::count(tiled_depth_after, tiled_depth_stale);
Require(
name, "tiled depth backing preservation",
clear_tiled_words == active_tiled_words &&
stale_tiled_words ==
tiled_depth_after.size() - active_tiled_words,
fmt::format("tiled depth readback changed clear={}/{} stale={}/{}",
clear_tiled_words, active_tiled_words, stale_tiled_words,
tiled_depth_after.size() - active_tiled_words)
.c_str());
TileBlockLayout depth_block{};
Require(name, "tiled depth block layout",
TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint32_t),
depth_block),
"failed to describe the tiled depth test surface");
std::array<GpuTileInfo, tiled_depth_layers> tiled_depth_tiles{};
for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) {
const uint64_t offset = tiled_depth_slice * layer;
tiled_depth_tiles[layer] = {depth_block.family,
depth_block.bytes_per_element,
offset,
tiled_depth_slice,
offset,
tiled_depth_slice,
0,
tiled_depth_width,
tiled_depth_height,
1,
tiled_depth_pitch};
tiled_depth_tiles[layer].surface_z = layer;
}
auto tiled_depth_input =
CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_depth_after);
auto tiled_depth_linear =
TextureCacheTestAccess::Tiler(texture_cache)
.Detile(tiled_depth_input.buffer, 0, tiled_depth_size,
tiled_depth_size, tiled_depth_tiles);
auto tiled_depth_output =
CreateHostBuffer(name, tiled_depth_size, AllFlags,
std::vector<uint32_t>(tiled_depth_guest.size(), 0));
const vk::BufferCopy tiled_depth_copy{tiled_depth_linear.offset, 0,
tiled_depth_size};
scheduler.Current().Handle().copyBuffer(tiled_depth_linear.buffer,
tiled_depth_output.buffer, 1,
&tiled_depth_copy);
HostReadBarrier(tiled_depth_output.buffer, tiled_depth_size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto tiled_depth_words =
ReadBuffer(name, tiled_depth_output, tiled_depth_guest.size());
bool tiled_depth_matches = true;
for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) {
for (uint32_t y = 0; y < tiled_depth_height; y++) {
for (uint32_t x = 0; x < tiled_depth_width; x++) {
const auto index = layer * (tiled_depth_slice / sizeof(uint32_t)) +
y * tiled_depth_pitch + x;
tiled_depth_matches &=
tiled_depth_words[index] == tiled_depth_clear;
}
}
}
Require(name, "tiled depth layered contents",
tiled_depth_matches &&
TextureCacheTestAccess::Contains(texture_cache,
tiled_depth_image) &&
texture_cache.GetImage(tiled_depth_image).IsGpuModified(),
"tiled depth readback lost a padded slice or retired its image");
DestroyBuffer(&tiled_depth_output);
DestroyBuffer(&tiled_depth_input);
constexpr uint64_t tiled_d16_offset = 0x3a0000;
constexpr uint64_t tiled_d16_stencil_offset = 0x3c0000;
constexpr uint16_t tiled_d16_clear = 0x8000u;
constexpr uint16_t tiled_d16_stale = 0xbeefu;
std::vector<uint16_t> tiled_d16_guest(tiled_depth_size / sizeof(uint16_t),
tiled_d16_stale);
constexpr std::array<uint8_t, 16> tiled_d16_stencil{
0x10, 0x21, 0x32, 0x43, 0x54, 0x65, 0x76, 0x87,
0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f};
std::memcpy(memory + tiled_d16_offset, tiled_d16_guest.data(),
tiled_depth_size);
std::memcpy(memory + tiled_d16_stencil_offset, tiled_d16_stencil.data(),
tiled_d16_stencil.size());
auto tiled_d16_desc = MakeLinearDesc(
base + tiled_d16_offset, tiled_depth_size,
vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k16UNorm,
Prospero::ImageType::kColor2D,
{tiled_depth_width, tiled_depth_height, 1}, tiled_depth_layers, 2, 1);
tiled_d16_desc.type = BindingType::DepthTarget;
tiled_d16_desc.info.pitch = tiled_depth_pitch;
tiled_d16_desc.info.tile_mode = Prospero::TileMode::kDepth;
tiled_d16_desc.info.stencil = {base + tiled_d16_stencil_offset,
tiled_d16_stencil.size()};
tiled_d16_desc.info.mip_layout[0] = {
0, tiled_depth_size, tiled_depth_pitch, tiled_depth_height};
tiled_d16_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
tiled_d16_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto tiled_d16_image = texture_cache.FindImage(tiled_d16_desc);
Require(name, "tiled D16 native clear",
texture_cache.ClearImageFromBuffer(
command, tiled_d16_desc.info.data.address,
tiled_d16_desc.info.data.size,
EncodeD16AsD32(tiled_d16_clear)),
"failed to clear a layered tiled D16 fallback image");
Require(
name, "tiled D16 download queue",
TextureCacheTestAccess::TryDownload(texture_cache, tiled_d16_image),
"layered tiled D16 fallback readback was rejected");
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::vector<uint16_t> tiled_d16_after(tiled_d16_guest.size());
std::array<uint8_t, tiled_d16_stencil.size()> tiled_d16_stencil_after{};
std::memcpy(tiled_d16_after.data(), memory + tiled_d16_offset,
tiled_depth_size);
std::memcpy(tiled_d16_stencil_after.data(),
memory + tiled_d16_stencil_offset,
tiled_d16_stencil_after.size());
Require(name, "tiled D16 backing preservation",
std::ranges::count(tiled_d16_after, tiled_d16_clear) ==
active_tiled_words &&
std::ranges::count(tiled_d16_after, tiled_d16_stale) ==
tiled_d16_after.size() - active_tiled_words &&
tiled_d16_stencil_after == tiled_d16_stencil,
"tiled D16 fallback overwrote inactive depth or stencil backing");
std::vector<uint32_t> tiled_d16_words(tiled_depth_size /
sizeof(uint32_t));
std::memcpy(tiled_d16_words.data(), tiled_d16_after.data(),
tiled_depth_size);
std::array<GpuTileInfo, tiled_depth_layers> tiled_d16_tiles{};
Require(name, "tiled D16 block layout",
TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint16_t),
depth_block),
"failed to describe the tiled D16 test surface");
for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) {
const uint64_t offset = tiled_depth_slice * layer;
tiled_d16_tiles[layer] = {depth_block.family,
depth_block.bytes_per_element,
offset,
tiled_depth_slice,
offset,
tiled_depth_slice,
0,
tiled_depth_width,
tiled_depth_height,
1,
tiled_depth_pitch};
tiled_d16_tiles[layer].surface_z = layer;
}
auto tiled_d16_input =
CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_d16_words);
auto tiled_d16_linear =
TextureCacheTestAccess::Tiler(texture_cache)
.Detile(tiled_d16_input.buffer, 0, tiled_depth_size,
tiled_depth_size, tiled_d16_tiles);
auto tiled_d16_output =
CreateHostBuffer(name, tiled_depth_size, AllFlags,
std::vector<uint32_t>(tiled_d16_words.size(), 0));
const vk::BufferCopy tiled_d16_copy{tiled_d16_linear.offset, 0,
tiled_depth_size};
scheduler.Current().Handle().copyBuffer(
tiled_d16_linear.buffer, tiled_d16_output.buffer, 1, &tiled_d16_copy);
HostReadBarrier(tiled_d16_output.buffer, tiled_depth_size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto tiled_d16_linear_words =
ReadBuffer(name, tiled_d16_output, tiled_d16_words.size());
std::vector<uint16_t> tiled_d16_linear_values(tiled_d16_guest.size());
std::memcpy(tiled_d16_linear_values.data(), tiled_d16_linear_words.data(),
tiled_depth_size);
bool tiled_d16_matches = true;
for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) {
for (uint32_t y = 0; y < tiled_depth_height; y++) {
for (uint32_t x = 0; x < tiled_depth_width; x++) {
const auto index = layer * (tiled_depth_slice / sizeof(uint16_t)) +
y * tiled_depth_pitch + x;
tiled_d16_matches &=
tiled_d16_linear_values[index] == tiled_d16_clear;
}
}
}
Require(name, "tiled D16 layered contents",
tiled_d16_matches &&
TextureCacheTestAccess::Contains(texture_cache,
tiled_d16_image) &&
texture_cache.GetImage(tiled_d16_image).IsGpuModified(),
"tiled D16 fallback lost a converted layer or retired its image");
DestroyBuffer(&tiled_d16_output);
DestroyBuffer(&tiled_d16_input);
constexpr uint64_t d16_fallback_offset = 0x33c000;
constexpr uint64_t d16_fallback_stencil_offset = 0x33d000;
constexpr std::array<uint16_t, 4> d16_fallback_values{0x0000u, 0x2468u,
0xabcdu, 0xffffu};
std::memcpy(memory + d16_fallback_offset, d16_fallback_values.data(),
sizeof(d16_fallback_values));
std::memset(memory + d16_fallback_stencil_offset, 0x6d, 4);
auto d16_depth_desc = MakeLinearDesc(
base + d16_fallback_offset, sizeof(d16_fallback_values),
vk::Format::eD24UnormS8Uint, Prospero::BufferFormat::k16UNorm,
Prospero::ImageType::kColor2D, {4, 1, 1}, 1, 2, 1);
d16_depth_desc.type = BindingType::DepthTarget;
d16_depth_desc.info.stencil = {base + d16_fallback_stencil_offset, 4};
d16_depth_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
d16_depth_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto d16_depth_image = texture_cache.FindImage(d16_depth_desc);
(void)texture_cache.FindDepthTarget(d16_depth_image, d16_depth_desc);
texture_cache.MarkGpuWritten(d16_depth_image);
auto d16_storage_desc = d16_depth_desc;
d16_storage_desc.type = BindingType::Storage;
d16_storage_desc.info.stencil = {};
d16_storage_desc.info.pixel_format = vk::Format::eR16Unorm;
d16_storage_desc.view_info.format = vk::Format::eR16Unorm;
d16_storage_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
d16_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto d16_storage_image =
texture_cache.FindImage(d16_storage_desc, true);
auto d16_storage_readback = CreateHostBuffer(
name, sizeof(d16_fallback_values),
vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>(2, 0));
auto &d16_storage_native = texture_cache.GetImage(d16_storage_image);
d16_storage_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
scheduler.Current().Handle());
vk::BufferImageCopy d16_storage_copy{};
d16_storage_copy.bufferRowLength = 4;
d16_storage_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0,
0, 1};
d16_storage_copy.imageExtent = {4, 1, 1};
scheduler.Current().Handle().copyImageToBuffer(
d16_storage_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal, d16_storage_readback.buffer, 1,
&d16_storage_copy);
HostReadBarrier(d16_storage_readback.buffer, d16_storage_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto d16_storage_words = ReadBuffer(name, d16_storage_readback, 2);
std::array<uint16_t, 4> d16_storage_values{};
std::memcpy(d16_storage_values.data(), d16_storage_words.data(),
sizeof(d16_storage_values));
Require(
name, "D16 fallback depth/storage copy",
d16_storage_values == d16_fallback_values &&
d16_storage_image != d16_depth_image &&
texture_cache.GetImage(d16_storage_image).IsGpuModified(),
"D16 fallback reinterpreted four-byte host depth as two-byte color");
DestroyBuffer(&d16_storage_readback);
if (!texture_cache.GetImage(combined_destination_image).IsGpuModified()) {
texture_cache.MarkGpuWritten(combined_destination_image);
}
constexpr float stale_added_stencil_depth = 0.125f;
Libs::LibKernel::Memory::WriteBacking(
combined_destination.info.data.address, &stale_added_stencil_depth,
sizeof(stale_added_stencil_depth));
const auto depth_gc_tick = scheduler.CurrentTick();
TextureCacheTestAccess::ConfigureGarbageCollection(
texture_cache, std::array{combined_destination_image}, 81, 0);
texture_cache.RunGarbageCollector();
float depth_before_completion = 0.0f;
Libs::LibKernel::Memory::TryReadBacking(
combined_destination.info.data.address, &depth_before_completion,
sizeof(depth_before_completion));
const bool depth_image_retired = !TextureCacheTestAccess::Contains(
texture_cache, combined_destination_image);
const bool depth_proxy_retired = !texture_cache.FindImageFromRange(
base + added_stencil_offset, added_stencil_size, false);
Require(
name, "depth/stencil deferred pressure retirement",
depth_image_retired && depth_proxy_retired &&
scheduler.CurrentTick() == depth_gc_tick &&
depth_before_completion == stale_added_stencil_depth,
fmt::format(
"GC failed to retire/defer depth: image={} proxy={} tick={}/{} "
"depth={}/{}",
depth_image_retired, depth_proxy_retired, scheduler.CurrentTick(),
depth_gc_tick, depth_before_completion, stale_added_stencil_depth)
.c_str());
scheduler.Finish();
scheduler.DrainPriorityOperations();
float depth_after_completion = 0.0f;
Libs::LibKernel::Memory::TryReadBacking(
combined_destination.info.data.address, &depth_after_completion,
sizeof(depth_after_completion));
Require(name, "depth/stencil depth-plane preservation",
depth_after_completion == added_stencil_depth_value,
"GC discarded the current depth plane of a depth/stencil "
"image");
constexpr size_t gc_depth_pair_count = 6;
constexpr uint64_t gc_depth_pair_offset = 0x350000;
constexpr uint64_t gc_depth_pair_stride = 0x1000;
std::array<ImageId, gc_depth_pair_count> gc_depth_images{};
std::array<ImageId, gc_depth_pair_count> gc_stencil_images{};
std::array<ImageId, gc_depth_pair_count * 2> gc_depth_lru{};
for (size_t index = 0; index < gc_depth_pair_count; index++) {
auto depth = MakeLinearDesc(
base + gc_depth_pair_offset + index * gc_depth_pair_stride,
sizeof(float), vk::Format::eD32SfloatS8Uint,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
depth.type = BindingType::DepthTarget;
depth.info.stencil = {depth.info.data.address +
gc_depth_pair_stride / 2,
sizeof(uint32_t)};
depth.view_info.format = vk::Format::eD32SfloatS8Uint;
depth.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
gc_depth_images[index] = texture_cache.FindImage(depth);
TextureCacheTestAccess::AssociateStencil(
texture_cache, gc_depth_images[index], depth.info.stencil);
gc_stencil_images[index] = texture_cache.FindImageFromRange(
depth.info.stencil.address, depth.info.stencil.size, false);
gc_depth_lru[index * 2] = gc_depth_images[index];
gc_depth_lru[index * 2 + 1] = gc_stencil_images[index];
}
Require(
name, "depth/stencil GC fixtures",
std::ranges::all_of(gc_depth_images,
[](ImageId image) { return bool{image}; }) &&
std::ranges::all_of(gc_stencil_images,
[](ImageId image) { return bool{image}; }),
"failed to create six depth/stencil association pairs");
TextureCacheTestAccess::ConfigureGarbageCollection(
texture_cache, gc_depth_lru, 81, UINT64_MAX);
texture_cache.RunGarbageCollector();
bool gc_depth_budget = true;
for (size_t index = 0; index < gc_depth_pair_count; index++) {
const bool expected_live = index == gc_depth_pair_count - 1;
gc_depth_budget &=
TextureCacheTestAccess::Contains(
texture_cache, gc_depth_images[index]) == expected_live &&
TextureCacheTestAccess::Contains(
texture_cache, gc_stencil_images[index]) == expected_live;
}
Require(name, "depth/stencil GC traversal budget", gc_depth_budget,
"recursive association deletion stopped LRU traversal or "
"exceeded the ten-entry deletion budget");
constexpr uint64_t large_offset = 0x400000;
constexpr uint32_t large_width = 4096;
constexpr uint32_t large_height = 2047;
constexpr uint64_t large_size =
uint64_t{large_width} * large_height * sizeof(uint32_t);
static_assert(large_size < (32ull << 20));
std::memset(memory + large_offset, 0, static_cast<size_t>(large_size));
auto large_desc = MakeLinearDesc(
base + large_offset, large_size, vk::Format::eR8G8B8A8Unorm,
Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
{large_width, large_height, 1}, 1, 4, 1);
const auto RunLargeReadback = [&](uint32_t clear_value) {
const auto image = texture_cache.FindImage(large_desc);
Require(name, "near-capacity image clear",
texture_cache.ClearImageFromBuffer(
command, large_desc.info.data.address,
large_desc.info.data.size, clear_value),
"failed to write the near-capacity readback source image");
const std::array<uint64_t, 3> sample_offsets{
0, large_size / 2, large_size - sizeof(uint32_t)};
constexpr uint32_t stale = 0;
for (const auto offset : sample_offsets) {
Libs::LibKernel::Memory::WriteBacking(base + large_offset + offset,
&stale, sizeof(stale));
}
TextureCacheTestAccess::ConfigureGarbageCollection(
texture_cache, std::array{image}, 81, 0);
const auto tick = scheduler.CurrentTick();
texture_cache.RunGarbageCollector();
const auto handle =
BufferCacheTestAccess::DownloadBuffer(resources.GetBufferCache())
.Handle();
Require(
name, "near-capacity deferred retirement",
!TextureCacheTestAccess::Contains(texture_cache, image) &&
scheduler.CurrentTick() == tick,
"near-capacity readback was rejected or synchronously submitted");
scheduler.Finish();
scheduler.DrainPriorityOperations();
bool content = true;
for (const auto offset : sample_offsets) {
uint32_t value = 0;
Libs::LibKernel::Memory::TryReadBacking(base + large_offset + offset,
&value, sizeof(value));
content &= value == clear_value;
}
Require(
name, "near-capacity readback content", content,
"near-capacity image readback did not publish its GPU contents");
return handle;
};
const auto large_download = RunLargeReadback(0xa5a5a5a5u);
const auto reused_large_download = RunLargeReadback(0x5a5a5a5au);
Require(name, "near-capacity shared download reuse",
large_download == reused_large_download,
"successive near-capacity image transfers replaced the shared "
"download buffer");
constexpr uint64_t tile_alias_offset = 0x2000000;
constexpr uint64_t tile_alias_size = 0x400000;
constexpr uint32_t tile_alias_extent = 1024;
std::memset(memory + tile_alias_offset, 0,
static_cast<size_t>(tile_alias_size));
auto render_target_alias = MakeLinearDesc(
base + tile_alias_offset, tile_alias_size, vk::Format::eR8G8B8A8Unorm,
Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
{tile_alias_extent, tile_alias_extent, 1}, 1, 4, 1);
render_target_alias.type = BindingType::Storage;
render_target_alias.info.tile_mode = Prospero::TileMode::kRenderTarget;
render_target_alias.view_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto render_target_alias_image =
texture_cache.FindImage(render_target_alias);
auto standard_4kb_alias = render_target_alias;
standard_4kb_alias.type = BindingType::Texture;
standard_4kb_alias.info.tile_mode = Prospero::TileMode::kStandard4KB;
standard_4kb_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto standard_4kb_alias_image =
texture_cache.FindImage(standard_4kb_alias);
auto repeated_standard_4kb_alias = standard_4kb_alias;
const auto repeated_standard_4kb_alias_image =
texture_cache.FindImage(repeated_standard_4kb_alias);
Require(
name, "equal-size tile-mode alias",
render_target_alias_image && standard_4kb_alias_image &&
standard_4kb_alias_image != render_target_alias_image &&
repeated_standard_4kb_alias_image == standard_4kb_alias_image &&
texture_cache.GetImage(render_target_alias_image)
.info.tile_mode == Prospero::TileMode::kRenderTarget &&
texture_cache.GetImage(standard_4kb_alias_image).info.tile_mode ==
Prospero::TileMode::kStandard4KB,
"equal address/size lookup reused an incompatible "
"tiled backing");
for (auto &output : ms_observer_outputs) {
DestroyBuffer(&output);
}
DestroyBuffer(&layered_readback);
DestroyBuffer(&bgra16_readback);
DestroyBuffer(&mixed_source_readback);
DestroyBuffer(&bridged_source_readback);
DestroyBuffer(&mip_formatted_readback);
DestroyBuffer(&mip_prefix_readback);
DestroyBuffer(&partial_cpu_refresh_readback);
DestroyBuffer(&mirror_cpu_readback);
DestroyBuffer(&exact_buffer_readback);
m_device.destroyDescriptorPool(observer_pool, nullptr);
m_device.destroyPipeline(observer_depth_pipeline, nullptr);
m_device.destroyPipelineLayout(observer_pipeline_layout, nullptr);
m_device.destroyDescriptorSetLayout(observer_set_layout, nullptr);
m_device.destroyShaderModule(ms_depth_module, nullptr);
constexpr uint64_t mapped_end = base + allocation_size;
constexpr uint64_t final_word_address = mapped_end - sizeof(uint32_t);
auto *final_word = reinterpret_cast<volatile uint32_t *>(final_word_address);
auto *final_byte = reinterpret_cast<volatile uint8_t *>(mapped_end - 1);
Require(name, "fault boundary containment",
resources.IsMapped(final_word_address, sizeof(uint32_t)) &&
!resources.IsMapped(final_word_address, sizeof(uint64_t)) &&
!resources.HandleFault(PageFaultAccess::Write, mapped_end) &&
!resources.HandleFault(PageFaultAccess::Read, mapped_end),
"fault handling changed the mapped range's exclusive end");
Require(name, "fault boundary setup",
resources.InvalidateMemory(final_word_address, sizeof(uint32_t)),
"the final mapped word could not be initialized");
*final_word = 0;
auto final_word_desc = MakeLinearDesc(
final_word_address, sizeof(uint32_t), vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
const auto final_word_image = texture_cache.FindImage(final_word_desc);
(void)texture_cache.FindTexture(final_word_image, final_word_desc);
Require(name, "fault boundary image tracking",
texture_cache.GetImage(final_word_image).IsTracked() &&
!texture_cache.GetImage(final_word_image).IsCpuDirty(),
"the final mapped word's image was not clean before its CPU store");
*final_word = 45;
Require(name, "fault boundary CPU store",
*final_word == 45 &&
texture_cache.GetImage(final_word_image).IsDefinitelyCpuDirty(),
"a valid four-byte store at the mapping end did not invalidate its image");
constexpr uint32_t final_gpu_value = 0xa1b2c3d4u;
auto &boundary_buffer_cache = resources.GetBufferCache();
(void)boundary_buffer_cache.ObtainBuffer(final_word_address, sizeof(uint32_t), true);
boundary_buffer_cache.FillBuffer(final_word_address, sizeof(uint32_t),
final_gpu_value, false);
Require(name, "fault boundary GPU ownership",
boundary_buffer_cache.HasGpuDirtyBytes(final_word_address, sizeof(uint32_t)) &&
!HostMemoryIsReadable(mapped_end - 1),
"the final mapped byte was not protected by GPU ownership");
const uint8_t final_gpu_byte = *final_byte;
Require(name, "fault boundary GPU readback",
final_gpu_byte == static_cast<uint8_t>(final_gpu_value >> 24u) &&
*final_word == final_gpu_value &&
!boundary_buffer_cache.HasGpuDirtyBytes(final_word_address, sizeof(uint32_t)),
"a final-byte CPU read did not publish the GPU-owned final word");
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
LibKernel::Memory::InstallGpuResources(nullptr);
}
context.ShutdownGpu();
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"cache direct-memory mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"cache direct-memory allocation release failed");
std::printf("[host] %-32s ok\n", name);
}
void CheckBgra16Readback() {
constexpr const char *name = "Bgra16Readback";
constexpr uintptr_t base = 0x0000000204000000ull;
constexpr uint64_t allocation_size = 0x200000;
constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float;
constexpr auto tile = Prospero::TileMode::kRenderTarget;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"BGRA16 direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 &&
mapped == reinterpret_cast<void *>(base),
"BGRA16 fixed mapping failed");
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
{
auto &resources = context;
resources.MapMemory(base, allocation_size);
const uint32_t pitch = TileGetTexturePitch(format, 1, tile);
TileSizeAlign total{};
TileSizeOffset mip{};
TilePaddedSize padded{};
TileGetTextureSize(format, 1, 1, 1, tile, &total, &mip, &padded);
Require(name, "tiled layout",
total.size >= 8 && total.size <= allocation_size,
"BGRA16 tiled layout exceeds its guest allocation");
std::memset(mapped, 0x5a, total.size);
constexpr std::array<uint16_t, 4> initial{0x3c00u, 0x4000u, 0x4200u, 0x4400u};
std::memcpy(mapped, initial.data(), sizeof(initial));
ImageDesc desc{};
desc.type = BindingType::RenderTarget;
desc.info.data = {base, total.size};
desc.info.pixel_format = vk::Format::eR16G16B16A16Sfloat;
desc.info.guest_format = format;
desc.info.type = Prospero::ImageType::kColor2D;
desc.info.extent = {1, 1, 1};
desc.info.resources = {1, 1};
desc.info.pitch = pitch;
desc.info.bytes_per_block = 8;
desc.info.samples = 1;
desc.info.tile_mode = tile;
desc.info.bgra16 = true;
desc.info.mip_layout[0] = {mip.offset, mip.size, pitch, 1};
desc.view_info.format = desc.info.pixel_format;
desc.view_info.type = vk::ImageViewType::e2D;
desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
auto &cache = resources.GetTextureCache();
const auto id = cache.FindImage(desc);
(void)cache.FindRenderTarget(id, desc);
Require(name, "tiled BGRA16 upload",
ReadCachedTexel(name, context, id) ==
std::vector<u32>{0x40004200u, 0x44003c00u},
"fused detiling lost the BGRA16 component transform");
auto &image = cache.GetImage(id);
image.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {},
scheduler.Current().Handle());
vk::ClearColorValue clear{};
clear.float32[0] = 1.0f;
clear.float32[1] = 2.0f;
clear.float32[2] = 3.0f;
clear.float32[3] = 4.0f;
const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eColor, 0,
1, 0, 1};
scheduler.Current().Handle().clearColorImage(
image.backing.image, vk::ImageLayout::eTransferDstOptimal, clear,
range);
cache.MarkGpuWritten(id);
Require(name, "guest readback queue",
TextureCacheTestAccess::TryDownload(cache, id),
"tiled BGRA16 guest readback was rejected");
auto mirror = resources.GetBufferCache().ObtainBuffer(base, total.size,
false, true);
Require(name, "mirror owner", mirror.first != nullptr,
"tiled BGRA16 mirror has no BufferCache owner");
auto mirror_readback = CreateHostBuffer(
name, 8, vk::BufferUsageFlagBits::eTransferDst, {0, 0});
const vk::BufferCopy copy{mirror.second, 0, 8};
scheduler.Current().Handle().copyBuffer(mirror.first->Handle(),
mirror_readback.buffer, 1, &copy);
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = mirror_readback.buffer;
barrier.size = mirror_readback.size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
nullptr);
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::vector<uint8_t> backing(total.size);
Require(name, "guest backing",
Libs::LibKernel::Memory::TryReadBacking(base, backing.data(),
backing.size()),
"tiled BGRA16 guest backing is unreadable");
const std::array<uint16_t, 4> expected{0x4200u, 0x4000u, 0x3c00u,
0x4400u};
std::array<uint16_t, 4> observed{};
std::memcpy(observed.data(), backing.data(), sizeof(observed));
Require(name, "guest component order",
observed == expected &&
std::all_of(backing.begin() + sizeof(observed), backing.end(),
[](uint8_t value) { return value == 0x5a; }),
"tiled BGRA16 guest readback changed component order or padding");
Require(name, "mirror component order",
ReadBuffer(name, mirror_readback, 2) ==
std::vector<u32>{0x40004200u, 0x44003c00u},
"tiled BGRA16 Buffer mirror changed guest component order");
DestroyBuffer(&mirror_readback);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"BGRA16 fixed mapping release failed");
Require(name, "release",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"BGRA16 direct-memory release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColorDiscovery() {
constexpr const char *name = "RenderExecutorColorDiscovery";
constexpr uintptr_t base = 0x0000000203c00000ull;
constexpr uint32_t width = 512;
constexpr uint32_t height = 1;
constexpr uint32_t bytes_per_element = 4;
constexpr uint64_t allocation_alignment = 0x10000;
EnsureRuntimeContext();
const auto pitch =
TileGetRenderTargetPitch(width, bytes_per_element, 0);
TileSizeAlign layout{};
Require(name, "tile layout",
pitch != 0 && TileGetRenderTargetSize(
width, height, pitch, bytes_per_element, layout, 0),
"1D render-target layout is unavailable");
struct TargetCase {
uint32_t width, height, dimension, pitch;
uint64_t size;
Prospero::TileMode tile;
uint32_t levels = 1;
bool float16 = false;
};
const std::array cases{
TargetCase{width, height, 0, pitch, layout.size,
Prospero::TileMode::kRenderTarget},
// PPSA28068: the 1440x720 agreement target must alias its sampled texture.
TargetCase{1440, 720, 1, 1472, 0x40b000, Prospero::TileMode::kLinear},
// PPSA01416: linear RGBA16F chains place the smallest mip first.
TargetCase{128, 128, 1, 128, 0x2be00, Prospero::TileMode::kLinear, 7, true},
// PPSA03541: the single texel still occupies a padded 128x64 block.
TargetCase{1, 1, 1, 128, 0x10000, Prospero::TileMode::kRenderTarget, 1, true},
};
constexpr uint64_t allocation_size = 0x410000;
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"color direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"color fixed mapping failed");
std::memset(mapped, 0, allocation_size);
for (const auto &target : cases) {
const auto width = target.width;
const auto height = target.height;
const bool is_1d = target.dimension == 0;
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetColorBase(0, {.addr = base});
registers.SetColorInfo(
0, {.format = target.float16 ? Prospero::ChannelLayout::k16_16_16_16
: Prospero::ChannelLayout::k8_8_8_8,
.channel_type = target.float16 ? Prospero::ChannelType::kFloat
: Prospero::ChannelType::kSrgb,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorAttrib2(0, {.height = height - 1, .width = width - 1,
.num_mip_levels = target.levels - 1});
registers.SetColorAttrib3(0,
{.tile_mode = target.tile,
.dimension = target.dimension,
.metadata_pipe_aligned = true});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
RenderColorInfo color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto attachment =
texture_cache.FindRenderTarget(color.image_id, color.desc);
const auto &image = texture_cache.GetImage(color.image_id);
RenderDepthInfo no_depth{};
const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &color, 1, no_depth);
scheduler.Current().BeginRendering(rendering);
scheduler.Current().EndRendering();
Require(name, "captured color target",
color.image_id && attachment != nullptr &&
rendering.color_attachments[0].image_view != nullptr &&
color.desc.info.type == (is_1d ? Prospero::ImageType::kColor1D
: Prospero::ImageType::kColor2D) &&
color.desc.info.extent == vk::Extent3D{width, height, 1} &&
color.desc.info.resources == ImageSubresources{target.levels, 1} &&
color.desc.info.pitch == target.pitch &&
color.desc.info.data.size == target.size &&
color.desc.info.mip_layout[0].offset ==
(target.levels == 1 ? 0 : 0xbe00) &&
color.desc.info.mip_layout[0].size ==
(target.levels == 1 ? target.size : 0x20000) &&
color.desc.view_info.type == (is_1d ? vk::ImageViewType::e1D
: vk::ImageViewType::e2D) &&
image.backing.image_type == (is_1d ? vk::ImageType::e1D
: vk::ImageType::e2D) &&
rendering.width == width && rendering.height == height &&
rendering.num_layers == 1 &&
rendering.num_color_attachments == 1,
"color attachment lost its dimensions or padded guest layout");
if (width == 1 && height == 1) {
Require(name, "single-texel tiled layout",
color.desc.info.mip_layout[0] == ImageMipInfo{0, 0x10000, 128, 64},
"the render target lost the expected padded block height");
}
if (target.levels == 7) {
constexpr std::array offsets{0xbe00u, 0x3e00u, 0x1e00u, 0xe00u,
0x600u, 0x200u, 0u};
constexpr std::array pitches{128u, 64u, 32u, 32u, 32u, 32u, 32u};
for (uint32_t mip = 0; mip < target.levels; ++mip) {
const auto &layout = color.desc.info.mip_layout[mip];
Require(name, "linear mip chain", layout.offset == offsets[mip] &&
layout.pitch == pitches[mip] &&
layout.size == pitches[mip] * (height >> mip) * 8,
"linear target mip offsets or padded pitches differ from the expected layout");
}
}
{
vk::ClearValue clear{};
clear.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, target.levels, 0, 1}, clear);
RenderExecutorTestAccess::ResetBindings(executor);
ShaderTextureResource descriptor{{
static_cast<uint32_t>(base >> 8u),
(static_cast<uint32_t>(target.float16
? Prospero::BufferFormat::k16_16_16_16Float
: Prospero::BufferFormat::k8_8_8_8Srgb) << 20u) |
(((width - 1u) & 3u) << 30u),
((width - 1u) >> 2u) | ((height - 1u) << 14u),
DstSel(4, 5, 6, 7) | ((target.levels - 1) << 16u) |
(static_cast<uint32_t>(target.tile) << 20u) |
(static_cast<uint32_t>(is_1d ? Prospero::ImageType::kColor1D
: Prospero::ImageType::kColor2D) << 28u),
0, 0x00700000u | ((target.levels - 1) << 4u), 0, 0}};
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = is_1d ? ShaderRecompiler::Decoder::ImageDimension::Dim1D
: ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = true;
const auto sampled =
RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
Require(name, "target sampled alias",
sampled.image_id == color.image_id &&
sampled.desc.info.pitch == target.pitch &&
sampled.desc.info.data.size == target.size &&
sampled.desc.info.mip_layout == color.desc.info.mip_layout &&
texture_cache.FindTexture(sampled.image_id, sampled.desc) != nullptr,
"sampling the target changed its padded layout or created a second image");
const auto expected = target.float16
? std::vector<u32>{0x00003c00u, 0x3c003c00u}
: std::vector<u32>{0xffff00ffu};
for (uint32_t mip = 0; mip < target.levels; ++mip) {
Require(name, "target sampled contents",
ReadCachedTexel(name, context, sampled.image_id,
{static_cast<int32_t>((width >> mip) - 1),
static_cast<int32_t>((height >> mip) - 1), 0},
{1, 1, 1}, 0, mip) == expected,
"the sampled target lost its GPU-written final texel");
}
if (width == 1 && height == 1) {
for (const bool uint_view : {true, false}) {
descriptor.fields[1] = static_cast<uint32_t>(
uint_view ? Prospero::BufferFormat::k16_16_16_16UInt
: Prospero::BufferFormat::k16_16_16_16Float) << 20u;
descriptor.fields[3] = 0x81b00facu; // Captured 1D view, tile27, RGBA swizzle.
std::copy_n(descriptor.fields, 8, value.dwords.begin());
resource.numeric_class = uint_view ? Prospero::TextureNumericClass::Uint
: Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim1D;
const auto one_d =
RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
const auto one_d_view = texture_cache.FindTexture(one_d.image_id, one_d.desc);
const auto repeated =
RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
Require(name, "1D sampled backing and reuse",
one_d.image_id != color.image_id && one_d_view != nullptr &&
one_d.desc.view_info.type == vk::ImageViewType::e1D &&
texture_cache.GetImage(one_d.image_id).backing.image_type ==
vk::ImageType::e1D &&
repeated.image_id == one_d.image_id &&
texture_cache.FindTexture(repeated.image_id, repeated.desc) == one_d_view &&
ReadCachedTexel(name, context, one_d.image_id) == expected,
"the 1D alias reused an incompatible backing or lost the rendered texel");
RenderExecutorTestAccess::ResetBindings(executor);
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto restored = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &color, 1, no_depth);
scheduler.Current().BeginRendering(restored);
scheduler.Current().EndRendering();
Require(name, "2D target restoration",
color.image_id != one_d.image_id &&
restored.color_attachments[0].image_view != nullptr &&
texture_cache.GetImage(color.image_id).backing.image_type ==
vk::ImageType::e2D &&
texture_cache.GetImage(color.image_id).info.pixel_format ==
vk::Format::eR16G16B16A16Sfloat &&
ReadCachedTexel(name, context, color.image_id) == expected,
"restoring the 2D target lost its view type or GPU-written texel");
RenderExecutorTestAccess::ResetBindings(executor);
}
}
}
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"color direct mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"color direct-memory allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColor1DArrayDiscovery() {
constexpr const char *name = "RenderExecutorColor1DArrayDiscovery";
constexpr uintptr_t base = 0x0000000203a00000ull;
constexpr uint32_t width = 512;
constexpr uint32_t height = 1;
constexpr uint32_t bytes_per_element = 4;
constexpr uint32_t first_slice = 2;
constexpr uint32_t last_slice = 5;
constexpr uint32_t image_layers = last_slice + 1;
constexpr uint32_t view_layers = last_slice - first_slice + 1;
constexpr uint64_t allocation_alignment = 0x10000;
EnsureRuntimeContext();
const auto pitch = TileGetRenderTargetPitch(width, bytes_per_element, 0);
TileSizeAlign layout{};
Require(name, "tile layout",
pitch == width &&
TileGetRenderTargetSize(width, height, pitch,
bytes_per_element, layout, 0) &&
layout.size == 0x40000 &&
layout.align == allocation_alignment,
"1D-array render-target layout is unavailable");
const auto allocation_size = layout.size * image_layers;
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"1D-array color direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"1D-array color fixed mapping failed");
std::memset(mapped, 0, allocation_size);
{
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetColorBase(0, {.addr = base});
registers.SetColorInfo(
0, {.format = Prospero::ChannelLayout::k8_8_8_8,
.channel_type = Prospero::ChannelType::kSrgb,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorView(
0, {.base_array_slice_index = first_slice,
.last_array_slice_index = last_slice});
registers.SetColorAttrib2(0, {.height = height - 1, .width = width - 1});
registers.SetColorAttrib3(0,
{.tile_mode = Prospero::TileMode::kRenderTarget,
.dimension = 0,
.metadata_pipe_aligned = true});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
RenderColorInfo color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto attachment =
texture_cache.FindRenderTarget(color.image_id, color.desc);
const auto &image = texture_cache.GetImage(color.image_id);
RenderDepthInfo no_depth{};
const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &color, 1, no_depth);
scheduler.Current().BeginRendering(rendering);
scheduler.Current().EndRendering();
Require(
name, "captured 1D-array target",
color.image_id && attachment != nullptr &&
rendering.color_attachments[0].image_view == attachment &&
color.guest_array_layer == first_slice &&
color.desc.info.type == Prospero::ImageType::kColor1D &&
color.desc.info.extent == vk::Extent3D{width, height, 1} &&
color.desc.info.resources == ImageSubresources{1, image_layers} &&
color.desc.info.pitch == pitch &&
color.desc.info.data.size == allocation_size &&
color.desc.info.mip_layout[0].offset == 0 &&
color.desc.info.mip_layout[0].size == allocation_size &&
color.desc.view_info.type == vk::ImageViewType::e1DArray &&
color.desc.view_info.base_layer == first_slice &&
color.desc.view_info.layer_count == view_layers &&
image.backing.image_type == vk::ImageType::e1D &&
image.backing.layers == image_layers &&
rendering.width == width && rendering.height == height &&
rendering.num_layers == view_layers &&
rendering.num_color_attachments == 1,
"dimension=0 slice range did not create a Vulkan 1D-array "
"attachment view");
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"1D-array color direct mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"1D-array color direct-memory allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColorVolumeDiscovery() {
constexpr const char *name = "RenderExecutorColorVolumeDiscovery";
constexpr uintptr_t base = 0x0000000203e00000ull;
constexpr uint64_t slice_size = 0x10000;
constexpr uint64_t color_size = 0x200000;
constexpr uint64_t metadata_size = 0x20000;
constexpr uint64_t allocation_size = 56 * slice_size;
constexpr uint64_t allocation_alignment = 0x10000;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"color-volume direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"color-volume fixed mapping failed");
std::memset(mapped, 0x5a, allocation_size);
{
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetColorBase(0, {.addr = base});
registers.SetColorInfo(
0, {.dcc_compression_enable = true,
.format = Prospero::ChannelLayout::k10_10_10_2,
.channel_type = Prospero::ChannelType::kUNorm,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorAttrib2(0, {.height = 31, .width = 31});
registers.SetColorAttrib3(0,
{.depth = 31,
.tile_mode = Prospero::TileMode::kRenderTarget,
.dimension = 2,
.metadata_pipe_aligned = true});
constexpr uint64_t dcc_address = base + color_size;
registers.SetColorDccAddr(0, {.addr = dcc_address});
registers.SetColorClearWord0(0, {.word0 = 0});
registers.SetColorClearWord1(0, {.word1 = 0});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
WriteMetadata(context, dcc_address, metadata_size, 0);
Require(name, "deferred DCC clear",
!texture_cache.IsMeta(dcc_address),
"a pre-registration DCC clear was not deferred");
RenderColorInfo color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto attachment =
texture_cache.FindRenderTarget(color.image_id, color.desc);
const auto &image = texture_cache.GetImage(color.image_id);
Require(
name, "captured 3D target",
color.image_id && attachment != nullptr &&
color.desc.info.type == Prospero::ImageType::kColor3D &&
color.desc.info.extent == vk::Extent3D{32, 32, 32} &&
color.desc.info.resources == ImageSubresources{1, 1} &&
color.desc.info.pitch == 128 &&
color.desc.info.data.size == color_size &&
color.desc.info.mip_layout[0].size == 0x10000 &&
color.desc.info.metadata.kind == ImageMetadataKind::Dcc &&
color.desc.info.metadata.range.address == dcc_address &&
!texture_cache.IsMeta(dcc_address) && !texture_cache.ClearMeta(dcc_address) &&
color.desc.view_info.type == vk::ImageViewType::e2D &&
color.desc.view_info.layer_count == 1 &&
image.backing.image_type == vk::ImageType::e3D &&
static_cast<bool>(image.backing.flags &
vk::ImageCreateFlagBits::e2DArrayCompatible) &&
image.usage.render_target && image.IsGpuModified(),
"dimension=2/depth=31 did not create the expected 32x32x32 "
"backing and 2D "
"attachment slice");
constexpr uint64_t metadata_slice_size = metadata_size / 32;
std::vector<uint8_t> metadata_bytes(metadata_size);
resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
Require(name, "first volume slice expanded metadata",
LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
metadata_bytes.size()) &&
std::all_of(metadata_bytes.begin(),
metadata_bytes.begin() + metadata_slice_size,
[](uint8_t byte) { return byte == 0xff; }) &&
std::all_of(metadata_bytes.begin() + metadata_slice_size,
metadata_bytes.end(), [](uint8_t byte) { return byte == 0; }),
"consuming volume slice 0 changed unrequested native metadata slices");
RenderExecutorTestAccess::ResetBindings(executor);
registers.SetColorView(
0, {.base_array_slice_index = 7, .last_array_slice_index = 7});
RenderColorInfo sliced_color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), sliced_color, 0);
RenderDepthInfo no_depth{};
const auto sliced_rendering =
RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &sliced_color, 1, no_depth);
Require(
name, "3D slice transition",
sliced_color.image_id == color.image_id &&
sliced_rendering.color_attachments[0].image_view != nullptr &&
sliced_color.desc.view_info.base_layer == 7 &&
sliced_rendering.num_color_attachments == 1 &&
sliced_rendering.num_layers == 1,
"a nonzero 3D attachment slice was treated as a Vulkan array layer");
resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
Require(name, "mixed volume metadata readback",
LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
metadata_bytes.size()),
"mixed volume metadata is unavailable");
for (uint32_t slice = 0; slice < 32; slice++) {
const uint8_t expected = slice == 0 || slice == 7 ? 0xff : 0;
const auto first = metadata_bytes.begin() + slice * metadata_slice_size;
Require(name, "independent volume metadata consumption",
std::all_of(first, first + metadata_slice_size,
[expected](uint8_t byte) { return byte == expected; }),
"mixed DCC metadata lost a pending slice or consumed an unrequested slice");
}
Require(name, "3D slice clear and preservation",
ReadCachedTexel(name, context, color.image_id, {0, 0, 7}) ==
std::vector<u32>{0} &&
ReadCachedTexel(name, context, color.image_id, {0, 0, 31}) ==
std::vector<u32>{0x5a5a5a5a},
"clearing one volume slice changed untouched slice contents");
auto storage_desc = color.desc;
storage_desc.type = BindingType::Storage;
storage_desc.info.metadata = {};
storage_desc.info.guest_format = Prospero::BufferFormat::k10_10_10_2UNorm;
storage_desc.view_info.type = vk::ImageViewType::e3D;
storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto storage_id = texture_cache.FindImage(storage_desc);
const auto storage_view =
texture_cache.FindTexture(storage_id, storage_desc);
const auto &shared_image = texture_cache.GetImage(storage_id);
Require(name, "storage alias reuse",
storage_id == color.image_id && storage_view != nullptr &&
shared_image.IsGpuModified() &&
shared_image.usage.render_target,
"the matching 3D storage binding did not reuse the live "
"render-target image");
Require(
name, "volume readback queue",
TextureCacheTestAccess::TryDownload(texture_cache, storage_id),
"the 3D render target could not be queued for guest-layout readback");
auto mirror = resources.GetBufferCache().ObtainBuffer(
base, color_size, false, true);
Require(name, "volume mirror", mirror.first != nullptr,
"the 3D render-target readback has no BufferCache owner");
auto slice_probe =
CreateHostBuffer(name, 4, vk::BufferUsageFlagBits::eTransferDst, {0});
const vk::BufferCopy copy{mirror.second + 31 * slice_size, 0, 4};
scheduler.Current().Handle().copyBuffer(mirror.first->Handle(),
slice_probe.buffer, 1, &copy);
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = slice_probe.buffer;
barrier.size = slice_probe.size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
nullptr);
scheduler.Finish();
scheduler.DrainPriorityOperations();
Require(name, "volume Z transfer",
ReadBuffer(name, slice_probe, 1) == std::vector<u32>{0x5a5a5a5a},
"render-target upload/readback lost the final Z slice");
DestroyBuffer(&slice_probe);
vk::ClearValue painted{};
painted.color.float32 = std::array{1.0f, 1.0f, 1.0f, 0.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
WriteMetadata(context, dcc_address, metadata_size, 0xffffffffu);
color.image_id = texture_cache.FindImage(color.desc);
(void)texture_cache.FindRenderTarget(color.image_id, color.desc);
resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
Require(name, "expanded volume metadata preserves rendering",
ReadCachedTexel(name, context, color.image_id) ==
std::vector<u32>{0x3fffffffu} &&
LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
metadata_bytes.size()) &&
std::all_of(metadata_bytes.begin(), metadata_bytes.end(),
[](uint8_t byte) { return byte == 0xff; }),
"uncompressed native keys reapplied an old volume clear");
WriteMetadata(context, dcc_address, metadata_size, 0);
color.image_id = texture_cache.FindImage(color.desc);
(void)texture_cache.FindRenderTarget(color.image_id, color.desc);
resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
Require(name, "native volume re-clear",
ReadCachedTexel(name, context, color.image_id) == std::vector<u32>{0} &&
LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
metadata_bytes.size()) &&
std::all_of(metadata_bytes.begin(),
metadata_bytes.begin() + metadata_slice_size,
[](uint8_t byte) { return byte == 0xff; }) &&
std::all_of(metadata_bytes.begin() + metadata_slice_size,
metadata_bytes.end(), [](uint8_t byte) { return byte == 0; }),
"a native metadata overwrite did not clear only its requested volume slice");
for (const uint32_t base_slice : {0u, 7u}) {
RenderExecutorTestAccess::ResetBindings(executor);
registers.SetColorView(0, {.base_array_slice_index = base_slice,
.last_array_slice_index = 32});
RenderColorInfo volume_color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), volume_color, 0);
const auto volume_rendering =
RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &volume_color, 1, no_depth);
Require(name, "3D export bound exceeds depth",
volume_color.image_id == color.image_id &&
volume_color.desc.info.extent == vk::Extent3D{32, 32, 32} &&
volume_color.desc.info.data.size == color_size &&
volume_color.desc.view_info.type == vk::ImageViewType::e2DArray &&
volume_color.desc.view_info.base_layer == base_slice &&
volume_color.desc.view_info.layer_count == 32 - base_slice &&
volume_rendering.num_layers == 32 - base_slice &&
registers.GetRenderTarget(0).view.last_array_slice_index == 32,
"3D export bounds changed the backing or exceeded its attachment depth");
scheduler.Current().BeginRendering(volume_rendering);
scheduler.Current().EndRendering();
}
resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
Require(name, "complete volume expansion",
LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
metadata_bytes.size()) &&
std::all_of(metadata_bytes.begin(), metadata_bytes.end(),
[](uint8_t byte) { return byte == 0xff; }),
"full volume acquisition left an unconsumed native clear slice");
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 31, 1}, painted);
auto expanded = storage_desc;
expanded.info.extent.depth = 56;
expanded.info.data.size = allocation_size;
const auto expanded_id = texture_cache.FindImage(expanded);
Require(name, "3D storage depth expansion",
expanded_id != color.image_id &&
!TextureCacheTestAccess::Contains(texture_cache, color.image_id) &&
texture_cache.GetImage(expanded_id).backing.extent.depth == 56 &&
texture_cache.GetImage(expanded_id).IsGpuModified(),
"a deeper storage binding reused the smaller native volume");
Require(name, "3D expansion preserves native prefix and guest suffix",
ReadCachedTexel(name, context, expanded_id, {0, 0, 31}) ==
std::vector<u32>{0x3fffffffu} &&
ReadCachedTexel(name, context, expanded_id, {0, 0, 55}) ==
std::vector<u32>{0x5a5a5a5a},
"volume expansion lost GPU writes or copied beyond the old depth");
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
context.ShutdownGpu();
LibKernel::Memory::InstallGpuResources(nullptr);
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"color-volume direct mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"color-volume direct-memory allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
static void WriteMetadata(RenderContext &context, uint64_t address,
uint64_t size, uint32_t value) {
Require("DccMetadata", "guest write range", size % sizeof(value) == 0 &&
context.InvalidateMemory(address, size),
"metadata write is outside the mapped allocation");
const std::vector<uint32_t> words(size / sizeof(value), value);
LibKernel::Memory::WriteBacking(address, words.data(), size);
}
static TextureCache::ImageDesc MakeLinearDesc(
uint64_t address, uint64_t size, vk::Format format,
Prospero::BufferFormat guest_format, Prospero::ImageType type,
vk::Extent3D extent, uint32_t layers, uint32_t bytes_per_block,
uint32_t samples) {
TextureCache::ImageDesc desc{};
desc.type = TextureCache::BindingType::Texture;
desc.info.data = {address, size};
desc.info.pixel_format = format;
desc.info.guest_format = guest_format;
desc.info.type = type;
desc.info.extent = extent;
desc.info.resources = {1, layers};
desc.info.pitch = extent.width;
desc.info.bytes_per_block = bytes_per_block;
desc.info.samples = samples;
desc.info.tile_mode = Prospero::TileMode::kLinear;
desc.info.mip_layout[0] = {0, size, extent.width, extent.height};
desc.view_info.format = format;
desc.view_info.type = type == Prospero::ImageType::kColor3D ? vk::ImageViewType::e3D
: layers > 1 ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D;
desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
desc.view_info.layer_count = layers;
desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
return desc;
}
std::vector<u32> ReadCachedTexel(const char *name, RenderContext &context,
ImageId id, vk::Offset3D offset = {},
vk::Extent3D extent = {1, 1, 1}, uint32_t layer = 0,
uint32_t mip = 0) {
auto &scheduler = context.GetCommandScheduler();
auto &image = context.GetTextureCache().GetImage(id);
const auto bytes = image.info.bytes_per_block * extent.width *
extent.height * extent.depth;
auto probe = CreateHostBuffer(name, bytes,
vk::BufferUsageFlagBits::eTransferDst, {});
scheduler.Current().EndRendering();
image.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
scheduler.Current().Handle());
vk::BufferImageCopy copy{};
copy.imageSubresource = {image.info.IsDepth() ? vk::ImageAspectFlagBits::eDepth
: vk::ImageAspectFlagBits::eColor, mip, layer, 1};
copy.imageOffset = offset;
copy.imageExtent = extent;
scheduler.Current().Handle().copyImageToBuffer(
image.backing.image, vk::ImageLayout::eTransferSrcOptimal,
probe.buffer, 1, &copy);
vk::BufferMemoryBarrier barrier{};
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = probe.buffer;
barrier.size = probe.size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
nullptr);
scheduler.Finish();
auto result = ReadBuffer(name, probe, bytes / 4);
DestroyBuffer(&probe);
return result;
}
void CheckNativeIndirectDispatch() {
constexpr const char *name = "NativeIndirectDispatch";
constexpr uintptr_t base = 0x0000000204600000ull;
constexpr uint64_t allocation_size = 0x100000;
constexpr uint64_t case_size = 0x10000;
constexpr uint32_t sentinel = 0xa5a5a5a5u;
struct DispatchCase {
std::array<uint32_t, 3> dimensions;
uint32_t mode;
uint32_t expected_threads;
bool transfer = false;
};
constexpr std::array cases{
DispatchCase{{2, 1, 1}, 0x41u, 8},
DispatchCase{{0, 1, 1}, 0x41u, 0},
DispatchCase{{1, 0, 1}, 0x41u, 0},
DispatchCase{{1, 1, 0}, 0x41u, 0},
DispatchCase{{3, 1, 1}, 0x41u, 12, true},
DispatchCase{{8, 1, 1}, 0x61u, 8},
};
// Separate DWORD descriptors preserve two owners until the indirect argument
// range spans them. The consumer's output aliases the second owner.
std::vector<u32> writer;
for (u32 i = 0; i < 3; i++) {
writer.push_back(EncodeVop1(0x01u, 1, 12u + i));
writer.push_back(EncodeMubuf0(0x1cu, 0, false, false));
writer.push_back(EncodeMubuf1(1, i, 0));
}
AppendEnd(&writer);
std::vector<u32> consumer;
AppendVop3(&consumer, 0x346u, 1, 4, InlineU32(2), Vgpr(0));
consumer.push_back(EncodeVop2(0x25u, 2, InlineU32(1), 1));
consumer.push_back(EncodeVop2(0x1au, 1, InlineU32(2), 1));
consumer.push_back(EncodeMubuf0(0x1cu));
consumer.push_back(EncodeMubuf1(2, 0, 1));
AppendEnd(&consumer);
std::vector<u32> counted_consumer{
EncodeMubuf0(0x0cu, 0, false, false), EncodeMubuf1(3, 1, 0),
EncodeSopp(0x0c, 0), EncodeVop1(0x02, 2, Vgpr(3))};
AppendVop3(&counted_consumer, 0x346u, 1, 8, InlineU32(2), Vgpr(0));
AppendVop3(&counted_consumer, 0x36du, 2, Vgpr(1), Vgpr(3), InlineU32(1));
counted_consumer.push_back(EncodeMubuf0(0x1cu, 0, true, false));
counted_consumer.push_back(EncodeMubuf1(2, 0, 1));
AppendEnd(&counted_consumer);
for (const auto *code : {&writer, &consumer, &counted_consumer}) {
ShaderMapUserData(reinterpret_cast<uint64_t>(code->data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<uint32_t>(code->size() * sizeof(u32))});
}
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, case_size, 0, &direct_offset) == 0,
"indirect argument allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset, case_size) == 0 &&
mapped == reinterpret_cast<void *>(base),
"indirect argument mapping failed");
std::memset(mapped, 0xa5, allocation_size);
const auto argument_address = [&](size_t index) {
return base + index * case_size +
(cases[index].transfer ? 0x200u : BufferCache::CACHING_PAGESIZE - 4u);
};
for (size_t i = 0; i < cases.size(); i++) {
std::memset(reinterpret_cast<void *>(argument_address(i)), 0, 12);
}
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
context.GetGpu().SendCommandSync([&] {
CommandProcessor processor(context, 0);
processor.BufferInit();
auto &scheduler = context.GetCommandScheduler();
auto &cache = context.GetBufferCache();
auto &shaders = processor.GetShCtx();
context.MapMemory(base, allocation_size);
const auto set_buffer = [&](u32 sgpr, uint64_t address, u32 records, u32 stride = 0) {
ShaderBufferResource descriptor{};
descriptor.UpdateAddress48(address);
descriptor.fields[1] |= stride << 16u;
descriptor.fields[2] = records;
descriptor.fields[3] = DstSel(4, 5, 6, 7) |
(static_cast<u32>(Prospero::BufferFormat::k32UInt) << 12u);
for (u32 i = 0; i < 4; i++) {
shaders.SetCsUserSgpr(sgpr + i, descriptor.fields[i],
HW::UserSgprType::Unknown);
}
};
for (size_t index = 0; index < cases.size(); index++) {
const auto &test = cases[index];
const auto args = argument_address(index);
const auto output = base + index * case_size + BufferCache::CACHING_PAGESIZE + 0x100u;
if (test.transfer) {
for (u32 i = 0; i < 3; i++) {
auto [buffer, offset] = cache.ObtainBuffer(args + i * 4u, 4, true);
buffer->Fill(offset, 4, test.dimensions[i]);
}
} else {
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(writer.data()),
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 15});
for (u32 i = 0; i < 3; i++) {
set_buffer(i * 4u, args + i * 4u, 4);
shaders.SetCsUserSgpr(12u + i, test.dimensions[i], HW::UserSgprType::Unknown);
}
processor.DispatchDirect(1, 1, 1, 0x41u);
}
std::array<u32, 3> stale{};
Require(name, "GPU-only argument write",
LibKernel::Memory::TryReadBacking(args, stale.data(), sizeof(stale)) &&
stale == std::array<u32, 3>{},
"argument publication changed the CPU backing before indirect dispatch");
const auto first_owner = BufferCacheTestAccess::PageOwner(cache, args);
const auto second_owner = BufferCacheTestAccess::PageOwner(cache, args + 8u);
if (!test.transfer) {
Require(name, "separate argument owners",
first_owner && second_owner && first_owner != second_owner,
"the indirect argument merge fixture already shared one owner");
}
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(consumer.data()),
.num_thread_x = 4, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 4, .tgid_x_en = true});
set_buffer(0, output, 16u * sizeof(u32));
const auto tick = scheduler.CurrentTick();
if (test.transfer) {
const std::array<u32, 3> packet{static_cast<u32>(args),
static_cast<u32>(args >> 32u), test.mode};
Require(name, "absolute indirect packet",
CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3,
"the absolute indirect packet was not consumed");
} else {
processor.SetDispatchIndirectArgsBaseAddress(base + index * case_size);
const std::array<u32, 2> packet{
static_cast<u32>(BufferCache::CACHING_PAGESIZE - 4u), test.mode};
Require(name, "offset indirect packet",
CpOpDispatchIndirect(processor, 0xc0011600u, packet.data(), 0, 0) == 2,
"the offset indirect packet was not consumed");
}
if (test.mode == 0x41u) {
Require(name, "asynchronous indirect dispatch", scheduler.CurrentTick() == tick,
"workgroup-count arguments caused a host submission or readback");
if (!test.transfer) {
const auto merged = BufferCacheTestAccess::PageOwner(cache, args);
Require(name, "argument/output owner merge",
merged && merged != first_owner && merged != second_owner &&
merged == BufferCacheTestAccess::PageOwner(cache, output),
"argument discovery did not merge the consumer's buffer owner");
}
}
cache.ReadMemory(output, 16u * sizeof(u32));
std::array<u32, 16> actual{};
Require(name, "output readback",
LibKernel::Memory::TryReadBacking(output, actual.data(), sizeof(actual)),
"consumer output could not be read back");
for (u32 i = 0; i < actual.size(); i++) {
const auto expected = i < test.expected_threads ? i + 1u : sentinel;
Require(name, "indirect invocation coverage", actual[i] == expected,
"case " + std::to_string(index) + " word " + std::to_string(i) +
" expected " + Hex(expected) + ", got " + Hex(actual[i]));
}
}
// The same GPU-written DWORD supplies indirect X and the output descriptor's
// native NUM_RECORDS. Four lanes per group exercise the descriptor's OOB bound.
constexpr auto image_address = base + cases.size() * case_size;
constexpr auto count_args = image_address + 4u * sizeof(u32);
constexpr auto count_output = image_address + 2u * BufferCache::CACHING_PAGESIZE;
constexpr u32 image_value = 0x13579bdfu;
auto &textures = context.GetTextureCache();
auto image_desc = MakeLinearDesc(
image_address, 8u * sizeof(u32), vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{8, 1, 1}, 1, 4, 1);
const auto image = textures.FindImage(image_desc);
ShaderProgram previous_program;
for (const auto count : {3u, 7u}) {
cache.FillBuffer(count_output, 16u * sizeof(u32), sentinel, false);
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(writer.data()),
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 15});
for (u32 i = 0; i < 3; i++) {
set_buffer(i * 4u, count_args + i * 4u, 4);
shaders.SetCsUserSgpr(12u + i, i == 0 ? count : 1u,
HW::UserSgprType::Unknown);
}
processor.DispatchDirect(1, 1, 1, 0x41u);
Require(name, "GPU-owned descriptor count", cache.HasGpuDirtyBytes(count_args, 4),
"the descriptor count was already visible to the CPU");
// Raw buffer access keeps its own contents even after a newer overlapping image write.
vk::ClearValue painted{};
painted.color.uint32 = std::array{image_value + count, 0u, 0u, 0u};
TextureCacheTestAccess::ClearImage(
textures, scheduler.Current(), image,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
u32 stale_count = 0;
Require(name, "independent raw buffer and image contents",
textures.GetImage(image).SafeToDownload() &&
cache.HasGpuDirtyBytes(count_args, sizeof(u32)) &&
LibKernel::Memory::TryReadBacking(count_args, &stale_count,
sizeof(stale_count)) &&
stale_count != count,
"the alias fixture did not retain GPU buffer data and a newer independent image");
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(counted_consumer.data()),
.num_thread_x = 4, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 8, .tgid_x_en = true});
set_buffer(0, count_output, 0, 4);
set_buffer(4, count_args, 1, 4);
const std::array<u32, 3> packet{static_cast<u32>(count_args),
static_cast<u32>(count_args >> 32u), 0x41u};
const auto dirty_tick = scheduler.CurrentTick();
Require(name, "GPU count indirect dispatch",
CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3 &&
scheduler.CurrentTick() > dirty_tick &&
!cache.HasGpuDirtyBytes(count_args, 4),
"the descriptor dependency did not synchronize its preceding GPU writer");
const auto clean_tick = scheduler.CurrentTick();
ShaderComputeInputInfo input{};
const auto program = context.GetPipelineCache().GetComputeProgram(
shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
if (previous_program.id != 0) {
Require(name, "count permutation reuse", program.id == previous_program.id,
"a changing GPU count compiled a new shader permutation");
}
previous_program = program;
Require(name, "clean count repeat",
CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3 &&
scheduler.CurrentTick() == clean_tick,
"a clean descriptor dependency caused another GPU drain");
cache.ReadMemory(count_output, 16u * sizeof(u32));
std::array<u32, 16> actual{};
Require(name, "counted output readback",
LibKernel::Memory::TryReadBacking(count_output, actual.data(), sizeof(actual)),
"the counted consumer output could not be read back");
for (u32 i = 0; i < actual.size(); i++) {
Require(name, "native descriptor count bound",
actual[i] == (i < count ? count + i + 1u : sentinel),
"GPU count " + std::to_string(count) + " wrote an incorrect output word " +
std::to_string(i));
}
Require(name, "raw count preserves independent image contents",
ReadCachedTexel(name, context, image, {}, {8, 1, 1}) ==
std::vector<u32>(8, image_value + count),
"raw buffer descriptor materialization overwrote the newer native image");
}
// PPSA24156 loads an image sharp from a 148-byte record selected by
// WorkGroupID.z, then uses the same descriptor words for dimensions.
constexpr auto parameters = base + 0x80000;
constexpr auto table_address = parameters + 0x100;
std::vector<u32> table_shader{
EncodeSop2(0x26, 8, 4, 255), 148,
EncodeSmem0(0x02, 16), EncodeSmem1(0, 125),
EncodeSmem0(0x0b, 32, 8), EncodeSmem1(0, 8),
EncodeSop2(0x1e, 8, 34, InlineU32(2)),
EncodeSop2(0x20, 9, 33, InlineU32(30)),
EncodeSop2(0x0e, 8, 8, 255), 0xfffcu,
EncodeSop2(0x10, 8, 8, 9),
EncodeSop2(0x27, 9, 34, 255), 0x000e000eu,
EncodeSop2(0x02, 8, 8, 9),
EncodeSop2(0x02, 8, 8, InlineU32(2)),
EncodeVop1(0x01, 0, 4), EncodeVop1(0x01, 1, InlineU32(0)),
EncodeVop1(0x01, 2, 8),
EncodeMimg0(0x08, 1), EncodeMimg1(2, 0, 8)};
AppendEnd(&table_shader);
ShaderMapUserData(reinterpret_cast<uint64_t>(table_shader.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(table_shader.size() * sizeof(u32))});
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(table_shader.data()),
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 2, .tgid_x_en = true,
.tgid_y_en = true, .tgid_z_en = true});
shaders.SetCsUserSgpr(0, static_cast<u32>(parameters), HW::UserSgprType::Unknown);
shaders.SetCsUserSgpr(1, static_cast<u32>(parameters >> 32u), HW::UserSgprType::Unknown);
const ShaderBufferResource table_buffer{{static_cast<u32>(table_address),
static_cast<u32>(table_address >> 32u) | (148u << 16u), 2, 0x5204u}};
LibKernel::Memory::WriteBacking(parameters, table_buffer.fields, sizeof(table_buffer.fields));
struct TableCase { u32 groups, width, height, image; };
uint64_t table_program_id = 0;
for (const auto test : {TableCase{1, 4, 2, 0}, TableCase{2, 4, 2, 0},
TableCase{2, 8, 3, 1}, TableCase{1, 4, 2, 0}}) {
const auto address = base + 0x90000 + test.image * 0x10000;
const ShaderTextureResource sharp{{static_cast<u32>(address >> 8u),
(static_cast<u32>(Prospero::BufferFormat::k32UInt) << 20u) |
(((test.width - 1u) & 3u) << 30u),
((test.width - 1u) >> 2u) | ((test.height - 1u) << 14u),
DstSel(4, 5, 6, 7) |
(static_cast<u32>(Prospero::TileMode::kLinear) << 20u) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u),
0, 0x00700000u, 0, 0}};
std::array<u32, 74> table{};
for (u32 group = 0; group < test.groups; group++) {
std::copy_n(sharp.fields, 8, table.begin() + group * 37);
}
Require(name, "descriptor table refresh", context.InvalidateMemory(table_address, sizeof(table)),
"the descriptor table escaped its mapped allocation");
LibKernel::Memory::WriteBacking(table_address, table.data(), sizeof(table));
ShaderComputeInputInfo input{};
input.workgroup_counts[0] = input.workgroup_counts[1] = 1;
input.workgroup_counts[2] = test.groups;
const auto program = context.GetPipelineCache().GetComputeProgram(
shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
Require(name, "workgroup table refresh and coalescing",
input.workgroup_counts[2] == test.groups &&
input.stage.program->info.images.size() == 1 &&
input.stage.program->info.images[0].indirect_resources.empty() &&
input.stage.resources->images.size() == 1 &&
std::equal(std::begin(sharp.fields), std::end(sharp.fields),
input.stage.resources->images[0].dwords.begin()) &&
!input.stage.program->info.buffers.empty(),
"workgroup bounds, refreshed sharp, or live descriptor dimension reads were lost");
Require(name, "workgroup table program reuse",
table_program_id == 0 || program.id == table_program_id,
"changing the workgroup count or identical sharp values recompiled the shader");
table_program_id = program.id;
const auto *snapshot = input.stage.resources;
const auto *image_storage = snapshot->images.data();
const auto flattened_capacity = snapshot->flattened_srt.capacity();
const auto repeated = context.GetPipelineCache().GetComputeProgram(
shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
Require(name, "workgroup table cache-hit storage reuse",
repeated.id == program.id && input.stage.resources == snapshot &&
snapshot->images.data() == image_storage &&
snapshot->flattened_srt.capacity() == flattened_capacity,
"a repeated descriptor table hit replaced its snapshot or storage");
auto binding = RenderExecutorTestAccess::ResolveTexture(
context.GetRenderExecutor(), input.stage.program->info.images[0],
input.stage.resources->images[0]);
const auto target = textures.FindImage(binding.desc);
TextureCacheTestAccess::ClearImage(textures, scheduler.Current(), target,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
processor.DispatchDirect(1, 1, test.groups, 0x41u);
const auto pixels = ReadCachedTexel(name, context, target, {},
{test.width, test.height, 1});
for (u32 pixel = 0; pixel < pixels.size(); pixel++) {
const auto expected = pixel < test.groups ? test.width + test.height : 0u;
Require(name, "workgroup table GPU dimension reads", pixels[pixel] == expected,
"a retained descriptor load used stale dimensions or the wrong record stride");
}
}
// Exchange values across two waves in the last 512 bytes of each
// workgroup's 64-KiB LDS, on a host restricted to 48-KiB native shared memory.
constexpr auto lds_output = base + 0xb0000;
constexpr auto lds_args = base + 0xc0000;
constexpr u32 lds_words = 8 * 128;
std::vector<u32> lds_shader{
EncodeSop2(0x1e, 8, 5, InlineU32(1)),
EncodeSop2(0x1e, 9, 6, InlineU32(2)),
EncodeSop2(0x02, 8, 8, 9), EncodeSop2(0x02, 8, 8, 4)};
AppendVop3(&lds_shader, 0x346, 2, 8, InlineU32(7), Vgpr(0));
lds_shader.push_back(EncodeVop2(0x1a, 5, InlineU32(2), 2));
lds_shader.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2));
lds_shader.push_back(EncodeVop2(0x1a, 1, InlineU32(2), 0));
AppendVMovLiteral(&lds_shader, 7, 64 * 1024 - 128 * 4);
lds_shader.insert(lds_shader.end(), {
EncodeVop2(0x25, 1, Vgpr(7), 1),
EncodeDs0(0x0d), EncodeDs1(0, 2, 1), EncodeSopp(0x0a, 0),
EncodeVop2(0x1d, 3, InlineU32(64), 0),
EncodeVop2(0x1a, 3, InlineU32(2), 3), EncodeVop2(0x25, 3, Vgpr(7), 3),
EncodeDs0(0x36), EncodeDs1(4, 0, 3),
EncodeMubuf0(0x1c), EncodeMubuf1(4, 0, 5)});
AppendEnd(&lds_shader);
ShaderMapUserData(reinterpret_cast<uint64_t>(lds_shader.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(lds_shader.size() * sizeof(u32))});
auto &host_lds_limit = context.GetGraphics().physical_device_properties.limits.maxComputeSharedMemorySize;
const auto saved_lds_limit = host_lds_limit;
host_lds_limit = std::min(host_lds_limit, 48u * 1024u);
for (u32 dispatch = 0; dispatch < 4; dispatch++) {
cache.FillBuffer(lds_output, lds_words * sizeof(u32), sentinel, false);
const bool indirect = dispatch == 1 || dispatch == 2;
const bool empty = dispatch == 2;
if (indirect) {
const auto groups = empty ? std::array{4096u, 0u, 1u} : std::array{2u, 2u, 2u};
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(writer.data()),
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 15});
for (u32 axis = 0; axis < 3; axis++) {
set_buffer(axis * 4, lds_args + axis * sizeof(u32), sizeof(u32));
shaders.SetCsUserSgpr(12 + axis, groups[axis], HW::UserSgprType::Unknown);
}
processor.DispatchDirect(1, 1, 1, 0x41u);
Require(name, "GPU-authored LDS dispatch arguments",
cache.HasGpuDirtyBytes(lds_args, 3 * sizeof(u32)),
"indirect LDS dimensions were already visible on the CPU");
}
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(lds_shader.data()),
.num_thread_x = 128, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 4, .tgid_x_en = true,
.tgid_y_en = true, .tgid_z_en = true, .lds_size = 128});
set_buffer(0, lds_output, lds_words * sizeof(u32));
if (indirect) {
const std::array<u32, 3> packet{static_cast<u32>(lds_args),
static_cast<u32>(lds_args >> 32u), 0x41u};
Require(name, "indirect storage LDS dispatch",
CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3,
"the GPU-authored LDS dispatch was not consumed");
} else {
processor.DispatchDirect(2, 2, 2, 0x41u);
}
cache.ReadMemory(lds_output, lds_words * sizeof(u32));
std::array<u32, lds_words> actual{};
Require(name, "storage LDS readback",
LibKernel::Memory::TryReadBacking(lds_output, actual.data(), sizeof(actual)),
"LDS result buffer could not be read back");
for (u32 word = 0; word < actual.size(); word++) {
const auto expected = empty ? sentinel : (word ^ 64u) + 1u;
Require(name, "storage LDS upper range, isolation and barrier", actual[word] == expected,
"64-KiB LDS lost upper bytes, workgroup isolation, or cross-wave writes");
}
}
host_lds_limit = saved_lds_limit;
context.UnmapMemory(base, allocation_size);
scheduler.Finish();
});
LibKernel::Memory::InstallGpuResources(nullptr);
context.ShutdownGpu();
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"indirect argument mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"indirect argument allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColorMetadataClear() {
constexpr const char *name = "RenderExecutorColorMetadataClear";
struct PackedCase {
vk::Format format;
uint32_t word;
std::array<float, 4> channels;
};
const std::array packed_cases{
PackedCase{vk::Format::eR16Sfloat, 0x7bff7bffu, {65504.0f}},
PackedCase{vk::Format::eR16G16Sfloat, 0xbc003c00u, {1.0f, -1.0f}},
PackedCase{vk::Format::eR16Sfloat, 0xffff0001u, {0x1p-24f}},
PackedCase{vk::Format::eR16Sfloat, 0x00008001u, {-0x1p-24f}},
PackedCase{vk::Format::eR16Sfloat, 0x000003ffu, {1023.0f * 0x1p-24f}},
PackedCase{vk::Format::eR16Sfloat, 0x00000400u, {0x1p-14f}},
PackedCase{vk::Format::eR16G16Sfloat, 0x80000000u, {0.0f, -0.0f}},
PackedCase{vk::Format::eR16G16Sfloat, 0xfc007c00u,
{std::numeric_limits<float>::infinity(),
-std::numeric_limits<float>::infinity()}},
PackedCase{vk::Format::eR16Unorm, 0x0000ffffu, {1.0f}},
PackedCase{vk::Format::eR16G16Unorm, 0x0000ffffu, {1.0f, 0.0f}},
PackedCase{vk::Format::eR16G16Unorm, 0x80000001u,
{1.0f / 65535.0f, 32768.0f / 65535.0f}},
};
for (const auto &test : packed_cases) {
vk::ClearColorValue clear{};
Require(name, "packed 16-bit clear",
DecodePackedColorClear(test.format, test.word, clear) &&
std::bit_cast<std::array<uint32_t, 4>>(clear.float32) ==
std::bit_cast<std::array<uint32_t, 4>>(test.channels),
"16-bit clear lost channel order, sign, range or unused-channel zeros");
}
vk::ClearColorValue clear{};
Require(name, "half NaN clear",
DecodePackedColorClear(vk::Format::eR16Sfloat, 0x7e00u, clear) &&
std::isnan(clear.float32[0]),
"half NaN did not remain NaN");
for (const auto format : {vk::Format::eR16Sfloat, vk::Format::eR16Unorm}) {
Require(name, "R16 DWORD fill",
!DecodeColorDwordFill(format, 0xbc003c00u, clear) &&
DecodeColorDwordFill(format, 0x3c003c00u, clear),
"alternating R16 pixels were treated as a uniform clear");
}
constexpr uintptr_t base = 0x0000000204100000ull;
constexpr uint64_t allocation_size = 0x200000;
constexpr uint64_t allocation_alignment = 0x10000;
struct FillCase {
uint32_t fill;
std::array<uint32_t, 2> texel;
bool reuse_unorm = false;
bool cmask = false;
bool reuse_mips = false;
Prospero::ChannelLayout layout = Prospero::ChannelLayout::k16_16_16_16;
Prospero::ChannelType type = Prospero::ChannelType::kFloat;
uint32_t clear_word = 0;
};
constexpr std::array cases{
FillCase{0x40404040u, {0, 0x3c000000u}},
FillCase{0x80808080u, {0x3c003c00u, 0x00003c00u}},
FillCase{0x40404040u, {0, 0x3c000000u}, true},
FillCase{0, {0x804020ffu, 0}, false, true},
FillCase{0x40404040u, {0, 0x3c000000u}, false, false, true},
FillCase{.fill = 0x20202020u, .texel = {0x7bff7bffu},
.layout = Prospero::ChannelLayout::k16, .clear_word = 0x7bff7bffu},
FillCase{.fill = 0x20202020u, .texel = {0xbc003c00u},
.layout = Prospero::ChannelLayout::k16_16, .clear_word = 0xbc003c00u},
FillCase{.fill = 0x20202020u, .texel = {0x80008000u},
.layout = Prospero::ChannelLayout::k16,
.type = Prospero::ChannelType::kUNorm, .clear_word = 0x80008000u},
FillCase{.fill = 0x20202020u, .texel = {0x0000ffffu},
.layout = Prospero::ChannelLayout::k16_16,
.type = Prospero::ChannelType::kUNorm, .clear_word = 0x0000ffffu},
};
EnsureRuntimeContext();
// Astro's generic metadata fill, through S_ENDPGM; trailing debug data is omitted.
static constexpr std::array<uint32_t, 69> native_fill{
0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u,
0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu,
0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u,
0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u,
0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u,
0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u,
0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d860609u,
0x7d8c02f9u, 0x06068c02u, 0x87ea6a0cu, 0x50000080u,
0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u,
0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u,
0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au,
0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u,
0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au,
0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u,
0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u,
0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u,
0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u,
0xbf810000u,
};
static constexpr auto raw_fill = [] {
auto words = native_fill;
for (auto &word : words) {
if (word == 0xe0102000u) {
word = 0xe0702000u; // BUFFER_STORE_DWORD in place of BUFFER_STORE_FORMAT_X.
}
}
return words;
}();
for (const auto *shader : {&native_fill, &raw_fill}) {
ShaderMapUserData(reinterpret_cast<uint64_t>(shader->data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = sizeof(native_fill)});
}
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"fixed-clear direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"fixed-clear direct mapping failed");
std::memset(mapped, 0, allocation_size);
for (const auto &fill_case : cases) {
const auto layout = fill_case.cmask ? Prospero::ChannelLayout::k8_8_8_8 : fill_case.layout;
const auto type = fill_case.cmask ? Prospero::ChannelType::kUNorm : fill_case.type;
const auto bytes_per_pixel = TextureGetRenderTargetFormat(
layout, type, Prospero::ChannelOrder::kStandard).bytes_per_element;
const uint32_t probe_width = bytes_per_pixel == 2 ? 2 : 1;
TileSizeAlign dcc_size{};
(void)TileGetDccSize(512, 256, 1, bytes_per_pixel, 1,
Prospero::TileMode::kRenderTarget, dcc_size, 0);
const uint64_t dcc_address = base + (fill_case.reuse_mips ? 0x160000 : 0x100000);
const uint32_t selected_layer = fill_case.cmask ? 1 : 0;
const uint64_t metadata_size = fill_case.cmask ? 0x2000 : dcc_size.size;
const std::vector<u32> expected(
fill_case.texel.begin(), fill_case.texel.begin() + std::max(bytes_per_pixel / 4u, 1u));
RenderContext context(m_runtime_context);
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
context.GetGpu().SendCommandSync([&] {
auto &scheduler = context.GetCommandScheduler();
registers.SetColorBase(0, {.addr = base});
registers.SetColorInfo(
0, {.cmask_fast_clear_enable = !fill_case.reuse_unorm,
.dcc_compression_enable = !fill_case.cmask,
.format = layout,
.channel_type = type,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorView(0, {.base_array_slice_index = selected_layer,
.last_array_slice_index = selected_layer});
registers.SetColorAttrib2(0, {.height = 255, .width = 511});
registers.SetColorAttrib3(0,
{.tile_mode = Prospero::TileMode::kRenderTarget,
.dimension = 1,
.metadata_pipe_aligned = true});
registers.SetColorDccAddr(0, {.addr = dcc_address});
registers.SetColorCmask(0, {.addr = dcc_address});
registers.SetColorClearWord0(0, {.word0 = fill_case.cmask ? expected[0] : fill_case.clear_word});
registers.SetColorClearWord1(0, {.word1 = 0});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
ImageId preexisting_id{};
if (fill_case.reuse_unorm || fill_case.reuse_mips) {
const auto float_info = registers.GetRenderTarget(0).info;
const auto single_mip = registers.GetRenderTarget(0).attrib2;
auto initial_info = float_info;
initial_info.dcc_compression_enable = false;
initial_info.cmask_fast_clear_enable = false;
if (fill_case.reuse_unorm) {
initial_info.channel_type = Prospero::ChannelType::kUNorm;
}
auto initial_mips = single_mip;
initial_mips.num_mip_levels = fill_case.reuse_mips ? 1 : 0;
registers.SetColorInfo(0, initial_info);
registers.SetColorAttrib2(0, initial_mips);
RenderColorInfo preexisting{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), preexisting, 0);
preexisting_id = preexisting.image_id;
(void)texture_cache.FindRenderTarget(preexisting_id, preexisting.desc);
if (fill_case.reuse_mips) {
Require(name, "mip chain excludes metadata",
preexisting.desc.info.data.address + preexisting.desc.info.data.size <=
dcc_address,
"the mip preservation fixture overlaps its color metadata");
vk::ClearValue clear{};
clear.color.float32 = std::array{1.0f, 0.0f, 1.0f, 0.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), preexisting_id,
{vk::ImageAspectFlagBits::eColor, 0, 2, 0, 1}, clear);
}
RenderExecutorTestAccess::ResetBindings(executor);
registers.SetColorInfo(0, float_info);
registers.SetColorAttrib2(0, single_mip);
}
const auto fill_metadata = [&](uint32_t count, bool raw = false) {
const auto *shader = &native_fill;
if (raw) {
shader = &raw_fill;
}
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(shader->data()),
.num_thread_x = 64, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 10, .tgid_x_en = true});
const std::array<uint32_t, 10> user_data{
static_cast<uint32_t>(dcc_address),
static_cast<uint32_t>(dcc_address >> 32u) | (4u << 16u),
count, 0x00014004u, fill_case.fill, 0, 0x200, 0, count, 1};
for (uint32_t i = 0; i < user_data.size(); i++) {
shaders.SetCsUserSgpr(i, user_data[i], HW::UserSgprType::Unknown);
}
executor.DispatchDirect(0, scheduler.Current(), (count + 63) / 64, 1, 1, 0x41u);
};
const auto metadata_words = static_cast<uint32_t>(metadata_size / 4);
fill_metadata(metadata_words);
Require(name, "metadata before descriptor",
!texture_cache.IsMeta(dcc_address),
"the native fill guessed an unbound metadata allocation's type");
RenderColorInfo color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
RenderDepthInfo no_depth{};
const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &color, 1, no_depth);
if (fill_case.reuse_unorm) {
Require(name, "FLOAT clear reuses UNORM image",
color.image_id == preexisting_id &&
texture_cache.GetImage(preexisting_id).backing.format ==
vk::Format::eR16G16B16A16Unorm,
"the aliased clear did not reuse its existing UNORM allocation");
}
Require(name, "color clear metadata discovery",
color.image_id &&
color.desc.info.metadata.kind ==
(fill_case.cmask ? ImageMetadataKind::Cmask : ImageMetadataKind::Dcc) &&
color.desc.info.metadata.range.address == dcc_address &&
color.desc.info.metadata.range.size == metadata_size &&
rendering.num_color_attachments == 1 &&
!texture_cache.IsMeta(dcc_address),
"color metadata did not retain its type and full native allocation");
const auto read_texel = [&] {
return ReadCachedTexel(name, context, color.image_id, {}, {probe_width, 1, 1}, selected_layer);
};
Require(name, "GPU color clear value",
read_texel() == expected &&
ReadCachedTexel(name, context, color.image_id,
{static_cast<int32_t>(512 - probe_width), 255, 0},
{probe_width, 1, 1}, selected_layer) == expected,
"native metadata did not materialize the expected color");
if (fill_case.reuse_mips) {
const auto &image = texture_cache.GetImage(color.image_id);
Require(name, "single-mip clear reuses mip chain",
color.image_id == preexisting_id && image.info.resources.levels == 2 &&
color.desc.info.resources.levels == 1 &&
color.desc.view_info.base_level == 0 && color.desc.view_info.level_count == 1,
"the single-mip target did not retain its cached image and view scope");
Require(name, "single-mip clear preserves higher mip",
ReadCachedTexel(name, context, color.image_id, {255, 127, 0},
{1, 1, 1}, 0, 1) ==
std::vector<u32>{0x00003c00u, 0x00003c00u},
"clearing mip zero changed its higher mip");
}
const auto check_expanded_metadata = [&] {
context.GetBufferCache().ReadMemory(dcc_address, metadata_size, false);
std::vector<uint8_t> bytes(metadata_size);
Require(name, "expanded metadata readback",
LibKernel::Memory::TryReadBacking(dcc_address, bytes.data(), bytes.size()),
"native color metadata could not be read back");
const auto selected = bytes.begin() + (fill_case.cmask ? 0x1000 : 0);
Require(name, "selected metadata slice expansion",
std::all_of(bytes.begin(), selected, [](uint8_t v) { return v == 0; }) &&
std::all_of(selected, bytes.end(), [](uint8_t v) { return v == 0xff; }),
"materialization retained a clear key or changed an unselected slice");
};
check_expanded_metadata();
const auto bind = [&] {
RenderExecutorTestAccess::ResetBindings(executor);
color.image_id = texture_cache.FindImage(color.desc);
(void)texture_cache.FindRenderTarget(color.image_id, color.desc);
};
const auto paint = [&] {
vk::ClearValue clear{};
clear.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, selected_layer, 1}, clear);
};
std::vector<u32> painted{0x00003c00u, 0x3c003c00u};
if (fill_case.reuse_unorm) {
painted = {0x0000ffffu, 0xffffffffu};
} else if (fill_case.cmask) {
painted = {0xffff00ffu};
} else if (bytes_per_pixel <= 4) {
const uint32_t one = type == Prospero::ChannelType::kFloat ? 0x3c00u : 0xffffu;
painted = {bytes_per_pixel == 2 ? one | (one << 16u) : one};
}
const auto retains_painted_texel = [&] {
const auto texels = ReadCachedTexel(name, context, color.image_id, {},
{512, 256, 1}, selected_layer);
for (size_t i = 0; i < texels.size(); i += painted.size()) {
if (std::equal(painted.begin(), painted.end(), texels.begin() + i)) {
return true;
}
}
return false;
};
paint();
bind();
Require(name, "unchanged metadata preserves rendering",
read_texel() == painted,
"rebinding unchanged color metadata reapplied an old clear");
if (fill_case.cmask) {
WriteMetadata(context, dcc_address, metadata_size, 0x40404040u);
bind();
Require(name, "nonzero CMASK is not a DCC clear", read_texel() == painted,
"CMASK bytes were decoded as a DCC fixed-color key");
}
WriteMetadata(context, dcc_address, metadata_size, fill_case.fill);
bind();
Require(name, "CPU metadata re-clear", read_texel() == expected,
"CPU metadata writes did not clear an already drawn target");
paint();
fill_metadata(metadata_words - 1);
bind();
Require(name, "partial same-value metadata fill",
retains_painted_texel(),
"a partial same-value write reused stale clear bytes after rendering");
fill_metadata(metadata_words, true);
bind();
Require(name, "same-value raw native re-clear",
read_texel() == expected,
"a repeated metadata fill did not clear an already drawn target");
paint();
WriteMetadata(context, dcc_address, metadata_size, UINT32_MAX);
fill_metadata(metadata_words - 1);
bind();
Require(name, "partial native metadata fill",
retains_painted_texel(),
"a nonuniform metadata allocation cleared the whole target");
fill_metadata(metadata_words);
bind();
Require(name, "complete fill after partial fill",
read_texel() == expected,
"completing the metadata overwrite did not restore its clear");
// Consuming a clear key updates metadata backing without invalidating
// a pooled image whose first texel precedes the metadata range.
constexpr uint32_t alias_value = 0x13579bdfu;
constexpr uint32_t expanded_words = 0x1000 / sizeof(uint32_t);
const auto selected_metadata = dcc_address + metadata_size - 0x1000;
ImageDesc metadata_alias{};
metadata_alias.info.data = {dcc_address - 0x1000, 0x4000};
metadata_alias.info.pixel_format = vk::Format::eR32Uint;
metadata_alias.info.guest_format = Prospero::BufferFormat::k32UInt;
metadata_alias.info.extent = {4096, 1, 1};
metadata_alias.info.pitch = 4096;
metadata_alias.info.bytes_per_block = sizeof(uint32_t);
metadata_alias.info.mip_layout[0] = {0, 0x4000, 4096, 1};
metadata_alias.view_info.format = vk::Format::eR32Uint;
const auto metadata_alias_id = texture_cache.FindImage(metadata_alias);
vk::ClearValue alias_paint{};
alias_paint.color.uint32[0] = alias_value;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), metadata_alias_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, alias_paint);
paint();
fill_metadata(metadata_words);
bind();
Require(name, "pooled metadata clear target", read_texel() == expected,
"the overlapping native image prevented the metadata clear");
auto metadata_source = resources.GetBufferCache().ObtainBuffer(
selected_metadata, 0x1000, false);
Require(name, "expanded metadata buffer source", metadata_source.first != nullptr,
"expanded metadata did not produce a GPU-readable buffer");
auto metadata_readback = CreateHostBuffer(
name, 0x1000, vk::BufferUsageFlagBits::eTransferDst, {});
vk::MemoryBarrier metadata_barrier{};
metadata_barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite;
metadata_barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eTransfer,
{}, 1, &metadata_barrier, 0, nullptr, 0, nullptr);
const vk::BufferCopy metadata_copy{metadata_source.second, 0, 0x1000};
scheduler.Current().Handle().copyBuffer(metadata_source.first->Handle(),
metadata_readback.buffer, 1, &metadata_copy);
metadata_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
metadata_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost,
{}, 1, &metadata_barrier, 0, nullptr, 0, nullptr);
scheduler.Finish();
Require(name, "expanded metadata GPU reader",
ReadBuffer(name, metadata_readback, expanded_words) ==
std::vector<u32>(expanded_words, UINT32_MAX),
"a later GPU buffer read retained the consumed clear key");
DestroyBuffer(&metadata_readback);
(void)texture_cache.FindTexture(metadata_alias_id, metadata_alias);
Require(name, "metadata consumption preserves pooled image",
ReadCachedTexel(name, context, metadata_alias_id) ==
std::vector<u32>{alias_value},
"internal metadata publication replaced an untouched GPU image texel");
ImageDesc previous_depth{};
previous_depth.type = BindingType::DepthTarget;
previous_depth.info.data = {base + 0x180000, 0x10000};
previous_depth.info.pixel_format = vk::Format::eD32Sfloat;
previous_depth.info.guest_format = Prospero::BufferFormat::k32Float;
previous_depth.info.type = Prospero::ImageType::kColor2D;
previous_depth.info.extent = {128, 128, 1};
previous_depth.info.resources = {1, 1};
previous_depth.info.pitch = 128;
previous_depth.info.bytes_per_block = 4;
previous_depth.info.samples = 1;
previous_depth.info.tile_mode = Prospero::TileMode::kLinear;
previous_depth.info.mip_layout[0] = {0, 0x10000, 128, 128};
previous_depth.info.metadata.kind = ImageMetadataKind::Htile;
previous_depth.info.metadata.range = {dcc_address, metadata_size};
previous_depth.info.htile_clear_mask = 0;
previous_depth.view_info.format = vk::Format::eD32Sfloat;
previous_depth.view_info.type = vk::ImageViewType::e2D;
previous_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
previous_depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto previous_depth_id = texture_cache.FindImage(previous_depth);
Require(name, "old HTile interpretation",
texture_cache.FindDepthTarget(previous_depth_id, previous_depth) != nullptr &&
texture_cache.IsMeta(dcc_address),
"the old depth target did not register its metadata allocation");
paint();
bind();
Require(name, "HTile allocation reused as expanded color metadata",
!texture_cache.IsMeta(dcc_address) && !texture_cache.ClearMeta(dcc_address) &&
read_texel() == painted,
"color discovery retained the old HTile classification or erased rendering");
fill_metadata(metadata_words);
bind();
Require(name, "native color clear after HTile reuse", read_texel() == expected,
"the former HTile entry swallowed the native color metadata fill");
check_expanded_metadata();
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
});
context.ShutdownGpu();
LibKernel::Memory::InstallGpuResources(nullptr);
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"fixed-clear direct mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"fixed-clear direct-memory allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckSampledDccClear() {
constexpr const char *name = "SampledDccClear";
constexpr uintptr_t base = 0x0000000205000000ull;
constexpr uint64_t color_size = 0x3fc0000;
constexpr uint64_t metadata_size = 0x48000;
constexpr uint64_t dcc_address = base + color_size;
constexpr uint64_t allocation_size = 0x4020000;
constexpr uint64_t alignment = 0x10000;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, alignment, 0, &direct_offset) == 0,
"captured scene allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset, alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"captured scene mapping failed");
for (const auto initial_size : {metadata_size - 0x1000, metadata_size}) {
std::memset(mapped, 0x5a, color_size);
std::memset(reinterpret_cast<void *>(dcc_address), 0xff, metadata_size);
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
WriteMetadata(context, dcc_address, initial_size, 0x40404040u);
Require(name, "deferred metadata fill",
!cache.IsMeta(dcc_address),
"unknown metadata was consumed before descriptor discovery");
// PPSA19244's first sampled scene descriptor, with relocated allocations.
ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
0xc4700000u, 0x021bc3bfu, 0x91b00facu, 0, 0x00700000u,
0x006b0000u | (static_cast<uint32_t>(dcc_address >> 8u) << 24u),
static_cast<uint32_t>(dcc_address >> 16u)}};
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = true;
auto binding = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
Require(name, "sampled metadata discovery",
binding.desc.info.extent == vk::Extent3D{3840, 2160, 1} &&
binding.desc.info.data.size == color_size &&
binding.desc.info.metadata.kind == ImageMetadataKind::Dcc &&
binding.desc.info.metadata.range.address == dcc_address &&
binding.desc.info.metadata.range.size == metadata_size,
"captured sampled descriptor lost its DCC allocation");
Require(name, "sampled view", cache.FindTexture(binding.image_id, binding.desc) != nullptr,
"captured scene did not produce a sampled view");
if (initial_size != metadata_size) {
Require(name, "partial metadata coverage",
ReadCachedTexel(name, context, binding.image_id) ==
std::vector<u32>{0x5a5a5a5au, 0x5a5a5a5au},
"an incomplete metadata fill cleared the whole scene");
WriteMetadata(context, dcc_address, metadata_size, 0x40404040u);
binding.image_id = cache.FindImage(binding.desc);
(void)cache.FindTexture(binding.image_id, binding.desc);
}
Require(name, "opaque black before first sample",
ReadCachedTexel(name, context, binding.image_id, {3839, 2159, 0}) ==
std::vector<u32>{0, 0x3c000000u},
"sampling read stale color bytes instead of the DCC opaque black clear");
context.GetBufferCache().ReadMemory(dcc_address, metadata_size);
std::vector<uint8_t> expanded_metadata(metadata_size);
Require(name, "sampled clear expands native metadata",
LibKernel::Memory::TryReadBacking(dcc_address, expanded_metadata.data(),
expanded_metadata.size()) &&
std::all_of(expanded_metadata.begin(), expanded_metadata.end(),
[](uint8_t byte) { return byte == 0xff; }),
"the sampled clear retained native clear keys after materialization");
WriteMetadata(context, dcc_address, metadata_size, 0x80808080u);
binding.image_id = cache.FindImage(binding.desc);
(void)cache.FindTexture(binding.image_id, binding.desc);
Require(name, "later sampled clear",
ReadCachedTexel(name, context, binding.image_id) ==
std::vector<u32>{0x3c003c00u, 0x00003c00u},
"an existing sampled image retained the previous DCC clear");
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
context.ShutdownGpu();
LibKernel::Memory::InstallGpuResources(nullptr);
}
Require(name, "unmap",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"captured scene mapping release failed");
Require(name, "release",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"captured scene allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckPackedTextureComponents() {
constexpr const char *name = "PackedTextureComponents";
constexpr uintptr_t base = 0x0000000204200000ull;
constexpr uint64_t allocation_size = 0x10000;
struct FormatCase {
Prospero::BufferFormat format;
std::array<uint16_t, 4> words;
std::array<std::array<uint32_t, 4>, 4> channels;
std::array<uint32_t, 4> maxima;
};
// Packed format channels start at the least significant bit.
// PPSA24515's blue, white, and masked black exposed the 5551 reversal.
constexpr std::array cases{
FormatCase{Prospero::BufferFormat::k5_5_5_1UNorm,
{0x7c00, 0x7fff, 0x8000, 0x8c41},
{{{0, 0, 31, 0}, {31, 31, 31, 0}, {0, 0, 0, 1}, {1, 2, 3, 1}}},
{31, 31, 31, 1}},
FormatCase{Prospero::BufferFormat::k4_4_4_4UNorm,
{0x000f, 0x00f0, 0x0f00, 0x4321},
{{{15, 0, 0, 0}, {0, 15, 0, 0}, {0, 0, 15, 0}, {1, 2, 3, 4}}},
{15, 15, 15, 15}},
};
const auto argb = TextureGetComponentMapping(DstSel(5, 1, 7, 0),
Prospero::ColorMappingArgb);
Require(name, "inverse mapping",
argb == vk::ComponentMapping{vk::ComponentSwizzle::eB,
vk::ComponentSwizzle::eOne, vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eZero},
"cyclic host mapping reordered output slots or descriptor constants");
const auto target = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k1_5_5_5, Prospero::ChannelType::kUNorm,
Prospero::ChannelOrder::kStandard);
Require(name, "1555 render target",
target.format == vk::Format::eR5G5B5A1UnormPack16 &&
target.export_mapping == Prospero::ColorMappingAbgr,
"shared format resolution changed the existing 1555 render target");
for (const auto native : {target, TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k8, Prospero::ChannelType::kSNorm,
Prospero::ChannelOrder::kStandard)}) {
const auto upload = TextureCalcUploadLayout(native.guest_format, 4, 1, 1, 1,
Prospero::TileMode::kLinear, 256, false, name);
Require(name, "render-target guest upload layout",
upload.pitch * native.bytes_per_element == 256 && upload.mips[0].size == 256,
"retaining the render-target format lost its physical byte width");
}
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"packed texture allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
"packed texture mapping failed");
for (const auto &format : cases) {
std::memset(mapped, 0, allocation_size);
std::memcpy(mapped, format.words.data(), sizeof(format.words));
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
for (const auto swizzle : {DstSel(4, 5, 6, 7), DstSel(7, 1, 4, 0)}) {
ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
(static_cast<uint32_t>(format.format) << 20u) | (3u << 30u), 0,
swizzle | (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u),
0, 0x00700000u, 0, 0}};
TestCase test;
test.name = name;
test.has_user_data = true;
test.image_descriptor_swizzle = swizzle;
std::copy_n(descriptor.fields, 8, test.user_data.begin());
test.user_data[50] = 16 * sizeof(uint32_t);
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_LOAD,
ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
for (uint32_t pixel = 0; pixel < format.words.size(); ++pixel) {
AppendVMovU32(&test.code, 20, pixel);
AppendVMovU32(&test.code, 21, 0);
test.code.push_back(EncodeMimg0(0x00, 0xf));
test.code.push_back(EncodeMimg1(0, 20));
for (uint32_t channel = 0; channel < 4; ++channel) {
AppendStoreVgpr(&test.code, channel, pixel * 4 + channel);
}
}
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
const auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images.at(0), value);
Image sampled;
sampled.view = cache.FindTexture(binding.image_id, binding.desc);
sampled.layout = cache.GetImage(binding.image_id).backing.state.layout;
scheduler.Finish();
auto output = CreateStorageBuffer(name, {}, 16);
Dispatch(test, compiled, output, nullptr, &sampled);
const auto actual = ReadBuffer(name, output, 16);
for (uint32_t pixel = 0; pixel < format.words.size(); ++pixel) {
for (uint32_t channel = 0; channel < 4; ++channel) {
const auto selector = GetDstSel(swizzle, channel);
const auto maximum = selector < 4 ? 1u : format.maxima[selector - 4];
const auto expected = selector < 4 ? selector : format.channels[pixel][selector - 4];
const float observed = std::bit_cast<float>(actual[pixel * 4 + channel]);
// Recover the source bits without testing the host's UNORM rounding precision.
const bool matches = maximum == 1 ? observed == expected
: observed >= 0.0f && observed <= 1.0f &&
std::round(observed * maximum) == expected;
if (!matches) {
std::ostringstream message;
message << "format=" << static_cast<uint32_t>(format.format)
<< " swizzle=" << swizzle << " word=" << format.words[pixel]
<< " channel=" << channel << " expected=" << expected
<< " actual=" << observed;
Fail(name, "GPU components", message.str());
}
}
}
DestroyBuffer(&output);
}
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
for (const auto tile : {Prospero::TileMode::kLinear, Prospero::TileMode::kRenderTarget}) {
constexpr std::array<u32, 4> words{0x883005e1u, 0x7c0e81c0u,
0x800f0000u, 0x98120280u};
constexpr std::array<std::array<float, 4>, 4> channels{{
{1.03125f, 2.03125f, 4.0625f, 1.f}, {0.5f, 0.75f, 1.5f, 1.f},
{0.f, 1.f, 2.f, 1.f}, {32.f, 8.f, 16.f, 1.f}}};
const auto target = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k10_11_11, Prospero::ChannelType::kFloat,
Prospero::ChannelOrder::kReversed);
Require(name, "10_11_11 reversed render target",
target.format == vk::Format::eB10G11R11UfloatPack32 &&
target.guest_format == Prospero::BufferFormat::k10_11_11Float &&
target.export_mapping == Prospero::ColorMappingRgba,
"PPSA24156 reversed packed-float target lost its physical encoding");
std::memset(mapped, 0, allocation_size);
std::array<u32, 4> offsets{0, 4, 8, 12};
if (tile == Prospero::TileMode::kRenderTarget) {
TileBlockLayout block{};
u32 block_xor = 0;
Require(name, "10_11_11 tile layout",
TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, 4, block) &&
TileGetBlockXor(block, 0, 0, 0, block_xor),
"packed-float render-target layout is unavailable");
for (u32 pixel = 0; pixel < words.size(); ++pixel) {
Require(name, "10_11_11 tile offset",
TileGetBlockOffset(block, pixel, 0, 0, offsets[pixel]),
"packed-float texel offset is unavailable");
offsets[pixel] ^= block_xor;
}
}
for (u32 pixel = 0; pixel < words.size(); ++pixel) {
std::memcpy(static_cast<uint8_t *>(mapped) + offsets[pixel], &words[pixel], sizeof(u32));
}
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.MapMemory(base, allocation_size);
auto &cache = context.GetTextureCache();
auto &executor = context.GetRenderExecutor();
constexpr u32 swizzle = DstSel(4, 5, 6, 1);
ShaderTextureResource descriptor{{static_cast<u32>(base >> 8u),
(static_cast<u32>(Prospero::BufferFormat::k10_11_11Float) << 20u) |
(3u << 30u), 0,
swizzle | (static_cast<u32>(tile) << 20u) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u),
0, 0x00700000u, 0, 0}};
TestCase test;
test.name = name;
test.has_user_data = true;
test.image_descriptor_swizzle = swizzle;
std::copy_n(descriptor.fields, 8, test.user_data.begin());
test.user_data[50] = 16 * sizeof(u32);
for (u32 pixel = 0; pixel < words.size(); ++pixel) {
AppendVMovU32(&test.code, 20, pixel);
AppendVMovU32(&test.code, 21, 0);
test.code.push_back(EncodeMimg0(0x00, 0xf));
test.code.push_back(EncodeMimg1(0, 20));
for (u32 channel = 0; channel < 4; ++channel) {
AppendStoreVgpr(&test.code, channel, pixel * 4 + channel);
test.expected.push_back(std::bit_cast<u32>(channels[pixel][channel]));
}
}
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
const auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images.at(0), value);
Image sampled;
sampled.view = cache.FindTexture(binding.image_id, binding.desc);
sampled.layout = cache.GetImage(binding.image_id).backing.state.layout;
scheduler.Finish();
auto output = CreateStorageBuffer(name, {}, 16);
Dispatch(test, compiled, output, nullptr, &sampled);
Require(name, "10_11_11 uploaded components",
ReadBuffer(name, output, 16) == test.expected,
"packed-float upload or sample changed a component's width or order");
DestroyBuffer(&output);
const auto clear = [&](std::array<float, 4> channels) {
auto &native = cache.GetImage(binding.image_id);
native.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {},
scheduler.Current().Handle());
const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
scheduler.Current().Handle().clearColorImage(native.backing.image,
vk::ImageLayout::eTransferDstOptimal, vk::ClearColorValue{channels}, range);
cache.MarkGpuWritten(binding.image_id);
};
clear({8.f, 4.f, 2.f, 1.f});
Require(name, "10_11_11 guest download",
TextureCacheTestAccess::TryDownload(cache, binding.image_id),
"packed-float native contents could not be downloaded");
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::array<u32, 4> guest{};
for (u32 pixel = 0; pixel < words.size(); ++pixel) {
Require(name, "10_11_11 guest texel",
Libs::LibKernel::Memory::TryReadBacking(base + offsets[pixel],
&guest[pixel], sizeof(u32)),
"packed-float guest texel is inaccessible");
}
Require(name, "10_11_11 exact guest words",
guest == std::array<u32, 4>{0x90110200u, 0x90110200u, 0x90110200u, 0x90110200u},
"native packed-float write lost the guest 10/11/11 bit layout");
clear({16.f, 8.f, 4.f, 1.f});
auto alias_desc = binding.desc;
alias_desc.info.guest_format = Prospero::BufferFormat::k11_11_10Float;
alias_desc.view_info.mapping = {};
Require(name, "same-format transform compatibility",
!alias_desc.info.IsCompatible(binding.desc.info),
"equal Vulkan formats hid incompatible guest color transforms");
const auto alias = cache.FindImage(alias_desc);
(void)cache.FindTexture(alias, alias_desc);
Require(name, "same-format transformed alias",
alias != binding.image_id &&
ReadCachedTexel(name, context, alias, {}, {4, 1, 1}) ==
std::vector<u32>(4, 0x98120220u),
"packed-float alias reused a transformed image or bypassed guest bytes");
RenderExecutorTestAccess::ResetBindings(executor);
context.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "release",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0 &&
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"packed texture backing release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckSamplerBorderColors() {
constexpr const char *name = "SamplerBorderColors";
auto &context = RuntimeRenderer();
constexpr std::array<std::array<float, 2>, 5> coordinates{{
{{0.5f, -1.f}}, {{-1.f, 0.5f}}, {{2.f, 0.5f}},
{{0.5f, 2.f}}, {{0.5f, 0.5f}}}};
std::array<vk::Sampler, 3> float_samplers{};
for (const bool integer : {false, true}) {
auto image = CreateImage2D(name, 2, 2,
integer ? vk::Format::eR8G8B8A8Uint : vk::Format::eR8G8B8A8Unorm,
vk::ImageUsageFlagBits::eSampled, std::vector<u32>(4, 0xffffffffu), 1,
vk::ImageLayout::eShaderReadOnlyOptimal);
for (const auto border : {Prospero::SamplerBorderColor::kOpaqueWhite,
Prospero::SamplerBorderColor::kOpaqueBlack,
Prospero::SamplerBorderColor::kTransBlack}) {
ShaderSamplerResource sampler_descriptor{};
const auto clamp = static_cast<u32>(Prospero::SamplerClampMode::kClampBorder);
sampler_descriptor.fields[0] = clamp | (clamp << 3u);
sampler_descriptor.fields[2] = 1u << 24u;
sampler_descriptor.fields[3] = static_cast<u32>(border) << 30u;
TestCase test;
test.name = name;
test.has_user_data = true;
test.user_data = MakeSampledTextureData(integer ? Prospero::BufferFormat::k8_8_8_8UInt
: Prospero::BufferFormat::k8_8_8_8UNorm);
test.user_data[2] = 1u | (1u << 14u);
std::copy_n(sampler_descriptor.fields, 4, test.user_data.begin() + 8);
test.user_data[50] = coordinates.size() * 4u * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
for (u32 sample = 0; sample < coordinates.size(); ++sample) {
AppendVMovLiteral(&test.code, 20, std::bit_cast<u32>(coordinates[sample][0]));
AppendVMovLiteral(&test.code, 21, std::bit_cast<u32>(coordinates[sample][1]));
test.code.push_back(EncodeMimg0(0x20, 0xf));
test.code.push_back(EncodeMimg1(0, 20, 0, 2));
for (u32 channel = 0; channel < 4; ++channel) {
AppendStoreVgpr(&test.code, channel, sample * 4u + channel);
const bool inside = sample == coordinates.size() - 1u;
const bool one = inside || border == Prospero::SamplerBorderColor::kOpaqueWhite ||
(border == Prospero::SamplerBorderColor::kOpaqueBlack && channel == 3u);
test.expected.push_back(one ? (integer ? (inside ? 255u : 1u) : 0x3f800000u) : 0u);
}
}
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
const auto &sampler_info = compiled.program.info.samplers[0];
Require(name, "sampler specialization", sampler_info.integer_border == integer,
"sampled image numeric type did not select its sampler border class");
auto output = CreateStorageBuffer(name, {}, test.expected.size());
const auto sampler = context.GetSamplerCache().GetSampler(
sampler_descriptor, sampler_info.integer_border);
Require(name, "sampler reuse",
sampler == context.GetSamplerCache().GetSampler(sampler_descriptor, integer),
"identical typed sampler descriptors did not reuse the cached sampler");
if (integer) {
Require(name, "typed border cache key", sampler != float_samplers[static_cast<u32>(border)],
"integer image reused the floating-point border sampler");
} else {
float_samplers[static_cast<u32>(border)] = sampler;
}
Dispatch(test, compiled, output, nullptr, &image, nullptr, nullptr, sampler);
const auto actual = ReadBuffer(name, output, test.expected.size());
for (u32 i = 0; i < actual.size(); ++i) {
if (actual[i] != test.expected[i]) {
Fail(name, "native border sample", "integer=" + std::to_string(integer) +
" border=" + std::to_string(static_cast<u32>(border)) + " word=" + std::to_string(i) +
" expected=" + std::to_string(test.expected[i]) + " actual=" + std::to_string(actual[i]));
}
}
DestroyBuffer(&output);
}
DestroyImage(&image);
}
std::printf("[gpu] %-32s ok\n", name);
}
void CheckComparisonDepthTexture() {
constexpr const char *name = "ComparisonDepthTexture";
constexpr uintptr_t base = 0x0000000204200000ull;
constexpr uint64_t allocation_size = 0x20000;
constexpr uint64_t ordinary_address = base + 0x10000;
constexpr std::array<uint16_t, 6> depths{0, 13107, 26214, 39321, 52428, 65535};
constexpr std::array<float, 5> references{0.1f, 0.3f, 0.5f, 0.7f, 0.9f};
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"comparison texture allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
"comparison texture mapping failed");
// Sea of Stars' 1D comparison texture and the existing six-face cube.
const auto check_layout = [&](bool cube, uint32_t swizzle) {
const uint32_t layers = cube ? depths.size() : 1;
const auto depth_at = [&](uint32_t layer, bool reversed) {
const auto index = cube ? layer : 2u;
return depths[reversed ? depths.size() - 1u - index : index];
};
std::memset(mapped, 0, allocation_size);
for (uint32_t face = 0; face < layers; face++) {
const auto depth = depth_at(face, false);
for (const auto address : {static_cast<uint64_t>(base), ordinary_address}) {
std::memcpy(reinterpret_cast<void *>(address + face * 0x100),
&depth, sizeof(depth));
}
}
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
0x00700000u, 0, (cube ? 0xb0100000u : 0x81800000u) | swizzle,
layers - 1u, 0x00700000u, 0, 0}};
ShaderSamplerResource sampler_descriptor{{1u << 12u, 0, 1u << 24u, 0}};
TestCase test;
test.name = name;
test.has_user_data = true;
test.image_descriptor_swizzle = descriptor.DstSelXYZW();
std::copy_n(descriptor.fields, 8, test.user_data.begin());
std::copy_n(sampler_descriptor.fields, 4, test.user_data.begin() + 8);
test.user_data[50] = layers * references.size() * sizeof(uint32_t);
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
test.required_spirv = {"OpImageSampleDrefExplicitLod"};
std::vector<uint32_t> promoted_expected;
for (uint32_t face = 0; face < layers; face++) {
for (const auto reference : references) {
AppendVMovLiteral(&test.code, 20, std::bit_cast<uint32_t>(reference));
AppendVMovLiteral(&test.code, 21,
std::bit_cast<uint32_t>(cube ? 1.5f : 0.5f));
if (cube) {
AppendVMovLiteral(&test.code, 22, std::bit_cast<uint32_t>(1.5f));
AppendVMovLiteral(&test.code, 23,
std::bit_cast<uint32_t>(static_cast<float>(face)));
}
test.code.push_back(EncodeMimg0(0x2f, 1, 0, false, cube ? 3 : 0));
test.code.push_back(EncodeMimg1(0, 20, 0, 2));
AppendStoreVgpr(&test.code, 0, static_cast<uint32_t>(test.expected.size()));
test.expected.push_back(
reference < static_cast<float>(depth_at(face, false)) / 65535.f
? 0x3f800000u : 0u);
promoted_expected.push_back(
reference < static_cast<float>(depth_at(face, true)) / 65535.f
? 0x3f800000u : 0u);
}
}
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
Require(name, "comparison resource", compiled.program.info.images.size() == 1 &&
compiled.program.info.images[0].depth_compare &&
compiled.program.info.images[0].cube == cube &&
compiled.program.info.images[0].dimension ==
(cube ? ShaderRecompiler::Decoder::ImageDimension::Dim2DArray
: ShaderRecompiler::Decoder::ImageDimension::Dim1D),
"native comparison did not retain its resource semantics");
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images[0], value);
const auto view = cache.FindTexture(binding.image_id, binding.desc);
auto &image = cache.GetImage(binding.image_id);
if (swizzle == DstSel(4, 4, 4, 1)) {
const vk::ComponentMapping mapping{vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eOne};
Require(name, "RRR1 depth view", std::ranges::any_of(image.views, [&](const auto &cached) {
return cached.view == view && cached.info.mapping == mapping;
}), "comparison depth view lost its replicated depth or constant alpha");
}
Require(name, "fresh depth backing", view != nullptr &&
image.info.pixel_format == vk::Format::eD16Unorm &&
image.info.guest_format == Prospero::BufferFormat::k16UNorm &&
image.info.tile_mode == descriptor.TileMode() &&
image.info.type == (cube ? Prospero::ImageType::kColor2D
: Prospero::ImageType::kColor1D) &&
image.info.data.size == (cube ? 0x600u : 0x10000u) &&
image.info.resources.layers == layers,
"comparison texture lost its guest layout or depth representation");
auto ordinary_resource = compiled.program.info.images[0];
ordinary_resource.depth_compare = false;
value.dwords[0] = static_cast<uint32_t>(ordinary_address >> 8u);
auto ordinary = RenderExecutorTestAccess::ResolveTexture(executor, ordinary_resource, value);
(void)cache.FindTexture(ordinary.image_id, ordinary.desc);
auto &color = cache.GetImage(ordinary.image_id);
Require(name, "ordinary R16", color.info.pixel_format == vk::Format::eR16Unorm,
"ordinary R16 sampling unexpectedly acquired a depth format");
scheduler.Finish();
auto output = CreateStorageBuffer(name, {}, test.expected.size());
const auto sampler = context.GetSamplerCache().GetSampler(
sampler_descriptor, compiled.program.info.samplers[0].integer_border);
Image sampled;
sampled.view = view;
sampled.layout = image.backing.state.layout;
Dispatch(test, compiled, output, nullptr, &sampled, nullptr, nullptr, sampler);
Require(name, "native comparisons",
ReadBuffer(name, output, test.expected.size()) == test.expected,
"comparison samples lost a depth value or guest layer stride");
image.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
{}, scheduler.Current().Handle());
color.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
{}, scheduler.Current().Handle());
for (uint32_t face = 0; face < layers; face++) {
const vk::ImageSubresourceRange depth_range{vk::ImageAspectFlagBits::eDepth, 0, 1, face, 1};
const vk::ClearDepthStencilValue clear_depth{
static_cast<float>(depth_at(face, true)) / 65535.f, 0};
scheduler.Current().Handle().clearDepthStencilImage(image.backing.image,
vk::ImageLayout::eTransferDstOptimal, &clear_depth, 1, &depth_range);
const vk::ImageSubresourceRange color_range{vk::ImageAspectFlagBits::eColor, 0, 1, face, 1};
const vk::ClearColorValue clear_color{std::array<float, 4>{
static_cast<float>(depth_at(face, true)) / 65535.f, 0.f, 0.f, 0.f}};
scheduler.Current().Handle().clearColorImage(color.backing.image,
vk::ImageLayout::eTransferDstOptimal, &clear_color, 1, &color_range);
}
cache.MarkGpuWritten(binding.image_id);
cache.MarkGpuWritten(ordinary.image_id);
Require(name, "depth readback", TextureCacheTestAccess::TryDownload(cache, binding.image_id),
"ordinary tiled depth readback did not use the shared transfer path");
scheduler.Finish();
scheduler.DrainPriorityOperations();
for (uint32_t face = 0; face < layers; face++) {
uint16_t actual = 0;
std::memcpy(&actual, reinterpret_cast<const void *>(base + face * 0x100), sizeof(actual));
Require(name, "depth guest roundtrip", actual == depth_at(face, true),
"GPU depth clear was not written to the correct guest face");
}
auto promoted = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images[0], value);
sampled.view = cache.FindTexture(promoted.image_id, promoted.desc);
auto &promoted_image = cache.GetImage(promoted.image_id);
Require(name, "GPU color owner promotion", promoted.image_id != ordinary.image_id &&
promoted_image.info.pixel_format == vk::Format::eD16Unorm &&
promoted_image.IsGpuModified(),
"comparison binding did not preserve the GPU-current color owner");
sampled.layout = promoted_image.backing.state.layout;
scheduler.Finish();
Dispatch(test, compiled, output, nullptr, &sampled, nullptr, nullptr, sampler);
Require(name, "promoted comparisons",
ReadBuffer(name, output, promoted_expected.size()) == promoted_expected,
"color-to-depth ownership conversion changed comparison results");
DestroyBuffer(&output);
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
};
check_layout(false, DstSel(4, 4, 4, 4));
check_layout(false, DstSel(4, 4, 4, 1));
check_layout(true, DstSel(4, 0, 0, 1));
Require(name, "unmap", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"comparison texture unmap failed");
Require(name, "release", Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"comparison texture release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckCubeFaceStorageExpansion() {
constexpr const char *name = "CubeFaceStorageExpansion";
constexpr uintptr_t base = 0x0000000204700000ull;
constexpr uint64_t allocation_size = 0x10000;
constexpr std::array<uint32_t, 6> half_red{0x3c00, 0x4000, 0x4200,
0x4400, 0x4500, 0x4600};
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"cube-face allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
"cube-face mapping failed");
std::memset(mapped, 0, allocation_size);
{
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.MapMemory(base, allocation_size);
auto &cache = context.GetTextureCache();
auto &executor = context.GetRenderExecutor();
for (uint32_t face = 0; face < half_red.size(); ++face) {
// PPSA25380 clears six single-face cube views at one guest address.
ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
0xc4700000u, 0x0001c001u, 0xb0000facu,
face | (face << 16u), 0x00700000u, 0, 0}};
TestCase test;
test.name = name;
test.has_user_data = true;
std::copy_n(descriptor.fields, 8, test.user_data.begin());
test.has_compute_info = true;
test.compute_info.threads_num[0] = test.compute_info.threads_num[1] = 8;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 2;
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_STORE,
ShaderOpcode::S_ENDPGM};
test.code = {EncodeVop1(0x01, 20, 256), EncodeVop1(0x01, 21, 257)};
AppendVMovU32(&test.code, 22, 0);
AppendVMovLiteral(&test.code, 0,
std::bit_cast<uint32_t>(static_cast<float>(face + 1)));
AppendVMovLiteral(&test.code, 1, std::bit_cast<uint32_t>(0.5f));
AppendVMovLiteral(&test.code, 2, std::bit_cast<uint32_t>(0.25f));
AppendVMovLiteral(&test.code, 3, std::bit_cast<uint32_t>(1.0f));
test.code.push_back(EncodeMimg0(0x08, 0xf, 0, false, 5));
test.code.push_back(EncodeMimg1(0, 20));
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
const auto &resource = compiled.program.info.images.at(0);
Require(name, "cube storage specialization", resource.cube &&
resource.written && resource.dimension ==
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray,
"single-face cube store lost its array coordinates");
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
const auto binding = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
Image storage;
storage.view = cache.FindTexture(binding.image_id, binding.desc);
auto &image = cache.GetImage(binding.image_id);
Require(name, "face view and backing", storage.view != nullptr &&
image.info.pixel_format == vk::Format::eR16G16B16A16Sfloat &&
image.info.bytes_per_block == 8 &&
image.info.resources.layers == face + 1 &&
image.backing.layers == face + 1 &&
binding.desc.view_info.type == vk::ImageViewType::e2DArray &&
binding.desc.view_info.base_layer == face &&
binding.desc.view_info.layer_count == 1,
"cube face did not expand the array or select its own layer");
image.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits2::eShaderWrite,
{}, scheduler.Current().Handle());
storage.layout = image.backing.state.layout;
scheduler.Finish();
Dispatch(test, compiled, {}, nullptr, nullptr, &storage);
cache.MarkGpuWritten(binding.image_id);
for (uint32_t prior = 0; prior <= face; ++prior) {
const auto pixels = ReadCachedTexel(name, context, binding.image_id,
{}, {8, 8, 1}, prior);
for (uint32_t pixel = 0; pixel < 64; ++pixel) {
Require(name, "stored and preserved faces",
pixels[pixel * 2] == (0x38000000u | half_red[prior]) &&
pixels[pixel * 2 + 1] == 0x3c003400u,
"face " + std::to_string(prior) + " pixel " +
std::to_string(pixel) + " changed after writing face " +
std::to_string(face));
}
}
}
RenderExecutorTestAccess::ResetBindings(executor);
context.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "release",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0 &&
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"cube-face backing release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColorStandardTileDiscovery() {
constexpr const char *name = "RenderExecutorColorStandardTile";
constexpr uintptr_t base = 0x0000000203b00000ull;
constexpr uint64_t allocation_size = 0x190000;
constexpr uint64_t allocation_alignment = 0x10000;
constexpr auto format = Prospero::BufferFormat::k32Float;
struct TileCase {
Prospero::TileMode tile;
uint64_t target_offset;
uint64_t alignment;
};
constexpr std::array cases{
TileCase{Prospero::TileMode::kStandard4KB, 0x4000, 0x1000},
TileCase{Prospero::TileMode::kStandard64KB, 0x10000, 0x10000},
};
EnsureRuntimeContext();
const auto odd_pitch =
TileGetTexturePitch(format, 257, Prospero::TileMode::kStandard4KB);
TileSizeAlign odd_slice{};
TileGetTextureSize(format, 257, 131, 1, Prospero::TileMode::kStandard4KB,
&odd_slice, nullptr, nullptr);
Require(name, "tile layout",
odd_pitch == 288 && odd_slice.size == 0x2d000 &&
odd_slice.align == 0x1000,
"Standard4KB pitch, footprint, or block alignment is incorrect");
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"standard-tiled color allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"standard-tiled color mapping failed");
std::memset(mapped, 0, allocation_size);
for (const auto &tile_case : cases) {
const auto pitch = TileGetTexturePitch(format, 256, tile_case.tile);
TileSizeAlign slice{};
TileGetTextureSize(format, 256, 256, 1, tile_case.tile, &slice, nullptr,
nullptr);
Require(
name, "tile layout",
pitch == 256 && slice.size == 0x40000 &&
slice.align == tile_case.alignment,
"standard tile pitch, footprint, or block alignment is incorrect");
const auto target_address = base + tile_case.target_offset;
const auto target_size = slice.size * 6ull;
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetColorBase(0, {.addr = target_address});
registers.SetColorInfo(
0, {.format = Prospero::ChannelLayout::k32,
.channel_type = Prospero::ChannelType::kFloat,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorView(
0, {.base_array_slice_index = 5, .last_array_slice_index = 5});
registers.SetColorAttrib2(0, {.height = 255, .width = 255});
registers.SetColorAttrib3(0, {.tile_mode = tile_case.tile,
.dimension = 1,
.metadata_pipe_aligned = true});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
RenderColorInfo color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto attachment =
texture_cache.FindRenderTarget(color.image_id, color.desc);
const auto &image = texture_cache.GetImage(color.image_id);
Require(name, "captured target",
color.image_id && attachment != nullptr &&
color.guest_array_layer == 5 &&
color.desc.view_info.base_layer == 5 &&
color.desc.info.data.address == target_address &&
color.desc.info.data.size == target_size &&
color.desc.info.tile_mode == tile_case.tile &&
color.desc.info.extent == vk::Extent3D{256, 256, 1} &&
color.desc.info.resources == ImageSubresources{1, 6} &&
color.desc.info.pitch == pitch &&
color.desc.info.mip_layout[0].offset == 0 &&
color.desc.info.mip_layout[0].size == target_size &&
color.desc.view_info.type == vk::ImageViewType::e2D &&
color.desc.view_info.base_layer == 5 &&
color.desc.view_info.layer_count == 1 &&
(target_address & (tile_case.alignment - 1)) == 0 &&
image.usage.render_target && image.IsGpuModified(),
"slice 5 standard-tiled render target was not resolved");
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"standard-tiled color mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"standard-tiled color allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColorDepthTileDiscovery() {
constexpr const char *name = "RenderExecutorColorDepthTile";
constexpr uintptr_t base = 0x0000000203d00000ull;
constexpr uint64_t allocation_size = 0x10000;
constexpr uint64_t allocation_alignment = 0x10000;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"depth-tiled color allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"depth-tiled color mapping failed");
constexpr uint32_t width = 64;
constexpr uint32_t height = 64;
TileTextureBlockLayout tile_layout{};
Require(name, "tile layout",
TileGetTextureBlockLayout(Prospero::BufferFormat::k32Float,
Prospero::TileMode::kDepth, false,
tile_layout),
"depth-tiled color block layout is unavailable");
uint32_t block_xor = 0;
Require(name, "block xor",
TileGetBlockXor(tile_layout.block, 0, 0, 0, block_xor),
"depth-tiled color block XOR is unavailable");
std::vector<uint32_t> expected(allocation_size / sizeof(uint32_t));
for (uint32_t y = 0; y < height; ++y) {
for (uint32_t x = 0; x < width; ++x) {
uint32_t offset = 0;
Require(name, "texel offset",
TileGetBlockOffset(tile_layout.block, x, y, 0, offset),
"depth-tiled color texel offset is unavailable");
const auto index = (offset ^ block_xor) / sizeof(uint32_t);
Require(name, "texel address", index < expected.size(),
"depth-tiled color texel address exceeds its backing");
expected[index] = 0x3f000000u + y * width + x;
}
}
std::memcpy(mapped, expected.data(), allocation_size);
{
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetColorBase(0, {.addr = base});
registers.SetColorInfo(
0, {.format = Prospero::ChannelLayout::k32,
.channel_type = Prospero::ChannelType::kFloat,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorAttrib2(0, {.height = 63, .width = 63});
registers.SetColorAttrib3(0, {.tile_mode = Prospero::TileMode::kDepth,
.dimension = 1,
.metadata_pipe_aligned = true});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
RenderColorInfo color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto attachment =
texture_cache.FindRenderTarget(color.image_id, color.desc);
const auto &image = texture_cache.GetImage(color.image_id);
Require(name, "captured target",
color.image_id && attachment != nullptr &&
color.desc.info.tile_mode == Prospero::TileMode::kDepth &&
color.desc.info.pitch == 128 &&
color.desc.info.data.size == allocation_size &&
color.desc.info.mip_layout[0].size == allocation_size &&
image.usage.render_target && image.IsGpuModified(),
"supported depth-tiled color target was not resolved");
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::vector<uint8_t> cleared(allocation_size);
Require(name, "clear guest backing",
Libs::LibKernel::Memory::TryWriteBacking(base, cleared.data(),
allocation_size),
"depth-tiled color backing could not be cleared");
Require(
name, "readback queue",
TextureCacheTestAccess::TryDownload(texture_cache, color.image_id),
"depth-tiled color target could not be queued for readback");
RenderExecutorTestAccess::ResetBindings(executor);
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::vector<uint32_t> observed(expected.size());
Require(name, "readback backing",
Libs::LibKernel::Memory::TryReadBacking(base, observed.data(),
allocation_size),
"depth-tiled color readback is unavailable");
bool recovered = true;
for (uint32_t y = 0; y < height; ++y) {
for (uint32_t x = 0; x < width; ++x) {
uint32_t offset = 0;
const auto valid =
TileGetBlockOffset(tile_layout.block, x, y, 0, offset);
const auto index = (offset ^ block_xor) / sizeof(uint32_t);
recovered &= valid && index < observed.size() &&
observed[index] == expected[index];
}
}
Require(name, "round trip", recovered,
"depth-tiled color upload/readback changed active texels");
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"depth-tiled color mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"depth-tiled color allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorStencilBindingDiscovery() {
constexpr const char *name = "RenderExecutorStencilBindingDiscovery";
constexpr uintptr_t base = 0x0000000203600000ull;
constexpr uint64_t allocation_size = 0x1000000;
constexpr uint64_t allocation_alignment = 0x10000;
constexpr uint64_t depth_address = base + 0x40000;
constexpr uint64_t stencil_address = base + 0x70000;
constexpr uint64_t mipped_storage_address = base + 0x100000;
constexpr uint32_t tail_backing_value = 0x2468ace0u;
constexpr uint32_t tail_gpu_value = 0x13579bdfu;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"descriptor discovery direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"descriptor discovery fixed mapping failed");
std::memset(mapped, 0, allocation_size);
std::fill_n(reinterpret_cast<uint32_t *>(mipped_storage_address),
0x10000 / 4, tail_backing_value);
{
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
context.InitializeGpu(nullptr);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
std::vector<PipelineCache::Pipeline> descriptor_pipelines;
// The virtual-texture atlas is written as raw BC3 blocks, then sampled as BC3.
constexpr uint64_t block_alias_address = base + 0xf0000;
constexpr std::array<uint32_t, 4> block_alias_data{
0x01234567u, 0x89abcdefu, 0xfedcba98u, 0x76543210u};
const auto resolve_block_alias = [&](bool compressed) {
const uint32_t side = compressed ? 256 : 64;
const auto format = compressed ? Prospero::BufferFormat::kBc3UNorm
: Prospero::BufferFormat::k32_32_32_32UInt;
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
value.dwords = {static_cast<uint32_t>(block_alias_address >> 8u),
(static_cast<uint32_t>(format) << 20u) | (3u << 30u),
((side - 1u) >> 2u) | ((side - 1u) << 14u),
0x90900facu, 0, 0x00700000u, 0, 0};
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = compressed
? ShaderRecompiler::IR::ImageResourceClass::Sampled
: ShaderRecompiler::IR::ImageResourceClass::Storage;
resource.numeric_class = compressed ? Prospero::TextureNumericClass::Float
: Prospero::TextureNumericClass::Uint;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = compressed;
resource.written = !compressed;
return RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
};
auto raw_blocks = resolve_block_alias(false);
(void)texture_cache.FindTexture(raw_blocks.image_id, raw_blocks.desc);
auto &raw_blocks_native = texture_cache.GetImage(raw_blocks.image_id);
raw_blocks_native.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {},
scheduler.Current().Handle());
vk::ClearColorValue block_clear{};
std::copy(block_alias_data.begin(), block_alias_data.end(),
block_clear.uint32.begin());
const vk::ImageSubresourceRange block_range{
vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
scheduler.Current().Handle().clearColorImage(
raw_blocks_native.backing.image, vk::ImageLayout::eTransferDstOptimal,
&block_clear, 1, &block_range);
texture_cache.MarkGpuWritten(raw_blocks.image_id);
auto compressed_blocks = resolve_block_alias(true);
(void)texture_cache.FindTexture(compressed_blocks.image_id,
compressed_blocks.desc);
auto raw_blocks_again = resolve_block_alias(false);
Require(name, "compressed atlas storage reuse",
compressed_blocks.image_id != raw_blocks.image_id &&
raw_blocks_again.image_id == compressed_blocks.image_id &&
texture_cache.FindTexture(raw_blocks_again.image_id,
raw_blocks_again.desc) != nullptr,
"BC3 sampling replaced the GPU atlas on its next raw-block write");
Require(name, "compressed atlas download",
TextureCacheTestAccess::TryDownload(texture_cache,
raw_blocks_again.image_id),
"the retained BC3 atlas could not publish its native contents");
scheduler.Finish();
scheduler.DrainPriorityOperations();
const auto *block_words = reinterpret_cast<const uint32_t *>(block_alias_address);
Require(name, "compressed atlas GPU contents",
std::equal(block_alias_data.begin(), block_alias_data.end(), block_words) &&
std::equal(block_alias_data.begin(), block_alias_data.end(),
block_words + 0x10000 / sizeof(uint32_t) - 4),
"compressed atlas aliases reloaded stale CPU bytes over GPU-written blocks");
const auto set_image_bindings = [](ShaderRecompiler::IR::Program &program) {
for (auto &image : program.info.images) {
const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(image);
const auto index = static_cast<size_t>(kind);
image.descriptor_index = program.bindings.descriptor_counts[index];
program.bindings.descriptor_counts[index] += image.mip_count;
program.bindings.descriptor_mask |= uint64_t{1} << index;
}
};
const auto make_buffer_program =
[](ShaderType stage, ShaderRecompiler::IR::BufferResource resource) {
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.stage = stage;
resource.descriptor_index = 0;
program.info.buffers.push_back(resource);
program.bindings.descriptor_counts[0] = 1;
program.bindings.descriptor_mask = 1;
program.bindings.push_data_start_dword = 0;
return program;
};
{
const auto buffer_program = make_buffer_program(ShaderType::Compute, {.read = true});
constexpr uint64_t buffer_address = base + allocation_size - 0x5000;
ShaderBufferResource buffer_descriptor{};
buffer_descriptor.UpdateAddress48(buffer_address);
buffer_descriptor.fields[2] = 0x40000000; // Only oversized descriptors are clamped.
ShaderRecompiler::IR::ResourceSnapshot buffer_snapshot;
ShaderStageRuntime buffer_runtime{&buffer_program, &buffer_snapshot};
auto &value = buffer_snapshot.buffers.emplace_back();
std::memcpy(value.dwords.data(), buffer_descriptor.fields,
sizeof(buffer_descriptor.fields));
value.dword_count = 4;
PreparedBindings buffer_bindings;
executor.PrepareBindings(buffer_runtime, buffer_bindings);
std::array<PreparedBindings *, 1> buffer_stages{&buffer_bindings};
executor.FindBuffers(buffer_stages);
const auto original_id = buffer_bindings.buffer_sources[0].id;
auto &buffer_cache = resources.GetBufferCache();
const auto merged_id =
buffer_cache.FindBuffer(base + allocation_size - 0x10000, 0x10000);
Require(name, "buffer discovery invalidation", merged_id != original_id,
"overlapping buffer discovery did not replace its smaller owner");
executor.RebindBuffers(buffer_bindings);
const auto &binding = buffer_bindings.buffers[0];
const auto &owner = buffer_cache.GetBuffer(merged_id);
Require(name, "clamped buffer rebind",
binding.buffer == owner.Handle() &&
binding.offset == owner.Offset(buffer_address) &&
binding.range == 0x5000,
"descriptor rebind lost its clamped guest range or retained a stale host owner");
}
{
const auto buffer_program = make_buffer_program(ShaderType::Compute, {.read = true, .written = true});
constexpr uint64_t buffer_address = base + 0xe0002;
ShaderBufferResource descriptor{};
descriptor.UpdateAddress48(buffer_address);
descriptor.fields[1] |= 6u << 16u;
descriptor.fields[2] = 2;
descriptor.fields[3] = 0x5204;
ShaderRecompiler::IR::ResourceSnapshot snapshot;
auto &value = snapshot.buffers.emplace_back();
std::copy_n(descriptor.fields, 4, value.dwords.begin());
value.dword_count = 4;
ShaderStageRuntime runtime{&buffer_program, &snapshot};
PreparedBindings bindings;
executor.PrepareBindings(runtime, bindings);
std::array<PreparedBindings *, 1> stages{&bindings};
executor.FindBuffers(stages);
executor.RebindBuffers(bindings);
const auto &owner = resources.GetBufferCache().GetBuffer(bindings.buffer_sources[0].id);
Require(name, "halfword buffer base and exact tail",
bindings.buffers[0].buffer == owner.Handle() &&
bindings.buffers[0].offset + 2 == owner.Offset(buffer_address) &&
bindings.buffers[0].range == 14 &&
bindings.shader_data[buffer_program.bindings.memory_offset_dword] == 2,
"native binding lost the byte adjustment or rounded the final halfword range");
}
{
constexpr uint64_t address = base + allocation_size - 0x4000 + 12;
constexpr uint64_t mapped_tail = base + allocation_size - address;
auto writer = make_buffer_program(ShaderType::Vertex, {.written = true});
ShaderRecompiler::IR::CompiledShaderInfo reader{};
reader.stage = ShaderType::Pixel;
ShaderRecompiler::IR::ResourceSnapshot writer_snapshot, reader_snapshot;
ShaderBufferResource descriptor{};
descriptor.UpdateAddress48(address);
descriptor.fields[2] = UINT32_MAX;
auto &value = writer_snapshot.buffers.emplace_back();
std::copy_n(descriptor.fields, 4, value.dwords.begin());
value.dword_count = 4;
ShaderStageRuntime writer_runtime{&writer, &writer_snapshot};
ShaderStageRuntime reader_runtime{&reader, &reader_snapshot};
PreparedBindings writer_bindings, reader_bindings;
const auto check_range = [&](uint64_t expected, const char *description) {
executor.PrepareBindings(writer_runtime, writer_bindings);
executor.PrepareBindings(reader_runtime, reader_bindings);
std::array<PreparedBindings *, 2> stages{&reader_bindings, &writer_bindings};
executor.FindBuffers(stages);
executor.RebindBuffers(writer_bindings);
const auto &source = writer_bindings.buffer_sources[0];
const auto &binding = writer_bindings.buffers[0];
const auto &owner = resources.GetBufferCache().GetBuffer(source.id);
Require(name, description, source.address == address && source.size == expected &&
binding.buffer == owner.Handle() && binding.offset <= owner.Offset(address) &&
binding.range == expected + owner.Offset(address) - binding.offset,
"descriptor discovery or rebind changed the write boundary or base adjustment");
};
writer_snapshot.specialization_reads = {{address + 0x500, 4},
{address - 0x40, 0x40}, {address + 0x80, 0}};
reader_snapshot.specialization_reads = {{address + 0x300, 4}, {address + 0x100, 4}};
check_range(0x100, "unbounded write stops before nearest cross-stage read");
reader_snapshot.specialization_reads.clear();
check_range(0x500, "unbounded write stops before same-stage read");
writer_snapshot.specialization_reads = {{address - 0x40, 0x40},
{address + mapped_tail + 0x100, 4}};
check_range(mapped_tail, "earlier and unmapped reads preserve mapped tail");
writer_snapshot.specialization_reads.clear();
check_range(mapped_tail, "unbounded write without reads preserves mapped tail");
reader_snapshot.specialization_reads = {{address + 0x100, 4}};
value.dwords[2] = 0x300;
check_range(0x300, "finite write preserves its declared range");
value.dwords[2] = UINT32_MAX;
writer.info.buffers[0].written = false;
writer.info.buffers[0].read = true;
check_range(mapped_tail, "read-only sentinel preserves mapped tail");
auto &graphics = context.GetGraphics();
auto &storage_limit = graphics.physical_device_properties.limits.maxStorageBufferRange;
const auto original_limit = storage_limit;
storage_limit = 0x2000;
value.dwords[1] |= 16u << 16u;
check_range(storage_limit - (graphics.StorageMinAlignment() - 1),
"read-only structured sentinel reserves native alignment prefix");
Require(name, "read-only native range fits device", writer_bindings.buffers[0].range <= storage_limit,
"read-only cap omitted the native descriptor alignment prefix");
value.dwords[2] = 0x300; // A finite 12 KiB descriptor remains finite despite the mock limit.
std::array<PreparedBindings *, 1> stages{&writer_bindings};
executor.FindBuffers(stages);
Require(name, "finite read exceeds mock cap unchanged", writer_bindings.buffer_sources[0].size == 0x3000,
"read-only window changed a finite descriptor's declared range");
storage_limit = original_limit;
check_range(0x3000, "finite structured read preserves native range");
value.dwords[2] = UINT32_MAX;
check_range(mapped_tail, "read-only structured sentinel preserves smaller mapping");
}
for (const auto format : {Prospero::BufferFormat::k8UNorm,
Prospero::BufferFormat::k16UNorm}) {
const uint64_t alias_address = base + 0xe00000 +
Prospero::NumBytesPerElement(format) * 0x10000;
constexpr uint8_t texel = 0x7f;
Libs::LibKernel::Memory::WriteBacking(alias_address, &texel, sizeof(texel));
u32 expected = texel;
if (format == Prospero::BufferFormat::k16UNorm) {
auto depth_desc = MakeLinearDesc(alias_address, 256, vk::Format::eD16Unorm,
format, Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 2, 1);
depth_desc.info.pitch = 128;
depth_desc.info.mip_layout[0] = {0, 256, 128, 1};
depth_desc.type = BindingType::DepthTarget;
depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
depth_desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto depth_id = texture_cache.FindImage(depth_desc);
auto &depth = texture_cache.GetImage(depth_id);
depth.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {}, scheduler.Current().Handle());
const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eDepth, 0, 1, 0, 1};
scheduler.Current().Handle().clearDepthStencilImage(depth.backing.image,
vk::ImageLayout::eTransferDstOptimal, vk::ClearDepthStencilValue{1.f, 0}, range);
texture_cache.MarkGpuWritten(depth_id);
expected = 0xffff;
}
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.stage = ShaderType::Compute;
ShaderRecompiler::IR::ResourceSnapshot snapshot;
// PPSA24156 binds one-texel R8/R16 allocations as 3D, then 2D.
for (const auto type : {Prospero::ImageType::kColor3D,
Prospero::ImageType::kColor2D}) {
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
resource.read = true;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
if (type == Prospero::ImageType::kColor3D) {
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
}
program.info.images.push_back(resource);
auto &value = snapshot.images.emplace_back();
value.dword_count = 8;
value.dwords = {static_cast<u32>(alias_address >> 8u),
static_cast<u32>(format) << 20u, 0,
DstSel(4, 4, 4, 1) | (static_cast<u32>(type) << 28u),
0, 0x00700000u, 0, 0};
}
ShaderStageRuntime runtime{&program, &snapshot};
PreparedBindings bindings;
executor.PrepareBindings(runtime, bindings);
Require(name, "overlapping dimension discovery",
!texture_cache.GetImage(bindings.images[0].image_id).registered,
"the 2D descriptor did not replace its earlier 3D alias");
executor.RebindImages(bindings);
auto output = CreateHostBuffer(name, 8, vk::BufferUsageFlagBits::eTransferDst, {0, 0});
constexpr std::array image_types{vk::ImageType::e3D, vk::ImageType::e2D};
for (u32 i = 0; i < bindings.images.size(); ++i) {
const auto &binding = bindings.images[i];
auto &image = texture_cache.GetImage(binding.image_id);
Require(name, "overlapping dimension views", binding.image_view != nullptr &&
image.backing.image_type == image_types[i],
"rediscovery lost a live view or the descriptor's image dimension");
image.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {}, scheduler.Current().Handle());
vk::BufferImageCopy copy{};
copy.bufferOffset = i * sizeof(u32);
copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1};
copy.imageExtent = {1, 1, 1};
scheduler.Current().Handle().copyImageToBuffer(image.backing.image,
vk::ImageLayout::eTransferSrcOptimal, output.buffer, 1, &copy);
}
vk::MemoryBarrier barrier{};
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 1, &barrier, 0, nullptr, 0, nullptr);
RenderExecutorTestAccess::ResetBindings(executor);
scheduler.Finish();
Require(name, "overlapping dimension GPU contents",
ReadBuffer(name, output, 2) == std::vector<u32>{expected, expected},
"an alias replacement retired image contents before queued GPU reads");
DestroyBuffer(&output);
}
for (const bool volume_first : {false, true}) {
// PPSA24156 shares a six-face R8 allocation with a smaller 3D placeholder.
const auto address = base + 0xe30000 + volume_first * 0x10000;
auto array_desc = MakeLinearDesc(address, 1536, vk::Format::eR8Unorm,
Prospero::BufferFormat::k8UNorm, Prospero::ImageType::kColor2D,
{1, 1, 1}, 6, 1, 1);
array_desc.info.pitch = 256;
array_desc.info.mip_layout[0] = {0, 1536, 256, 1};
auto volume_desc = array_desc;
volume_desc.info.type = Prospero::ImageType::kColor3D;
volume_desc.info.data.size = 256;
volume_desc.info.resources.layers = 1;
volume_desc.info.mip_layout[0].size = 256;
volume_desc.view_info.type = vk::ImageViewType::e3D;
volume_desc.view_info.layer_count = 1;
const auto first_id = texture_cache.FindImage(volume_first ? volume_desc : array_desc);
const auto second_id = texture_cache.FindImage(volume_first ? array_desc : volume_desc);
const auto array_id = volume_first ? second_id : first_id;
const auto volume_id = volume_first ? first_id : second_id;
Require(name, "smaller volume native image", volume_id != array_id &&
texture_cache.GetImage(volume_id).backing.image_type == vk::ImageType::e3D,
"a smaller 3D descriptor reused the six-layer 2D owner");
(void)texture_cache.FindTexture(array_id, array_desc);
(void)texture_cache.FindTexture(volume_id, volume_desc);
Require(name, "retained dimensional aliases",
texture_cache.FindImage(array_desc) == array_id &&
texture_cache.FindImage(volume_desc) == volume_id &&
texture_cache.GetImage(array_id).info.resources.layers == 6,
"dimensional alias lookup discarded the array owner or lost its faces");
auto subarray_desc = array_desc;
subarray_desc.info.data.size = 1280;
subarray_desc.info.resources.layers = 5;
subarray_desc.info.mip_layout[0].size = 1280;
subarray_desc.view_info.layer_count = 5;
const auto subarray_id = texture_cache.FindImage(subarray_desc);
Require(name, "preserved dimensional overlap selection", subarray_id == array_id &&
texture_cache.GetImage(array_id).info.resources.layers == 6,
"a smaller volume alias displaced the selected array owner");
(void)texture_cache.FindTexture(subarray_id, subarray_desc);
}
constexpr auto stencil_format = Prospero::BufferFormat::k8UInt;
constexpr auto linear = Prospero::TileMode::kLinear;
TileSizeAlign stencil_layout{};
TileGetTextureTotalSize(stencil_format, 1, 1, 1, 1, linear, false,
stencil_layout);
Require(name, "stencil footprint",
stencil_layout.size != 0 && stencil_layout.align != 0 &&
(stencil_address & (stencil_layout.align - 1u)) == 0,
"linear stencil descriptor produced an invalid footprint");
ShaderRecompiler::IR::Program null_program{};
null_program.stage = ShaderType::Compute;
null_program.resource_tracking_complete = true;
ShaderRecompiler::IR::ImageResource null_resource{};
null_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
null_resource.numeric_class = Prospero::TextureNumericClass::Uint;
null_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
null_resource.read = true;
null_program.info.images.push_back(null_resource);
auto null_volume = null_resource;
null_volume.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
null_program.info.images.push_back(null_volume);
auto null_storage = null_resource;
null_storage.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Storage;
null_storage.read = false;
null_storage.written = true;
null_program.info.images.push_back(null_storage);
ShaderRecompiler::IR::ResourceSnapshot null_snapshot{};
ShaderRecompiler::IR::DescriptorValue null_descriptor{};
null_descriptor.dword_count = 8;
null_snapshot.images.assign(3, null_descriptor);
ShaderRecompiler::IR::ResourceSpecialization null_specialization{};
null_specialization.images.resize(3);
for (auto &image : null_specialization.images) {
image.numeric_class = Prospero::TextureNumericClass::Float;
image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
}
ShaderRecompiler::IR::ApplyResourceSpecialization(
null_program, null_specialization);
set_image_bindings(null_program);
ShaderRecompiler::IR::CompiledShaderInfo null_info{};
null_info.stage = null_program.stage;
null_info.info = std::move(null_program.info);
null_info.bindings = std::move(null_program.bindings);
ShaderStageRuntime null_runtime{&null_info, &null_snapshot};
PreparedBindings null_bindings;
executor.PrepareBindings(null_runtime, null_bindings);
executor.RebindImages(null_bindings);
Require(name, "null descriptor count",
null_bindings.images.size() == 3,
"null descriptor preparation lost an image binding");
const auto null_image_id = null_bindings.images[0].image_id;
const auto &null_image = texture_cache.GetImage(null_image_id);
Require(name, "one null image per format",
null_bindings.images[1].image_id == null_image_id &&
null_bindings.images[2].image_id == null_image_id &&
TextureCacheTestAccess::NullImageCount(texture_cache) == 1,
"null view dimension or usage created another image allocation");
Require(name, "null descriptors share final cache acquisition",
null_image.info.data.Empty() && !null_image.binding.is_bound &&
!null_image.binding.is_target &&
!null_image.binding.needs_rebind &&
null_image.usage.texture && null_image.usage.storage &&
null_image.IsGpuModified() &&
!TextureCacheTestAccess::PendingDownload(texture_cache,
null_image_id),
"the shared null image did not preserve texture/storage "
"acquisition without guest readback or RenderExecutor ownership");
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), null_bindings));
Require(name, "null descriptor general layouts",
std::ranges::all_of(null_bindings.images,
[](const auto &binding) {
const auto info = MakeImageInfo(binding);
return binding.layout ==
vk::ImageLayout::eGeneral &&
info.imageView ==
binding.image_view &&
info.imageLayout == binding.layout;
}) &&
null_image.backing.state.layout == vk::ImageLayout::eGeneral,
"shared sampled/storage null descriptors did not retain "
"the general layout through descriptor generation");
constexpr uint64_t target_mip_size = 0x10000;
const auto make_target_desc = [&](uint64_t address, uint64_t size,
vk::Extent3D extent) {
ImageDesc desc{};
desc.type = BindingType::RenderTarget;
desc.info.data = {address, size};
desc.info.pixel_format = vk::Format::eR32Uint;
desc.info.guest_format = Prospero::BufferFormat::k32UInt;
desc.info.type = Prospero::ImageType::kColor2D;
desc.info.extent = extent;
desc.info.resources = {1, 1};
desc.info.pitch = extent.width;
desc.info.bytes_per_block = 4;
desc.info.samples = 1;
desc.info.tile_mode = linear;
desc.info.mip_layout[0] = {0, size, extent.width, extent.height};
desc.view_info.format = desc.info.pixel_format;
desc.view_info.type = vk::ImageViewType::e2D;
desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
return desc;
};
TextureCacheTestAccess::SetLinearReadback(texture_cache, true);
ShaderTextureResource storage{};
const uint64_t storage_address = base + 0x60000;
constexpr uint32_t storage_native_value = 0x13579bdfu;
constexpr uint32_t storage_stale_value = 0x2468ace0u;
std::memcpy(reinterpret_cast<uint8_t *>(mapped) + 0x60000,
&storage_native_value, sizeof(storage_native_value));
const uint64_t encoded_storage_address = storage_address >> 8u;
storage.fields[0] = static_cast<uint32_t>(encoded_storage_address);
storage.fields[1] =
static_cast<uint32_t>(encoded_storage_address >> 32u) |
(static_cast<uint32_t>(stencil_format) << 20u);
storage.fields[3] =
DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
storage.fields[5] = 0x00700000u;
ShaderRecompiler::IR::Program storage_program{};
storage_program.stage = ShaderType::Vertex;
storage_program.resource_tracking_complete = true;
ShaderRecompiler::IR::ImageResource storage_resource{};
storage_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Storage;
storage_resource.numeric_class = Prospero::TextureNumericClass::Uint;
storage_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
storage_resource.written = true;
storage_program.info.images.push_back(storage_resource);
set_image_bindings(storage_program);
ShaderRecompiler::IR::DescriptorValue storage_descriptor{};
std::copy(std::begin(storage.fields), std::end(storage.fields),
storage_descriptor.dwords.begin());
storage_descriptor.dword_count = 8;
auto mipped_storage = storage;
const auto encoded_mipped_storage_address = mipped_storage_address >> 8u;
mipped_storage.fields[0] =
static_cast<uint32_t>(encoded_mipped_storage_address);
mipped_storage.fields[1] =
static_cast<uint32_t>(encoded_mipped_storage_address >> 32u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u) |
(3u << 30u);
mipped_storage.fields[2] = 15u | (63u << 14u);
mipped_storage.fields[3] =
DstSel(4, 5, 6, 7) | (1u << 12u) | (3u << 16u) |
(static_cast<uint32_t>(Prospero::TileMode::kStandard64KB) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
mipped_storage.fields[5] = 0x00700000u | (3u << 4u);
ShaderRecompiler::IR::DescriptorValue mipped_storage_descriptor{};
std::copy(std::begin(mipped_storage.fields),
std::end(mipped_storage.fields),
mipped_storage_descriptor.dwords.begin());
mipped_storage_descriptor.dword_count = 8;
auto overwide_mipped_storage = mipped_storage;
overwide_mipped_storage.fields[3] =
(overwide_mipped_storage.fields[3] & ~(0xfu << 16u)) | (4u << 16u);
ShaderRecompiler::IR::DescriptorValue
overwide_mipped_storage_descriptor{};
std::copy(std::begin(overwide_mipped_storage.fields),
std::end(overwide_mipped_storage.fields),
overwide_mipped_storage_descriptor.dwords.begin());
overwide_mipped_storage_descriptor.dword_count = 8;
// The captured IMAGE_STORE addresses only BASE_LEVEL=1. Including
// LAST_LEVEL=4 would shift this four-mip render-target allocation.
auto fixed_storage_resource = storage_resource;
fixed_storage_resource.numeric_class = Prospero::TextureNumericClass::Float;
ShaderRecompiler::IR::DescriptorValue fixed_storage_descriptor{};
fixed_storage_descriptor.dwords = {
static_cast<uint32_t>((base + 0x300000) >> 8u),
0xc4700000u, 0x001fc7ffu, 0x91b41facu,
0, 0x00700030u, 0, 0};
fixed_storage_descriptor.dword_count = 8;
const auto fixed_storage_binding = RenderExecutorTestAccess::ResolveTexture(
executor, fixed_storage_resource, fixed_storage_descriptor);
Require(name, "fixed storage preserves physical mip layout",
fixed_storage_binding.desc.info.resources.levels == 4 &&
fixed_storage_binding.desc.info.data.size == 0xb80000 &&
fixed_storage_binding.desc.info.mip_layout[1].offset == 0x180000 &&
fixed_storage_binding.desc.view_info.base_level == 1 &&
fixed_storage_binding.desc.view_info.level_count == 1 &&
texture_cache.FindTexture(fixed_storage_binding.image_id,
fixed_storage_binding.desc) != nullptr,
"IMAGE_STORE expanded an inaccessible mip and shifted physical storage");
auto mipped_storage_resource = storage_resource;
mipped_storage_resource.mip_mode =
ShaderRecompiler::IR::ImageMipMode::Dynamic;
mipped_storage_resource.mip_count = 3;
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
ExpectFatal("MipViewPhysicalLayoutChange", [&] {
auto linear_view = overwide_mipped_storage_descriptor;
linear_view.dwords[3] &= ~(0x1fu << 20u);
(void)RenderExecutorTestAccess::ResolveTexture(executor, mipped_storage_resource,
linear_view);
});
#endif
auto sampled_overwide_resource = storage_resource;
sampled_overwide_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
sampled_overwide_resource.numeric_class =
Prospero::TextureNumericClass::Uint;
sampled_overwide_resource.read = true;
sampled_overwide_resource.written = false;
{
// MLB The Show 21 compresses a standalone final block through mip 9.
constexpr uint64_t address = base + 0x180000;
ShaderRecompiler::IR::DescriptorValue descriptor{};
descriptor.dwords = {static_cast<uint32_t>(address >> 8u),
0x03e00000u, 0, 0x9019902cu, 0, 0x00700000u, 0, 0};
descriptor.dword_count = 8;
TestCase test;
test.name = "RebasedLastMipQueryStore";
test.has_user_data = true;
std::copy_n(descriptor.dwords.begin(), 8, test.user_data.begin());
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_GET_RESINFO,
ShaderOpcode::IMAGE_STORE, ShaderOpcode::S_ENDPGM};
AppendVMovU32(&test.code, 24, 0);
AppendVMovU32(&test.code, 25, 0);
test.code.push_back(EncodeMimg0(0x0e, 0x3));
test.code.push_back(EncodeMimg1(0, 24));
test.code.push_back(EncodeMimg0(0x08, 0x3));
test.code.push_back(EncodeMimg1(0, 24));
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
auto sampled_descriptor = descriptor;
sampled_descriptor.dwords[1] |= 9u << 16u;
const auto query = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images.at(0), sampled_descriptor);
const auto store = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images.at(1), descriptor);
Require(name, "rebased last mip",
query.image_id == store.image_id &&
query.desc.info.data.size == 256 &&
query.desc.info.resources.levels == 1 &&
query.desc.view_info.base_level == 0 &&
query.desc.view_info.level_count == 1 &&
query.desc.view_info.min_lod == 0 &&
store.desc.view_info.base_level == 0,
"equivalent selected mip changed backing or relative LOD");
Image sampled;
sampled.view = texture_cache.FindTexture(query.image_id, query.desc);
sampled.layout = vk::ImageLayout::eGeneral;
Image storage_view;
storage_view.view = texture_cache.FindTexture(store.image_id, store.desc);
storage_view.layout = vk::ImageLayout::eGeneral;
texture_cache.GetImage(store.image_id).Transit(
vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
{}, scheduler.Current().Handle());
scheduler.Finish();
Dispatch(test, compiled, {}, nullptr, &sampled, nullptr, &storage_view);
texture_cache.MarkGpuWritten(store.image_id);
Require(name, "rebased mip query and store",
ReadCachedTexel(name, context, store.image_id) ==
std::vector<uint32_t>{1, 1},
"query/store did not access the standalone 1x1 mip");
}
auto plain_mipped_storage_binding =
RenderExecutorTestAccess::ResolveTexture(executor, storage_resource,
mipped_storage_descriptor);
vk::ClearValue tail_clear{};
tail_clear.color.uint32[0] = tail_gpu_value;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), plain_mipped_storage_binding.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 4, 0, 1}, tail_clear);
auto mipped_storage_binding = RenderExecutorTestAccess::ResolveTexture(
executor, mipped_storage_resource, mipped_storage_descriptor);
auto overwide_mipped_storage_binding =
RenderExecutorTestAccess::ResolveTexture(
executor, storage_resource, overwide_mipped_storage_descriptor);
auto sampled_overwide_resolved = RenderExecutorTestAccess::ResolveTexture(
executor, sampled_overwide_resource,
overwide_mipped_storage_descriptor);
const auto tail0 = ReadCachedTexel(name, context, sampled_overwide_resolved.image_id);
const auto tail3 = ReadCachedTexel(name, context, sampled_overwide_resolved.image_id,
{}, {1, 1, 1}, 0, 3);
const auto tail4 = ReadCachedTexel(name, context, sampled_overwide_resolved.image_id,
{}, {1, 1, 1}, 0, 4);
Require(name, "expanded tail preserves GPU and backing data",
tail0 == std::vector<u32>{tail_gpu_value} &&
tail3 == std::vector<u32>{tail_gpu_value} &&
tail4 == std::vector<u32>{tail_backing_value},
"expanding a shared mip tail lost GPU writes or the added mip's backing bytes");
PreparedBindings mipped_prepared{};
ShaderRecompiler::IR::CompiledShaderInfo mipped_program{};
mipped_program.info.images.push_back(storage_resource);
mipped_program.info.images.push_back(mipped_storage_resource);
mipped_program.info.images.push_back(storage_resource);
mipped_program.info.images.push_back(sampled_overwide_resource);
ShaderRecompiler::IR::ResourceSnapshot mipped_snapshot{};
mipped_snapshot.images = {mipped_storage_descriptor,
mipped_storage_descriptor,
overwide_mipped_storage_descriptor,
overwide_mipped_storage_descriptor};
ShaderStageRuntime mipped_runtime{&mipped_program, &mipped_snapshot};
mipped_prepared.runtime = &mipped_runtime;
mipped_prepared.images.push_back(
std::move(plain_mipped_storage_binding));
mipped_prepared.images.push_back(
std::move(mipped_storage_binding));
mipped_prepared.images.push_back(
std::move(overwide_mipped_storage_binding));
mipped_prepared.images.push_back(
std::move(sampled_overwide_resolved));
executor.RebindImages(mipped_prepared);
auto &plain_mipped_binding = mipped_prepared.images[0];
auto &mipped_binding = mipped_prepared.images[1];
auto &overwide_mipped_binding = mipped_prepared.images[2];
auto &sampled_overwide_binding = mipped_prepared.images[3];
plain_mipped_binding.layout = vk::ImageLayout::eGeneral;
mipped_binding.layout = vk::ImageLayout::eGeneral;
overwide_mipped_binding.layout = vk::ImageLayout::eGeneral;
sampled_overwide_binding.layout = vk::ImageLayout::eGeneral;
Require(name, "dynamic storage mip views",
plain_mipped_binding.desc.view_info.base_level == 1 &&
plain_mipped_binding.desc.view_info.level_count == 1 &&
plain_mipped_binding.mip_views.empty() &&
plain_mipped_binding.image_view ==
texture_cache.FindTexture(plain_mipped_binding.image_id,
plain_mipped_binding.desc) &&
mipped_binding.desc.info.resources.levels == 4 &&
mipped_binding.desc.view_info.base_level == 1 &&
mipped_binding.desc.view_info.level_count == 3 &&
mipped_binding.mip_views.size() == 3 &&
mipped_binding.mip_views[0] != nullptr &&
mipped_binding.mip_views[1] != nullptr &&
mipped_binding.mip_views[2] != nullptr &&
mipped_binding.mip_views[0] != mipped_binding.mip_views[1] &&
mipped_binding.mip_views[1] != mipped_binding.mip_views[2] &&
MakeImageInfo(mipped_binding, 0).imageView ==
mipped_binding.mip_views[0] &&
MakeImageInfo(mipped_binding, 1).imageView ==
mipped_binding.mip_views[1] &&
MakeImageInfo(mipped_binding, 2).imageView ==
mipped_binding.mip_views[2],
"base-1 through last-3 storage view did not create three "
"ordered one-level descriptors");
Require(name, "over-wide fixed storage mip view",
overwide_mipped_storage.LastLevel() == 4 &&
overwide_mipped_storage.MaxMip() == 3 &&
overwide_mipped_binding.image_id ==
plain_mipped_binding.image_id &&
overwide_mipped_binding.desc.info.resources.levels == 4 &&
overwide_mipped_binding.desc.view_info.base_level == 1 &&
overwide_mipped_binding.desc.view_info.level_count == 1 &&
overwide_mipped_binding.mip_views.empty() &&
overwide_mipped_binding.image_view ==
plain_mipped_binding.image_view,
"fixed storage view included mips that IMAGE_STORE cannot address");
Require(name, "over-wide sampled mip view",
sampled_overwide_binding.image_id ==
plain_mipped_binding.image_id &&
sampled_overwide_binding.desc.info.resources.levels == 5 &&
sampled_overwide_binding.desc.view_info.base_level == 1 &&
sampled_overwide_binding.desc.view_info.level_count == 4 &&
sampled_overwide_binding.mip_views.empty() &&
sampled_overwide_binding.image_view != nullptr,
"sampled view lost addressable mips in the allocated tail");
const auto *image_storage = mipped_prepared.images.data();
const auto *mip_storage = mipped_prepared.images[1].mip_views.data();
executor.PrepareBindings(mipped_runtime, mipped_prepared);
Require(name, "prepared mip scratch",
mipped_prepared.images.data() == image_storage &&
mipped_prepared.images[1].mip_views.empty() &&
mipped_prepared.images[1].mip_views.capacity() >= 3,
"repeated image preparation retained stale mip views or reallocated scratch");
executor.RebindImages(mipped_prepared);
Require(name, "prepared mip views rebound",
mipped_prepared.images[1].mip_views.size() == 3 &&
mipped_prepared.images[1].mip_views.data() == mip_storage,
"reused image scratch did not rebuild dynamic storage mip views");
// Streaming T# clamps apply after S# max LOD, without changing the view
// base.
{
constexpr uint64_t lod_address = base + 0x170000;
auto lod_descriptor = mipped_storage_descriptor;
lod_descriptor.dwords[0] = static_cast<uint32_t>(lod_address >> 8u);
lod_descriptor.dwords[1] =
static_cast<uint32_t>(lod_address >> 40u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u) |
(3u << 30u);
lod_descriptor.dwords[3] &= ~(0xfu << 12u);
ShaderSamplerResource lod_sampler{
{0, 0,
static_cast<uint32_t>(Prospero::SamplerMipFilter::kLinear) << 26u,
0}};
TestCase lod_test;
lod_test.name = "TextureMinLodAfterSamplerClamp";
lod_test.has_user_data = true;
lod_test.image_descriptor_swizzle = mipped_storage.DstSelXYZW();
std::copy_n(lod_descriptor.dwords.begin(), 8,
lod_test.user_data.begin());
std::copy_n(lod_sampler.fields, 4, lod_test.user_data.begin() + 8);
lod_test.user_data[50] = sizeof(uint32_t);
lod_test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
ShaderOpcode::BUFFER_STORE_DWORD,
ShaderOpcode::S_ENDPGM};
lod_test.required_spirv = {"OpImageSampleExplicitLod"};
AppendVMovLiteral(&lod_test.code, 20, std::bit_cast<uint32_t>(0.5f));
AppendVMovLiteral(&lod_test.code, 21, std::bit_cast<uint32_t>(0.5f));
lod_test.code.push_back(EncodeMimg0(0x27, 1));
lod_test.code.push_back(EncodeMimg1(0, 20, 0, 2));
AppendStoreVgpr(&lod_test.code, 0, 0);
AppendEnd(&lod_test.code);
const auto lod_program = CompileCase(lod_test, SubgroupSize());
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
ExpectFatal("TextureMinLodBeyondView", [&] {
auto invalid_lod = lod_descriptor;
invalid_lod.dwords[1] |= 1024u << 8u;
(void)RenderExecutorTestAccess::ResolveTexture(
executor, lod_program.program.info.images[0], invalid_lod);
});
#endif
const auto lod_binding = RenderExecutorTestAccess::ResolveTexture(
executor, lod_program.program.info.images[0], lod_descriptor);
for (uint32_t mip = 0; mip < 4; ++mip) {
vk::ClearValue clear{};
clear.color.float32[0] = static_cast<float>(1u << mip);
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), lod_binding.image_id,
{vk::ImageAspectFlagBits::eColor, mip, 1, 0, 1}, clear);
}
const auto sampler = context.GetSamplerCache().GetSampler(
lod_sampler, lod_program.program.info.samplers[0].integer_border);
auto output = CreateStorageBuffer(lod_test.name, {}, 1);
struct LodCase {
uint32_t base_level;
uint32_t min_lod;
float expected;
float integer_min_lod_expected;
};
constexpr std::array lod_cases{
LodCase{0, 0, 1.0f, 1.0f}, LodCase{0, 256, 2.0f, 2.0f},
LodCase{0, 384, 3.0f, 2.0f}, LodCase{1, 384, 3.0f, 2.0f},
LodCase{1, 512, 4.0f, 4.0f}, LodCase{1, 0, 2.0f, 2.0f},
LodCase{0, 256, 2.0f, 2.0f}};
std::array<vk::ImageView, lod_cases.size()> views{};
for (size_t index = 0; index < lod_cases.size(); ++index) {
const auto &test = lod_cases[index];
lod_descriptor.dwords[1] =
(lod_descriptor.dwords[1] & ~0xfff00u) | (test.min_lod << 8u);
lod_descriptor.dwords[3] =
(lod_descriptor.dwords[3] & ~(0xfu << 12u)) |
(test.base_level << 12u);
std::copy_n(lod_descriptor.dwords.begin(), 8,
lod_test.user_data.begin());
const auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, lod_program.program.info.images[0], lod_descriptor);
views[index] =
texture_cache.FindTexture(binding.image_id, binding.desc);
auto &image = texture_cache.GetImage(binding.image_id);
image.Transit(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, {},
scheduler.Current().Handle());
Image sampled;
sampled.view = views[index];
sampled.layout = image.backing.state.layout;
scheduler.Finish();
Dispatch(lod_test, lod_program, output, nullptr, &sampled, nullptr,
nullptr, sampler);
const auto result = ReadBuffer(lod_test.name, output, 1)[0];
// Vulkan permits flooring imageViewMinLod instead of retaining its
// fraction.
Require(lod_test.name, "sampled mip",
result == std::bit_cast<uint32_t>(test.expected) ||
result == std::bit_cast<uint32_t>(
test.integer_min_lod_expected),
"texture clamp was lost, applied before sampler max LOD, or "
"rebased incorrectly");
}
Require(lod_test.name, "view identity",
views[0] != views[1] && views[1] != views[2] &&
views[2] != views[3] && views[1] == views.back(),
"different texture clamps aliased or identical clamped views "
"were not reused");
ShaderRecompiler::IR::CompiledShaderInfo gather_info{};
gather_info.stage = ShaderType::Compute;
gather_info.info = lod_program.program.info;
gather_info.info.images[0].mip_mode =
ShaderRecompiler::IR::ImageMipMode::Dynamic;
gather_info.info.images[0].mip_count = 4;
auto gather_snapshot = lod_program.resources;
gather_snapshot.images[0] = lod_descriptor;
ShaderStageRuntime gather_runtime{&gather_info, &gather_snapshot};
PreparedBindings gather_bindings;
executor.PrepareBindings(gather_runtime, gather_bindings);
executor.RebindImages(gather_bindings);
const auto &gather_binding = gather_bindings.images[0];
Require(lod_test.name, "dynamic sampled mip views",
gather_binding.desc.view_info.min_lod == 256u &&
gather_binding.mip_views.size() == 4,
"dynamic gather lost the guest minimum or accessible mip range");
for (uint32_t mip = 0; mip < 4; ++mip) {
auto single_mip = gather_binding.desc;
single_mip.view_info.base_level = mip;
single_mip.view_info.level_count = 1;
single_mip.view_info.min_lod = 0;
Require(lod_test.name, "gather minimum applied once",
gather_binding.mip_views[mip] == texture_cache.FindTexture(
gather_binding.image_id, single_mip),
"the gather mip view reapplied the shader's streaming clamp");
}
DestroyBuffer(&output);
RenderExecutorTestAccess::ResetBindings(executor);
}
auto srgb_storage = storage;
constexpr auto srgb_format =
static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8Srgb);
constexpr uint64_t srgb_storage_address = base + 0xc0000;
const auto encoded_srgb_storage_address = srgb_storage_address >> 8u;
srgb_storage.fields[0] =
static_cast<uint32_t>(encoded_srgb_storage_address);
srgb_storage.fields[1] =
static_cast<uint32_t>(encoded_srgb_storage_address >> 32u) |
(srgb_format << 20u);
ShaderRecompiler::IR::DescriptorValue srgb_storage_descriptor{};
std::copy(std::begin(srgb_storage.fields), std::end(srgb_storage.fields),
srgb_storage_descriptor.dwords.begin());
srgb_storage_descriptor.dword_count = 8;
auto srgb_storage_resource = storage_resource;
srgb_storage_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Storage;
srgb_storage_resource.numeric_class =
Prospero::TextureNumericClass::Float;
const auto srgb_storage_binding =
RenderExecutorTestAccess::ResolveTexture(
executor, srgb_storage_resource, srgb_storage_descriptor);
const auto srgb_storage_view = texture_cache.FindTexture(
srgb_storage_binding.image_id, srgb_storage_binding.desc);
Require(name, "sRGB storage view",
srgb_storage_view != nullptr &&
srgb_storage_binding.desc.info.pixel_format ==
vk::Format::eR8G8B8A8Srgb &&
srgb_storage_binding.desc.view_info.format ==
vk::Format::eR8G8B8A8Unorm &&
texture_cache.GetImage(srgb_storage_binding.image_id)
.backing.format == vk::Format::eR8G8B8A8Srgb,
"storage descriptor did not preserve its sRGB backing and "
"select an UNORM Vulkan view");
ShaderTextureResource sint_storage{{0x01514b00u, 0xc1500000u, 0x000bc00bu,
0x91b00204u, 0x00000000u, 0x00700000u,
0x102b0000u, 0x0001514au}};
Require(name, "PPSA06888 R32 SINT descriptor",
sint_storage.Base40() == 0x1514b0000ull &&
sint_storage.Width5() + 1u == 48 &&
sint_storage.Height5() + 1u == 48 &&
sint_storage.Depth() + 1u == 1 &&
sint_storage.Format() == Prospero::BufferFormat::k32SInt &&
sint_storage.Type() == Prospero::ImageType::kColor2D &&
sint_storage.TileMode() ==
Prospero::TileMode::kRenderTarget &&
sint_storage.DstSelXYZW() == DstSel(4, 0, 0, 1),
"captured write-only signed storage descriptor was decoded "
"incorrectly");
const uint64_t mapped_sint_address = base + 0xe0000;
const auto encoded_sint_address = mapped_sint_address >> 8u;
sint_storage.fields[0] = static_cast<uint32_t>(encoded_sint_address);
sint_storage.fields[1] =
(sint_storage.fields[1] & ~0xffu) |
static_cast<uint32_t>(encoded_sint_address >> 32u);
constexpr uint64_t sint_metadata_address = base + 0x160000;
const auto encoded_sint_metadata = sint_metadata_address >> 8u;
sint_storage.fields[6] = (sint_storage.fields[6] & 0x00ffffffu) |
(static_cast<uint32_t>(encoded_sint_metadata & 0xffu) << 24u);
sint_storage.fields[7] = static_cast<uint32_t>(encoded_sint_metadata >> 8u);
WriteMetadata(context, sint_metadata_address, 0x1000, UINT32_MAX);
ShaderRecompiler::IR::DescriptorValue sint_storage_descriptor{};
std::copy(std::begin(sint_storage.fields), std::end(sint_storage.fields),
sint_storage_descriptor.dwords.begin());
sint_storage_descriptor.dword_count = 8;
auto sint_storage_resource = srgb_storage_resource;
sint_storage_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Storage;
sint_storage_resource.numeric_class = Prospero::TextureNumericClass::Uint;
const auto sint_storage_binding =
RenderExecutorTestAccess::ResolveTexture(
executor, sint_storage_resource, sint_storage_descriptor);
const auto sint_storage_view = texture_cache.FindTexture(
sint_storage_binding.image_id, sint_storage_binding.desc);
Require(name, "PPSA06888 raw R32 SINT storage view",
sint_storage_view != nullptr &&
sint_storage_binding.desc.info.metadata.range ==
GuestRange{sint_metadata_address, 0x1000} &&
sint_storage_binding.desc.info.data.size == 0x10000 &&
sint_storage_binding.desc.info.pixel_format ==
vk::Format::eR32Sint &&
sint_storage_binding.desc.view_info.format ==
vk::Format::eR32Uint &&
texture_cache.GetImage(sint_storage_binding.image_id)
.backing.format == vk::Format::eR32Sint,
"write-only R32 SINT storage did not select a bit-compatible "
"uint view");
auto narrowed_storage = storage;
constexpr uint64_t narrowed_storage_address = base + 0xd0000;
const auto encoded_narrowed_address = narrowed_storage_address >> 8u;
narrowed_storage.fields[0] =
static_cast<uint32_t>(encoded_narrowed_address);
narrowed_storage.fields[1] =
static_cast<uint32_t>(encoded_narrowed_address >> 32u) |
(static_cast<uint32_t>(stencil_format) << 20u);
narrowed_storage.fields[3] =
DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor1DArray) << 28u);
narrowed_storage.fields[4] = 1u | (1u << 16u);
ShaderRecompiler::IR::DescriptorValue narrowed_storage_descriptor{};
std::copy(std::begin(narrowed_storage.fields),
std::end(narrowed_storage.fields),
narrowed_storage_descriptor.dwords.begin());
narrowed_storage_descriptor.dword_count = 8;
auto narrowed_storage_resource = storage_resource;
narrowed_storage_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim1D;
const auto narrowed_storage_binding =
RenderExecutorTestAccess::ResolveTexture(
executor, narrowed_storage_resource, narrowed_storage_descriptor);
const auto narrowed_storage_view = texture_cache.FindTexture(
narrowed_storage_binding.image_id, narrowed_storage_binding.desc);
const auto &narrowed_storage_image =
texture_cache.GetImage(narrowed_storage_binding.image_id);
Require(name, "narrowed 1D array storage view",
narrowed_storage_view != nullptr &&
narrowed_storage_binding.desc.info.type ==
Prospero::ImageType::kColor1D &&
narrowed_storage_binding.desc.info.resources.layers == 2 &&
narrowed_storage_binding.desc.view_info.type ==
vk::ImageViewType::e1D &&
narrowed_storage_binding.desc.view_info.base_layer == 1 &&
narrowed_storage_binding.desc.view_info.layer_count == 1 &&
narrowed_storage_image.backing.image_type ==
vk::ImageType::e1D,
"non-array 1D specialization did not select the descriptor "
"base layer from its 1D-array backing");
ShaderRecompiler::IR::ResourceSnapshot storage_snapshot{};
storage_snapshot.images.push_back(storage_descriptor);
ShaderRecompiler::IR::CompiledShaderInfo storage_info{};
storage_info.stage = storage_program.stage;
storage_info.info = std::move(storage_program.info);
storage_info.bindings = std::move(storage_program.bindings);
ShaderStageRuntime storage_runtime{&storage_info, &storage_snapshot};
PreparedBindings storage_discovery;
executor.PrepareBindings(storage_runtime, storage_discovery);
const auto storage_id = storage_discovery.images[0].image_id;
Require(name, "storage prefetch purity",
storage_discovery.images[0].image_view == nullptr &&
texture_cache.GetImage(storage_id).binding.is_bound &&
!texture_cache.GetImage(storage_id).usage.storage &&
!TextureCacheTestAccess::PendingDownload(texture_cache,
storage_id),
"the first descriptor pass performed final storage acquisition");
RenderExecutorTestAccess::ResetBindings(executor);
ShaderRecompiler::IR::Program sampled_program{};
sampled_program.stage = ShaderType::Pixel;
sampled_program.resource_tracking_complete = true;
ShaderRecompiler::IR::ImageResource sampled_resource{};
sampled_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
sampled_resource.numeric_class = Prospero::TextureNumericClass::Uint;
sampled_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
sampled_resource.read = true;
sampled_program.info.images.push_back(sampled_resource);
set_image_bindings(sampled_program);
ShaderRecompiler::IR::ResourceSnapshot sampled_snapshot{};
sampled_snapshot.images.push_back(storage_descriptor);
ShaderRecompiler::IR::CompiledShaderInfo sampled_info{};
sampled_info.stage = sampled_program.stage;
sampled_info.info = std::move(sampled_program.info);
sampled_info.bindings = std::move(sampled_program.bindings);
ShaderStageRuntime sampled_runtime{&sampled_info, &sampled_snapshot};
constexpr uint64_t ordered_sampled_address = base + 0x10000;
const uint32_t ordered_sampled_value = 0x89abcdefu;
std::memcpy(reinterpret_cast<uint8_t *>(mapped) + 0x10000,
&ordered_sampled_value, sizeof(ordered_sampled_value));
auto ordered_sampled = storage;
const auto encoded_ordered_address = ordered_sampled_address >> 8u;
ordered_sampled.fields[0] =
static_cast<uint32_t>(encoded_ordered_address);
ordered_sampled.fields[1] =
static_cast<uint32_t>(encoded_ordered_address >> 32u) |
(static_cast<uint32_t>(stencil_format) << 20u);
ShaderRecompiler::IR::DescriptorValue ordered_descriptor{};
std::copy(std::begin(ordered_sampled.fields),
std::end(ordered_sampled.fields),
ordered_descriptor.dwords.begin());
ordered_descriptor.dword_count = 8;
ShaderRecompiler::IR::ResourceSnapshot ordered_snapshot{};
ordered_snapshot.images.push_back(ordered_descriptor);
ShaderStageRuntime ordered_sampled_runtime{sampled_runtime.program,
&ordered_snapshot};
auto ordered_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, storage_runtime, ordered_sampled_runtime, true);
const auto ordered_sampled_id =
ordered_bindings.pixel->images[0].image_id;
Require(
name, "VS-before-PS retained-owner order",
ordered_bindings.vertex[0].images[0].image_id == storage_id &&
ordered_sampled_id != storage_id &&
RenderExecutorTestAccess::BoundImagesInOrder(executor, storage_id,
ordered_sampled_id),
"production graphics binding did not retain vertex resources "
"before pixel resources");
RenderExecutorTestAccess::ResetBindings(executor);
auto graphics_bindings =
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, storage_runtime, sampled_runtime, true);
const auto &storage_binding =
graphics_bindings.vertex[0].images[0];
const auto &sampled_binding =
graphics_bindings.pixel->images[0];
Require(name, "storage final acquisition",
storage_binding.image_view != nullptr &&
texture_cache.GetImage(storage_id).usage.storage &&
texture_cache.GetImage(storage_id).IsGpuModified() &&
TextureCacheTestAccess::PendingDownload(texture_cache,
storage_id),
"the production graphics binding path did not establish GPU "
"ownership before storage readback");
Require(name, "VS-to-PS image acquisition order",
storage_binding.image_id == storage_id &&
sampled_binding.image_id == storage_id &&
texture_cache.GetImage(storage_id).binding.force_general &&
texture_cache.GetImage(storage_id).binding.shader_write &&
texture_cache.GetImage(storage_id).IsGpuModified() &&
texture_cache.GetImage(storage_id).usage.storage &&
texture_cache.GetImage(storage_id).usage.texture,
"the production graphics binding path did not complete vertex "
"storage acquisition before pixel sampling");
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), graphics_bindings.vertex[0],
*graphics_bindings.pixel));
Require(name, "early writable alias retention",
texture_cache.GetImage(storage_id).backing.state.access_mask ==
(vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eShaderWrite),
"a later sampled alias dropped an earlier storage write access");
RenderExecutorTestAccess::ResetBindings(executor);
ShaderRecompiler::IR::Program vertex_sampled_program{};
vertex_sampled_program.stage = ShaderType::Vertex;
vertex_sampled_program.resource_tracking_complete = true;
vertex_sampled_program.info.images.push_back(sampled_resource);
set_image_bindings(vertex_sampled_program);
ShaderRecompiler::IR::CompiledShaderInfo vertex_sampled_info{};
vertex_sampled_info.stage = vertex_sampled_program.stage;
vertex_sampled_info.info = std::move(vertex_sampled_program.info);
vertex_sampled_info.bindings =
std::move(vertex_sampled_program.bindings);
ShaderStageRuntime vertex_sampled_runtime{
&vertex_sampled_info, sampled_runtime.resources};
ShaderRecompiler::IR::Program pixel_storage_program{};
pixel_storage_program.stage = ShaderType::Pixel;
pixel_storage_program.resource_tracking_complete = true;
pixel_storage_program.info.images.push_back(storage_resource);
set_image_bindings(pixel_storage_program);
ShaderRecompiler::IR::CompiledShaderInfo pixel_storage_info{};
pixel_storage_info.stage = pixel_storage_program.stage;
pixel_storage_info.info = std::move(pixel_storage_program.info);
pixel_storage_info.bindings = std::move(pixel_storage_program.bindings);
ShaderStageRuntime pixel_storage_runtime{&pixel_storage_info,
storage_runtime.resources};
auto writable_alias_bindings =
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, vertex_sampled_runtime, pixel_storage_runtime, true);
Require(name, "late writable alias promotion",
texture_cache.GetImage(storage_id).binding.force_general,
"an already sampled image was not promoted when a later "
"storage alias bound the same backing");
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), writable_alias_bindings.vertex[0],
*writable_alias_bindings.pixel));
Require(
name, "forced-general descriptor capture",
writable_alias_bindings.vertex[0].images[0].layout ==
vk::ImageLayout::eGeneral &&
writable_alias_bindings.pixel->images[0].layout ==
vk::ImageLayout::eGeneral &&
MakeImageInfo(writable_alias_bindings.vertex[0].images[0])
.imageLayout == vk::ImageLayout::eGeneral &&
MakeImageInfo(writable_alias_bindings.pixel->images[0])
.imageLayout == vk::ImageLayout::eGeneral &&
texture_cache.GetImage(storage_id).backing.state.layout ==
vk::ImageLayout::eGeneral &&
texture_cache.GetImage(storage_id).backing.state.access_mask ==
(vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eShaderWrite),
"sampled/storage aliases did not retain the promoted general "
"layout and writable access in both generated descriptors");
RenderExecutorTestAccess::ResetBindings(executor);
const std::array<uint32_t, 3> split_mip_data{0x01020304u, 0x05060708u,
0x090a0b0cu};
std::memcpy(reinterpret_cast<uint8_t *>(mapped) + 0x50000,
split_mip_data.data(), sizeof(split_mip_data));
ImageDesc split_desc{};
split_desc.type = BindingType::Texture;
split_desc.info.data = {base + 0x50000, sizeof(split_mip_data)};
split_desc.info.pixel_format = vk::Format::eR32Uint;
split_desc.info.guest_format = Prospero::BufferFormat::k32UInt;
split_desc.info.type = Prospero::ImageType::kColor2D;
split_desc.info.extent = {2, 1, 1};
split_desc.info.resources = {2, 1};
split_desc.info.pitch = 2;
split_desc.info.bytes_per_block = 4;
split_desc.info.samples = 1;
split_desc.info.tile_mode = linear;
split_desc.info.mip_layout[0] = {0, 8, 2, 1};
split_desc.info.mip_layout[1] = {8, 4, 1, 1};
split_desc.view_info.format = split_desc.info.pixel_format;
split_desc.view_info.type = vk::ImageViewType::e2D;
split_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
split_desc.view_info.level_count = 2;
split_desc.view_info.layer_count = 1;
split_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto split_id = texture_cache.FindImage(split_desc);
auto split_storage_desc = split_desc;
split_storage_desc.type = BindingType::Storage;
split_storage_desc.view_info.base_level = 0;
split_storage_desc.view_info.level_count = 1;
split_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
auto split_sampled_desc = split_desc;
split_sampled_desc.view_info.base_level = 1;
split_sampled_desc.view_info.level_count = 1;
split_sampled_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
ShaderRecompiler::IR::Program split_ir{};
split_ir.stage = ShaderType::Vertex;
split_ir.resource_tracking_complete = true;
split_ir.info.images = {storage_resource, sampled_resource};
set_image_bindings(split_ir);
ShaderRecompiler::IR::CompiledShaderInfo split_program{};
split_program.stage = split_ir.stage;
split_program.info = std::move(split_ir.info);
split_program.bindings = std::move(split_ir.bindings);
PreparedBindings split_bindings{};
ShaderRecompiler::IR::ResourceSnapshot split_snapshot;
ShaderStageRuntime split_runtime{&split_program, &split_snapshot};
split_bindings.runtime = &split_runtime;
split_bindings.images.push_back(
{split_id, texture_cache.FindTexture(split_id, split_storage_desc),
split_storage_desc});
split_bindings.images.push_back(
{split_id, texture_cache.FindTexture(split_id, split_sampled_desc),
split_sampled_desc});
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), split_bindings));
const auto &split_image = texture_cache.GetImage(split_id);
Require(
name, "per-binding subresource layouts",
split_bindings.images[0].layout ==
vk::ImageLayout::eGeneral &&
split_bindings.images[1].layout ==
vk::ImageLayout::eShaderReadOnlyOptimal &&
MakeImageInfo(split_bindings.images[0]).imageLayout ==
vk::ImageLayout::eGeneral &&
MakeImageInfo(split_bindings.images[1]).imageLayout ==
vk::ImageLayout::eShaderReadOnlyOptimal &&
split_image.backing.subresource_states.size() == 2 &&
split_image.backing.subresource_states[0].layout ==
vk::ImageLayout::eGeneral &&
split_image.backing.subresource_states[1].layout ==
vk::ImageLayout::eShaderReadOnlyOptimal,
"descriptor layouts were queried after a later mip transition "
"instead of being captured at each binding");
auto disjoint_depth_desc = split_desc;
disjoint_depth_desc.type = BindingType::DepthTarget;
disjoint_depth_desc.info.data.address = base + 0xe00000;
disjoint_depth_desc.info.pixel_format = vk::Format::eD32Sfloat;
disjoint_depth_desc.info.guest_format = Prospero::BufferFormat::k32Float;
disjoint_depth_desc.view_info.format = vk::Format::eD32Sfloat;
disjoint_depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
disjoint_depth_desc.view_info.level_count = 1;
disjoint_depth_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto disjoint_depth_id = texture_cache.FindImage(disjoint_depth_desc);
auto disjoint_sampled_desc = disjoint_depth_desc;
disjoint_sampled_desc.type = BindingType::Texture;
disjoint_sampled_desc.view_info.base_level = 1;
disjoint_sampled_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto disjoint_sampled_view =
texture_cache.FindTexture(disjoint_depth_id, disjoint_sampled_desc);
RenderDepthInfo disjoint_depth{};
disjoint_depth.desc = disjoint_depth_desc;
disjoint_depth.image_id = disjoint_depth_id;
disjoint_depth.depth_test_enable = true;
disjoint_depth.depth_write_enable = true;
disjoint_depth.depth_compare_op = vk::CompareOp::eAlways;
PreparedBindings disjoint_binding{};
disjoint_binding.runtime = &sampled_runtime;
disjoint_binding.images.push_back(
{disjoint_depth_id, disjoint_sampled_view, disjoint_sampled_desc});
RenderExecutorTestAccess::BindRenderTarget(executor, disjoint_depth_id);
vk::ImageAspectFlags disjoint_feedback;
std::array<PreparedBindings *, 1> disjoint_stages{&disjoint_binding};
RenderColorInfo no_disjoint_color{};
const auto disjoint_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_disjoint_color, 0, disjoint_depth,
disjoint_stages, &disjoint_feedback);
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), disjoint_binding));
const auto &disjoint_image = texture_cache.GetImage(disjoint_depth_id);
Require(name, "disjoint depth sampling during depth writes",
!disjoint_feedback &&
disjoint_rendering.depth_stencil_attachment.image_layout ==
vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT &&
disjoint_binding.images[0].layout ==
disjoint_rendering.depth_stencil_attachment.image_layout &&
disjoint_image.backing.state.layout ==
disjoint_rendering.depth_stencil_attachment.image_layout &&
disjoint_image.backing.subresource_states.empty(),
"disjoint sampling used inconsistent image layouts or enabled feedback");
RenderExecutorTestAccess::ResetBindings(executor);
auto overlapping_sampled_desc = disjoint_sampled_desc;
overlapping_sampled_desc.view_info.base_level = 0;
const auto overlapping_view = texture_cache.FindTexture(
disjoint_depth_id, overlapping_sampled_desc);
ShaderRecompiler::IR::Program depth_views_ir{};
depth_views_ir.stage = ShaderType::Pixel;
depth_views_ir.resource_tracking_complete = true;
depth_views_ir.info.images = {sampled_resource, sampled_resource};
set_image_bindings(depth_views_ir);
ShaderRecompiler::IR::CompiledShaderInfo depth_views_program{};
depth_views_program.stage = depth_views_ir.stage;
depth_views_program.info = std::move(depth_views_ir.info);
depth_views_program.bindings = std::move(depth_views_ir.bindings);
ShaderRecompiler::IR::ResourceSnapshot depth_views_snapshot;
ShaderStageRuntime depth_views_runtime{&depth_views_program, &depth_views_snapshot};
PreparedBindings depth_views_binding{};
depth_views_binding.runtime = &depth_views_runtime;
depth_views_binding.images.push_back(
{disjoint_depth_id, overlapping_view, overlapping_sampled_desc});
depth_views_binding.images.push_back(
{disjoint_depth_id, disjoint_sampled_view, disjoint_sampled_desc});
RenderExecutorTestAccess::BindRenderTarget(executor, disjoint_depth_id);
std::array<PreparedBindings *, 1> depth_views_stages{&depth_views_binding};
vk::ImageAspectFlags depth_views_feedback;
const auto depth_views_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_disjoint_color, 0, disjoint_depth,
depth_views_stages, &depth_views_feedback);
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), depth_views_binding));
Require(name, "overlapping and disjoint depth views",
overlapping_view != nullptr &&
depth_views_feedback == vk::ImageAspectFlagBits::eDepth &&
depth_views_rendering.depth_stencil_attachment.image_layout ==
vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT &&
MakeImageInfo(depth_views_binding.images[0]).imageLayout ==
depth_views_rendering.depth_stencil_attachment.image_layout &&
MakeImageInfo(depth_views_binding.images[1]).imageLayout ==
depth_views_rendering.depth_stencil_attachment.image_layout,
"two depth views chose different descriptor layouts or feedback aspects");
Libs::LibKernel::Memory::WriteBacking(
storage_address, &storage_stale_value, sizeof(storage_stale_value));
texture_cache.ProcessDownloadImages();
Require(
name, "storage acquisition download consumption",
!TextureCacheTestAccess::PendingDownload(texture_cache, storage_id),
"submit-time processing retained an acquired storage request");
RenderExecutorTestAccess::ResetBindings(executor);
scheduler.Finish();
scheduler.DrainPriorityOperations();
uint32_t storage_downloaded_value = 0;
Libs::LibKernel::Memory::TryReadBacking(storage_address,
&storage_downloaded_value,
sizeof(storage_downloaded_value));
const uint32_t storage_expected_value =
(storage_stale_value & 0xffffff00u) | (storage_native_value & 0xffu);
Require(name, "storage acquisition readback window",
storage_downloaded_value == storage_expected_value &&
texture_cache.GetImage(storage_id).IsGpuModified(),
"readback enrollment was consumed before final storage "
"ownership became visible");
constexpr uint64_t phased_depth_address = base + 0x90000;
constexpr uint64_t phased_stencil_address = base + 0xa0000;
constexpr uint64_t phased_htile_address = base + 0xb0000;
HW::DepthRenderTarget phased_depth_target{};
phased_depth_target.z_info.format = Prospero::DepthFormat::kZ32F;
phased_depth_target.z_info.texture_compatibility =
Prospero::TextureCompatiblePlaneCompression::kEnable;
phased_depth_target.z_info.htile_acceleration = true;
phased_depth_target.z_info.z_compare_base = Prospero::ZCompareBase::kZMax;
phased_depth_target.stencil_info.format = Prospero::StencilFormat::k8UInt;
phased_depth_target.stencil_info.texture_compatibility =
Prospero::TextureCompatibleStencil::kEnable;
phased_depth_target.stencil_info.htile_stencil_disabled = false;
phased_depth_target.z_read_base_addr = phased_depth_address;
phased_depth_target.z_write_base_addr = phased_depth_address;
phased_depth_target.stencil_read_base_addr = phased_stencil_address;
phased_depth_target.stencil_write_base_addr = phased_stencil_address;
phased_depth_target.htile_data_base_addr = phased_htile_address;
phased_depth_target.size.valid = true;
registers.SetDepthRenderTarget(phased_depth_target);
HW::DepthControl phased_depth_control{};
phased_depth_control.z_write_enable = true;
registers.SetDepthControl(phased_depth_control);
registers.SetRenderControl({});
RenderDepthInfo disabled_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), disabled_depth);
Require(name, "disabled tests with stale depth write enable",
!disabled_depth.image_id,
"a dormant depth write bit bound a depth attachment");
phased_depth_control.z_enable = true;
registers.SetDepthControl(phased_depth_control);
HW::RenderControl phased_render_control{};
phased_render_control.resummarize_enable = true;
registers.SetRenderControl(phased_render_control);
RenderDepthInfo phased_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), phased_depth);
// PRT permits missing backing; a fully resident target keeps the same layout and image.
phased_depth_target.z_info.partially_resident = true;
phased_depth_target.stencil_info.partially_resident = true;
registers.SetDepthRenderTarget(phased_depth_target);
RenderDepthInfo resident_prt_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), resident_prt_depth);
Require(name, "fully resident PRT depth identity",
resident_prt_depth.image_id == phased_depth.image_id &&
resident_prt_depth.desc.info.data == phased_depth.desc.info.data &&
resident_prt_depth.desc.info.stencil == phased_depth.desc.info.stencil,
"PRT permission changed a fully backed depth/stencil attachment");
auto non_texture_compatible_target = phased_depth_target;
non_texture_compatible_target.z_info.texture_compatibility =
Prospero::TextureCompatiblePlaneCompression::kDisable;
non_texture_compatible_target.stencil_info.texture_compatibility =
Prospero::TextureCompatibleStencil::kDisable;
registers.SetDepthRenderTarget(non_texture_compatible_target);
RenderDepthInfo non_texture_compatible_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), non_texture_compatible_depth);
Require(
name, "depth texture compatibility identity",
phased_depth.image_id && (phased_depth.desc.info.metadata.kind == ImageMetadataKind::Htile) &&
non_texture_compatible_depth.image_id == phased_depth.image_id &&
non_texture_compatible_depth.desc.info.data ==
phased_depth.desc.info.data &&
non_texture_compatible_depth.desc.info.stencil ==
phased_depth.desc.info.stencil &&
non_texture_compatible_depth.desc.info.metadata.range ==
phased_depth.desc.info.metadata.range &&
phased_depth.desc.info.metadata.stencil_compressed &&
!phased_depth.depth_clear_enable &&
!texture_cache.IsMeta(phased_depth.desc.info.metadata.range.address) &&
!texture_cache.GetImage(phased_depth.image_id).IsGpuModified() &&
!texture_cache.GetImage(phased_depth.image_id).usage.depth_target,
"valid PS5 texture-compatibility policy changed logical depth image "
"identity or discovery side effects");
auto read_only_depth_target = phased_depth_target;
read_only_depth_target.z_write_base_addr = 0;
read_only_depth_target.stencil_write_base_addr = 0;
read_only_depth_target.depth_view.depth_write_disable = true;
read_only_depth_target.depth_view.stencil_write_disable = true;
registers.SetDepthRenderTarget(read_only_depth_target);
auto read_only_depth_control = phased_depth_control;
read_only_depth_control.z_write_enable = false;
read_only_depth_control.zfunc =
static_cast<uint8_t>(vk::CompareOp::eAlways);
read_only_depth_control.stencil_enable = true;
read_only_depth_control.stencilfunc =
static_cast<uint8_t>(vk::CompareOp::eAlways);
registers.SetDepthControl(read_only_depth_control);
HW::StencilControl read_only_stencil_control{};
read_only_stencil_control.stencil_zpass =
static_cast<uint8_t>(Prospero::StencilOp::kReplaceOp);
registers.SetStencilControl(read_only_stencil_control);
HW::StencilMask read_only_stencil_mask{};
read_only_stencil_mask.stencil_testval = 0x12;
read_only_stencil_mask.stencil_mask = 0xff;
read_only_stencil_mask.stencil_writemask = 0xff;
read_only_stencil_mask.stencil_opval = 0x34;
registers.SetStencilMask(read_only_stencil_mask);
RenderDepthInfo read_only_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), read_only_depth);
Require(
name, "read-only depth/stencil target without write addresses",
read_only_depth.image_id == phased_depth.image_id &&
read_only_depth.desc.info.data.address == phased_depth_address &&
read_only_depth.desc.info.stencil.address == phased_stencil_address &&
!read_only_depth.depth_write_enable &&
!read_only_depth.stencil_clear_enable &&
read_only_depth.stencil_test_enable &&
read_only_depth.stencil_front.writeMask == 0 &&
read_only_depth.stencil_back.writeMask == 0 &&
read_only_depth.stencil_front.passOp ==
vk::StencilOp::eKeep &&
!read_only_depth.AttachmentWriteAspects() &&
depth_attachment_layout(read_only_depth) ==
vk::ImageLayout::eDepthStencilReadOnlyOptimal,
"a PS5 read-only depth/stencil target required write addresses or "
"retained host writes");
auto read_only_render_control = phased_render_control;
read_only_render_control.stencil_clear_enable = true;
registers.SetRenderControl(read_only_render_control);
RenderDepthInfo read_only_clear_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), read_only_clear_depth);
Require(name, "read-only stencil clear suppression",
read_only_clear_depth.image_id == phased_depth.image_id &&
!read_only_clear_depth.stencil_clear_enable &&
!read_only_clear_depth.AttachmentWriteAspects() &&
depth_attachment_layout(read_only_clear_depth) ==
vk::ImageLayout::eDepthStencilReadOnlyOptimal,
"a PS5 target-level stencil write disable retained a host clear");
registers.SetDepthRenderTarget(phased_depth_target);
registers.SetDepthControl(phased_depth_control);
registers.SetStencilControl({});
registers.SetStencilMask({});
registers.SetRenderControl(phased_render_control);
RenderColorInfo no_color{};
auto phased_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_color, 0, phased_depth);
Require(
name, "HTile final acquisition",
phased_rendering.depth_stencil_attachment.image_view != nullptr &&
phased_rendering.num_color_attachments == 0 &&
phased_rendering.depth_stencil_attachment.has_depth &&
phased_rendering.depth_stencil_attachment.depth_clear &&
phased_depth.depth_load_clear_enable &&
texture_cache.IsMeta(phased_depth.desc.info.metadata.range.address) &&
!texture_cache.IsMetaCleared(
phased_depth.desc.info.metadata.range.address,
phased_depth.desc.view_info.base_layer) &&
texture_cache.GetImage(phased_depth.image_id).IsGpuModified() &&
texture_cache.GetImage(phased_depth.image_id).usage.depth_target,
"final depth acquisition did not publish and consume HTile "
"state at the final acquisition boundary");
RenderExecutorTestAccess::ResetBindings(executor);
constexpr uint64_t depth_only_address = base + 0x140000;
constexpr uint64_t depth_only_htile_address = base + 0x150000;
constexpr uint32_t captured_depth_only_z_info = 0x22900983u;
constexpr uint32_t captured_depth_only_stencil_info = 0x00100980u;
HW::DepthRenderTarget depth_only_target{};
depth_only_target.z_info =
HW::DepthZInfo::Decode(captured_depth_only_z_info);
depth_only_target.stencil_info =
HW::DepthStencilInfo::Decode(captured_depth_only_stencil_info);
depth_only_target.z_read_base_addr = depth_only_address;
depth_only_target.z_write_base_addr = depth_only_address;
depth_only_target.htile_data_base_addr = depth_only_htile_address;
depth_only_target.size = {63, 63, true};
registers.SetDepthRenderTarget(depth_only_target);
HW::DepthControl depth_only_control{};
depth_only_control.stencil_enable = true;
depth_only_control.z_enable = true;
depth_only_control.z_write_enable = true;
depth_only_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eAlways);
registers.SetDepthControl(depth_only_control);
HW::RenderControl depth_only_render_control{};
depth_only_render_control.depth_clear_enable = true;
depth_only_render_control.stencil_clear_enable = true;
registers.SetRenderControl(depth_only_render_control);
RenderDepthInfo depth_only{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), depth_only);
Require(
name, "depth-only target with stale stencil state",
depth_only.image_id && depth_only.desc.view_info.format == vk::Format::eD32Sfloat &&
(depth_only.desc.info.metadata.kind == ImageMetadataKind::Htile) &&
depth_only.depth_test_enable && !depth_only.depth_write_enable &&
depth_only.depth_clear_enable &&
!depth_only.stencil_test_enable &&
!depth_only.stencil_clear_enable &&
depth_only.desc.info.stencil.Empty() &&
depth_only.desc.info.metadata.range.address ==
depth_only_htile_address &&
depth_only.desc.info.data.size != 0 &&
depth_only_address + depth_only.desc.info.data.size <=
base + allocation_size &&
depth_only.AttachmentWriteAspects() ==
vk::ImageAspectFlagBits::eDepth,
"inactive stencil state changed a depth-only HTile attachment");
RenderExecutorTestAccess::ResetBindings(executor);
const auto sampled_width = depth_only.desc.info.extent.width - 1u;
const auto sampled_height = depth_only.desc.info.extent.height - 1u;
const auto sampled_address = depth_only_address >> 8u;
ShaderTextureResource sampled_depth_descriptor{};
sampled_depth_descriptor.fields[0] =
static_cast<uint32_t>(sampled_address);
sampled_depth_descriptor.fields[1] =
static_cast<uint32_t>(sampled_address >> 32u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u) |
((sampled_width & 3u) << 30u);
sampled_depth_descriptor.fields[2] =
(sampled_width >> 2u) | (sampled_height << 14u);
sampled_depth_descriptor.fields[3] =
DstSel(4, 4, 4, 4) |
(static_cast<uint32_t>(Prospero::TileMode::kDepth) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
sampled_depth_descriptor.fields[5] = 0x00700000u;
ShaderRecompiler::IR::DescriptorValue sampled_depth_value{};
std::copy(std::begin(sampled_depth_descriptor.fields),
std::end(sampled_depth_descriptor.fields),
sampled_depth_value.dwords.begin());
sampled_depth_value.dword_count = 8;
ShaderRecompiler::IR::ImageResource sampled_depth_resource{};
sampled_depth_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
sampled_depth_resource.numeric_class =
Prospero::TextureNumericClass::Uint;
sampled_depth_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
sampled_depth_resource.read = true;
const auto sampled_depth_backing =
texture_cache.GetImage(depth_only.image_id).backing.image;
auto sampled_depth_binding = RenderExecutorTestAccess::ResolveTexture(
executor, sampled_depth_resource, sampled_depth_value);
const auto sampled_depth_view = texture_cache.FindTexture(
sampled_depth_binding.image_id, sampled_depth_binding.desc);
const auto &sampled_depth_image =
texture_cache.GetImage(sampled_depth_binding.image_id);
const bool normalized_depth_view = std::ranges::any_of(
sampled_depth_image.views, [&](const auto &cached) {
return cached.view == sampled_depth_view &&
cached.info.format == vk::Format::eD32Sfloat &&
cached.info.aspect == vk::ImageAspectFlagBits::eDepth &&
cached.info.usage == vk::ImageUsageFlags{};
});
Require(
name, "R32 uint depth-backed view",
sampled_depth_binding.image_id == depth_only.image_id &&
sampled_depth_binding.desc.info.guest_format ==
Prospero::BufferFormat::k32UInt &&
sampled_depth_binding.desc.info.pixel_format ==
vk::Format::eR32Uint &&
sampled_depth_binding.desc.view_info.format ==
vk::Format::eR32Uint &&
sampled_depth_binding.desc.view_info.aspect ==
vk::ImageAspectFlagBits::eColor &&
sampled_depth_binding.desc.view_info.usage ==
vk::ImageUsageFlagBits::eSampled &&
sampled_depth_image.info.pixel_format == vk::Format::eD32Sfloat &&
sampled_depth_image.backing.format == vk::Format::eD32Sfloat &&
sampled_depth_image.backing.image == sampled_depth_backing &&
sampled_depth_view != nullptr && normalized_depth_view,
"uint T# did not retain its requested format while the concrete "
"view followed the D32 backing");
RenderExecutorTestAccess::ResetBindings(executor);
// Attachment acquisition consumes pending HTile clears using DB_DEPTH_CLEAR.
(void)texture_cache.FindDepthTarget(depth_only.image_id, depth_only.desc);
auto bounds_target = depth_only_target;
bounds_target.z_write_base_addr = 0;
bounds_target.depth_view.depth_write_disable = true;
registers.SetDepthRenderTarget(bounds_target);
HW::DepthControl bounds_control{};
bounds_control.depth_bounds_enable = true;
registers.SetDepthControl(bounds_control);
registers.SetRenderControl({});
registers.SetDepthClearValue(0.375f);
ShaderRecompiler::IR::ResourceSnapshot bounds_snapshot;
bounds_snapshot.images.push_back(sampled_depth_value);
ShaderStageRuntime bounds_vertex{&vertex_sampled_info, &bounds_snapshot};
ShaderStageRuntime bounds_pixel{&sampled_info, &bounds_snapshot};
const auto depth_texels = depth_only.desc.info.extent.width *
depth_only.desc.info.extent.height;
const auto depth_bytes = depth_texels * sizeof(uint32_t);
auto bounds_readback = CreateHostBuffer(name, depth_bytes * 2,
vk::BufferUsageFlagBits::eTransferDst, {});
for (uint32_t pass = 0; pass < 2; ++pass) {
Require(name, "bounds prior depth contents",
texture_cache.ClearImageFromBuffer(scheduler.Current(),
depth_only.desc.info.data.address, depth_only.desc.info.data.size,
std::bit_cast<uint32_t>(0.625f)),
"failed to seed depth before the deferred HTile clear");
if (pass == 0) {
Require(name, "bounds deferred HTile clear",
texture_cache.ClearMeta(depth_only_htile_address),
"depth-only HTile was not registered");
} else {
bounds_target.z_write_base_addr = depth_only_address;
bounds_target.depth_view.depth_write_disable = false;
bounds_control.z_write_enable = true;
registers.SetDepthRenderTarget(bounds_target);
registers.SetDepthControl(bounds_control);
}
RenderDepthInfo bounds_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), bounds_depth);
auto bounds_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, bounds_vertex, bounds_pixel, true);
std::array<PreparedBindings *, 2> bounds_stages{
&bounds_bindings.vertex[0], &*bounds_bindings.pixel};
vk::ImageAspectFlags bounds_feedback;
const auto bounds_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_color, 0, bounds_depth,
bounds_stages, &bounds_feedback);
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), bounds_bindings.vertex[0], *bounds_bindings.pixel));
const auto &vertex_depth = bounds_bindings.vertex[0].images[0];
const auto &pixel_depth = bounds_bindings.pixel->images[0];
const auto expected_layout = pass == 0
? vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT
: vk::ImageLayout::eDepthReadOnlyOptimal;
Require(name, "deferred clear with sampled depth bounds",
bounds_depth.image_id == depth_only.image_id &&
bounds_depth.depth_bounds_test_enable &&
!bounds_depth.depth_write_enable &&
bounds_depth.depth_load_clear_enable == (pass == 0) &&
!texture_cache.IsMetaCleared(depth_only_htile_address, 0) &&
bounds_feedback == (pass == 0
? vk::ImageAspectFlagBits::eDepth
: vk::ImageAspectFlags{}) &&
bounds_rendering.depth_stencil_attachment.image_layout == expected_layout &&
bounds_rendering.depth_stencil_attachment.depth_clear == (pass == 0) &&
vertex_depth.image_id == depth_only.image_id &&
pixel_depth.image_id == depth_only.image_id &&
MakeImageInfo(vertex_depth).imageLayout == expected_layout &&
MakeImageInfo(pixel_depth).imageLayout == expected_layout,
"a deferred clear sampled by vertex and pixel stages missed depth feedback");
scheduler.BeginRendering(bounds_rendering);
scheduler.EndRendering();
RenderExecutorTestAccess::ResetBindings(executor);
const vk::BufferImageCopy copy{pass * depth_bytes, 0, 0,
{vk::ImageAspectFlagBits::eDepth, 0, 0, 1}, {}, depth_only.desc.info.extent};
texture_cache.GetImage(depth_only.image_id).Download(
std::span{&copy, 1}, bounds_readback.buffer, 0, bounds_readback.size);
}
vk::MemoryBarrier2 bounds_host_barrier{};
bounds_host_barrier.sType = vk::StructureType::eMemoryBarrier2;
bounds_host_barrier.srcStageMask = vk::PipelineStageFlagBits2::eTransfer;
bounds_host_barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
bounds_host_barrier.dstStageMask = vk::PipelineStageFlagBits2::eHost;
bounds_host_barrier.dstAccessMask = vk::AccessFlagBits2::eHostRead;
vk::DependencyInfo bounds_dependency{};
bounds_dependency.sType = vk::StructureType::eDependencyInfo;
bounds_dependency.memoryBarrierCount = 1;
bounds_dependency.pMemoryBarriers = &bounds_host_barrier;
scheduler.Current().Handle().pipelineBarrier2(bounds_dependency);
scheduler.Finish();
const auto bounds_result = ReadBuffer(name, bounds_readback, depth_texels * 2);
Require(name, "deferred HTile clear occurs once",
std::ranges::all_of(std::span{bounds_result}.first(depth_texels),
[](uint32_t value) { return value == std::bit_cast<uint32_t>(0.375f); }) &&
std::ranges::all_of(std::span{bounds_result}.subspan(depth_texels),
[](uint32_t value) { return value == std::bit_cast<uint32_t>(0.625f); }),
"pending clear did not use DB_DEPTH_CLEAR or cleared the next acquisition again");
DestroyBuffer(&bounds_readback);
registers.SetDepthClearValue(0.0f);
// DB override copies run with depth and stencil tests disabled.
constexpr uint64_t copy_source_depth = base + 0x180000;
constexpr uint64_t copy_source_stencil = base + 0x1a0000;
constexpr uint64_t copy_destination_depth = base + 0x1c0000;
constexpr uint64_t copy_destination_stencil = base + 0x1e0000;
constexpr uint32_t copy_side = 8;
constexpr uint32_t copy_layer_texels = copy_side * copy_side;
constexpr uint32_t copy_depth_bytes =
copy_layer_texels * 2 * sizeof(uint32_t);
constexpr uint32_t copy_readback_stride =
copy_depth_bytes + copy_layer_texels * 2;
HW::DepthRenderTarget copy_target{};
copy_target.z_info.format = Prospero::DepthFormat::kZ32F;
copy_target.z_info.texture_compatibility =
Prospero::TextureCompatiblePlaneCompression::kEnable;
copy_target.stencil_info.format = Prospero::StencilFormat::k8UInt;
copy_target.stencil_info.texture_compatibility =
Prospero::TextureCompatibleStencil::kEnable;
copy_target.stencil_info.htile_stencil_disabled = true;
copy_target.depth_view.slice_max = 1;
copy_target.size = {copy_side - 1, copy_side - 1, true};
HW::DepthControl copy_seed_control{};
copy_seed_control.z_enable = true;
copy_seed_control.z_write_enable = true;
copy_seed_control.stencil_enable = true;
registers.SetDepthControl(copy_seed_control);
registers.SetRenderControl({});
const auto make_copy_target = [&](uint64_t depth, uint64_t stencil) {
auto target = copy_target;
target.z_read_base_addr = depth;
target.z_write_base_addr = depth;
target.stencil_read_base_addr = stencil;
target.stencil_write_base_addr = stencil;
registers.SetDepthRenderTarget(target);
RenderDepthInfo resolved{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), resolved);
(void)texture_cache.FindDepthTarget(resolved.image_id, resolved.desc);
RenderExecutorTestAccess::ResetBindings(executor);
return resolved;
};
const auto copy_source =
make_copy_target(copy_source_depth, copy_source_stencil);
const auto copy_destination =
make_copy_target(copy_destination_depth, copy_destination_stencil);
const auto clear_copy_target = [&](ImageId id, bool source) {
auto &image = texture_cache.GetImage(id);
image.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {},
scheduler.Current().Handle());
for (uint32_t layer = 0; layer < 2; ++layer) {
const vk::ClearDepthStencilValue clear{
source ? (layer == 0 ? 0.75f : 0.25f)
: (layer == 0 ? 0.875f : 0.625f),
source ? (layer == 0 ? 0xa3u : 0x5au)
: (layer == 0 ? 0x31u : 0xc4u)};
const vk::ImageSubresourceRange range{
vk::ImageAspectFlagBits::eDepth |
vk::ImageAspectFlagBits::eStencil,
0, 1, layer, 1};
scheduler.Current().Handle().clearDepthStencilImage(
image.backing.image, vk::ImageLayout::eTransferDstOptimal, &clear,
1, &range);
}
texture_cache.MarkGpuWritten(id);
};
struct CopyCase {
const char *label;
HW::DepthRenderOverride override;
uint32_t color_mode;
bool depth;
bool stencil;
};
constexpr std::array copy_cases{
CopyCase{"no override", {}, 0, false, false},
CopyCase{
"depth valid only", {true, false, false, false}, 0, false, false},
CopyCase{
"depth dirty only", {false, true, false, false}, 0, false, false},
CopyCase{"stencil valid only",
{false, false, true, false},
0,
false,
false},
CopyCase{"stencil dirty only",
{false, false, false, true},
0,
false,
false},
CopyCase{
"unpaired aspects", {true, false, false, true}, 0, false, false},
CopyCase{"color enabled", {true, true, true, true}, 1, false, false},
CopyCase{"depth only", {true, true, false, false}, 0, true, false},
CopyCase{"stencil only", {false, false, true, true}, 0, false, true},
CopyCase{
"depth and stencil", {true, true, true, true}, 0, true, true}};
auto copy_readback =
CreateHostBuffer(name, copy_cases.size() * copy_readback_stride,
vk::BufferUsageFlagBits::eTransferDst, {});
copy_target.z_read_base_addr = copy_source_depth;
copy_target.z_write_base_addr = copy_destination_depth;
copy_target.stencil_read_base_addr = copy_source_stencil;
copy_target.stencil_write_base_addr = copy_destination_stencil;
copy_target.depth_view.slice_start = 1;
registers.SetDepthRenderTarget(copy_target);
registers.SetDepthControl({});
RenderDepthInfo copy_disabled_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), copy_disabled_depth);
Require(name, "copy without attachment tests",
!copy_disabled_depth.image_id,
"disabled depth/stencil tests acquired a regular attachment");
for (size_t index = 0; index < copy_cases.size(); ++index) {
const auto &test = copy_cases[index];
clear_copy_target(copy_source.image_id, true);
clear_copy_target(copy_destination.image_id, false);
registers.SetDepthRenderOverride(test.override);
auto color_control = registers.GetColorControl();
color_control.mode = test.color_mode;
registers.SetColorControl(color_control);
Require(name, test.label,
RenderExecutorTestAccess::DepthStencilCopy(
executor, scheduler.Current()) ==
(test.depth || test.stencil),
"DB override copy did not honor paired flags and disabled "
"color output");
RenderExecutorTestAccess::ResetBindings(executor);
const auto offset = index * copy_readback_stride;
const std::array<vk::BufferImageCopy, 2> copies{
{{offset,
0,
0,
{vk::ImageAspectFlagBits::eDepth, 0, 0, 2},
{},
copy_destination.desc.info.extent},
{offset + copy_depth_bytes,
0,
0,
{vk::ImageAspectFlagBits::eStencil, 0, 0, 2},
{},
copy_destination.desc.info.extent}}};
texture_cache.GetImage(copy_destination.image_id)
.Download(copies, copy_readback.buffer, 0, copy_readback.size);
}
scheduler.Current().Handle().pipelineBarrier2(bounds_dependency);
scheduler.Finish();
const auto copy_result = ReadBuffer(
name, copy_readback,
copy_cases.size() * copy_readback_stride / sizeof(uint32_t));
for (size_t index = 0; index < copy_cases.size(); ++index) {
const auto &test = copy_cases[index];
const auto *depth = copy_result.data() +
index * copy_readback_stride / sizeof(uint32_t);
const auto *stencil =
reinterpret_cast<const uint8_t *>(depth) + copy_depth_bytes;
for (uint32_t layer = 0; layer < 2; ++layer) {
const auto expected_depth =
std::bit_cast<uint32_t>(layer == 0 ? 0.875f
: test.depth ? 0.25f
: 0.625f);
const auto expected_stencil = layer == 0 ? 0x31u
: test.stencil ? 0x5au
: 0xc4u;
for (uint32_t texel = 0; texel < copy_layer_texels; ++texel) {
const auto position = layer * copy_layer_texels + texel;
Require(name, test.label,
depth[position] == expected_depth &&
stencil[position] == expected_stencil,
"DB override copy changed an unselected layer/aspect or "
"missed copied texels");
}
}
}
DestroyBuffer(&copy_readback);
ShaderTextureResource copy_sample_descriptor{};
copy_sample_descriptor.fields[0] =
static_cast<uint32_t>(copy_destination_depth >> 8u);
copy_sample_descriptor.fields[1] =
static_cast<uint32_t>(copy_destination_depth >> 40u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u) |
(((copy_side - 1u) & 3u) << 30u);
copy_sample_descriptor.fields[2] =
((copy_side - 1u) >> 2u) | ((copy_side - 1u) << 14u);
copy_sample_descriptor.fields[3] =
DstSel(4, 4, 4, 4) |
(static_cast<uint32_t>(Prospero::TileMode::kDepth) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DArray) << 28u);
copy_sample_descriptor.fields[4] = 1;
copy_sample_descriptor.fields[5] = 0x00700000u;
TestCase copy_sample_test;
copy_sample_test.name = "DepthOverrideR32Sampling";
copy_sample_test.has_user_data = true;
copy_sample_test.image_descriptor_swizzle =
copy_sample_descriptor.DstSelXYZW();
std::copy_n(copy_sample_descriptor.fields, 8,
copy_sample_test.user_data.begin());
copy_sample_test.user_data[50] = 2 * sizeof(uint32_t);
copy_sample_test.opcodes = {
ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
copy_sample_test.required_spirv = {"OpImageSampleExplicitLod"};
for (uint32_t layer = 0; layer < 2; ++layer) {
AppendVMovLiteral(&copy_sample_test.code, 20,
std::bit_cast<uint32_t>(0.5f));
AppendVMovLiteral(&copy_sample_test.code, 21,
std::bit_cast<uint32_t>(0.5f));
AppendVMovLiteral(&copy_sample_test.code, 22,
std::bit_cast<uint32_t>(static_cast<float>(layer)));
copy_sample_test.code.push_back(EncodeMimg0(0x20, 1, 0, false, 5));
copy_sample_test.code.push_back(EncodeMimg1(0, 20, 0, 2));
AppendStoreVgpr(&copy_sample_test.code, 0, layer);
copy_sample_test.expected.push_back(
std::bit_cast<uint32_t>(layer == 0 ? 0.875f : 0.25f));
}
AppendEnd(&copy_sample_test.code);
const auto copy_sample_program =
CompileCase(copy_sample_test, SubgroupSize());
ShaderRecompiler::IR::DescriptorValue copy_sample_value{};
copy_sample_value.dword_count = 8;
std::copy_n(copy_sample_descriptor.fields, 8,
copy_sample_value.dwords.begin());
const auto copy_sample_binding = RenderExecutorTestAccess::ResolveTexture(
executor, copy_sample_program.program.info.images[0],
copy_sample_value);
const auto copy_sample_view = texture_cache.FindTexture(
copy_sample_binding.image_id, copy_sample_binding.desc);
Require(
name, "copied depth sampled owner",
copy_sample_binding.image_id == copy_destination.image_id &&
copy_sample_binding.desc.info.pixel_format ==
vk::Format::eR32Sfloat,
"R32 sampling did not reuse the GPU-written depth-copy destination");
auto &copy_sample_image = texture_cache.GetImage(copy_destination.image_id);
copy_sample_image.Transit(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, {},
scheduler.Current().Handle());
Image copied_depth_sample;
copied_depth_sample.view = copy_sample_view;
copied_depth_sample.layout = copy_sample_image.backing.state.layout;
scheduler.Finish();
auto copy_sample_output = CreateStorageBuffer(name, {}, 2);
const auto copy_sampler = CreateSampler(name);
Dispatch(copy_sample_test, copy_sample_program, copy_sample_output,
nullptr, &copied_depth_sample, nullptr, nullptr, copy_sampler);
Require(
name, "copied depth R32 samples",
ReadBuffer(name, copy_sample_output, 2) == copy_sample_test.expected,
"R32 post-effect sampling saw stale depth after the override copy");
m_device.destroySampler(copy_sampler);
DestroyBuffer(&copy_sample_output);
RenderExecutorTestAccess::ResetBindings(executor);
registers.SetDepthRenderOverride({});
registers.SetColorControl({});
auto shared_depth_descriptor = sampled_depth_descriptor;
shared_depth_descriptor.fields[0] =
static_cast<uint32_t>(phased_depth_address >> 8u);
shared_depth_descriptor.fields[1] =
static_cast<uint32_t>(phased_depth_address >> 40u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u);
shared_depth_descriptor.fields[2] = 0;
ShaderRecompiler::IR::DescriptorValue shared_depth_value{};
std::copy(std::begin(shared_depth_descriptor.fields),
std::end(shared_depth_descriptor.fields),
shared_depth_value.dwords.begin());
shared_depth_value.dword_count = 8;
ShaderRecompiler::IR::ResourceSnapshot shared_depth_snapshot;
shared_depth_snapshot.images.push_back(shared_depth_value);
ShaderStageRuntime shared_depth_vertex{&vertex_sampled_info,
&shared_depth_snapshot};
ShaderStageRuntime shared_depth_pixel{&sampled_info,
&shared_depth_snapshot};
for (const bool stencil_write : {true, false}) {
registers.SetDepthRenderTarget(phased_depth_target);
registers.SetRenderControl({});
registers.SetDepthControl(read_only_depth_control);
HW::StencilControl shared_stencil_control{};
shared_stencil_control.stencil_zpass =
static_cast<uint8_t>(Prospero::StencilOp::kReplaceTest);
registers.SetStencilControl(shared_stencil_control);
HW::StencilMask shared_stencil_mask{};
shared_stencil_mask.stencil_testval = 0x20;
shared_stencil_mask.stencil_mask = 0xff;
shared_stencil_mask.stencil_writemask = stencil_write ? 0xf4 : 0;
registers.SetStencilMask(shared_stencil_mask);
RenderDepthInfo shared_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), shared_depth);
auto shared_bindings =
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, shared_depth_vertex, shared_depth_pixel, true);
auto& shared_image = texture_cache.GetImage(shared_depth.image_id);
auto stencil_view_info = shared_depth.desc.view_info;
stencil_view_info.aspect = vk::ImageAspectFlagBits::eStencil;
stencil_view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto stencil_view = shared_image.FindView(stencil_view_info);
auto& pixel_image = shared_bindings.pixel->images[0];
pixel_image.image_view = stencil_view;
pixel_image.desc.view_info = stencil_view_info;
std::array<PreparedBindings *, 2> shared_stages{
&shared_bindings.vertex[0], &*shared_bindings.pixel};
vk::ImageAspectFlags shared_feedback;
const auto shared_rendering =
RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_color, 0, shared_depth,
shared_stages, &shared_feedback);
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), shared_bindings.vertex[0],
*shared_bindings.pixel));
const auto expected_layout =
stencil_write
? vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT
: vk::ImageLayout::eDepthStencilReadOnlyOptimal;
const auto expected_access =
vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eDepthStencilAttachmentRead |
vk::AccessFlagBits2::eDepthStencilAttachmentWrite;
const auto &vertex_image = shared_bindings.vertex[0].images[0];
Require(name, "sampled depth and stencil attachment layout",
shared_depth.image_id == phased_depth.image_id &&
stencil_view != nullptr &&
shared_feedback == (stencil_write
? vk::ImageAspectFlagBits::eStencil
: vk::ImageAspectFlags{}) &&
vertex_image.image_id == shared_depth.image_id &&
pixel_image.image_id == shared_depth.image_id &&
pixel_image.image_view == stencil_view &&
MakeImageInfo(vertex_image).imageLayout == expected_layout &&
MakeImageInfo(pixel_image).imageLayout == expected_layout &&
shared_rendering.depth_stencil_attachment.image_layout ==
expected_layout &&
shared_image.backing.state.layout == expected_layout &&
shared_image.backing.state.access_mask == expected_access,
"stencil sampling and writes lost the feedback aspect, shared "
"layout, or attachment access");
scheduler.BeginRendering(shared_rendering);
scheduler.EndRendering();
RenderExecutorTestAccess::ResetBindings(executor);
Require(name, "attachment binding reset",
!shared_image.binding.is_target &&
shared_image.binding.attachment_layout ==
vk::ImageLayout::eUndefined &&
!shared_image.binding.attachment_access,
"attachment usage leaked into the next draw");
}
// PPSA03541 reuses a tile24 depth allocation through a tile27 RGB10A2 T#.
auto pooled_resource = sampled_depth_resource;
pooled_resource.numeric_class = Prospero::TextureNumericClass::Float;
ImageId pooled_color_id{};
for (const bool color : {false, true, false, true}) {
auto value = shared_depth_value;
value.dwords[1] = static_cast<uint32_t>(phased_depth_address >> 40u) |
(static_cast<uint32_t>(color ? Prospero::BufferFormat::k10_10_10_2UNorm
: Prospero::BufferFormat::k32Float) << 20u);
value.dwords[3] = color ? 0x91b00facu : 0x91800204u;
const auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, pooled_resource, value);
const auto view = texture_cache.FindTexture(binding.image_id, binding.desc);
if (color && !pooled_color_id) {
pooled_color_id = binding.image_id;
}
Require(name, "pooled depth/color tile identity",
view != nullptr &&
binding.image_id == (color ? pooled_color_id : phased_depth.image_id) &&
(!color || binding.image_id != phased_depth.image_id) &&
texture_cache.GetImage(binding.image_id).backing.format ==
(color ? vk::Format::eA2B10G10R10UnormPack32
: vk::Format::eD32SfloatS8Uint) &&
binding.desc.info.tile_mode ==
(color ? Prospero::TileMode::kRenderTarget
: Prospero::TileMode::kDepth),
"a changed tile family reused depth or repeated sampling lost its backing");
RenderExecutorTestAccess::ResetBindings(executor);
}
auto video_subresource =
make_target_desc(base + 0x20000, target_mip_size, {1, 1, 1});
video_subresource.type = BindingType::VideoOut;
video_subresource.info.pixel_format = vk::Format::eR8G8B8A8Srgb;
video_subresource.info.guest_format =
Prospero::BufferFormat::k8_8_8_8Srgb;
video_subresource.view_info.format = video_subresource.info.pixel_format;
video_subresource.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto video_subresource_id =
texture_cache.FindImage(video_subresource);
const auto video_subresource_owner =
TextureCacheTestAccess::Owner(texture_cache, video_subresource_id);
auto video_parent =
make_target_desc(base + 0x20000, target_mip_size * 2, {2, 2, 1});
video_parent.type = BindingType::VideoOut;
video_parent.info.pixel_format = vk::Format::eR8G8B8A8Srgb;
video_parent.info.guest_format = Prospero::BufferFormat::k8_8_8_8Srgb;
video_parent.info.resources.levels = 2;
video_parent.info.mip_layout[0] = {0, target_mip_size, 2, 2};
video_parent.info.mip_layout[1] = {target_mip_size, target_mip_size, 1,
1};
video_parent.view_info.format = video_parent.info.pixel_format;
video_parent.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto video_parent_id = texture_cache.FindImage(video_parent);
const auto video_parent_owner =
TextureCacheTestAccess::Owner(texture_cache, video_parent_id);
Require(name, "video-out replacement remains cache-only",
video_subresource_owner != nullptr &&
video_parent_owner != nullptr &&
video_parent_id != video_subresource_id &&
!video_subresource_owner->binding.is_bound &&
!video_subresource_owner->binding.is_target &&
!video_subresource_owner->binding.needs_rebind &&
!video_subresource_owner->usage.video_out &&
!video_parent_owner->binding.is_bound &&
!video_parent_owner->binding.is_target &&
!video_parent_owner->binding.needs_rebind &&
!video_parent_owner->usage.video_out,
"FindImage leaked caller-owned state across a video-out "
"replacement");
constexpr uint64_t array_target_address = base + 0xd0000;
auto array_target = make_target_desc(array_target_address,
target_mip_size, {128, 128, 1});
const auto array_target_id = texture_cache.FindImage(array_target);
const auto array_target_owner =
TextureCacheTestAccess::Owner(texture_cache, array_target_id);
RenderExecutorTestAccess::BindRenderTarget(executor, array_target_id);
const auto array_target_view =
texture_cache.FindRenderTarget(array_target_id, array_target);
ShaderTextureResource array_texture{};
const auto encoded_array_address = array_target_address >> 8u;
constexpr auto array_format =
static_cast<uint32_t>(Prospero::BufferFormat::k32UInt);
array_texture.fields[0] = static_cast<uint32_t>(encoded_array_address);
array_texture.fields[1] =
static_cast<uint32_t>(encoded_array_address >> 32u) |
(array_format << 20u) | (3u << 30u);
array_texture.fields[2] = (127u >> 2u) | (127u << 14u);
array_texture.fields[3] =
DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DArray) << 28u);
array_texture.fields[4] = 1;
ShaderRecompiler::IR::CompiledShaderInfo array_program{};
array_program.stage = ShaderType::Compute;
ShaderRecompiler::IR::ImageResource array_resource{};
array_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
array_resource.numeric_class = Prospero::TextureNumericClass::Uint;
array_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
array_resource.read = true;
array_program.info.images.push_back(array_resource);
ShaderRecompiler::IR::DescriptorValue array_descriptor{};
std::copy(std::begin(array_texture.fields),
std::end(array_texture.fields),
array_descriptor.dwords.begin());
array_descriptor.dword_count = 8;
ShaderRecompiler::IR::ResourceSnapshot array_snapshot{};
array_snapshot.images.push_back(array_descriptor);
ShaderStageRuntime array_runtime{&array_program, &array_snapshot};
PreparedBindings array_binding;
executor.PrepareBindings(array_runtime, array_binding);
executor.RebindImages(array_binding);
const auto expanded_array_id = array_binding.images[0].image_id;
const auto &expanded_array = texture_cache.GetImage(expanded_array_id);
Require(
name, "2D target to array backing expansion",
array_target_view != nullptr && array_target_owner != nullptr &&
array_target_owner->binding.needs_rebind &&
expanded_array_id != array_target_id &&
!TextureCacheTestAccess::Contains(texture_cache,
array_target_id) &&
array_binding.images[0].image_view != nullptr &&
array_binding.images[0].desc.info.type ==
Prospero::ImageType::kColor2D &&
array_binding.images[0].desc.info.resources ==
ImageSubresources{1, 2} &&
array_binding.images[0].desc.view_info.type ==
vk::ImageViewType::e2DArray &&
array_binding.images[0].desc.view_info.layer_count ==
2 &&
expanded_array.backing.layers == 2 &&
expanded_array.IsGpuModified() && expanded_array.usage.texture,
"raw array view did not expand and reuse the Color2D backing");
RenderColorInfo rebound_array_target{};
rebound_array_target.desc = array_target;
rebound_array_target.image_id = array_target_id;
RenderDepthInfo no_array_depth{};
const auto acquired_array_view = array_binding.images[0].image_view;
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, std::span<PreparedBindings *const>{},
std::span{&rebound_array_target, 1u});
auto array_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &rebound_array_target, 1,
no_array_depth);
Require(name, "expanded array target rebind",
rebound_array_target.image_id == expanded_array_id &&
array_binding.images[0].image_id == expanded_array_id &&
array_binding.images[0].image_view == acquired_array_view &&
array_rendering.color_attachments[0].image_view != nullptr &&
array_rendering.num_color_attachments == 1 &&
array_rendering.width == 128 &&
array_rendering.height == 128 &&
array_rendering.num_layers == 1 &&
texture_cache.GetImage(expanded_array_id).binding.is_bound &&
texture_cache.GetImage(expanded_array_id).binding.is_target,
"the retained single-layer target did not rebind to the expanded "
"backing");
RenderExecutorTestAccess::ResetBindings(executor);
Require(name, "raw image synchronization default",
!Config::SyncRawImageBuffersEnabled(),
"raw image synchronization should be disabled by default");
for (const auto [sync_raw, formatted] :
{std::pair{false, false}, {true, false}, {false, true}, {true, true}}) {
Config::Load({.sync_raw_image_buffers = sync_raw});
const bool sync_image = sync_raw || formatted;
const uint32_t before = 0x13579bd0u | (uint32_t(sync_raw) << 1u) | uint32_t(formatted);
constexpr uint32_t guard = 0x2468ace0u;
constexpr uint32_t after = 0xa1b2c3d4u;
vk::ClearValue clear{};
clear.color.uint32[0] = before;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), expanded_array_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 2}, clear);
clear.color.uint32[0] = guard;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), expanded_array_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 1, 1}, clear);
const auto buffer_program = make_buffer_program(
ShaderType::Vertex, {.read = true, .formatted = formatted});
constexpr uint64_t buffer_size = 2 * target_mip_size;
static_assert(buffer_size > BufferCache::CACHING_PAGESIZE);
ShaderBufferResource descriptor{};
descriptor.UpdateAddress48(array_target_address);
descriptor.fields[1] |= 4u << 16u;
descriptor.fields[2] = buffer_size / sizeof(uint32_t);
descriptor.fields[3] = 0x00014004u;
ShaderRecompiler::IR::ResourceSnapshot buffer_snapshot;
ShaderStageRuntime buffer_runtime{&buffer_program, &buffer_snapshot};
auto &value = buffer_snapshot.buffers.emplace_back();
std::memcpy(value.dwords.data(), descriptor.fields, sizeof(descriptor.fields));
value.dword_count = 4;
PreparedBindings buffer_bindings;
executor.PrepareBindings(buffer_runtime, buffer_bindings);
std::array stages{&buffer_bindings};
RenderColorInfo target{};
target.desc = array_target;
target.image_id = array_target_id;
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, stages, std::span{&target, 1u});
const auto &buffer = buffer_bindings.buffers[0];
Require(name, "rediscovered target buffer read",
target.image_id == expanded_array_id &&
buffer.range == buffer_size &&
buffer_bindings.images.empty() &&
texture_cache.GetImage(target.image_id).backing.state.layout ==
(sync_image ? vk::ImageLayout::eTransferSrcOptimal
: vk::ImageLayout::eTransferDstOptimal),
"image-to-buffer synchronization ignored its setting or formatted reads");
auto readback = CreateHostBuffer(name, 2 * sizeof(uint32_t),
vk::BufferUsageFlagBits::eTransferDst, {0, 0});
const std::array copies{
vk::BufferCopy{buffer.offset, 0, sizeof(uint32_t)},
vk::BufferCopy{buffer.offset + target_mip_size, sizeof(uint32_t),
sizeof(uint32_t)}};
scheduler.Current().Handle().copyBuffer(buffer.buffer, readback.buffer,
copies.size(), copies.data());
vk::BufferMemoryBarrier readback_barrier{};
readback_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
readback_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
readback_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
readback_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
readback_barrier.buffer = readback.buffer;
readback_barrier.size = readback.size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost,
{}, 0, nullptr, 1, &readback_barrier, 0, nullptr);
const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &target, 1, no_array_depth);
Require(name, "attachment layout after buffer read",
rendering.color_attachments[0].image_layout ==
vk::ImageLayout::eColorAttachmentOptimal &&
texture_cache.GetImage(target.image_id).backing.state.layout ==
vk::ImageLayout::eColorAttachmentOptimal,
"the buffer copy left the render attachment in its transfer layout");
vk::ClearAttachment clear_attachment{};
clear_attachment.aspectMask = vk::ImageAspectFlagBits::eColor;
clear_attachment.colorAttachment = 0;
clear_attachment.clearValue.color.uint32[0] = after;
const vk::ClearRect clear_rect{{{0, 0}, {128, 128}}, 0, 1};
scheduler.BeginRendering(rendering);
scheduler.Current().Handle().clearAttachments(1, &clear_attachment, 1, &clear_rect);
scheduler.EndRendering();
scheduler.Finish();
Require(name, "buffer alias observes prior GPU contents",
ReadBuffer(name, readback, 2) ==
(sync_image ? std::vector<u32>{before, guard} : std::vector<u32>{0, 0}),
"buffer contents did not match the selected image synchronization mode");
DestroyBuffer(&readback);
Require(name, "rendering after buffer acquisition",
ReadCachedTexel(name, context, target.image_id) == std::vector<u32>{after} &&
ReadCachedTexel(name, context, target.image_id, {}, {1, 1, 1}, 1) ==
std::vector<u32>{guard},
"attachment rendering lost its new color or changed the untouched array layer");
RenderExecutorTestAccess::ResetBindings(executor);
}
Config::Load({});
auto colliding_msaa_texture = array_texture;
colliding_msaa_texture.fields[3] =
DstSel(4, 5, 6, 7) | (1u << 16u) |
(static_cast<uint32_t>(Prospero::TileMode::kRenderTarget) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DMsaa) << 28u);
colliding_msaa_texture.fields[4] = 0;
colliding_msaa_texture.fields[5] = (7u << 20u) | (1u << 4u);
ShaderRecompiler::IR::DescriptorValue colliding_msaa_descriptor{};
std::copy(std::begin(colliding_msaa_texture.fields),
std::end(colliding_msaa_texture.fields),
colliding_msaa_descriptor.dwords.begin());
colliding_msaa_descriptor.dword_count = 8;
ShaderRecompiler::IR::CompiledShaderInfo colliding_msaa_program{};
colliding_msaa_program.stage = ShaderType::Compute;
auto colliding_msaa_resource = array_resource;
colliding_msaa_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa;
colliding_msaa_program.info.images.push_back(colliding_msaa_resource);
ShaderRecompiler::IR::ResourceSnapshot colliding_msaa_snapshot{};
colliding_msaa_snapshot.images.push_back(colliding_msaa_descriptor);
ShaderStageRuntime colliding_msaa_runtime{
&colliding_msaa_program, &colliding_msaa_snapshot};
PreparedBindings colliding_msaa_binding;
executor.PrepareBindings(colliding_msaa_runtime, colliding_msaa_binding);
executor.RebindImages(colliding_msaa_binding);
const auto &resolved_colliding_msaa =
colliding_msaa_binding.images[0];
Require(
name, "equal-footprint sample-count identity",
resolved_colliding_msaa.image_id != expanded_array_id &&
resolved_colliding_msaa.image_view != nullptr &&
resolved_colliding_msaa.desc.info.type ==
Prospero::ImageType::kColor2D &&
resolved_colliding_msaa.desc.info.resources ==
ImageSubresources{1, 1} &&
resolved_colliding_msaa.desc.info.samples == 2 &&
resolved_colliding_msaa.desc.info.data.size == 0x20000 &&
texture_cache.GetImage(resolved_colliding_msaa.image_id)
.backing.samples == 2 &&
texture_cache.GetImage(expanded_array_id).backing.samples == 1 &&
texture_cache.GetImage(expanded_array_id).backing.layers == 2,
"equal-size 1x array and 2x image shared an incompatible backing");
RenderExecutorTestAccess::ResetBindings(executor);
constexpr uint64_t msaa_target_address = base + 0x100000;
constexpr uint32_t msaa_samples = 4;
auto msaa_target =
make_target_desc(msaa_target_address, 0x40000, {128, 128, 1});
msaa_target.info.samples = msaa_samples;
msaa_target.info.tile_mode = Prospero::TileMode::kRenderTarget;
msaa_target.info.mip_layout[0].size = msaa_target.info.data.size;
const auto msaa_target_id = texture_cache.FindImage(msaa_target);
RenderExecutorTestAccess::BindRenderTarget(executor, msaa_target_id);
const auto msaa_target_view =
texture_cache.FindRenderTarget(msaa_target_id, msaa_target);
auto msaa_texture = array_texture;
const auto encoded_msaa_address = msaa_target_address >> 8u;
msaa_texture.fields[0] = static_cast<uint32_t>(encoded_msaa_address);
msaa_texture.fields[1] =
static_cast<uint32_t>(encoded_msaa_address >> 32u) |
(array_format << 20u) | (3u << 30u);
msaa_texture.fields[3] =
DstSel(4, 5, 6, 7) | (2u << 16u) |
(static_cast<uint32_t>(Prospero::TileMode::kRenderTarget) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DMsaa) << 28u);
msaa_texture.fields[4] = 0;
msaa_texture.fields[5] = (7u << 20u) | (2u << 4u);
ShaderRecompiler::IR::DescriptorValue msaa_descriptor{};
std::copy(std::begin(msaa_texture.fields), std::end(msaa_texture.fields),
msaa_descriptor.dwords.begin());
msaa_descriptor.dword_count = 8;
ShaderRecompiler::IR::CompiledShaderInfo msaa_program{};
msaa_program.stage = ShaderType::Compute;
auto msaa_resource = array_resource;
msaa_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa;
msaa_program.info.images.push_back(msaa_resource);
ShaderRecompiler::IR::ResourceSnapshot msaa_snapshot{};
msaa_snapshot.images.push_back(msaa_descriptor);
ShaderStageRuntime msaa_runtime{&msaa_program, &msaa_snapshot};
PreparedBindings msaa_binding;
executor.PrepareBindings(msaa_runtime, msaa_binding);
executor.RebindImages(msaa_binding);
const auto &resolved_msaa = msaa_binding.images[0];
Require(
name, "MSAA descriptor backing reuse",
msaa_target_view != nullptr &&
resolved_msaa.image_id == msaa_target_id &&
resolved_msaa.image_view != nullptr &&
resolved_msaa.desc.info.type == Prospero::ImageType::kColor2D &&
resolved_msaa.desc.info.resources == ImageSubresources{1, 1} &&
resolved_msaa.desc.info.samples == msaa_samples &&
resolved_msaa.desc.info.data.size == 0x40000 &&
resolved_msaa.desc.view_info.level_count == 1 &&
texture_cache.GetImage(msaa_target_id).backing.samples ==
msaa_samples,
"MSAA view fields did not resolve to the matching multisample "
"backing");
RenderExecutorTestAccess::ResetBindings(executor);
auto msaa_array_texture = msaa_texture;
msaa_array_texture.fields[3] =
(msaa_array_texture.fields[3] & ~(0xfu << 28u)) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DMsaaArray)
<< 28u);
msaa_array_texture.fields[4] = 1;
ShaderRecompiler::IR::DescriptorValue msaa_array_descriptor{};
std::copy(std::begin(msaa_array_texture.fields),
std::end(msaa_array_texture.fields),
msaa_array_descriptor.dwords.begin());
msaa_array_descriptor.dword_count = 8;
ShaderRecompiler::IR::CompiledShaderInfo msaa_array_program{};
msaa_array_program.stage = ShaderType::Compute;
auto msaa_array_resource = array_resource;
msaa_array_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray;
msaa_array_program.info.images.push_back(msaa_array_resource);
ShaderRecompiler::IR::ResourceSnapshot msaa_array_snapshot{};
msaa_array_snapshot.images.push_back(msaa_array_descriptor);
ShaderStageRuntime msaa_array_runtime{&msaa_array_program, &msaa_array_snapshot};
PreparedBindings msaa_array_binding;
executor.PrepareBindings(msaa_array_runtime, msaa_array_binding);
executor.RebindImages(msaa_array_binding);
const auto &resolved_msaa_array = msaa_array_binding.images[0];
Require(name, "MSAA array backing expansion",
resolved_msaa_array.image_id != msaa_target_id &&
resolved_msaa_array.image_view != nullptr &&
resolved_msaa_array.desc.info.type ==
Prospero::ImageType::kColor2D &&
resolved_msaa_array.desc.info.resources ==
ImageSubresources{1, 2} &&
resolved_msaa_array.desc.info.samples == msaa_samples &&
resolved_msaa_array.desc.info.data.size == 0x80000 &&
resolved_msaa_array.desc.view_info.type ==
vk::ImageViewType::e2DArray &&
resolved_msaa_array.desc.view_info.layer_count == 2 &&
texture_cache.GetImage(resolved_msaa_array.image_id)
.backing.layers == 2 &&
texture_cache.GetImage(resolved_msaa_array.image_id)
.backing.samples == msaa_samples,
"MSAA array view did not expand the matching Color2D backing");
RenderExecutorTestAccess::ResetBindings(executor);
ImageDesc depth{};
depth.type = BindingType::DepthTarget;
depth.info.data = {depth_address, 0x10000};
depth.info.stencil = {stencil_address, stencil_layout.size};
depth.info.pixel_format = vk::Format::eD32SfloatS8Uint;
depth.info.guest_format = Prospero::BufferFormat::k32Float;
depth.info.type = Prospero::ImageType::kColor2D;
depth.info.extent = {1, 1, 1};
depth.info.resources = {1, 1};
depth.info.pitch = 1;
depth.info.bytes_per_block = 4;
depth.info.samples = 1;
depth.info.tile_mode = linear;
depth.info.mip_layout[0] = {0, depth.info.data.size, 1, 1};
depth.view_info.format = depth.info.pixel_format;
depth.view_info.type = vk::ImageViewType::e2D;
depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto depth_id = texture_cache.FindImage(depth);
Require(name, "cache-only depth discovery",
!texture_cache.FindImageFromRange(stencil_address,
stencil_layout.size, false) &&
!texture_cache.GetImage(depth_id).binding.is_bound &&
!texture_cache.GetImage(depth_id).binding.is_target &&
!texture_cache.GetImage(depth_id).usage.depth_target,
"FindImage created depth acquisition state or a stencil proxy");
RenderExecutorTestAccess::BindRenderTarget(executor, depth_id);
Require(name, "depth prefetch purity",
texture_cache.GetImage(depth_id).binding.is_target &&
!texture_cache.GetImage(depth_id).usage.depth_target &&
!texture_cache.FindImageFromRange(stencil_address,
stencil_layout.size, false),
"depth prefetch performed final target acquisition");
auto target_parent = make_target_desc(base, 1536, {4, 4, 1});
target_parent.type = BindingType::RenderTarget;
target_parent.info.resources.levels = 2;
// PS5 linear rows align to 256 bytes; smaller mips precede larger ones.
target_parent.info.pitch = 64;
target_parent.info.mip_layout[0] = {512, 1024, 64, 4};
target_parent.info.mip_layout[1] = {0, 512, 64, 2};
target_parent.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
auto target_base_subresource =
make_target_desc(base + 512, 1024, {4, 4, 1});
target_base_subresource.info.pitch = 64;
target_base_subresource.info.mip_layout[0] = {0, 1024, 64, 4};
const auto target_base_subresource_id =
texture_cache.FindImage(target_base_subresource);
auto target_subresource = make_target_desc(base, 512, {2, 2, 1});
target_subresource.info.pitch = 64;
target_subresource.info.mip_layout[0] = {0, 512, 64, 2};
const auto target_subresource_id =
texture_cache.FindImage(target_subresource);
const auto *target_subresource_owner =
TextureCacheTestAccess::Owner(texture_cache, target_subresource_id);
const auto &discovered_target =
texture_cache.GetImage(target_subresource_id);
Require(name, "cache-only target discovery",
!discovered_target.binding.is_bound &&
!discovered_target.binding.is_target &&
!discovered_target.binding.needs_rebind &&
!discovered_target.usage.render_target &&
!TextureCacheTestAccess::PendingDownload(
texture_cache, target_subresource_id),
"FindImage claimed RenderExecutor-owned render-target state");
vk::ClearValue target_clear{};
target_clear.color.uint32[0] = 0x12345678;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), target_base_subresource_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, target_clear);
target_clear.color.uint32[0] = 0x89abcdef;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), target_subresource_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, target_clear);
RenderExecutorTestAccess::BindRenderTarget(executor,
target_subresource_id);
Require(name, "target prefetch purity",
texture_cache.GetImage(target_subresource_id).binding.is_target &&
!texture_cache.GetImage(target_subresource_id)
.usage.render_target &&
!TextureCacheTestAccess::PendingDownload(
texture_cache, target_subresource_id),
"target prefetch performed final target acquisition");
const auto target_parent_id = texture_cache.FindImage(target_parent);
Require(name, "active target overlap",
target_parent_id &&
target_parent_id != target_base_subresource_id &&
target_parent_id != target_subresource_id &&
target_subresource_owner != nullptr &&
target_subresource_owner->binding.needs_rebind &&
texture_cache.GetImage(target_parent_id).binding.is_target &&
!TextureCacheTestAccess::PendingDownload(texture_cache,
target_parent_id),
"target overlap did not transfer target state to the merged "
"owner");
auto target_readback = CreateHostBuffer(
name, 2 * sizeof(u32), vk::BufferUsageFlagBits::eTransferDst,
{0xaaaaaaaa, 0xaaaaaaaa});
const std::array target_probes{
vk::BufferImageCopy{0,
0,
0,
{vk::ImageAspectFlagBits::eColor, 0, 0, 1},
{},
{1, 1, 1}},
vk::BufferImageCopy{sizeof(u32),
0,
0,
{vk::ImageAspectFlagBits::eColor, 1, 0, 1},
{},
{1, 1, 1}}};
texture_cache.GetImage(target_parent_id)
.Download(target_probes, target_readback.buffer, 0,
target_readback.size);
const vk::MemoryBarrier2 target_host_barrier{
.srcStageMask = vk::PipelineStageFlagBits2::eCopy,
.srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
.dstStageMask = vk::PipelineStageFlagBits2::eHost,
.dstAccessMask = vk::AccessFlagBits2::eHostRead};
vk::DependencyInfo target_dependency{};
target_dependency.memoryBarrierCount = 1;
target_dependency.pMemoryBarriers = &target_host_barrier;
scheduler.Current().Handle().pipelineBarrier2(target_dependency);
RenderColorInfo rebound_color{};
rebound_color.desc = target_subresource;
rebound_color.image_id = target_subresource_id;
RenderDepthInfo rebound_depth{};
rebound_depth.desc = depth;
rebound_depth.image_id = depth_id;
scheduler.Finish();
Require(name, "promoted target mip contents",
ReadBuffer(name, target_readback, 2) ==
std::vector<u32>{0x12345678, 0x89abcdef},
"promotion discarded GPU contents from a child mip");
DestroyBuffer(&target_readback);
Require(name, "deferred target slot erasure",
TextureCacheTestAccess::Owner(texture_cache,
target_subresource_id) == nullptr,
"the displaced target slot was not erased after GPU completion");
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, std::span<PreparedBindings *const>{}, std::span{&rebound_color, 1u});
auto rebound_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &rebound_color, 1, rebound_depth);
Require(name, "guest target extent after host mip remap",
rebound_color.desc.view_info.base_level == 1 &&
rebound_color.guest_mip_level == 0 &&
rebound_color.Extent() == vk::Extent2D{2, 2},
"remapping a guest target into its parent mip shrank the guest "
"render area twice");
const auto stencil_proxy_id = texture_cache.FindImageFromRange(
stencil_address, stencil_layout.size, false);
Require(
name, "production target rebind",
rebound_color.image_id == target_parent_id &&
rebound_rendering.color_attachments[0].image_view != nullptr &&
rebound_rendering.depth_stencil_attachment.image_view != nullptr &&
rebound_rendering.num_color_attachments == 1 &&
rebound_rendering.width == 1 && rebound_rendering.height == 1 &&
texture_cache.GetImage(target_parent_id).binding.is_target &&
texture_cache.GetImage(target_parent_id).usage.render_target &&
texture_cache.GetImage(target_parent_id).IsGpuModified() &&
texture_cache.GetImage(depth_id).usage.depth_target &&
texture_cache.GetImage(depth_id).IsGpuModified() &&
TextureCacheTestAccess::PendingDownload(texture_cache,
target_parent_id) &&
stencil_proxy_id &&
texture_cache.GetImage(stencil_proxy_id).depth_id == depth_id,
"the final target pass did not acquire color before depth and "
"build the dynamic render state");
scheduler.BeginRendering(rebound_rendering);
scheduler.EndRendering();
RenderExecutorTestAccess::ResetBindings(executor);
Require(name, "draw-scoped target reset",
TextureCacheTestAccess::Owner(texture_cache,
target_subresource_id) == nullptr &&
!texture_cache.GetImage(target_parent_id).binding.is_target &&
!texture_cache.GetImage(depth_id).binding.is_target,
"RenderExecutor retained target binding state after reset");
constexpr uint64_t ordered_color_address = base + 0x80000;
auto ordered_color_desc =
make_target_desc(ordered_color_address, target_mip_size, {1, 1, 1});
const auto stale_ordered_color =
texture_cache.FindImage(ordered_color_desc);
RenderExecutorTestAccess::BindRenderTarget(executor, stale_ordered_color);
resources.UnmapMemory(ordered_color_address, target_mip_size);
resources.MapMemory(ordered_color_address, target_mip_size);
auto ordered_depth_desc = depth;
ordered_depth_desc.info.stencil = {ordered_color_address,
target_mip_size};
const auto ordered_depth_id = texture_cache.FindImage(ordered_depth_desc);
RenderExecutorTestAccess::BindRenderTarget(executor, ordered_depth_id);
RenderColorInfo ordered_color{};
ordered_color.desc = ordered_color_desc;
ordered_color.image_id = stale_ordered_color;
RenderDepthInfo ordered_depth{};
ordered_depth.desc = ordered_depth_desc;
ordered_depth.image_id = ordered_depth_id;
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, std::span<PreparedBindings *const>{}, std::span{&ordered_color, 1u});
auto ordered_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &ordered_color, 1, ordered_depth);
Require(name, "color-before-depth acquisition order",
ordered_color.image_id != stale_ordered_color &&
ordered_rendering.color_attachments[0].image_view != nullptr &&
ordered_rendering.depth_stencil_attachment.image_view != nullptr &&
texture_cache.GetImage(ordered_color.image_id).depth_id ==
ordered_depth.image_id,
"depth acquisition ran before the stale color target was "
"recreated and associated");
RenderExecutorTestAccess::ResetBindings(executor);
TextureCacheTestAccess::SetLinearReadback(texture_cache, false);
ShaderTextureResource stencil{};
const uint64_t encoded_address = stencil_address >> 8u;
stencil.fields[0] = static_cast<uint32_t>(encoded_address);
stencil.fields[1] = static_cast<uint32_t>(encoded_address >> 32u) |
(static_cast<uint32_t>(stencil_format) << 20u);
stencil.fields[3] =
DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.stage = ShaderType::Compute;
ShaderRecompiler::IR::ImageResource stencil_resource{};
stencil_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
stencil_resource.numeric_class = Prospero::TextureNumericClass::Uint;
stencil_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
stencil_resource.read = true;
program.info.images.push_back(stencil_resource);
ShaderRecompiler::IR::ResourceSnapshot snapshot{};
ShaderRecompiler::IR::DescriptorValue descriptor{};
std::copy(std::begin(stencil.fields), std::end(stencil.fields),
descriptor.dwords.begin());
descriptor.dword_count = 8;
snapshot.images.push_back(descriptor);
ShaderStageRuntime runtime{&program, &snapshot};
PreparedBindings prepared;
context.GetRenderExecutor().PrepareBindings(runtime, prepared);
const auto sampled_stencil_id = prepared.images[0].image_id;
Require(name, "first stencil discovery",
prepared.images.size() == 1 &&
sampled_stencil_id != depth_id &&
prepared.images[0].image_view == nullptr &&
texture_cache.GetImage(sampled_stencil_id).binding.is_bound &&
!texture_cache.GetImage(stencil_proxy_id).binding.is_bound &&
!texture_cache.GetImage(stencil_proxy_id).binding.is_target,
"the first sampled-stencil lookup did not remain an ordinary "
"discovery before final depth acquisition");
context.GetRenderExecutor().RebindImages(prepared);
Require(name, "first stencil acquisition",
prepared.images[0].image_id == sampled_stencil_id &&
prepared.images[0].image_view != nullptr &&
texture_cache.GetImage(sampled_stencil_id).usage.texture,
"the first sampled-stencil image was not finally acquired");
RenderExecutorTestAccess::ResetBindings(executor);
RenderDepthInfo reassociated_depth = rebound_depth;
RenderColorInfo no_reassociated_color{};
(void)RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_reassociated_color, 0,
reassociated_depth);
Require(name, "existing stencil association selection",
texture_cache.GetImage(sampled_stencil_id).depth_id ==
reassociated_depth.image_id,
"final depth acquisition did not associate the existing image "
"at the stencil guest address");
RenderExecutorTestAccess::ResetBindings(executor);
PreparedBindings redirected;
context.GetRenderExecutor().PrepareBindings(runtime, redirected);
Require(name, "redirected owner discovery",
redirected.images.size() == 1 &&
redirected.images[0].image_id == depth_id &&
redirected.images[0].image_view == nullptr &&
texture_cache.GetImage(depth_id).binding.is_bound &&
!texture_cache.GetImage(sampled_stencil_id).binding.is_bound,
"the established stencil association did not redirect the next "
"discovery to the depth owner");
context.GetRenderExecutor().RebindImages(redirected);
Require(name, "second-pass stencil acquisition",
redirected.images[0].image_id == depth_id &&
redirected.images[0].image_view != nullptr &&
texture_cache.GetImage(depth_id).usage.texture,
"RebindImages did not acquire the associated depth owner");
RenderExecutorTestAccess::ResetBindings(executor);
// Execute the captured IMAGE_STORE clear through production fill recognition.
const std::array<uint32_t, 12> stencil_shader{
0xd7460000u, 0x0401060cu, 0xf4201a84u, 0xfa000000u,
0xbf8cc07fu, 0x7e06026au, 0xd7460001u, 0x0405060du,
0x7e04020eu, 0xf0200128u, 0x00000300u, 0xbf810000u};
uint32_t stencil_byte = 0x35;
const auto value_address = reinterpret_cast<uint64_t>(&stencil_byte);
auto storage_stencil = stencil;
storage_stencil.fields[3] = (storage_stencil.fields[3] & 0x0fffffffu) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DArray) << 28u);
storage_stencil.fields[5] = 0x00700000u;
std::array<uint32_t, 12> stencil_userdata{};
std::copy_n(storage_stencil.fields, 8, stencil_userdata.begin());
stencil_userdata[8] = static_cast<uint32_t>(value_address);
stencil_userdata[9] = static_cast<uint32_t>(value_address >> 32) | (4u << 16);
stencil_userdata[10] = 1;
stencil_userdata[11] = 0x14204u;
ShaderComputeInputInfo stencil_compute{};
stencil_compute.threads_num[0] = stencil_compute.threads_num[1] = 8;
stencil_compute.threads_num[2] = 1;
stencil_compute.workgroup_register = 12;
stencil_compute.thread_ids_num = 2;
std::fill_n(stencil_compute.group_id, 3, true);
ShaderRecompiler::CompileOptions stencil_options;
stencil_options.stage = ShaderType::Compute;
stencil_options.user_data = stencil_userdata;
stencil_options.input_info.compute = &stencil_compute;
auto stencil_translated = ShaderRecompiler::TranslateProgram(stencil_shader, stencil_options);
auto stencil_plan = ShaderRecompiler::IR::ExtractResourcePlan(stencil_translated.program);
ShaderRecompiler::IR::ResourceSpecialization stencil_specialization;
ShaderRecompiler::IR::ResourceSnapshot stencil_snapshot;
stencil_compute.stage.resources = &stencil_snapshot;
Require(name, "stencil fill materialization",
ShaderRecompiler::IR::MaterializeResources(stencil_plan,
{.user_data = stencil_userdata, .userdata = &stencil_byte,
.read_specialization_memory = +[](void *data, uint64_t address, std::span<uint32_t> words) {
if (address != reinterpret_cast<uint64_t>(data) || words.size() != 1u) return false;
words[0] = *static_cast<uint32_t *>(data);
return true;
}}, stencil_snapshot, stencil_specialization),
"captured stencil shader could not resolve its clear byte");
ShaderRecompiler::IR::ApplyResourceSpecialization(stencil_translated.program,
stencil_specialization);
ShaderRecompiler::IR::CompiledShaderInfo stencil_program{};
stencil_program.stage = ShaderType::Compute;
stencil_program.info = std::move(stencil_translated.program.info);
stencil_compute.stage.program = &stencil_program;
vk::ClearValue stencil_initial{};
stencil_initial.depthStencil = vk::ClearDepthStencilValue{0.625f, 0x91};
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), depth_id,
{vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1},
stencil_initial);
Require(name, "stencil fill coverage",
!RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
scheduler.Current(), 0, 1, 1, 0x41u) &&
!RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
scheduler.Current(), 2, 1, 1, 0x41u),
"incomplete or excessive stencil dispatch was consumed");
auto &stencil_source = stencil_snapshot.buffers[0];
const auto clean_source = stencil_source;
stencil_source.dwords[0] = static_cast<uint32_t>(stencil_address);
stencil_source.dwords[1] = static_cast<uint32_t>(stencil_address >> 32) | (4u << 16);
Require(name, "stencil fill source alias",
!RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
scheduler.Current(), 1, 1, 1, 0x41u),
"an aliased scalar load was consumed as a uniform image clear");
stencil_source = clean_source;
Require(name, "stencil IMAGE_STORE clear",
RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
scheduler.Current(), 1, 1, 1, 0x41u),
"captured clear attempted an unsupported depth/stencil storage binding");
auto stencil_readback = CreateHostBuffer(name, 8,
vk::BufferUsageFlagBits::eTransferDst, {});
const std::array<vk::BufferImageCopy, 2> stencil_copies{{
{0, 0, 0, {vk::ImageAspectFlagBits::eDepth, 0, 0, 1}, {}, {1, 1, 1}},
{4, 0, 0, {vk::ImageAspectFlagBits::eStencil, 0, 0, 1}, {}, {1, 1, 1}}}};
texture_cache.GetImage(depth_id).Download(stencil_copies, stencil_readback.buffer, 0, 8);
vk::MemoryBarrier2 stencil_host_barrier{};
stencil_host_barrier.sType = vk::StructureType::eMemoryBarrier2;
stencil_host_barrier.srcStageMask = vk::PipelineStageFlagBits2::eTransfer;
stencil_host_barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
stencil_host_barrier.dstStageMask = vk::PipelineStageFlagBits2::eHost;
stencil_host_barrier.dstAccessMask = vk::AccessFlagBits2::eHostRead;
vk::DependencyInfo stencil_dependency{};
stencil_dependency.sType = vk::StructureType::eDependencyInfo;
stencil_dependency.memoryBarrierCount = 1;
stencil_dependency.pMemoryBarriers = &stencil_host_barrier;
scheduler.Current().Handle().pipelineBarrier2(stencil_dependency);
scheduler.Finish();
const auto stencil_result = ReadBuffer(name, stencil_readback, 2);
Require(name, "stencil clear retains depth",
stencil_result[0] == std::bit_cast<uint32_t>(0.625f) &&
(stencil_result[1] & 0xffu) == 0x35 &&
!texture_cache.GetImage(depth_id).usage.storage,
"stencil clear altered depth, lost its scalar value, or used a storage view");
DestroyBuffer(&stencil_readback);
// Buffer writes to the independent tiled stencil plane must reach the
// combined native attachment, including values that are not a clear.
{
LibKernel::Memory::InstallGpuResources(&resources);
constexpr uint64_t plane_size = 0x10000;
constexpr uint64_t plane_address = base + 0x910000;
constexpr uint32_t texels = 6;
auto plane_depth = depth;
plane_depth.info.data = {base + 0x900000, plane_size};
plane_depth.info.stencil = {plane_address, plane_size};
plane_depth.info.extent = {3, 2, 1};
plane_depth.info.pitch = TileGetDepthPitch(3, sizeof(uint32_t));
plane_depth.info.tile_mode = Prospero::TileMode::kDepth;
plane_depth.info.mip_layout[0] = {0, plane_size, plane_depth.info.pitch, 2};
const auto plane_depth_id = texture_cache.FindImage(plane_depth);
auto sampled_plane = plane_depth;
sampled_plane.type = BindingType::Texture;
sampled_plane.info.data = plane_depth.info.stencil;
sampled_plane.info.stencil = {};
sampled_plane.info.metadata = {};
sampled_plane.info.pixel_format = vk::Format::eR8Uint;
sampled_plane.info.guest_format = Prospero::BufferFormat::k8UInt;
sampled_plane.info.bytes_per_block = 1;
sampled_plane.info.pitch = TileGetDepthPitch(3, 1);
sampled_plane.info.mip_layout[0] = {0, plane_size, sampled_plane.info.pitch, 2};
sampled_plane.view_info.format = vk::Format::eR8Uint;
sampled_plane.view_info.aspect = vk::ImageAspectFlagBits::eColor;
sampled_plane.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto sampled_plane_id = texture_cache.FindImage(sampled_plane);
(void)texture_cache.FindTexture(sampled_plane_id, sampled_plane);
(void)texture_cache.FindDepthTarget(plane_depth_id, plane_depth);
Require(name, "sampled stencil association",
texture_cache.GetImage(sampled_plane_id).depth_id == plane_depth_id,
"depth acquisition did not associate the sampled stencil plane");
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), plane_depth_id,
{vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1},
stencil_initial);
TileBlockLayout stencil_block{};
Require(name, "independent stencil tile pitch",
TileGetBlockLayout(TileBlockFamily::Depth64KB, 1, stencil_block) &&
plane_depth.info.pitch == 128 && TileGetDepthPitch(3, 1) == 256,
"stencil must use its own one-byte tile layout");
std::vector<uint32_t> plane_words(plane_size / sizeof(uint32_t), 0);
auto *plane_bytes = reinterpret_cast<uint8_t *>(plane_words.data());
std::array<uint32_t, texels> plane_offsets{};
for (uint32_t i = 0; i < texels; ++i) {
Require(name, "stencil tile offset",
TileGetBlockOffset(stencil_block, i % 3, i / 3, 0, plane_offsets[i]),
"stencil texel offset is unavailable");
plane_bytes[plane_offsets[i]] = static_cast<uint8_t>(0x21 + i * 7);
}
auto plane_upload = CreateHostBuffer(name, plane_size,
vk::BufferUsageFlagBits::eTransferSrc, plane_words);
const auto plane_program = make_buffer_program(
ShaderType::Compute, {.written = true, .formatted = true});
ShaderBufferResource plane_descriptor{};
plane_descriptor.UpdateAddress48(plane_address);
plane_descriptor.fields[1] |= 4u << 16u;
plane_descriptor.fields[2] = plane_size / sizeof(uint32_t);
plane_descriptor.fields[3] = 0x00014204u;
ShaderRecompiler::IR::ResourceSnapshot plane_snapshot;
auto &plane_value = plane_snapshot.buffers.emplace_back();
std::memcpy(plane_value.dwords.data(), plane_descriptor.fields,
sizeof(plane_descriptor.fields));
plane_value.dword_count = 4;
const ShaderStageRuntime plane_runtime{&plane_program, &plane_snapshot};
PreparedBindings plane_bindings;
executor.PrepareBindings(plane_runtime, plane_bindings);
vk::MemoryBarrier2 plane_barrier{};
plane_barrier.srcStageMask = plane_barrier.dstStageMask =
vk::PipelineStageFlagBits2::eAllCommands;
plane_barrier.srcAccessMask = plane_barrier.dstAccessMask =
vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite;
vk::DependencyInfo plane_dependency{};
plane_dependency.memoryBarrierCount = 1;
plane_dependency.pMemoryBarriers = &plane_barrier;
const auto write_plane = [&] {
std::array<PreparedBindings *, 1> stages{&plane_bindings};
executor.FindBuffers(stages);
executor.RebindBuffers(plane_bindings);
const auto &plane_buffer = plane_bindings.buffers[0];
const vk::BufferCopy plane_copy{0, plane_buffer.offset, plane_size};
scheduler.Current().Handle().pipelineBarrier2(plane_dependency);
scheduler.Current().Handle().copyBuffer(plane_upload.buffer, plane_buffer.buffer,
1, &plane_copy);
scheduler.Current().Handle().pipelineBarrier2(plane_dependency);
RenderExecutorTestAccess::ResetBindings(executor);
};
write_plane();
auto plane_readback = CreateHostBuffer(name, texels * 4 + 8,
vk::BufferUsageFlagBits::eTransferDst, {});
const std::array<vk::BufferImageCopy, 2> plane_copies{{
{0, 0, 0, {vk::ImageAspectFlagBits::eDepth, 0, 0, 1}, {}, {3, 2, 1}},
{texels * 4, 0, 0, {vk::ImageAspectFlagBits::eStencil, 0, 0, 1}, {}, {3, 2, 1}}}};
const auto check_plane = [&](const char *phase, bool native_clear) {
(void)texture_cache.FindDepthTarget(plane_depth_id, plane_depth);
texture_cache.GetImage(plane_depth_id).Download(
plane_copies, plane_readback.buffer, 0, plane_readback.size);
scheduler.Current().Handle().pipelineBarrier2(stencil_dependency);
scheduler.Finish();
const auto values = ReadBuffer(name, plane_readback, texels + 2);
const auto *bytes = reinterpret_cast<const uint8_t *>(values.data() + texels);
for (uint32_t i = 0; i < texels; ++i) {
Require(name, phase,
values[i] == std::bit_cast<uint32_t>(0.625f) &&
bytes[i] == (native_clear ? 0x67 : plane_bytes[plane_offsets[i]]),
"stencil acquisition lost bytes or overwrote the independent depth plane");
}
};
check_plane("GPU stencil buffer write", false);
write_plane();
vk::ClearValue stencil_again{};
stencil_again.depthStencil.stencil = 0x67;
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), plane_depth_id,
{vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1}, stencil_again);
check_plane("consumed stencil dirtiness", true);
check_plane("repeated clean stencil acquisition", true);
for (uint32_t i = 0; i < texels; ++i) {
plane_bytes[plane_offsets[i]] += 0x10;
}
std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
check_plane("CPU stencil plane write", false);
std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), plane_depth_id,
{vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1}, stencil_again);
for (uint32_t i = 0; i < texels; ++i) {
plane_bytes[plane_offsets[i]] += 0x10;
}
std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
check_plane("CPU stencil write after native clear", false);
for (uint32_t i = 0; i < texels; ++i) {
plane_bytes[plane_offsets[i]] += 0x10;
}
std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
Require(name, "sampled stencil view",
texture_cache.FindTexture(plane_depth_id, sampled_plane) != nullptr,
"an associated stencil plane could not be sampled");
texture_cache.GetImage(plane_depth_id).Download(
plane_copies, plane_readback.buffer, 0, plane_readback.size);
scheduler.Current().Handle().pipelineBarrier2(stencil_dependency);
scheduler.Finish();
const auto sampled_values = ReadBuffer(name, plane_readback, texels + 2);
const auto *sampled_bytes =
reinterpret_cast<const uint8_t *>(sampled_values.data() + texels);
for (uint32_t i = 0; i < texels; ++i) {
Require(name, "sampled stencil plane refresh",
sampled_bytes[i] == plane_bytes[plane_offsets[i]],
"sampling without another depth acquisition used stale stencil bytes");
}
DestroyBuffer(&plane_readback);
DestroyBuffer(&plane_upload);
LibKernel::Memory::InstallGpuResources(nullptr);
}
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
RenderExecutorTestAccess::DestroyDescriptorPipelines(
executor, descriptor_pipelines);
context.ShutdownGpu();
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"descriptor discovery direct mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"descriptor discovery direct-memory allocation release failed");
std::printf("[host] %-32s ok\n", name);
}
Buffer CreateStorageBuffer(const char *shader_name,
const std::vector<u32> &initial,
size_t dword_count, bool device_address = false) {
Buffer ret;
ret.size = static_cast<vk::DeviceSize>(std::max<size_t>(dword_count, 1u) *
sizeof(u32));
vk::BufferCreateInfo buffer_info{};
buffer_info.sType = vk::StructureType::eBufferCreateInfo;
buffer_info.size = ret.size;
buffer_info.usage = vk::BufferUsageFlagBits::eStorageBuffer;
if (device_address) {
buffer_info.usage |= vk::BufferUsageFlagBits::eShaderDeviceAddress;
}
buffer_info.sharingMode = vk::SharingMode::eExclusive;
RequireVk(shader_name, "dispatch",
m_device.createBuffer(&buffer_info, nullptr, &ret.buffer),
"vkCreateBuffer");
vk::MemoryRequirements req{};
m_device.getBufferMemoryRequirements(ret.buffer, &req);
u32 memory_type = 0;
ret.coherent = FindMemoryType(req.memoryTypeBits,
vk::MemoryPropertyFlagBits::eHostVisible |
vk::MemoryPropertyFlagBits::eHostCoherent,
&memory_type);
if (!ret.coherent) {
Require(shader_name, "dispatch",
FindMemoryType(req.memoryTypeBits,
vk::MemoryPropertyFlagBits::eHostVisible,
&memory_type),
"no host-visible memory type for storage buffer");
}
vk::MemoryAllocateInfo alloc{};
alloc.sType = vk::StructureType::eMemoryAllocateInfo;
alloc.allocationSize = req.size;
alloc.memoryTypeIndex = memory_type;
vk::MemoryAllocateFlagsInfo flags{};
if (device_address) {
flags.sType = vk::StructureType::eMemoryAllocateFlagsInfo;
flags.flags = vk::MemoryAllocateFlagBits::eDeviceAddress;
alloc.pNext = &flags;
}
RequireVk(shader_name, "dispatch",
m_device.allocateMemory(&alloc, nullptr, &ret.memory),
"vkAllocateMemory");
RequireVk(shader_name, "dispatch",
m_device.bindBufferMemory(ret.buffer, ret.memory, 0),
"vkBindBufferMemory");
if (device_address) {
vk::BufferDeviceAddressInfo address_info{};
address_info.sType = vk::StructureType::eBufferDeviceAddressInfo;
address_info.buffer = ret.buffer;
ret.device_address = m_device.getBufferAddress(address_info);
Require(shader_name, "dispatch", ret.device_address != 0,
"storage buffer has no device address");
}
std::vector<u32> contents(dword_count, 0);
for (size_t i = 0; i < initial.size() && i < contents.size(); i++) {
contents[i] = initial[i];
}
WriteBuffer(shader_name, ret, contents);
return ret;
}
void DestroyBuffer(Buffer *buffer) {
if (buffer == nullptr) {
return;
}
if (buffer->buffer != nullptr) {
m_device.destroyBuffer(buffer->buffer, nullptr);
buffer->buffer = nullptr;
}
if (buffer->memory != nullptr) {
m_device.freeMemory(buffer->memory, nullptr);
buffer->memory = nullptr;
}
}
Image CreateImage2D(const char *shader_name, u32 width, u32 height,
vk::Format format, vk::ImageUsageFlags usage,
const std::vector<u32> &initial, u32 dwords_per_pixel,
vk::ImageLayout final_layout,
vk::Format view_format = vk::Format::eUndefined) {
std::vector<std::vector<u32>> mips;
if (!initial.empty()) {
mips.push_back(initial);
}
return CreateImageMips(shader_name, width, height, format, usage, mips,
dwords_per_pixel, final_layout, vk::ImageType::e2D,
vk::ImageViewType::e2D, 1, 0, 0,
vk::SampleCountFlagBits::e1, view_format);
}
static u32 MipExtent(u32 value, u32 level) {
for (u32 i = 0; i < level && value > 1u; i++) {
value >>= 1u;
}
return std::max(value, 1u);
}
static size_t ImageMipDwordCount(u32 width, u32 height, u32 dwords_per_pixel,
u32 level, u32 layers = 1) {
return static_cast<size_t>(MipExtent(width, level)) *
static_cast<size_t>(MipExtent(height, level)) * layers *
dwords_per_pixel;
}
Image CreateImageMips(const char *shader_name, u32 width, u32 height,
vk::Format format, vk::ImageUsageFlags usage,
const std::vector<std::vector<u32>> &initial_mips,
u32 dwords_per_pixel, vk::ImageLayout final_layout,
vk::ImageType image_type, vk::ImageViewType view_type,
u32 layers, u32 view_base_layer = 0,
u32 view_layers = 0,
vk::SampleCountFlagBits samples = vk::SampleCountFlagBits::e1,
vk::Format view_format = vk::Format::eUndefined,
u32 view_base_mip = 0) {
Image ret;
ret.format = format;
ret.width = width;
ret.height = height;
ret.layers = layers;
ret.mip_levels = std::max<u32>(static_cast<u32>(initial_mips.size()), 1u);
ret.dwords_per_pixel = dwords_per_pixel;
vk::ImageCreateInfo image_info{};
image_info.sType = vk::StructureType::eImageCreateInfo;
image_info.imageType = image_type;
image_info.format = format;
image_info.extent.width = width;
image_info.extent.height = height;
image_info.extent.depth = 1;
image_info.mipLevels = ret.mip_levels;
image_info.arrayLayers = layers;
image_info.samples = samples;
image_info.tiling = vk::ImageTiling::eOptimal;
image_info.usage = usage | vk::ImageUsageFlagBits::eTransferDst |
vk::ImageUsageFlagBits::eTransferSrc;
image_info.sharingMode = vk::SharingMode::eExclusive;
image_info.initialLayout = vk::ImageLayout::eUndefined;
if (view_format != vk::Format::eUndefined && view_format != format) {
image_info.flags |= vk::ImageCreateFlagBits::eMutableFormat;
}
RequireVk(shader_name, "dispatch",
m_device.createImage(&image_info, nullptr, &ret.image),
"vkCreateImage");
vk::MemoryRequirements req{};
m_device.getImageMemoryRequirements(ret.image, &req);
u32 memory_type = 0;
if (!FindMemoryType(req.memoryTypeBits,
vk::MemoryPropertyFlagBits::eDeviceLocal,
&memory_type)) {
Require(shader_name, "dispatch",
FindMemoryType(req.memoryTypeBits, {}, &memory_type),
"no memory type for image");
}
vk::MemoryAllocateInfo alloc{};
alloc.sType = vk::StructureType::eMemoryAllocateInfo;
alloc.allocationSize = req.size;
alloc.memoryTypeIndex = memory_type;
RequireVk(shader_name, "dispatch",
m_device.allocateMemory(&alloc, nullptr, &ret.memory),
"vk::Device::allocateMemory(image)");
RequireVk(shader_name, "dispatch",
m_device.bindImageMemory(ret.image, ret.memory, 0),
"vkBindImageMemory");
vk::ImageViewCreateInfo view_info{};
view_info.sType = vk::StructureType::eImageViewCreateInfo;
view_info.image = ret.image;
view_info.viewType = view_type;
view_info.format = view_format == vk::Format::eUndefined ? format : view_format;
view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor;
Require(shader_name, "dispatch", view_base_mip < ret.mip_levels,
"image view base mip is out of bounds");
view_info.subresourceRange.baseMipLevel = view_base_mip;
view_info.subresourceRange.levelCount = ret.mip_levels - view_base_mip;
Require(shader_name, "dispatch", view_base_layer < layers,
"image view base layer is out of bounds");
if (view_layers == 0) {
view_layers = layers - view_base_layer;
}
Require(shader_name, "dispatch", view_layers <= layers - view_base_layer,
"image view layer count is out of bounds");
view_info.subresourceRange.baseArrayLayer = view_base_layer;
view_info.subresourceRange.layerCount = view_layers;
RequireVk(shader_name, "dispatch",
m_device.createImageView(&view_info, nullptr, &ret.view),
"vkCreateImageView");
if (!initial_mips.empty()) {
size_t total_dwords = 0;
for (u32 level = 0; level < ret.mip_levels; level++) {
total_dwords +=
ImageMipDwordCount(width, height, dwords_per_pixel, level, layers);
}
std::vector<u32> contents(total_dwords, 0);
size_t offset = 0;
for (u32 level = 0; level < ret.mip_levels; level++) {
const auto level_dwords =
ImageMipDwordCount(width, height, dwords_per_pixel, level, layers);
const auto &src = initial_mips[level];
for (size_t i = 0; i < src.size() && i < level_dwords; i++) {
contents[offset + i] = src[i];
}
offset += level_dwords;
}
auto staging =
CreateHostBuffer(shader_name, contents.size() * sizeof(u32),
vk::BufferUsageFlagBits::eTransferSrc, contents);
UploadImageMips(shader_name, &ret, staging.buffer, final_layout);
DestroyBuffer(&staging);
} else {
TransitionImage(shader_name, &ret, final_layout,
vk::PipelineStageFlagBits::eTopOfPipe,
vk::PipelineStageFlagBits::eComputeShader, {},
AccessForLayout(final_layout));
}
return ret;
}
void DestroyImage(Image *image) {
if (image == nullptr) {
return;
}
if (image->view != nullptr) {
m_device.destroyImageView(image->view, nullptr);
image->view = nullptr;
}
if (image->image != nullptr) {
m_device.destroyImage(image->image, nullptr);
image->image = nullptr;
}
if (image->memory != nullptr) {
m_device.freeMemory(image->memory, nullptr);
image->memory = nullptr;
}
}
std::vector<u32> ReadImage(const char *shader_name, Image *image) {
const auto dword_count = static_cast<size_t>(image->width) *
static_cast<size_t>(image->height) *
image->layers * image->dwords_per_pixel;
auto staging = CreateHostBuffer(shader_name, dword_count * sizeof(u32),
vk::BufferUsageFlagBits::eTransferDst, {});
vk::CommandBuffer cmd = BeginCommands(shader_name, "readback");
AddImageBarrier(
cmd, image->image, image->layout, vk::ImageLayout::eTransferSrcOptimal,
vk::PipelineStageFlagBits::eAllCommands,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
vk::AccessFlagBits::eTransferRead, image->mip_levels, image->layers);
vk::BufferImageCopy copy{};
copy.bufferOffset = 0;
copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
copy.imageSubresource.mipLevel = 0;
copy.imageSubresource.baseArrayLayer = 0;
copy.imageSubresource.layerCount = image->layers;
copy.imageExtent.width = image->width;
copy.imageExtent.height = image->height;
copy.imageExtent.depth = 1;
cmd.copyImageToBuffer(image->image, vk::ImageLayout::eTransferSrcOptimal,
staging.buffer, 1, &copy);
EndSubmitAndFree(shader_name, "readback", cmd);
image->layout = vk::ImageLayout::eTransferSrcOptimal;
auto ret = ReadBuffer(shader_name, staging, dword_count);
DestroyBuffer(&staging);
return ret;
}
vk::Sampler CreateSampler(const char *shader_name, u32 mip_levels = 1,
vk::Filter filter = vk::Filter::eNearest) {
vk::SamplerCreateInfo sampler_info{};
sampler_info.sType = vk::StructureType::eSamplerCreateInfo;
sampler_info.magFilter = filter;
sampler_info.minFilter = filter;
sampler_info.mipmapMode = vk::SamplerMipmapMode::eNearest;
sampler_info.addressModeU = vk::SamplerAddressMode::eClampToEdge;
sampler_info.addressModeV = vk::SamplerAddressMode::eClampToEdge;
sampler_info.addressModeW = vk::SamplerAddressMode::eClampToEdge;
sampler_info.minLod = 0.0f;
sampler_info.maxLod = static_cast<float>(mip_levels - 1u);
vk::Sampler sampler = nullptr;
RequireVk(shader_name, "dispatch",
m_device.createSampler(&sampler_info, nullptr, &sampler),
"vkCreateSampler");
return sampler;
}
std::vector<u32> ReadBuffer(const char *shader_name, const Buffer &buffer,
size_t dword_count) {
if (!buffer.coherent) {
vk::MappedMemoryRange range{};
range.sType = vk::StructureType::eMappedMemoryRange;
range.memory = buffer.memory;
range.offset = 0;
range.size = VK_WHOLE_SIZE;
RequireVk(shader_name, "readback",
m_device.invalidateMappedMemoryRanges(1, &range),
"vkInvalidateMappedMemoryRanges");
}
void *data = nullptr;
RequireVk(shader_name, "readback",
m_device.mapMemory(buffer.memory, 0, buffer.size, {}, &data),
"vkMapMemory");
std::vector<u32> ret(dword_count, 0);
std::memcpy(ret.data(), data, dword_count * sizeof(u32));
m_device.unmapMemory(buffer.memory);
return ret;
}
void Dispatch(const TestCase &test, const CompiledShader &compiled,
const Buffer &buffer, const Buffer *gds_buffer = nullptr,
const Image *sampled_image = nullptr,
const Image *storage_image = nullptr,
const Image *storage_image_uint = nullptr,
vk::Sampler sampler = nullptr,
std::span<const Image> sampled_resources = {}) {
using Kind = ShaderRecompiler::IR::DescriptorBindingKind;
const auto &layout = compiled.program.bindings;
auto shader_data = compiled.packed_user_data;
if (layout.dispatch_thread_dword != ShaderRecompiler::IR::PushData::NoStart) {
Require(test.name, "dispatch dimensions", test.has_compute_info,
"runtime thread extents are required by the compiled shader");
std::copy_n(test.compute_info.dispatch_threads_num, 3,
shader_data.begin() + layout.dispatch_thread_dword);
}
auto Count = [&](Kind kind) {
return layout.descriptor_counts[static_cast<size_t>(kind)];
};
auto Native = [](Kind kind) {
return ShaderRecompiler::IR::NativeBinding(ShaderType::Compute, kind);
};
for (const auto &image : compiled.program.info.images) {
bool supported =
image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
if (image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) {
supported = supported ||
image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1D ||
image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1DArray;
}
Require(test.name, "dispatch", supported,
"unsupported image dimension needs a matching Vulkan test view");
}
vk::ShaderModuleCreateInfo module_info{};
module_info.sType = vk::StructureType::eShaderModuleCreateInfo;
module_info.codeSize = compiled.spirv.size() * sizeof(u32);
module_info.pCode = compiled.spirv.data();
vk::ShaderModule module = nullptr;
RequireVk(test.name, "SPIR-V module",
m_device.createShaderModule(&module_info, nullptr, &module),
"vkCreateShaderModule");
std::vector<vk::DescriptorSetLayoutBinding> layout_bindings;
auto add_layout_binding =
[&layout_bindings](u32 binding, vk::DescriptorType type, u32 count) {
if (count == 0) {
return;
}
vk::DescriptorSetLayoutBinding item{};
item.binding = binding;
item.descriptorType = type;
item.descriptorCount = count;
item.stageFlags = vk::ShaderStageFlagBits::eCompute;
layout_bindings.push_back(item);
};
for (auto mask = layout.descriptor_mask; mask != 0; mask &= mask - 1) {
const auto index = std::countr_zero(mask);
const auto kind = static_cast<Kind>(index);
add_layout_binding(Native(kind), NativeDescriptorType(kind), layout.descriptor_counts[index]);
}
vk::DescriptorSetLayoutCreateInfo layout_info{};
layout_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
layout_info.bindingCount = static_cast<u32>(layout_bindings.size());
layout_info.pBindings =
layout_bindings.empty() ? nullptr : layout_bindings.data();
vk::DescriptorSetLayout descriptor_layout = nullptr;
RequireVk(test.name, "dispatch",
m_device.createDescriptorSetLayout(&layout_info, nullptr,
&descriptor_layout),
"vkCreateDescriptorSetLayout");
vk::PipelineLayoutCreateInfo pipeline_layout_info{};
pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
pipeline_layout_info.setLayoutCount = 1;
pipeline_layout_info.pSetLayouts = &descriptor_layout;
vk::PushConstantRange push_range{};
if (layout.UsesPushData()) {
push_range.stageFlags = vk::ShaderStageFlagBits::eCompute;
push_range.offset = 0;
push_range.size = ShaderRecompiler::IR::NativePushConstantSize;
pipeline_layout_info.pushConstantRangeCount = 1;
pipeline_layout_info.pPushConstantRanges = &push_range;
}
vk::PipelineLayout pipeline_layout = nullptr;
RequireVk(test.name, "dispatch",
m_device.createPipelineLayout(&pipeline_layout_info, nullptr,
&pipeline_layout),
"vkCreatePipelineLayout");
vk::PipelineShaderStageCreateInfo stage{};
stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
stage.stage = vk::ShaderStageFlagBits::eCompute;
stage.module = module;
stage.pName = "main";
vk::ComputePipelineCreateInfo pipeline_info{};
pipeline_info.sType = vk::StructureType::eComputePipelineCreateInfo;
pipeline_info.stage = stage;
pipeline_info.layout = pipeline_layout;
vk::Pipeline pipeline = nullptr;
RequireVk(test.name, "dispatch",
m_device.createComputePipelines(nullptr, 1, &pipeline_info,
nullptr, &pipeline),
"vkCreateComputePipelines");
std::vector<vk::DescriptorPoolSize> pool_sizes;
auto add_pool_size = [&pool_sizes](vk::DescriptorType type, u32 count) {
if (count == 0) {
return;
}
for (auto &size : pool_sizes) {
if (size.type == type) {
size.descriptorCount += count;
return;
}
}
pool_sizes.push_back({type, count});
};
for (auto mask = layout.descriptor_mask; mask != 0; mask &= mask - 1) {
const auto index = std::countr_zero(mask);
add_pool_size(NativeDescriptorType(static_cast<Kind>(index)), layout.descriptor_counts[index]);
}
vk::DescriptorPoolCreateInfo pool_info{};
pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo;
pool_info.maxSets = 1;
pool_info.poolSizeCount = static_cast<u32>(pool_sizes.size());
pool_info.pPoolSizes = pool_sizes.empty() ? nullptr : pool_sizes.data();
vk::DescriptorPool descriptor_pool = nullptr;
RequireVk(
test.name, "dispatch",
m_device.createDescriptorPool(&pool_info, nullptr, &descriptor_pool),
"vkCreateDescriptorPool");
vk::DescriptorSetAllocateInfo set_info{};
set_info.sType = vk::StructureType::eDescriptorSetAllocateInfo;
set_info.descriptorPool = descriptor_pool;
set_info.descriptorSetCount = 1;
set_info.pSetLayouts = &descriptor_layout;
vk::DescriptorSet descriptor_set = nullptr;
RequireVk(test.name, "dispatch",
m_device.allocateDescriptorSets(&set_info, &descriptor_set),
"vkAllocateDescriptorSets");
std::vector<vk::WriteDescriptorSet> writes;
std::vector<vk::DescriptorBufferInfo> buffer_infos;
std::array<std::vector<vk::DescriptorImageInfo>, ShaderRecompiler::IR::ImageBindingCount> image_infos;
std::vector<vk::ImageView> sampled_mip_views;
std::vector<vk::DescriptorImageInfo> sampler_infos;
Buffer flattened_buffer;
Buffer user_data_buffer;
Buffer shared_buffer;
vk::DescriptorBufferInfo flattened_info{};
vk::DescriptorBufferInfo user_data_info{};
vk::DescriptorBufferInfo shared_info{};
vk::DescriptorBufferInfo gds_info{};
vk::DescriptorBufferInfo bda_pagetable_info{};
vk::DescriptorBufferInfo fault_buffer_info{};
if (Count(Kind::SharedMemory) != 0) {
const auto bytes = uint64_t{test.compute_info.lds_size_dwords} * sizeof(u32) *
test.dispatch_x * test.dispatch_y * test.dispatch_z;
shared_buffer = CreateHostBuffer(test.name, bytes, vk::BufferUsageFlagBits::eStorageBuffer, {});
shared_info = {shared_buffer.buffer, 0, shared_buffer.size};
vk::WriteDescriptorSet write{};
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::SharedMemory);
write.descriptorCount = 1;
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = &shared_info;
writes.push_back(write);
}
const bool uses_bda = Count(Kind::BdaPagetable) != 0;
Require(test.name, "dispatch",
uses_bda == (Count(Kind::FaultBuffer) != 0),
"BDA page table and fault buffer must be bound together");
if (uses_bda) {
EnsureBdaBuffers(test.name);
bda_pagetable_info = {m_bda_pagetable_buffer.buffer, 0,
m_bda_pagetable_buffer.size};
fault_buffer_info = {m_fault_buffer.buffer, 0, m_fault_buffer.size};
const std::array special_buffers{
std::pair{Kind::BdaPagetable, &bda_pagetable_info},
std::pair{Kind::FaultBuffer, &fault_buffer_info},
};
for (const auto &[kind, info] : special_buffers) {
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(kind);
write.descriptorCount = 1;
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = info;
writes.push_back(write);
}
}
if (Count(Kind::Buffers) != 0) {
buffer_infos.resize(Count(Kind::Buffers));
for (u32 resource = 0; resource < compiled.program.info.buffers.size(); ++resource) {
const auto index = compiled.program.info.buffers[resource].descriptor_index;
if (index == UINT32_MAX) continue;
auto &info = buffer_infos[index];
info.buffer = buffer.buffer;
info.offset = 0;
info.range = buffer.size;
if (test.storage_buffer_range_bytes != 0) {
const auto offset = resource < test.storage_buffer_offsets.size()
? test.storage_buffer_offsets[resource]
: 0u;
info.range = static_cast<vk::DeviceSize>(
test.storage_buffer_range_bytes + offset);
Require(test.name, "dispatch", info.range <= buffer.size,
"storage buffer descriptor range exceeds backing buffer");
}
}
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::Buffers);
write.descriptorCount = static_cast<u32>(buffer_infos.size());
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = buffer_infos.data();
writes.push_back(write);
}
if (Count(Kind::FlattenedSrt) != 0) {
flattened_buffer =
CreateStorageBuffer(test.name, compiled.resources.flattened_srt,
compiled.resources.flattened_srt.size());
flattened_info = {flattened_buffer.buffer, 0, flattened_buffer.size};
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::FlattenedSrt);
write.descriptorCount = 1;
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = &flattened_info;
writes.push_back(write);
}
if (Count(Kind::ShaderData) != 0) {
user_data_buffer =
CreateStorageBuffer(test.name, shader_data, shader_data.size());
user_data_info = {user_data_buffer.buffer, 0, user_data_buffer.size};
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::ShaderData);
write.descriptorCount = 1;
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = &user_data_info;
writes.push_back(write);
}
if (Count(Kind::Gds) != 0) {
Require(test.name, "dispatch", gds_buffer != nullptr,
"GDS descriptor requested but no GDS buffer was provided");
gds_info = {gds_buffer->buffer, 0, gds_buffer->size};
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::Gds);
write.descriptorCount = 1;
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = &gds_info;
writes.push_back(write);
}
Require(test.name, "dispatch", sampled_resources.empty() ||
sampled_resources.size() == compiled.program.info.images.size(),
"sampled resources must match the logical images");
for (u32 resource = 0; resource < compiled.program.info.images.size(); ++resource) {
const auto &image = compiled.program.info.images[resource];
const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(image);
auto &infos = image_infos[ShaderRecompiler::IR::ImageBindingIndex(kind)];
infos.resize(Count(kind));
const bool sampled = image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
const auto *source_image = sampled
? (sampled_resources.empty() ? sampled_image : &sampled_resources[resource])
: (image.numeric_class == Prospero::TextureNumericClass::Float
? storage_image : storage_image_uint);
Require(test.name, "dispatch", source_image != nullptr,
"image descriptor requested but no matching image was provided");
for (u32 mip_index = 0; mip_index < image.mip_count; ++mip_index) {
auto &info = infos[image.descriptor_index + mip_index];
info.imageView = source_image->view;
info.imageLayout = source_image->layout;
if (sampled && image.mip_mode == ShaderRecompiler::IR::ImageMipMode::Dynamic) {
const auto mip = test.sampled_image_view_base_mip + mip_index;
Require(test.name, "dispatch", mip < source_image->mip_levels,
"sampled mip descriptor exceeds the supplied image");
vk::ImageViewCreateInfo view{};
view.image = source_image->image;
view.viewType = test.sampled_image_view_type;
view.format = source_image->format;
view.subresourceRange = {
vk::ImageAspectFlagBits::eColor, mip, 1,
test.sampled_image_view_base_layer,
test.sampled_image_view_layers != 0
? test.sampled_image_view_layers
: source_image->layers - test.sampled_image_view_base_layer};
RequireVk(test.name, "dispatch",
m_device.createImageView(&view, nullptr, &info.imageView),
"vkCreateImageView(sampled mip)");
sampled_mip_views.push_back(info.imageView);
}
}
}
for (u32 group = 0; group < image_infos.size(); ++group) {
const auto &infos = image_infos[group];
if (infos.empty()) continue;
const auto kind = static_cast<Kind>(ShaderRecompiler::IR::FirstImageBinding + group);
vk::WriteDescriptorSet write{};
write.dstSet = descriptor_set;
write.dstBinding = Native(kind);
write.descriptorCount = static_cast<u32>(infos.size());
write.descriptorType = NativeDescriptorType(kind);
write.pImageInfo = infos.data();
writes.push_back(write);
}
if (Count(Kind::Samplers) != 0) {
Require(test.name, "dispatch", sampler != nullptr,
"sampler descriptor requested but no sampler was provided");
sampler_infos.resize(Count(Kind::Samplers));
for (auto &info : sampler_infos) {
info.sampler = sampler;
}
vk::WriteDescriptorSet write{};
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::Samplers);
write.descriptorCount = static_cast<u32>(sampler_infos.size());
write.descriptorType = vk::DescriptorType::eSampler;
write.pImageInfo = sampler_infos.data();
writes.push_back(write);
}
if (!writes.empty()) {
m_device.updateDescriptorSets(static_cast<u32>(writes.size()),
writes.data(), 0, nullptr);
}
vk::CommandBuffer cmd = BeginCommands(test.name, "dispatch");
if (uses_bda) {
Require(test.name, "dispatch", buffer.device_address != 0,
"BDA test backing has no device address");
Require(test.name, "dispatch", !test.bda_mappings.empty(),
"BDA test has no explicit guest mapping");
cmd.fillBuffer(m_bda_pagetable_buffer.buffer, 0,
m_bda_pagetable_buffer.size, 0);
cmd.fillBuffer(m_fault_buffer.buffer, 0, m_fault_buffer.size, 0);
std::array<vk::BufferMemoryBarrier, 2> barriers{};
barriers[0].sType = vk::StructureType::eBufferMemoryBarrier;
barriers[0].srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barriers[0].dstAccessMask = vk::AccessFlagBits::eTransferWrite;
barriers[0].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barriers[0].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barriers[0].buffer = m_bda_pagetable_buffer.buffer;
barriers[0].offset = 0;
barriers[0].size = m_bda_pagetable_buffer.size;
barriers[1] = barriers[0];
barriers[1].dstAccessMask = vk::AccessFlagBits::eShaderRead |
vk::AccessFlagBits::eShaderWrite;
barriers[1].buffer = m_fault_buffer.buffer;
barriers[1].size = m_fault_buffer.size;
cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr,
1, barriers.data(), 0, nullptr);
for (const auto &[guest_base, backing] : test.bda_mappings) {
const auto page_offset = guest_base &
(BufferCache::CACHING_PAGESIZE - 1);
Require(test.name, "dispatch",
backing >= page_offset && backing < buffer.size,
"BDA test mapping begins outside its backing buffer");
const auto bytes = page_offset + buffer.size - backing;
const auto pages =
(bytes + BufferCache::CACHING_PAGESIZE - 1) /
BufferCache::CACHING_PAGESIZE;
auto address = buffer.device_address + backing - page_offset;
auto page = BufferCache::PageIndex(guest_base);
for (uint64_t mapped = 0; mapped < pages; mapped++) {
cmd.updateBuffer(m_bda_pagetable_buffer.buffer,
(page + mapped) * sizeof(vk::DeviceAddress),
sizeof(address), &address);
address += BufferCache::CACHING_PAGESIZE;
}
}
barriers[0].dstAccessMask = vk::AccessFlagBits::eShaderRead;
cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eComputeShader, {}, 0,
nullptr, static_cast<u32>(barriers.size()),
barriers.data(), 0, nullptr);
}
cmd.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline);
cmd.bindDescriptorSets(vk::PipelineBindPoint::eCompute, pipeline_layout, 0,
1, &descriptor_set, 0, nullptr);
if (layout.UsesPushData()) {
ShaderRecompiler::IR::PushData push_data;
std::copy(shader_data.begin(), shader_data.end(),
push_data.dwords.begin() + layout.push_data_start_dword);
cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0,
sizeof(push_data), push_data.dwords.data());
}
cmd.dispatch(test.dispatch_x, test.dispatch_y, test.dispatch_z);
if (Count(Kind::Buffers) != 0) {
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask =
vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = buffer.buffer;
barrier.offset = 0;
barrier.size = buffer.size;
cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
&barrier, 0, nullptr);
}
if (gds_buffer != nullptr) {
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask =
vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = gds_buffer->buffer;
barrier.offset = 0;
barrier.size = gds_buffer->size;
cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
&barrier, 0, nullptr);
}
EndSubmitAndFree(test.name, "dispatch", cmd);
for (const auto view : sampled_mip_views) {
m_device.destroyImageView(view, nullptr);
}
if (flattened_buffer.buffer != nullptr) {
DestroyBuffer(&flattened_buffer);
}
if (user_data_buffer.buffer != nullptr) {
DestroyBuffer(&user_data_buffer);
}
DestroyBuffer(&shared_buffer);
m_device.destroyDescriptorPool(descriptor_pool, nullptr);
m_device.destroyPipeline(pipeline, nullptr);
m_device.destroyPipelineLayout(pipeline_layout, nullptr);
m_device.destroyDescriptorSetLayout(descriptor_layout, nullptr);
m_device.destroyShaderModule(module, nullptr);
}
void CheckRasterization(
bool depth_feedback,
Prospero::BufferFormat color_format = Prospero::BufferFormat::k32_32_32_32Float) {
const char *name = depth_feedback ? "DepthAttachmentFeedback" : "PolygonModeRasterization";
uint32_t packed_color = 0;
std::array<float, 4> packed_expected{};
switch (color_format) {
case Prospero::BufferFormat::k11_11_10Float:
name = "PackedFloatVertexColor";
packed_color = 0x801e0380u;
packed_expected = {0.5f, 1.0f, 2.0f, 1.0f};
break;
case Prospero::BufferFormat::k10_10_10_2UScaled:
name = "PackedUscaledVertexColor";
packed_color = 513u | (257u << 10u) | (1023u << 20u) | (3u << 30u);
packed_expected = {513.0f, 257.0f, 1023.0f, 3.0f};
break;
default: break;
}
const bool packed_vertex_color = color_format != Prospero::BufferFormat::k32_32_32_32Float;
const uint32_t extent = depth_feedback ? 8 : 32;
constexpr uintptr_t depth_address = 0x0000000204400000ull;
constexpr uint64_t allocation_size = 0x40000;
constexpr uint64_t rect_address = depth_address + 0x8000;
constexpr uint64_t index_address = rect_address + 0x1000;
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetRenderTargetMask(0xf);
registers.SetShaderMask(0xf);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &cache = context.GetTextureCache();
auto &executor = context.GetRenderExecutor();
int64_t direct_offset = -1;
RenderDepthInfo depth{};
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"depth allocation failed");
void *mapped = reinterpret_cast<void *>(depth_address);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 &&
mapped == reinterpret_cast<void *>(depth_address),
"depth mapping failed");
std::memset(mapped, 0, allocation_size);
const auto initial_depth = [](uint32_t x, uint32_t y) {
return 0.125f + static_cast<float>(x + 2 * y) / 128.0f;
};
for (uint32_t y = 0; y < extent; y++) {
for (uint32_t x = 0; x < extent; x++) {
static_cast<float *>(mapped)[y * 64 + x] = initial_depth(x, y);
}
}
if (!depth_feedback && !packed_vertex_color) {
// Rectangle input contains three position/UV records.
constexpr std::array<float, 24> rect_vertices{
-0.75f, -0.75f, 0, 0, 0, 0, 0, 0,
0.75f, -0.75f, 0, 0, 1, 0, 0, 0,
-0.75f, 0.75f, 0, 0, 0, 1, 0, 0};
std::memcpy(reinterpret_cast<void *>(rect_address), rect_vertices.data(),
sizeof(rect_vertices));
constexpr std::array<uint32_t, 3> indices{0, 1, 2};
std::memcpy(reinterpret_cast<void *>(index_address), indices.data(), sizeof(indices));
}
resources.MapMemory(depth_address, allocation_size);
if (depth_feedback) {
depth.desc.type = BindingType::DepthTarget;
depth.desc.info.data = {depth_address, 256 * extent};
depth.desc.info.pixel_format = vk::Format::eD32Sfloat;
depth.desc.info.guest_format = Prospero::BufferFormat::k32Float;
depth.desc.info.extent = {extent, extent, 1};
depth.desc.info.pitch = 64;
depth.desc.info.bytes_per_block = 4;
depth.desc.info.tile_mode = Prospero::TileMode::kLinear;
depth.desc.info.mip_layout[0] = {0, 256 * extent, 64, extent};
depth.desc.view_info.format = depth.desc.info.pixel_format;
depth.desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
depth.desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
depth.depth_test_enable = depth.depth_write_enable = true;
depth.depth_compare_op = vk::CompareOp::eAlways;
depth.image_id = cache.FindImage(depth.desc);
}
GraphicsCase test;
test.name = name;
test.pixel_depth_export = depth_feedback;
if (depth_feedback) {
const ShaderTextureResource descriptor{{
static_cast<uint32_t>(depth_address >> 8u),
(static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u) |
(((extent - 1u) & 3u) << 30u),
((extent - 1u) >> 2u) | ((extent - 1u) << 14u),
DstSel(4, 4, 4, 4) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u),
0, 0, 0, 0}};
test.has_user_data = true;
std::copy_n(descriptor.fields, 8, test.user_data.begin());
// Each fragment samples its own texel through interpolated screen UVs.
for (uint32_t component = 0; component < 2; component++) {
test.fragment_code.push_back(EncodeVintrp(0, 20 + component, 0, component, 0));
test.fragment_code.push_back(EncodeVintrp(1, 20 + component, 0, component, 1));
}
test.fragment_code.push_back(EncodeMimg0(0x27, 1));
test.fragment_code.push_back(EncodeMimg1(0, 20, 0, 2));
AppendVMovLiteral(&test.fragment_code, 1, 0x3e000000u);
test.fragment_code.push_back(EncodeVop2(0x03, 1, Vgpr(0), 1));
test.fragment_code.push_back(EncodeExp0(0x08, 0x1, false));
test.fragment_code.push_back(EncodeExp1(1, 1, 1, 1));
} else {
test.pixel_interpolator_settings = {0x400u};
for (uint32_t component = 0; component < (packed_vertex_color ? 4u : 1u); component++) {
test.fragment_code.push_back(EncodeVintrp(0x02, component, 0, component, 2));
}
}
test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
test.fragment_code.push_back(packed_vertex_color ? EncodeExp1(0, 1, 2, 3)
: EncodeExp1(0, 0, 0, 0));
AppendEnd(&test.fragment_code);
auto fragment = CompileFragmentCase(test);
const auto vertex_spirv = TestSpv::MakePassthroughVertexSpirv(false);
ShaderProgram vertex_shader{1, CreateShaderModule(name, vertex_spirv)};
ShaderProgram pixel_shader{2, CreateShaderModule(name, fragment.spirv)};
ShaderRecompiler::IR::CompiledShaderInfo vertex_program{};
vertex_program.stage = ShaderType::Vertex;
vertex_program.info.vertex_fetch_components[0] = 2;
vertex_program.info.vertex_fetch_components[1] = packed_vertex_color ? 3 : 4;
ShaderRecompiler::IR::CompiledShaderInfo pixel_program{};
pixel_program.stage = ShaderType::Pixel;
pixel_program.info = fragment.program.info;
pixel_program.bindings = fragment.program.bindings;
ShaderVertexInputInfo vertex{};
ShaderRecompiler::IR::ResourceSnapshot vertex_snapshot;
vertex.stage.program = &vertex_program;
vertex.stage.resources = &vertex_snapshot;
vertex.resources_num = 2;
vertex.buffers_num = 1;
vertex.buffers[0].stride = 6 * sizeof(float);
vertex.resources[1].UpdateAddress48(2 * sizeof(float));
for (uint32_t i = 0; i < 2; i++) {
const auto format = i == 0 ? Prospero::BufferFormat::k32_32Float : color_format;
vertex.resources[i].fields[3] = DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(format) << 12u);
vertex.resources_dst[i].registers_num =
i == 0 ? 2 : vertex_program.info.vertex_fetch_components[1];
}
ShaderPixelInputInfo pixel{};
pixel.stage.program = &pixel_program;
pixel.stage.resources = &fragment.resources;
RenderColorInfo color{};
color.desc.type = BindingType::RenderTarget;
color.desc.info.data = {depth_address + 0x10000, extent * extent * 16};
color.desc.info.mip_layout[0] = {0, color.desc.info.data.size, extent, extent};
color.desc.info.pixel_format = vk::Format::eR32G32B32A32Sfloat;
color.desc.info.guest_format = Prospero::BufferFormat::k32_32_32_32Float;
color.desc.info.extent = {extent, extent, 1};
color.desc.info.pitch = extent;
color.desc.info.bytes_per_block = 16;
color.desc.view_info.format = color.desc.info.pixel_format;
color.desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
color.image_id = cache.FindImage(color.desc);
std::array<float, 18> vertices{
-0.75f, -0.75f, 1, 1, 1, 1, 0.75f, -0.75f, 0.5f, 1, 1, 1,
0.0f, 0.75f, 0.25f, 1, 1, 1};
if (depth_feedback) {
vertices = {-1, -1, 0, 0, 0, 1, 3, -1, 2, 0, 0, 1, -1, 3, 0, 2, 0, 1};
}
std::vector<u32> vertex_words(vertices.size());
std::memcpy(vertex_words.data(), vertices.data(), sizeof(vertices));
if (packed_vertex_color) {
for (uint32_t i = 0; i < 3; i++) {
vertex_words[i * 6 + 2] = packed_color;
}
}
auto buffer = CreateHostBuffer(name, sizeof(vertices), vk::BufferUsageFlagBits::eVertexBuffer,
vertex_words);
const auto pipeline = [&](bool enabled, uint8_t front, uint8_t back,
bool provoking_last = false,
bool clockwise = false,
vk::PrimitiveTopology topology = vk::PrimitiveTopology::eTriangleList)
-> PipelineCache::Pipeline & {
HW::ModeControl mode{};
mode.face = clockwise;
mode.poly_mode = enabled;
mode.polymode_front_ptype = front;
mode.polymode_back_ptype = back;
mode.provoking_vtx_last = provoking_last;
registers.SetModeControl(mode);
return context.GetPipelineCache().GetGraphicsPipeline(
std::span{&color, 1u}, depth, std::span{&vertex, 1u}, scheduler.Current(), &pixel,
topology, false,
PipelineCache::GraphicsPrograms{{vertex_shader}, pixel_shader});
};
auto &filled = pipeline(true, 2, 2);
const auto draw = [&](const PipelineCache::Pipeline &selected, uint32_t vertex_count = 3,
std::array<u32, 4> clear_value = {}) {
RenderExecutorTestAccess::BindRenderTarget(executor, color.image_id);
if (depth.image_id) {
RenderExecutorTestAccess::BindRenderTarget(executor, depth.image_id);
}
auto bindings = RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, vertex.stage, pixel.stage, true);
Require(name, "sampled attachment identity",
!depth_feedback || bindings.pixel->images[0].image_id == depth.image_id,
"the fragment must sample the depth attachment's native image");
auto &command = scheduler.Current();
vk::ImageAspectFlags feedback_aspects;
std::array<PreparedBindings *, 2> stages{
&bindings.vertex[0], &*bindings.pixel};
auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, command, &color, 1, depth, stages, &feedback_aspects);
const bool feedback_enabled = rendering.depth_stencil_attachment.image_layout ==
vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT;
Require(name, "per-draw depth feedback",
feedback_enabled == (depth_feedback && depth.depth_write_enable) &&
feedback_aspects == (feedback_enabled
? vk::ImageAspectFlagBits::eDepth
: vk::ImageAspectFlags{}),
"feedback was not selected only for an overlapping fragment depth read/write");
RenderExecutorTestAccess::CommitBindings(
executor, command, selected, bindings.vertex[0], *bindings.pixel);
rendering.color_attachments[0].is_clear = true;
rendering.color_attachments[0].clear_value = clear_value;
command.BeginRendering(rendering);
auto cmd = command.Handle();
cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, selected.pipeline);
const vk::Viewport viewport{0, 0, static_cast<float>(extent),
static_cast<float>(extent), 0, 1};
const vk::Rect2D scissor{{0, 0}, {extent, extent}};
cmd.setViewportWithCount(1, &viewport);
cmd.setScissorWithCount(1, &scissor);
cmd.setLineWidth(1);
cmd.setDepthTestEnable(depth.depth_test_enable);
cmd.setDepthWriteEnable(depth.depth_write_enable);
cmd.setDepthCompareOp(depth.depth_compare_op);
#if !defined(__APPLE__)
cmd.setDepthBoundsTestEnable(depth.depth_bounds_test_enable);
if (depth.depth_bounds_test_enable) {
cmd.setDepthBounds(registers.GetDepthBoundsMin(), registers.GetDepthBoundsMax());
}
#endif
cmd.setDepthBiasEnable(false);
cmd.setStencilTestEnable(depth.stencil_test_enable);
if (depth.stencil_test_enable) {
const auto set_stencil = [&](vk::StencilFaceFlagBits face,
const vk::StencilOpState &state) {
cmd.setStencilOp(face, state.failOp, state.passOp, state.depthFailOp,
state.compareOp);
cmd.setStencilCompareMask(face, state.compareMask);
cmd.setStencilWriteMask(face, state.writeMask);
cmd.setStencilReference(face, state.reference);
};
set_stencil(vk::StencilFaceFlagBits::eFront, depth.stencil_front);
set_stencil(vk::StencilFaceFlagBits::eBack, depth.stencil_back);
}
const vk::Bool32 write = true;
cmd.setColorWriteEnableEXT(1, &write);
cmd.setAttachmentFeedbackLoopEnableEXT(feedback_aspects);
const vk::DeviceSize offset = 0;
cmd.bindVertexBuffers(0, 1, &buffer.buffer, &offset);
cmd.draw(vertex_count, 1, 0, 0);
command.EndRendering();
RenderExecutorTestAccess::ResetBindings(executor);
};
const auto read_color = [&] {
return ReadCachedTexel(name, context, color.image_id, {}, {extent, extent, 1});
};
draw(filled);
const auto solid_pixels = read_color();
if (packed_vertex_color) {
const auto center = 4 * ((extent / 2) * extent + extent / 2);
for (uint32_t component = 0; component < packed_expected.size(); component++) {
Require(name, "packed vertex fetch and coverage",
std::abs(std::bit_cast<float>(solid_pixels[center + component]) -
packed_expected[component]) < 0.0001f && solid_pixels[component] == 0,
"packed vertex color changed channels, clamped HDR, or filled outside the triangle");
}
} else if (depth_feedback) {
// Consecutive draws in the same submission must see each earlier depth write.
draw(filled);
draw(filled);
depth.depth_write_enable = false;
draw(filled); // Reset feedback in the same command buffer and preserve the depth writes.
const auto repeated_pixels = read_color();
const auto stored_depth = ReadCachedTexel(
name, context, depth.image_id, {}, {extent, extent, 1});
for (uint32_t y = 0; y < extent; y++) {
for (uint32_t x = 0; x < extent; x++) {
const auto index = y * extent + x;
Require(name, "sampled and stored depth",
solid_pixels[index * 4] == std::bit_cast<uint32_t>(initial_depth(x, y)) &&
repeated_pixels[index * 4] ==
std::bit_cast<uint32_t>(initial_depth(x, y) + 0.375f) &&
stored_depth[index] == repeated_pixels[index * 4],
"depth feedback lost a prior write or the read-only draw changed depth");
}
}
depth.depth_test_enable = false;
for (uint32_t pass = 0; pass < 3; pass++) {
depth.depth_bounds_test_enable = pass != 2;
registers.SetDepthBoundsMin(pass == 0 ? 0.5f : 0.625f);
registers.SetDepthBoundsMax(pass == 0 ? 0.5625f : 0.6875f);
Require(name, "dynamic depth bounds pipeline reuse",
pipeline(true, 2, 2).pipeline == filled.pipeline,
"changing depth bounds or their enable state created another pipeline");
draw(filled);
const auto bounded_pixels = read_color();
for (size_t index = 0; index < stored_depth.size(); index++) {
const auto z = std::bit_cast<float>(stored_depth[index]);
const bool passes = !depth.depth_bounds_test_enable ||
(z >= registers.GetDepthBoundsMin() &&
z <= registers.GetDepthBoundsMax());
Require(name, "dynamic stored-depth bounds",
bounded_pixels[index * 4] == (passes ? stored_depth[index] : 0u),
"depth bounds did not filter stored depth independently of the depth test");
}
}
} else {
auto &wireframe = pipeline(true, 1, 1);
Require(name, "pipeline cache", filled.pipeline != wireframe.pipeline &&
pipeline(true, 2, 2).pipeline == filled.pipeline &&
pipeline(false, 1, 0).pipeline == filled.pipeline,
"polygon mode was lost from the cache key or dormant face modes were applied");
draw(wireframe);
const auto line_pixels = read_color();
const auto interior = 4 * (16 * extent + 16);
Require(name, "GPU coverage", solid_pixels[interior] == 0x3f800000u &&
line_pixels[interior] == 0 &&
std::ranges::any_of(line_pixels, [](u32 value) { return value == 0x3f800000u; }),
"wireframe did not preserve triangle edges while leaving its interior empty");
auto &last_vertex = pipeline(true, 2, 2, true);
Require(name, "provoking vertex pipeline cache",
last_vertex.pipeline != filled.pipeline &&
pipeline(true, 2, 2, true).pipeline == last_vertex.pipeline &&
pipeline(true, 2, 2).pipeline == filled.pipeline,
"first and last provoking vertices did not keep distinct cached pipelines");
draw(last_vertex);
const auto last_pixels = read_color();
for (size_t i = 0; i < solid_pixels.size(); i++) {
Require(name, "flat provoking vertex output",
last_pixels[i] == (solid_pixels[i] == 0 ? 0 : 0x3e800000u),
"last-vertex flat shading changed coverage or did not use vertex two");
}
const auto blend_pipeline = [&](bool enabled, bool bypass,
bool alternate = false) -> PipelineCache::Pipeline & {
auto blend = registers.GetBlendControl(0);
blend.enable = enabled;
blend.color_srcblend = static_cast<uint8_t>(alternate ? Prospero::BlendFactor::kOne
: Prospero::BlendFactor::kZero);
blend.alpha_srcblend = blend.color_srcblend;
blend.separate_alpha_blend = alternate;
registers.SetBlendControl(0, blend);
auto target = registers.GetRenderTarget(0).info;
target.blend_bypass = bypass;
registers.SetColorInfo(0, target);
return pipeline(true, 2, 2);
};
auto &disabled_blend = blend_pipeline(false, false);
Require(name, "effective blend pipeline cache",
blend_pipeline(false, true).pipeline == disabled_blend.pipeline &&
blend_pipeline(true, true).pipeline == disabled_blend.pipeline &&
blend_pipeline(false, false, true).pipeline == disabled_blend.pipeline &&
blend_pipeline(true, true, true).pipeline == disabled_blend.pipeline,
"disabled and bypassed blending did not share the same host pipeline");
auto &enabled_blend = blend_pipeline(true, false);
Require(name, "enabled blend pipeline cache",
enabled_blend.pipeline != disabled_blend.pipeline &&
blend_pipeline(true, false).pipeline == enabled_blend.pipeline,
"active blending was lost from the pipeline key or missed its cached pipeline");
auto dormant_alpha = registers.GetBlendControl(0);
dormant_alpha.alpha_srcblend = static_cast<uint8_t>(Prospero::BlendFactor::kOne);
registers.SetBlendControl(0, dormant_alpha);
Require(name, "dormant alpha blend pipeline cache",
pipeline(true, 2, 2).pipeline == enabled_blend.pipeline,
"inactive separate-alpha state recompiled the pipeline");
draw(enabled_blend);
const auto blended_pixels = read_color();
Require(name, "active blend output",
std::ranges::all_of(blended_pixels, [](u32 value) { return value == 0; }),
"zero-factor blending did not clear the fragment color");
draw(blend_pipeline(true, true));
Require(name, "bypassed blend output", read_color() == solid_pixels,
"blend bypass did not restore the unblended fragment color and coverage");
HW::DepthRenderTarget stencil_target{};
stencil_target.z_info.format = Prospero::DepthFormat::kZ32F;
stencil_target.stencil_info.format = Prospero::StencilFormat::k8UInt;
stencil_target.stencil_info.htile_stencil_disabled = true;
stencil_target.z_read_base_addr = stencil_target.z_write_base_addr = depth_address;
stencil_target.stencil_read_base_addr = stencil_target.stencil_write_base_addr =
depth_address + 0x20000;
stencil_target.size = {static_cast<uint16_t>(extent - 1),
static_cast<uint16_t>(extent - 1), true};
registers.SetDepthRenderTarget(stencil_target);
HW::DepthControl stencil_control{};
// Keep the same depth/stencil attachment active when stencil testing is disabled.
stencil_control.z_enable = true;
stencil_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eAlways);
stencil_control.stencil_enable = stencil_control.backface_enable = true;
stencil_control.stencilfunc = stencil_control.stencilfunc_bf =
static_cast<uint8_t>(vk::CompareOp::eAlways);
registers.SetDepthControl(stencil_control);
// PPSA01325 sets bit 7 on front faces and clears it on back faces.
registers.SetStencilControl({3, 3, 3, 4, 4, 4});
registers.SetStencilMask({0x80, 0, 0x80, 1, 0x80, 0, 0x80, 1});
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
auto stencil_readback = CreateHostBuffer(name, extent * extent,
vk::BufferUsageFlagBits::eTransferDst, {});
const auto read_stencil = [&] {
const vk::BufferImageCopy copy{0, 0, 0,
{vk::ImageAspectFlagBits::eStencil, 0, 0, 1}, {}, {extent, extent, 1}};
cache.GetImage(depth.image_id).Download(std::span{&copy, 1},
stencil_readback.buffer, 0, stencil_readback.size);
const vk::MemoryBarrier2 barrier{
.srcStageMask = vk::PipelineStageFlagBits2::eCopy,
.srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
.dstStageMask = vk::PipelineStageFlagBits2::eHost,
.dstAccessMask = vk::AccessFlagBits2::eHostRead};
vk::DependencyInfo dependency{};
dependency.memoryBarrierCount = 1;
dependency.pMemoryBarriers = &barrier;
scheduler.Current().Handle().pipelineBarrier2(dependency);
scheduler.Finish();
return ReadBuffer(name, stencil_readback, extent * extent / 4);
};
const std::array stencil_pipelines{
pipeline(true, 2, 2, false, true).pipeline,
pipeline(true, 2, 2, false, false).pipeline};
const auto check_stencil = [&](const char *label, bool back_face,
uint8_t initial, uint8_t written) {
vk::ClearValue clear{};
clear.depthStencil = vk::ClearDepthStencilValue{0.625f, initial};
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), depth.image_id,
{vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil,
0, 1, 0, 1}, clear);
const auto &selected = pipeline(true, 2, 2, false, !back_face);
Require(name, "dynamic stencil pipeline reuse",
selected.pipeline == stencil_pipelines[back_face],
"stencil enable, operations or references changed the graphics pipeline");
draw(selected);
const auto result = read_stencil();
const auto *stencil_bytes = reinterpret_cast<const uint8_t *>(result.data());
for (uint32_t texel = 0; texel < extent * extent; ++texel) {
const uint8_t expected = solid_pixels[texel * 4] != 0 ? written : initial;
Require(name, label,
stencil_bytes[texel] == expected,
"captured stencil operations changed masked bits or triangle coverage");
}
};
check_stencil("front-face ReplaceTest", false, 0x35, 0xb5);
check_stencil("back-face masked ReplaceOp", true, 0xb5, 0x35);
stencil_control.stencil_enable = false;
registers.SetDepthControl(stencil_control);
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
check_stencil("disabled front stencil", false, 0x35, 0x35);
check_stencil("disabled back stencil", true, 0xb5, 0xb5);
stencil_control.stencil_enable = true;
registers.SetDepthControl(stencil_control);
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
check_stencil("re-enabled front stencil", false, 0x35, 0xb5);
check_stencil("re-enabled back stencil", true, 0xb5, 0x35);
// Frame 6115 uses separate test/op replacement values with Always comparisons.
stencil_control.z_enable = stencil_control.z_write_enable = true;
stencil_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eLessOrEqual);
registers.SetDepthControl(stencil_control);
registers.SetStencilControl({0, 3, 9, 0, 4, 8});
registers.SetStencilMask({8, 0xff, 0xff, 1, 8, 0xff, 0xff, 1});
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
Require(name, "independent stencil references",
depth.stencil_front.reference == 8 &&
depth.stencil_back.reference == 1,
"always-pass faces must preserve their distinct replacement values");
check_stencil("front test-value replacement", false, 0x35, 8);
check_stencil("back operation-value replacement", true, 0x35, 1);
stencil_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eNever);
registers.SetDepthControl(stencil_control);
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
check_stencil("front depth-fail subtract wrap", false, 0, 0xff);
check_stencil("back depth-fail add wrap", true, 0xff, 0);
// A compare on bit 3 and replacement on bit 0 share one Vulkan reference
// without changing either native operation or the untouched stencil bits.
stencil_control.z_enable = stencil_control.z_write_enable = false;
stencil_control.stencilfunc = stencil_control.stencilfunc_bf =
static_cast<uint8_t>(vk::CompareOp::eEqual);
registers.SetDepthControl(stencil_control);
registers.SetStencilControl({0, 4, 0, 0, 4, 0});
registers.SetStencilMask({8, 8, 1, 1, 8, 8, 1, 1});
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
Require(name, "masked compare and replace reference",
depth.stencil_front.reference == 9 &&
depth.stencil_back.reference == 9,
"disjoint comparison and replacement bits must combine into one reference");
check_stencil("masked compare passes", false, 0xb8, 0xb9);
check_stencil("masked compare fails", false, 0xb0, 0xb0);
// SPI_PS_IN_CONTROL must select the native wave width through the actual
// program cache. In wave32 a low-word compare preserves scalar VCC_HI.
static const auto native_vertex = [] {
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0xbf800000u);
AppendVMovLiteral(&code, 2, 0x40400000u);
code.push_back(EncodeVopc(0xc2, InlineU32(1), 5));
code.push_back(EncodeVop2(0x01, 3, Vgpr(1), 2));
code.push_back(EncodeVopc(0xc2, InlineU32(2), 5));
code.push_back(EncodeVop2(0x01, 4, Vgpr(1), 2));
AppendVMovU32(&code, 0, 0);
AppendVMovLiteral(&code, 6, 0x3f800000u);
code.push_back(EncodeExp0(0x0c, 0xf));
code.push_back(EncodeExp1(3, 4, 0, 6));
for (u32 parameter = 0; parameter < 8; parameter++) {
code.push_back(EncodeExp0(0x20 + parameter, 0xf));
code.push_back(EncodeExp1(0, 0, 0, 0));
}
AppendEnd(&code);
return code;
}();
static const auto native_pixel = [] {
std::vector<u32> code;
AppendVMovU32(&code, 0, 0);
AppendSMovLiteral(&code, 107, 0x3e800000u);
code.push_back(EncodeVopc(0xc2, InlineU32(1), 0));
code.push_back(EncodeVop1(0x01, 1, 107));
AppendVMovLiteral(&code, 2, 0x3f800000u);
code.push_back(EncodeExp0(0x00, 0xf));
code.push_back(EncodeExp1(1, 1, 1, 2));
AppendEnd(&code);
return code;
}();
static ShaderUserData native_user_data{};
const auto vertex_address = reinterpret_cast<uint64_t>(native_vertex.data());
const auto pixel_address = reinterpret_cast<uint64_t>(native_pixel.data());
ShaderMapUserData(vertex_address,
{.type = Prospero::ShaderBinaryType::kGs,
.user_data = &native_user_data,
.code_size_bytes = static_cast<uint32_t>(native_vertex.size() * sizeof(u32))});
ShaderMapUserData(pixel_address,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<uint32_t>(native_pixel.size() * sizeof(u32))});
HW::VertexShaderInfo native_vertex_regs{};
native_vertex_regs.es_regs.data_addr = vertex_address;
HW::PixelShaderInfo native_pixel_regs{};
native_pixel_regs.ps_regs.data_addr = pixel_address;
const auto owned_vertex_shader = vertex_shader;
const auto owned_pixel_shader = pixel_shader;
depth.stencil_test_enable = false;
registers.SetViewportTransformControl(0x300);
user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriList);
std::array<Prospero::ColorComponentMapping, 8> export_mapping{};
std::array<ShaderVertexInputInfo, 3> native_vertex_info{};
std::array<uint64_t, 2> wave_ids{};
std::array<std::vector<u32>, 2> wave_keys;
struct PixelWaveCase { uint32_t control, width, color; };
for (const auto wave : {PixelWaveCase{0x0008, 64, 0},
PixelWaveCase{0x8008, 32, 0x3e800000},
PixelWaveCase{0x0008, 64, 0}}) {
registers.SetPsInControl(wave.control);
const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
registers, user_config, export_mapping, true, native_vertex_info, pixel);
Require(name, "pixel wave metadata propagation",
pixel.input_num == 8 && pixel.stage.program->wave_size == wave.width,
"SPI_PS_IN_CONTROL width did not reach the compiled pixel program");
const auto index = static_cast<uint32_t>(wave.width == 32);
std::vector<u32> key;
BuildStageStaticKey(pixel, key);
if (wave_ids[index] != 0) {
Require(name, "pixel wave cache reuse",
programs.pixel.id == wave_ids[index] && key == wave_keys[index],
"restoring wave64 did not reuse its original program");
}
wave_ids[index] = programs.pixel.id;
wave_keys[index] = std::move(key);
vertex_shader = programs.vertex[0];
pixel_shader = programs.pixel;
vertex = native_vertex_info[0];
draw(pipeline(true, 2, 2));
const auto wave_pixels = read_color();
for (size_t component = 0; component < wave_pixels.size(); component++) {
const auto expected = component % 4 == 3 ? 0x3f800000u : wave.color;
Require(name, "pixel wave VCC_HI readback", wave_pixels[component] == expected,
"pixel comparison clobbered VCC_HI in wave32 or retained it in wave64");
}
}
Require(name, "pixel wave cache distinction",
wave_ids[0] != wave_ids[1] && wave_keys[0] != wave_keys[1],
"wave32 and wave64 pixel programs shared a cache key");
// Reversed RGBA targets must use logical Sa for RGB, with a separate
// zero or unit source factor for separate alpha attenuation or accumulation.
static const auto blend_pixel = [] {
std::vector<u32> code;
constexpr std::array<float, 4> source{0.8f, 0.6f, 0.4f, 0.5f};
for (u32 component = 0; component < source.size(); component++) {
AppendVMovLiteral(&code, component, std::bit_cast<u32>(source[component]));
}
code.push_back(EncodeExp0(0, 0xf));
code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&code);
return code;
}();
native_pixel_regs.ps_regs.data_addr = reinterpret_cast<uint64_t>(blend_pixel.data());
ShaderMapUserData(native_pixel_regs.ps_regs.data_addr,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<uint32_t>(blend_pixel.size() * sizeof(u32))});
const auto saved_blend = registers.GetBlendControl(0);
const auto saved_target_info = registers.GetRenderTarget(0).info;
const auto saved_output_mode = registers.GetShaderRegisters().target_output_mode[0];
registers.SetTargetOutputMode(0, 4);
auto blend_target_info = saved_target_info;
blend_target_info.blend_bypass = false;
registers.SetColorInfo(0, blend_target_info);
export_mapping[0] = color.export_mapping = Prospero::ColorMappingAbgr;
HW::BlendControl alpha_blend{};
alpha_blend.enable = true;
alpha_blend.color_srcblend = static_cast<uint8_t>(Prospero::BlendFactor::kSrcAlpha);
alpha_blend.alpha_srcblend = static_cast<uint8_t>(Prospero::BlendFactor::kOne);
alpha_blend.color_destblend = alpha_blend.alpha_destblend =
static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusSrcAlpha);
struct AlphaBlendCase {
bool separate;
Prospero::BlendFactor alpha_source;
ShaderAlphaBlendSource mode;
float alpha;
};
uint64_t broadcast_program = 0;
vk::Pipeline broadcast_pipeline{};
for (const auto test : {
AlphaBlendCase{false, Prospero::BlendFactor::kOne, ShaderAlphaBlendSource::SourceAlpha, 0.4f},
AlphaBlendCase{true, Prospero::BlendFactor::kSrcAlpha, ShaderAlphaBlendSource::SourceAlpha, 0.4f},
AlphaBlendCase{true, Prospero::BlendFactor::kOne, ShaderAlphaBlendSource::SourceAlphaOne, 0.65f},
AlphaBlendCase{true, Prospero::BlendFactor::kZero, ShaderAlphaBlendSource::SourceAlphaZero, 0.15f}}) {
alpha_blend.separate_alpha_blend = test.separate;
alpha_blend.alpha_srcblend = static_cast<uint8_t>(test.alpha_source);
registers.SetBlendControl(0, alpha_blend);
const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
registers, user_config, export_mapping, true, native_vertex_info, pixel);
Require(name, "logical alpha blend program", pixel.alpha_blend_source == test.mode &&
pixel.dual_source_blending,
"the separate alpha equation did not reach the compiled pixel program");
vertex_shader = programs.vertex[0];
pixel_shader = programs.pixel;
vertex = native_vertex_info[0];
auto &selected = pipeline(true, 2, 2);
if (test.mode == ShaderAlphaBlendSource::SourceAlpha) {
Require(name, "equal blend equation cache reuse",
broadcast_program == 0 || (programs.pixel.id == broadcast_program &&
selected.pipeline == broadcast_pipeline),
"equivalent separate alpha state created another shader or pipeline");
broadcast_program = programs.pixel.id;
broadcast_pipeline = selected.pipeline;
}
constexpr std::array<float, 4> destination{0.3f, 0.3f, 0.1f, 0.2f};
draw(selected, 3, std::bit_cast<std::array<u32, 4>>(destination));
const auto blend_pixels = read_color();
const std::array<float, 4> expected{test.alpha, 0.35f, 0.35f, 0.5f};
for (size_t component = 0; component < blend_pixels.size(); component++) {
Require(name, "logical alpha blend readback",
std::abs(std::bit_cast<float>(blend_pixels[component]) -
expected[component % 4]) < 0.00001f,
"reversed blending changed logical RGB or the separate alpha equation");
}
}
native_pixel_regs.ps_regs.data_addr = pixel_address;
export_mapping[0] = color.export_mapping = {};
registers.SetBlendControl(0, saved_blend);
registers.SetColorInfo(0, saved_target_info);
registers.SetTargetOutputMode(0, saved_output_mode);
// Captured Playroom strips contain the fullscreen triangle followed by an
// out-of-bounds fetch exporting (0,0,0,0). Additive color reveals any extra triangle.
static constexpr std::array<std::array<u32, 4>, 4> fourth_positions{{
{0, 0, 0, 0}, {0x80000000u, 0, 0x80000000u, 0x80000000u},
{0, 0, 0, 0x3f800000u}, {0xbf800000u, 0xbf800000u, 0, 0}}};
static const auto position_shaders = [&] {
std::array<std::vector<u32>, fourth_positions.size()> result;
for (size_t i = 0; i < result.size(); i++) {
auto &code = result[i];
const auto position_export = std::ranges::find(native_vertex, EncodeExp0(0x0c, 0xf));
code.assign(native_vertex.begin(), position_export);
code.push_back(EncodeVopc(0xc2, InlineU32(3), 5));
constexpr std::array<u32, 4> position_registers{3, 4, 0, 6};
for (size_t component = 0; component < position_registers.size(); component++) {
AppendVMovLiteral(&code, 7, fourth_positions[i][component]);
const auto reg = position_registers[component];
code.push_back(EncodeVop2(0x01, reg, Vgpr(reg), 7));
}
code.insert(code.end(), position_export, native_vertex.end());
}
return result;
}();
auto additive_blend = registers.GetBlendControl(0);
additive_blend.enable = true;
additive_blend.color_srcblend = additive_blend.color_destblend =
additive_blend.alpha_srcblend = additive_blend.alpha_destblend =
static_cast<uint8_t>(Prospero::BlendFactor::kOne);
registers.SetBlendControl(0, additive_blend);
auto target_info = registers.GetRenderTarget(0).info;
target_info.blend_bypass = false;
registers.SetColorInfo(0, target_info);
registers.SetPsInControl(0x8008);
user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriStrip);
for (size_t i = 0; i < position_shaders.size(); i++) {
const auto &code = position_shaders[i];
native_vertex_regs.es_regs.data_addr = reinterpret_cast<uint64_t>(code.data());
ShaderMapUserData(native_vertex_regs.es_regs.data_addr,
{.type = Prospero::ShaderBinaryType::kGs,
.user_data = &native_user_data,
.code_size_bytes = static_cast<uint32_t>(code.size() * sizeof(u32))});
const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
registers, user_config, export_mapping, true, native_vertex_info, pixel);
vertex_shader = programs.vertex[0];
pixel_shader = programs.pixel;
vertex = native_vertex_info[0];
draw(pipeline(true, 2, 2, false, false, vk::PrimitiveTopology::eTriangleStrip), 4);
const auto pixels = read_color();
bool second_triangle = false;
for (size_t component = 0; component < pixels.size(); component += 4) {
Require(name, "fullscreen triangle preserved",
pixels[component] == 0x3e800000u || pixels[component] == 0x3f000000u,
"clipping-error culling removed valid fullscreen coverage");
second_triangle |= pixels[component] == 0x3f000000u;
}
Require(name, "zero homogeneous position culling", second_triangle == (i >= 2),
"zero positions drew an extra triangle, or a valid position was culled");
}
// A NULL-export stencil pass must ignore smaller stale color targets,
// materialize the whole pending HTile clear, and still execute pixel discard.
static const auto null_pixels = [] {
std::array<std::vector<u32>, 2> result;
for (uint32_t discard = 0; discard < result.size(); discard++) {
auto &code = result[discard];
if (discard != 0) {
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
}
code.push_back(EncodeExp0(0x09, 0, true, false, true));
code.push_back(EncodeExp1(0, 0, 0, 0));
AppendEnd(&code);
}
return result;
}();
registers.Reset();
registers.SetViewportTransformControl(0x300);
registers.SetViewportScaleOffset(0, extent / 2, extent / 2,
extent / 2, extent / 2, 1, 0);
registers.SetViewportZMax(0, 1);
registers.SetScreenScissor(0, 0, extent, extent);
registers.SetWindowScissor(0, 0, extent, extent, false);
registers.SetGenericScissor(0, 0, extent, extent, false);
registers.SetViewportScissor(0, 0, 0, extent, extent, false);
registers.SetRenderTargetMask(0xf00f);
registers.SetShaderMask(0xf00f);
registers.SetPsInControl(0x8000);
registers.SetDepthShaderControl({.shader_kill_enable = true});
for (const auto slot : {0u, 3u}) {
registers.SetColorBase(slot,
{.addr = depth_address + (slot == 0 ? 0x38000 : 0x10000)});
registers.SetColorInfo(slot,
{.format = Prospero::ChannelLayout::k32_32_32_32,
.channel_type = Prospero::ChannelType::kFloat,
.channel_order = Prospero::ChannelOrder::kStandard});
const auto side = slot == 0 ? extent / 4 : extent;
registers.SetColorAttrib2(slot, {.height = side - 1, .width = side - 1});
registers.SetColorAttrib3(slot,
{.tile_mode = Prospero::TileMode::kLinear, .dimension = 1});
}
registers.SetTargetOutputMode(0, 4);
registers.SetTargetOutputMode(1, 4);
stencil_target.z_info.htile_acceleration = true;
stencil_target.htile_data_base_addr = depth_address + 0x30000;
registers.SetDepthRenderTarget(stencil_target);
registers.SetDepthClearValue(1);
stencil_control = {};
stencil_control.stencil_enable = true;
stencil_control.stencilfunc = static_cast<uint8_t>(vk::CompareOp::eAlways);
registers.SetDepthControl(stencil_control);
registers.SetStencilControl({3, 3, 3, 3, 3, 3});
registers.SetStencilMask({0x80, 0, 0x80, 1, 0x80, 0, 0x80, 1});
user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriList);
shaders.SetEsShaderBase(vertex_address);
for (uint32_t discard = 0; discard < null_pixels.size(); discard++) {
const auto &code = null_pixels[discard];
const auto address = reinterpret_cast<uint64_t>(code.data());
ShaderMapUserData(address,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<uint32_t>(code.size() * sizeof(u32))});
shaders.SetPsShaderBase(address);
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
(void)cache.FindDepthTarget(depth.image_id, depth.desc);
vk::ClearValue zero{};
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), depth.image_id,
{vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil,
0, 1, 0, 1}, zero);
Require(name, "pending stencil-pass HTile clear",
cache.ClearMeta(stencil_target.htile_data_base_addr),
"the stencil pass did not retain its HTile metadata");
RenderExecutorTestAccess::DrawAuto(
executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
const auto depth_pixels = ReadCachedTexel(name, context, depth.image_id,
{}, {extent, extent, 1});
Require(name, "NULL-export full depth clear",
std::ranges::all_of(depth_pixels, [](u32 v) { return v == 0x3f800000u; }),
"an unexported color target clipped the pending depth clear");
const auto stencil = read_stencil();
const auto expected = discard == 0 ? 0x80808080u : 0u;
Require(name, "NULL-export stencil coverage and discard",
std::ranges::all_of(stencil, [=](u32 v) { return v == expected; }),
"a stale color target clipped stencil coverage or the NULL-export shader was skipped");
}
// A depth-only draw retains the previous PS address and write masks, but
// SPI_SHADER_COL_FORMAT disables its exports. Compiling this stale PS
// with the new zero-input metadata would fail before the depth draw.
static const auto stale_pixel = [] {
std::vector<u32> code{EncodeVintrp(0, 0, 0, 0, 0),
EncodeVintrp(1, 0, 0, 0, 1),
EncodeExp0(0, 0xf), EncodeExp1(0, 0, 0, 0)};
AppendEnd(&code);
return code;
}();
const auto stale_address = reinterpret_cast<uint64_t>(stale_pixel.data());
ShaderMapUserData(stale_address,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<u32>(stale_pixel.size() * sizeof(u32))});
shaders.SetPsShaderBase(stale_address);
registers.SetPsInControl(0);
registers.SetDepthShaderControl({});
registers.SetDepthControl({.z_enable = true, .z_write_enable = true,
.zfunc = static_cast<uint8_t>(vk::CompareOp::eAlways)});
for (const auto slot : {0u, 1u}) {
registers.SetTargetOutputMode(slot, 0);
}
RenderExecutorTestAccess::DrawAuto(
executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
const auto depth_only_pixels = ReadCachedTexel(
name, context, depth.image_id, {}, {extent, extent, 1});
Require(name, "disabled stale PS preserves fixed-function depth",
std::ranges::all_of(depth_only_pixels, [](u32 v) { return v == 0; }),
"disabled color exports invoked the stale PS or clipped fixed-function depth");
registers.SetPsInControl(0x8000);
registers.SetTargetOutputMode(0, 4);
// Export ordinals follow CB_SHADER_MASK. A disabled CB_TARGET_MASK slot
// still consumes an ordinal, and stale MRT0 must not clip physical MRT3.
static const auto sparse_pixels_code = [] {
std::array<std::vector<u32>, 2> result{native_pixel, native_pixel};
auto &code = result[1];
const auto exp = std::ranges::find(code, EncodeExp0(0, 0xf));
*exp = EncodeExp0(0, 0xf, false);
*(exp + 1) = EncodeExp1(0, 0, 0, 0);
code.insert(exp + 2, {EncodeExp0(1, 0xf), EncodeExp1(1, 1, 1, 2)});
return result;
}();
registers.SetRenderTargetMask(0xf000);
registers.SetDepthControl({});
registers.SetDepthShaderControl({});
RenderColorInfo sparse_color{};
for (uint32_t index = 0; index < sparse_pixels_code.size(); index++) {
const auto &code = sparse_pixels_code[index];
const auto address = reinterpret_cast<uint64_t>(code.data());
ShaderMapUserData(address,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<uint32_t>(code.size() * sizeof(u32))});
shaders.SetPsShaderBase(address);
registers.SetShaderMask(index == 0 ? 0xf000 : 0xf00f);
registers.SetTargetOutputMode(1, index == 0 ? 0 : 4);
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), sparse_color, 3);
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), sparse_color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
RenderExecutorTestAccess::DrawAuto(
executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
const auto pixels = ReadCachedTexel(name, context, sparse_color.image_id,
{}, {extent, extent, 1});
for (size_t component = 0; component < pixels.size(); component++) {
const auto expected = component % 4 == 3 ? 0x3f800000u : 0x3e800000u;
Require(name, "compact export to sparse MRT3", pixels[component] == expected,
"shader output routing used the write mask or clipped MRT3 with stale MRT0");
}
}
static const auto snorm_pixel = [] {
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xc0008000u);
AppendVMovLiteral(&code, 1, 0x7fff4000u);
code.insert(code.end(), {EncodeExp0(0, 0xf, true, true), EncodeExp1(0, 1, 0, 0)});
AppendEnd(&code);
return code;
}();
const auto snorm_address = reinterpret_cast<uint64_t>(snorm_pixel.data());
ShaderMapUserData(snorm_address,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<uint32_t>(snorm_pixel.size() * sizeof(u32))});
shaders.SetPsShaderBase(snorm_address);
registers.SetShaderMask(0xf000);
registers.SetTargetOutputMode(0, 6);
registers.SetTargetOutputMode(1, 0);
RenderExecutorTestAccess::DrawAuto(
executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
const auto snorm_pixels = ReadCachedTexel(name, context, sparse_color.image_id,
{}, {extent, extent, 1});
constexpr std::array<float, 4> snorm_expected{-1, -16384.0f / 32767,
16384.0f / 32767, 1};
for (size_t component = 0; component < snorm_pixels.size(); component++) {
Require(name, "compressed SNORM16 color export",
std::abs(std::bit_cast<float>(snorm_pixels[component]) -
snorm_expected[component % 4]) < 0.00001f,
"SNORM16 export lost its sign, normalization or negative endpoint clamp");
}
registers.SetTargetOutputMode(0, 4);
// Indirect arguments also reach s8/s9. Native VS prologs must add them
// exactly once, for both indexed and automatically generated indices.
static const auto indirect_vertex = [&] {
auto code = native_vertex;
for (auto &word : code) {
if (word == EncodeVopc(0xc2, InlineU32(1), 5))
word = EncodeVopc(0xc2, InlineU32(8), 5);
else if (word == EncodeVopc(0xc2, InlineU32(2), 5))
word = EncodeVopc(0xc2, InlineU32(9), 5);
}
code.insert(code.begin(), {EncodeVop2(0x25, 5, 8, 5),
EncodeVop2(0x25, 8, 9, 8)});
const auto position = std::ranges::find(code, EncodeExp0(0x0c, 0xf));
code.insert(position, {EncodeVopc(0xc2, InlineU32(5), 8),
EncodeVop2(0x01, 3, Vgpr(1), 3),
EncodeVop2(0x01, 4, Vgpr(1), 4)});
return code;
}();
const auto indirect_vs = reinterpret_cast<uint64_t>(indirect_vertex.data());
ShaderMapUserData(indirect_vs,
{.type = Prospero::ShaderBinaryType::kGs, .user_data = &native_user_data,
.code_size_bytes = static_cast<uint32_t>(indirect_vertex.size() * sizeof(u32))});
shaders.SetEsShaderBase(indirect_vs);
shaders.SetPsShaderBase(pixel_address);
shaders.SetGsShaderResource2({.user_sgpr = 2});
shaders.SetGsUserSgpr(0, 7, HW::UserSgprType::Unknown);
shaders.SetGsUserSgpr(1, 5, HW::UserSgprType::Unknown);
user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriList);
for (const bool indexed : {false, true}) {
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), sparse_color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
if (indexed) {
RenderExecutorTestAccess::DrawIndex(executor, scheduler.Current(),
{.index_count = 3, .index_addr = reinterpret_cast<const void *>(index_address),
.instance_count = 1, .index_type_and_size = 1, .base_vertex = 7, .first_instance = 5,
.offset_source = DrawOffsetSource::IndirectArgs});
} else {
RenderExecutorTestAccess::DrawAuto(executor, scheduler.Current(),
{.vertex_count = 3, .instance_count = 1, .first_vertex = 7, .first_instance = 5,
.offset_source = DrawOffsetSource::IndirectArgs});
}
const auto pixels = ReadCachedTexel(name, context, sparse_color.image_id,
{}, {extent, extent, 1});
for (size_t component = 0; component < pixels.size(); component++) {
Require("NativeIndirectDrawOffsets", indexed ? "indexed" : "auto",
pixels[component] == (component % 4 == 3 ? 0x3f800000u : 0x3e800000u),
"native vertex or instance offset was applied twice");
}
}
// A fourth VS invocation reads beyond the descriptor instead of reconstructing the corner.
const ShaderBufferResource rect_buffer{{
static_cast<u32>(rect_address),
static_cast<u32>(rect_address >> 32u) | (32u << 16u), 3,
DstSel(4, 5, 6, 7) |
(static_cast<u32>(Prospero::BufferFormat::k32Float) << 12u)}};
static const auto rect_vertex_code = [] {
std::vector<u32> code{
EncodeMubuf0(0x0d, 0, true, false), EncodeMubuf1(0, 2, 5),
EncodeMubuf0(0x0d, 16, true, false), EncodeMubuf1(2, 2, 5)};
AppendVMovU32(&code, 4, 0);
AppendVMovLiteral(&code, 6, 0x3f800000u);
code.insert(code.end(), {EncodeExp0(0x0c, 0xf), EncodeExp1(0, 1, 4, 6),
EncodeExp0(0x20, 0xf), EncodeExp1(2, 3, 4, 6)});
AppendEnd(&code);
return code;
}();
static const auto rect_pixel_code = [] {
std::vector<u32> code;
for (u32 component = 0; component < 2; component++) {
code.push_back(EncodeVintrp(0, 20 + component, 0, component, 0));
code.push_back(EncodeVintrp(1, 20 + component, 0, component, 1));
}
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 23, 0x3f800000u);
code.insert(code.end(), {EncodeExp0(0, 0xf), EncodeExp1(20, 21, 22, 23)});
AppendEnd(&code);
return code;
}();
const auto rect_vs = reinterpret_cast<uint64_t>(rect_vertex_code.data());
const auto rect_ps = reinterpret_cast<uint64_t>(rect_pixel_code.data());
ShaderMapUserData(rect_vs,
{.type = Prospero::ShaderBinaryType::kGs, .user_data = &native_user_data,
.code_size_bytes = static_cast<u32>(rect_vertex_code.size() * sizeof(u32))});
ShaderMapUserData(rect_ps,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<u32>(rect_pixel_code.size() * sizeof(u32))});
shaders.SetEsShaderBase(rect_vs);
shaders.SetPsShaderBase(rect_ps);
shaders.SetGsShaderResource2({.user_sgpr = 4});
for (u32 i = 0; i < 4; i++) {
shaders.SetGsUserSgpr(i, rect_buffer.fields[i], HW::UserSgprType::Vsharp);
}
registers.SetPsInControl(0x8001);
registers.SetPsInputEna(2);
registers.SetPsInputAddr(2);
registers.SetPsInputSettings(0, 0);
for (const auto primitive : {Prospero::PrimitiveType::kRectList,
Prospero::PrimitiveType::kRectListLegacy}) {
const char *rect_name = primitive == Prospero::PrimitiveType::kRectList
? "RectListThreeRecords" : "LegacyRectListThreeRecords";
user_config.SetPrimitiveType(primitive);
registers.SetModeControl({});
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), sparse_color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
RenderExecutorTestAccess::DrawAuto(
executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
const auto pixels = ReadCachedTexel(name, context, sparse_color.image_id,
{}, {extent, extent, 1});
for (u32 y = 0; y < extent; y++) {
for (u32 x = 0; x < extent; x++) {
const bool inside = x >= 4 && x < 28 && y >= 4 && y < 28;
const auto offset = 4 * (y * extent + x);
const std::array<float, 4> expected{
inside ? (x + 0.5f - 4) / 24 : 0,
inside ? (y + 0.5f - 4) / 24 : 0, 0, inside ? 1.0f : 0};
for (u32 component = 0; component < 4; component++) {
Require(rect_name, "three-vertex rectangle coverage and UV",
std::abs(std::bit_cast<float>(pixels[offset + component]) -
expected[component]) < 0.0001f,
"rectangle reconstruction lost its fourth corner or interpolated UVs");
}
}
}
}
DestroyBuffer(&stencil_readback);
vertex_shader = owned_vertex_shader;
pixel_shader = owned_pixel_shader;
}
resources.UnmapMemory(depth_address, allocation_size);
scheduler.Finish();
DestroyBuffer(&buffer);
m_device.destroyShaderModule(pixel_shader.module);
m_device.destroyShaderModule(vertex_shader.module);
Require(name, "release depth memory",
Libs::LibKernel::Memory::KernelMunmap(depth_address, allocation_size) == 0 &&
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"depth mapping or allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
std::vector<u32> RenderFragment(const GraphicsCase &test,
const CompiledShader &fragment) {
const auto vertex_spirv =
TestSpv::MakePassthroughVertexSpirv(test.layers > 1, test.vertex_clip_w);
ValidateSpirv(test.name, vertex_spirv);
Image target =
CreateImageMips(test.name, 1, 1, vk::Format::eR32G32B32A32Sfloat,
vk::ImageUsageFlagBits::eColorAttachment, {}, 4,
vk::ImageLayout::eGeneral, vk::ImageType::e2D,
test.layers > 1 ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D,
test.layers, 0, 0, test.samples);
Image resolved;
if (test.samples != vk::SampleCountFlagBits::e1) {
resolved = CreateImageMips(
test.name, 1, 1, target.format, vk::ImageUsageFlagBits::eColorAttachment,
{}, 4, vk::ImageLayout::eGeneral, vk::ImageType::e2D,
test.layers > 1 ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D,
test.layers);
}
const std::vector<u32> default_vertices = {
0xbf800000u, 0xbf800000u, 0x3e800000u, 0x3f000000u, 0x3f400000u,
0x3f800000u, 0x40400000u, 0xbf800000u, 0x3e800000u, 0x3f000000u,
0x3f400000u, 0x3f800000u, 0xbf800000u, 0x40400000u, 0x3e800000u,
0x3f000000u, 0x3f400000u, 0x3f800000u,
};
const auto &vertices =
test.vertices.empty() ? default_vertices : test.vertices;
Require(test.name, "graphics", vertices.size() == 18u,
"graphics vertex buffer must contain three pos2/color4 vertices");
auto vertex_buffer =
CreateHostBuffer(test.name, vertices.size() * sizeof(u32),
vk::BufferUsageFlagBits::eVertexBuffer, vertices);
vk::ShaderModule vertex_module = CreateShaderModule(test.name, vertex_spirv);
vk::ShaderModule fragment_module = CreateShaderModule(test.name, fragment.spirv);
const auto &fragment_bind = fragment.program.bindings;
vk::PushConstantRange push_constant_range{};
if (fragment_bind.UsesPushData()) {
Require(test.name, "graphics",
test.push_constants.size() * sizeof(u32) ==
fragment_bind.ShaderDataDwords() * sizeof(u32),
"fragment push constant data size does not match reflection");
push_constant_range.stageFlags = vk::ShaderStageFlagBits::eFragment;
push_constant_range.offset = 0;
push_constant_range.size = ShaderRecompiler::IR::NativePushConstantSize;
}
vk::PipelineLayoutCreateInfo pipeline_layout_info{};
pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
pipeline_layout_info.pushConstantRangeCount =
push_constant_range.size != 0 ? 1u : 0u;
pipeline_layout_info.pPushConstantRanges =
push_constant_range.size != 0 ? &push_constant_range : nullptr;
vk::PipelineLayout pipeline_layout = nullptr;
RequireVk(test.name, "graphics",
m_device.createPipelineLayout(&pipeline_layout_info, nullptr,
&pipeline_layout),
"vkCreatePipelineLayout");
vk::PipelineShaderStageCreateInfo stages[2] = {};
stages[0].sType = vk::StructureType::ePipelineShaderStageCreateInfo;
stages[0].stage = vk::ShaderStageFlagBits::eVertex;
stages[0].module = vertex_module;
stages[0].pName = "main";
stages[1].sType = vk::StructureType::ePipelineShaderStageCreateInfo;
stages[1].stage = vk::ShaderStageFlagBits::eFragment;
stages[1].module = fragment_module;
stages[1].pName = "main";
vk::VertexInputBindingDescription vertex_binding{};
vertex_binding.binding = 0;
vertex_binding.stride = 6u * sizeof(float);
vertex_binding.inputRate = vk::VertexInputRate::eVertex;
vk::VertexInputAttributeDescription attributes[2] = {};
attributes[0].location = 0;
attributes[0].binding = 0;
attributes[0].format = vk::Format::eR32G32Sfloat;
attributes[0].offset = 0;
attributes[1].location = 1;
attributes[1].binding = 0;
attributes[1].format = vk::Format::eR32G32B32A32Sfloat;
attributes[1].offset = 2u * sizeof(float);
vk::PipelineVertexInputStateCreateInfo vertex_input{};
vertex_input.sType = vk::StructureType::ePipelineVertexInputStateCreateInfo;
vertex_input.vertexBindingDescriptionCount = 1;
vertex_input.pVertexBindingDescriptions = &vertex_binding;
vertex_input.vertexAttributeDescriptionCount = 2;
vertex_input.pVertexAttributeDescriptions = attributes;
vk::PipelineInputAssemblyStateCreateInfo input_assembly{};
input_assembly.sType =
vk::StructureType::ePipelineInputAssemblyStateCreateInfo;
input_assembly.topology = vk::PrimitiveTopology::eTriangleList;
vk::Viewport viewport{};
viewport.x = 0.0f;
viewport.y = 0.0f;
viewport.width = 1.0f;
viewport.height = 1.0f;
viewport.minDepth = 0.0f;
viewport.maxDepth = 1.0f;
vk::Rect2D scissor{};
scissor.extent.width = 1;
scissor.extent.height = 1;
vk::PipelineViewportStateCreateInfo viewport_state{};
viewport_state.sType = vk::StructureType::ePipelineViewportStateCreateInfo;
viewport_state.viewportCount = 1;
viewport_state.pViewports = &viewport;
viewport_state.scissorCount = 1;
viewport_state.pScissors = &scissor;
vk::PipelineRasterizationStateCreateInfo raster{};
raster.sType = vk::StructureType::ePipelineRasterizationStateCreateInfo;
raster.polygonMode = vk::PolygonMode::eFill;
raster.cullMode = vk::CullModeFlagBits::eNone;
raster.frontFace = vk::FrontFace::eCounterClockwise;
raster.lineWidth = 1.0f;
vk::PipelineRasterizationProvokingVertexStateCreateInfoEXT provoking_vertex{};
if (test.provoking_vtx_last) {
provoking_vertex.provokingVertexMode = vk::ProvokingVertexModeEXT::eLastVertex;
raster.pNext = &provoking_vertex;
}
vk::PipelineMultisampleStateCreateInfo multisample{};
multisample.sType = vk::StructureType::ePipelineMultisampleStateCreateInfo;
multisample.rasterizationSamples = test.samples;
vk::PipelineColorBlendAttachmentState color_attachment{};
color_attachment.colorWriteMask =
vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG |
vk::ColorComponentFlagBits::eB | vk::ColorComponentFlagBits::eA;
vk::PipelineColorBlendStateCreateInfo color_blend{};
color_blend.sType = vk::StructureType::ePipelineColorBlendStateCreateInfo;
color_blend.attachmentCount = 1;
color_blend.pAttachments = &color_attachment;
vk::GraphicsPipelineCreateInfo pipeline_info{};
const vk::Format color_format = vk::Format::eR32G32B32A32Sfloat;
vk::PipelineRenderingCreateInfo rendering_pipeline{};
rendering_pipeline.sType = vk::StructureType::ePipelineRenderingCreateInfo;
rendering_pipeline.colorAttachmentCount = 1;
rendering_pipeline.pColorAttachmentFormats = &color_format;
pipeline_info.sType = vk::StructureType::eGraphicsPipelineCreateInfo;
pipeline_info.pNext = &rendering_pipeline;
pipeline_info.stageCount = 2;
pipeline_info.pStages = stages;
pipeline_info.pVertexInputState = &vertex_input;
pipeline_info.pInputAssemblyState = &input_assembly;
pipeline_info.pViewportState = &viewport_state;
pipeline_info.pRasterizationState = &raster;
pipeline_info.pMultisampleState = &multisample;
pipeline_info.pColorBlendState = &color_blend;
pipeline_info.layout = pipeline_layout;
vk::Pipeline pipeline = nullptr;
RequireVk(test.name, "graphics",
m_device.createGraphicsPipelines(nullptr, 1, &pipeline_info,
nullptr, &pipeline),
"vkCreateGraphicsPipelines");
vk::CommandBuffer cmd = BeginCommands(test.name, "graphics");
vk::RenderingAttachmentInfo color{};
color.sType = vk::StructureType::eRenderingAttachmentInfo;
color.imageView = target.view;
color.imageLayout = vk::ImageLayout::eGeneral;
color.loadOp = vk::AttachmentLoadOp::eClear;
color.storeOp = vk::AttachmentStoreOp::eStore;
if (resolved.image != nullptr) {
color.resolveMode = vk::ResolveModeFlagBits::eAverage;
color.resolveImageView = resolved.view;
color.resolveImageLayout = vk::ImageLayout::eGeneral;
}
vk::RenderingInfo rendering{};
rendering.sType = vk::StructureType::eRenderingInfo;
rendering.renderArea.extent = {1, 1};
rendering.layerCount = test.layers;
rendering.colorAttachmentCount = 1;
rendering.pColorAttachments = &color;
cmd.beginRendering(rendering);
cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline);
if (push_constant_range.size != 0) {
ShaderRecompiler::IR::PushData push_data;
std::copy(test.push_constants.begin(), test.push_constants.end(),
push_data.dwords.begin() + fragment_bind.push_data_start_dword);
cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eFragment, 0,
sizeof(push_data), push_data.dwords.data());
}
vk::DeviceSize offset = 0;
cmd.bindVertexBuffers(0, 1, &vertex_buffer.buffer, &offset);
cmd.draw(3, test.layers, 0, 0);
cmd.endRendering();
EndSubmitAndFree(test.name, "graphics", cmd);
target.layout = vk::ImageLayout::eGeneral;
auto pixel = ReadImage(test.name, resolved.image != nullptr ? &resolved : &target);
pixel.resize(4 * test.layers);
m_device.destroyPipeline(pipeline, nullptr);
m_device.destroyPipelineLayout(pipeline_layout, nullptr);
m_device.destroyShaderModule(fragment_module, nullptr);
m_device.destroyShaderModule(vertex_module, nullptr);
DestroyBuffer(&vertex_buffer);
DestroyImage(&resolved);
DestroyImage(&target);
return pixel;
}
void CheckGpuTilerCpuParity() {
constexpr const char *name = "GpuTilerCpuParity";
EnsureRuntimeContext();
CommandScheduler scheduler(Renderer(), m_runtime_context);
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
StreamBuffer parameters(m_runtime_context, scheduler, MemoryUsage::Stream,
1u << 20);
TileManager tile_manager(m_runtime_context, scheduler, parameters);
constexpr uint64_t conversion_limit = 128ull << 20;
Require(name, "D16 conversion chunk boundaries",
TileManagerTestAccess::ConversionRows(0, 32ull << 10, 32ull << 10,
4096, 256, conversion_limit,
65535) == 4096 &&
TileManagerTestAccess::ConversionRows(
0, 32ull << 10, 32ull << 10, 4097, 256, conversion_limit,
65535) == 4096 &&
TileManagerTestAccess::ConversionRows(
257, 32ull << 10, 32ull << 10, 4097, 256, conversion_limit,
65535) == 4095,
"D16 conversion planner exceeds descriptor or dispatch limits");
const auto align_dword = [](uint64_t size) {
return (size + sizeof(u32) - 1u) & ~(uint64_t{sizeof(u32) - 1u});
};
const auto to_dwords = [&](const std::vector<uint8_t> &bytes,
uint64_t padded_size) {
std::vector<u32> words(static_cast<size_t>(padded_size / sizeof(u32)), 0);
std::memcpy(words.data(), bytes.data(), bytes.size());
return words;
};
const auto read_bytes = [&](const Buffer &buffer, uint64_t size) {
const auto words = ReadBuffer(
name, buffer, static_cast<size_t>(align_dword(size) / sizeof(u32)));
std::vector<uint8_t> bytes(static_cast<size_t>(size));
std::memcpy(bytes.data(), words.data(), bytes.size());
return bytes;
};
const auto host_barrier = [&](vk::Buffer buffer, uint64_t size,
vk::PipelineStageFlags source_stage,
vk::AccessFlags source_access) {
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = source_access;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = buffer;
barrier.offset = 0;
barrier.size = size;
scheduler.Current().Handle().pipelineBarrier(
source_stage, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
&barrier, 0, nullptr);
};
auto scratch_input = CreateHostBuffer(name, 32, AllFlags,
{0x00020001u, 0x00040003u, 0x00060005u, 0x00080007u,
0x000a0009u, 0x000c000bu, 0x000e000du, 0x0010000fu});
auto scratch_output = CreateHostBuffer(name, 56, AllFlags, {});
const auto copy_scratch = [&](TileManager::Result result, uint64_t offset) {
const vk::BufferCopy copy{result.offset, offset, result.size};
scheduler.Current().Handle().copyBuffer(result.buffer, scratch_output.buffer, 1, &copy);
};
// Keep all four conversions in flight to check growth, overwrite ordering,
// the requested span on reuse, and two simultaneously live workspaces.
const auto first = tile_manager.TransformColor(
{scratch_input.buffer, 0, 8}, ColorTransform::SwapBgra16, true);
copy_scratch(first, 0);
const auto grown = tile_manager.TransformColor(
{scratch_input.buffer, 0, 32}, ColorTransform::SwapBgra16, true);
copy_scratch(grown, 8);
const auto reused = tile_manager.TransformColor(
{scratch_input.buffer, 8, 8}, ColorTransform::SwapBgra16, true);
copy_scratch(reused, 40);
const auto paired = tile_manager.TransformColor(reused, ColorTransform::SwapBgra16, true);
copy_scratch(paired, 48);
Require(name, "scratch workspace reuse",
first.buffer != grown.buffer && grown.buffer == reused.buffer &&
reused.buffer != paired.buffer && reused.size == 8 && paired.size == 8,
"scratch growth, input exclusion, or requested span was lost");
host_barrier(scratch_output.buffer, scratch_output.size,
vk::PipelineStageFlagBits::eTransfer, vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "scratch workspace GPU contents",
ReadBuffer(name, scratch_output, 14) == std::vector<u32>{
0x00020003u, 0x00040001u,
0x00020003u, 0x00040001u, 0x00060007u, 0x00080005u,
0x000a000bu, 0x000c0009u, 0x000e000fu, 0x0010000du,
0x00060007u, 0x00080005u, 0x00060005u, 0x00080007u},
"scratch reuse overwrote an earlier result or its live conversion input");
DestroyBuffer(&scratch_output);
DestroyBuffer(&scratch_input);
const auto gpu_detile =
[&](const std::vector<uint8_t> &tiled, std::vector<uint8_t> *linear,
uint64_t tiled_capacity, uint64_t linear_capacity,
std::span<const GpuTileInfo> infos) {
const uint64_t padded_tiled = align_dword(tiled_capacity);
const uint64_t padded_linear = align_dword(linear_capacity);
auto input = CreateHostBuffer(name, padded_tiled, AllFlags,
to_dwords(tiled, padded_tiled));
auto output = CreateHostBuffer(
name, padded_linear, AllFlags,
std::vector<u32>(static_cast<size_t>(padded_linear / sizeof(u32)),
0xabababab));
const auto result = tile_manager.Detile(input.buffer, 0, padded_tiled,
padded_linear, infos);
const vk::BufferCopy copy{result.offset, 0, padded_linear};
scheduler.Current().Handle().copyBuffer(result.buffer, output.buffer,
1, &copy);
host_barrier(output.buffer, padded_linear,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
*linear = read_bytes(output, linear_capacity);
DestroyBuffer(&output);
DestroyBuffer(&input);
};
const auto gpu_tile = [&](const std::vector<uint8_t> &linear,
std::vector<uint8_t> *tiled,
uint64_t tiled_capacity, uint64_t linear_capacity,
std::span<const GpuTileInfo> infos) {
const uint64_t padded_tiled = align_dword(tiled_capacity);
const uint64_t padded_linear = align_dword(linear_capacity);
auto input = CreateHostBuffer(name, padded_linear, AllFlags,
to_dwords(linear, padded_linear));
auto output = CreateHostBuffer(
name, padded_tiled, AllFlags,
std::vector<u32>(static_cast<size_t>(padded_tiled / sizeof(u32)),
0xabababab));
tile_manager.Tile(input.buffer, 0, padded_linear, output.buffer, 0,
padded_tiled, infos);
host_barrier(output.buffer, padded_tiled,
vk::PipelineStageFlagBits::eComputeShader,
vk::AccessFlagBits::eShaderWrite);
scheduler.Finish();
*tiled = read_bytes(output, tiled_capacity);
DestroyBuffer(&output);
DestroyBuffer(&input);
};
for (const bool d32 : std::array{true, false}) {
constexpr uint32_t width = 5;
constexpr uint32_t height = 3;
constexpr uint32_t pitch = 8;
constexpr uint32_t layers = 2;
constexpr uint64_t d16_row = pitch * sizeof(uint16_t);
constexpr uint64_t d32_row = pitch * sizeof(uint32_t);
constexpr uint64_t d16_slice = d16_row * height;
constexpr uint64_t d32_slice = d32_row * height;
constexpr uint64_t d16_size = d16_slice * layers;
constexpr uint64_t d32_size = d32_slice * layers;
StreamBuffer conversion_upload(m_runtime_context, scheduler,
MemoryUsage::Upload, 1u << 20);
auto [prefix, prefix_offset] = conversion_upload.Map(4096, 256);
Require(name, "D16 conversion prefix",
prefix != nullptr && prefix_offset == 0,
"failed to advance the D16 conversion upload ring");
std::memset(prefix, 0x7b, 4096);
conversion_upload.Commit();
auto [source_bytes, source_offset] = conversion_upload.Map(d16_size, 4);
Require(name, "D16 conversion source",
source_bytes != nullptr && source_offset != 0,
"failed to reserve a nonzero D16 conversion source");
std::memset(source_bytes, 0x3c, d16_size);
for (uint32_t layer = 0; layer < layers; layer++) {
for (uint32_t y = 0; y < height; y++) {
auto *row = reinterpret_cast<uint16_t *>(
source_bytes + layer * d16_slice + y * d16_row);
for (uint32_t x = 0; x < width; x++) {
row[x] = static_cast<uint16_t>(
0x1111u * (1u + x + y * width + layer * width * height));
}
}
}
conversion_upload.Commit();
auto promoted = CreateHostBuffer(
name, d32_size, AllFlags,
std::vector<u32>(static_cast<size_t>(d32_size / sizeof(u32)), 0));
auto round_trip = CreateHostBuffer(
name, d16_size, AllFlags,
std::vector<u32>(static_cast<size_t>(d16_size / sizeof(u32)),
0xa5a5a5a5u));
const TileManager::D16Layout promote_layout{
.width = width,
.height = height,
.layers = layers,
.source_row_stride = d16_row,
.target_row_stride = d32_row,
.source_slice_stride = d16_slice,
.target_slice_stride = d32_slice,
};
tile_manager.ConvertD16({conversion_upload.Handle(), source_offset,
conversion_upload.Size() - source_offset},
{promoted.buffer, 0, d32_size},
TileManager::D16Direction::Promote, d32,
promote_layout);
tile_manager.ConvertD16({promoted.buffer, 0, d32_size},
{round_trip.buffer, 0, d16_size},
TileManager::D16Direction::Demote, d32,
{.width = width,
.height = height,
.layers = layers,
.source_row_stride = d32_row,
.target_row_stride = d16_row,
.source_slice_stride = d32_slice,
.target_slice_stride = d16_slice});
host_barrier(round_trip.buffer, d16_size,
vk::PipelineStageFlagBits::eComputeShader,
vk::AccessFlagBits::eShaderWrite);
scheduler.Finish();
const auto result = read_bytes(round_trip, d16_size);
bool conversion_matches = true;
for (uint32_t layer = 0; layer < layers; layer++) {
for (uint32_t y = 0; y < height; y++) {
for (uint32_t x = 0; x < pitch; x++) {
const auto offset =
layer * d16_slice + y * d16_row + x * sizeof(uint16_t);
conversion_matches &=
x < width
? std::memcmp(result.data() + offset, source_bytes + offset,
sizeof(uint16_t)) == 0
: result[offset] == 0xa5 && result[offset + 1] == 0xa5;
}
}
}
Require(name,
d32 ? "D32 conversion round trip" : "D24 conversion round trip",
conversion_matches,
"D16 conversion lost rows, layers, active values, or padding");
DestroyBuffer(&round_trip);
DestroyBuffer(&promoted);
}
auto fill = [](std::vector<uint8_t> *bytes, u32 salt) {
for (size_t i = 0; i < bytes->size(); i++) {
uint64_t value =
i + static_cast<uint64_t>(salt) * 0x9e3779b97f4a7c15ull;
value = (value ^ (value >> 30u)) * 0xbf58476d1ce4e5b9ull;
value = (value ^ (value >> 27u)) * 0x94d049bb133111ebull;
(*bytes)[i] = static_cast<uint8_t>((value ^ (value >> 31u)) >> 56u);
}
};
auto compare = [&](const char *stage, const std::vector<uint8_t> &expected,
const std::vector<uint8_t> &actual) {
if (expected != actual) {
const auto mismatch = static_cast<size_t>(
std::mismatch(expected.begin(), expected.end(), actual.begin())
.first -
expected.begin());
std::ostringstream out;
out << "first mismatch at " << mismatch << " of " << expected.size();
Fail(name, stage, out.str());
}
};
auto convert_reference = [&](bool to_tiled, std::vector<uint8_t> *dst,
const std::vector<uint8_t> &src,
const GpuTileInfo &info) {
TileBlockLayout block{};
Require(name, "reference layout",
TileGetBlockLayout(info.family, info.bytes_per_element, block),
"CPU reference rejected GPU tile info");
const u32 tiled_width =
info.tiled_width != 0 ? info.tiled_width : info.pitch;
const u32 tiled_height =
info.tiled_height != 0 ? info.tiled_height : info.height;
const uint64_t columns =
(tiled_width + block.block_width - 1u) / block.block_width;
const uint64_t rows =
(tiled_height + block.block_height - 1u) / block.block_height;
const uint64_t slice = info.linear_slice_stride != 0
? info.linear_slice_stride
: static_cast<uint64_t>(info.pitch) *
info.height * info.bytes_per_element;
for (u32 z = 0; z < info.depth; ++z) {
for (u32 y = 0; y < info.height; ++y) {
for (u32 x = 0; x < info.width; ++x) {
const u32 bx = info.tail ? 0 : x / block.block_width;
const u32 by = info.tail ? 0 : y / block.block_height;
const u32 bz = info.tail ? 0 : z / block.block_depth;
const u32 lx = info.tail ? x + info.tail_x : x % block.block_width;
const u32 ly = info.tail ? y + info.tail_y : y % block.block_height;
const u32 lz = z % block.block_depth;
u32 local = 0, block_xor = 0;
Require(name, "reference offset",
TileGetBlockOffset(block, lx, ly, lz, local) &&
TileGetBlockXor(block, bx, by, info.surface_z + bz,
block_xor),
"CPU reference address lookup failed");
const uint64_t block_index =
static_cast<uint64_t>(bz) * columns * rows + by * columns + bx;
const uint64_t tiled = info.tiled_offset +
block_index * block.block_size +
(local ^ block_xor);
const uint64_t linear =
info.linear_offset + static_cast<uint64_t>(z) * slice +
static_cast<uint64_t>(y) * info.pitch * info.bytes_per_element +
static_cast<uint64_t>(x) * info.bytes_per_element;
const uint64_t dst_offset = to_tiled ? tiled : linear;
const uint64_t src_offset = to_tiled ? linear : tiled;
Require(name, "reference range",
dst_offset + info.bytes_per_element <= dst->size() &&
src_offset + info.bytes_per_element <= src.size(),
"CPU reference address escaped storage");
std::memcpy(dst->data() + dst_offset, src.data() + src_offset,
info.bytes_per_element);
}
}
}
};
struct FamilyCase {
TileBlockFamily family;
u32 max_bpe;
};
constexpr FamilyCase families[] = {
{TileBlockFamily::Standard256B, 16},
{TileBlockFamily::Standard4KB, 16},
{TileBlockFamily::Standard4KB3D, 16},
{TileBlockFamily::Standard64KB, 16},
{TileBlockFamily::Standard64KB3D, 16},
{TileBlockFamily::Prt64KB, 16},
{TileBlockFamily::Prt64KB3D, 16},
{TileBlockFamily::RenderTarget64KB, 16},
{TileBlockFamily::Depth64KB, 8},
};
{
TileBlockLayout standard{}, prt{}, standard_3d{}, prt_3d{}, color{},
depth{};
u32 standard_offset = 0, prt_offset = 0, standard_3d_offset = 0,
prt_3d_offset = 0, color_z = 0, depth_z = 0;
bool fixed_addresses =
TileGetBlockLayout(TileBlockFamily::Standard64KB, 4, standard) &&
TileGetBlockLayout(TileBlockFamily::Prt64KB, 4, prt) &&
TileGetBlockLayout(TileBlockFamily::Standard64KB3D, 4, standard_3d) &&
TileGetBlockLayout(TileBlockFamily::Prt64KB3D, 4, prt_3d) &&
TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, 4, color) &&
TileGetBlockLayout(TileBlockFamily::Depth64KB, 4, depth) &&
TileGetBlockOffset(standard, 64, 0, 0, standard_offset) &&
TileGetBlockOffset(prt, 64, 0, 0, prt_offset) &&
TileGetBlockOffset(standard_3d, 16, 0, 0, standard_3d_offset) &&
TileGetBlockOffset(prt_3d, 16, 0, 0, prt_3d_offset) &&
TileGetBlockXor(color, 0, 0, 1, color_z) &&
TileGetBlockXor(depth, 0, 0, 15, depth_z);
Require(name, "fixed address vectors",
fixed_addresses && standard_offset == 0x8000 &&
prt_offset == 0x8100 && standard_3d_offset == 0x8000 &&
prt_3d_offset == 0x8400 && standard_3d.block_width == 32 &&
standard_3d.block_height == 32 &&
standard_3d.block_depth == 16 && color_z == 0x800 &&
depth_z == 0xf00,
"a fixed block address changed");
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr u32 levels = 7;
TileSizeAlign total{};
TileSizeOffset mip[levels]{};
TilePaddedSize padded[levels]{};
TileGetTextureSize(format, 65, 33, levels,
Prospero::TileMode::kStandard256B, &total, mip,
padded);
constexpr u32 offsets[levels] = {0x1a00, 0xb00, 0x500, 0x300,
0x200, 0x100, 0};
constexpr u32 sizes[levels] = {0x2d00, 0xf00, 0x600, 0x200,
0x100, 0x100, 0x100};
bool layout_matches = total.size == 0x4700 && total.align == 0x100 &&
padded[0].width == 72 && padded[0].height == 40;
for (u32 level = 0; level < levels; ++level) {
layout_matches &= mip[level].offset == offsets[level] &&
mip[level].size == sizes[level];
}
Require(name, "fixed mip vector", layout_matches,
"the reverse-packed small-block mip layout changed");
}
u32 case_index = 0;
auto check_round_trip = [&](const char *stage, uint64_t tiled_size,
std::span<const GpuTileInfo> infos) {
uint64_t linear_size = 0;
for (const auto &info : infos) {
linear_size =
std::max(linear_size, info.linear_offset + info.linear_size);
}
std::vector<uint8_t> tiled(tiled_size);
std::vector<uint8_t> cpu(linear_size, 0);
std::vector<uint8_t> gpu(linear_size, 0xab);
fill(&tiled, ++case_index);
for (const auto &info : infos) {
convert_reference(false, &cpu, tiled, info);
}
gpu_detile(tiled, &gpu, tiled_size, linear_size, infos);
compare((std::string(stage) + " detile bytes").c_str(), cpu, gpu);
std::vector<uint8_t> linear(linear_size);
std::vector<uint8_t> cpu_tiled(tiled_size, 0xab);
std::vector<uint8_t> gpu_tiled(tiled_size, 0xab);
fill(&linear, 0x280u + case_index);
for (const auto &info : infos) {
convert_reference(true, &cpu_tiled, linear, info);
}
gpu_tile(linear, &gpu_tiled, tiled_size, linear_size, infos);
compare((std::string(stage) + " tile bytes").c_str(), cpu_tiled,
gpu_tiled);
};
for (const auto family : families) {
for (u32 bpe = 1; bpe <= family.max_bpe; bpe <<= 1u) {
TileBlockLayout block{};
Require(name, "block layout",
TileGetBlockLayout(family.family, bpe, block),
"admitted family/BPE has no block layout");
const bool volume = block.block_depth > 1;
const u32 width =
block.block_width * 3u + std::min(block.block_width, 3u);
const u32 height =
block.block_height * 3u + std::min(block.block_height, 3u);
const u32 depth = volume ? block.block_depth + 1u : 1u;
const u32 pitch = block.block_width * 4u;
const uint64_t block_columns =
(pitch + block.block_width - 1u) / block.block_width;
const uint64_t block_rows =
(height + block.block_height - 1u) / block.block_height;
const uint64_t block_slices =
(depth + block.block_depth - 1u) / block.block_depth;
const uint64_t storage_size =
block_columns * block_rows * block_slices * block.block_size;
const uint64_t slice_stride =
static_cast<uint64_t>(pitch) * height * bpe;
std::vector<uint8_t> tiled(storage_size);
std::vector<uint8_t> cpu(storage_size, 0);
std::vector<uint8_t> gpu(storage_size, 0xab);
fill(&tiled, ++case_index);
GpuTileInfo info{};
info.family = block.family;
info.bytes_per_element = block.bytes_per_element;
info.linear_size = storage_size;
info.tiled_size = storage_size;
info.linear_slice_stride = volume ? slice_stride : 0;
info.width = width;
info.height = height;
info.depth = depth;
info.pitch = pitch;
info.surface_z = family.family == TileBlockFamily::RenderTarget64KB ||
family.family == TileBlockFamily::Depth64KB
? 3
: 0;
convert_reference(false, &cpu, tiled, info);
gpu_detile(tiled, &gpu, storage_size, storage_size,
std::span<const GpuTileInfo>(&info, 1));
const auto family_label = [&](const char *operation) {
std::ostringstream out;
out << operation << " family=" << static_cast<u32>(family.family)
<< " bpe=" << bpe;
return out.str();
};
compare(family_label("detile bytes").c_str(), cpu, gpu);
{
std::vector<uint8_t> linear(storage_size);
std::vector<uint8_t> cpu_tiled(storage_size, 0xab);
std::vector<uint8_t> gpu_tiled(storage_size, 0xab);
fill(&linear, 0x80u + case_index);
convert_reference(true, &cpu_tiled, linear, info);
gpu_tile(linear, &gpu_tiled, storage_size, storage_size,
std::span<const GpuTileInfo>(&info, 1));
compare(family_label("tile bytes").c_str(), cpu_tiled, gpu_tiled);
}
}
}
// Exercise the real texture-layout/info-building seam for every format
// admitted by each standard tile mode. Formats sharing a byte/block width
// intentionally share a shader, but this loop still validates their
// texel-to-element conversion (notably every BCn format).
struct StandardMode {
Prospero::TileMode tile;
TileBlockFamily family;
};
constexpr StandardMode standard_modes[] = {
{Prospero::TileMode::kStandard256B, TileBlockFamily::Standard256B},
{Prospero::TileMode::kStandard4KB, TileBlockFamily::Standard4KB},
{Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB},
{Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB},
};
struct RenderTargetFormatCase {
Prospero::BufferFormat format;
u32 bytes_per_element;
};
constexpr RenderTargetFormatCase render_target_formats[] = {
{Prospero::BufferFormat::k8Srgb, 1},
{Prospero::BufferFormat::k8_8Srgb, 2},
{Prospero::BufferFormat::k9_9_9_5Float, 0},
};
for (const auto &test : render_target_formats) {
for (const auto tile :
{Prospero::TileMode::kDepth, Prospero::TileMode::kRenderTarget}) {
TileTextureBlockLayout texture{};
const bool supported =
TileGetTextureBlockLayout(test.format, tile, false, texture);
Require(name, "RT format policy",
supported == (test.bytes_per_element != 0) &&
(!supported || texture.block.bytes_per_element ==
test.bytes_per_element),
"RT/depth tile format support or element size is incorrect");
}
}
{
TileSurfaceLayout invalid{};
const TileSurfaceDescription description{
Prospero::BufferFormat::k32Float,
Prospero::TileMode::kStandard256B,
TileSurfaceDimension::Dim3D,
16,
16,
16,
1,
1,
};
Require(name, "3D tile policy",
!TileGetTiledTextureLayout(description, invalid),
"Standard256B volume silently fell back to a 2D array layout");
}
{
constexpr auto format = Prospero::BufferFormat::kFmask8_S4_F4;
constexpr auto tile = Prospero::TileMode::kStandard64KB;
TileSizeAlign total{};
TileGetTextureSize(format, 128, 128, 1, tile, &total, nullptr, nullptr);
const auto layout = TextureCalcUploadLayout(
format, 128, 128, 1, 1, tile, total.size, false, name);
const auto regions = TextureBuildImageCopies(layout);
std::vector<GpuTileInfo> infos;
Require(name, "FMASK policy",
!TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos),
"FMASK entered the texel tiler through a non-depth family");
}
u32 format_cases = 0;
for (u32 raw_format = 1;
raw_format <= static_cast<uint32_t>(Prospero::BufferFormat::kBc7Srgb);
++raw_format) {
const auto format = static_cast<Prospero::BufferFormat>(raw_format);
// FMASK contains sample-to-fragment metadata, not texels for the GPU tiler.
if (Prospero::IsFmaskTextureFormat(format)) {
continue;
}
for (const auto &mode : standard_modes) {
TileTextureBlockLayout texture{};
if (!TileGetTextureBlockLayout(format, mode.tile, false, texture)) {
continue;
}
const u32 bpe = texture.block.bytes_per_element;
Require(name, "format block", texture.block.family == mode.family,
"CPU-supported format selected the wrong block family");
constexpr u32 width = 67;
constexpr u32 height = 51;
const u32 pitch = TileGetTexturePitch(format, width, mode.tile);
TileSizeAlign total{};
TileGetTextureSize(format, width, height, 1, mode.tile, &total, nullptr,
nullptr);
Require(name, "format size", total.size != 0,
"supported format has an empty layout");
const auto layout =
TextureCalcUploadLayout(format, width, height, 1, 1, mode.tile,
total.size, false, name);
const auto regions = TextureBuildImageCopies(layout);
std::vector<GpuTileInfo> infos;
if (!TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos)) {
std::ostringstream out;
out << "format=" << raw_format
<< " tile=" << static_cast<uint32_t>(mode.tile)
<< " size=" << total.size << " pitch=" << pitch;
Fail(name, "format infos", out.str());
}
Require(name, "format family",
infos.size() == 1 && infos[0].family == mode.family &&
infos[0].bytes_per_element == bpe,
"texture info selected the wrong shader family");
std::vector<uint8_t> tiled(total.size);
std::vector<uint8_t> cpu(total.size, 0);
std::vector<uint8_t> gpu(total.size, 0xab);
fill(&tiled, ++case_index);
convert_reference(false, &cpu, tiled, infos[0]);
gpu_detile(tiled, &gpu, total.size, total.size, infos);
compare("format bytes", cpu, gpu);
++format_cases;
}
}
Require(name, "format coverage", format_cases != 0,
"no CPU-supported standard formats were tested");
{
constexpr auto format = Prospero::BufferFormat::k11_11_10UInt;
constexpr auto tile = Prospero::TileMode::kStandard64KB;
constexpr u32 width = 128, height = 128, levels = 8;
TileSurfaceLayout surface{};
const TileSurfaceDescription description{
format, tile, TileSurfaceDimension::Dim2D, width, height, 1,
levels, 1};
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, 1, levels, tile, false,
total);
const bool built = TileGetTiledTextureLayout(description, surface);
const auto surface_format = TextureGetSurfaceFormatInfo(format);
constexpr u32 tail_xy[7][2] = {{64, 0}, {0, 64}, {32, 0}, {0, 32},
{16, 0}, {8, 16}, {0, 24}};
bool valid = built && Prospero::NumBytesPerElement(format) == 4 &&
Prospero::BlockCompressedBytesPerBlock(format) == 0 &&
Prospero::SampledTextureNumericClass(format) ==
Prospero::TextureNumericClass::Uint &&
Prospero::RemapTextureFormat(format) ==
Prospero::BufferFormat::k32UInt &&
surface_format.vk_format == vk::Format::eR32Uint &&
surface_format.conversion_format == format &&
VulkanFormat(format) == vk::Format::eUndefined &&
surface.texture.block.block_width == 128 &&
surface.texture.block.block_height == 128 &&
surface.texture.block.block_depth == 1 &&
surface.first_tail_level == 1 &&
surface.mips[0].offset == 0x10000 && total.size == 0x20000 &&
total.align == 0x10000;
for (u32 level = 1; level < levels && valid; level++) {
valid &= surface.mips[level].offset == 0 &&
surface.mips[level].tail_x == tail_xy[level - 1][0] &&
surface.mips[level].tail_y == tail_xy[level - 1][1];
}
Require(name, "11_11_10 uint layout", valid,
"PS5 packed integer texture metadata or Standard64KB mip tail "
"changed");
const auto signed_format = Prospero::BufferFormat::k32SInt;
const auto signed_surface = TextureGetSurfaceFormatInfo(signed_format);
Require(name, "signed sampled surface mapping",
Prospero::SampledTextureNumericClass(signed_format) ==
Prospero::TextureNumericClass::Sint &&
Prospero::RemapTextureFormat(signed_format) ==
signed_format &&
signed_surface.vk_format == vk::Format::eR32Sint &&
signed_surface.conversion_format ==
Prospero::BufferFormat::kInvalid,
"signed sampled texture class or native backing changed");
}
{
constexpr auto format = Prospero::BufferFormat::k32_32_32Float;
TileTextureElementLayout element{};
Require(name, "RGB32 texture policy",
!TileGetTextureElementLayout(format, element),
"12-byte texel layouts must be rejected");
}
{
constexpr auto format = Prospero::BufferFormat::kBc1UNorm;
constexpr auto tile = Prospero::TileMode::kStandard64KB;
constexpr u32 width = 256, height = 256, levels = 9;
TileSizeAlign total{};
TileGetTextureSize(format, width, height, levels, tile, &total, nullptr,
nullptr);
const auto layout =
TextureCalcUploadLayout(format, width, height, levels, 1, tile,
total.size, false, name);
const auto regions = TextureBuildImageCopies(layout);
std::vector<GpuTileInfo> infos;
const bool built =
TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos);
uint64_t linear_size = 0;
for (const auto &info : infos) {
linear_size =
std::max(linear_size, info.linear_offset + info.linear_size);
}
Require(name, "BC1 mip-tail capacities",
built && total.size == 0x10000 &&
layout.surface.first_tail_level == 0 &&
linear_size == 0x15560 && linear_size > total.size,
"BC1 mip tail conflated tiled and linear capacities");
check_round_trip("BC1 mip tail", total.size, infos);
}
{
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr auto tile = Prospero::TileMode::kRenderTarget;
constexpr u32 width = 129, height = 65, layers = 3;
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, layers, 1, tile, false,
total);
const auto layout =
TextureCalcUploadLayout(format, width, height, 1, layers, tile,
total.size, false, name);
const auto regions = TextureBuildImageCopies(layout);
std::vector<GpuTileInfo> infos;
const bool built =
TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos);
Require(name, "array infos",
built && infos.size() == layers && infos[0].surface_z == 0 &&
infos[1].surface_z == 1 && infos[2].surface_z == 2,
"array slices lost their absolute surface Z");
check_round_trip("array", total.size, infos);
}
for (const auto &mode : standard_modes) {
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr u32 levels = 2;
TileBlockLayout block{};
Require(name, "odd mip block", TileGetBlockLayout(mode.family, 4, block),
"odd multi-mip format has no block layout");
const u32 width = block.block_width * 2u + 1u;
const u32 height = block.block_height * 2u + 1u;
TileSizeAlign total{};
TileGetTextureSize(format, width, height, levels, mode.tile, &total,
nullptr, nullptr);
const auto layout =
TextureCalcUploadLayout(format, width, height, levels, 1, mode.tile,
total.size, false, name);
const auto regions = TextureBuildImageCopies(layout);
std::vector<GpuTileInfo> infos;
Require(name, "odd mip infos",
TextureBuildGpuTileInfos(total.size, regions, layout, levels,
infos) &&
infos.size() == 2 && infos[1].tiled_width >= infos[1].pitch &&
infos[1].tiled_height >= infos[1].height &&
(mode.family == TileBlockFamily::Standard256B ||
infos[1].tiled_height > infos[1].height),
"odd multi-mip physical stride collapsed to the active linear "
"extent");
check_round_trip("odd multi-mip", total.size, infos);
}
struct TailMode {
Prospero::TileMode tile;
TileBlockFamily family;
};
constexpr TailMode tail_modes[] = {
{Prospero::TileMode::kStandard4KB, TileBlockFamily::Standard4KB},
{Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB},
{Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB},
{Prospero::TileMode::kRenderTarget, TileBlockFamily::RenderTarget64KB},
{Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB},
};
for (const auto &mode : tail_modes) {
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr u32 levels = 7;
TileBlockLayout block{};
Require(name, "2D tail block", TileGetBlockLayout(mode.family, 4, block),
"2D tail mode has no block layout");
const u32 width = block.block_width * 2u;
const u32 height = block.block_height * 2u;
TileSizeAlign total{};
TileGetTextureSize(format, width, height, levels, mode.tile, &total,
nullptr, nullptr);
const auto layout =
TextureCalcUploadLayout(format, width, height, levels, 1, mode.tile,
total.size, false, name);
const auto regions = TextureBuildImageCopies(layout);
std::vector<GpuTileInfo> infos;
Require(name, "2D mip tail seam",
layout.surface.first_tail_level == 2 &&
TextureBuildGpuTileInfos(total.size, regions, layout, levels,
infos) &&
infos.size() == levels && !infos[0].tail && !infos[1].tail &&
std::all_of(infos.begin() + 2, infos.end(),
[](const auto &info) { return info.tail; }),
"2D mip chain lost its linear/tiled tail boundary");
check_round_trip("2D mip tail seam", total.size, infos);
}
{
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr auto tile = Prospero::TileMode::kStandard4KB;
constexpr u32 width = 65;
constexpr u32 height = 129;
constexpr u32 depth = 17;
constexpr u32 levels = 2;
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, depth, levels, tile, true,
total);
const auto layout =
TextureCalcUploadLayout(format, width, height, levels, depth, tile,
total.size, true, name);
const auto regions = TextureBuildImageCopies(layout);
std::vector<GpuTileInfo> infos;
const bool built =
TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos);
Require(name, "3D mip infos",
regions.size() == depth + (depth >> 1u) && built &&
infos.size() == 4 && infos[0].depth == 8 &&
infos[1].depth == 8 && infos[2].depth == 1 &&
infos[3].depth == 8 && infos[0].tiled_offset == 0x19000 &&
infos[1].tiled_offset == 0x83000 &&
infos[2].tiled_offset == 0xed000 &&
infos[3].tiled_offset == 0 && infos[3].pitch == 32 &&
infos[3].height == 64 && infos[3].tiled_width == 40 &&
infos[3].tiled_height == 80,
"Standard4KB3D mip depth, packing, or physical stride changed");
check_round_trip("3D mip", total.size, infos);
}
{
constexpr auto format = Prospero::BufferFormat::kBc1UNorm;
constexpr auto tile = Prospero::TileMode::kStandard4KB;
constexpr u32 width = 65;
constexpr u32 height = 129;
constexpr u32 depth = 17;
constexpr u32 levels = 2;
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, depth, levels, tile, true,
total);
const auto layout =
TextureCalcUploadLayout(format, width, height, levels, depth, tile,
total.size, true, name);
const auto regions = TextureBuildImageCopies(layout);
std::vector<GpuTileInfo> infos;
Require(name, "3D BC mip infos",
TextureBuildGpuTileInfos(total.size, regions, layout, levels,
infos) &&
infos.size() == 4 && infos[3].pitch == 8 &&
infos[3].height == 16 && infos[3].tiled_width == 16 &&
infos[3].tiled_height == 24,
"block-compressed 3D mip lost its physical row or slice stride");
check_round_trip("3D BC mip", total.size, infos);
}
{
constexpr auto format = Prospero::BufferFormat::kBc3UNorm;
constexpr auto tile = Prospero::TileMode::kLinear;
constexpr u32 width = 8, height = 8, levels = 4;
TileSizeAlign total{};
TileGetTextureSize(format, width, height, levels, tile, &total, nullptr,
nullptr);
const auto layout =
TextureCalcUploadLayout(format, width, height, levels, 1, tile,
total.size, false, name);
const auto regions = TextureBuildImageCopies(layout);
Require(name, "linear BC native regions",
regions.size() == levels &&
std::all_of(regions.begin(), regions.end(),
[](const auto &region) {
return region.bufferRowLength >=
region.imageExtent.width &&
region.bufferImageHeight >=
region.imageExtent.height &&
region.bufferRowLength % 4 == 0 &&
region.bufferImageHeight % 4 == 0;
}),
"linear BC mip tails emitted invalid Vulkan copy strides");
}
struct VolumeModeCase {
Prospero::TileMode tile;
TileBlockFamily family;
Prospero::BufferFormat format;
};
constexpr VolumeModeCase volume_modes[] = {
{Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB3D,
Prospero::BufferFormat::k32Float},
{Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB3D,
Prospero::BufferFormat::k32Float},
{Prospero::TileMode::kRenderTarget, TileBlockFamily::RenderTarget64KB,
Prospero::BufferFormat::k32Float},
{Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB,
Prospero::BufferFormat::k32Float},
{Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB3D,
Prospero::BufferFormat::kBc1UNorm},
{Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB3D,
Prospero::BufferFormat::kBc3UNorm},
};
for (const auto test : volume_modes) {
constexpr u32 width = 65, height = 33, depth = 37, levels = 6;
TileSizeAlign total{};
TileGetTextureTotalSize(test.format, width, height, depth, levels,
test.tile, true, total);
const auto layout =
TextureCalcUploadLayout(test.format, width, height, levels, depth,
test.tile, total.size, true, name);
const auto regions = TextureBuildImageCopies(layout);
std::vector<GpuTileInfo> infos;
const bool built =
TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos);
const bool uses_z = test.family == TileBlockFamily::RenderTarget64KB ||
test.family == TileBlockFamily::Depth64KB;
Require(name, "volume family infos",
built && !infos.empty() &&
std::all_of(infos.begin(), infos.end(),
[&](const auto &info) {
return info.family == test.family;
}) &&
(!uses_z || std::any_of(infos.begin(), infos.end(),
[](const auto &info) {
return info.surface_z != 0;
})),
"volume mode selected the wrong family or lost its surface Z");
check_round_trip("volume family", total.size, infos);
}
struct VolumeTailCase {
Prospero::BufferFormat format;
u32 bytes;
};
constexpr VolumeTailCase volume_tails[] = {
{Prospero::BufferFormat::k8UNorm, 1},
{Prospero::BufferFormat::k16UNorm, 2},
{Prospero::BufferFormat::k32Float, 4},
{Prospero::BufferFormat::k16_16_16_16Float, 8},
{Prospero::BufferFormat::k32_32_32_32Float, 16},
{Prospero::BufferFormat::kBc1UNorm, 8},
{Prospero::BufferFormat::kBc3UNorm, 16},
};
constexpr u32 volume_tail_xy[5][5][2] = {
{{0, 8}, {8, 4}, {8, 0}, {0, 4}, {0, 0}},
{{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}},
{{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}},
{{0, 4}, {4, 2}, {4, 0}, {0, 2}, {0, 0}},
{{0, 4}, {2, 2}, {2, 0}, {0, 2}, {0, 0}},
};
for (const auto &tail : volume_tails) {
constexpr auto tile = Prospero::TileMode::kStandard4KB;
constexpr u32 levels = 5;
TileBlockLayout block{};
Require(
name, "3D tail block",
TileGetBlockLayout(TileBlockFamily::Standard4KB3D, tail.bytes, block),
"Standard4KB3D tail format has no block layout");
const bool compressed =
Prospero::BlockCompressedBytesPerBlock(tail.format) != 0;
const u32 width = block.block_width * (compressed ? 4u : 1u);
const u32 height = block.block_height / 2u * (compressed ? 4u : 1u);
const u32 depth = block.block_depth + 1u;
TileSizeAlign total{};
TileGetTextureTotalSize(tail.format, width, height, depth, levels, tile,
true, total);
const auto layout =
TextureCalcUploadLayout(tail.format, width, height, levels, depth,
tile, total.size, true, name);
const auto regions = TextureBuildImageCopies(layout);
std::vector<GpuTileInfo> infos;
bool valid = total.size == 8192 &&
regions.size() == depth + std::max(depth >> 1u, 1u) +
std::max(depth >> 2u, 1u) +
std::max(depth >> 3u, 1u) +
std::max(depth >> 4u, 1u) &&
TextureBuildGpuTileInfos(total.size, regions, layout, levels,
infos) &&
infos.size() == 6;
const u32 table = std::countr_zero(tail.bytes);
for (u32 level = 0; level < levels && valid; level++) {
const auto &info = infos[level == 0 ? 0 : level + 1];
valid &= info.tail && info.tail_x == volume_tail_xy[table][level][0] &&
info.tail_y == volume_tail_xy[table][level][1] &&
info.tiled_offset == 0;
}
if (valid) {
valid = infos[1].tail &&
infos[1].tail_x == volume_tail_xy[table][0][0] &&
infos[1].tail_y == volume_tail_xy[table][0][1] &&
infos[1].tiled_offset == 4096;
}
Require(
name, "3D mip tail infos", valid,
"Standard4KB3D mip tail coordinates or block-slice packing changed");
check_round_trip("3D mip tail", total.size, infos);
}
for (const auto family :
{TileBlockFamily::Standard4KB, TileBlockFamily::Standard64KB,
TileBlockFamily::Prt64KB, TileBlockFamily::RenderTarget64KB,
TileBlockFamily::Depth64KB}) {
for (u32 bpe = 1; bpe <= 16; bpe <<= 1u) {
if (family == TileBlockFamily::Depth64KB && bpe == 16)
continue;
TileBlockLayout block{};
Require(name, "tail layout", TileGetBlockLayout(family, bpe, block),
"tail family/BPE has no block layout");
const u32 width = std::max(block.block_width / 4u, 1u);
const u32 height = std::max(block.block_height / 4u, 1u);
const u32 x = block.block_width / 2u;
const u32 y = block.block_height / 2u;
const u32 pitch = width;
const uint64_t linear_size =
static_cast<uint64_t>(pitch) * height * bpe;
std::vector<uint8_t> tiled(block.block_size);
std::vector<uint8_t> cpu(linear_size, 0xcd);
std::vector<uint8_t> gpu(linear_size, 0xab);
fill(&tiled, ++case_index);
GpuTileInfo info{};
info.family = block.family;
info.bytes_per_element = block.bytes_per_element;
info.linear_size = linear_size;
info.tiled_size = block.block_size;
info.width = width;
info.height = height;
info.pitch = pitch;
info.tail = true;
info.tail_x = x;
info.tail_y = y;
info.surface_z = family == TileBlockFamily::RenderTarget64KB ||
family == TileBlockFamily::Depth64KB
? 2
: 0;
convert_reference(false, &cpu, tiled, info);
gpu_detile(tiled, &gpu, block.block_size, linear_size,
std::span<const GpuTileInfo>(&info, 1));
compare("tail bytes", cpu, gpu);
std::vector<uint8_t> linear(linear_size);
std::vector<uint8_t> cpu_tiled(block.block_size, 0xab);
std::vector<uint8_t> gpu_tiled(block.block_size, 0xab);
fill(&linear, 0x400u + case_index);
convert_reference(true, &cpu_tiled, linear, info);
gpu_tile(linear, &gpu_tiled, block.block_size, linear_size,
std::span<const GpuTileInfo>(&info, 1));
compare("tail tile bytes", cpu_tiled, gpu_tiled);
}
}
TileBlockLayout small_block{};
Require(name, "small layout",
TileGetBlockLayout(TileBlockFamily::Standard256B, 4, small_block),
"small fixture layout is unavailable");
std::vector<uint8_t> small_input(small_block.block_size);
std::vector<uint8_t> small_expected(small_block.block_size, 0);
std::vector<uint8_t> small_output(small_block.block_size, 0xab);
fill(&small_input, 0xee);
GpuTileInfo small_info{};
small_info.family = small_block.family;
small_info.bytes_per_element = small_block.bytes_per_element;
small_info.linear_size = small_output.size();
small_info.tiled_size = small_input.size();
small_info.width = 1;
small_info.height = 1;
small_info.pitch = small_block.block_width;
convert_reference(false, &small_expected, small_input, small_info);
gpu_detile(small_input, &small_output, small_input.size(),
small_output.size(),
std::span<const GpuTileInfo>(&small_info, 1));
compare("small detile", small_expected, small_output);
std::fill(small_output.begin(), small_output.end(), 0xab);
gpu_detile(small_input, &small_output, small_input.size(),
small_output.size(),
std::span<const GpuTileInfo>(&small_info, 1));
compare("scheduler-owned reuse", small_expected, small_output);
constexpr auto volume_format = Prospero::BufferFormat::k32UInt;
constexpr auto volume_tile = Prospero::TileMode::kLinear;
constexpr u32 volume_width = 8, volume_height = 4, volume_depth = 5;
constexpr u32 volume_levels = 3;
const u32 volume_pitch =
TileGetTexturePitch(volume_format, volume_width, volume_tile);
TileSizeAlign volume_size{};
TileGetTextureTotalSize(volume_format, volume_width, volume_height,
volume_depth, volume_levels, volume_tile, true,
volume_size);
const auto volume_layout = TextureCalcUploadLayout(
volume_format, volume_width, volume_height, volume_levels, volume_depth,
volume_tile, volume_size.size, true, name);
const auto volume_copies = TextureBuildImageCopies(volume_layout);
std::vector<u32> volume_source(volume_size.size / sizeof(u32), 0);
std::vector<std::pair<size_t, u32>> volume_probes;
for (const auto &copy : volume_copies) {
for (u32 y = 0; y < copy.imageExtent.height; y++) {
for (u32 x = 0; x < copy.imageExtent.width; x++) {
const auto index =
(copy.bufferOffset +
(static_cast<uint64_t>(y) * copy.bufferRowLength + x) *
sizeof(u32)) /
sizeof(u32);
const u32 value =
0xa0000000u | (copy.imageSubresource.mipLevel << 24u) |
(static_cast<u32>(copy.imageOffset.z) << 16u) | (y << 8u) | x;
Require(name, "linear volume native bounds",
index < volume_source.size(),
"linear volume copy escaped its padded guest layout");
volume_source[index] = value;
volume_probes.emplace_back(static_cast<size_t>(index), value);
}
}
}
auto volume_upload =
CreateHostBuffer(name, volume_size.size,
vk::BufferUsageFlagBits::eTransferSrc, volume_source);
auto volume_download = CreateHostBuffer(
name, volume_size.size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(volume_source.size(), 0));
ImageInfo volume_info{};
volume_info.data = {0x10000, volume_size.size};
volume_info.pixel_format = vk::Format::eR32Uint;
volume_info.guest_format = volume_format;
volume_info.type = Prospero::ImageType::kColor3D;
volume_info.extent = {volume_width, volume_height, volume_depth};
volume_info.resources = {volume_levels, 1};
volume_info.pitch = volume_pitch;
volume_info.bytes_per_block = sizeof(u32);
volume_info.tile_mode = volume_tile;
for (u32 level = 0; level < volume_levels; level++) {
volume_info.mip_layout[level] = {volume_layout.mips[level].offset,
volume_layout.mips[level].size,
volume_layout.mips[level].row_length,
volume_layout.mips[level].image_height};
}
Libs::Graphics::Image volume_image(m_runtime_context, scheduler,
volume_info);
volume_image.Upload(volume_copies, volume_upload.buffer, 0,
volume_size.size);
volume_image.Download(volume_copies, volume_download.buffer, 0,
volume_size.size);
scheduler.Finish();
const auto volume_observed =
ReadBuffer(name, volume_download, volume_source.size());
Require(name, "linear volume native content",
std::ranges::all_of(volume_probes,
[&](const auto &probe) {
return volume_observed[probe.first] ==
probe.second;
}),
"native upload/download did not preserve padded 3D mip rows");
DestroyBuffer(&volume_download);
DestroyBuffer(&volume_upload);
std::printf("[gpu] %-32s ok (%u cases, %u format/mode pairs)\n", name,
case_index, format_cases);
}
private:
bool m_rasterization_supported = true;
u32 m_skipped_cases = 0;
RenderContext &Renderer() {
EXIT_IF(m_renderer == nullptr);
return *m_renderer;
}
void EnsureRuntimeContext() {
if (m_runtime_context.allocator != nullptr) {
return;
}
namespace Exception = Common::HostException;
Require("VulkanHarness", "guest fault handler",
Exception::InstallHandler([](const Exception::ExceptionInfo &info) {
if (info.type != Exception::ExceptionType::AccessViolation ||
(info.access_violation_type != Exception::AccessViolationType::Read &&
info.access_violation_type != Exception::AccessViolationType::Write)) {
return false;
}
const auto access =
info.access_violation_type == Exception::AccessViolationType::Write
? PageFaultAccess::Write : PageFaultAccess::Read;
return LibKernel::Memory::HandleGpuFault(access, info.access_violation_vaddr);
}),
"failed to install the production guest-memory fault route");
m_runtime_context.instance = m_instance;
m_runtime_context.physical_device = m_physical_device;
m_runtime_context.device = m_device;
m_physical_device.getProperties(
&m_runtime_context.physical_device_properties);
m_runtime_context.physical_device_memory_properties = m_memory_properties;
m_runtime_context.queue_family = m_queue_family;
m_runtime_context.queue = m_queue;
m_runtime_context.shader_image_int64_atomics_enabled = true;
m_runtime_context.attachment_feedback_loop_enabled = m_rasterization_supported;
m_runtime_context.provoking_vertex_last_enabled = m_rasterization_supported;
const vk::PhysicalDeviceImageFormatInfo2 block_texel_view_info{
.format = vk::Format::eBc1RgbaUnormBlock,
.type = vk::ImageType::e2D,
.tiling = vk::ImageTiling::eOptimal,
.usage = vk::ImageUsageFlagBits::eSampled,
.flags = vk::ImageCreateFlagBits::eBlockTexelViewCompatible,
};
const auto block_texel_view_props =
m_physical_device.getImageFormatProperties2(block_texel_view_info);
m_runtime_context.supports_block_texel_view =
block_texel_view_props.result == vk::Result::eSuccess;
VmaVulkanFunctions functions{};
functions.vkGetInstanceProcAddr =
VULKAN_HPP_DEFAULT_DISPATCHER.vkGetInstanceProcAddr;
functions.vkGetDeviceProcAddr =
VULKAN_HPP_DEFAULT_DISPATCHER.vkGetDeviceProcAddr;
VmaAllocatorCreateInfo allocator_info{};
allocator_info.instance = m_instance;
allocator_info.physicalDevice = m_physical_device;
allocator_info.device = m_device;
allocator_info.pVulkanFunctions = &functions;
allocator_info.vulkanApiVersion = VK_API_VERSION_1_3;
allocator_info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT;
RequireVk("VulkanHarness", "runtime context",
static_cast<vk::Result>(vmaCreateAllocator(
&allocator_info, &m_runtime_context.allocator)),
"vmaCreateAllocator");
m_renderer = std::make_unique<RenderContext>(m_runtime_context);
}
void Init() {
static vk::detail::DynamicLoader loader;
const auto get_instance_proc_addr =
loader.getProcAddress<PFN_vkGetInstanceProcAddr>(
"vkGetInstanceProcAddr");
Require("VulkanHarness", "dispatch", get_instance_proc_addr != nullptr,
"could not load the Vulkan loader");
VULKAN_HPP_DEFAULT_DISPATCHER.init(get_instance_proc_addr);
vk::ApplicationInfo app{};
app.sType = vk::StructureType::eApplicationInfo;
app.pApplicationName = "ShaderRecompilerComputeTests";
app.apiVersion = VK_API_VERSION_1_3;
vk::InstanceCreateInfo instance_info{};
instance_info.sType = vk::StructureType::eInstanceCreateInfo;
instance_info.pApplicationInfo = &app;
RequireVk("VulkanHarness", "dispatch",
vk::createInstance(&instance_info, nullptr, &m_instance),
"vkCreateInstance");
VULKAN_HPP_DEFAULT_DISPATCHER.init(m_instance);
u32 physical_count = 0;
RequireVk("VulkanHarness", "dispatch",
m_instance.enumeratePhysicalDevices(&physical_count, nullptr),
"vkEnumeratePhysicalDevices");
Require("VulkanHarness", "dispatch", physical_count != 0,
"no Vulkan physical devices");
std::vector<vk::PhysicalDevice> physical_devices(physical_count);
RequireVk("VulkanHarness", "dispatch",
m_instance.enumeratePhysicalDevices(&physical_count,
physical_devices.data()),
"vkEnumeratePhysicalDevices");
for (auto physical : physical_devices) {
u32 queue_count = 0;
physical.getQueueFamilyProperties(&queue_count, nullptr);
std::vector<vk::QueueFamilyProperties> queues(queue_count);
physical.getQueueFamilyProperties(&queue_count, queues.data());
for (u32 i = 0; i < queue_count; i++) {
if ((queues[i].queueFlags &
(vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) ==
(vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) {
vk::PhysicalDeviceVulkan11Features features11{};
vk::PhysicalDeviceVulkan12Features features12{};
features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{};
features12.pNext = &features11;
features11.pNext = &workgroup_layout;
vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{};
workgroup_layout.pNext = &image_atomic64;
vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{};
barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
barycentric.pNext = &features12;
vk::PhysicalDeviceFeatures2 features{};
features.sType = vk::StructureType::ePhysicalDeviceFeatures2;
features.pNext = &barycentric;
physical.getFeatures2(&features);
if (barycentric.fragmentShaderBarycentric != true ||
features.features.shaderInt64 != true ||
features11.storageBuffer16BitAccess != true ||
features12.storageBuffer8BitAccess != true ||
features12.samplerMirrorClampToEdge != true ||
features12.shaderOutputViewportIndex != true ||
features12.shaderBufferInt64Atomics != true ||
features12.shaderSampledImageArrayNonUniformIndexing != true ||
features12.shaderSharedInt64Atomics != true ||
image_atomic64.shaderImageInt64Atomics != true ||
workgroup_layout.workgroupMemoryExplicitLayout != true ||
features12.bufferDeviceAddress != true) {
continue;
}
m_physical_device = physical;
m_queue_family = i;
break;
}
}
if (m_physical_device != nullptr) {
break;
}
}
Require("VulkanHarness", "dispatch", m_physical_device != nullptr,
"no Vulkan graphics+compute device with fragment barycentrics and 64-bit LDS/image atomics");
m_physical_device.getMemoryProperties(&m_memory_properties);
vk::PhysicalDeviceFeatures available_features{};
m_physical_device.getFeatures(&available_features);
vk::PhysicalDeviceVulkan11Features available_features11{};
vk::PhysicalDeviceVulkan12Features available_features12{};
available_features12.pNext = &available_features11;
available_features12.sType =
vk::StructureType::ePhysicalDeviceVulkan12Features;
vk::PhysicalDeviceVulkan13Features available_features13{};
available_features13.sType =
vk::StructureType::ePhysicalDeviceVulkan13Features;
available_features13.pNext = &available_features12;
vk::PhysicalDeviceComputeShaderDerivativesFeaturesKHR available_derivatives{};
available_derivatives.pNext = &available_features13;
vk::PhysicalDeviceDepthClipEnableFeaturesEXT available_depth_clip{};
available_depth_clip.pNext = &available_derivatives;
vk::PhysicalDeviceDepthClipControlFeaturesEXT available_clip_control{};
available_clip_control.pNext = &available_depth_clip;
vk::PhysicalDeviceColorWriteEnableFeaturesEXT available_color_write{};
available_color_write.pNext = &available_clip_control;
vk::PhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT available_feedback_layout{};
available_feedback_layout.pNext = &available_color_write;
vk::PhysicalDeviceAttachmentFeedbackLoopDynamicStateFeaturesEXT available_feedback_dynamic{};
available_feedback_dynamic.pNext = &available_feedback_layout;
vk::PhysicalDeviceProvokingVertexFeaturesEXT available_provoking_vertex{};
available_provoking_vertex.pNext = &available_feedback_dynamic;
vk::PhysicalDeviceImageViewMinLodFeaturesEXT available_min_lod{};
available_min_lod.pNext = &available_provoking_vertex;
vk::PhysicalDeviceFeatures2 available_features2{};
available_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
available_features2.pNext = &available_min_lod;
m_physical_device.getFeatures2(&available_features2);
Require("VulkanHarness", "dispatch",
available_features.shaderStorageImageWriteWithoutFormat == true,
"shaderStorageImageWriteWithoutFormat is not supported");
Require("VulkanHarness", "dispatch",
available_features.shaderImageGatherExtended == true &&
available_derivatives.computeDerivativeGroupQuads == true,
"image gather or compute derivative quads are not supported");
Require("VulkanHarness", "dispatch",
available_features11.storageBuffer16BitAccess == true &&
available_features12.storageBuffer8BitAccess == true,
"8-bit and 16-bit storage buffer access are not supported");
Require("VulkanHarness", "dispatch",
available_features12.timelineSemaphore == true,
"timeline semaphores are not supported");
Require("VulkanHarness", "dispatch",
available_features13.dynamicRendering == true,
"dynamic rendering is not supported");
Require("VulkanHarness", "dispatch",
available_features13.synchronization2 == true,
"synchronization2 is not supported");
Require("VulkanHarness", "dispatch",
available_features.sampleRateShading == true,
"sample-rate shading is not supported");
Require("VulkanHarness", "dispatch", available_features.shaderInt64 == true,
"shaderInt64 is not supported");
Require("VulkanHarness", "dispatch",
available_features12.bufferDeviceAddress == true,
"bufferDeviceAddress is not supported");
Require("VulkanHarness", "dispatch",
available_features12.shaderSampledImageArrayNonUniformIndexing == true,
"nonuniform sampled image indexing is not supported");
Require("VulkanHarness", "dispatch", available_min_lod.minLod == true,
"image view minimum LOD is not supported");
Require("VulkanHarness", "graphics", available_features12.shaderOutputLayer == true,
"vertex layer output is not supported");
m_rasterization_supported = available_features.fillModeNonSolid &&
available_features.tessellationShader &&
available_features.depthBounds &&
available_depth_clip.depthClipEnable &&
available_clip_control.depthClipControl &&
available_color_write.colorWriteEnable &&
available_feedback_layout.attachmentFeedbackLoopLayout &&
available_feedback_dynamic.attachmentFeedbackLoopDynamicState &&
available_provoking_vertex.provokingVertexLast;
if (!m_rasterization_supported) {
std::printf(
"[host] ProductionRasterization unavailable, rasterization cases "
"will be skipped\n");
}
float priority = 1.0f;
vk::DeviceQueueCreateInfo queue_info{};
queue_info.sType = vk::StructureType::eDeviceQueueCreateInfo;
queue_info.queueFamilyIndex = m_queue_family;
queue_info.queueCount = 1;
queue_info.pQueuePriorities = &priority;
vk::DeviceCreateInfo device_info{};
device_info.sType = vk::StructureType::eDeviceCreateInfo;
device_info.queueCreateInfoCount = 1;
device_info.pQueueCreateInfos = &queue_info;
auto device_features11 = WindowContext::RequiredVulkan11Features();
auto device_features12 = WindowContext::RequiredVulkan12Features();
device_features12.shaderSharedInt64Atomics = true;
vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{};
workgroup_layout.workgroupMemoryExplicitLayout = true;
device_features12.pNext = &device_features11;
device_features11.pNext = &workgroup_layout;
vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{};
image_atomic64.shaderImageInt64Atomics = true;
workgroup_layout.pNext = &image_atomic64;
vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{};
barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
barycentric.pNext = &device_features12;
barycentric.fragmentShaderBarycentric = true;
vk::PhysicalDeviceVulkan13Features device_features13{};
device_features13.sType =
vk::StructureType::ePhysicalDeviceVulkan13Features;
device_features13.pNext = &barycentric;
device_features13.dynamicRendering = true;
device_features13.synchronization2 = true;
vk::PhysicalDeviceComputeShaderDerivativesFeaturesKHR derivatives{};
derivatives.pNext = &device_features13;
derivatives.computeDerivativeGroupQuads = true;
// Requesting a feature the device does not support fails device creation, so the
// rasterization feature chain is only chained in when every part is available.
vk::PhysicalDeviceDepthClipEnableFeaturesEXT depth_clip{};
vk::PhysicalDeviceDepthClipControlFeaturesEXT clip_control{};
vk::PhysicalDeviceColorWriteEnableFeaturesEXT color_write{};
vk::PhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT feedback_layout{};
vk::PhysicalDeviceAttachmentFeedbackLoopDynamicStateFeaturesEXT feedback_dynamic{};
vk::PhysicalDeviceProvokingVertexFeaturesEXT provoking_vertex{};
if (m_rasterization_supported) {
depth_clip.pNext = &derivatives;
depth_clip.depthClipEnable = true;
clip_control.pNext = &depth_clip;
clip_control.depthClipControl = true;
color_write.pNext = &clip_control;
color_write.colorWriteEnable = true;
feedback_layout.pNext = &color_write;
feedback_layout.attachmentFeedbackLoopLayout = true;
feedback_dynamic.pNext = &feedback_layout;
feedback_dynamic.attachmentFeedbackLoopDynamicState = true;
provoking_vertex.pNext = &feedback_dynamic;
provoking_vertex.provokingVertexLast = available_provoking_vertex.provokingVertexLast;
}
vk::PhysicalDeviceImageViewMinLodFeaturesEXT min_lod{};
min_lod.minLod = true;
min_lod.pNext = m_rasterization_supported
? static_cast<void *>(&provoking_vertex)
: static_cast<void *>(&derivatives);
device_info.pNext = &min_lod;
vk::PhysicalDeviceFeatures device_features{};
device_features.shaderStorageImageWriteWithoutFormat = true;
device_features.shaderImageGatherExtended = true;
device_features.sampleRateShading = true;
device_features.shaderInt64 = true;
device_features.shaderFloat64 = available_features.shaderFloat64;
device_features.fillModeNonSolid = m_rasterization_supported;
device_features.tessellationShader = m_rasterization_supported;
device_features.depthBounds = m_rasterization_supported;
device_info.pEnabledFeatures = &device_features;
std::vector<const char *> device_extensions{
VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME,
VK_KHR_COMPUTE_SHADER_DERIVATIVES_EXTENSION_NAME,
VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME,
VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME,
VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME,
VK_EXT_IMAGE_VIEW_MIN_LOD_EXTENSION_NAME};
if (m_rasterization_supported) {
device_extensions.push_back(VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME);
device_extensions.push_back(VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME);
device_extensions.push_back(VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
device_extensions.push_back(VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME);
device_extensions.push_back(
VK_EXT_ATTACHMENT_FEEDBACK_LOOP_DYNAMIC_STATE_EXTENSION_NAME);
device_extensions.push_back(VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME);
}
device_info.enabledExtensionCount =
static_cast<uint32_t>(device_extensions.size());
device_info.ppEnabledExtensionNames = device_extensions.data();
RequireVk("VulkanHarness", "dispatch",
m_physical_device.createDevice(&device_info, nullptr, &m_device),
"vkCreateDevice");
VULKAN_HPP_DEFAULT_DISPATCHER.init(m_device);
m_device.getQueue(m_queue_family, 0, &m_queue);
vk::CommandPoolCreateInfo pool_info{};
pool_info.sType = vk::StructureType::eCommandPoolCreateInfo;
pool_info.queueFamilyIndex = m_queue_family;
pool_info.flags = vk::CommandPoolCreateFlagBits::eResetCommandBuffer;
RequireVk("VulkanHarness", "dispatch",
m_device.createCommandPool(&pool_info, nullptr, &m_command_pool),
"vkCreateCommandPool");
}
void Destroy() {
if (m_device != nullptr) {
RequireVulkanSuccess(m_device.waitIdle(), "vkDeviceWaitIdle");
DestroyBuffer(&m_fault_buffer);
DestroyBuffer(&m_bda_pagetable_buffer);
if (m_runtime_context.allocator != nullptr) {
m_renderer.reset();
vmaDestroyAllocator(m_runtime_context.allocator);
m_runtime_context.allocator = nullptr;
}
if (m_command_pool != nullptr) {
m_device.destroyCommandPool(m_command_pool, nullptr);
}
m_device.destroy(nullptr);
}
if (m_instance != nullptr) {
m_instance.destroy(nullptr);
}
}
bool FindMemoryType(u32 type_bits, vk::MemoryPropertyFlags required,
u32 *index) const {
for (u32 i = 0; i < m_memory_properties.memoryTypeCount; i++) {
if ((type_bits & (1u << i)) == 0) {
continue;
}
if ((m_memory_properties.memoryTypes[i].propertyFlags & required) ==
required) {
*index = i;
return true;
}
}
return false;
}
static vk::AccessFlags AccessForLayout(vk::ImageLayout layout) {
switch (layout) {
case vk::ImageLayout::eTransferDstOptimal:
return vk::AccessFlagBits::eTransferWrite;
case vk::ImageLayout::eTransferSrcOptimal:
return vk::AccessFlagBits::eTransferRead;
case vk::ImageLayout::eShaderReadOnlyOptimal:
return vk::AccessFlagBits::eShaderRead;
case vk::ImageLayout::eGeneral:
return vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite;
default:
return {};
}
}
vk::ShaderModule CreateShaderModule(const char *shader_name, const std::vector<u32> &spirv) {
vk::ShaderModuleCreateInfo info{};
info.codeSize = spirv.size() * sizeof(u32);
info.pCode = spirv.data();
vk::ShaderModule module{};
RequireVk(shader_name, "graphics", m_device.createShaderModule(&info, nullptr, &module),
"vkCreateShaderModule");
return module;
}
vk::CommandBuffer BeginCommands(const char *shader_name, const char *stage) {
vk::CommandBufferAllocateInfo cmd_alloc{};
cmd_alloc.sType = vk::StructureType::eCommandBufferAllocateInfo;
cmd_alloc.commandPool = m_command_pool;
cmd_alloc.level = vk::CommandBufferLevel::ePrimary;
cmd_alloc.commandBufferCount = 1;
vk::CommandBuffer cmd = nullptr;
RequireVk(shader_name, stage,
m_device.allocateCommandBuffers(&cmd_alloc, &cmd),
"vkAllocateCommandBuffers");
vk::CommandBufferBeginInfo begin{};
begin.sType = vk::StructureType::eCommandBufferBeginInfo;
begin.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit;
RequireVk(shader_name, stage, cmd.begin(&begin), "vkBeginCommandBuffer");
return cmd;
}
void EndSubmitAndFree(const char *shader_name, const char *stage,
vk::CommandBuffer cmd) {
RequireVk(shader_name, stage, cmd.end(), "vkEndCommandBuffer");
vk::FenceCreateInfo fence_info{};
fence_info.sType = vk::StructureType::eFenceCreateInfo;
vk::Fence fence = nullptr;
RequireVk(shader_name, stage,
m_device.createFence(&fence_info, nullptr, &fence),
"vkCreateFence");
vk::SubmitInfo submit{};
submit.sType = vk::StructureType::eSubmitInfo;
submit.commandBufferCount = 1;
submit.pCommandBuffers = &cmd;
RequireVk(shader_name, stage, m_queue.submit(1, &submit, fence),
"vkQueueSubmit");
RequireVk(shader_name, stage,
m_device.waitForFences(1, &fence, true, UINT64_MAX),
"vkWaitForFences");
m_device.destroyFence(fence, nullptr);
m_device.freeCommandBuffers(m_command_pool, 1, &cmd);
}
void AddImageBarrier(vk::CommandBuffer cmd, vk::Image image,
vk::ImageLayout old_layout, vk::ImageLayout new_layout,
vk::PipelineStageFlags src_stage,
vk::PipelineStageFlags dst_stage,
vk::AccessFlags src_access, vk::AccessFlags dst_access,
u32 mip_levels = 1, u32 layers = 1) {
vk::ImageMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eImageMemoryBarrier;
barrier.srcAccessMask = src_access;
barrier.dstAccessMask = dst_access;
barrier.oldLayout = old_layout;
barrier.newLayout = new_layout;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.image = image;
barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor;
barrier.subresourceRange.baseMipLevel = 0;
barrier.subresourceRange.levelCount = mip_levels;
barrier.subresourceRange.baseArrayLayer = 0;
barrier.subresourceRange.layerCount = layers;
cmd.pipelineBarrier(src_stage, dst_stage, {}, 0, nullptr, 0, nullptr, 1,
&barrier);
}
void TransitionImage(const char *shader_name, Image *image,
vk::ImageLayout new_layout,
vk::PipelineStageFlags src_stage,
vk::PipelineStageFlags dst_stage,
vk::AccessFlags src_access, vk::AccessFlags dst_access) {
vk::CommandBuffer cmd = BeginCommands(shader_name, "dispatch");
AddImageBarrier(cmd, image->image, image->layout, new_layout, src_stage,
dst_stage, src_access, dst_access, image->mip_levels,
image->layers);
EndSubmitAndFree(shader_name, "dispatch", cmd);
image->layout = new_layout;
}
void UploadImage(const char *shader_name, Image *image, vk::Buffer staging,
vk::ImageLayout final_layout) {
UploadImageMips(shader_name, image, staging, final_layout);
}
void UploadImageMips(const char *shader_name, Image *image,
vk::Buffer staging, vk::ImageLayout final_layout) {
vk::CommandBuffer cmd = BeginCommands(shader_name, "dispatch");
AddImageBarrier(cmd, image->image, vk::ImageLayout::eUndefined,
vk::ImageLayout::eTransferDstOptimal,
vk::PipelineStageFlagBits::eTopOfPipe,
vk::PipelineStageFlagBits::eTransfer, {},
vk::AccessFlagBits::eTransferWrite, image->mip_levels,
image->layers);
std::vector<vk::BufferImageCopy> copies;
copies.reserve(image->mip_levels);
vk::DeviceSize offset = 0;
for (u32 level = 0; level < image->mip_levels; level++) {
vk::BufferImageCopy copy{};
copy.bufferOffset = offset;
copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
copy.imageSubresource.mipLevel = level;
copy.imageSubresource.baseArrayLayer = 0;
copy.imageSubresource.layerCount = image->layers;
copy.imageExtent.width = MipExtent(image->width, level);
copy.imageExtent.height = MipExtent(image->height, level);
copy.imageExtent.depth = 1;
copies.push_back(copy);
offset += static_cast<vk::DeviceSize>(
ImageMipDwordCount(image->width, image->height,
image->dwords_per_pixel, level, image->layers) *
sizeof(u32));
}
cmd.copyBufferToImage(staging, image->image,
vk::ImageLayout::eTransferDstOptimal,
static_cast<u32>(copies.size()), copies.data());
AddImageBarrier(cmd, image->image, vk::ImageLayout::eTransferDstOptimal,
final_layout, vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eComputeShader,
vk::AccessFlagBits::eTransferWrite,
AccessForLayout(final_layout), image->mip_levels,
image->layers);
EndSubmitAndFree(shader_name, "dispatch", cmd);
image->layout = final_layout;
}
Buffer CreateDeviceBuffer(const char *shader_name, vk::DeviceSize size,
vk::BufferUsageFlags usage) {
Buffer ret;
ret.size = size;
vk::BufferCreateInfo buffer_info{};
buffer_info.sType = vk::StructureType::eBufferCreateInfo;
buffer_info.size = size;
buffer_info.usage = usage;
buffer_info.sharingMode = vk::SharingMode::eExclusive;
RequireVk(shader_name, "dispatch",
m_device.createBuffer(&buffer_info, nullptr, &ret.buffer),
"vkCreateBuffer");
vk::MemoryRequirements requirements{};
m_device.getBufferMemoryRequirements(ret.buffer, &requirements);
u32 memory_type = 0;
Require(shader_name, "dispatch",
FindMemoryType(requirements.memoryTypeBits,
vk::MemoryPropertyFlagBits::eDeviceLocal,
&memory_type) ||
FindMemoryType(requirements.memoryTypeBits, {}, &memory_type),
"no memory type for BDA support buffer");
vk::MemoryAllocateInfo allocation{};
allocation.sType = vk::StructureType::eMemoryAllocateInfo;
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = memory_type;
RequireVk(shader_name, "dispatch",
m_device.allocateMemory(&allocation, nullptr, &ret.memory),
"vkAllocateMemory");
RequireVk(shader_name, "dispatch",
m_device.bindBufferMemory(ret.buffer, ret.memory, 0),
"vkBindBufferMemory");
return ret;
}
void EnsureBdaBuffers(const char *shader_name) {
if (m_bda_pagetable_buffer.buffer != nullptr) {
return;
}
const auto usage = vk::BufferUsageFlagBits::eStorageBuffer |
vk::BufferUsageFlagBits::eTransferDst;
m_bda_pagetable_buffer =
CreateDeviceBuffer(shader_name, BufferCache::BDA_PAGETABLE_SIZE, usage);
m_fault_buffer = CreateDeviceBuffer(
shader_name, BufferCache::CACHING_NUMPAGES / 8, usage);
}
Buffer CreateHostBuffer(const char *shader_name, vk::DeviceSize size,
vk::BufferUsageFlags usage,
const std::vector<u32> &contents) {
Buffer ret;
ret.size = std::max<vk::DeviceSize>(size, sizeof(u32));
vk::BufferCreateInfo buffer_info{};
buffer_info.sType = vk::StructureType::eBufferCreateInfo;
buffer_info.size = ret.size;
buffer_info.usage = usage;
buffer_info.sharingMode = vk::SharingMode::eExclusive;
RequireVk(shader_name, "dispatch",
m_device.createBuffer(&buffer_info, nullptr, &ret.buffer),
"vkCreateBuffer");
vk::MemoryRequirements req{};
m_device.getBufferMemoryRequirements(ret.buffer, &req);
u32 memory_type = 0;
ret.coherent = FindMemoryType(req.memoryTypeBits,
vk::MemoryPropertyFlagBits::eHostVisible |
vk::MemoryPropertyFlagBits::eHostCoherent,
&memory_type);
if (!ret.coherent) {
Require(shader_name, "dispatch",
FindMemoryType(req.memoryTypeBits,
vk::MemoryPropertyFlagBits::eHostVisible,
&memory_type),
"no host-visible memory type for staging buffer");
}
vk::MemoryAllocateInfo alloc{};
alloc.sType = vk::StructureType::eMemoryAllocateInfo;
alloc.allocationSize = req.size;
alloc.memoryTypeIndex = memory_type;
RequireVk(shader_name, "dispatch",
m_device.allocateMemory(&alloc, nullptr, &ret.memory),
"vkAllocateMemory");
RequireVk(shader_name, "dispatch",
m_device.bindBufferMemory(ret.buffer, ret.memory, 0),
"vkBindBufferMemory");
if (!contents.empty()) {
WriteBuffer(shader_name, ret, contents);
}
return ret;
}
void WriteBuffer(const char *shader_name, const Buffer &buffer,
const std::vector<u32> &contents) {
void *data = nullptr;
RequireVk(shader_name, "dispatch",
m_device.mapMemory(buffer.memory, 0, buffer.size, {}, &data),
"vkMapMemory");
std::memcpy(data, contents.data(), contents.size() * sizeof(u32));
if (!buffer.coherent) {
vk::MappedMemoryRange range{};
range.sType = vk::StructureType::eMappedMemoryRange;
range.memory = buffer.memory;
range.offset = 0;
range.size = VK_WHOLE_SIZE;
RequireVk(shader_name, "dispatch",
m_device.flushMappedMemoryRanges(1, &range),
"vkFlushMappedMemoryRanges");
}
m_device.unmapMemory(buffer.memory);
}
vk::Instance m_instance = nullptr;
vk::PhysicalDevice m_physical_device = nullptr;
vk::Device m_device = nullptr;
vk::Queue m_queue = nullptr;
vk::CommandPool m_command_pool = nullptr;
u32 m_queue_family = 0;
vk::PhysicalDeviceMemoryProperties m_memory_properties{};
Buffer m_bda_pagetable_buffer;
Buffer m_fault_buffer;
GraphicContext m_runtime_context{};
std::unique_ptr<RenderContext> m_renderer;
};
void CompareWords(const TestCase &test, const char *stage,
const std::vector<u32> &expected,
const std::vector<u32> &actual) {
if (actual == expected) {
return;
}
if (test.expected_float_tolerance > 0.0f &&
std::equal(expected.begin(), expected.end(), actual.begin(), actual.end(),
[&](u32 expected_word, u32 actual_word) {
return std::abs(std::bit_cast<float>(expected_word) -
std::bit_cast<float>(actual_word)) <=
test.expected_float_tolerance;
})) {
return;
}
std::ostringstream out;
out << "expected [";
for (size_t i = 0; i < expected.size(); i++) {
out << (i == 0 ? "" : ", ") << Hex(expected[i]);
}
out << "] actual [";
for (size_t i = 0; i < actual.size(); i++) {
out << (i == 0 ? "" : ", ") << Hex(actual[i]);
}
out << "]";
Fail(test.name, stage, out.str());
}
void CompareGraphicsWords(const GraphicsCase &test,
const std::vector<u32> &actual) {
if (actual == test.expected_pixel) {
return;
}
std::ostringstream out;
out << "expected [";
for (size_t i = 0; i < test.expected_pixel.size(); i++) {
out << (i == 0 ? "" : ", ") << Hex(test.expected_pixel[i]);
}
out << "] actual [";
for (size_t i = 0; i < actual.size(); i++) {
out << (i == 0 ? "" : ", ") << Hex(actual[i]);
}
out << "]";
Fail(test.name, "graphics readback", out.str());
}
void RunCase(VulkanHarness *vulkan, const TestCase &test) {
auto compiled = CompileCase(test, vulkan != nullptr ? vulkan->SubgroupSize() : 64u);
if (test.image_descriptor_swizzle != DstSel(4, 5, 6, 7) &&
!compiled.program.info.images.empty()) {
Require(test.name, "resource specialization",
compiled.program.info.images[0].shader_swizzle ==
test.image_descriptor_swizzle,
"storage image descriptor swizzle did not reach the specialized "
"program");
}
if (test.expected_force_point_sampler) {
Require(test.name, "sampler specialization",
compiled.program.info.samplers.size() == 1u &&
compiled.program.info.samplers[0].force_point_filtering,
"bit-packed sampled image did not force point filtering");
}
if (test.sampler_filter == vk::Filter::eLinear) {
Require(test.name, "sampler specialization",
std::ranges::none_of(compiled.program.info.samplers,
[](const auto &sampler) { return sampler.force_point_filtering; }),
"filterable sampled image incorrectly forced point filtering");
}
if (test.compile_only) {
std::printf("[compute] %-32s ok\n", test.name);
return;
}
using Kind = ShaderRecompiler::IR::DescriptorBindingKind;
auto Has = [&](Kind kind) {
return compiled.program.bindings.descriptor_counts[static_cast<size_t>(kind)] != 0;
};
const auto dwords = std::max<size_t>(
{test.initial.size(), test.expected.size(), static_cast<size_t>(1)});
auto buffer = vulkan->CreateStorageBuffer(
test.name, test.initial, dwords, Has(Kind::BdaPagetable));
VulkanHarness::Image sampled_image;
VulkanHarness::Image storage_image;
VulkanHarness::Image storage_image_uint;
VulkanHarness::Buffer gds_buffer;
vk::Sampler sampler = nullptr;
bool needs_sampled_image = false;
bool needs_storage_image = false;
for (const auto &image : compiled.program.info.images) {
const auto resource_class = image.resource_class;
if (resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) {
needs_sampled_image = true;
} else if (resource_class ==
ShaderRecompiler::IR::ImageResourceClass::Storage) {
needs_storage_image = true;
}
}
const bool needs_sampler = Has(Kind::Samplers);
const bool needs_gds = Has(Kind::Gds);
if (needs_gds) {
const auto gds_dwords = std::max<size_t>(
{test.gds_initial.size(), test.expected_gds.size(), 1u});
gds_buffer =
vulkan->CreateStorageBuffer(test.name, test.gds_initial, gds_dwords);
}
if (needs_sampled_image) {
auto sampled_mips = test.sampled_image_rgba_mips;
auto sampled_format = test.sampled_image_format;
auto sampled_dwords_per_pixel = test.sampled_image_dwords_per_pixel;
if (!test.sampled_image_rgba_mips.empty()) {
sampled_format = vk::Format::eR32G32B32A32Sfloat;
sampled_dwords_per_pixel = 4;
} else if (!test.sampled_image_rgba.empty()) {
sampled_mips.push_back(test.sampled_image_rgba);
}
sampled_image = vulkan->CreateImageMips(
test.name, test.image_width, test.image_height, sampled_format,
vk::ImageUsageFlagBits::eSampled, sampled_mips,
sampled_dwords_per_pixel, vk::ImageLayout::eShaderReadOnlyOptimal,
test.sampled_image_type, test.sampled_image_view_type,
test.sampled_image_layers, test.sampled_image_view_base_layer,
test.sampled_image_view_layers, vk::SampleCountFlagBits::e1,
vk::Format::eUndefined, test.sampled_image_view_base_mip);
}
if (needs_storage_image) {
const bool atomic64 = std::ranges::any_of(
compiled.program.info.images,
[](const auto &image) { return image.atomic64; });
storage_image = vulkan->CreateImage2D(
test.name, test.image_width, test.image_height,
test.storage_image_format, vk::ImageUsageFlagBits::eStorage,
test.storage_image_rgba, test.storage_image_dwords_per_pixel,
vk::ImageLayout::eGeneral);
storage_image_uint = vulkan->CreateImage2D(
test.name, test.image_width, test.image_height,
atomic64 ? vk::Format::eR32G32Uint : vk::Format::eR32Uint,
vk::ImageUsageFlagBits::eStorage, test.storage_image_r32ui, atomic64 ? 2 : 1,
vk::ImageLayout::eGeneral,
atomic64 ? vk::Format::eR64Uint : vk::Format::eR32Uint);
}
if (needs_sampler) {
sampler = vulkan->CreateSampler(
test.name, sampled_image.mip_levels - test.sampled_image_view_base_mip,
test.sampler_filter);
}
vulkan->Dispatch(test, compiled, buffer, needs_gds ? &gds_buffer : nullptr,
needs_sampled_image ? &sampled_image : nullptr,
needs_storage_image ? &storage_image : nullptr,
needs_storage_image ? &storage_image_uint : nullptr,
sampler);
auto actual = vulkan->ReadBuffer(test.name, buffer, test.expected.size());
if (!test.expected_gds.empty()) {
const auto gds_actual =
vulkan->ReadBuffer(test.name, gds_buffer, test.expected_gds.size());
CompareWords(test, "GDS readback", test.expected_gds, gds_actual);
}
if (!test.expected_storage_image_rgba.empty()) {
auto image_actual = vulkan->ReadImage(test.name, &storage_image);
image_actual.resize(test.expected_storage_image_rgba.size());
CompareWords(test, "storage image readback",
test.expected_storage_image_rgba, image_actual);
}
if (!test.expected_storage_image_r32ui.empty()) {
auto image_actual = vulkan->ReadImage(test.name, &storage_image_uint);
image_actual.resize(test.expected_storage_image_r32ui.size());
CompareWords(test, "uint storage image readback",
test.expected_storage_image_r32ui, image_actual);
}
if (sampler != nullptr) {
vulkan->Device().destroySampler(sampler, nullptr);
}
vulkan->DestroyImage(&sampled_image);
vulkan->DestroyImage(&storage_image);
vulkan->DestroyImage(&storage_image_uint);
vulkan->DestroyBuffer(&gds_buffer);
vulkan->DestroyBuffer(&buffer);
CompareWords(test, "readback", test.expected, actual);
std::printf("[compute] %-32s ok\n", test.name);
}
void RunGraphicsCase(VulkanHarness *vulkan, const GraphicsCase &test) {
if (vulkan != nullptr && !vulkan->RasterizationSupported()) {
vulkan->SkipRasterizationCases(1);
return;
}
auto compiled = CompileFragmentCase(test);
auto actual = vulkan->RenderFragment(test, compiled);
CompareGraphicsWords(test, actual);
std::printf("[graphics] %-31s ok\n", test.name);
}
enum class CoverageClass {
Covered,
ControlOrMarker,
NeedsAluCase,
NeedsFloatCase,
NeedsMemoryCase,
NeedsImageCase,
NeedsGraphicsStageCase,
};
CoverageClass ClassifyOpcode(ShaderOpcode opcode,
const std::set<ShaderOpcode> &covered) {
using ShaderRecompiler::Decoder::Opcode;
if (covered.contains(opcode)) {
return CoverageClass::Covered;
}
switch (opcode) {
case Opcode::S_GETPC_B64:
case Opcode::S_SETPC_B64:
case Opcode::S_NOP:
case Opcode::S_WAITCNT:
case Opcode::S_BARRIER:
case Opcode::S_BRANCH:
case Opcode::S_CBRANCH_SCC0:
case Opcode::S_CBRANCH_SCC1:
case Opcode::S_CBRANCH_VCCZ:
case Opcode::S_CBRANCH_VCCNZ:
case Opcode::S_CBRANCH_EXECZ:
case Opcode::S_CBRANCH_EXECNZ:
case Opcode::S_SENDMSG:
case Opcode::S_SETREG_B32:
case Opcode::S_SLEEP:
case Opcode::S_TTRACEDATA:
case Opcode::S_INST_PREFETCH:
case Opcode::S_ENDPGM:
return CoverageClass::ControlOrMarker;
case Opcode::V_ADD_F32:
case Opcode::V_SUB_F32:
case Opcode::V_SUBREV_F32:
case Opcode::V_MUL_F32:
case Opcode::V_MAC_F32:
case Opcode::V_MADMK_F32:
case Opcode::V_MADAK_F32:
case Opcode::V_FMAC_F32:
case Opcode::V_FMAMK_F32:
case Opcode::V_FMAAK_F32:
case Opcode::V_MIN_F32:
case Opcode::V_MAX_F32:
case Opcode::V_MAD_F32:
case Opcode::V_FMA_F32:
case Opcode::V_MIN3_F32:
case Opcode::V_MAX3_F32:
case Opcode::V_MED3_F32:
case Opcode::V_DOT2C_F32_F16:
case Opcode::V_CVT_F64_I32:
case Opcode::V_CVT_F32_F64:
case Opcode::V_CVT_F64_F32:
case Opcode::V_CVT_F64_U32:
case Opcode::V_RCP_F64:
case Opcode::V_FRACT_F64:
case Opcode::V_MUL_F64:
case Opcode::V_FMA_F64:
case Opcode::V_CVT_F32_I32:
case Opcode::V_CVT_F32_U32:
case Opcode::V_CVT_U32_F32:
case Opcode::V_CVT_I32_F32:
case Opcode::V_CVT_F16_F32:
case Opcode::V_CVT_F32_F16:
case Opcode::V_CVT_U16_F16:
case Opcode::V_CVT_RPI_I32_F32:
case Opcode::V_CVT_FLR_I32_F32:
case Opcode::V_CVT_OFF_F32_I4:
case Opcode::V_CVT_F32_UBYTE0:
case Opcode::V_CVT_F32_UBYTE1:
case Opcode::V_CVT_F32_UBYTE2:
case Opcode::V_CVT_F32_UBYTE3:
case Opcode::V_RCP_F32:
case Opcode::V_FRACT_F32:
case Opcode::V_TRUNC_F32:
case Opcode::V_CEIL_F32:
case Opcode::V_RNDNE_F32:
case Opcode::V_FLOOR_F32:
case Opcode::V_EXP_F32:
case Opcode::V_LOG_F32:
case Opcode::V_RSQ_F32:
case Opcode::V_SQRT_F32:
case Opcode::V_SIN_F32:
case Opcode::V_COS_F32:
case Opcode::V_CUBEID_F32:
case Opcode::V_CUBESC_F32:
case Opcode::V_CUBETC_F32:
case Opcode::V_CUBEMA_F32:
case Opcode::V_LDEXP_F32:
case Opcode::V_CVT_PK_U8_F32:
case Opcode::V_CVT_PKNORM_I16_F32:
case Opcode::V_CVT_PKNORM_U16_F32:
case Opcode::V_CVT_PKRTZ_F16_F32:
case Opcode::V_PK_ADD_F16:
case Opcode::V_PK_MUL_F16:
case Opcode::V_PK_MIN_F16:
case Opcode::V_PK_MAX_F16:
case Opcode::V_PK_FMA_F16:
case Opcode::V_ADD_F16:
case Opcode::V_SUB_F16:
case Opcode::V_SUBREV_F16:
case Opcode::V_MUL_F16:
case Opcode::V_MAX_F16:
case Opcode::V_MIN_F16:
case Opcode::V_MIN3_F16:
case Opcode::V_MAX3_F16:
case Opcode::V_MED3_F16:
case Opcode::V_RCP_F16:
case Opcode::V_RSQ_F16:
case Opcode::V_LOG_F16:
case Opcode::V_EXP_F16:
case Opcode::V_SIN_F16:
case Opcode::V_COS_F16:
case Opcode::V_MAD_MIXLO_F16:
case Opcode::V_MAD_MIXHI_F16:
case Opcode::DS_MIN_F32:
case Opcode::DS_MAX_F32:
case Opcode::V_CMP_F_F32:
case Opcode::V_CMP_LT_F32:
case Opcode::V_CMP_EQ_F32:
case Opcode::V_CMP_LE_F32:
case Opcode::V_CMP_GT_F32:
case Opcode::V_CMP_LG_F32:
case Opcode::V_CMP_GE_F32:
case Opcode::V_CMP_O_F32:
case Opcode::V_CMP_U_F32:
case Opcode::V_CMP_NGE_F32:
case Opcode::V_CMP_NLG_F32:
case Opcode::V_CMP_NGT_F32:
case Opcode::V_CMP_NLE_F32:
case Opcode::V_CMP_NEQ_F32:
case Opcode::V_CMP_NLT_F32:
case Opcode::V_CMP_TRU_F32:
case Opcode::V_CMPX_LT_F32:
case Opcode::V_CMPX_EQ_F32:
case Opcode::V_CMPX_LE_F32:
case Opcode::V_CMPX_GT_F32:
case Opcode::V_CMPX_LG_F32:
case Opcode::V_CMPX_GE_F32:
case Opcode::V_CMPX_O_F32:
case Opcode::V_CMPX_NGE_F32:
case Opcode::V_CMPX_NLG_F32:
case Opcode::V_CMPX_NGT_F32:
case Opcode::V_CMPX_NLE_F32:
case Opcode::V_CMPX_NEQ_F32:
case Opcode::V_CMPX_NLT_F32:
case Opcode::V_CMP_CLASS_F32:
case Opcode::V_CMPX_CLASS_F32:
case Opcode::V_CMP_CLASS_F16:
case Opcode::V_CMPX_CLASS_F16:
case Opcode::V_CMP_LT_F16:
case Opcode::V_CMP_EQ_F16:
case Opcode::V_CMP_LE_F16:
case Opcode::V_CMP_GT_F16:
case Opcode::V_CMP_LG_F16:
case Opcode::V_CMP_GE_F16:
case Opcode::V_CMP_NGT_F16:
case Opcode::V_CMP_NEQ_F16:
case Opcode::V_CMP_NLT_F16:
case Opcode::V_CMPX_LT_F16:
case Opcode::V_CMPX_EQ_F16:
case Opcode::V_CMPX_LE_F16:
case Opcode::V_CMPX_GT_F16:
case Opcode::V_CMPX_GE_F16:
case Opcode::V_CMPX_NEQ_F16:
case Opcode::V_CMPX_NLT_F16:
return CoverageClass::NeedsFloatCase;
case Opcode::S_LOAD_DWORD:
case Opcode::S_LOAD_DWORDX2:
case Opcode::S_LOAD_DWORDX4:
case Opcode::S_LOAD_DWORDX8:
case Opcode::S_LOAD_DWORDX16:
case Opcode::S_BUFFER_LOAD_DWORD:
case Opcode::S_BUFFER_LOAD_DWORDX2:
case Opcode::S_BUFFER_LOAD_DWORDX4:
case Opcode::S_BUFFER_LOAD_DWORDX8:
case Opcode::S_BUFFER_LOAD_DWORDX16:
case Opcode::BUFFER_LOAD_FORMAT_X:
case Opcode::BUFFER_LOAD_FORMAT_XY:
case Opcode::BUFFER_LOAD_FORMAT_XYZ:
case Opcode::BUFFER_LOAD_FORMAT_XYZW:
case Opcode::BUFFER_STORE_FORMAT_X:
case Opcode::BUFFER_STORE_FORMAT_XY:
case Opcode::BUFFER_STORE_FORMAT_XYZ:
case Opcode::BUFFER_STORE_FORMAT_XYZW:
case Opcode::BUFFER_LOAD_UBYTE:
case Opcode::BUFFER_LOAD_SBYTE:
case Opcode::BUFFER_LOAD_USHORT:
case Opcode::BUFFER_LOAD_SSHORT:
case Opcode::BUFFER_LOAD_DWORDX2:
case Opcode::BUFFER_LOAD_DWORDX3:
case Opcode::BUFFER_LOAD_DWORDX4:
case Opcode::BUFFER_STORE_BYTE:
case Opcode::BUFFER_STORE_SHORT:
case Opcode::BUFFER_STORE_DWORDX2:
case Opcode::BUFFER_STORE_DWORDX3:
case Opcode::BUFFER_STORE_DWORDX4:
case Opcode::TBUFFER_LOAD_FORMAT_X:
case Opcode::TBUFFER_LOAD_FORMAT_XY:
case Opcode::TBUFFER_LOAD_FORMAT_XYZ:
case Opcode::TBUFFER_LOAD_FORMAT_XYZW:
case Opcode::TBUFFER_STORE_FORMAT_X:
case Opcode::TBUFFER_STORE_FORMAT_XY:
case Opcode::TBUFFER_STORE_FORMAT_XYZ:
case Opcode::TBUFFER_STORE_FORMAT_XYZW:
case Opcode::BUFFER_ATOMIC_SWAP:
case Opcode::BUFFER_ATOMIC_CMPSWAP:
case Opcode::BUFFER_ATOMIC_ADD:
case Opcode::BUFFER_ATOMIC_SUB:
case Opcode::BUFFER_ATOMIC_SMIN:
case Opcode::BUFFER_ATOMIC_UMIN:
case Opcode::BUFFER_ATOMIC_SMAX:
case Opcode::BUFFER_ATOMIC_UMAX:
case Opcode::BUFFER_ATOMIC_AND:
case Opcode::BUFFER_ATOMIC_AND_X2:
case Opcode::BUFFER_ATOMIC_OR:
case Opcode::BUFFER_ATOMIC_XOR:
case Opcode::BUFFER_ATOMIC_FMIN:
case Opcode::BUFFER_ATOMIC_FMAX:
case Opcode::FLAT_LOAD_UBYTE:
case Opcode::FLAT_LOAD_SBYTE:
case Opcode::FLAT_LOAD_USHORT:
case Opcode::FLAT_LOAD_SSHORT:
case Opcode::FLAT_LOAD_SHORT_D16:
case Opcode::FLAT_LOAD_DWORD:
case Opcode::FLAT_LOAD_DWORDX2:
case Opcode::FLAT_LOAD_DWORDX3:
case Opcode::FLAT_LOAD_DWORDX4:
case Opcode::FLAT_STORE_BYTE:
case Opcode::FLAT_STORE_SHORT:
case Opcode::FLAT_STORE_DWORD:
case Opcode::FLAT_STORE_DWORDX2:
case Opcode::FLAT_STORE_DWORDX3:
case Opcode::FLAT_STORE_DWORDX4:
case Opcode::DS_ADD_U32:
case Opcode::DS_ADD_U64:
case Opcode::DS_ADD_RTN_U32:
case Opcode::DS_SUB_U32:
case Opcode::DS_SUB_RTN_U32:
case Opcode::DS_INC_RTN_U32:
case Opcode::DS_DEC_RTN_U32:
case Opcode::DS_MIN_I32:
case Opcode::DS_MIN_RTN_I32:
case Opcode::DS_MAX_I32:
case Opcode::DS_MAX_RTN_I32:
case Opcode::DS_MIN_U32:
case Opcode::DS_MIN_RTN_U32:
case Opcode::DS_MAX_U32:
case Opcode::DS_MAX_RTN_U32:
case Opcode::DS_AND_B32:
case Opcode::DS_AND_RTN_B32:
case Opcode::DS_OR_B32:
case Opcode::DS_OR_RTN_B32:
case Opcode::DS_XOR_B32:
case Opcode::DS_XOR_RTN_B32:
case Opcode::DS_MSKOR_B32:
case Opcode::DS_WRXCHG_RTN_B32:
case Opcode::DS_SWIZZLE_B32:
case Opcode::DS_PERMUTE_B32:
case Opcode::DS_BPERMUTE_B32:
case Opcode::DS_READ_I8:
case Opcode::DS_READ_U8:
case Opcode::DS_READ_I16:
case Opcode::DS_READ_U16:
case Opcode::DS_READ_U16_D16:
case Opcode::DS_READ_U16_D16_HI:
case Opcode::DS_READ2_B32:
case Opcode::DS_READ_B32:
case Opcode::DS_READ_B64:
case Opcode::DS_READ2_B64:
case Opcode::DS_READ_B96:
case Opcode::DS_READ_B128:
case Opcode::DS_WRITE_B8:
case Opcode::DS_WRITE_B16:
case Opcode::DS_WRITE_B8_D16_HI:
case Opcode::DS_WRITE_B16_D16_HI:
case Opcode::DS_WRITE2_B32:
case Opcode::DS_WRITE2ST64_B32:
case Opcode::DS_WRITE_B32:
case Opcode::DS_WRITE_B64:
case Opcode::DS_WRITE_B96:
case Opcode::DS_WRITE_B128:
case Opcode::DS_WRITE_ADDTID_B32:
case Opcode::DS_READ_ADDTID_B32:
return CoverageClass::NeedsMemoryCase;
case Opcode::IMAGE_GET_RESINFO:
case Opcode::IMAGE_GET_LOD:
case Opcode::IMAGE_LOAD:
case Opcode::IMAGE_LOAD_MIP:
case Opcode::IMAGE_STORE:
case Opcode::IMAGE_STORE_MIP:
case Opcode::IMAGE_ATOMIC_SWAP:
case Opcode::IMAGE_ATOMIC_ADD:
case Opcode::IMAGE_ATOMIC_SMIN:
case Opcode::IMAGE_ATOMIC_SMAX:
case Opcode::IMAGE_ATOMIC_UMIN:
case Opcode::IMAGE_ATOMIC_AND:
case Opcode::IMAGE_ATOMIC_OR:
case Opcode::IMAGE_ATOMIC_XOR:
case Opcode::IMAGE_ATOMIC_FMIN:
case Opcode::IMAGE_ATOMIC_FMAX:
case Opcode::IMAGE_SAMPLE:
case Opcode::IMAGE_GATHER4_LZ:
case Opcode::IMAGE_GATHER4_C:
case Opcode::IMAGE_GATHER4_C_LZ:
case Opcode::IMAGE_GATHER4_LZ_O:
case Opcode::IMAGE_GATHER4_C_O:
case Opcode::IMAGE_GATHER4_C_LZ_O:
return CoverageClass::NeedsImageCase;
case Opcode::V_INTERP_P1_F32:
case Opcode::V_INTERP_P2_F32:
case Opcode::V_INTERP_MOV_F32:
case Opcode::EXP:
return CoverageClass::NeedsGraphicsStageCase;
default:
return CoverageClass::NeedsAluCase;
}
}
const char *CoverageClassName(CoverageClass status) {
switch (status) {
case CoverageClass::Covered:
return "covered";
case CoverageClass::ControlOrMarker:
return "control";
case CoverageClass::NeedsAluCase:
return "alu";
case CoverageClass::NeedsFloatCase:
return "float";
case CoverageClass::NeedsMemoryCase:
return "memory";
case CoverageClass::NeedsImageCase:
return "image";
case CoverageClass::NeedsGraphicsStageCase:
return "graphics";
default:
return "unknown";
}
}
void PrintPendingOpcodes(CoverageClass status,
const std::vector<ShaderOpcode> &opcodes) {
if (opcodes.empty()) {
return;
}
std::printf("[coverage] pending_%s:", CoverageClassName(status));
for (auto opcode : opcodes) {
const auto name = magic_enum::enum_name(opcode);
std::printf(" %.*s", static_cast<int>(name.size()), name.data());
}
std::printf("\n");
}
void CheckOpcodeCoverage(const std::vector<TestCase> &tests,
const std::vector<GraphicsCase> &graphics_tests) {
using ShaderRecompiler::Decoder::Opcode;
std::set<Opcode> covered;
for (const auto &test : tests) {
for (auto opcode : test.opcodes) {
covered.insert(opcode);
}
}
for (const auto &test : graphics_tests) {
for (auto opcode : test.opcodes) {
covered.insert(opcode);
}
}
uint32_t counts[7] = {};
std::vector<ShaderOpcode> pending[7];
for (auto value = static_cast<int>(Opcode::S_MOV_B32);
value < static_cast<int>(Opcode::COUNT); value++) {
const auto opcode = static_cast<Opcode>(value);
const auto status = ClassifyOpcode(opcode, covered);
counts[static_cast<uint32_t>(status)]++;
if (status != CoverageClass::Covered &&
status != CoverageClass::ControlOrMarker) {
pending[static_cast<uint32_t>(status)].push_back(opcode);
}
}
std::printf(
"[coverage] decoder opcodes: covered=%u control=%u alu_pending=%u "
"float_pending=%u memory_pending=%u image_pending=%u "
"graphics_pending=%u\n",
counts[static_cast<uint32_t>(CoverageClass::Covered)],
counts[static_cast<uint32_t>(CoverageClass::ControlOrMarker)],
counts[static_cast<uint32_t>(CoverageClass::NeedsAluCase)],
counts[static_cast<uint32_t>(CoverageClass::NeedsFloatCase)],
counts[static_cast<uint32_t>(CoverageClass::NeedsMemoryCase)],
counts[static_cast<uint32_t>(CoverageClass::NeedsImageCase)],
counts[static_cast<uint32_t>(CoverageClass::NeedsGraphicsStageCase)]);
PrintPendingOpcodes(
CoverageClass::NeedsAluCase,
pending[static_cast<uint32_t>(CoverageClass::NeedsAluCase)]);
PrintPendingOpcodes(
CoverageClass::NeedsFloatCase,
pending[static_cast<uint32_t>(CoverageClass::NeedsFloatCase)]);
PrintPendingOpcodes(
CoverageClass::NeedsMemoryCase,
pending[static_cast<uint32_t>(CoverageClass::NeedsMemoryCase)]);
PrintPendingOpcodes(
CoverageClass::NeedsImageCase,
pending[static_cast<uint32_t>(CoverageClass::NeedsImageCase)]);
PrintPendingOpcodes(
CoverageClass::NeedsGraphicsStageCase,
pending[static_cast<uint32_t>(CoverageClass::NeedsGraphicsStageCase)]);
}
TestCase IntegerAddSubMul() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSMovB32(0, InlineU32(7)),
EncodeSMovB32(1, InlineU32(3)),
EncodeSop2(0x00, 2, 0, 1),
EncodeSop2(0x01, 3, 2, InlineU32(1)),
EncodeSop2(0x26, 4, 3, InlineU32(2)),
EncodeVop1(0x01, 0, 4),
};
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"IntegerAddSubMul",
code,
{},
{18},
{O::S_MOV_B32, O::S_ADD_U32, O::S_SUB_U32, O::S_MUL_I32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BitwiseOps() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSMovB32(0, InlineU32(60)), EncodeSMovB32(1, InlineU32(15)),
EncodeSop2(0x0e, 2, 0, 1), EncodeSop2(0x10, 3, 0, 1),
EncodeSop2(0x12, 4, 3, 2), EncodeVop1(0x37, 0, 4),
};
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"BitwiseAndOrXorNot",
code,
{},
{~0x33u},
{O::S_MOV_B32, O::S_AND_B32, O::S_OR_B32, O::S_XOR_B32, O::V_NOT_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Shifts() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSMovB32(0, InlineU32(3)),
EncodeSop2(0x1e, 1, 0, InlineU32(2)),
EncodeSop2(0x20, 2, 1, InlineU32(1)),
EncodeVop1(0x01, 0, 2),
};
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"Shifts",
code,
{},
{6},
{O::S_MOV_B32, O::S_LSHL_B32, O::S_LSHR_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarZeroShiftWithRuntimeCount() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSop2(0x20, 8, InlineU32(0), 4),
EncodeSop2(0x20, 9, 6, 4),
};
AppendStoreSgpr(&code, 8, 0);
AppendStoreSgpr(&code, 9, 1);
AppendEnd(&code);
TestCase test;
test.name = "ScalarZeroShiftWithRuntimeCount";
test.code = std::move(code);
test.expected = {0, 0x40000000u};
test.opcodes = {O::S_LSHR_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 2 * sizeof(u32));
test.user_data[4] = 33;
test.user_data[6] = 0x80000000u;
test.has_user_data = true;
test.ir_counts = {{" = ShiftRightLogical32 ", 1}};
test.required_spirv = {"OpShiftRightLogical"};
return test;
}
TestCase ExactPushConstantExtent() {
using O = ShaderOpcode;
constexpr std::array<u32, 10> values{
0x10203040u, 0x21314151u, 0x32425262u, 0x43536373u, 0x54647484u,
0x65758595u, 0x768696a6u, 0x8797a7b7u, 0x98a8b8c8u, 0xa9b9c9d9u};
std::vector<u32> code;
for (u32 i = 0; i < values.size(); i++) {
AppendStoreSgpr(&code, 4u + i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "ExactPushConstantExtent";
test.code = std::move(code);
test.initial.resize(values.size());
test.expected.assign(values.begin(), values.end());
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data =
MakeStructuredStorageBufferData(0, values.size() * sizeof(u32));
std::copy(values.begin(), values.end(), test.user_data.begin() + 4);
test.has_user_data = true;
return test;
}
TestCase ScalarShiftCountsMaskLowBits() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(1)));
AppendSMovLiteral(&code, 1, 0x80000000u);
code.push_back(EncodeSMovB32(2, InlineU32(32)));
code.push_back(EncodeSMovB32(3, InlineU32(33)));
code.push_back(EncodeSop2(0x1e, 10, 0, 2));
code.push_back(EncodeSop2(0x1e, 11, 0, 3));
code.push_back(EncodeSop2(0x20, 12, 1, 2));
code.push_back(EncodeSop2(0x20, 13, 1, 3));
code.push_back(EncodeSop2(0x22, 14, 1, 2));
code.push_back(EncodeSop2(0x22, 15, 1, 3));
for (u32 i = 0; i < 6u; i++) {
AppendStoreSgpr(&code, 10u + i, i);
}
AppendEnd(&code);
return {"ScalarShiftCountsMaskLowBits",
code,
{},
{1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u},
{O::S_MOV_B32, O::S_LSHL_B32, O::S_LSHR_B32, O::S_ASHR_I32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Rdna2ScalarOpcodes() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(2, InlineU32(7)));
code.push_back(EncodeSMovB32(106, InlineU32(16)));
code.push_back(EncodeSop1(0x1d, 106, InlineU32(0)));
AppendStoreSgpr(&code, 106, 0);
code.push_back(EncodeSopk(0x13, 106, 0x1019u));
AppendStoreSgpr(&code, 106, 1);
code.push_back(EncodeSop2(0x02, 106, 2, 239u));
AppendStoreSgpr(&code, 106, 2);
code.push_back(EncodeSopp(0x0e, 0));
code.push_back(EncodeSop2(0x02, 106, 239u, 2));
AppendStoreSgpr(&code, 106, 3);
AppendEnd(&code);
return {"Rdna2ScalarOpcodes",
code,
{},
{0x11u, 0x11u, 7u, 7u},
{O::S_MOV_B32, O::S_BITSET1_B32, O::S_SETREG_B32, O::S_ADD_I32,
O::S_SLEEP, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarTrapDisabledFallsThroughExactRaw() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 7);
code.push_back(0xbf920001u); // s_trap 1 from the game shader
AppendVMovU32(&code, 0, 42);
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ScalarTrapDisabledFallsThroughExactRaw";
test.code = code;
test.expected = {42};
test.opcodes = {O::V_MOV_B32, O::S_TRAP, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_TRAP 0x00000001", 1}};
test.compute_info.threads_num[0] = 1;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase ScalarWaitcntDepctrCapturedVmVsrc() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 7, 7);
AppendVMovU32(&code, 31, 0);
AppendBufferStoreDword(&code, 7, 31);
code.push_back(0xbfa3ffe3u); // s_waitcnt_depctr vm_vsrc(0)
AppendVMovU32(&code, 7, 42);
AppendStoreVgpr(&code, 7, 1);
AppendEnd(&code);
TestCase test;
test.name = "ScalarWaitcntDepctrCapturedVmVsrc";
test.code = std::move(code);
test.initial = {0, 0};
test.expected = {7, 42};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_WAITCNT_DEPCTR,
O::S_ENDPGM};
test.decoded_counts = {{"0x00000010: S_WAITCNT_DEPCTR 0x0000ffe3\n", 1}};
test.forbidden_spirv = {"OpControlBarrier", "OpMemoryBarrier"};
return test;
}
TestCase ScalarExtendedArithmetic() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0xffffffffu);
code.push_back(EncodeSMovB32(1, InlineU32(1)));
code.push_back(EncodeSop2(0x00, 2, 0, 1));
code.push_back(EncodeSMovB32(3, InlineU32(5)));
code.push_back(EncodeSMovB32(4, InlineU32(6)));
code.push_back(EncodeSop2(0x04, 5, 3, 4));
code.push_back(EncodeSop2(0x02, 6, InlineU32(7), InlineU32(8)));
code.push_back(EncodeSop2(0x03, 7, InlineU32(7), InlineU32(9)));
AppendSMovLiteral(&code, 8, 0xfffffffbu);
code.push_back(EncodeSMovB32(9, InlineU32(3)));
code.push_back(EncodeSop2(0x06, 10, 8, 9));
code.push_back(EncodeSop2(0x08, 11, 8, 9));
AppendSMovLiteral(&code, 12, 0xfffffffeu);
code.push_back(EncodeSMovB32(13, InlineU32(3)));
code.push_back(EncodeSop2(0x07, 14, 12, 13));
code.push_back(EncodeSop2(0x09, 15, 12, 13));
code.push_back(EncodeSMovB32(16, 8));
code.push_back(EncodeSop1(0x34, 16, 16));
code.push_back(EncodeSopk(0x00, 17, 0xfff5u));
code.push_back(EncodeSopk(0x10, 17, 0xfffdu));
const u32 results[] = {2, 5, 6, 7, 10, 11, 14, 15, 16, 17};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreSgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"ScalarExtendedArithmetic",
code,
{},
{0, 12, 15, 0xfffffffeu, 0xfffffffbu, 3, 3, 0xfffffffeu, 5, 33},
{O::S_MOV_B32, O::S_ADD_U32, O::S_ADDC_U32, O::S_ADD_I32,
O::S_SUB_I32, O::S_MIN_I32, O::S_MAX_I32, O::S_MIN_U32, O::S_MAX_U32,
O::S_ABS_I32, O::S_MOVK_I32, O::S_MULK_I32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarArithmeticSccCarryBorrowOverflow() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) {
set_scc(prior_scc);
code.push_back(instruction);
capture_scc(out_sgpr);
};
code.push_back(EncodeSMovB32(0, InlineU32(0)));
code.push_back(EncodeSMovB32(1, InlineU32(1)));
append_case(false, EncodeSop2(0x01, 10, 0, 1), 20);
code.push_back(EncodeSMovB32(2, InlineU32(5)));
code.push_back(EncodeSMovB32(3, InlineU32(3)));
append_case(true, EncodeSop2(0x01, 11, 2, 3), 21);
AppendSMovLiteral(&code, 4, 0x7fffffffu);
code.push_back(EncodeSMovB32(5, InlineU32(1)));
append_case(false, EncodeSop2(0x02, 12, 4, 5), 22);
code.push_back(EncodeSMovB32(6, InlineU32(1)));
code.push_back(EncodeSMovB32(7, InlineU32(2)));
append_case(true, EncodeSop2(0x02, 13, 6, 7), 23);
AppendSMovLiteral(&code, 8, 0x80000000u);
code.push_back(EncodeSMovB32(9, InlineU32(1)));
append_case(false, EncodeSop2(0x03, 14, 8, 9), 24);
code.push_back(EncodeSMovB32(15, InlineU32(5)));
code.push_back(EncodeSMovB32(16, InlineU32(3)));
append_case(true, EncodeSop2(0x03, 17, 15, 16), 25);
AppendSMovLiteral(&code, 18, 0x7fffffffu);
set_scc(false);
code.push_back(EncodeSopk(0x0f, 18, 1));
capture_scc(26);
code.push_back(EncodeSMovB32(19, InlineU32(4)));
set_scc(true);
code.push_back(EncodeSopk(0x0f, 19, 0xfffeu));
capture_scc(27);
for (u32 i = 0; i < 8u; i++) {
AppendStoreSgpr(&code, 20u + i, i);
}
AppendEnd(&code);
return {"ScalarArithmeticSccCarryBorrowOverflow",
code,
{},
{1, 0, 1, 0, 1, 0, 1, 0},
{O::S_MOV_B32, O::S_SUB_U32, O::S_ADD_I32, O::S_SUB_I32,
O::S_CMP_EQ_U32, O::S_CSELECT_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarMinMaxSccComparisonEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) {
set_scc(prior_scc);
code.push_back(instruction);
capture_scc(out_sgpr);
};
AppendSMovLiteral(&code, 0, 0xfffffffeu);
code.push_back(EncodeSMovB32(1, InlineU32(3)));
append_case(false, EncodeSop2(0x06, 10, 0, 1), 20);
append_case(true, EncodeSop2(0x06, 11, 1, 0), 21);
append_case(false, EncodeSop2(0x08, 12, 1, 0), 22);
append_case(true, EncodeSop2(0x08, 13, 0, 1), 23);
code.push_back(EncodeSMovB32(2, InlineU32(2)));
code.push_back(EncodeSMovB32(3, InlineU32(3)));
append_case(false, EncodeSop2(0x07, 14, 2, 3), 24);
append_case(true, EncodeSop2(0x07, 15, 3, 2), 25);
append_case(false, EncodeSop2(0x09, 16, 3, 2), 26);
append_case(true, EncodeSop2(0x09, 17, 2, 3), 27);
for (u32 i = 0; i < 8u; i++) {
AppendStoreSgpr(&code, 20u + i, i);
}
AppendEnd(&code);
return {"ScalarMinMaxSccComparisonEdges",
code,
{},
{1, 0, 1, 0, 1, 0, 1, 0},
{O::S_MOV_B32, O::S_MIN_I32, O::S_MAX_I32, O::S_MIN_U32, O::S_MAX_U32,
O::S_CMP_EQ_U32, O::S_CSELECT_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarAbsI32UpdatesScc() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
AppendSmemLoadOpcode(&code, 0x08, 16, 0);
set_scc(true);
code.push_back(EncodeSop1(0x34, 17, 16));
capture_scc(18);
AppendSmemLoadOpcode(&code, 0x08, 19, 4);
set_scc(false);
code.push_back(EncodeSop1(0x34, 20, 19));
capture_scc(21);
AppendSmemLoadOpcode(&code, 0x08, 22, 8);
set_scc(false);
code.push_back(EncodeSop1(0x34, 23, 22));
capture_scc(24);
AppendStoreSgpr(&code, 17, 0);
AppendStoreSgpr(&code, 18, 1);
AppendStoreSgpr(&code, 20, 2);
AppendStoreSgpr(&code, 21, 3);
AppendStoreSgpr(&code, 23, 4);
AppendStoreSgpr(&code, 24, 5);
AppendEnd(&code);
return {"ScalarAbsI32UpdatesScc",
code,
{0, 0xfffffffbu, 0x80000000u},
{0, 0, 5, 1, 0x80000000u, 1},
{O::S_BUFFER_LOAD_DWORD, O::S_CMP_EQ_U32, O::S_ABS_I32, O::S_CSELECT_B32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarAbsdiffBitset64AndSetprio() {
using O = ShaderOpcode;
std::vector<u32> code;
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(
EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
code.push_back(EncodeSMovB32(0, InlineU32(2)));
code.push_back(EncodeSMovB32(1, InlineU32(5)));
code.push_back(EncodeSop2(0x2c, 10, 0, 1));
capture_scc(20);
AppendSMovLiteral(&code, 0, 0xffffffffu);
code.push_back(EncodeSMovB32(1, InlineU32(0)));
code.push_back(EncodeSop2(0x2c, 11, 0, 1));
capture_scc(21);
AppendSMovLiteral(&code, 0, 0x80000000u);
code.push_back(EncodeSop2(0x2c, 12, 0, 1));
capture_scc(22);
code.push_back(EncodeSMovB32(1, InlineU32(1)));
code.push_back(EncodeSop2(0x2c, 13, 0, 1));
capture_scc(23);
code.push_back(EncodeSop2(0x2c, 14, 0, 0));
capture_scc(24);
AppendSMovLiteral(&code, 2, 65u);
AppendSMovLiteral(&code, 3, 101u);
code.push_back(EncodeSMovB32(4, InlineU32(0)));
code.push_back(EncodeSMovB32(5, InlineU32(0)));
code.push_back(EncodeSop1(0x1e, 4, 2));
AppendStoreSgprPair(&code, 4, 10);
code.push_back(EncodeSop1(0x1e, 4, 3));
AppendStoreSgprPair(&code, 4, 12);
code.push_back(EncodeSop1(0x1c, 4, 2));
AppendStoreSgprPair(&code, 4, 14);
code.push_back(EncodeSop1(0x1c, 4, 3));
AppendStoreSgprPair(&code, 4, 16);
code.push_back(EncodeSopp(0x0f, 3));
AppendStoreSgpr(&code, 10, 0);
AppendStoreSgpr(&code, 11, 1);
AppendStoreSgpr(&code, 12, 2);
AppendStoreSgpr(&code, 13, 3);
AppendStoreSgpr(&code, 14, 4);
AppendStoreSgpr(&code, 20, 5);
AppendStoreSgpr(&code, 21, 6);
AppendStoreSgpr(&code, 22, 7);
AppendStoreSgpr(&code, 23, 8);
AppendStoreSgpr(&code, 24, 9);
AppendEnd(&code);
TestCase test;
test.name = "ScalarAbsdiffBitset64AndSetprio";
test.code = std::move(code);
test.expected = {3u, 1u, 0x80000000u, 0x7fffffffu, 0u,
1u, 1u, 1u, 1u, 0u,
2u, 0u, 2u, 32u, 0u,
32u, 0u, 0u};
test.opcodes = {O::S_MOV_B32, O::S_ABSDIFF_I32, O::S_CSELECT_B32,
O::S_BITSET1_B64, O::S_BITSET0_B64, O::S_SETPRIO,
O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_ABSDIFF_I32", 5},
{"S_BITSET1_B64", 2},
{"S_BITSET0_B64", 2},
{"S_SETPRIO 0x00000003", 1}};
return test;
}
TestCase ScalarShiftLeftAddSccCarryEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) {
set_scc(prior_scc);
code.push_back(instruction);
capture_scc(out_sgpr);
};
AppendSMovLiteral(&code, 0, 0x80000000u);
AppendSMovLiteral(&code, 1, 0x40000000u);
AppendSMovLiteral(&code, 2, 0x20000000u);
AppendSMovLiteral(&code, 3, 0x10000000u);
code.push_back(EncodeSMovB32(4, InlineU32(1)));
code.push_back(EncodeSMovB32(5, InlineU32(0)));
append_case(false, EncodeSop2(0x2e, 10, 0, 5), 20);
append_case(true, EncodeSop2(0x2e, 11, 4, 4), 21);
append_case(false, EncodeSop2(0x2f, 12, 1, 5), 22);
append_case(true, EncodeSop2(0x2f, 13, 4, 4), 23);
append_case(false, EncodeSop2(0x30, 14, 2, 5), 24);
append_case(true, EncodeSop2(0x30, 15, 4, 4), 25);
append_case(false, EncodeSop2(0x31, 16, 3, 5), 26);
append_case(true, EncodeSop2(0x31, 17, 4, 4), 27);
for (u32 i = 0; i < 8u; i++) {
AppendStoreSgpr(&code, 20u + i, i);
}
AppendEnd(&code);
return {"ScalarShiftLeftAddSccCarryEdges",
code,
{},
{1, 0, 1, 0, 1, 0, 1, 0},
{O::S_MOV_B32, O::S_LSHL1_ADD_U32, O::S_LSHL2_ADD_U32,
O::S_LSHL3_ADD_U32, O::S_LSHL4_ADD_U32, O::S_CMP_EQ_U32,
O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarCompareOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0xfffffffeu);
code.push_back(EncodeSMovB32(1, InlineU32(1)));
code.push_back(EncodeSMovB32(2, InlineU32(5)));
code.push_back(EncodeSMovB32(3, InlineU32(2)));
code.push_back(EncodeSMovB32(4, InlineU32(3)));
AppendSMovLiteral(&code, 10, 0x00000001u);
code.push_back(EncodeSMovB32(11, InlineU32(0)));
code.push_back(EncodeSMovB32(12, InlineU32(2)));
code.push_back(EncodeSMovB32(13, InlineU32(0)));
code.push_back(EncodeSMovB32(14, InlineU32(1)));
code.push_back(EncodeSMovB32(15, InlineU32(1)));
u32 dst = 20;
auto append_compare = [&](u32 opcode, u32 src0, u32 src1) {
code.push_back(EncodeSopc(opcode, src0, src1));
code.push_back(EncodeSop2(0x0a, dst++, InlineU32(1), InlineU32(0)));
};
append_compare(0x00, 2, 2);
append_compare(0x01, 0, 1);
append_compare(0x02, 1, 0);
append_compare(0x03, 1, 1);
append_compare(0x04, 0, 1);
append_compare(0x05, 0, 0);
append_compare(0x07, 3, 4);
append_compare(0x08, 4, 3);
append_compare(0x09, 4, 4);
append_compare(0x0b, 3, 4);
append_compare(0x12, 10, 10);
append_compare(0x12, 10, 12);
append_compare(0x12, 10, 14);
append_compare(0x13, 10, 12);
append_compare(0x13, 10, 14);
for (u32 i = 0; i < 15u; i++) {
AppendStoreSgpr(&code, 20u + i, i);
}
AppendEnd(&code);
return {"ScalarCompareOps",
code,
{},
{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1},
{O::S_MOV_B32, O::S_CMP_EQ_I32, O::S_CMP_LG_I32, O::S_CMP_GT_I32,
O::S_CMP_GE_I32, O::S_CMP_LT_I32, O::S_CMP_LE_I32, O::S_CMP_LG_U32,
O::S_CMP_GT_U32, O::S_CMP_GE_U32, O::S_CMP_LE_U32, O::S_CMP_EQ_U64,
O::S_CMP_LG_U64, O::S_CSELECT_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarShiftAddAndMaskOps() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(2)));
code.push_back(EncodeSMovB32(1, InlineU32(3)));
code.push_back(EncodeSop2(0x2e, 2, 0, 1));
code.push_back(EncodeSop2(0x2f, 3, 0, 1));
code.push_back(EncodeSop2(0x30, 4, 0, 1));
code.push_back(EncodeSop2(0x31, 5, 0, 1));
AppendSMovLiteral(&code, 6, 0xfffffff8u);
code.push_back(EncodeSop2(0x22, 7, 6, InlineU32(2)));
AppendSMovLiteral(&code, 8, 0xffffffffu);
code.push_back(EncodeSMovB32(9, InlineU32(2)));
code.push_back(EncodeSop2(0x35, 10, 8, 9));
AppendSMovLiteral(&code, 12, 0x80000011u);
AppendSMovLiteral(&code, 13, 0x80000001u);
code.push_back(EncodeSop1(0x08, 14, 12));
code.push_back(EncodeSop1(0x0a, 16, 12));
AppendStoreSgpr(&code, 2, 0);
AppendStoreSgpr(&code, 3, 1);
AppendStoreSgpr(&code, 4, 2);
AppendStoreSgpr(&code, 5, 3);
AppendStoreSgpr(&code, 7, 4);
AppendStoreSgpr(&code, 10, 5);
AppendStoreSgprPair(&code, 14, 6);
AppendStoreSgprPair(&code, 16, 8);
AppendEnd(&code);
return {"ScalarShiftAddAndMaskOps",
code,
{},
{7, 11, 19, 35, 0xfffffffeu, 1, 0x7fffffeeu, 0x7ffffffeu, 0xf00000ffu,
0xf000000fu},
{O::S_MOV_B32, O::S_LSHL1_ADD_U32, O::S_LSHL2_ADD_U32,
O::S_LSHL3_ADD_U32, O::S_LSHL4_ADD_U32, O::S_ASHR_I32,
O::S_MUL_HI_U32, O::S_NOT_B64, O::S_WQM_B64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarNotB64UpdatesScc() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
AppendSMovLiteral(&code, 0, 0xffffffffu);
AppendSMovLiteral(&code, 1, 0xffffffffu);
set_scc(true);
code.push_back(EncodeSop1(0x08, 2, 0));
capture_scc(4);
AppendSMovLiteral(&code, 6, 0xfffffffeu);
AppendSMovLiteral(&code, 7, 0xffffffffu);
set_scc(false);
code.push_back(EncodeSop1(0x08, 8, 6));
capture_scc(10);
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 12, 106));
code.push_back(EncodeSop1(0x08, 14, 12));
capture_scc(16);
AppendStoreSgpr(&code, 4, 0);
AppendStoreSgpr(&code, 10, 1);
AppendStoreSgprPair(&code, 2, 2);
AppendStoreSgprPair(&code, 8, 4);
AppendStoreSgpr(&code, 16, 6);
AppendStoreSgprPair(&code, 14, 7);
AppendEnd(&code);
return {"ScalarNotB64UpdatesScc",
code,
{},
{0, 1, 0, 0, 1, 0, 1, 0xfffffffeu, 0xffffffffu},
{O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_NOT_B64, O::S_CSELECT_B32,
O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarQuadmaskB64() {
using O = ShaderOpcode;
std::vector<u32> code;
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
AppendSMovLiteral(&code, 106, 0x80000011u);
AppendSMovLiteral(&code, 107, 0x000000f0u);
code.push_back(EncodeSop1(0x2d, 106, 106));
capture_scc(0);
AppendStoreSgprPair(&code, 106, 0);
AppendStoreSgpr(&code, 0, 2);
code.push_back(EncodeSMovB32(106, InlineU32(0)));
code.push_back(EncodeSMovB32(107, InlineU32(0)));
code.push_back(EncodeSop1(0x2d, 106, 106));
capture_scc(0);
AppendStoreSgprPair(&code, 106, 3);
AppendStoreSgpr(&code, 0, 5);
AppendEnd(&code);
return {"ScalarQuadmaskB64",
code,
{},
{0x00000283u, 0, 1, 0, 0, 0},
{O::S_MOV_B32, O::S_QUADMASK_B64, O::S_CSELECT_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarFlbitI32B64Gpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0x00000000u);
AppendSMovLiteral(&code, 1, 0x80000000u);
code.push_back(EncodeSop1(0x16, 20, 0));
AppendSMovLiteral(&code, 2, 0x00008000u);
AppendSMovLiteral(&code, 3, 0x00000000u);
code.push_back(EncodeSop1(0x16, 21, 2));
AppendSMovLiteral(&code, 4, 0x00000000u);
AppendSMovLiteral(&code, 5, 0x00000000u);
code.push_back(EncodeSop1(0x16, 22, 4));
AppendSMovLiteral(&code, 14, 0x00000008u);
AppendSMovLiteral(&code, 15, 0x00000000u);
code.push_back(EncodeSop1(0x16, 106, 14));
AppendStoreSgpr(&code, 20, 0);
AppendStoreSgpr(&code, 21, 1);
AppendStoreSgpr(&code, 22, 2);
AppendStoreSgpr(&code, 106, 3);
AppendEnd(&code);
return {"ScalarFlbitI32B64Gpu",
code,
{},
{0, 48, 0xffffffffu, 60},
{O::S_MOV_B32, O::S_FLBIT_I32_B64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarFlbitI32B32Gpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0x00000000u);
code.push_back(EncodeSop1(0x15, 20, 0));
AppendSMovLiteral(&code, 0, 0x0000ccccu);
code.push_back(EncodeSop1(0x15, 21, 0));
AppendSMovLiteral(&code, 0, 0x80000000u);
code.push_back(EncodeSop1(0x15, 22, 0));
AppendSMovLiteral(&code, 0, 0x7fffffffu);
code.push_back(EncodeSop1(0x15, 23, 0));
AppendStoreSgpr(&code, 20, 0);
AppendStoreSgpr(&code, 21, 1);
AppendStoreSgpr(&code, 22, 2);
AppendStoreSgpr(&code, 23, 3);
AppendEnd(&code);
return {"ScalarFlbitI32B32Gpu",
code,
{},
{0xffffffffu, 16, 0, 1},
{O::S_MOV_B32, O::S_FLBIT_I32_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarSaveExecOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 106, 0xffffffffu);
AppendSMovLiteral(&code, 107, 0xffffffffu);
code.push_back(0xbe80246au);
code.push_back(EncodeSop1(0x28, 2, 106));
code.push_back(EncodeSMovB32(106, InlineU32(0)));
code.push_back(EncodeSMovB32(107, InlineU32(0)));
code.push_back(EncodeSop1(0x37, 4, 106));
AppendSMovLiteral(&code, 106, 0x00000003u);
code.push_back(EncodeSop1(0x3c, 8, 106));
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeSMovB32(106, InlineU32(1)));
code.push_back(EncodeSop1(0x44, 12, 106));
code.push_back(EncodeSop1(0x04, 14, 126));
AppendSMovLiteral(&code, 126, 0xffffffffu);
AppendSMovLiteral(&code, 127, 0xffffffffu);
AppendStoreSgprPair(&code, 0, 0);
AppendStoreSgprPair(&code, 2, 2);
AppendStoreSgprPair(&code, 4, 4);
AppendStoreSgpr(&code, 8, 6);
AppendStoreSgprPair(&code, 10, 7);
AppendStoreSgpr(&code, 12, 9);
AppendStoreSgprPair(&code, 14, 10);
AppendStoreSgpr(&code, 253, 12);
AppendEnd(&code);
TestCase test;
test.name = "ScalarSaveExecOps";
test.code = code;
// A full wave enters with EXEC set for all 64 lanes, so the two saved EXEC
// pairs are 0xffffffff/0xffffffff rather than the 1/0 a single-invocation
// dispatch would report.
test.expected = {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu,
0xffffffffu, 0xffffffffu, 0xffffffffu, 3, 0xffffffffu,
3, 2, 0xffffffffu, 1};
test.opcodes = {O::S_MOV_B32, O::S_AND_SAVEEXEC_B64, O::S_ORN2_SAVEEXEC_B64,
O::S_ANDN1_SAVEEXEC_B64, O::S_AND_SAVEEXEC_B32,
O::S_ANDN1_SAVEEXEC_B32, O::S_MOV_B64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.has_compute_info = true;
return test;
}
TestCase ScalarOrn2SaveexecUsesSourceOrNotExec() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 126, 0x0000000cu);
AppendSMovLiteral(&code, 127, 0x80000000u);
AppendSMovLiteral(&code, 0, 0x00000001u);
AppendSMovLiteral(&code, 1, 0x00000001u);
code.push_back(EncodeSop1(0x28, 2, 0));
code.push_back(EncodeSop1(0x04, 4, 126));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeSMovB32(127, InlineU32(0)));
AppendStoreSgprPair(&code, 2, 0);
AppendStoreSgprPair(&code, 4, 2);
AppendStoreSgpr(&code, 253, 4);
AppendEnd(&code);
TestCase test;
test.name = "ScalarOrn2SaveexecUsesSourceOrNotExec";
test.code = code;
test.expected = {0x0000000cu, 0x80000000u, 0xfffffff3u, 0x7fffffffu, 1};
test.opcodes = {O::S_MOV_B32, O::S_ORN2_SAVEEXEC_B64, O::S_MOV_B64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
// The case reads a whole 64-bit EXEC mask, so it needs a populated subgroup.
// A 1x1x1 dispatch leaves OpGroupNonUniformBallot with a single invocation
// and every lane above zero reads back as zero.
test.compute_info.threads_num[0] = 64;
test.has_compute_info = true;
return test;
}
TestCase ScalarOrn2SaveexecB32(u32 wave_size, u32 threads) {
using O = ShaderOpcode;
constexpr u32 exec_hi = 0xa5a55a5au;
constexpr u32 vcc_hi = 0x13579bdfu;
struct Case {
u32 exec_lo, source, destination, result;
};
constexpr Case cases[] = {
{0x8000000cu, 126, 106, 0xffffffffu}, // Captured vcc_lo, exec_lo.
{0x0000000fu, 6, 6, 0xfffffff3u}, // Aliased SGPR source/destination.
{0xffffffffu, 128, 8, 0}, // SCC ignores the nonzero high half.
{0xffffffffu, 107, 8, vcc_hi},
{0xffffffffu, 127, 8, exec_hi},
{0xffffffffu, 253, 8, 1}, // SCC is a scalar value, not a mask.
{0xffffffffu, 251, 8, 0}, // VCCZ summarizes the whole mask.
{0x00000001u, 252, 8, 0xfffffffeu}, // EXECZ is uniform across lanes.
};
TestCase test;
test.name = wave_size == 64 ? "ScalarOrn2SaveexecB32Wave64"
: threads == 32 ? "ScalarOrn2SaveexecB32Wave32"
: "ScalarOrn2SaveexecB32PartialWave";
auto &code = test.code;
// Zero flags ignore high bits only in wave32 and include inactive low bits.
constexpr u32 masks[][2] = {{0, 0}, {0x80000000u, 0}, {0, 0x80000000u}};
for (const auto &mask : masks) {
AppendSMovLiteral(&code, 106, mask[0]);
AppendSMovLiteral(&code, 107, mask[1]);
AppendSMovLiteral(&code, 126, mask[0]);
AppendSMovLiteral(&code, 127, mask[1]);
code.push_back(EncodeSMovB32(20, 251));
code.push_back(EncodeSMovB32(21, 252));
code.push_back(EncodeSop1(0x04, 126, 193u));
const u32 zero = mask[0] == 0 && (wave_size == 32 || mask[1] == 0);
for (u32 reg = 20; reg <= 21; ++reg) {
AppendStoreSgprAtLaneDwordOffset(&code, reg, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), threads, zero);
}
}
for (const auto &item : cases) {
AppendVMovU32(&code, 2, 0);
AppendSMovLiteral(&code, 126, item.exec_lo);
AppendSMovLiteral(&code, 127, exec_hi);
AppendSMovLiteral(&code, 106, 1);
AppendSMovLiteral(&code, 107, vcc_hi);
AppendSMovLiteral(&code, 6, 3);
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(0)));
code.push_back(item.source == 126 ? 0xbeea407eu
: EncodeSop1(0x40, item.destination, item.source));
const u32 snapshots[] = {item.destination, 126, 127, 107, 253};
for (u32 i = 0; i < std::size(snapshots); ++i) {
code.push_back(EncodeSMovB32(20 + i, snapshots[i]));
}
AppendVMovU32(&code, 2, 1);
code.push_back(EncodeSop1(0x04, 126, 193u)); // Restore EXEC for readback.
const u32 expected[] = {item.exec_lo, item.result, exec_hi, vcc_hi,
item.result != 0 ? 1u : 0u};
for (u32 i = 0; i < std::size(expected); ++i) {
AppendStoreSgprAtLaneDwordOffset(&code, 20 + i, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), threads, expected[i]);
}
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
static_cast<u32>(test.expected.size()));
for (u32 lane = 0; lane < threads; ++lane) {
const u32 mask = lane < 32 ? item.result : exec_hi;
test.expected.push_back((mask >> (lane % 32)) & 1u);
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_CMP_EQ_U32,
O::S_ORN2_SAVEEXEC_B32, O::V_MOV_B32, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_ORN2_SAVEEXEC_B32 vcc_lo, exec_lo", 1}};
test.compute_info.threads_num[0] = threads;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarSubvectorLoops(u32 wave_size) {
using O = ShaderOpcode;
using namespace ShaderRecompiler::Decoder;
TestCase test;
test.name = wave_size == 64 ? "ScalarSubvectorLoopsWave64"
: "ScalarSubvectorLoopsWave32";
// The captured mesh shader's signed SOPK targets are relative to PC + 4.
std::array<u32, 47> captured{};
captured[0x30 / 4] = 0xbda60022u;
captured[0xb8 / 4] = 0xbe26ffdeu;
Instruction begin, end;
DecodeInstruction(captured, 0x30 / 4, begin);
DecodeInstruction(captured, 0xb8 / 4, end);
Require(test.name, "captured branch decode",
begin.opcode == O::S_SUBVECTOR_LOOP_BEGIN && begin.dst.reg == 38 &&
begin.dst.kind == OperandKind::Sgpr && begin.branch_target == 0xbc &&
end.opcode == O::S_SUBVECTOR_LOOP_END && end.dst.reg == 38 &&
end.dst.kind == OperandKind::Sgpr && end.branch_target == 0x34 &&
begin.word_count == 1 && end.word_count == 1,
"subvector SDST or signed PC-relative target was decoded incorrectly");
constexpr u32 sentinel = 0x5a13abcdu;
constexpr u32 low = 0x80000005u, high = 0x42000002u;
struct Case {
u32 lo, hi, body_mode, saved, final_lo, final_hi, passes;
};
constexpr Case cases[] = {
{0, 0, 0, sentinel, 0, 0, 0},
{low, 0, 0, 0, low, 0, 1},
{0, high, 0, 0, 0, high, 1},
{low, high, 0, low, low, high, 2},
{~0u, ~0u, 0, ~0u, ~0u, ~0u, 2},
// The game resets the full EXEC mask before END, selecting its HI != 0 arm.
{low, high, 1, high, high, ~0u, 1},
// END must restore the low mask saved after the body has changed it.
{low, high, 2, 0, 0, high, 2},
// BEGIN EXEC_LO first copies HI to its aliased destination, then clears HI.
{low, high, 3, 0, 0, high, 2},
};
auto &code = test.code;
for (u32 index = 0; index < std::size(cases); ++index) {
const auto &item = cases[index];
const u32 scc = index & 1u;
const u32 saved_reg = item.body_mode == 3 ? 126u : 38u;
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 3, 0);
AppendSMovLiteral(&code, 38, sentinel);
AppendSMovLiteral(&code, 32, 1); // Scalar 2^passes; MULK preserves SCC.
AppendSMovLiteral(&code, 33, 0xff);
AppendSMovLiteral(&code, 126, item.lo);
AppendSMovLiteral(&code, 127, item.hi);
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(scc == 0 ? 1 : 0)));
const auto begin_word = static_cast<u32>(code.size());
code.push_back(0);
const auto body_word = static_cast<u32>(code.size());
code.push_back(EncodeSMovB32(33, 253)); // Observe SCC immediately after BEGIN.
code.push_back(EncodeSopk(0x10, 32, 2));
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2));
code.push_back(EncodeVop1(0x01, 3, 32));
if (item.body_mode == 1) {
code.push_back(EncodeSop1(0x04, 126, 193)); // S_MOV_B64 EXEC, -1.
} else if (item.body_mode == 2) {
code.push_back(EncodeSMovB32(126, InlineU32(0)));
}
const auto end_word = static_cast<u32>(code.size());
code.push_back(EncodeSopk(0x1c, saved_reg, body_word - end_word - 1u));
code[begin_word] = EncodeSopk(0x1b, saved_reg, end_word - begin_word);
const u32 sources[] = {saved_reg, 126, 127, 253, 32, 33};
for (u32 i = 0; i < std::size(sources); ++i) {
code.push_back(EncodeSMovB32(20 + i, sources[i]));
}
code.push_back(EncodeSop1(0x04, 126, 193)); // Restore all lanes for readback.
const u32 expected[] = {item.saved, item.final_lo, item.final_hi, scc,
1u << item.passes, item.passes == 0 ? 0xff : scc};
for (u32 i = 0; i < std::size(expected); ++i) {
AppendStoreSgprAtLaneDwordOffset(&code, 20 + i, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, expected[i]);
}
for (u32 reg : {2u, 3u}) {
AppendStoreVgprAtLaneDwordOffset(&code, reg, 0,
static_cast<u32>(test.expected.size()));
for (u32 lane = 0; lane < wave_size; ++lane) {
const bool upper = lane >= 32;
const u32 mask = upper || item.body_mode == 3 ? item.hi : item.lo;
const bool active = ((mask >> (lane % 32)) & 1u) != 0 &&
!(upper && item.body_mode == 1);
const u32 pass_value = upper && item.lo != 0 ? 4u : 2u;
test.expected.push_back(active ? (reg == 2 ? 1u : pass_value) : 0u);
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::S_SUBVECTOR_LOOP_BEGIN, O::S_SUBVECTOR_LOOP_END,
O::S_MOV_B32, O::S_MOV_B64, O::S_MULK_I32, O::S_CMP_EQ_U32,
O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarGetpcWritesNextInstructionPc() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSop1(0x1f, 0, 0));
AppendStoreSgprPair(&code, 0, 0);
AppendEnd(&code);
return {"ScalarGetpcWritesNextInstructionPc",
code,
{},
{4, 0},
{O::S_GETPC_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarBitcmpB64DynamicOperands() {
using O = ShaderOpcode;
struct Input {
u32 low;
u32 high;
u32 index;
};
const Input inputs[] = {
{1u, 0u, 0u}, {0x80000000u, 0u, 31u}, {0u, 1u, 32u},
{0u, 0x80000000u, 63u}, {1u, 0u, 64u},
{0x80000000u, 0u, 95u}, {0u, 1u, 96u},
{0u, 0x80000000u, 127u}, {1u, 0u, 128u},
{0u, 0x80000000u, 0xffffffffu}, {0u, 0u, 17u},
{0xffffffffu, 0xffffffffu, 42u},
{0x89abcdefu, 0x76543210u, 4u},
{0x89abcdefu, 0x76543210u, 36u},
};
TestCase test;
test.name = "ScalarBitcmpB64DynamicOperands";
for (const auto &input : inputs) {
test.initial.insert(test.initial.end(), {input.low, input.high, input.index});
}
test.expected = test.initial;
for (u32 i = 0; i < std::size(inputs); ++i) {
// Load the value and index on the GPU so the compares cannot constant-fold.
for (u32 component = 0; component < 3; ++component) {
AppendVMovU32(&test.code, 30, (i * 3 + component) * 4);
AppendBufferLoadDword(&test.code, 0, 30);
test.code.push_back(EncodeVop1(0x02, 20 + component, Vgpr(0)));
}
const uint64_t value = uint64_t{inputs[i].low} | (uint64_t{inputs[i].high} << 32);
const u32 bit = static_cast<u32>((value >> (inputs[i].index & 63u)) & 1u);
for (u32 expected_bit = 0; expected_bit < 2; ++expected_bit) {
const bool scc = bit == expected_bit;
test.code.push_back(EncodeSopc(0x0e + expected_bit, 20, 22));
test.code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
test.expected.push_back(scc ? 1u : 0u);
// Exercise SCC-driven control flow as well as S_CSELECT_B32.
test.code.push_back(EncodeSopp(0x05, 2)); // s_cbranch_scc1 to the true arm.
test.code.push_back(EncodeSMovB32(24, InlineU32(3)));
test.code.push_back(EncodeSopp(0x02, 1)); // s_branch past the true arm.
test.code.push_back(EncodeSMovB32(24, InlineU32(7)));
AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
test.expected.push_back(scc ? 7u : 3u);
}
}
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_BITCMP0_B64, O::S_BITCMP1_B64, O::S_CSELECT_B32,
O::S_CBRANCH_SCC1, O::S_BRANCH, O::S_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitFieldUExtract", "OpULessThan", "OpSelect"};
return test;
}
TestCase ScalarBitcmpB64IntegerConstants() {
using O = ShaderOpcode;
struct Source {
u32 encoding;
uint64_t bits;
};
// Negative integer inlines sign-extend; B64 literals zero-extend.
const Source sources[] = {
{128u, 0u}, {129u, 1u}, {192u, 64u},
{193u, 0xffffffffffffffffull}, {208u, 0xfffffffffffffff0ull},
{255u, 0xffffffffu}, {255u, 0x80000000u},
};
const u32 indices[] = {0u, 4u, 6u, 31u, 32u, 63u, 64u, 127u};
TestCase test;
test.name = "ScalarBitcmpB64IntegerConstants";
test.initial.assign(std::begin(indices), std::end(indices));
test.expected = test.initial;
for (u32 i = 0; i < std::size(indices); ++i) {
AppendVMovU32(&test.code, 30, i * 4);
AppendBufferLoadDword(&test.code, 0, 30);
test.code.push_back(EncodeVop1(0x02, 22, Vgpr(0)));
for (const auto &source : sources) {
const u32 bit = static_cast<u32>((source.bits >> (indices[i] & 63u)) & 1u);
for (u32 expected_bit = 0; expected_bit < 2; ++expected_bit) {
test.code.push_back(EncodeSopc(0x0e + expected_bit, source.encoding, 22));
if (source.encoding == 255u) {
test.code.push_back(static_cast<u32>(source.bits));
}
test.code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
test.expected.push_back(bit == expected_bit ? 1u : 0u);
}
}
}
// An instruction has one literal word even when both sources select it.
for (u32 expected_bit = 0; expected_bit < 2; ++expected_bit) {
test.code.push_back(EncodeSopc(0x0e + expected_bit, 255u, 255u));
test.code.push_back(0x8000001fu); // Select bit 31 of the same zero-extended literal.
test.code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
test.expected.push_back(expected_bit);
}
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_BITCMP0_B64, O::S_BITCMP1_B64, O::S_CSELECT_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitFieldUExtract"};
return test;
}
TestCase ScalarBitfieldPack() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(15)));
code.push_back(EncodeSop1(0x0b, 1, 0));
AppendSMovLiteral(&code, 2, 0xf0000000u);
code.push_back(EncodeSop1(0x10, 3, 0));
AppendSMovLiteral(&code, 19, 0xf0f00001u);
code.push_back(EncodeSop1(0x0f, 20, 19));
code.push_back(EncodeSop2(0x0a, 21, InlineU32(1), InlineU32(0)));
code.push_back(EncodeSMovB32(22, InlineU32(0)));
code.push_back(EncodeSop1(0x0f, 23, 22));
code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
code.push_back(EncodeSMovB32(4, InlineU32(3)));
code.push_back(EncodeSop1(0x3b, 5, 4));
code.push_back(EncodeSop2(0x24, 7, InlineU32(4), InlineU32(8)));
AppendSMovLiteral(&code, 8, 0x00f00000u);
AppendSMovLiteral(&code, 9, 0x00040014u);
code.push_back(EncodeSop2(0x27, 10, 8, 9));
AppendSMovLiteral(&code, 11, 0xaaaabbbbu);
AppendSMovLiteral(&code, 12, 0xccccddddu);
code.push_back(EncodeSop2(0x32, 13, 11, 12));
code.push_back(EncodeSop2(0x33, 14, 11, 12));
code.push_back(EncodeSop2(0x34, 15, 11, 12));
code.push_back(EncodeSopc(0x0d, InlineU32(4), InlineU32(2)));
code.push_back(EncodeSop2(0x0a, 17, InlineU32(1), InlineU32(0)));
code.push_back(EncodeSopc(0x0c, InlineU32(4), InlineU32(1)));
code.push_back(EncodeSop2(0x0a, 18, InlineU32(1), InlineU32(0)));
AppendSMovLiteral(&code, 25, 0u);
AppendSMovLiteral(&code, 26, 0x00000100u);
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(1)));
code.push_back(EncodeSop1(0x14, 31, 25));
code.push_back(EncodeSop2(0x0a, 34, InlineU32(1), InlineU32(0)));
AppendSMovLiteral(&code, 27, 0x00000020u);
AppendSMovLiteral(&code, 28, 0x00000100u);
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1)));
code.push_back(EncodeSop1(0x14, 32, 27));
code.push_back(EncodeSop2(0x0a, 35, InlineU32(1), InlineU32(0)));
AppendSMovLiteral(&code, 29, 0u);
AppendSMovLiteral(&code, 30, 0u);
code.push_back(EncodeSop1(0x14, 33, 29));
AppendStoreSgpr(&code, 1, 0);
AppendStoreSgpr(&code, 3, 1);
AppendStoreSgpr(&code, 20, 2);
AppendStoreSgpr(&code, 21, 3);
AppendStoreSgpr(&code, 23, 4);
AppendStoreSgpr(&code, 24, 5);
AppendStoreSgprPair(&code, 5, 6);
AppendStoreSgpr(&code, 7, 8);
AppendStoreSgpr(&code, 10, 9);
AppendStoreSgpr(&code, 13, 10);
AppendStoreSgpr(&code, 14, 11);
AppendStoreSgpr(&code, 15, 12);
AppendStoreSgpr(&code, 17, 13);
AppendStoreSgpr(&code, 18, 14);
AppendStoreSgpr(&code, 31, 15);
AppendStoreSgpr(&code, 32, 16);
AppendStoreSgpr(&code, 33, 17);
AppendStoreSgpr(&code, 34, 18);
AppendStoreSgpr(&code, 35, 19);
AppendEnd(&code);
return {"ScalarBitfieldPack",
code,
{},
{0xf0000000u, 8, 9, 1, 0,
0, 0x0000000fu, 0, 0x00000f00u, 0x0000000fu,
0xddddbbbbu, 0xccccbbbbu, 0xccccaaaau, 1, 1,
40, 5, 0xffffffffu, 0, 1},
{O::S_MOV_B32, O::S_BREV_B32, O::S_BCNT1_I32_B32, O::S_BCNT1_I32_B64,
O::S_FF1_I32_B64, O::S_BITREPLICATE_B64_B32, O::S_BFM_B32,
O::S_BFE_U32, O::S_PACK_LL_B32_B16, O::S_PACK_LH_B32_B16,
O::S_PACK_HH_B32_B16, O::S_BITCMP0_B32, O::S_BITCMP1_B32,
O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarBrevB32PreservesScc() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
code.push_back(EncodeSMovB32(0, InlineU32(1)));
set_scc(false);
code.push_back(EncodeSop1(0x0b, 1, 0));
capture_scc(2);
code.push_back(EncodeSMovB32(3, InlineU32(0)));
set_scc(true);
code.push_back(EncodeSop1(0x0b, 4, 3));
capture_scc(5);
AppendStoreSgpr(&code, 1, 0);
AppendStoreSgpr(&code, 2, 1);
AppendStoreSgpr(&code, 4, 2);
AppendStoreSgpr(&code, 5, 3);
AppendEnd(&code);
return {"ScalarBrevB32PreservesScc",
code,
{},
{0x80000000u, 0, 0, 1},
{O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_BREV_B32, O::S_CSELECT_B32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarSextMasks(u32 bits, u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = bits == 8 ? (wave_size == 32 ? "ScalarSextI8MasksWave32"
: "ScalarSextI8MasksWave64")
: (wave_size == 32 ? "ScalarSextI16MasksWave32"
: "ScalarSextI16MasksWave64");
constexpr u32 high = 0x80000001u;
test.initial = bits == 8
? std::vector<u32>{0xdeadff00u, 0xabcdff7fu, 0x12340080u, 0xabcf00ffu}
: std::vector<u32>{0xdead0000u, 0xabcd7fffu, 0x12348000u, 0xabcfffffu};
const u32 extended_values[] = {0u, bits == 8 ? 0x7fu : 0x7fffu,
bits == 8 ? 0xffffff80u : 0xffff8000u, 0xffffffffu};
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < 4u; ++i) {
const auto extended = extended_values[i];
for (const bool scc : {false, true}) {
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 3, 30);
code.push_back(EncodeVop1(0x02, 11, Vgpr(3)));
AppendSMovLiteral(&code, 107, high);
code.push_back(EncodeSopc(0x06, InlineU32(scc ? 1u : 0u), InlineU32(1)));
code.push_back(EncodeSop1(0x04, 126, 128)); // Scalar operations execute with empty EXEC.
// The I16 word is captured s_sext_i32_i16 vcc_lo, s11.
code.push_back(bits == 8 ? 0xbeea190bu : 0xbeea1a0bu);
code.push_back(EncodeSMovB32(20, 106));
code.push_back(EncodeSMovB32(21, 107));
code.push_back(EncodeSMovB32(22, 253));
code.push_back(EncodeSop1(0x04, 126, 193));
AppendVMovU32(&code, 1, 1);
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
const u32 expected[] = {extended, high, scc ? 1u : 0u};
for (u32 word = 0; word < 3u; ++word) {
AppendStoreSgprAtLaneDwordOffset(&code, 20u + word, 0, test.expected.size());
test.expected.insert(test.expected.end(), wave_size, expected[word]);
}
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, test.expected.size());
const uint64_t mask = (uint64_t{high} << 32u) | extended;
for (u32 lane = 0; lane < wave_size; ++lane)
test.expected.push_back(static_cast<u32>((mask >> lane) & 1u));
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_MOV_B32,
O::S_MOV_B64, O::S_CMP_EQ_U32,
bits == 8 ? O::S_SEXT_I32_I8 : O::S_SEXT_I32_I16,
O::V_CNDMASK_B32, O::V_MOV_B32, O::V_ADD_NC_U32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{bits == 8 ? "S_SEXT_I32_I8 vcc_lo, s11"
: "S_SEXT_I32_I16 vcc_lo, s11", 8u}};
test.required_spirv = {"OpBitFieldSExtract"};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarSextConstantsAndAliases() {
using O = ShaderOpcode;
TestCase test;
test.name = "ScalarSextConstantsAndAliases";
auto &code = test.code;
for (const u32 opcode : {0x19u, 0x1au}) {
const u32 literal = opcode == 0x19u ? 0x12345680u : 0x12348000u;
const u32 extended = opcode == 0x19u ? 0xffffff80u : 0xffff8000u;
for (const bool scc : {false, true}) {
code.push_back(EncodeSopc(0x06, InlineU32(scc ? 1u : 0u), InlineU32(1)));
code.push_back(EncodeSop1(opcode, 10, 255));
code.push_back(literal);
code.push_back(EncodeSop1(opcode, 10, 10)); // In-place source/destination.
code.push_back(EncodeSop1(opcode, 11, InlineU32(0)));
code.push_back(EncodeSop1(opcode, 12, InlineU32(64)));
code.push_back(EncodeSop1(opcode, 13, 193)); // Inline -1.
code.push_back(EncodeSMovB32(14, 253));
const u32 expected[] = {extended, 0, 64, 0xffffffffu, scc ? 1u : 0u};
for (u32 word = 0; word < 5u; ++word) {
AppendStoreSgpr(&code, 10u + word, test.expected.size());
test.expected.push_back(expected[word]);
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::S_CMP_EQ_U32, O::S_SEXT_I32_I8, O::S_SEXT_I32_I16,
O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_SEXT_I32_I8", 10u}, {"S_SEXT_I32_I16", 10u}};
return test;
}
TestCase ScalarBrevB64OperandsAndMasks(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 64 ? "ScalarBrevB64OperandsAndMasksWave64"
: "ScalarBrevB64OperandsAndMasksWave32";
constexpr uint64_t values[] = {0, 1, 0x8000000000000000ull,
0x0123456789abcdefull};
for (const auto value : values) {
test.initial.push_back(static_cast<u32>(value));
test.initial.push_back(static_cast<u32>(value >> 32u));
}
test.expected = test.initial;
auto &code = test.code;
size_t instructions = 0;
const auto check = [&](u32 dst, u32 src, uint64_t value, bool scc,
std::optional<u32> literal = {}) {
uint64_t reversed = 0;
for (u32 bit = 0; bit < 64; ++bit) {
reversed |= ((value >> bit) & 1ull) << (63u - bit);
}
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 0);
if (dst == 106u || dst == 126u) {
code.push_back(EncodeSop1(0x04, dst, src));
src = dst;
}
code.push_back(EncodeSopc(0x06, InlineU32(scc ? 1u : 0u), InlineU32(1)));
code.push_back(EncodeSop1(0x0c, dst, src));
if (literal) {
code.push_back(*literal);
}
code.push_back(EncodeSMovB32(24, dst));
code.push_back(EncodeSMovB32(25, dst + 1u));
code.push_back(EncodeSMovB32(26, 253));
if (dst == 106u) {
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
} else if (dst == 126u) {
code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
}
code.push_back(EncodeSop1(0x04, 126, 193)); // Restore EXEC for readback.
const u32 expected[] = {static_cast<u32>(reversed),
static_cast<u32>(reversed >> 32u), scc ? 1u : 0u};
for (u32 i = 0; i < std::size(expected); ++i) {
AppendStoreSgprAtLaneDwordOffset(
&code, 24 + i, 0, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, expected[i]);
}
if (dst == 106u || dst == 126u) {
AppendStoreVgprAtLaneDwordOffset(
&code, 2, 0, static_cast<u32>(test.expected.size()));
for (u32 lane = 0; lane < wave_size; ++lane) {
test.expected.push_back(static_cast<u32>((reversed >> lane) & 1u));
}
}
++instructions;
};
for (const u32 dst : {20u, 106u, 126u}) {
for (u32 i = 0; i < std::size(values); ++i) {
// Runtime inputs exercise the emitter, including in-place mask writes.
for (u32 word = 0; word < 2; ++word) {
AppendVMovU32(&code, 30, (2 * i + word) * 4u);
AppendBufferLoadDword(&code, 3, 30);
code.push_back(EncodeVop1(0x02, 20 + word, Vgpr(3)));
}
check(dst, 20, values[i], values[i] == 0);
}
}
check(20, 129, 1, false);
check(20, 193, ~uint64_t{0}, false);
check(20, 125, 0, true);
check(20, 255, 0x80000001u, true, 0x80000001u);
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_MOV_B32,
O::S_MOV_B64, O::S_BREV_B64, O::S_CMP_EQ_U32, O::V_MOV_B32,
O::V_CNDMASK_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_BREV_B64", instructions}};
test.required_spirv = {"OpBitReverse"};
test.forbidden_spirv = {"OpTypeInt 64"};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarBfeI32CapturedRawSignExtends() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 0, 30);
code.push_back(EncodeVop1(0x02, 20, Vgpr(0)));
code.push_back(0x946aff14u); // s_bfe_i32 vcc_lo, s20, literal
code.push_back(0x0004001cu); // offset 28, width 4
code.push_back(EncodeSop2(0x0a, 21, InlineU32(1), InlineU32(0)));
code.push_back(EncodeSop2(0x28, 22, 20, 255u));
code.push_back(0u); // zero width produces zero and clears SCC
code.push_back(EncodeSop2(0x0a, 23, InlineU32(1), InlineU32(0)));
AppendStoreSgpr(&code, 106, 1);
AppendStoreSgpr(&code, 21, 2);
AppendStoreSgpr(&code, 22, 3);
AppendStoreSgpr(&code, 23, 4);
AppendEnd(&code);
TestCase test;
test.name = "ScalarBfeI32CapturedRawSignExtends";
test.code = std::move(code);
test.initial = {0xf0000000u, 0, 0, 0, 0};
test.expected = {0xf0000000u, 0xffffffffu, 1, 0, 0};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_BFE_I32, O::S_CSELECT_B32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpBitFieldSExtract"};
return test;
}
TestCase BitfieldExtractWidthPastEndEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0xf0000000u);
AppendSMovLiteral(&code, 1, (20u << 16u) | 20u);
code.push_back(EncodeSop2(0x27, 2, 0, 1));
AppendSMovLiteral(&code, 3, 0x80000000u);
AppendSMovLiteral(&code, 4, (7u << 16u) | 31u);
code.push_back(EncodeSop2(0x27, 5, 3, 4));
AppendVMovLiteral(&code, 6, 0xf0000000u);
AppendVMovU32(&code, 7, 28);
AppendVMovU32(&code, 8, 8);
AppendVop3(&code, 0x148, 9, Vgpr(6), Vgpr(7), Vgpr(8));
AppendStoreSgpr(&code, 2, 0);
AppendStoreSgpr(&code, 5, 1);
AppendStoreVgpr(&code, 9, 2);
AppendEnd(&code);
return {"BitfieldExtractWidthPastEndEdges",
code,
{},
{0x00000f00u, 1, 0x0000000fu},
{O::S_MOV_B32, O::S_BFE_U32, O::V_MOV_B32, O::V_BFE_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Scalar64BitOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0x0f0f0f0fu);
AppendSMovLiteral(&code, 1, 0x00ff00ffu);
AppendSMovLiteral(&code, 2, 0x33333333u);
AppendSMovLiteral(&code, 3, 0x0f0f0f0fu);
code.push_back(EncodeSop2(0x0f, 4, 0, 2));
code.push_back(EncodeSop2(0x15, 6, 0, 2));
code.push_back(EncodeSop2(0x11, 8, 0, 2));
code.push_back(EncodeSop2(0x17, 10, 0, 2));
code.push_back(EncodeSop2(0x13, 12, 0, 2));
code.push_back(EncodeSop2(0x19, 14, 0, 2));
code.push_back(EncodeSop2(0x1b, 16, 0, 2));
code.push_back(EncodeSop2(0x1d, 18, 0, 2));
code.push_back(EncodeSop1(0x04, 20, 0));
code.push_back(EncodeSop2(0x1f, 22, 0, InlineU32(4)));
code.push_back(EncodeSop2(0x21, 24, 0, InlineU32(8)));
code.push_back(EncodeSop2(0x25, 26, InlineU32(36), InlineU32(4)));
code.push_back(EncodeSop2(0x21, 34, 193u, InlineU32(1)));
AppendSMovLiteral(&code, 28, 0x000c0004u);
code.push_back(EncodeSop2(0x29, 30, 0, 28));
code.push_back(EncodeSopc(0x06, 0, 0));
code.push_back(EncodeSop2(0x0b, 32, 0, 2));
const u32 result_pairs[] = {4, 6, 8, 10, 12, 14, 16, 18,
20, 22, 24, 26, 34, 30, 32};
u32 out = 0;
for (auto sgpr : result_pairs) {
AppendStoreSgprPair(&code, sgpr, out);
out += 2;
}
AppendEnd(&code);
return {"Scalar64BitOps",
code,
{},
{0x03030303u, 0x000f000fu, 0x0c0c0c0cu, 0x00f000f0u, 0x3f3f3f3fu,
0x0fff0fffu, 0xcfcfcfcfu, 0xf0fff0ffu, 0x3c3c3c3cu, 0x0ff00ff0u,
0xfcfcfcfcu, 0xfff0fff0u, 0xc0c0c0c0u, 0xf000f000u, 0xc3c3c3c3u,
0xf00ff00fu, 0x0f0f0f0fu, 0x00ff00ffu, 0xf0f0f0f0u, 0x0ff00ff0u,
0xff0f0f0fu, 0x0000ff00u, 0xfffffff0u, 0x000000ffu, 0xffffffffu,
0x7fffffffu, 0x000000f0u, 0, 0x0f0f0f0fu, 0x00ff00ffu},
{O::S_MOV_B32, O::S_MOV_B64, O::S_AND_B64, O::S_ANDN2_B64,
O::S_OR_B64, O::S_ORN2_B64, O::S_XOR_B64, O::S_NAND_B64,
O::S_NOR_B64, O::S_XNOR_B64, O::S_LSHL_B64, O::S_LSHR_B64,
O::S_BFM_B64, O::S_BFE_U64, O::S_CMP_EQ_U32, O::S_CSELECT_B64,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarDynamic64BitOps() {
using O = ShaderOpcode;
struct Input {
uint64_t lhs;
uint64_t rhs;
u32 count;
u32 offset;
};
constexpr Input inputs[] = {
{0, UINT64_MAX, 0, 0},
{UINT64_MAX, 0, 63, 0},
{0x8000000000000000ull, 1, 64, 0},
{0x0123456789abcdefull, 0xfedcba9876543210ull, 127, 0},
{UINT64_MAX, 0xffffffffull, 64, 1},
{0x0123456789abcdefull, 0x0123456789abcdefull, 63, 32},
{UINT64_MAX, 0x100000000ull, 127, 63},
{0, 0, 0, 127},
{0x8000000000000001ull, UINT64_MAX, UINT32_MAX, 64},
};
TestCase test;
test.name = "ScalarDynamic64BitOps";
for (const auto &input : inputs) {
test.initial.insert(test.initial.end(),
{u32(input.lhs), u32(input.lhs >> 32), u32(input.rhs),
u32(input.rhs >> 32), input.count, input.offset,
((input.count & 127u) << 16) | input.offset});
}
test.expected = test.initial;
auto &code = test.code;
const auto store = [&](uint64_t value, bool scc) {
const auto output = static_cast<u32>(test.expected.size());
code.push_back(EncodeSMovB32(18, 253));
AppendStoreSgprPair(&code, 16, output);
AppendStoreSgpr(&code, 18, output + 2);
test.expected.insert(test.expected.end(),
{u32(value), u32(value >> 32), u32(scc)});
};
for (u32 i = 0; i < std::size(inputs); ++i) {
const auto [lhs, rhs, count, offset] = inputs[i];
for (u32 word = 0; word < 7; ++word) {
AppendVMovU32(&code, 30, (i * 7 + word) * sizeof(u32));
AppendBufferLoadDword(&code, 1, 30);
code.push_back(EncodeVop1(0x02, 8 + word, Vgpr(1)));
}
for (const auto [opcode, value] :
std::array<std::pair<u32, uint64_t>, 8>{{{0x0f, lhs & rhs},
{0x11, lhs | rhs},
{0x13, lhs ^ rhs},
{0x15, lhs & ~rhs},
{0x17, lhs | ~rhs},
{0x19, ~(lhs & rhs)},
{0x1b, ~(lhs | rhs)},
{0x1d, ~(lhs ^ rhs)}}}) {
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(value != 0)));
code.push_back(EncodeSop2(opcode, 16, 8, 10));
store(value, value != 0);
}
for (const bool select_lhs : {false, true}) {
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(!select_lhs)));
code.push_back(EncodeSop2(0x0b, 16, 8, 10));
store(select_lhs ? lhs : rhs, select_lhs);
}
// BFM preserves SCC; BFE sets it from the entire extracted result.
code.push_back(EncodeSop2(0x25, 16, 12, 13));
store(((uint64_t{1} << (count & 63u)) - 1u) << (offset & 63u), true);
code.push_back(EncodeSop2(0x29, 16, 8, 14));
const auto width = std::min(count & 127u, 64u - (offset & 63u));
const auto mask = width == 64 ? UINT64_MAX : (uint64_t{1} << width) - 1u;
const auto extracted = (lhs >> (offset & 63u)) & mask;
store(extracted, extracted != 0);
for (const bool set : {false, true}) {
code.push_back(EncodeSop1(0x04, 16, 8));
code.push_back(EncodeSop1(set ? 0x1e : 0x1c, 16, 12));
const auto bit = uint64_t{1} << (count & 63u);
store(set ? lhs | bit : lhs & ~bit, extracted != 0);
}
code.push_back(EncodeSop1(0x2d, 16, 8));
uint64_t quads = 0;
for (u32 quad = 0; quad < 16; ++quad) {
quads |= uint64_t((lhs >> (quad * 4) & 15u) != 0) << quad;
}
store(quads, quads != 0);
code.push_back(EncodeSop1(0x3b, 16, 8));
uint64_t replicated = 0;
for (u32 bit = 0; bit < 32; ++bit) {
replicated |= ((lhs >> bit) & 1u) * (uint64_t{3} << (2 * bit));
}
store(replicated, quads != 0);
code.push_back(EncodeSMovB32(17, InlineU32(0)));
code.push_back(EncodeSop1(0x10, 16, 8));
store(std::popcount(lhs), lhs != 0);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {
O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_AND_B64,
O::S_OR_B64, O::S_XOR_B64, O::S_ANDN2_B64,
O::S_ORN2_B64, O::S_NAND_B64, O::S_NOR_B64,
O::S_XNOR_B64, O::S_CSELECT_B64, O::S_BFM_B64,
O::S_BFE_U64, O::S_MOV_B64, O::S_BITSET0_B64,
O::S_BITSET1_B64, O::S_QUADMASK_B64, O::S_BITREPLICATE_B64_B32,
O::S_BCNT1_I32_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpTypeInt 64 0", "OpBitwiseAnd", "OpBitwiseOr",
"OpBitwiseXor"};
return test;
}
uint64_t ReferenceAshrI64(uint64_t bits, u32 count) {
const auto shift = count & 63u;
// Complementing negative values expresses sign fill using unsigned shifts.
return (bits & (uint64_t{1} << 63u)) != 0 ? ~(~bits >> shift) : bits >> shift;
}
TestCase ScalarAshrI64Edges(bool dynamic) {
using O = ShaderOpcode;
TestCase test;
test.name =
dynamic ? "ScalarAshrI64DynamicEdges" : "ScalarAshrI64ConstantEdges";
constexpr uint64_t sources[] = {0,
1,
0x80000000ull,
0x100000000ull,
0x7fffffffffffffffull,
0x8000000000000000ull,
0xffffffffffffffffull,
0x0123456789abcdefull,
0xfedcba9876543210ull};
constexpr u32 counts[] = {0, 1, 7, 31, 32, 33, 63,
64, 65, 95, 96, 127, 0xffffffffu};
test.initial.assign(std::begin(counts), std::end(counts));
for (const auto source : sources) {
test.initial.push_back(static_cast<u32>(source));
test.initial.push_back(static_cast<u32>(source >> 32u));
}
test.expected = test.initial;
auto &code = test.code;
const auto load_scalar = [&](u32 reg, u32 index) {
AppendVMovU32(&code, 30, index * sizeof(u32));
AppendBufferLoadDword(&code, 1, 30);
code.push_back(EncodeVop1(0x02, reg, Vgpr(1)));
};
for (u32 source_index = 0; source_index < std::size(sources);
++source_index) {
const auto source = sources[source_index];
// Runtime sources exercise both constant-count and dynamic-count emission.
load_scalar(20, static_cast<u32>(std::size(counts)) + source_index * 2u);
load_scalar(21,
static_cast<u32>(std::size(counts)) + source_index * 2u + 1u);
for (u32 index = 0; index < std::size(counts); ++index) {
const auto expected = ReferenceAshrI64(source, counts[index]);
if (dynamic) {
load_scalar(22, index);
}
// Start SCC at the opposite value so every result must update it.
code.push_back(
EncodeSopc(0x06, InlineU32(0), InlineU32(expected != 0 ? 1u : 0u)));
const auto count_src = dynamic ? 22u
: counts[index] <= 64u ? InlineU32(counts[index])
: 255u;
code.push_back(EncodeSop2(0x23, 24, 20, count_src));
if (!dynamic && count_src == 255u) {
code.push_back(counts[index]);
}
code.push_back(EncodeSMovB32(26, 253));
const auto output = static_cast<u32>(test.expected.size());
AppendStoreSgprPair(&code, 24, output);
AppendStoreSgpr(&code, 26, output + 2u);
test.expected.push_back(static_cast<u32>(expected));
test.expected.push_back(static_cast<u32>(expected >> 32u));
test.expected.push_back(expected != 0 ? 1u : 0u);
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_MOV_B32, O::S_CMP_EQ_U32,
O::S_ASHR_I64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {
{"S_ASHR_I64", std::size(sources) * std::size(counts)}};
test.required_spirv = {"OpShiftRightArithmetic", "OpTypeInt 64 0"};
return test;
}
TestCase ScalarAshrI64OperandsAndAliases() {
using O = ShaderOpcode;
TestCase test;
test.name = "ScalarAshrI64OperandsAndAliases";
test.initial = {33}; // Runtime shift for signed literal source tests.
test.expected = test.initial;
auto &code = test.code;
AppendVMovU32(&code, 30, 0);
AppendBufferLoadDword(&code, 1, 30);
code.push_back(EncodeVop1(0x02, 22, Vgpr(1)));
size_t instructions = 0;
const auto check_result = [&](u32 dst, u32 src0, u32 src1, uint64_t value,
u32 count, std::optional<u32> literal = {}) {
code.push_back(EncodeSop2(0x23, dst, src0, src1));
if (literal) {
code.push_back(*literal);
}
code.push_back(EncodeSMovB32(26, 253));
const auto output = static_cast<u32>(test.expected.size());
AppendStoreSgprPair(&code, dst, output);
AppendStoreSgpr(&code, 26, output + 2u);
const auto expected = ReferenceAshrI64(value, count);
test.expected.push_back(static_cast<u32>(expected));
test.expected.push_back(static_cast<u32>(expected >> 32u));
test.expected.push_back(expected != 0 ? 1u : 0u);
++instructions;
};
for (const u32 literal : {0u, 0x7fffffffu, 0x80000000u, 0xffffffffu}) {
const uint64_t value =
literal | ((literal & 0x80000000u) != 0 ? 0xffffffff00000000ull : 0ull);
for (const u32 count : {0u, 1u, 31u, 32u, 63u, 64u}) {
check_result(24, 255, InlineU32(count), value, count, literal);
}
check_result(24, 255, 22, value, 33, literal);
}
for (const u32 source : {128u, 129u, 192u, 193u, 208u, 125u}) {
const uint64_t value = source == 125u ? 0ull
: source <= 192u ? source - 128u
: uint64_t{192} - source;
for (const u32 count : {0u, 1u, 32u, 63u}) {
check_result(24, source, InlineU32(count), value, count);
}
}
for (const u32 scc : {0u, 1u}) {
// SCC may be either source and must be read before it is overwritten.
code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
check_result(24, 253, InlineU32(1), scc, 1);
AppendSMovLiteral(&code, 20, 1);
AppendSMovLiteral(&code, 21, 0);
code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
check_result(24, 20, 253, 1, scc);
}
constexpr uint64_t value = 0xfedcba9889abcdefull;
for (const u32 count : {0u, 1u, 31u, 32u, 33u, 63u, 65u}) {
AppendSMovLiteral(&code, 20, static_cast<u32>(value));
AppendSMovLiteral(&code, 21, static_cast<u32>(value >> 32u));
AppendSMovLiteral(&code, 23, count);
check_result(20, 20, 23, value, count); // In-place source/destination pair.
for (const u32 count_reg : {24u, 25u}) {
AppendSMovLiteral(&code, 20, static_cast<u32>(value));
AppendSMovLiteral(&code, 21, static_cast<u32>(value >> 32u));
AppendSMovLiteral(&code, count_reg, count);
check_result(24, 20, count_reg, value,
count); // Count aliases destination.
}
}
for (const u32 count_reg : {20u, 21u}) {
const uint64_t alias_value =
count_reg == 20u ? 0x8000000000000021ull : 0x0000002100000001ull;
AppendSMovLiteral(&code, 20, static_cast<u32>(alias_value));
AppendSMovLiteral(&code, 21, static_cast<u32>(alias_value >> 32u));
check_result(20, 20, count_reg, alias_value, 33);
}
// Both operands may refer to the single literal word in the instruction.
check_result(24, 255, 255, 0xffffffffffffffffull, 0xffffffffu, 0xffffffffu);
for (const u32 source : {0u, 1u}) {
// Discarding the destination must still update SCC and leave M0 untouched.
AppendSMovLiteral(&code, 124, 0x5a13abcdu);
AppendSMovLiteral(&code, 20, 0);
AppendSMovLiteral(&code, 21, source);
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(source)));
code.push_back(EncodeSop2(0x23, 125, 20, InlineU32(0)));
code.push_back(EncodeSMovB32(26, 253));
const auto output = static_cast<u32>(test.expected.size());
AppendStoreSgpr(&code, 26, output);
AppendStoreSgpr(&code, 124, output + 1u);
AppendStoreSgprPair(&code, 20, output + 2u);
test.expected.insert(test.expected.end(), {source, 0x5a13abcdu, 0, source});
++instructions;
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_MOV_B32, O::S_CMP_EQ_U32,
O::S_ASHR_I64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_ASHR_I64", instructions}};
test.required_spirv.push_back("OpTypeInt 64 0");
return test;
}
TestCase ScalarAshrI64Masks(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name =
wave_size == 64 ? "ScalarAshrI64MasksWave64" : "ScalarAshrI64MasksWave32";
auto &code = test.code;
struct Case {
uint64_t source;
u32 count;
};
constexpr Case cases[] = {
{0x8000000100000001ull, 1}, {0x8000000100000001ull, 32},
{0x8000000100000001ull, 63}, {0x0000000100000000ull, 0},
{0x0000000100000000ull, 32}, {1, 1}};
for (const u32 mask_reg : {106u, 126u}) {
for (const auto &item : cases) {
const auto result = ReferenceAshrI64(item.source, item.count);
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 0);
AppendSMovLiteral(&code, mask_reg, static_cast<u32>(item.source));
AppendSMovLiteral(&code, mask_reg + 1u,
static_cast<u32>(item.source >> 32u));
code.push_back(
EncodeSopc(0x06, InlineU32(0), InlineU32(result != 0 ? 1u : 0u)));
code.push_back(
EncodeSop2(0x23, mask_reg, mask_reg, InlineU32(item.count)));
code.push_back(EncodeSMovB32(20, mask_reg));
code.push_back(EncodeSMovB32(21, mask_reg + 1u));
code.push_back(EncodeSMovB32(22, 253));
if (mask_reg == 106u) {
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
} else {
code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
}
code.push_back(EncodeSop1(0x04, 126, 193)); // Restore EXEC for readback.
const u32 expected[] = {static_cast<u32>(result),
static_cast<u32>(result >> 32u),
result != 0 ? 1u : 0u};
for (u32 i = 0; i < std::size(expected); ++i) {
AppendStoreSgprAtLaneDwordOffset(
&code, 20 + i, 0, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, expected[i]);
}
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
static_cast<u32>(test.expected.size()));
for (u32 lane = 0; lane < wave_size; ++lane) {
test.expected.push_back(static_cast<u32>((result >> lane) & 1u));
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_CMP_EQ_U32,
O::S_ASHR_I64, O::V_MOV_B32, O::V_CNDMASK_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {{"S_ASHR_I64 vcc_lo, vcc_lo", std::size(cases)},
{"S_ASHR_I64 exec_lo, exec_lo", std::size(cases)}};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarConditionalMoveB64() {
using O = ShaderOpcode;
TestCase test;
test.name = "ScalarConditionalMoveB64";
auto &code = test.code;
for (u32 scc : {0u, 1u}) {
AppendSMovLiteral(&code, 0, 0x13579bdfu);
AppendSMovLiteral(&code, 1, 0x2468ace0u);
AppendSMovLiteral(&code, 2, 0xabcdef01u);
AppendSMovLiteral(&code, 3, 0xfedcba98u);
code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
code.push_back(EncodeSop1(0x06, 0, 2));
code.push_back(EncodeSop2(0x0a, 4, InlineU32(1), InlineU32(0)));
AppendStoreSgprPair(&code, 0, scc * 3);
AppendStoreSgpr(&code, 4, scc * 3 + 2);
}
AppendEnd(&code);
test.expected = {0x13579bdfu, 0x2468ace0u, 0,
0xabcdef01u, 0xfedcba98u, 1};
test.opcodes = {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CMOV_B64,
O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase ScalarConditionalMoveB64PreservesMasks() {
using O = ShaderOpcode;
TestCase test;
test.name = "ScalarConditionalMoveB64PreservesMasks";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 16, 126)); // Save full EXEC.
AppendVMovU32(&code, 1, 3);
AppendVMovU32(&code, 2, 9);
u32 offset = 0;
for (u32 destination : {126u, 106u, 8u}) {
for (u32 scc : {0u, 1u}) {
code.push_back(EncodeVopc(0xc2, Vgpr(0), 1)); // Only lane 3.
code.push_back(EncodeSop1(0x04, destination, 106));
code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
// EXEC encoding is the captured Pathless instruction, 0xbefe06c1.
code.push_back(EncodeSop1(0x06, destination, 193u));
code.push_back(EncodeSop2(0x0a, 4, InlineU32(1), InlineU32(0)));
code.push_back(EncodeSop1(0x04, 126, destination));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
code.push_back(EncodeSop1(0x04, 126, 16));
AppendStoreSgprAtLaneDwordOffset(&code, 4, 0, offset + 64);
for (u32 lane = 0; lane < 64; ++lane) {
test.expected.push_back(scc || lane == 3 ? 9u : 0xdeadbeefu);
}
test.expected.insert(test.expected.end(), 64, scc);
offset += 128;
}
}
AppendEnd(&code);
test.initial.assign(test.expected.size(), 0xdeadbeefu);
test.opcodes = {O::S_MOV_B64, O::V_MOV_B32, O::V_CMP_EQ_U32,
O::S_CMP_EQ_U32, O::S_CMOV_B64, O::S_CSELECT_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = 64;
test.has_compute_info = true;
return test;
}
TestCase ScalarAndn2B64SccUsesMaskShadow() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVopc(0xc2, InlineU32(0), 0),
EncodeSop1(0x04, 0, 106),
EncodeSop2(0x15, 4, 0, InlineU32(0)),
EncodeVop1(0x01, 0, InlineU32(1)),
EncodeSopp(0x04, 1),
EncodeVop1(0x01, 0, InlineU32(7)),
};
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"ScalarAndn2B64SccUsesMaskShadow",
code,
{},
{7},
{O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_ANDN2_B64, O::V_MOV_B32,
O::S_CBRANCH_SCC0, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarMaskHighWriteInvalidatesProvenance() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVopc(0xc2, Vgpr(0), 0), EncodeSop1(0x04, 0, 106),
EncodeSMovB32(1, InlineU32(1)), EncodeSop1(0x08, 2, 0),
EncodeVop1(0x01, 0, InlineU32(1)), EncodeSopp(0x04, 1),
EncodeVop1(0x01, 0, InlineU32(7)),
};
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"ScalarMaskHighWriteInvalidatesProvenance",
code,
{},
{7},
{O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_MOV_B32, O::S_NOT_B64,
O::V_MOV_B32, O::S_CBRANCH_SCC0, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarSelectB64PreservesMaskProvenance(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 32 ? "ScalarSelectB64PreservesMaskProvenanceWave32"
: "ScalarSelectB64PreservesMaskProvenanceWave64";
auto &code = test.code;
const auto set_scc = [&] {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1)));
};
const auto capture_scc = [&](u32 dst) {
code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
};
AppendSMovLiteral(&code, 0, 0xfffffffeu);
AppendSMovLiteral(&code, 1, 0xffffffffu);
code.push_back(EncodeSMovB32(2, InlineU32(0)));
code.push_back(EncodeSMovB32(3, InlineU32(0)));
set_scc();
code.push_back(EncodeSop2(0x0b, 4, 0, 2));
code.push_back(EncodeSop1(0x08, 6, 4));
capture_scc(8);
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0)); // Only lane zero is set.
code.push_back(EncodeSop1(0x04, 10, 106));
set_scc();
code.push_back(EncodeSop2(0x0b, 12, 10, 2));
code.push_back(EncodeSop1(0x08, 14, 12));
capture_scc(16);
AppendStoreSgprAtLaneDwordOffset(&code, 8, 0, 0);
AppendStoreSgprAtLaneDwordOffset(&code, 16, 0, wave_size);
test.expected.assign(2u * wave_size, 1u);
code.push_back(EncodeSop1(0x04, 16, 126)); // Save full EXEC.
AppendVMovU32(&code, 2, 9);
constexpr std::array<uint32_t, 4> words{
0x5aa55aa5u, 0xc39ac39au, 0xa55aa55au, 0x3c653c65u};
for (u32 i = 0; i < words.size(); ++i) AppendSMovLiteral(&code, i, words[i]);
for (const u32 destination : {106u, 126u}) {
for (const u32 scc : {0u, 1u}) {
code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
code.push_back(EncodeSop2(0x0b, destination, 0, 2));
code.push_back(EncodeSop1(0x04, 8, destination)); // Preserve both raw words.
code.push_back(EncodeSop1(0x04, 126, destination));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
static_cast<u32>(test.expected.size()));
const u32 selected = scc ? 0u : 2u;
for (u32 lane = 0; lane < wave_size; ++lane)
test.expected.push_back((words[selected + lane / 32u] >> (lane & 31u)) & 1u
? 9u : 0xdeadbeefu);
code.push_back(EncodeSop1(0x04, 126, 16));
for (u32 component = 0; component < 2; ++component) {
AppendStoreSgprAtLaneDwordOffset(&code, 8 + component, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, words[selected + component]);
}
}
}
AppendEnd(&code);
test.initial.assign(test.expected.size(), 0xdeadbeefu);
test.opcodes = {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CSELECT_B64, O::S_NOT_B64,
O::S_CSELECT_B32, O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarWqmB32Masks(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 64 ? "ScalarWqmB32MasksWave64" : "ScalarWqmB32MasksWave32";
auto &code = test.code;
const auto store_scalar = [&](u32 reg, u32 expected) {
AppendStoreSgprAtLaneDwordOffset(&code, reg, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, expected);
};
const auto store_marker = [&](u32 lo, u32 hi) {
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
static_cast<u32>(test.expected.size()));
for (u32 lane = 0; lane < wave_size; ++lane) {
const u32 mask = lane < 32 ? lo : hi;
test.expected.push_back(((mask >> (lane % 32)) & 1u) != 0 ? 9u : 0u);
}
};
constexpr u32 other_half = 0x2468ace0u;
constexpr u32 masks[] = {0, 1, 8, 0x10, 0x100, 0x80000000u, 0x12481248u, ~0u};
for (u32 mask : masks) {
u32 expanded = 0;
for (u32 bit = 0; bit < 32; bit += 4) {
if ((mask & (0xfu << bit)) != 0) expanded |= 0xfu << bit;
}
AppendVMovU32(&code, 2, 0);
AppendSMovLiteral(&code, 126, mask);
AppendSMovLiteral(&code, 127, other_half);
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(mask == 0 ? 0 : 1)));
code.push_back(0xbefe097eu); // Captured S_WQM_B32 EXEC_LO, EXEC_LO.
code.push_back(EncodeSMovB32(20, 126));
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(22, 253));
AppendVMovU32(&code, 2, 9);
code.push_back(EncodeSop1(0x04, 126, 193));
store_scalar(20, expanded);
store_scalar(21, other_half);
store_scalar(22, mask != 0 ? 1u : 0u);
store_marker(expanded, other_half);
}
// VALU-produced masks keep WQM dynamic and carry provenance through S_MOV_B64.
// Updating either word must invalidate an old SGPR-pair predicate and preserve
// the other word. Wave32 still reads/writes the scalar high half independently.
for (u32 base : {8u, 106u, 126u}) {
for (u32 half : {0u, 1u}) {
AppendVMovU32(&code, 2, 0);
code.push_back(EncodeVop2(0x1b, 1, InlineU32(31), 0));
code.push_back(EncodeVopc(0xc2, InlineU32(3), 1)); // Lanes 3 and 35.
code.push_back(EncodeSop1(0x04, base, 106));
code.push_back(EncodeSop1(0x09, base + half, base + half));
code.push_back(EncodeSMovB32(20, base));
code.push_back(EncodeSMovB32(21, base + 1));
code.push_back(EncodeSMovB32(22, 253));
code.push_back(EncodeSop1(0x04, 126, base));
AppendVMovU32(&code, 2, 9);
code.push_back(EncodeSop1(0x04, 126, 193));
const u32 lo = half == 0 ? 0xfu : 8u;
const u32 hi = wave_size == 32 ? 0u : half == 1 ? 0xfu : 8u;
store_scalar(20, lo);
store_scalar(21, hi);
store_scalar(22, wave_size == 32 && half == 1 ? 0u : 1u);
store_marker(lo, hi);
}
}
for (u32 base : {106u, 126u}) {
for (u32 half : {0u, 1u}) {
AppendVMovU32(&code, 2, 0);
code.push_back(EncodeSMovB32(107, InlineU32(0)));
code.push_back(EncodeVopc(0xc2, InlineU32(3), 0)); // Only lane 3.
code.push_back(EncodeSop1(0x04, base, 106));
code.push_back(EncodeSop1(0x09, base + half, base + (half ^ 1u)));
code.push_back(EncodeSMovB32(20, base));
code.push_back(EncodeSMovB32(21, base + 1));
code.push_back(EncodeSop1(0x04, 126, base));
AppendVMovU32(&code, 2, 9);
code.push_back(EncodeSop1(0x04, 126, 193));
const u32 lo = half == 0 ? 0u : 8u;
const u32 hi = half == 0 ? 0u : 0xfu;
store_scalar(20, lo);
store_scalar(21, hi);
store_marker(lo, hi);
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::S_WQM_B32, O::S_MOV_B32, O::S_MOV_B64, O::S_CMP_EQ_U32,
O::V_CMP_EQ_U32, O::V_MOV_B32, O::V_AND_B32, O::V_ADD_NC_U32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_WQM_B32 exec_lo, exec_lo", 9}};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarWqmB64SelectsSccDomain() {
using O = ShaderOpcode;
std::vector<u32> code;
const auto capture_scc = [&](u32 dst) {
code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
};
code.push_back(EncodeSMovB32(0, InlineU32(0)));
code.push_back(EncodeSMovB32(1, InlineU32(1)));
code.push_back(EncodeSop1(0x0a, 2, 0));
capture_scc(4);
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 6, 106));
code.push_back(EncodeSop1(0x08, 8, 6));
code.push_back(EncodeSop1(0x0a, 10, 8));
capture_scc(12);
AppendStoreSgpr(&code, 4, 0);
AppendStoreSgpr(&code, 12, 1);
AppendEnd(&code);
return {"ScalarWqmB64SelectsSccDomain",
code,
{},
{1, 1},
{O::S_MOV_B32, O::S_WQM_B64, O::S_CSELECT_B32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::S_NOT_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarWqmB64PreservesPartialMasks() {
using O = ShaderOpcode;
TestCase test;
test.name = "ScalarWqmB64PreservesPartialMasks";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 16, 126)); // Save full EXEC.
AppendVMovU32(&code, 1, 3);
AppendVMovU32(&code, 2, 9);
u32 offset = 0;
for (const u32 destination : {126u, 106u, 8u}) {
code.push_back(EncodeVopc(0xc2, Vgpr(0), 1)); // Only lane 3.
code.push_back(EncodeSop1(0x04, destination, 106));
code.push_back(EncodeSop1(0x0a, destination, destination));
code.push_back(EncodeSop1(0x04, 126, destination));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
code.push_back(EncodeSop1(0x04, 126, 16));
offset += 64;
}
code.push_back(EncodeSMovB32(8, InlineU32(32))); // Raw bits 5 and 33.
code.push_back(EncodeSMovB32(9, InlineU32(2)));
code.push_back(EncodeSop1(0x0a, 10, 8));
code.push_back(EncodeSop1(0x04, 126, 10));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
code.push_back(EncodeSop1(0x04, 126, 16));
for (u32 source : {253u, 127u, 107u}) {
if (source == 253u) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1))); // SCC=1.
} else {
AppendSMovLiteral(&code, source - 1, ~0u);
code.push_back(EncodeSMovB32(source, InlineU32(1)));
}
code.push_back(EncodeSop1(0x0a, 126, source));
offset += 64;
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
code.push_back(EncodeSop1(0x04, 126, 16));
}
AppendEnd(&code);
test.initial.assign(448, 0xdeadbeef);
test.expected = test.initial;
for (u32 lane = 0; lane < 64; ++lane) {
for (u32 region = 0; region < 3; ++region) {
if (lane < 4) test.expected[region * 64 + lane] = 9;
}
if ((lane >= 4 && lane < 8) || (lane >= 32 && lane < 36)) {
test.expected[192 + lane] = 9;
}
for (u32 region = 4; region < 7; ++region) {
if (lane < 4) test.expected[region * 64 + lane] = 9;
}
}
test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::S_WQM_B64, O::V_MOV_B32,
O::V_CMP_EQ_U32, O::S_CMP_EQ_U32, O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase ScalarMaskProvenanceOverlapAndMixedBinary() {
using O = ShaderOpcode;
std::vector<u32> code;
const auto capture_scc = [&](u32 dst) {
code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
};
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 0, 106));
code.push_back(EncodeSop1(0x04, 1, 0));
code.push_back(EncodeSop1(0x08, 3, 1));
capture_scc(5);
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 8, 106));
code.push_back(EncodeSMovB32(10, InlineU32(1)));
code.push_back(EncodeSMovB32(11, InlineU32(0)));
code.push_back(EncodeSop2(0x19, 12, 8, 10));
capture_scc(14);
AppendStoreSgpr(&code, 5, 0);
AppendStoreSgpr(&code, 14, 1);
AppendEnd(&code);
return {"ScalarMaskProvenanceOverlapAndMixedBinary",
code,
{},
{1, 1},
{O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_NOT_B64, O::S_CSELECT_B32,
O::S_MOV_B32, O::S_NAND_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarB32MaskDataflow(u32 wave_size) {
using O = ShaderOpcode;
constexpr u32 mask = 0xffff0000u, rhs = 0x0f0f0f0fu;
constexpr u32 sentinel = 0xdeadbeefu, marker = 0x12345678u;
struct Operation {
u32 encoding;
O opcode;
u32 expected;
u32 source = rhs;
};
const std::array operations{
Operation{0x0e, O::S_AND_B32, mask & rhs},
Operation{0x10, O::S_OR_B32, mask | rhs},
Operation{0x12, O::S_XOR_B32, mask ^ rhs},
Operation{0x14, O::S_ANDN2_B32, mask & ~rhs},
Operation{0x16, O::S_ORN2_B32, mask | ~rhs},
Operation{0x18, O::S_NAND_B32, ~(mask & rhs)},
Operation{0x1a, O::S_NOR_B32, ~(mask | rhs)},
Operation{0x1c, O::S_XNOR_B32, ~(mask ^ rhs)},
Operation{0x07, O::S_NOT_B32, ~mask},
Operation{0x12, O::S_XOR_B32, 0, mask}};
TestCase test;
test.name = wave_size == 32 ? "ScalarB32MaskDataflowWave32" : "ScalarB32MaskDataflowWave64";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 16, 126)); // Preserve both EXEC halves.
AppendVMovU32(&code, 2, marker);
AppendVMovU32(&code, 1, 4);
for (const bool predicate_source : {false, true}) {
for (const auto &operation : operations) {
code.push_back(EncodeVopc(0xc1, InlineU32(15), 0)); // Lanes greater than 15.
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1)));
code.push_back(EncodeSop2(0x0a, 107, 106, InlineU32(0))); // VCC_HI is a wave32 mask temporary.
code.push_back(EncodeSMovB32(8, 107));
if (wave_size == 32) {
code.push_back(EncodeSMovB32(127, 8));
code.push_back(EncodeSMovB32(8, 127)); // EXEC_HI must preserve its own scalar word.
code.push_back(EncodeSMovB32(127, InlineU32(0)));
}
AppendSMovLiteral(&code, 9, sentinel); // Adjacent write must preserve a wave32 predicate.
if (predicate_source) {
if (operation.source == mask) {
code.push_back(EncodeVopc(0xc1, InlineU32(15), 0));
} else {
code.push_back(EncodeVop2(0x1b, 3, InlineU32(7), 0));
code.push_back(EncodeVopc(0xc1, Vgpr(3), 1)); // (lane & 7) < 4.
}
code.push_back(EncodeSMovB32(10, 106));
} else {
code.push_back(EncodeSMovB32(10, operation.source == rhs ? 60u : 61u));
}
code.push_back(operation.opcode == O::S_NOT_B32
? EncodeSop1(operation.encoding, 8, 8)
: EncodeSop2(operation.encoding, 8, 8, 10));
code.push_back(EncodeSop2(0x0a, 12, InlineU32(1), InlineU32(0))); // Capture scalar SCC.
code.push_back(EncodeSop2(0x0a, 8, 8, 10)); // Aliased scalar select, including SCC0.
code.push_back(EncodeSMovB32(126, 8));
const auto offset = static_cast<u32>(test.expected.size());
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
code.push_back(EncodeSop1(0x04, 126, 16));
const auto selected = operation.expected != 0 ? operation.expected : operation.source;
for (u32 lane = 0; lane < wave_size; ++lane) {
test.expected.push_back(lane >= 32 || (selected & (1u << lane)) != 0
? marker : sentinel);
}
for (const auto [reg, value] : std::array<std::pair<u32, u32>, 3>{{
{8, selected}, {9, sentinel}, {12, operation.expected != 0}}}) {
AppendStoreSgprAtLaneDwordOffset(&code, reg, 0, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, value);
}
test.opcodes.push_back(operation.opcode);
}
}
if (wave_size == 32) {
for (u32 low : {106u, 126u}) {
code.push_back(EncodeVopc(0xc1, InlineU32(15), 0));
code.push_back(EncodeSMovB32(low + 1, 106)); // A previously known high-word predicate.
code.push_back(EncodeSop1(0x04, low, 193u)); // Full scalar -1 overwrites both words.
code.push_back(EncodeSMovB32(8, low + 1));
code.push_back(EncodeSop1(0x04, 126, 16));
code.push_back(EncodeSMovB32(126, 8));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, marker);
code.push_back(EncodeSop1(0x04, 126, 16));
AppendStoreSgprAtLaneDwordOffset(&code, 8, 0, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, UINT32_MAX);
}
}
AppendEnd(&code);
test.initial.assign(test.expected.size(), sentinel);
test.user_data = MakeNativeUserData(nullptr);
test.user_data[50] = static_cast<u32>(test.expected.size() * sizeof(u32));
test.user_data[60] = rhs; // Raw masks initialized at shader entry.
test.user_data[61] = mask;
test.has_user_data = true;
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes.insert(test.opcodes.end(), {O::S_MOV_B32, O::S_MOV_B64, O::S_CSELECT_B32, O::S_CMP_EQ_U32,
O::V_CMP_LT_U32, O::V_MOV_B32, O::V_AND_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM});
return test;
}
TestCase ScalarLiteral() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0x12345678u);
code.push_back(EncodeVop1(0x01, 0, 0));
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"ScalarLiteral",
code,
{},
{0x12345678u},
{O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorMoves() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x00, 0, 0),
EncodeVop1(0x01, 0, InlineU32(5)),
EncodeVop1(0x01, 2, Vgpr(0)),
};
AppendBufferStoreDword(&code, 2, 30);
AppendEnd(&code);
return {"VectorRegisterMoves",
code,
{},
{5},
{O::V_NOP, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVop3MoveAppliesFloatSourceModifiers() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 24, 0x40000000u);
AppendVop3(&code, 0x181, 1, 24, 0, 0, 0, 0, false, 0, 0x1);
AppendStoreVgpr(&code, 1, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3MoveAppliesFloatSourceModifiers";
test.code = code;
test.expected = {0xc0000000u};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpFNegate"};
return test;
}
TestCase VectorIntegerOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xfffffff8u);
code.push_back(EncodeVop1(0x01, 1, InlineU32(5)));
code.push_back(EncodeVop1(0x01, 4, InlineU32(7)));
AppendVMovLiteral(&code, 10, 0x01000001u);
code.push_back(EncodeVop1(0x01, 11, InlineU32(2)));
AppendVMovLiteral(&code, 12, 0x0f0f0f0fu);
AppendVMovLiteral(&code, 13, 0x33333333u);
AppendVMovLiteral(&code, 14, 0xfffffff0u);
code.push_back(EncodeVop1(0x01, 15, InlineU32(4)));
code.push_back(EncodeVop1(0x01, 29, InlineU32(15)));
code.push_back(EncodeVop2(0x11, 2, Vgpr(0), 1));
code.push_back(EncodeVop2(0x12, 3, Vgpr(0), 1));
code.push_back(EncodeVop2(0x13, 5, Vgpr(0), 1));
code.push_back(EncodeVop2(0x14, 6, Vgpr(0), 1));
code.push_back(EncodeVop2(0x25, 7, Vgpr(4), 1));
code.push_back(EncodeVop2(0x26, 8, Vgpr(4), 1));
code.push_back(EncodeVop2(0x27, 9, InlineU32(3), 4));
code.push_back(EncodeVop2(0x26, 35, 249, 11));
code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6));
code.push_back(EncodeVop2(0x27, 36, 249, 11));
code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6));
code.push_back(EncodeVop2(0x0b, 16, Vgpr(10), 11));
code.push_back(EncodeVop2(0x09, 37, Vgpr(0), 1));
code.push_back(EncodeVop2(0x1b, 17, Vgpr(12), 13));
code.push_back(EncodeVop2(0x1c, 18, Vgpr(12), 13));
code.push_back(EncodeVop2(0x1d, 19, Vgpr(12), 13));
code.push_back(EncodeVop2(0x1e, 20, Vgpr(12), 13));
code.push_back(EncodeVop2(0x19, 21, Vgpr(11), 15));
code.push_back(EncodeVop2(0x1a, 22, InlineU32(4), 11));
code.push_back(EncodeVop2(0x15, 23, Vgpr(14), 15));
code.push_back(EncodeVop2(0x16, 24, InlineU32(4), 14));
code.push_back(EncodeVop2(0x17, 25, Vgpr(14), 15));
code.push_back(EncodeVop2(0x18, 26, InlineU32(4), 14));
code.push_back(EncodeVop1(0x37, 27, Vgpr(12)));
code.push_back(EncodeVop1(0x38, 28, Vgpr(29)));
code.push_back(EncodeVop1(0x3a, 30, Vgpr(11)));
code.push_back(EncodeVop1(0x39, 32, Vgpr(11)));
code.push_back(EncodeVopc(0xc2, Vgpr(4), 4));
code.push_back(EncodeVop2(0x01, 33, InlineU32(3), 4));
code.push_back(EncodeVop2(0x11, 34, 249, 4));
code.push_back(EncodeVop2Sdwa(0));
const u32 results[] = {2, 3, 5, 6, 7, 8, 9, 35, 36, 16, 37, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28, 30, 32, 33, 34};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"VectorIntegerOps",
code,
{},
{0xfffffff8u, 5, 5, 0xfffffff8u, 12,
2, 4, 13, 0xfffffff3u, 2,
0xffffffd8u, 0x03030303u, 0x3f3f3f3fu, 0x3c3c3c3cu, 0xc3c3c3c3u,
32, 32, 0x0fffffffu, 0x0fffffffu, 0xffffffffu,
0xffffffffu, 0xf0f0f0f0u, 0xf0000000u, 1, 30,
7, 0xfffffff8u},
{O::V_MOV_B32, O::V_MIN_I32, O::V_MAX_I32,
O::V_MIN_U32, O::V_MAX_U32, O::V_ADD_NC_U32,
O::V_SUB_NC_U32, O::V_SUBREV_NC_U32, O::V_MUL_U32_U24,
O::V_MUL_I32_I24, O::V_AND_B32, O::V_OR_B32,
O::V_XOR_B32, O::V_XNOR_B32, O::V_LSHL_B32,
O::V_LSHLREV_B32, O::V_LSHR_B32, O::V_LSHRREV_B32,
O::V_ASHR_I32, O::V_ASHRREV_I32, O::V_NOT_B32,
O::V_BFREV_B32, O::V_FFBL_B32, O::V_FFBH_U32,
O::V_CMP_EQ_U32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorIntegerExtremaRuntimeEdges() {
using O = ShaderOpcode;
constexpr std::array<std::array<u32, 3>, 8> inputs{{
{0x80000000u, 0u, 0x7fffffffu},
{0xffffffffu, 1u, 0x80000000u},
{1u, 0x80000000u, 0xffffffffu},
{1u, 0xffffffffu, 0x80000000u},
{0x7fffffffu, 0x7fffffffu, 0x80000000u},
{0x80000000u, 0x80000000u, 0xffffffffu},
{0xffffffffu, 0xffffffffu, 0xffffffffu},
{0u, 0u, 0u},
}};
TestCase test;
test.name = "VectorIntegerExtremaRuntimeEdges";
for (const auto &values : inputs)
test.initial.insert(test.initial.end(), values.begin(), values.end());
test.expected = test.initial;
for (u32 i = 0; i < inputs.size(); ++i) {
for (u32 reg = 0; reg < 3; ++reg) {
AppendVMovU32(&test.code, 30, (i * 3u + reg) * 4u);
AppendBufferLoadDword(&test.code, reg, 30);
}
for (u32 op = 0; op < 4; ++op)
test.code.push_back(EncodeVop2(0x11u + op, 4u + op, Vgpr(0), 1));
constexpr std::array opcodes{0x152u, 0x155u, 0x158u, 0x153u, 0x156u, 0x159u};
for (u32 op = 0; op < opcodes.size(); ++op)
AppendVop3(&test.code, opcodes[op], 8u + op, Vgpr(0), Vgpr(1), Vgpr(2));
const auto output = static_cast<u32>(test.expected.size());
for (u32 result = 0; result < 10; ++result)
AppendStoreVgpr(&test.code, 4u + result, output + result);
auto unsigned_values = inputs[i];
std::array<int32_t, 3> signed_values;
for (u32 j = 0; j < 3; ++j)
signed_values[j] = std::bit_cast<int32_t>(unsigned_values[j]);
test.expected.insert(test.expected.end(), {
static_cast<u32>(std::min(signed_values[0], signed_values[1])),
static_cast<u32>(std::max(signed_values[0], signed_values[1])),
std::min(unsigned_values[0], unsigned_values[1]),
std::max(unsigned_values[0], unsigned_values[1])});
std::sort(signed_values.begin(), signed_values.end());
std::sort(unsigned_values.begin(), unsigned_values.end());
test.expected.insert(test.expected.end(), {
static_cast<u32>(signed_values[0]), static_cast<u32>(signed_values[2]),
static_cast<u32>(signed_values[1]), unsigned_values[0],
unsigned_values[2], unsigned_values[1]});
}
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MIN_I32, O::V_MAX_I32,
O::V_MIN_U32, O::V_MAX_U32, O::V_MIN3_I32, O::V_MAX3_I32,
O::V_MED3_I32, O::V_MIN3_U32, O::V_MAX3_U32, O::V_MED3_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {" SMin ", " SMax ", " UMin ", " UMax "};
return test;
}
TestCase VectorFfbhI32NativeAndVop3OnGpu() {
using O = ShaderOpcode;
constexpr std::array inputs = {0x00000000u, 0x40000000u, 0x80000000u,
0x0fffffffu, 0xffff0000u, 0xfffffffeu,
0xffffffffu, 0x80000000u};
std::vector<u32> code;
for (u32 i = 0; i < inputs.size() - 1u; i++) {
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 14, 30);
code.push_back(0x7e1c770eu); // v_ffbh_i32 v14, v14
AppendStoreVgpr(&code, 14, i);
}
AppendVMovU32(&code, 30, (inputs.size() - 1u) * 4u);
AppendBufferLoadDword(&code, 14, 30);
AppendVop3(&code, 0x1bbu, 15, Vgpr(14), 0);
AppendStoreVgpr(&code, 15, inputs.size() - 1u);
AppendEnd(&code);
TestCase test;
test.name = "VectorFfbhI32NativeAndVop3OnGpu";
test.code = std::move(code);
test.initial.assign(inputs.begin(), inputs.end());
test.expected = {0xffffffffu, 1u, 1u, 4u, 16u, 31u, 0xffffffffu, 1u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FFBH_I32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_FFBH_I32 v14, v14", inputs.size() - 1u},
{"V_FFBH_I32 v15, v14", 1u}};
return test;
}
TestCase Vop1SdwaBfrevSourceSelectors() {
using O = ShaderOpcode;
constexpr u32 expected[2][7] = {
{0x8f000000u, 0xd5000000u, 0x21000000u, 0x49000000u,
0x8fd50000u, 0x21490000u, 0x8fd52149u},
{0x8fffffffu, 0xd5ffffffu, 0x21ffffffu, 0x49ffffffu,
0x8fd5ffffu, 0x2149ffffu, 0x8fd52149u}};
TestCase test;
test.name = "Vop1SdwaBfrevSourceSelectors";
for (u32 sext = 0; sext < 2; sext++) {
for (u32 selector = 0; selector < 7; selector++) {
const u32 index = static_cast<u32>(test.initial.size());
AppendVMovU32(&test.code, 30, index * 4u);
AppendBufferLoadDword(&test.code, 0, 30);
// selector=4, sext=0 reproduces captured words 7e0070f9, 00040600.
test.code.insert(test.code.end(),
{0x7e0070f9u, EncodeVop1Sdwa(0, 6, 0, selector, sext)});
AppendStoreVgpr(&test.code, 0, index);
test.initial.push_back(0x9284abf1u);
test.expected.push_back(expected[sext][selector]);
}
}
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_BFREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_BFREV_B32 v0, v0.sdwa(sel=4,sext=0)", 1}};
test.required_spirv = {"OpBitFieldUExtract", "OpBitFieldSExtract", "OpBitReverse"};
return test;
}
TestCase Vop1SdwaFfbhCapturedScalarLowWordSource() {
using O = ShaderOpcode;
constexpr std::array inputs = {0xdead0000u, 0xffff0001u, 0x00008000u,
0x00010008u, 0xffff4000u, 0x0000ffffu};
std::vector<u32> code;
for (u32 i = 0; i < inputs.size(); i++) {
AppendSmemLoadOpcode(&code, 0x08, 39, i * 4u);
code.insert(code.end(), {0x7e0072f9u, 0x00840627u}); // v_ffbh_u32 v0, s39.word0
AppendStoreVgpr(&code, 0, i);
}
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaFfbhCapturedScalarLowWordSource";
test.code = std::move(code);
test.initial.assign(inputs.begin(), inputs.end());
test.expected = {0xffffffffu, 31u, 16u, 28u, 17u, 16u};
test.opcodes = {O::S_BUFFER_LOAD_DWORD, O::V_FFBH_U32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_FFBH_U32 v0, s39.sdwa(sel=4,sext=0)", inputs.size()}};
test.required_spirv = {"OpBitFieldUExtract", "FindUMsb"};
return test;
}
TestCase Vop1SdwaFfblCapturedHighWordSource() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 2, 30);
code.push_back(0x7e2074f9u);
code.push_back(0x00050602u); // v_ffbl_b32 v16, v2.word1
AppendStoreVgpr(&code, 16, 0);
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaFfblCapturedHighWordSource";
test.code = std::move(code);
test.initial = {0x00400000u};
test.expected = {6u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_FFBL_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {{"V_FFBL_B32 v16, v2.sdwa(sel=5,sext=0)", 1}};
test.ir_counts = {{" = BitFieldUExtract ", 1}, {" = FindILsb32 ", 1}};
test.required_spirv = {"OpBitFieldUExtract"};
return test;
}
TestCase Vop1SdwaNotCapturedByte0Source() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 2, 30);
code.push_back(0x7e046ef9u);
code.push_back(0x00000602u); // v_not_b32 v2, v2.byte0
AppendStoreVgpr(&code, 2, 0);
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaNotCapturedByte0Source";
test.code = std::move(code);
test.initial = {0x12345678u};
test.expected = {0xffffff87u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_NOT_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {{"V_NOT_B32 v2, v2.sdwa(sel=0,sext=0)", 1}};
test.ir_counts = {{" = BitFieldUExtract ", 1}, {" = BitwiseNot32 ", 1}};
test.required_spirv = {"OpBitFieldUExtract", "OpNot"};
return test;
}
TestCase Vop1SdwaNotPreservesHighWordDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 0, 30);
AppendVMovLiteral(&code, 3, 0xabcd5555u);
code.push_back(0x7e066ef9u);
code.push_back(0x00061400u);
AppendStoreVgpr(&code, 3, 0);
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaNotPreservesHighWordDestination";
test.code = std::move(code);
test.initial = {0x12345678u};
test.expected = {0xabcda987u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::V_NOT_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.ir_counts = {{" = BitwiseNot32 ", 1}};
test.required_spirv = {"OpNot"};
return test;
}
TestCase Vop1SdwaNotPartialSourcesAndDestinations() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 8, 30);
AppendVMovLiteral(&code, 28, 0xabcd5555u);
code.push_back(EncodeVop1(0x37, 28, 249));
code.push_back(EncodeVop1Sdwa(8, 4, 2, 5)); // v28.word0 = ~v8.word1, preserve word1
AppendStoreVgpr(&code, 28, 0);
AppendVMovLiteral(&code, 3, 0xa1b2c3d4u);
code.push_back(EncodeVop1(0x37, 3, 249));
code.push_back(EncodeVop1Sdwa(8, 2, 2, 0)); // v3.byte2 = ~v8.byte0, preserve others
AppendStoreVgpr(&code, 3, 1);
code.push_back(EncodeVop1(0x37, 4, 249));
code.push_back(EncodeVop1Sdwa(8, 0, 0, 0)); // v4.byte0 = ~v8.byte0, zero others
AppendStoreVgpr(&code, 4, 2);
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaNotPartialSourcesAndDestinations";
test.code = std::move(code);
test.initial = {0x12345678u, 0u, 0u};
test.expected = {0xabcdedcbu, 0xa187c3d4u, 0x00000087u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::V_NOT_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_NOT_B32", 3u}};
test.required_spirv = {"OpBitFieldUExtract", "OpNot"};
return test;
}
TestCase Vop2SdwaBitwiseByteDestinations(u32 wave_size) {
using O = ShaderOpcode;
// Low result bytes of AND, OR, XOR, XNOR for the two runtime inputs.
constexpr std::array<u32, 4> result_bytes{0xa4u, 0xbeu, 0x1au, 0xe5u};
constexpr std::array<u32, 4> alias_expected{
0xc350e5a6u, 0xc3fce5a6u, 0xc3ace5a6u, 0xc353e5a6u};
constexpr u32 sentinel = 0xa1b2c3d4u;
TestCase test;
test.name = wave_size == 64 ? "Vop2SdwaBitwiseByteDestinationsWave64"
: "Vop2SdwaBitwiseByteDestinationsWave32";
test.initial = {0xc3d4e5a6u, 0x9b785abcu};
test.expected = test.initial;
auto &code = test.code;
const auto store = [&](u32 reg, u32 expected) {
AppendStoreVgpr(&code, reg, static_cast<u32>(test.expected.size()));
test.expected.push_back(expected);
};
for (u32 op = 0; op < 4; ++op) {
AppendVMovU32(&code, 30, 0);
AppendBufferLoadDword(&code, 12, 30);
AppendVMovU32(&code, 30, 4);
AppendBufferLoadDword(&code, 13, 30);
for (u32 selector = 0; selector < 4; ++selector) {
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 26, sentinel);
code.push_back(EncodeVop2(0x1b + op, 26, 249, 13));
code.push_back(EncodeVop2Sdwa(12, selector, unused));
// RDNA2 table 88: pad, sign extend above/zero below, or preserve.
const u32 shift = selector * 8u;
u32 expected = result_bytes[op] << shift;
if (unused == 1 && (result_bytes[op] & 0x80u) != 0) {
expected = (0xffffff00u | result_bytes[op]) << shift;
} else if (unused == 2) {
expected |= sentinel & ~(0xffu << shift);
}
store(26, expected);
}
}
// Read source byte2 and word1 before overwriting source0's byte2.
code.push_back(EncodeVop2(0x1b + op, 12, 249, 13));
code.push_back(EncodeVop2Sdwa(12, 2, 2, 2, 5));
store(12, alias_expected[op]);
// Source1 may also alias the destination.
AppendVMovU32(&code, 30, 0);
AppendBufferLoadDword(&code, 12, 30);
code.push_back(EncodeVop2(0x1b + op, 13, 249, 13));
code.push_back(EncodeVop2Sdwa(12, 1, 2));
store(13, 0x9b7800bcu | (result_bytes[op] << 8u));
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 26, sentinel);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.push_back(EncodeVop2(0x1b + op, 26, 249, 13));
code.push_back(EncodeVop2Sdwa(12, 2, unused));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
store(26, sentinel);
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_AND_B32, O::V_OR_B32, O::V_XOR_B32, O::V_XNOR_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitwiseAnd", "OpBitwiseOr", "OpBitwiseXor", "OpNot",
"OpBitFieldInsert", "OpBitFieldSExtract"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop1SdwaMovByteDestinations() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 3, 30);
AppendVMovU32(&code, 30, 4);
AppendBufferLoadDword(&code, 2, 30);
code.push_back(EncodeVop1(0x01, 1, Vgpr(3)));
code.push_back(0x7e0202f9u);
code.push_back(0x00861280u); // Captured v_mov_b32 v1.byte2, 0, preserve.
AppendStoreVgpr(&code, 1, 2);
// RDNA2 table 88: select any byte, then pad, sign extend, or preserve.
for (u32 dst_u = 0; dst_u < 3; dst_u++) {
for (u32 dst_sel = 0; dst_sel < 4; dst_sel++) {
code.push_back(EncodeVop1(0x01, 1, Vgpr(3)));
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(2, dst_sel, dst_u));
AppendStoreVgpr(&code, 1, 3 + dst_u * 4 + dst_sel);
}
}
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaMovByteDestinations";
test.code = std::move(code);
test.initial = {0xa1b2c3d4u, 0x12345680u};
test.expected = {0xa1b2c3d4u, 0x12345680u, 0xa100c3d4u,
0x00000080u, 0x00008000u, 0x00800000u, 0x80000000u,
0xffffff80u, 0xffff8000u, 0xff800000u, 0x80000000u,
0xa1b2c380u, 0xa1b280d4u, 0xa180c3d4u, 0x80b2c3d4u};
test.initial.resize(test.expected.size());
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {{"V_MOV_B32 v1.sdwa(sel=2,sext=0), 0", 1}};
test.required_spirv = {"OpBitFieldInsert", "OpBitFieldSExtract", "OpBitwiseOr"};
return test;
}
TestCase Vop1SdwaMovByteSources(u32 wave_size) {
using O = ShaderOpcode;
// Select 0x80 from each byte position, with nonzero neighboring bytes.
constexpr std::array<u32, 4> sources{
0x12345680u, 0x12348056u, 0x12803456u, 0x80341256u};
// RDNA2 table 88 / PS5 SDWA-related options: PAD, SEXT, PRESERVE.
constexpr std::array<u32, 18> partial_expected{
0x00000080u, 0xffffff80u, 0xa1b2c380u,
0x00008000u, 0xffff8000u, 0xa1b280d4u,
0x00800000u, 0xff800000u, 0xa180c3d4u,
0x80000000u, 0x80000000u, 0x80b2c3d4u,
0x00000080u, 0x00000080u, 0xa1b20080u,
0x00800000u, 0x00800000u, 0x0080c3d4u};
constexpr std::array<u32, 6> sext_word_expected{
0x0000ff80u, 0xffffff80u, 0xa1b2ff80u,
0xff800000u, 0xff800000u, 0xff80c3d4u};
TestCase test;
test.name = wave_size == 64 ? "Vop1SdwaMovByteSourcesWave64"
: "Vop1SdwaMovByteSourcesWave32";
test.initial.assign(sources.begin(), sources.end());
test.expected = test.initial;
auto &code = test.code;
const auto store = [&](u32 expected) {
AppendStoreVgpr(&code, 1, static_cast<u32>(test.expected.size()));
test.expected.push_back(expected);
};
for (u32 src_sel = 0; src_sel < sources.size(); ++src_sel) {
AppendVMovU32(&code, 30, src_sel * sizeof(u32));
AppendBufferLoadDword(&code, 2, 30);
for (u32 sext = 0; sext < 2; ++sext) {
for (u32 dst_sel = 0; dst_sel < 6; ++dst_sel) {
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 1, 0xa1b2c3d4u);
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(2, dst_sel, unused, src_sel, sext));
store(sext && dst_sel >= 4 ? sext_word_expected[(dst_sel - 4u) * 3u + unused]
: partial_expected[dst_sel * 3u + unused]);
}
}
}
}
// Aliasing must read the source byte before replacing the destination byte.
AppendVMovU32(&code, 30, 0);
AppendBufferLoadDword(&code, 1, 30);
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(1, 3, 2, 1));
store(0x56345680u);
// Scalar and inline sources use the same independent selectors.
AppendSMovLiteral(&code, 18, 0x12348056u);
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(18, 5, 0, 1, 1, 0, 0, 1));
store(0xff800000u);
AppendVMovLiteral(&code, 1, 0xa1b2c3d4u);
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(193, 0, 2, 3, 0, 0, 0, 1)); // Inline -1.
store(0xa1b2c3ffu);
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 1, 0xa1b2c3d4u);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(2, 4, unused, 0));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
store(0xa1b2c3d4u);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitFieldUExtract", "OpBitFieldSExtract", "OpBitFieldInsert"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop2SdwaSubNcExactByte2Destination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 11, 2);
code.push_back(0x4c1616f9u);
code.push_back(0x0686128du);
AppendStoreVgpr(&code, 11, 0);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaSubNcExactByte2Destination";
test.code = code;
test.expected = {0x000b0002u};
test.opcodes = {O::V_MOV_B32, O::V_SUB_NC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase Vop2SdwaAddNcCapturedHighWordDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 0, 30);
code.push_back(EncodeVop1(0x02, 106, Vgpr(0)));
AppendVMovLiteral(&code, 97, 0x00030004u);
AppendVMovLiteral(&code, 4, 0xa1b2c3d4u);
code.push_back(0x4a08c2f9u);
code.push_back(0x0686156au);
AppendStoreVgpr(&code, 4, 1);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaAddNcCapturedHighWordDestination";
test.code = std::move(code);
test.initial = {0x00010002u, 0};
test.expected = {0x00010002u, 0x0006c3d4u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::V_MOV_B32,
O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpIAdd", "OpShiftLeftLogical", "OpBitwiseOr"};
return test;
}
TestCase Vop2SdwaAshrrevCapturedWord0SignExtends() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 1, 30);
code.push_back(0x303202f9u);
code.push_back(0x0c860686u); // v_ashrrev_i32 v25, 6, sign_extend(v1.word0)
AppendStoreVgpr(&code, 25, 1);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaAshrrevCapturedWord0SignExtends";
test.code = std::move(code);
test.initial = {0xabcd8000u, 0};
test.expected = {0xabcd8000u, 0xfffffe00u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_ASHRREV_I32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_ASHRREV_I32 v25, 6, v1.sdwa(sel=4,sext=1)", 1}};
test.ir_counts = {{" = BitFieldSExtract ", 1},
{" = ShiftRightArithmetic32 ", 1}};
test.required_spirv = {"OpBitFieldSExtract", "OpShiftRightArithmetic"};
return test;
}
TestCase Vop2SdwaMaxI32CapturedHighWord(u32 wave_size) {
using O = ShaderOpcode;
struct MaxCase { u32 lhs, packed_rhs, expected, exec = 1; };
constexpr std::array<MaxCase, 9> cases{{
{0xfffffffbu, 0xfffe1234u, 0xfffffffeu}, // -5 versus sign-extended -2.
{0xffffffffu, 0x80007fffu, 0xffffffffu},
{3, 0x0007ffffu, 7}, // Ignore the low word.
{0xffffffffu, 0x0000ffffu, 0},
{0x80000000u, 0x80000000u, 0xffff8000u},
{0x7fffffffu, 0x8000ffffu, 0x7fffffffu},
{0x0000ffffu, 0x7fffffffu, 0x0000ffffu}, // Source 0 stays 32-bit.
{0x00007fffu, 0x7fff8000u, 0x00007fffu},
{0x80000001u, 0x7fff0123u, 0x80000001u, 0}, // Inactive destination.
}};
TestCase test;
test.name = wave_size == 64 ? "Vop2SdwaMaxI32HighWordWave64"
: "Vop2SdwaMaxI32HighWordWave32";
for (const auto &entry : cases) {
test.initial.insert(test.initial.end(), {entry.lhs, entry.packed_rhs});
}
test.initial.resize(cases.size() * 3u, 0xdeadbeefu);
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
AppendVMovU32(&code, 30, i * 8u);
AppendBufferLoadDword(&code, 11, 30);
AppendVMovU32(&code, 30, i * 8u + 4u);
AppendBufferLoadDword(&code, 22, 30);
code.push_back(EncodeSMovB32(126, InlineU32(cases[i].exec)));
// Exact game encoding: v11 = max(v11, sign_extend(v22.word1)).
code.insert(code.end(), {0x24162cf9u, 0x0d06060bu});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = cases.size() * 2u + i;
AppendStoreVgpr(&code, 11, out);
test.expected[out] = cases[i].expected;
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_MAX_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MAX_I32 v11, v11, v22.sdwa(sel=5,sext=1)", cases.size()}};
test.required_spirv = {"OpBitFieldSExtract"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop2SdwaMulI24Destinations(u32 wave_size) {
using O = ShaderOpcode;
constexpr std::array<u32, 5> sources{
0xdead1234u, 0xabcdffffu, 0x98768001u, 0x34560000u, 0x56789abcu};
constexpr std::array<u32, 10> captured_expected{
0xdead91a0u, 0xdead3400u, 0xabcdfff8u, 0xabcdff00u,
0x98760008u, 0x98760100u, 0x34560000u, 0x34560000u,
0x56789abcu, 0x56789abcu};
constexpr std::array<u32, 18> partial_expected{
0x00000080u, 0xffffff80u, 0xa1b2c380u,
0x00008000u, 0xffff8000u, 0xa1b280d4u,
0x00800000u, 0xff800000u, 0xa180c3d4u,
0x80000000u, 0x80000000u, 0x80b2c3d4u,
0x0000a180u, 0xffffa180u, 0xa1b2a180u,
0xa1800000u, 0xa1800000u, 0xa180c3d4u};
TestCase test;
test.name = wave_size == 64 ? "Vop2SdwaMulI24DestinationsWave64"
: "Vop2SdwaMulI24DestinationsWave32";
test.initial.assign(sources.begin(), sources.end());
test.initial.push_back(0xabcde080u);
test.expected = test.initial;
auto &code = test.code;
for (u32 index = 0; index < sources.size(); ++index) {
AppendVMovU32(&code, 30, index * sizeof(u32));
AppendBufferLoadDword(&code, 12, 30);
AppendBufferLoadDword(&code, 26, 30);
AppendSMovLiteral(&code, 106, 256);
code.push_back(EncodeSMovB32(126, InlineU32(index + 1u == sources.size() ? 0 : 1)));
// Captured WORD0/PRESERVE encodings alias their source and destination.
code.insert(code.end(), {0x121818f9u, 0x04861488u,
0x123434f9u, 0x0486146au});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
AppendStoreVgpr(&code, 12, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&code, 26, static_cast<u32>(test.expected.size()) + 1u);
test.expected.push_back(captured_expected[index * 2u]);
test.expected.push_back(captured_expected[index * 2u + 1u]);
}
AppendVMovU32(&code, 30, sources.size() * sizeof(u32));
AppendBufferLoadDword(&code, 12, 30);
for (u32 selector = 0; selector < 6; ++selector) {
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 26, 0xa1b2c3d4u);
code.push_back(EncodeVop2(0x09, 26, 249, 12));
code.push_back(EncodeVop2Sdwa(InlineU32(3), selector, unused, 6, 4,
0, 0, 0, 0, 0, 0, 1));
AppendStoreVgpr(&code, 26, static_cast<u32>(test.expected.size()));
test.expected.push_back(partial_expected[selector * 3u + unused]);
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_MUL_I32_I24, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpIMul", "OpBitFieldUExtract"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop2SdwaMulU24Destinations(u32 wave_size) {
using O = ShaderOpcode;
// The high source bytes are discarded: 3 * 0xe080 = 0x2a180.
constexpr std::array<u32, 18> partial_expected{
0x00000080u, 0xffffff80u, 0xa1b2c380u,
0x00008000u, 0xffff8000u, 0xa1b280d4u,
0x00800000u, 0xff800000u, 0xa180c3d4u,
0x80000000u, 0x80000000u, 0x80b2c3d4u,
0x0000a180u, 0xffffa180u, 0xa1b2a180u,
0xa1800000u, 0xa1800000u, 0xa180c3d4u};
TestCase test;
test.name = wave_size == 64 ? "Vop2SdwaMulU24DestinationsWave64"
: "Vop2SdwaMulU24DestinationsWave32";
test.initial = {0xab000003u, 0xcd00e080u, 0xffffffffu};
test.expected = test.initial;
auto &code = test.code;
const auto store = [&](u32 reg, u32 expected) {
AppendStoreVgpr(&code, reg, static_cast<u32>(test.expected.size()));
test.expected.push_back(expected);
};
for (u32 i = 0; i < test.initial.size(); ++i) {
AppendVMovU32(&code, 30, i * sizeof(u32));
AppendBufferLoadDword(&code, 12u + i, 30);
}
for (u32 selector = 0; selector < 6; ++selector) {
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 26, 0xa1b2c3d4u);
code.push_back(EncodeVop2(0x0b, 26, 249, 13));
code.push_back(EncodeVop2Sdwa(12, selector, unused));
store(26, partial_expected[selector * 3u + unused]);
}
}
// Full result still truncates the 48-bit unsigned product to 32 bits.
code.push_back(EncodeVop2(0x0b, 26, 249, 14));
code.push_back(EncodeVop2Sdwa(14));
store(26, 0xfe000001u);
// Either multiplicand may alias the partial destination.
code.push_back(EncodeVop2(0x0b, 12, 249, 13));
code.push_back(EncodeVop2Sdwa(12, 5, 2));
store(12, 0xa1800003u);
code.push_back(EncodeVop2(0x0b, 13, 249, 13));
code.push_back(EncodeVop2Sdwa(InlineU32(3), 2, 2, 6, 4,
0, 0, 0, 0, 0, 0, 1));
store(13, 0xcd80e080u);
// Source-byte sign extension happens before the unsigned 24-bit multiply.
AppendSMovLiteral(&code, 18, 0x12348056u);
code.push_back(EncodeVop2(0x0b, 26, 249, InlineU32(3)));
code.push_back(EncodeVop2Sdwa(18, 5, 0, 1, 6, 1, 0, 0, 0, 0, 0, 1, 1));
store(26, 0xfe800000u);
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 26, 0xa1b2c3d4u);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.push_back(EncodeVop2(0x0b, 26, 249, 13));
code.push_back(EncodeVop2Sdwa(12, 4, unused));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
store(26, 0xa1b2c3d4u);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_MUL_U32_U24, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MUL_U32_U24", 25u}};
test.required_spirv = {"OpIMul", "OpBitFieldUExtract", "OpBitFieldSExtract"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop2SdwaLshrrevCapturedByte1Source() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 106, 4);
AppendBufferLoadDword(&code, 8, 30);
code.push_back(0x2c1210f9u);
code.push_back(0x0186066au); // v_lshrrev_b32 v9, vcc_lo, v8.byte1
AppendStoreVgpr(&code, 9, 1);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaLshrrevCapturedByte1Source";
test.code = std::move(code);
test.initial = {0x12348678u, 0};
test.expected = {0x12348678u, 8};
test.opcodes = {O::S_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHRREV_B32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {
{"V_LSHRREV_B32 v9, vcc_lo, v8.sdwa(sel=1,sext=0)", 1}};
test.ir_counts = {{" = BitFieldUExtract ", 1},
{" = ShiftRightLogical32 ", 1}};
test.required_spirv = {"OpBitFieldUExtract", "OpShiftRightLogical"};
return test;
}
TestCase Vop2SdwaSubNcPreservesByteAndWordDestinations() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 3);
for (u32 dst = 10; dst <= 15; dst++) {
AppendVMovLiteral(&code, dst, 0xa1b2c3d4u);
}
for (u32 sel = 0; sel <= 5; sel++) {
code.push_back(EncodeVop2(0x26, 10 + sel, 249, 0));
code.push_back(
EncodeVop2Sdwa(InlineU32(20), sel, 2, 6, 6, 0, 0, 0, 0, 0, 0, 1));
AppendStoreVgpr(&code, 10 + sel, sel);
}
AppendEnd(&code);
return {"Vop2SdwaSubNcPreservesByteAndWordDestinations",
code,
{},
{0xa1b2c311u, 0xa1b211d4u, 0xa111c3d4u, 0x11b2c3d4u, 0xa1b20011u,
0x0011c3d4u},
{O::V_MOV_B32, O::V_SUB_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop3CvtPkI16I32Captured() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 7, 30);
AppendVMovU32(&code, 30, 4);
AppendBufferLoadDword(&code, 16, 30);
code.push_back(0xd76b0005u);
code.push_back(0x00022107u); // v_cvt_pk_i16_i32 v5, v7, v16
AppendStoreVgpr(&code, 5, 2);
AppendEnd(&code);
TestCase test;
test.name = "Vop3CvtPkI16I32Captured";
test.code = std::move(code);
test.initial = {0x00018001u, 0xfffe7ffeu, 0};
test.expected = {0x00018001u, 0xfffe7ffeu, 0x7ffe8001u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::V_CVT_PK_I16_I32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"0x00000014: V_CVT_PK_I16_I32 v5, v7, v16\n", 1}};
test.ir_counts = {{" = BitwiseAnd32 ", 2},
{" = ShiftLeftLogical32 ", 1},
{" = BitwiseOr32 ", 1}};
test.required_spirv = {"OpBitwiseAnd", "OpShiftLeftLogical", "OpBitwiseOr"};
return test;
}
TestCase Vop3MulLoU16CapturedAndSelectors() {
using O = ShaderOpcode;
TestCase test;
test.name = "Vop3MulLoU16CapturedAndSelectors";
// Distinct runtime halves expose source selection and prevent constant folding.
test.initial = {0x00138001u, 0x0005ffffu, 0xffffffffu};
test.expected = {0x00138001u, 0x0005ffffu, 0xffffffffu,
0xa5a57fffu, 0xa5a5ffedu, 0xa5a58005u, 0xa5a5005fu,
0x7fff1234u, 0xffed1234u, 0x80051234u, 0x005f1234u,
0xffff0001u, 0x00231234u};
auto &code = test.code;
AppendBufferLoadDword(&code, 12, 30);
AppendVMovU32(&code, 30, 4);
AppendBufferLoadDword(&code, 24, 30);
u32 output = 3;
for (u32 selectors : {0u, 1u, 2u, 3u, 8u, 9u, 10u, 11u}) {
AppendVMovLiteral(&code, 2, 0xa5a51234u);
if (selectors == 8u) {
code.push_back(0xd7054002u);
code.push_back(0x0002310cu); // Captured v_mul_lo_u16 v2.hi, v12.lo, v24.lo.
} else {
AppendVop3(&code, 0x305, 2, Vgpr(12), Vgpr(24), 0, 0, selectors);
}
AppendStoreVgpr(&code, 2, output++);
}
// 0xffff * 0xffff wraps to 1; aliasing preserves the original high half.
AppendVMovU32(&code, 30, 8);
AppendBufferLoadDword(&code, 12, 30);
AppendVop3(&code, 0x305, 12, Vgpr(12), Vgpr(12), 0, 0, 1);
AppendStoreVgpr(&code, 12, output++);
AppendVMovLiteral(&code, 2, 0xa5a51234u);
AppendVop3(&code, 0x305, 2, 255, Vgpr(24), 0, 0, 11);
code.push_back(0x0007deadu); // Select literal high half: 7 * 5 = 35.
AppendStoreVgpr(&code, 2, output);
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MUL_LO_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MUL_LO_U16", 10}};
test.required_spirv = {"OpIMul"};
return test;
}
TestCase Vop3MadI16CapturedSelectorsAndSaturation() {
using O = ShaderOpcode;
struct MadCase {
u32 lhs, rhs, addend, expected;
u32 selectors = 0;
bool clamp = false;
u32 dst = 1;
};
constexpr u32 a = 0x8000fffdu, b = 0xfffe0005u, c = 0xfff90007u;
const std::array<MadCase, 14> cases{{
{a, b, c, 0xa5a5fff8u}, // Captured: -3 * 5 + 7 = -8.
{a, b, c, 0x8007beefu, 9}, // Source 0 high; destination high.
{a, b, c, 0xa5a5000du, 2}, // Source 1 high: -3 * -2 + 7.
{a, b, c, 0xa5a5ffeau, 4}, // Source 2 high: -3 * 5 - 7.
{a, b, c, 0xfff9beefu, 15}, // All high: -32768 * -2 - 7 wraps.
{a, b, c, 0xfff8fffdu, 8, false, 9}, // Destination aliases source 0.
{0x7fffu, 1, 1, 0xa5a58000u},
{0x7fffu, 1, 1, 0x7fffbeefu, 8, true}, // Positive saturation.
{0x8000u, 1, 0xffffu, 0xa5a57fffu},
{0x8000u, 1, 0xffffu, 0x8000beefu, 8, true}, // Negative saturation.
{0x8000u, 0x8000u, 0xffffu, 0xa5a5ffffu},
{0x8000u, 0x8000u, 0xffffu, 0xa5a57fffu, 0, true},
{200, 200, 0xd8f0u, 0xa5a57530u, 0, true}, // 40000 - 10000 = 30000.
{200, 0xff38u, 10000, 0xa5a58ad0u, 0, true}, // -40000 + 10000 = -30000.
}};
TestCase test;
test.name = "Vop3MadI16CapturedSelectorsAndSaturation";
for (const auto &entry : cases) {
test.initial.insert(test.initial.end(), {entry.lhs, entry.rhs, entry.addend});
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
u32 offset = i * 12u;
for (u32 reg : {9u, 6u, 15u}) {
AppendVMovU32(&code, 30, offset);
AppendBufferLoadDword(&code, reg, 30); // Runtime inputs prevent folding.
offset += 4u;
}
AppendVMovLiteral(&code, 1, 0xa5a5beefu);
if (i == 0u) {
code.insert(code.end(), {0xd75e0001u, 0x043e0d09u});
} else {
AppendVop3(&code, 0x35e, entry.dst, Vgpr(9), Vgpr(6), Vgpr(15), 0,
entry.selectors, entry.clamp);
}
AppendStoreVgpr(&code, entry.dst, static_cast<u32>(test.initial.size()) + i);
test.expected.push_back(entry.expected);
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_I16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MAD_I16", cases.size()}};
test.required_spirv = {"OpIMul", "OpIAdd"};
return test;
}
TestCase Vop3Min3U16CapturedAndSelectors() {
using O = ShaderOpcode;
struct MinCase {
u32 lhs, rhs, third, expected;
u32 selectors = 0;
u32 dst = 17;
bool active = true;
};
constexpr u32 a = 0x0009ea60u, b = 0x0007c350u, c = 0x00059c40u;
const std::array<MinCase, 10> cases{{
{a, b, c, 0x0005beefu, 14}, // Captured: min(60000, 7, 5) into v17.hi.
{a, b, c, 0xa5a59c40u}, // Unsigned values above INT16_MAX.
{a, b, c, 0xa5a50009u, 1},
{a, b, c, 0xa5a50007u, 2},
{a, b, c, 0xa5a50005u, 4},
{a, b, c, 0x0005beefu, 15},
{a, b, c, 0x9c40ea60u, 8, 4}, // Destination aliases source 0.
{0xabcdffffu, 0xffff0000u, 0x98768000u, 0xa5a50000u},
{0x0000ffffu, 0x7fffffffu, 0x8000ffffu, 0xa5a5ffffu},
{a, b, c, 0xa5a5beefu, 14, 17, false},
}};
TestCase test;
test.name = "Vop3Min3U16CapturedAndSelectors";
for (const auto &entry : cases) {
test.initial.insert(test.initial.end(), {entry.lhs, entry.rhs, entry.third});
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
u32 offset = i * 12u;
for (u32 reg : {4u, 0u, 7u}) {
AppendVMovU32(&code, 30, offset);
AppendBufferLoadDword(&code, reg, 30); // Runtime inputs prevent folding.
offset += 4u;
}
AppendVMovLiteral(&code, 17, 0xa5a5beefu);
if (!entry.active) {
code.push_back(EncodeSop1(0x04, 24, 126)); // Preserve the initial active lanes.
code.push_back(EncodeSop1(0x04, 126, 128));
}
if (i == 0u) {
code.insert(code.end(), {0xd7537011u, 0x041e0104u});
} else {
AppendVop3(&code, 0x353, entry.dst, Vgpr(4), Vgpr(0), Vgpr(7), 0,
entry.selectors);
}
if (!entry.active) code.push_back(EncodeSop1(0x04, 126, 24));
AppendStoreVgpr(&code, entry.dst, static_cast<u32>(test.initial.size()) + i);
test.expected.push_back(entry.expected);
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MIN3_U16,
O::S_MOV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MIN3_U16", cases.size()}};
test.required_spirv = {" UMin "};
return test;
}
TestCase Vop3Med3I16Captured() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 5, 0x8ad003e8u); // high=-30000, low=1000
AppendSMovLiteral(&code, 25, 0xb1e04e20u); // high=-20000, low=20000
AppendSMovLiteral(&code, 27, 0x7530fc18u); // high=30000, low=-1000
AppendVMovLiteral(&code, 3, 0x1234cafeu);
code.push_back(0xd7584803u);
code.push_back(0x006c3305u); // v_med3_i16 v3.hi, v5.hi, s25.lo, s27.lo
AppendStoreVgpr(&code, 3, 0);
AppendEnd(&code);
TestCase test;
test.name = "Vop3Med3I16Captured";
test.code = std::move(code);
test.expected = {0xfc18cafeu};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_MED3_I16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"0x00000020: V_MED3_I16 v3.sdwa(sel=5,sext=0), "
"v5.opsel(lo=1,hi=0,neghi=0), s25, s27\n",
1}};
test.required_spirv = {" SMin ", " SMax "};
return test;
}
TestCase Vop2SdwaMinU32PreservesWordDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 10, 0xa1b2c3d4u);
AppendVMovU32(&code, 12, 7);
code.push_back(0x261418f9u);
code.push_back(0x0686149fu);
AppendStoreVgpr(&code, 10, 0);
AppendVMovLiteral(&code, 10, 0xa1b2c3d4u);
AppendVMovU32(&code, 12, 64);
code.push_back(0x261418f9u);
code.push_back(0x0686149fu);
AppendStoreVgpr(&code, 10, 1);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaMinU32PreservesWordDestination";
test.code = code;
test.expected = {0xa1b20007u, 0xa1b2001fu};
test.opcodes = {O::V_MOV_B32, O::V_MIN_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {" UMin ", "OpSelect"};
return test;
}
TestCase VectorShiftCountsMaskLowBits() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 1);
AppendVMovU32(&code, 1, 32);
AppendVMovU32(&code, 2, 33);
AppendVMovLiteral(&code, 3, 0x80000000u);
code.push_back(EncodeVop2(0x19, 10, Vgpr(0), 1));
code.push_back(EncodeVop2(0x1a, 11, Vgpr(2), 0));
code.push_back(EncodeVop2(0x15, 12, Vgpr(3), 1));
code.push_back(EncodeVop2(0x16, 13, Vgpr(2), 3));
code.push_back(EncodeVop2(0x17, 14, Vgpr(3), 1));
code.push_back(EncodeVop2(0x18, 15, Vgpr(2), 3));
for (u32 i = 0; i < 6u; i++) {
AppendStoreVgpr(&code, 10u + i, i);
}
AppendEnd(&code);
return {"VectorShiftCountsMaskLowBits",
code,
{},
{1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u},
{O::V_MOV_B32, O::V_LSHL_B32, O::V_LSHLREV_B32, O::V_LSHR_B32,
O::V_LSHRREV_B32, O::V_ASHR_I32, O::V_ASHRREV_I32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVop3LshrrevB64Captured() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 106, 0x89abcdefu);
AppendSMovLiteral(&code, 107, 0x01234567u);
AppendVMovU32(&code, 12, 68);
code.push_back(0xd7000001u);
code.push_back(0x0000d50cu); // v_lshrrev_b64 v[1:2], v12, vcc
AppendStoreVgpr(&code, 1, 0);
AppendStoreVgpr(&code, 2, 1);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3LshrrevB64Captured";
test.code = std::move(code);
test.expected = {0x789abcdeu, 0x00123456u};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::V_LSHRREV_B64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftRightLogical"};
return test;
}
TestCase VectorVop3LshlrevB64Captured() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 3, 30);
code.push_back(0xd6ff0021u);
code.push_back(0x00010303u); // v_lshlrev_b64 v[33:34], v3, 1
AppendStoreVgpr(&code, 33, 1);
AppendStoreVgpr(&code, 34, 2);
code.push_back(EncodeVop2(0x03, 3, InlineU32(31), 3));
code.push_back(0xd6ff0021u);
code.push_back(0x00010303u);
AppendStoreVgpr(&code, 33, 3);
AppendStoreVgpr(&code, 34, 4);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3LshlrevB64Captured";
test.code = std::move(code);
test.initial = {33u, 0u, 0u, 0u, 0u};
test.expected = {33u, 0u, 2u, 1u, 0u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftLeftLogical", "OpTypeInt 64 0"};
return test;
}
TestCase VectorDynamicU64ShiftEdges() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorDynamicU64ShiftEdges";
test.initial = {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u, 127u, UINT32_MAX};
test.expected = test.initial;
auto& code = test.code;
for (const uint64_t source : {uint64_t{0}, uint64_t{1},
uint64_t{0x8000000000000000ull},
uint64_t{0xffffffffffffffffull},
uint64_t{0x0123456789abcdefull}}) {
AppendVMovLiteral(&code, 4, static_cast<u32>(source));
AppendVMovLiteral(&code, 5, static_cast<u32>(source >> 32u));
for (u32 index = 0; index < test.initial.size(); ++index) {
// A buffer load prevents the constant-shift specialization.
AppendVMovU32(&code, 30, index * sizeof(u32));
AppendBufferLoadDword(&code, 3, 30);
for (const bool left : {true, false}) {
AppendVop3(&code, left ? 0x2ff : 0x300, 33, Vgpr(3), Vgpr(4));
const auto amount = test.initial[index] & 63u;
const auto expected = left ? source << amount : source >> amount;
const auto output = static_cast<u32>(test.expected.size());
AppendStoreVgpr(&code, 33, output);
AppendStoreVgpr(&code, 34, output + 1u);
test.expected.push_back(static_cast<u32>(expected));
test.expected.push_back(static_cast<u32>(expected >> 32u));
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64,
O::V_LSHRREV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftLeftLogical", "OpShiftRightLogical",
"OpTypeInt 64 0"};
return test;
}
TestCase VectorVop3IntegerOps() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 0, InlineU32(2)));
code.push_back(EncodeVop1(0x01, 1, InlineU32(3)));
code.push_back(EncodeVop1(0x01, 2, InlineU32(4)));
AppendVMovLiteral(&code, 3, 0xfffffff8u);
code.push_back(EncodeVop1(0x01, 4, InlineU32(5)));
AppendVMovLiteral(&code, 5, 0x11223344u);
AppendVMovLiteral(&code, 6, 0x55667788u);
AppendVMovLiteral(&code, 7, 0x0f0f0f0fu);
AppendVMovLiteral(&code, 8, 0x33333333u);
AppendVMovLiteral(&code, 9, 0xaaaaaaaau);
AppendVMovLiteral(&code, 14, 0x00f00000u);
code.push_back(EncodeVop1(0x01, 15, InlineU32(8)));
AppendVMovLiteral(&code, 30, 0x00010000u);
AppendVMovLiteral(&code, 31, 0x00010000u);
AppendVop3(&code, 0x36d, 10, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x152, 11, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x155, 12, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x158, 13, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x153, 16, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x156, 17, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x159, 18, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x15d, 19, Vgpr(0), Vgpr(4), Vgpr(2));
AppendVop3(&code, 0x346, 20, Vgpr(1), Vgpr(0), Vgpr(2));
AppendVop3(&code, 0x347, 21, Vgpr(0), Vgpr(1), Vgpr(0));
AppendVop3(&code, 0x345, 22, Vgpr(7), Vgpr(8), Vgpr(0));
AppendVop3(&code, 0x36f, 23, Vgpr(1), Vgpr(2), Vgpr(0));
AppendVop3(&code, 0x371, 24, Vgpr(7), Vgpr(8), Vgpr(2));
AppendVop3(&code, 0x372, 25, Vgpr(7), Vgpr(8), Vgpr(2));
AppendVop3(&code, 0x178, 26, Vgpr(7), Vgpr(8), Vgpr(2));
AppendVop3(&code, 0x148, 27, Vgpr(14), InlineU32(20), Vgpr(2));
AppendVop3(&code, 0x149, 28, Vgpr(3), Vgpr(2), Vgpr(2));
AppendVop3(&code, 0x14a, 29, Vgpr(8), Vgpr(7), Vgpr(9));
AppendVop3(&code, 0x14e, 32, Vgpr(5), Vgpr(6), Vgpr(15));
AppendVop3(&code, 0x363, 33, Vgpr(2), Vgpr(15));
AppendVop3(&code, 0x169, 34, Vgpr(30), Vgpr(31));
AppendVop3(&code, 0x16a, 35, Vgpr(30), Vgpr(31));
AppendVop3(&code, 0x16b, 36, Vgpr(30), Vgpr(31));
AppendVop3(&code, 0x16c, 44, Vgpr(3), Vgpr(30));
AppendVop3(&code, 0x142, 37, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x143, 38, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x30f, 39, Vgpr(0), Vgpr(1));
AppendVop3(&code, 0x310, 40, Vgpr(0), Vgpr(1));
AppendVop3(&code, 0x319, 41, Vgpr(0), Vgpr(1));
const u32 results[] = {10, 11, 12, 13, 16, 17, 18, 19, 20, 21,
22, 23, 24, 25, 26, 27, 28, 29, 32, 33,
34, 35, 36, 44, 37, 38, 39, 40, 41};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"VectorVop3IntegerOps",
code,
{},
{9,
0xfffffff8u,
5,
2,
2,
0xfffffff8u,
5,
7,
16,
20,
0x3c3c3c3eu,
0x32,
0x03030307u,
0x3f3f3f3fu,
0x3c3c3c38u,
0x0fu,
0xffffffffu,
0x8b8b8b8bu,
0x44556677u,
0x00000f00u,
0,
1,
0,
0xffffffffu,
10,
10,
5,
0xffffffffu,
1},
{O::V_MOV_B32, O::V_ADD3_U32, O::V_MIN3_I32,
O::V_MAX3_I32, O::V_MED3_I32, O::V_MIN3_U32,
O::V_MAX3_U32, O::V_MED3_U32, O::V_SAD_U32,
O::V_LSHL_ADD_U32, O::V_ADD_LSHL_U32, O::V_XAD_U32,
O::V_LSHL_OR_B32, O::V_AND_OR_B32, O::V_OR3_B32,
O::V_XOR3_B32, O::V_BFE_U32, O::V_BFE_I32,
O::V_BFI_B32, O::V_ALIGNBIT_B32, O::V_BFM_B32,
O::V_MUL_LO_U32, O::V_MUL_HI_U32, O::V_MUL_LO_I32,
O::V_MUL_HI_I32, O::V_MAD_I32_I24, O::V_MAD_U32_U24,
O::V_ADD_I32, O::V_SUB_I32, O::V_SUBREV_I32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorBfeI32SignExtendsField() {
using O = ShaderOpcode;
std::vector<u32> code;
for (u32 vgpr = 0; vgpr < 6; vgpr++) {
AppendVMovU32(&code, 30, vgpr * sizeof(u32));
AppendBufferLoadDword(&code, vgpr, 30);
}
AppendVop3(&code, 0x149, 10, Vgpr(0), Vgpr(2), Vgpr(3));
AppendVop3(&code, 0x149, 11, Vgpr(1), Vgpr(4), Vgpr(5));
AppendStoreVgpr(&code, 10, 0);
AppendStoreVgpr(&code, 11, 1);
AppendEnd(&code);
TestCase test;
test.name = "VectorBfeI32SignExtendsField";
test.code = std::move(code);
test.initial = {0x80000000u, 0xfffffff8u, 31, 1, 3, 4};
test.expected = {0xffffffffu, 0xffffffffu};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_BFE_I32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitFieldSExtract"};
return test;
}
TestCase VectorAlignByteUsesTwoBitByteOffset() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x11223344u);
AppendVMovLiteral(&code, 1, 0x55667788u);
const std::vector<u32> offsets = {0, 1, 2, 3, 4, 5, 7, 8, 31, 32, 0xffffffffu};
for (u32 i = 0; i < static_cast<u32>(std::size(offsets)); i++) {
AppendVMovU32(&code, 2, i * sizeof(u32));
AppendBufferLoadDword(&code, 2, 2);
AppendVop3(&code, 0x14f, 10u + i, Vgpr(0), Vgpr(1), Vgpr(2));
AppendStoreVgpr(&code, 10u + i, i);
}
AppendEnd(&code);
TestCase test{
"VectorAlignByteUsesTwoBitByteOffset",
code,
offsets,
{0x55667788u, 0x44556677u, 0x33445566u, 0x22334455u, 0x55667788u,
0x44556677u, 0x22334455u, 0x55667788u, 0x22334455u, 0x55667788u,
0x22334455u},
{O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_ALIGNBYTE_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.forbidden_spirv = {"OpTypeInt 64"};
return test;
}
TestCase VectorCarryAndBitCountOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xffffffffu);
code.push_back(EncodeVop1(0x01, 1, InlineU32(0)));
AppendVMovLiteral(&code, 3, 0x0000f0f0u);
code.push_back(EncodeVop1(0x01, 4, InlineU32(5)));
code.push_back(EncodeVopc(0xc7, Vgpr(1), 1));
code.push_back(EncodeVop2(0x28, 2, Vgpr(0), 1));
code.push_back(EncodeVop2(0x22, 5, Vgpr(3), 4));
AppendVMovLiteral(&code, 6, 0xffffffffu);
code.push_back(EncodeVop2(0x23, 7, Vgpr(6), 4));
code.push_back(EncodeVop2(0x24, 8, Vgpr(6), 4));
AppendStoreVgpr(&code, 2, 0);
AppendStoreSgpr(&code, 106, 1);
AppendStoreVgpr(&code, 5, 2);
AppendStoreVgpr(&code, 7, 3);
AppendStoreVgpr(&code, 8, 4);
AppendEnd(&code);
return {"VectorCarryAndBitCountOps",
code,
{},
{0, 1, 13, 5, 5},
{O::V_MOV_B32, O::V_CMP_T_U32, O::V_ADDC_U32, O::V_BCNT_U32_B32,
O::V_MBCNT_LO_U32_B32, O::V_MBCNT_HI_U32_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorMbcntUsesThreadMask() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1),
EncodeVop2(0x1a, 3, InlineU32(2), 1),
};
AppendVMovLiteral(&code, 2, 0xffffffffu);
code.push_back(EncodeVop1(0x01, 4, InlineU32(0)));
code.push_back(EncodeVop2(0x23, 5, Vgpr(2), 4));
code.push_back(EncodeVop2(0x24, 6, Vgpr(2), 5));
AppendBufferStoreDword(&code, 6, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorMbcntUsesThreadMask";
test.code = code;
test.expected = {0, 1, 2, 3, 0, 1, 2, 3};
test.opcodes = {O::V_MOV_B32,
O::V_LSHLREV_B32,
O::V_ADD_NC_U32,
O::V_MBCNT_LO_U32_B32,
O::V_MBCNT_HI_U32_B32,
O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase VectorAddcUsesPerLaneCarryIn() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1),
EncodeVopc(0xc7, Vgpr(0), 0),
EncodeVop1(0x01, 2, InlineU32(0)),
EncodeVop2(0x28, 3, Vgpr(2), 2),
EncodeVop2(0x1a, 4, InlineU32(2), 1),
};
AppendBufferStoreDword(&code, 3, 4);
AppendEnd(&code);
TestCase test;
test.name = "VectorAddcUsesPerLaneCarryIn";
test.code = code;
test.expected = std::vector<u32>(8, 1);
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_T_U32, O::V_ADDC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase VectorAddcWritesPerLaneCarryOut() {
using O = ShaderOpcode;
std::vector<u32> code = {EncodeVop1(0x01, 1, Vgpr(0)),
EncodeVop2(0x1a, 4, InlineU32(2), 1),
EncodeVopc(0xc7, Vgpr(0), 0)};
AppendBufferLoadDword(&code, 26, 4);
code.push_back(0x500434f9u);
code.push_back(0x0c860680u);
code.push_back(EncodeVop1(0x01, 6, 106));
AppendBufferStoreDword(&code, 6, 4);
AppendEnd(&code);
TestCase test;
test.name = "VectorAddcWritesPerLaneCarryOut";
test.code = code;
test.initial = {0x0000ffffu, 0u, 0x0000ffffu, 0u};
test.expected = {5u, 5u, 5u, 5u};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_CMP_T_U32,
O::BUFFER_LOAD_DWORD, O::V_ADDC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {
{"V_ADDC_U32 v2, vcc_lo, 0, v26.sdwa(sel=4,sext=1), vcc_lo", 1}};
test.ir_counts = {{" = BitFieldSExtract ", 1}, {" = IAddCarry32 ", 2}};
test.required_spirv = {"OpBitFieldSExtract"};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorSubCoCiU32CompactAndVop3() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
AppendBufferLoadDword(&code, 1, 4);
code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4));
AppendBufferLoadDword(&code, 2, 5);
code.push_back(EncodeVop2(0x1b, 3, InlineU32(1), 0));
code.push_back(EncodeVopc(0xc5, InlineU32(0), 3));
code.push_back(EncodeSop1(0x04, 20, 106));
code.push_back(EncodeVop2(0x29, 10, Vgpr(1), 2));
AppendVMovU32(&code, 6, 1);
code.push_back(EncodeVop2(0x01, 11, InlineU32(0), 6));
AppendStoreVgprAtLaneDwordOffset(&code, 10, 0, 0);
AppendStoreVgprAtLaneDwordOffset(&code, 11, 0, 4);
code.push_back(EncodeVopc(0xc2, InlineU32(0), 6));
AppendVop3B(&code, 0x129u, 12, 22, Vgpr(1), Vgpr(2), 20);
AppendVop3(&code, 0x101u, 13, InlineU32(0), Vgpr(6), 22);
code.push_back(EncodeVop2(0x01, 14, InlineU32(0), 6));
AppendStoreVgprAtLaneDwordOffset(&code, 12, 0, 8);
AppendStoreVgprAtLaneDwordOffset(&code, 13, 0, 12);
AppendStoreVgprAtLaneDwordOffset(&code, 14, 0, 16);
AppendEnd(&code);
TestCase test;
test.name = "VectorSubCoCiU32CompactAndVop3";
test.code = std::move(code);
test.initial = {5u, 5u, 3u, 3u, 3u, 3u, 5u, 3u, 0u, 0u,
0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u};
test.expected = {2u, 1u, 0xfffffffeu, 0xffffffffu, 0u, 0u, 1u,
1u, 2u, 1u, 0xfffffffeu, 0xffffffffu, 0u, 0u,
1u, 1u, 0u, 0u, 0u, 0u};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_AND_B32, O::V_CMP_NE_U32, O::V_CMP_EQ_U32,
O::V_SUB_CO_CI_U32, O::V_CNDMASK_B32, O::S_MOV_B64,
O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_SUB_CO_CI_U32", 2}};
test.required_spirv = {"OpISub", "OpUGreaterThan", "OpLogicalOr"};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase VectorSubrevCoCiU32ExactRawOnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 24, Vgpr(0)));
code.push_back(EncodeSop1(0x04, 20, 126)); // s_mov_b64 s[20:21], exec
code.push_back(EncodeVop1(0x01, 1, InlineU32(2)));
code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4));
code.push_back(EncodeVopc(0xc1, Vgpr(0), 1));
code.push_back(EncodeSop1(0x04, 22, 106)); // s_mov_b64 s[22:23], vcc
code.push_back(EncodeVopc(0xc5, InlineU32(0), 0));
AppendVMovLiteral(&code, 0, 0xdeadbeefu);
code.push_back(EncodeSop1(0x04, 126, 106)); // s_mov_b64 exec, vcc
code.push_back(EncodeSop1(0x04, 106, 22)); // s_mov_b64 vcc, s[22:23]
code.push_back(0x54003080u); // v_subrev_co_ci_u32 v0, vcc, 0, v24, vcc
code.push_back(EncodeSop1(0x04, 126, 20)); // s_mov_b64 exec, s[20:21]
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 2));
AppendBufferStoreDword(&code, 0, 4);
AppendBufferStoreDword(&code, 3, 5);
AppendEnd(&code);
TestCase test;
test.name = "VectorSubrevCoCiU32ExactRawOnGpu";
test.code = code;
test.expected = {0xdeadbeefu, 0, 2, 3, 0, 0, 0, 0};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::V_CMP_LT_U32,
O::S_MOV_B64, O::V_CMP_NE_U32,
O::V_SUBREV_CO_CI_U32, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpISub", "OpUGreaterThan"};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
return test;
}
TestCase VectorVop3BSubrevCoCiUsesEncodedMasks() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 24, Vgpr(0)));
code.push_back(EncodeVop2(0x25, 24, InlineU32(3), 24));
code.push_back(EncodeVop1(0x01, 1, InlineU32(2)));
code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4));
code.push_back(EncodeVopc(0xc2, InlineU32(1), 0));
code.push_back(EncodeSop1(0x04, 20, 106)); // s_mov_b64 s[20:21], vcc
AppendSMovLiteral(&code, 106, 0);
AppendSMovLiteral(&code, 107, 0);
AppendVop3B(&code, 0x12au, 10, 22, InlineU32(5), Vgpr(24), 20);
AppendVop3(&code, 0x101u, 3, InlineU32(0), Vgpr(2), 22);
AppendBufferStoreDword(&code, 10, 4);
AppendBufferStoreDword(&code, 3, 5);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3BSubrevCoCiUsesEncodedMasks";
test.code = code;
test.expected = {0xfffffffeu, 0xfffffffeu, 0, 1, 1, 1, 0, 0};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::S_MOV_B32,
O::V_SUBREV_CO_CI_U32, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
return test;
}
TestCase VectorVop3BCarryOutWritesSgprMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0xffffffffu);
AppendVMovU32(&code, 2, 1);
AppendVMovU32(&code, 3, 0);
AppendVop3B(&code, 0x30fu, 10, 0, Vgpr(1), Vgpr(2));
AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 0);
AppendVop3B(&code, 0x310u, 11, 0, Vgpr(3), Vgpr(2));
AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 4);
AppendVop3B(&code, 0x319u, 12, 0, Vgpr(2), Vgpr(3));
AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 8);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3BCarryOutWritesSgprMask";
test.code = code;
test.expected = std::vector<u32>(12, 15u);
test.opcodes = {O::V_MOV_B32, O::V_ADD_I32, O::V_SUB_I32,
O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
return test;
}
TestCase VectorVop3BCarryOutUsesEncodedSdst() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0xffffffffu);
AppendVMovU32(&code, 2, 1);
AppendVMovU32(&code, 3, 0);
AppendVop3B(&code, 0x30fu, 10, 20, Vgpr(1), Vgpr(2));
AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 0);
AppendVop3B(&code, 0x310u, 11, 22, Vgpr(3), Vgpr(2));
AppendStoreSgprAtLaneDwordOffset(&code, 22, 0, 4);
AppendVop3B(&code, 0x319u, 12, 24, Vgpr(2), Vgpr(3));
AppendStoreSgprAtLaneDwordOffset(&code, 24, 0, 8);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3BCarryOutUsesEncodedSdst";
test.code = code;
test.expected = std::vector<u32>(12, 15u);
test.opcodes = {O::V_MOV_B32, O::V_ADD_I32, O::V_SUB_I32,
O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
return test;
}
TestCase VectorVop3BSubCoU32UsesRdna2Opcode310() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovU32(&code, 2, 1);
AppendVMovU32(&code, 3, 1);
AppendVMovLiteral(&code, 4, 0xffffffffu);
AppendVMovLiteral(&code, 5, 0x80000000u);
code.push_back(EncodeVopc(0xc2, InlineU32(1), 0));
code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(3)));
code.push_back(EncodeVopc(0xc2, InlineU32(2), 0));
code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(4)));
code.push_back(EncodeVopc(0xc2, InlineU32(3), 0));
code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(5)));
code.push_back(EncodeVop2(0x01, 2, Vgpr(2), Vgpr(4)));
AppendVop3B(&code, 0x310u, 10, 20, Vgpr(1), Vgpr(2));
AppendStoreVgprAtLaneDwordOffset(&code, 10, 0, 0);
AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 4);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3BSubCoU32UsesRdna2Opcode310";
test.code = code;
test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 9, 9, 9, 9};
test.opcodes = {O::V_MOV_B32, O::V_CMP_EQ_U32, O::V_CNDMASK_B32,
O::V_SUB_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
return test;
}
TestCase VectorMadU64U32UnsignedCarryOut() {
using O = ShaderOpcode;
struct Input {
u32 lhs;
u32 rhs;
uint64_t addend;
};
constexpr Input inputs[] = {
{0, UINT32_MAX, UINT64_MAX},
{UINT32_MAX, 2, 0xffffffff00000002ull},
{UINT32_MAX, UINT32_MAX, UINT64_MAX},
{0x80000000u, 0x80000000u, 0x4000000000000000ull},
{3, 5, 7},
};
TestCase test;
test.name = "VectorMadU64U32UnsignedCarryOut";
for (const auto &input : inputs) {
test.initial.insert(
test.initial.end(),
{input.lhs, input.rhs, u32(input.addend), u32(input.addend >> 32)});
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < std::size(inputs); ++i) {
for (u32 word = 0; word < 4; ++word) {
AppendVMovU32(&code, 30, (i * 4 + word) * sizeof(u32));
AppendBufferLoadDword(&code, word < 2 ? word + 1 : word + 2, 30);
}
AppendVop3B(&code, 0x176u, 10, 20, Vgpr(1), Vgpr(2), Vgpr(4));
const auto output = static_cast<u32>(test.expected.size());
AppendStoreVgpr(&code, 10, output);
AppendStoreVgpr(&code, 11, output + 1);
AppendStoreSgprPair(&code, 20, output + 2);
const auto product = uint64_t(inputs[i].lhs) * inputs[i].rhs;
const auto sum = product + inputs[i].addend;
test.expected.insert(test.expected.end(),
{u32(sum), u32(sum >> 32), u32(sum < product), 0});
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_U64_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
// NVIDIA rejected the native widening multiply in a large Astro's Playroom
// vertex shader. Keep its 32x32 product portable and its 64-bit addition
// native.
test.required_spirv = {"OpTypeInt 64 0", "OpUMulExtended", "OpIMul %uint",
"OpIAdd %ulong", "OpULessThan"};
test.forbidden_spirv = {"OpIMul %ulong", "OpIMul %long"};
return test;
}
TestCase VectorLaneAndPackedOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x12345678u);
code.push_back(EncodeVop1(0x01, 1, InlineU32(0)));
AppendVMovLiteral(&code, 4, 0x40003c00u);
AppendVMovLiteral(&code, 5, 0x44004200u);
AppendVMovLiteral(&code, 6, 0x3c003c00u);
AppendVMovLiteral(&code, 7, 0x3f800000u);
AppendVMovLiteral(&code, 8, 0x40000000u);
AppendVMovLiteral(&code, 9, 0x12345678u);
AppendVMovLiteral(&code, 10, 0xabcdef01u);
code.push_back(EncodeVop1(0x02, 0, Vgpr(0)));
AppendVop3(&code, 0x360, 1, Vgpr(0), InlineU32(0));
AppendVop3(&code, 0x361, 1, 0, InlineU32(0));
AppendVop3(&code, 0x377, 2, Vgpr(0), InlineU32(0), InlineU32(0));
code.push_back(EncodeVop2(0x2f, 3, Vgpr(7), 8));
AppendVop3p(&code, 0x0f, 11, Vgpr(4), Vgpr(5), 0, 0x3);
AppendVop3p(&code, 0x10, 12, Vgpr(4), Vgpr(5), 0, 0x3);
AppendVop3p(&code, 0x11, 13, Vgpr(4), Vgpr(5), 0, 0x3);
AppendVop3p(&code, 0x12, 14, Vgpr(4), Vgpr(5), 0, 0x3);
AppendVop3p(&code, 0x0e, 15, Vgpr(4), Vgpr(5), Vgpr(6), 0x7);
AppendVop3(&code, 0x362, 16, Vgpr(7), InlineU32(1));
AppendVop3(&code, 0x36a, 17, Vgpr(9), Vgpr(10));
AppendStoreSgpr(&code, 0, 0);
AppendStoreSgpr(&code, 1, 1);
const u32 results[] = {1, 2, 3, 11, 12, 13, 14, 15, 16, 17};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i + 2u);
}
AppendEnd(&code);
return {"VectorLaneAndPackedOps",
code,
{},
{0x12345678u, 0x12345678u, 0x12345678u, 0x12345678u, 0x40003c00u,
0x46004400u, 0x48004200u, 0x40003c00u, 0x44004200u, 0x48804400u,
0x40000000u, 0xef015678u},
{O::V_MOV_B32, O::V_READFIRSTLANE_B32, O::V_READLANE_B32,
O::V_WRITELANE_B32, O::V_PERMLANE16_B32, O::V_CVT_PKRTZ_F16_F32,
O::V_PK_ADD_F16, O::V_PK_MUL_F16, O::V_PK_MIN_F16, O::V_PK_MAX_F16,
O::V_PK_FMA_F16, O::V_LDEXP_F32, O::V_CVT_PK_U16_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop2PkFmacF16AccumulatesPackedHalvesIndependently() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x50004c00u); // low=16.0h, high=32.0h
AppendVMovLiteral(&code, 1, 0x42003c00u); // low=1.0h, high=3.0h
AppendVMovLiteral(&code, 2, 0x48004400u); // low=4.0h, high=8.0h
code.push_back(EncodeVop2(0x3c, 0, Vgpr(1), 2));
AppendStoreVgpr(&code, 0, 0);
code.push_back(EncodeVop2(0x3c, 0, Vgpr(1), 2));
AppendStoreVgpr(&code, 0, 1);
AppendVMovLiteral(&code, 3, 0x50004c00u);
code.push_back(EncodeVop2(0x3c, 3, 242, 2)); // Inline 1.0h is {1.0h, 0.0h}.
AppendStoreVgpr(&code, 3, 2);
AppendEnd(&code);
return {"Vop2PkFmacF16AccumulatesPackedHalvesIndependently",
code,
{},
{0x53004d00u, 0x55004e00u, 0x50004d00u},
{O::V_MOV_B32, O::V_PK_FMAC_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase Vop2PkFmacF16DppNegatesBothPackedHalves() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x50004c00u); // low=16.0h, high=32.0h
AppendVMovLiteral(&code, 1, 0x42003c00u); // low=1.0h, high=3.0h
AppendVMovLiteral(&code, 2, 0x48004400u); // low=4.0h, high=8.0h
code.push_back(EncodeVop2(0x3c, 0, 250, 2));
code.push_back(EncodeVop2Dpp(1, 0, 0xf, 0xf, 1));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"Vop2PkFmacF16DppNegatesBothPackedHalves",
code,
{},
{0x48004a00u},
{O::V_MOV_B32, O::V_PK_FMAC_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase Vop3pOpselHiUsesArchitecturalSourceBits() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x42003c00u); // low=1.0h, high=3.0h
AppendVMovLiteral(&code, 1, 0x48004400u); // low=4.0h, high=8.0h
AppendVMovLiteral(&code, 2, 0x50004c00u); // low=16.0h, high=32.0h
// Raw V_PK_FMA_F16 words keep this test independent of the encoders above.
code.insert(code.end(), {0xcc0e000au, 0x0c0a0300u}); // src0 high: 3*4+16=28
code.insert(code.end(), {0xcc0e000bu, 0x140a0300u}); // src1 high: 1*8+16=24
code.insert(code.end(), {0xcc0e400cu, 0x040a0300u}); // src2 high: 1*4+32=36
for (u32 i = 0; i < 3; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"Vop3pOpselHiUsesArchitecturalSourceBits",
code,
{},
{0x4f004d00u, 0x4e004d00u, 0x50804d00u},
{O::V_MOV_B32, O::V_PK_FMA_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop3pIntegerNegationCapturedAndSelectedHalves() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 6, 0xfed4ff38u); // (-200, -300)
AppendVMovLiteral(&code, 3, 0x012c00c8u); // (200, 300)
// Captured temporal shader: NEG_LO/HI turns 0x7fff into 0xffff (-1).
// Adding that clamped offset must select (199,299), not collapse to (0,0).
code.insert(code.end(), {0xcc074107u, 0x30020cffu, 0x00007fffu});
code.insert(code.end(), {0xcc0a4010u, 0x18020f03u});
AppendStoreVgpr(&code, 7, 0);
AppendStoreVgpr(&code, 16, 1);
AppendVMovLiteral(&code, 0, 0x80011234u);
AppendVMovLiteral(&code, 1, 0xbeef5678u);
// Independently negate src0 low and src1 high, then reverse both the
// modified sources and src1 half selection. NEG flips the selected sign bit.
AppendVop3p(&code, 0x0a, 10, Vgpr(0), Vgpr(1), 0, 0x3, 0, 0x2, 0x1);
AppendVop3p(&code, 0x0a, 11, Vgpr(0), Vgpr(1), 0, 0x1, 0x2, 0x1, 0x2);
AppendStoreVgpr(&code, 10, 2);
AppendStoreVgpr(&code, 11, 3);
AppendEnd(&code);
return {"Vop3pIntegerNegationCapturedAndSelectedHalves",
code,
{},
{0xffffffffu, 0x012b00c7u, 0xbef0e8acu, 0x56795123u},
{O::V_MOV_B32, O::V_PK_MAX_I16, O::V_PK_ADD_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop3pInt16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0x80007fffu);
AppendVMovLiteral(&code, 2, 0x00010001u);
AppendVMovLiteral(&code, 3, 0xfffe0003u);
AppendVMovLiteral(&code, 4, 0x00130004u);
AppendVMovLiteral(&code, 5, 0x8010f00fu);
AppendVMovLiteral(&code, 6, 0x00050007u);
AppendVop3p(&code, 0x00u, 10, Vgpr(1), Vgpr(3), Vgpr(6), 0x7u);
AppendVop3p(&code, 0x09u, 11, Vgpr(1), Vgpr(3), Vgpr(6), 0x7u);
AppendVop3p(&code, 0x01u, 12, Vgpr(1), Vgpr(3), 0, 0x3u);
AppendVop3p(&code, 0x02u, 13, Vgpr(1), Vgpr(2), 0, 0x3u);
AppendVop3p(&code, 0x0bu, 14, Vgpr(2), Vgpr(3), 0, 0x3u);
AppendVop3p(&code, 0x04u, 15, Vgpr(4), Vgpr(5), 0, 0x3u);
AppendVop3p(&code, 0x05u, 16, Vgpr(4), Vgpr(5), 0, 0x3u);
AppendVop3p(&code, 0x06u, 17, Vgpr(4), Vgpr(5), 0, 0x3u);
AppendVop3p(&code, 0x08u, 18, Vgpr(1), Vgpr(2), 0, 0x3u);
AppendVop3p(&code, 0x0cu, 19, Vgpr(1), Vgpr(2), 0, 0x3u);
AppendVop3p(&code, 0x0du, 20, Vgpr(1), Vgpr(2), 0, 0x3u);
for (u32 i = 0; i < 11; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "Vop3pInt16Ops";
test.code = std::move(code);
test.expected = {0x00058004u, 0x00058004u, 0x00007ffdu, 0x80018000u,
0x0003fffeu, 0x008000f0u, 0x10020f00u, 0xf002ff00u,
0x80000001u, 0x80007fffu, 0x00010001u};
test.opcodes = {O::V_MOV_B32, O::V_PK_MAD_I16, O::V_PK_MAD_U16,
O::V_PK_MUL_LO_U16, O::V_PK_ADD_I16, O::V_PK_SUB_U16,
O::V_PK_LSHLREV_B16, O::V_PK_LSHRREV_B16, O::V_PK_ASHRREV_I16,
O::V_PK_MIN_I16, O::V_PK_MAX_U16, O::V_PK_MIN_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase CvtPkU8F32PacksSelectedByte() {
using O = ShaderOpcode;
std::vector<u32> code;
// RDNA2 V_CVT_PK_U8_F32: uint8(S0) replaces byte S1[1:0] in S2.
AppendVMovLiteral(&code, 0, 0x414c0000u); // 12.75f truncates to 12.
AppendVMovLiteral(&code, 1, 0x43960000u); // 300.0f saturates to 255.
AppendVMovLiteral(&code, 2, 0xbf800000u); // Negative values saturate to zero.
AppendVMovLiteral(&code, 3, 0x7fc00000u); // NaN saturates to zero.
AppendVMovLiteral(&code, 4, 0x11223344u);
AppendVop3(&code, 0x15eu, 10, Vgpr(0), InlineU32(0), Vgpr(4));
AppendVop3(&code, 0x15eu, 11, Vgpr(1), InlineU32(5), Vgpr(4));
AppendVop3(&code, 0x15eu, 12, Vgpr(2), InlineU32(2), Vgpr(4));
AppendVop3(&code, 0x15eu, 13, Vgpr(3), InlineU32(3), Vgpr(4));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {
"CvtPkU8F32PacksSelectedByte",
code,
{},
{0x1122330cu, 0x1122ff44u, 0x11003344u, 0x00223344u},
{O::V_MOV_B32, O::V_CVT_PK_U8_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase CvtPkrtzF16F32SubnormalRoundsTowardZero() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x33c00000u);
AppendVMovLiteral(&code, 1, 0xb3c00000u);
code.push_back(EncodeVop2(0x2f, 2, Vgpr(0), 1));
AppendStoreVgpr(&code, 2, 0);
AppendEnd(&code);
return {"CvtPkrtzF16F32SubnormalRoundsTowardZero",
code,
{},
{0x80010001u},
{O::V_MOV_B32, O::V_CVT_PKRTZ_F16_F32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase CvtPkrtzF16F32SdwaAndOutputModifiers() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f000000u); // 0.5f
AppendVMovLiteral(&code, 1, 0x40000000u); // 2.0f
AppendVMovLiteral(&code, 2, 0x3f802000u); // exactly 0x3c01h
AppendVMovLiteral(&code, 3, 0x00000000u);
AppendVMovLiteral(&code, 4, 0xbf000000u); // -0.5f
for (u32 dst = 10; dst <= 18; dst++) {
AppendVMovLiteral(&code, dst, 0xa1b2c3d4u);
}
code.push_back(EncodeVop2(0x2f, 10, 249, 1));
code.push_back(EncodeVop2Sdwa(0, 6, 2, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0,
1));
code.push_back(EncodeVop2(0x2f, 11, 249, 3));
code.push_back(EncodeVop2Sdwa(2, 0, 0));
code.push_back(EncodeVop2(0x2f, 12, 249, 1));
code.push_back(EncodeVop2Sdwa(0, 6, 0, 3, 5));
code.push_back(EncodeVop2(0x2f, 13, 249, 1));
code.push_back(EncodeVop2Sdwa(4, 6, 0, 6, 6, 0, 0, 0, 1, 1));
code.push_back(EncodeVop2(0x2f, 14, 250, 1));
code.push_back(EncodeVop2Dpp(0, 0, 0xf, 0xf, 1));
AppendVop3(&code, 0x12f, 15, Vgpr(0), Vgpr(1), 0, 0, 0, true);
code.push_back(EncodeVop2(0x2f, 16, 249, 1));
code.push_back(EncodeVop2Sdwa(0, 4, 2));
code.push_back(EncodeVop2(0x2f, 17, 249, 1));
code.push_back(EncodeVop2Sdwa(0, 6, 2, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0,
0, 1));
AppendVop3(&code, 0x12f, 18, Vgpr(0), Vgpr(1), 0, 0, 0, false, 1);
for (u32 dst = 10; dst <= 18; dst++) {
AppendStoreVgpr(&code, dst, dst - 10);
}
AppendEnd(&code);
return {"CvtPkrtzF16F32SdwaAndOutputModifiers",
code,
{},
{0x3c003800u, 0x00000001u, 0x00000000u, 0xc0003800u,
0x4000b800u, 0x3c003800u, 0xa1b23800u, 0x44003c00u,
0x44003c00u},
{O::V_MOV_B32, O::V_CVT_PKRTZ_F16_F32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase PackedMinMaxF16NanAndSignedZeroEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x80004000u);
AppendVMovLiteral(&code, 1, 0x00007e00u);
AppendVMovLiteral(&code, 2, 0x00004000u);
AppendVMovLiteral(&code, 3, 0x80007e00u);
AppendVMovLiteral(&code, 4, 0x40007d01u);
AppendVMovLiteral(&code, 5, 0x7d014000u);
AppendVop3p(&code, 0x11, 10, Vgpr(0), Vgpr(1), 0, 0x3);
AppendVop3p(&code, 0x12, 11, Vgpr(2), Vgpr(3), 0, 0x3);
AppendVop3p(&code, 0x11, 12, Vgpr(4), Vgpr(5), 0, 0x3);
AppendVop3p(&code, 0x12, 13, Vgpr(4), Vgpr(5), 0, 0x3);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"PackedMinMaxF16NanAndSignedZeroEdges",
code,
{},
{0x80004000u, 0x00004000u, 0x7f017f01u, 0x7f017f01u},
{O::V_MOV_B32, O::V_PK_MIN_F16, O::V_PK_MAX_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorMinMaxF16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xaaaa4000u); // low=2.0h
AppendVMovLiteral(&code, 1, 0xbbbb3c00u); // low=1.0h
AppendVMovLiteral(&code, 2, 0x12345678u);
AppendVMovLiteral(&code, 3, 0x87654321u);
AppendVMovLiteral(&code, 4, 0x5555aaaau);
AppendVMovLiteral(&code, 5, 0x6666bbbbu);
AppendVMovLiteral(&code, 6, 0x7777bbbbu);
AppendVMovLiteral(&code, 7, 0x8888bbbbu);
AppendVMovLiteral(&code, 8, 0x9999bbbbu);
AppendVMovLiteral(&code, 9, 0x99990000u);
code.push_back(EncodeVop2(0x32, 5, Vgpr(0), 1));
code.push_back(EncodeVop2(0x33, 6, Vgpr(0), 1));
code.push_back(EncodeVop2(0x34, 7, Vgpr(0), 1));
code.push_back(EncodeVop2(0x35, 4, Vgpr(0), 1));
code.push_back(EncodeVop2(0x39, 2, Vgpr(0), 1));
code.push_back(EncodeVop2(0x3a, 3, Vgpr(0), 1));
code.push_back(EncodeVop2(0x35, 9, 249, 1));
code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4));
code.push_back(0x660000f9u);
code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4));
AppendStoreVgpr(&code, 5, 0);
AppendStoreVgpr(&code, 6, 1);
AppendStoreVgpr(&code, 7, 2);
AppendStoreVgpr(&code, 0, 3);
AppendStoreVgpr(&code, 4, 4);
AppendStoreVgpr(&code, 2, 5);
AppendStoreVgpr(&code, 3, 6);
AppendStoreVgpr(&code, 9, 7);
AppendEnd(&code);
return {"VectorMinMaxF16Ops",
code,
{},
{0x66664200u, 0x77773c00u, 0x8888bc00u, 0xaaaa0000u, 0x55554000u,
0x12344000u, 0x87653c00u, 0x99994000u},
{O::V_MOV_B32, O::V_ADD_F16, O::V_SUB_F16, O::V_SUBREV_F16,
O::V_MUL_F16, O::V_MAX_F16, O::V_MIN_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorCvtF16U16ByteSource() {
using O = ShaderOpcode;
constexpr std::array inputs = {0xff123456u, 0x80123456u, 0x00123456u};
std::vector<u32> code;
AppendVMovLiteral(&code, 16, 0xa5a55a5au);
for (u32 i = 0; i < inputs.size(); i++) {
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 18, 30);
// Convert unsigned byte3 to FP16 word0 while preserving word1 of v16.
code.insert(code.end(), {0x7e20a0f9u, 0x00031412u});
AppendStoreVgpr(&code, 16, i);
}
AppendEnd(&code);
TestCase test;
test.name = "VectorCvtF16U16ByteSource";
test.code = std::move(code);
test.initial.assign(inputs.begin(), inputs.end());
test.expected = {0xa5a55bf8u, 0xa5a55800u, 0xa5a50000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_CVT_F16_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitFieldUExtract", "OpConvertUToF"};
return test;
}
TestCase VectorCvtU16F16Sdwa() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x45004200u); // low=3.0h, high=5.0h
AppendVMovLiteral(&code, 1, 0xbc007c00u); // low=+inf, high=-1.0h
AppendVMovLiteral(&code, 2, 0x00007e00u); // low=qNaN
AppendVMovLiteral(&code, 10, 0x12345678u);
AppendVMovLiteral(&code, 11, 0x87654321u);
AppendVMovLiteral(&code, 12, 0xabcd1111u);
AppendVMovLiteral(&code, 13, 0x77772222u);
AppendVMovLiteral(&code, 14, 0x5555aaaau);
code.push_back(EncodeVop1(0x52, 10, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 4));
code.push_back(EncodeVop1(0x52, 11, 249));
code.push_back(EncodeVop1Sdwa(0, 5, 2, 5));
code.push_back(EncodeVop1(0x52, 12, 249));
code.push_back(EncodeVop1Sdwa(1, 4, 2, 4));
code.push_back(EncodeVop1(0x52, 13, 249));
code.push_back(EncodeVop1Sdwa(1, 4, 2, 5));
code.push_back(EncodeVop1(0x52, 14, 249));
code.push_back(EncodeVop1Sdwa(2, 4, 2, 4));
for (u32 i = 0; i < 5; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"VectorCvtU16F16Sdwa",
code,
{},
{0x12340003u, 0x00054321u, 0xabcdffffu, 0x77770000u, 0x55550000u},
{O::V_MOV_B32, O::V_CVT_U16_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase NativeAndSdwa16BitDestinationWrites() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3fc00000u); // 1.5f -> 0x3e00h
AppendVMovLiteral(&code, 1, 0x00004200u); // low=3.0h
AppendVMovLiteral(&code, 2, 0x0000c000u); // low=-2.0h
AppendVMovLiteral(&code, 3, 0x00003c00u); // low=1.0h
AppendVMovLiteral(&code, 4, 0x00004000u); // low=2.0h
const u32 sentinels[] = {
0x3c001234u, 0xbbbb2345u, 0xcccc3456u, 0xdddd4567u, 0xeeee5678u,
0xffff6789u, 0xf00d789au, 0x11118888u, 0x22229999u, 0x3333aaaau,
0xaaaa0000u, 0xbbbb0000u, 0xcccc0000u, 0xabcd1234u, 0xbcde2345u,
0xcdef3456u, 0xdef01234u, 0xef012345u, 0xf0123456u, 0x01234567u,
0xaaaa1234u, 0xbbbb5678u, 0x24681234u, 0x13572468u, 0x89abcdefu,
0xfedcba98u, 0x0bad1234u,
};
// AppendStoreVgpr uses v31 as its address scratch register.
const auto output_vgpr = [](u32 index) { return 10u + index + (index >= 21u ? 1u : 0u); };
for (u32 i = 0; i < static_cast<u32>(std::size(sentinels)); i++) {
AppendVMovLiteral(&code, output_vgpr(i), sentinels[i]);
}
code.push_back(EncodeVop1(0x0a, 10, Vgpr(0)));
code.push_back(EncodeVop1(0x0a, 11, 250));
code.push_back(EncodeVop1Dpp(0));
AppendVop3(&code, 0x18a, 12, Vgpr(0), 0);
code.push_back(EncodeVop1(0x52, 13, Vgpr(1)));
code.push_back(EncodeVop1(0x53, 14, Vgpr(2)));
code.push_back(EncodeVop1(0x0a, 15, 249));
code.push_back(EncodeVop1Sdwa(0));
code.push_back(EncodeVop1(0x54, 16, 249));
code.push_back(EncodeVop1Sdwa(10, 4, 2, 5));
code.push_back(EncodeVop1(0x52, 17, 249));
code.push_back(EncodeVop1Sdwa(1, 6, 0, 4));
code.push_back(EncodeVop1(0x52, 18, 249));
code.push_back(EncodeVop1Sdwa(1, 4, 0, 4));
code.push_back(EncodeVop1(0x53, 19, 249));
code.push_back(EncodeVop1Sdwa(2, 4, 1, 4));
code.push_back(EncodeVop2(0x32, 20, 250, 4));
code.push_back(EncodeVop2Dpp(3));
AppendVop3(&code, 0x132, 21, Vgpr(3), Vgpr(4));
code.push_back(EncodeVop2(0x32, 22, 249, 4));
code.push_back(EncodeVop2Sdwa(3, 4, 0, 4, 4));
code.push_back(EncodeVop1(0x0a, 23, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 3));
code.push_back(EncodeVop1(0x0a, 24, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 5));
code.push_back(EncodeVop1(0x0a, 25, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 1, 1));
code.push_back(EncodeVop1(0x0a, 26, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 0, 0, 0, 0, 1));
code.push_back(EncodeVop1(0x0a, 27, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 0, 0, 0, 1));
AppendVop3(&code, 0x18a, 28, Vgpr(0), 0, 0, 0, 0, false, 1);
AppendVop3(&code, 0x18a, 29, Vgpr(0), 0, 0, 0, 0, true);
AppendVop3(&code, 0x34b, 30, Vgpr(3), Vgpr(4), Vgpr(3));
AppendVop3(&code, 0x34b, 32, Vgpr(3), Vgpr(4), Vgpr(3), 0, 0x8);
code.push_back(EncodeVop1(0x52, 33, 249));
code.push_back(EncodeVop1Sdwa(1, 4, 2, 4, 0, 1, 1, 0, 1));
AppendVop3(&code, 0x1d3, 34, Vgpr(1), 0, 0, 1, 0, true, 0, 1);
code.push_back(EncodeVop1(0x07, 35, 249));
code.push_back(EncodeVop1Sdwa(0, 6, 0, 6, 0, 1, 1, 0, 1));
AppendVop3(&code, 0x188, 36, Vgpr(0), 0, 0, 1, 0, true, 0, 1);
code.push_back(EncodeVop1(0x54, 37, 249));
code.push_back(EncodeVop1Sdwa(4, 4, 2, 4, 0, 1, 1));
for (u32 i = 0; i < static_cast<u32>(std::size(sentinels)); i++) {
AppendStoreVgpr(&code, output_vgpr(i), i);
}
AppendEnd(&code);
return {"NativeAndSdwa16BitDestinationWrites",
code,
{},
{0x3c003e00u, 0xbbbb3e00u, 0xcccc3e00u, 0xdddd0003u,
0xeeeefffeu, 0x00003e00u, 0xf00d3c00u, 0x00000003u,
0x00000003u, 0xfffffffeu, 0xaaaa4200u, 0xbbbb4200u,
0x00004200u, 0xabcd0000u, 0xbcde0000u, 0xcdefbe00u,
0xdef04200u, 0xef013c00u, 0xf0124200u, 0x01233c00u,
0xaaaa4200u, 0x42005678u, 0x24680000u, 0x1357fffdu,
0x00000000u, 0xffffffffu, 0x0badb800u},
{O::V_MOV_B32, O::V_CVT_F16_F32, O::V_CVT_U32_F32,
O::V_CVT_I32_F32, O::V_CVT_U16_F16, O::V_CVT_I16_F16,
O::V_RCP_F16, O::V_ADD_F16, O::V_FMA_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorMinMaxMed3F16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x40003c00u); // low=1.0h, high=2.0h
AppendVMovLiteral(&code, 1, 0x44004200u); // low=3.0h, high=4.0h
AppendVMovLiteral(&code, 2, 0xc8004000u); // low=2.0h, high=-8.0h
AppendVMovLiteral(&code, 3, 0x00007e00u); // low=qNaN
AppendVMovLiteral(&code, 4, 0x00004000u); // low=2.0h
AppendVMovLiteral(&code, 5, 0x00003c00u); // low=1.0h
AppendVMovLiteral(&code, 10, 0xaaaa5555u);
AppendVMovLiteral(&code, 11, 0x12345678u);
AppendVMovLiteral(&code, 12, 0x77772222u);
AppendVop3(&code, 0x351, 10, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x354, 11, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0xf, false, 0,
0x4);
AppendVop3(&code, 0x357, 12, Vgpr(3), Vgpr(4), Vgpr(5));
for (u32 i = 0; i < 3; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"VectorMinMaxMed3F16Ops",
code,
{},
{0xaaaa3c00u, 0x48005678u, 0x77773c00u},
{O::V_MOV_B32, O::V_MIN3_F16, O::V_MAX3_F16, O::V_MED3_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorSpecialF16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xc000fc00u); // low=-inf, high=-2.0h
AppendVMovLiteral(&code, 1, 0x00004400u); // low=+4.0h
AppendVMovLiteral(&code, 2, 0x00003c00u); // low=+1.0h
AppendVMovLiteral(&code, 3, 0x0000bc00u); // low=-1.0h
AppendVMovLiteral(&code, 4, 0x0000fc00u); // low=-inf
AppendVMovLiteral(&code, 11, 0x12345678u);
code.push_back(EncodeVop1(0x54, 10, Vgpr(0)));
code.push_back(EncodeVop1(0x54, 11, 249));
code.push_back(EncodeVop1Sdwa(0, 5, 2, 5));
code.push_back(EncodeVop1(0x56, 12, Vgpr(1)));
code.push_back(EncodeVop1(0x57, 13, Vgpr(2)));
code.push_back(EncodeVop1(0x57, 14, Vgpr(3)));
code.push_back(EncodeVop1(0x58, 15, Vgpr(4)));
AppendVop3(&code, 0x1d7, 16, Vgpr(3), 0, 0, 0, 0, true);
for (u32 i = 0; i < 7; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"VectorSpecialF16Ops",
code,
{},
{0x00008000u, 0xb8005678u, 0x00003800u, 0x00000000u, 0x0000fe00u,
0x00000000u, 0x00000000u},
{O::V_MOV_B32, O::V_RCP_F16, O::V_RSQ_F16, O::V_LOG_F16, O::V_EXP_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorFractF16CapturedAndEdges() {
using O = ShaderOpcode;
// RDNA2 section 12.8: x - floor(x), rounded to half precision.
// Load at runtime to exercise the emitted operation, including half subnormals.
const std::array<u32, 18> inputs{
0x3d00u, 0xbd00u, 0x0000u, 0x8000u, 0x0001u, 0x8001u,
0x03ffu, 0x83ffu, 0x7bffu, 0xfbffu, 0x3800u, 0xb800u,
0x8c00u, 0x8c01u, 0x7c00u, 0xfc00u, 0x7e55u, 0x7d01u};
const std::array<u32, 18> fractions{
0x3400u, 0x3a00u, 0x0000u, 0x0000u, 0x0001u, 0x3c00u,
0x03ffu, 0x3c00u, 0x0000u, 0x0000u, 0x3800u, 0x3800u,
0x3c00u, 0x3bffu, 0x7e00u, 0x7e00u, 0x7e00u, 0x7e00u};
TestCase test;
test.name = "VectorFractF16CapturedAndEdges";
for (u32 bits : inputs) {
test.initial.push_back(0x42000000u | bits); // Distinct high half must be ignored.
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < inputs.size(); i++) {
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 7, 30);
AppendVMovLiteral(&code, 9, 0xa5a51234u);
code.push_back(0x7e12bf07u); // Captured v_fract_f16 v9, v7.
if (i >= 14u) {
// Check quiet NaN class without constraining its sign or payload.
code.push_back(EncodeVop2(0x1b, 9, 255u, 9));
code.push_back(0xffff7e00u);
}
AppendStoreVgpr(&code, 9, static_cast<u32>(inputs.size()) + i);
test.expected.push_back(0xa5a50000u | fractions[i]);
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FRACT_F16,
O::V_AND_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_FRACT_F16 v9, v7", inputs.size()}};
test.ir_counts = {{" = FPFract32 ", inputs.size()}};
test.required_spirv = {" Fract ", " PackHalf2x16 ", " UnpackHalf2x16 "};
test.forbidden_spirv = {"OpCapability Float16"};
return test;
}
TestCase VectorFractF16Modifiers() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 7, 0xbd003d00u); // high=-1.25h, low=+1.25h
for (u32 dst = 10; dst <= 19; dst++) {
AppendVMovLiteral(&code, dst, 0xabcd1234u);
}
code.push_back(EncodeVop1(0x5f, 10, 249));
code.push_back(EncodeVop1Sdwa(7, 5, 2, 5)); // high -> high, preserve low
code.push_back(EncodeVop1(0x5f, 11, 249));
code.push_back(EncodeVop1Sdwa(7, 4, 0, 5, 0, 1, 1)); // -abs(-1.25), pad high
code.push_back(EncodeVop1(0x5f, 12, 249));
code.push_back(EncodeVop1Sdwa(7, 4, 2, 5, 0, 0, 1)); // abs(-1.25)
code.push_back(EncodeVop1(0x5f, 13, 249));
code.push_back(EncodeVop1Sdwa(7, 4, 2, 5, 0, 0, 0, 0, 1, 1)); // *2, clamp
code.push_back(EncodeVop1(0x5f, 14, 250));
code.push_back(EncodeVop1Dpp(7, 0x0e4) | (1u << 20u)); // DPP identity, negate
AppendVop3(&code, 0x1df, 15, Vgpr(7), 0, 0, 1, 0, true, 1, 1);
code.push_back(EncodeVop1(0x5f, 16, 255));
code.push_back(0x4200bd00u); // literal -1.25h
code.push_back(EncodeVop1(0x5f, 17, 240)); // inline +0.5, interpreted as FP16
AppendSMovLiteral(&code, 20, 0x4200bd00u);
code.push_back(EncodeVop1(0x5f, 18, 20)); // SGPR source
AppendVop3(&code, 0x1df, 19, Vgpr(7), 0, 0, 0, 0, false, 3); // *0.5
for (u32 i = 0; i < 10; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
TestCase test{"VectorFractF16Modifiers", code, {},
{0x3a001234u, 0x00003a00u, 0xabcd3400u, 0xabcd3c00u,
0xabcd3a00u, 0xabcd3c00u, 0xabcd3a00u, 0xabcd3800u,
0xabcd3a00u, 0xabcd3000u},
{O::V_MOV_B32, O::S_MOV_B32, O::V_FRACT_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"V_FRACT_F16", 10}};
test.ir_counts = {{" = FPFract32 ", 10}};
test.required_spirv = {" Fract ", " PackHalf2x16 "};
return test;
}
TestCase VectorCosF16CapturedSdwaAndEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 4, 0x34001234u); // high=+0.25h, low sentinel
code.push_back(0x7e08c2f9u);
code.push_back(0x00051504u); // captured v_cos_f16 v4.word1, v4.word1
AppendVMovLiteral(&code, 5, 0x00008000u); // -0.0h
code.push_back(EncodeVop1(0x61, 6, Vgpr(5)));
AppendVMovLiteral(&code, 7, 0x00007bffu); // max finite
code.push_back(EncodeVop1(0x61, 8, Vgpr(7)));
AppendVMovLiteral(&code, 9, 0x00007c00u); // +inf
code.push_back(EncodeVop1(0x61, 10, Vgpr(9)));
AppendVMovLiteral(&code, 11, 0x00000001u); // minimum positive denormal
code.push_back(EncodeVop1(0x61, 12, Vgpr(11)));
AppendVMovLiteral(&code, 13, 0x00003800u); // +0.5h
code.push_back(EncodeVop1(0x61, 14, Vgpr(13)));
for (u32 i = 0; i < 6; i++) {
AppendStoreVgpr(&code, 4 + i * 2, i);
}
AppendEnd(&code);
TestCase test{"VectorCosF16CapturedSdwaAndEdges",
code,
{},
{0x00001234u, 0x00003c00u, 0x00003c00u, 0x0000fe00u,
0x00003c00u, 0x0000bc00u},
{O::V_MOV_B32, O::V_COS_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
test.decoded_counts = {{"V_COS_F16", 6},
{"V_COS_F16 v4.sdwa(sel=5,sext=0), "
"v4.sdwa(sel=5,sext=0)",
1}};
test.ir_counts = {{" = FPCos ", 6}};
test.required_spirv = {" Fract ", " Cos ", " PackHalf2x16 "};
test.forbidden_spirv = {"OpCapability Float16"};
return test;
}
TestCase VectorSinF16SdwaAndEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 4, 0x34001234u); // high=+0.25h, low sentinel
code.push_back(0x7e08c0f9u);
code.push_back(0x00051504u); // v_sin_f16 v4.word1, v4.word1
AppendVMovLiteral(&code, 5, 0x00008000u); // -0.0h
code.push_back(EncodeVop1(0x60, 6, Vgpr(5)));
AppendVMovLiteral(&code, 7, 0x00007bffu); // max finite
code.push_back(EncodeVop1(0x60, 8, Vgpr(7)));
AppendVMovLiteral(&code, 9, 0x0000fbffu); // minimum finite
code.push_back(EncodeVop1(0x60, 10, Vgpr(9)));
AppendVMovLiteral(&code, 11, 0x00007c00u); // +inf
code.push_back(EncodeVop1(0x60, 12, Vgpr(11)));
AppendVMovLiteral(&code, 13, 0x0000fc00u); // -inf
code.push_back(EncodeVop1(0x60, 14, Vgpr(13)));
AppendVMovLiteral(&code, 15, 0x00003800u); // +0.5h
code.push_back(EncodeVop1(0x60, 16, Vgpr(15)));
AppendVMovLiteral(&code, 17, 0x00003a00u); // +0.75h
code.push_back(EncodeVop1(0x60, 18, Vgpr(17)));
AppendVMovLiteral(&code, 19, 0x00003400u); // +0.25h
AppendVop3(&code, 0x1e0, 20, Vgpr(19), 0);
for (u32 i = 0; i < 9; i++) {
AppendStoreVgpr(&code, 4 + i * 2, i);
}
AppendEnd(&code);
TestCase test{"VectorSinF16SdwaAndEdges",
code,
{},
{0x3c001234u, 0x00008000u, 0x00000000u, 0x00000000u,
0x0000fe00u, 0x0000fe00u, 0x00000000u, 0x0000bc00u,
0x00003c00u},
{O::V_MOV_B32, O::V_SIN_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
test.decoded_counts = {{"V_SIN_F16", 9},
{"V_SIN_F16 v4.sdwa(sel=5,sext=0), "
"v4.sdwa(sel=5,sext=0)",
1}};
test.ir_counts = {{" = FPSin ", 9}};
test.required_spirv = {" Fract ", " Sin ", " PackHalf2x16 "};
test.forbidden_spirv = {"OpCapability Float16"};
return test;
}
TestCase VectorWritelaneIgnoresExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 2, 0xaaaaaaaau);
AppendSMovLiteral(&code, 4, 0x12345678u);
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeSMovB32(127, InlineU32(0)));
AppendVop3(&code, 0x361, 2, 4, InlineU32(1));
code.push_back(EncodeSMovB32(126, InlineU32(0xf)));
code.push_back(EncodeSMovB32(127, InlineU32(0)));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorWritelaneIgnoresExecMask";
test.code = code;
test.expected = {0xaaaaaaaau, 0x12345678u, 0xaaaaaaaau, 0xaaaaaaaau};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_WRITELANE_B32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorReadlaneFromInactiveWrittenLane() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 4, 0x12345678u);
AppendVop3(&code, 0x361, 2, 4, InlineU32(4));
AppendSMovLiteral(&code, 4, 0xdeadbeefu);
AppendVop3(&code, 0x360, 5, Vgpr(2), InlineU32(4));
AppendStoreSgpr(&code, 5, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorReadlaneInactiveWrittenLane";
test.code = code;
test.expected = {0x12345678u};
test.opcodes = {O::S_MOV_B32, O::V_WRITELANE_B32, O::V_READLANE_B32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.forbidden_spirv = {"OpGroupNonUniformShuffle"};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorLaneWave32RuntimeSelectorWraps() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSop2(0x00, 5, 4, InlineU32(33)));
AppendSMovLiteral(&code, 6, 0x12345678u);
AppendVMovU32(&code, 2, 0);
AppendVop3(&code, 0x361, 2, 6, 5);
AppendVop3(&code, 0x360, 7, Vgpr(2), InlineU32(1));
AppendStoreSgpr(&code, 7, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorLaneWave32RuntimeSelectorWraps";
test.code = code;
test.expected = {0x12345678u};
test.opcodes = {O::S_ADD_U32, O::S_MOV_B32, O::V_WRITELANE_B32,
O::V_READLANE_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformShuffle"};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 4;
test.has_compute_info = true;
return test;
}
TestCase VectorReadlaneSelectsTwoKeysWithinWave() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x16, 1, InlineU32(4), 0)); // lanes 0..15: 0, 16..31: 1
AppendVop3(&code, 0x360, 20, Vgpr(1), InlineU32(0));
AppendVop3(&code, 0x360, 21, Vgpr(1), InlineU32(16));
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 3, 11);
AppendVMovU32(&code, 4, 22);
code.push_back(EncodeVopc(0xc2, 20, 1)); // selected lane 0 key == local key
code.push_back(EncodeVop2(0x01, 2, Vgpr(2), 3));
code.push_back(EncodeVopc(0xc2, 21, 1)); // selected lane 16 key == local key
code.push_back(EncodeVop2(0x01, 2, Vgpr(2), 4));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorReadlaneSelectsTwoKeysWithinWave";
test.code = std::move(code);
test.expected.resize(32);
std::fill_n(test.expected.begin(), 16, 11u);
std::fill_n(test.expected.begin() + 16, 16, 22u);
test.opcodes = {O::V_LSHRREV_B32, O::V_READLANE_B32, O::V_MOV_B32,
O::V_CMP_EQ_U32, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_READLANE_B32", 2}};
test.required_spirv = {"OpGroupNonUniformShuffle"};
test.compute_info.threads_num[0] = 32;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorPermlanex16() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 3, 0x76543210u);
AppendVMovLiteral(&code, 4, 0xfedcba98u);
code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0));
AppendVMovLiteral(&code, 5, 0xfeedbabeu);
AppendVop3(&code, 0x378, 1, Vgpr(5), Vgpr(3), Vgpr(4));
AppendBufferStoreDword(&code, 1, 2);
AppendEnd(&code);
TestCase test;
test.name = "VectorPermlanex16";
test.code = code;
test.expected = std::vector<u32>(32, 0xfeedbabeu);
test.opcodes = {O::V_MOV_B32, O::V_PERMLANEX16_B32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 32;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorPermlane16FetchInactiveZero() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x25, 1, InlineU32(10), 1),
EncodeVopc(0xc2, InlineU32(0), 0),
EncodeSop1(0x04, 126, 106),
EncodeSMovB32(0, InlineU32(1)),
EncodeSMovB32(1, InlineU32(0)),
};
AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1);
AppendStoreVgpr(&code, 2, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorPermlane16FetchInactiveZero";
test.code = code;
test.expected = {0};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::S_MOV_B32, O::V_PERMLANE16_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorPermlane16FetchInactiveFi() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, Vgpr(0)), EncodeVop2(0x25, 1, InlineU32(10), 1),
EncodeVopc(0xc2, InlineU32(0), 0), EncodeSop1(0x04, 126, 106),
EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)),
};
AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1, 0, 1);
AppendStoreVgpr(&code, 2, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorPermlane16FetchInactiveFi";
test.code = code;
test.expected = {11};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::S_MOV_B32, O::V_PERMLANE16_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorDpp8Captured(bool masked_exec) {
using O = ShaderOpcode;
constexpr u32 sentinel = 0xaaaaaaaau;
const u32 masks[] = {masked_exec ? 0x0f0f0f0fu : 0xffffffffu,
masked_exec ? 0xf0f0f0f0u : 0xffffffffu};
std::vector<u32> code;
AppendVMovU32(&code, 5, 100);
code.push_back(EncodeVop2(0x25, 5, Vgpr(0), 5));
code.push_back(EncodeVop1(0x01, 7, Vgpr(5)));
AppendVMovLiteral(&code, 2, sentinel);
AppendVMovLiteral(&code, 3, sentinel);
AppendVMovLiteral(&code, 8, sentinel);
AppendSMovLiteral(&code, 126, masks[0]);
AppendSMovLiteral(&code, 127, masks[1]);
// Captured DPP8 selectors broadcast each group's upper/lower four lanes.
code.push_back(EncodeVop1(0x01, 2, 233));
code.push_back(0xfacfac05u);
code.push_back(EncodeVop1(0x01, 3, 233));
code.push_back(0x68868805u);
// CS 6babb2a8915d35c4, pc 0xcc: add lane 3 of each group to its own v7.
code.push_back(0x4a100ee9u);
code.push_back(0x6db6db07u);
AppendSMovLiteral(&code, 126, 0xffffffffu);
AppendSMovLiteral(&code, 127, 0xffffffffu);
code.push_back(EncodeVop2(0x1a, 6, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 6);
AppendVMovU32(&code, 7, 64u * sizeof(u32));
code.push_back(EncodeVop2(0x25, 6, Vgpr(7), 6));
AppendBufferStoreDword(&code, 3, 6);
code.push_back(EncodeVop2(0x25, 6, Vgpr(7), 6));
AppendBufferStoreDword(&code, 8, 6);
AppendEnd(&code);
TestCase test;
test.name = masked_exec ? "VectorDpp8CapturedMaskedExec"
: "VectorDpp8Captured";
test.code = std::move(code);
test.expected.resize(192, sentinel);
for (u32 lane = 0; lane < 64; ++lane) {
const auto mask = masks[lane / 32u];
if ((mask & (1u << (lane % 32u))) == 0) {
continue;
}
const u32 sources[] = {(lane & ~7u) | 4u | (lane & 3u),
(lane & ~7u) | (lane & 3u)};
for (u32 permutation = 0; permutation < 2; ++permutation) {
const auto source = sources[permutation];
test.expected[permutation * 64u + lane] =
(mask & (1u << (source % 32u))) != 0 ? 100u + source : 0u;
}
const u32 source = (lane & ~7u) | 3u;
test.expected[128u + lane] = 100u + lane;
if ((mask & (1u << (source % 32u))) != 0) {
test.expected[128u + lane] += 100u + source;
}
}
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{".dpp8(", 3}};
test.compute_info.wave_size = 64;
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorDppQuadPermuteReverse() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeVop2(0x25, 2, 250, 1));
code.push_back(EncodeVop2Dpp(0, 0x01b));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorDppQuadPermuteReverse";
test.code = code;
test.expected = {103, 102, 101, 100, 107, 106, 105, 104};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorDppRowXmask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeVop2(0x25, 2, 250, 1));
code.push_back(EncodeVop2Dpp(0, 0x164));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorDppRowXmask";
test.code = code;
test.expected = {104, 105, 106, 107, 100, 101, 102, 103};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorDppBankMaskPreservesDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 2, 0xaaaaaaaau);
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeVop2(0x25, 2, 250, 1));
code.push_back(EncodeVop2Dpp(0, 0x0e4, 0xf, 0xe));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorDppBankMaskPreservesDestination";
test.code = code;
test.expected = {0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau,
104, 105, 106, 107};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorDppBoundsControlZeroPreservesDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 2, 0xaaaaaaaau);
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeVop2(0x25, 2, 250, 1));
code.push_back(EncodeVop2Dpp(0, 0x111));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorDppBoundsControlZeroPreservesDestination";
test.code = code;
test.expected = {0xaaaaaaaau, 100, 101, 102, 103, 104, 105, 106};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorMadF32RoundsProduct() {
using O = ShaderOpcode;
std::vector<u32> code;
for (u32 source = 0; source < 3; source++) {
AppendVMovU32(&code, 30, source * sizeof(u32));
AppendBufferLoadDword(&code, source, 30);
}
AppendVMovLiteral(&code, 10, 0xbf800000u);
AppendVMovLiteral(&code, 11, 0xbf800000u);
AppendVMovLiteral(&code, 15, 0xbf800000u);
AppendVMovLiteral(&code, 16, 0xbf800000u);
code.push_back(EncodeVop2(0x1f, 10, Vgpr(0), 1));
AppendVop3(&code, 0x11f, 11, Vgpr(0), Vgpr(1), 0);
AppendVop3(&code, 0x141, 12, Vgpr(0), Vgpr(1), Vgpr(2));
code.push_back(EncodeVop2(0x20, 13, Vgpr(0), 2));
code.push_back(0x3f7ffffeu);
code.push_back(EncodeVop2(0x21, 14, Vgpr(0), 1));
code.push_back(0xbf800000u);
code.push_back(EncodeVop2(0x2b, 15, Vgpr(0), 1));
AppendVop3(&code, 0x12b, 16, Vgpr(0), Vgpr(1), 0);
AppendVop3(&code, 0x14b, 17, Vgpr(0), Vgpr(1), Vgpr(2));
code.push_back(EncodeVop2(0x2c, 18, Vgpr(0), 2));
code.push_back(0x3f7ffffeu);
code.push_back(EncodeVop2(0x2d, 19, Vgpr(0), 1));
code.push_back(0xbf800000u);
for (u32 i = 0; i < 10; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "VectorMadF32RoundsProduct";
test.code = std::move(code);
test.initial = {0x3f800001u, 0x3f7ffffeu, 0xbf800000u};
// (1+2^-23)*(1-2^-23)-1 rounds to zero for MAD, but -2^-46 for FMA.
test.expected = {0u, 0u, 0u, 0u, 0u, 0xa8800000u, 0xa8800000u,
0xa8800000u, 0xa8800000u, 0xa8800000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAC_F32,
O::V_MAD_F32, O::V_MADMK_F32, O::V_MADAK_F32,
O::V_FMAC_F32, O::V_FMA_F32, O::V_FMAMK_F32, O::V_FMAAK_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpFMul", "OpFAdd", "NoContraction", "Fma"};
return test;
}
TestCase VectorMadF32FlushesDenorms() {
using O = ShaderOpcode;
std::vector<u32> code;
for (u32 sample = 0; sample < 5; sample++) {
for (u32 source = 0; source < 3; source++) {
AppendVMovU32(&code, 30, (sample * 3 + source) * sizeof(u32));
AppendBufferLoadDword(&code, source, 30);
}
AppendVop3(&code, 0x141, 10 + sample, Vgpr(0), Vgpr(1), Vgpr(2));
}
for (u32 sample = 0; sample < 5; sample++) {
AppendStoreVgpr(&code, 10 + sample, sample);
}
AppendEnd(&code);
TestCase test;
test.name = "VectorMadF32FlushesDenorms";
test.code = std::move(code);
// Input denorms in each operand, an intermediate denorm, and a negative result denorm.
test.initial = {0x00400000u, 0x40000000u, 0u,
0x40000000u, 0x00400000u, 0u,
0u, 0x3f800000u, 0x00400000u,
0x00800000u, 0x3f000000u, 0x00800000u,
0x00800000u, 0x3f800000u, 0x80800001u};
test.expected = {0u, 0u, 0u, 0x00800000u, 0x80000000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
// Legacy MAD always flushes, even when the shader requests denorm preservation.
test.compute_info.float_mode = 0xf0;
test.has_compute_info = true;
return test;
}
TestCase Vop3FmacF32NegatedSourceAccumulates() {
using O = ShaderOpcode;
std::vector<u32> code;
constexpr u32 registers[] = {13, 20, 29};
for (u32 sample = 0; sample < 2; sample++) {
for (u32 source = 0; source < 3; source++) {
AppendVMovU32(&code, 30, (sample * 3 + source) * sizeof(u32));
AppendBufferLoadDword(&code, registers[source], 30);
}
code.push_back(0xd52b001du);
code.push_back(0x4002290du); // v_fmac_f32_e64 v29, v13, -v20
AppendStoreVgpr(&code, 29, sample);
}
AppendEnd(&code);
TestCase test;
test.name = "Vop3FmacF32NegatedSourceAccumulates";
test.code = std::move(code);
test.initial = {0x40000000u, 0x40400000u, 0x41200000u,
0x3f800001u, 0x3f7ffffeu, 0x3f800000u};
// 10 - 2*3 = 4; 1 - (1+2^-23)*(1-2^-23) = 2^-46, not rounded zero.
test.expected = {0x40800000u, 0x28800000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FMAC_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpFNegate", "Fma"};
return test;
}
TestCase Vop3LdexpSourceModifier() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 9, 0xbfc00000u);
code.push_back(0xd7620107u);
code.push_back(0x00018509u);
AppendStoreVgpr(&code, 7, 0);
AppendEnd(&code);
return {"Vop3LdexpSourceModifier",
code,
{},
{0x3ec00000u},
{O::V_MOV_B32, O::V_LDEXP_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop1MoveRelSource() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 12, 0x11111111u);
AppendVMovLiteral(&code, 13, 0x22222222u);
AppendVMovLiteral(&code, 14, 0x33333333u);
code.push_back(EncodeSMovB32(124, InlineU32(2)));
code.push_back(0x7e6e870cu);
AppendStoreVgpr(&code, 55, 0);
AppendEnd(&code);
return {"Vop1MoveRelSource",
code,
{},
{0x33333333u},
{O::V_MOV_B32, O::S_MOV_B32, O::V_MOVRELS_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase Vop1MoveRelDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0x12345678u);
AppendVMovLiteral(&code, 5, 0xaaaaaaaau);
AppendVMovLiteral(&code, 6, 0xbbbbbbbbu);
AppendVMovLiteral(&code, 7, 0xccccccccu);
code.push_back(EncodeSMovB32(124, InlineU32(2)));
code.push_back(EncodeSop1(0x04, 8, 126));
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(EncodeVop1(0x42, 5, Vgpr(1)));
code.push_back(EncodeSop1(0x04, 126, 8));
code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
AppendBufferStoreDword(&code, 7, 4);
AppendEnd(&code);
TestCase test;
test.name = "Vop1MoveRelDestination";
test.code = code;
test.expected = {0x12345678u, 0xccccccccu, 0xccccccccu, 0xccccccccu};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::V_MOVRELD_B32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase CubeIdCapturedNegationAndOutputScale() {
using O = ShaderOpcode;
constexpr std::array directions = {
std::array{4.0f, 2.0f, 1.0f}, std::array{-4.0f, 2.0f, 1.0f},
std::array{1.0f, -4.0f, 2.0f}, std::array{1.0f, 4.0f, 2.0f},
std::array{1.0f, 2.0f, -4.0f}, std::array{1.0f, 2.0f, 4.0f}};
constexpr std::array registers = {47u, 49u, 48u};
TestCase test;
test.name = "CubeIdCapturedNegationAndOutputScale";
for (u32 i = 0; i < directions.size(); i++) {
for (u32 component = 0; component < registers.size(); component++) {
test.initial.push_back(std::bit_cast<u32>(directions[i][component]));
AppendVMovU32(&test.code, 30, (i * 3u + component) * 4u);
AppendBufferLoadDword(&test.code, registers[component], 30);
}
// The captured instruction negates Y/Z and divides the floating face ID by two.
test.code.insert(test.code.end(), {0xd544000bu, 0xdcc2632fu});
AppendStoreVgpr(&test.code, 11, i);
}
AppendEnd(&test.code);
test.expected = {0x00000000u, 0x3f000000u, 0x3f800000u,
0x3fc00000u, 0x40000000u, 0x40200000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_CUBEID_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {
{"V_CUBEID_F32 v11.omod(3), v47, v49.neg, v48.neg", directions.size()}};
test.required_spirv = {"OpFNegate", "OpFMul"};
return test;
}
TestCase VectorFloatSpecialOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x40000000u);
AppendVMovLiteral(&code, 2, 0x40400000u);
AppendVMovLiteral(&code, 3, 0x40800000u);
AppendVMovLiteral(&code, 4, 0x40003c00u);
AppendVMovLiteral(&code, 5, 0x44004200u);
AppendVMovLiteral(&code, 6, 0x3f800000u);
AppendVMovLiteral(&code, 7, 0x3f800000u);
AppendVMovLiteral(&code, 8, 0xbf800000u);
AppendVMovLiteral(&code, 10, 0x3f000000u);
AppendVMovLiteral(&code, 12, 0xaaaabbbbu);
AppendVMovLiteral(&code, 13, 0xaaaabbbbu);
code.push_back(EncodeVop2(0x02, 6, Vgpr(4), 5));
AppendVop3(&code, 0x368, 9, Vgpr(7), Vgpr(8));
AppendVop3(&code, 0x369, 11, Vgpr(7), Vgpr(10));
AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(2), Vgpr(0));
AppendVop3p(&code, 0x22, 13, Vgpr(1), Vgpr(2), Vgpr(0));
AppendVop3(&code, 0x144, 14, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x145, 15, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x146, 16, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x147, 17, Vgpr(0), Vgpr(1), Vgpr(2));
const u32 results[] = {6, 9, 11, 12, 13, 14, 15, 16, 17};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"VectorFloatSpecialOps",
code,
{},
{0x41400000u, 0x80017fffu, 0x8000ffffu, 0xaaaa4700u, 0x4700bbbbu,
0x40800000u, 0x3f800000u, 0xc0000000u, 0x40c00000u},
{O::V_MOV_B32, O::V_DOT2C_F32_F16, O::V_CVT_PKNORM_I16_F32,
O::V_CVT_PKNORM_U16_F32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16,
O::V_CUBEID_F32, O::V_CUBESC_F32, O::V_CUBETC_F32, O::V_CUBEMA_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase MadMixF16LiteralHalfSourceUsesOpsel() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 12, 0xaaaabbbbu);
AppendVMovLiteral(&code, 13, 0x11112222u);
AppendVop3p(&code, 0x21, 12, 255u, Vgpr(0), Vgpr(0), 0x1, 0x1);
code.push_back(0x40003c00u);
AppendVop3p(&code, 0x22, 13, 255u, Vgpr(0), Vgpr(0), 0x1, 0x0);
code.push_back(0x40003c00u);
AppendStoreVgpr(&code, 12, 0);
AppendStoreVgpr(&code, 13, 1);
AppendEnd(&code);
return {"MadMixF16LiteralHalfSourceUsesOpsel",
code,
{},
{0xaaaa4200u, 0x40002222u},
{O::V_MOV_B32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase MadMixF16NegHiIsAbsAndNegIsIndependent() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0xc0000000u);
AppendVMovLiteral(&code, 2, 0x40000000u);
AppendVMovLiteral(&code, 12, 0xaaaabbbbu);
AppendVMovLiteral(&code, 13, 0x11112222u);
AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(0), Vgpr(0), 0, 0, 0x1);
AppendVop3p(&code, 0x22, 13, Vgpr(2), Vgpr(0), Vgpr(0), 0, 0, 0, 0x1);
AppendStoreVgpr(&code, 12, 0);
AppendStoreVgpr(&code, 13, 1);
AppendEnd(&code);
return {"MadMixF16NegHiIsAbsAndNegIsIndependent",
code,
{},
{0xaaaa4200u, 0xbc002222u},
{O::V_MOV_B32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVop3FmaF16UsesRdna2Opcode34b() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0x40003c00u);
AppendVMovLiteral(&code, 2, 0x44004200u);
AppendVMovLiteral(&code, 3, 0x3c003c00u);
AppendVMovLiteral(&code, 10, 0xaaaa5555u);
AppendVMovLiteral(&code, 11, 0xbbbb5555u);
AppendVop3(&code, 0x34bu, 10, Vgpr(1), Vgpr(2), Vgpr(3));
AppendVop3(&code, 0x34bu, 11, Vgpr(1), Vgpr(2), Vgpr(3), 0, 0xfu);
AppendStoreVgpr(&code, 10, 0);
AppendStoreVgpr(&code, 11, 1);
AppendEnd(&code);
return {"VectorVop3FmaF16UsesRdna2Opcode34b",
code,
{},
{0xaaaa4400u, 0x48805555u},
{O::V_MOV_B32, O::V_FMA_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase FloatInlineConstantF16UsesNumericValue() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0x38003800u); // 0.5h, 0.5h
AppendVMovLiteral(&code, 2, 0x3c003c00u); // 1.0h, 1.0h
AppendVMovLiteral(&code, 10, 0xaaaa0000u);
AppendVMovLiteral(&code, 12, 0xbbbb0000u);
AppendVMovLiteral(&code, 14, 0xcccc0000u);
AppendVMovLiteral(&code, 15, 0xdddd0000u);
AppendVop3(&code, 0x34bu, 10, 246u, Vgpr(1), Vgpr(2));
AppendVop3p(&code, 0x0eu, 11, 246u, Vgpr(1), Vgpr(2), 0x1u);
AppendVop3(&code, 0x34bu, 12, 248u, 242u, 128u);
AppendVop3(&code, 0x311u, 13, 242u, 248u);
AppendVop3(&code, 0x34bu, 14, 246u, Vgpr(1), Vgpr(2), 0, 1);
AppendVop3(&code, 0x303u, 15, 240u, InlineU32(1));
AppendStoreVgpr(&code, 10, 0);
AppendStoreVgpr(&code, 11, 1);
AppendStoreVgpr(&code, 12, 2);
AppendStoreVgpr(&code, 13, 3);
AppendStoreVgpr(&code, 14, 4);
AppendStoreVgpr(&code, 15, 5);
AppendEnd(&code);
TestCase test;
test.name = "FloatInlineConstantF16UsesNumericValue";
test.code = std::move(code);
test.expected = {0xaaaa4200u, 0x3c004200u, 0xbbbb3118u,
0x31183c00u, 0xcccc3c00u, 0xdddd3801u};
test.opcodes = {O::V_MOV_B32, O::V_FMA_F16, O::V_PK_FMA_F16,
O::V_PACK_B32_F16, O::V_ADD_NC_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"4.000000", 3}, {"0.159155", 2}};
return test;
}
TestCase Vop3Int16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0xabcd0001u);
AppendVMovLiteral(&code, 2, 0x12340002u);
AppendVMovLiteral(&code, 3, 0x55557fffu);
AppendVMovLiteral(&code, 4, 0x66668000u);
AppendVMovLiteral(&code, 5, 0x77770013u);
AppendVMovLiteral(&code, 6, 0x88888010u);
AppendVMovLiteral(&code, 10, 0xaaaa0000u);
AppendVMovLiteral(&code, 11, 0xbbbb0000u);
AppendVMovLiteral(&code, 12, 0xcccc0000u);
AppendVMovLiteral(&code, 13, 0xdddd0000u);
AppendVMovLiteral(&code, 14, 0xeeee0000u);
AppendVMovLiteral(&code, 15, 0x11110000u);
AppendVop3(&code, 0x304u, 10, Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x30du, 11, Vgpr(3), Vgpr(1));
AppendVop3(&code, 0x308u, 12, Vgpr(5), Vgpr(6));
AppendVop3(&code, 0x309u, 13, Vgpr(3), Vgpr(4));
AppendVop3(&code, 0x30bu, 14, Vgpr(3), Vgpr(4));
AppendVop3(&code, 0x308u, 15, Vgpr(5), Vgpr(3));
for (u32 i = 0; i < 6; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "Vop3Int16Ops";
test.code = std::move(code);
test.expected = {0xaaaaffffu, 0xbbbb8000u, 0xccccf002u,
0xdddd8000u, 0xeeee7fffu, 0x11110fffu};
test.opcodes = {O::V_MOV_B32, O::V_SUB_NC_U16, O::V_ADD_NC_I16,
O::V_ASHRREV_I16, O::V_MAX_U16, O::V_MIN_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase VectorFloatControlContractPreservesInfNan() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSop2(0x24, 0, InlineU32(8), InlineU32(23)));
code.push_back(EncodeVop1(0x01, 0, InlineU32(0)));
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 3);
AppendVop3(&code, 0x108, 10, 0, Vgpr(0));
code.push_back(EncodeVopc(0x88, Vgpr(10), 2));
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorFloatControlContractPreservesInfNan";
test.code = std::move(code);
test.expected = {1u};
test.opcodes = {O::S_BFM_B32, O::V_MOV_B32, O::V_MUL_F32,
O::V_CMP_CLASS_F32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"SPV_KHR_float_controls",
"SignedZeroInfNanPreserve 32"};
return test;
}
TestCase VectorFloatArithmeticOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x40000000u);
AppendVMovLiteral(&code, 2, 0x40800000u);
AppendVMovLiteral(&code, 3, 0x40a00000u);
AppendVMovLiteral(&code, 4, 0x40400000u);
AppendVMovLiteral(&code, 11, 0x40800000u);
AppendVMovLiteral(&code, 17, 0x40800000u);
code.push_back(EncodeVop2(0x03, 5, Vgpr(0), 1));
code.push_back(EncodeVop2(0x04, 6, Vgpr(3), 1));
code.push_back(EncodeVop2(0x05, 7, Vgpr(1), 3));
code.push_back(EncodeVop2(0x08, 8, Vgpr(1), 2));
code.push_back(EncodeVop1(0x01, 21, Vgpr(8)));
code.push_back(EncodeVop2(0x0f, 9, Vgpr(3), 1));
code.push_back(EncodeVop2(0x10, 10, Vgpr(3), 1));
code.push_back(EncodeVop2(0x1f, 11, Vgpr(1), 4));
AppendVop3(&code, 0x141, 12, Vgpr(1), Vgpr(4), Vgpr(2));
AppendVop3(&code, 0x14b, 13, Vgpr(1), Vgpr(4), Vgpr(2));
AppendVop3(&code, 0x151, 14, Vgpr(3), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x154, 15, Vgpr(3), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x157, 16, Vgpr(3), Vgpr(1), Vgpr(2));
code.push_back(EncodeVop2(0x20, 18, Vgpr(1), 17));
code.push_back(0x3f800000u);
code.push_back(EncodeVop2(0x21, 19, Vgpr(1), 4));
code.push_back(0x40000000u);
code.push_back(EncodeVop2(0x03, 20, 249, 1));
code.push_back(EncodeVop2Sdwa(0));
AppendSMovLiteral(&code, 70, 0x40000000u);
code.push_back(0x06088cf9u);
code.push_back(0x868606f2u);
AppendSMovLiteral(&code, 26, 0x40000000u);
code.push_back(0x081034f9u);
code.push_back(0x868606f2u);
code.push_back(0xd51f8011u);
code.push_back(0x00020d0bu);
AppendVMovLiteral(&code, 22, 0xbf800000u);
code.push_back(EncodeVop2(0x05, 23, 249, 26));
code.push_back(EncodeVop2Sdwa(22, 6, 0, 6, 6, 0, 0, 0, 1, 0, 0, 0, 1));
AppendVMovLiteral(&code, 24, 0x3f800000u);
AppendVMovLiteral(&code, 25, 0x3f800000u);
code.push_back(EncodeVop2(0x03, 26, 249, 25));
code.push_back(EncodeVop2Sdwa(24, 6, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1));
AppendVMovLiteral(&code, 27, 0xbf800000u);
AppendVMovLiteral(&code, 28, 0x40000000u);
AppendVop3(&code, 0x104, 29, Vgpr(27), Vgpr(28), 0, 1);
AppendVop3(&code, 0x103, 30, Vgpr(0), Vgpr(1), 0, 0, 0, false, 1);
AppendVop3(&code, 0x151, 32, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0, false, 1);
AppendVMovLiteral(&code, 33, 0xbf800000u);
AppendVMovLiteral(&code, 34, 0x40000000u);
code.push_back(EncodeVop2(0x03, 35, 250, 34));
code.push_back(EncodeVop2Dpp(33, 0, 0xf, 0xf, 0, 1));
const u32 results[] = {5, 6, 7, 21, 9, 10, 11, 12, 13, 14, 15, 16,
18, 19, 20, 4, 8, 17, 23, 26, 29, 30, 32, 35};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"VectorFloatArithmeticOps",
code,
{},
{0x40400000u, 0x40400000u, 0x40400000u, 0x41000000u, 0x40000000u,
0x40a00000u, 0x41200000u, 0x41200000u, 0x41200000u, 0x40000000u,
0x40a00000u, 0x40800000u, 0x40c00000u, 0x41000000u, 0x40400000u,
0x40400000u, 0xbf800000u, 0x3f800000u, 0x3f800000u, 0x40800000u,
0xbf800000u, 0x40c00000u, 0x40000000u, 0x40400000u},
{O::S_MOV_B32, O::V_MOV_B32, O::V_ADD_F32, O::V_SUB_F32,
O::V_SUBREV_F32, O::V_MUL_F32, O::V_MIN_F32, O::V_MAX_F32,
O::V_MAC_F32, O::V_MAD_F32, O::V_FMA_F32, O::V_MIN3_F32,
O::V_MAX3_F32, O::V_MED3_F32, O::V_MADMK_F32, O::V_MADAK_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorMinMaxF32NanAndSignedZeroEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x40000000u);
AppendVMovLiteral(&code, 1, 0x7fc00000u);
AppendVMovLiteral(&code, 2, 0x00000000u);
AppendVMovLiteral(&code, 3, 0x80000000u);
AppendVMovLiteral(&code, 4, 0x40400000u);
AppendVMovLiteral(&code, 5, 0x3f800000u);
AppendVMovLiteral(&code, 6, 0x7fa00001u);
code.push_back(EncodeVop2(0x0f, 10, Vgpr(0), 1));
code.push_back(EncodeVop2(0x10, 11, Vgpr(0), 1));
code.push_back(EncodeVop2(0x0f, 12, Vgpr(3), 2));
code.push_back(EncodeVop2(0x10, 13, Vgpr(2), 3));
code.push_back(EncodeVop2(0x0f, 14, Vgpr(6), 0));
code.push_back(EncodeVop2(0x10, 15, Vgpr(0), 6));
AppendVop3(&code, 0x151, 16, Vgpr(0), Vgpr(1), Vgpr(4));
AppendVop3(&code, 0x154, 17, Vgpr(0), Vgpr(1), Vgpr(5));
for (u32 i = 0; i < 8; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"VectorMinMaxF32NanAndSignedZeroEdges",
code,
{},
{0x40000000u, 0x40000000u, 0x80000000u, 0x00000000u, 0x40000000u,
0x40000000u, 0x40000000u, 0x40000000u},
{O::V_MOV_B32, O::V_MIN_F32, O::V_MAX_F32, O::V_MIN3_F32,
O::V_MAX3_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorMed3F32NanUsesMin3Path() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x40000000u);
AppendVMovLiteral(&code, 1, 0x40400000u);
AppendVMovLiteral(&code, 2, 0x7fa00001u);
AppendVMovLiteral(&code, 3, 0x7fc00000u);
AppendVop3(&code, 0x157, 10, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x157, 11, Vgpr(0), Vgpr(1), Vgpr(3));
AppendVop3(&code, 0x157, 12, Vgpr(2), Vgpr(0), Vgpr(1));
AppendStoreVgpr(&code, 10, 0);
AppendStoreVgpr(&code, 11, 1);
AppendStoreVgpr(&code, 12, 2);
AppendEnd(&code);
return {"VectorMed3F32NanUsesMin3Path",
code,
{},
{0x40000000u, 0x40000000u, 0x40000000u},
{O::V_MOV_B32, O::V_MED3_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorFloatConversionOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xfffffffdu);
code.push_back(EncodeVop1(0x01, 1, InlineU32(7)));
AppendVMovLiteral(&code, 2, 0x40e00000u);
AppendVMovLiteral(&code, 3, 0xc0400000u);
AppendVMovLiteral(&code, 4, 0x3f800000u);
AppendVMovLiteral(&code, 5, 0x00003c00u);
AppendVMovLiteral(&code, 6, 0x40700000u);
AppendVMovLiteral(&code, 7, 0x44332211u);
AppendVMovLiteral(&code, 8, 0x40000000u);
AppendVMovLiteral(&code, 9, 0x40800000u);
AppendVMovLiteral(&code, 10, 0x40100000u);
AppendVMovLiteral(&code, 11, 0x40300000u);
code.push_back(EncodeVop1(0x01, 12, InlineU32(0)));
code.push_back(EncodeVop1(0x01, 37, InlineU32(15)));
code.push_back(EncodeVop1(0x05, 13, Vgpr(0)));
code.push_back(EncodeVop1(0x06, 14, Vgpr(1)));
code.push_back(EncodeVop1(0x07, 15, Vgpr(2)));
code.push_back(EncodeVop1(0x08, 16, Vgpr(3)));
code.push_back(EncodeVop1(0x0a, 17, Vgpr(4)));
code.push_back(EncodeVop1(0x0b, 18, Vgpr(5)));
code.push_back(EncodeVop1(0x0c, 42, Vgpr(11)));
code.push_back(EncodeVop1(0x0d, 19, Vgpr(6)));
code.push_back(EncodeVop1(0x11, 20, Vgpr(7)));
code.push_back(EncodeVop1(0x12, 21, Vgpr(7)));
code.push_back(EncodeVop1(0x13, 22, Vgpr(7)));
code.push_back(EncodeVop1(0x14, 23, Vgpr(7)));
code.push_back(EncodeVop1(0x2a, 24, Vgpr(8)));
code.push_back(EncodeVop1(0x20, 25, Vgpr(10)));
code.push_back(EncodeVop1(0x21, 26, Vgpr(11)));
code.push_back(EncodeVop1(0x22, 27, Vgpr(10)));
code.push_back(EncodeVop1(0x23, 28, Vgpr(10)));
code.push_back(EncodeVop1(0x24, 29, Vgpr(11)));
code.push_back(EncodeVop1(0x25, 30, Vgpr(4)));
code.push_back(EncodeVop1(0x27, 36, Vgpr(9)));
code.push_back(EncodeVop1(0x2e, 32, Vgpr(9)));
code.push_back(EncodeVop1(0x33, 33, Vgpr(9)));
code.push_back(EncodeVop1(0x35, 34, Vgpr(12)));
code.push_back(EncodeVop1(0x36, 35, Vgpr(12)));
code.push_back(EncodeVop1(0x0e, 38, Vgpr(37)));
code.push_back(EncodeVop1(0x2a, 39, 249));
code.push_back(EncodeVop1Sdwa(8, 6, 0, 6, 0, 0, 0, 0, 0, 1));
AppendVMovLiteral(&code, 40, 0xc0000000u);
AppendVop3(&code, 0x1aa, 41, Vgpr(40), 0, 0, 1, 0, false, 1);
AppendVMovLiteral(&code, 2, 0xc0003c00u); // low=1.0h, high=-2.0h
code.push_back(0x7e1016f9u);
code.push_back(0x00250602u);
const u32 results[] = {13, 14, 15, 16, 17, 18, 42, 19, 20, 21, 22, 23, 24, 25,
26, 27, 28, 29, 30, 36, 32, 33, 34, 35, 38, 39, 41, 8};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"VectorFloatConversionOps",
code,
{},
{0xc0400000u, 0x40e00000u, 7, 0xfffffffdu, 0x00003c00u,
0x3f800000u, 3, 3, 0x41880000u, 0x42080000u,
0x424c0000u, 0x42880000u, 0x3f000000u, 0x3e800000u, 0x40000000u,
0x40400000u, 0x40000000u, 0x40000000u, 0x40000000u, 0x40000000u,
0x3f000000u, 0x40000000u, 0, 0x3f800000u, 0xbd800000u,
0x3f800000u, 0x3f800000u, 0x40000000u},
{O::V_MOV_B32, O::V_CVT_F32_I32, O::V_CVT_F32_U32,
O::V_CVT_U32_F32, O::V_CVT_I32_F32, O::V_CVT_F16_F32,
O::V_CVT_F32_F16, O::V_CVT_RPI_I32_F32, O::V_CVT_FLR_I32_F32,
O::V_CVT_OFF_F32_I4, O::V_CVT_F32_UBYTE0, O::V_CVT_F32_UBYTE1,
O::V_CVT_F32_UBYTE2, O::V_CVT_F32_UBYTE3, O::V_RCP_F32,
O::V_FRACT_F32, O::V_TRUNC_F32, O::V_CEIL_F32,
O::V_RNDNE_F32, O::V_FLOOR_F32, O::V_EXP_F32,
O::V_LOG_F32, O::V_RSQ_F32, O::V_SQRT_F32,
O::V_SIN_F32, O::V_COS_F32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorFrexpF32Edges() {
using O = ShaderOpcode;
const std::vector<u32> inputs = {
0x00000000u, 0x80000000u, 0x3f000000u, 0xbf800000u, 0x40800000u,
0xc1000000u, 0x00800000u, 0x007fffffu, 0x00000001u, 0x7f800000u,
0x7fc00000u, 0xff800000u, 0x7fa00001u,
};
std::vector<u32> code;
for (u32 i = 0; i < static_cast<u32>(inputs.size()); i++) {
AppendVMovU32(&code, 31, i * sizeof(u32));
AppendBufferLoadDword(&code, i == 3u ? 23u : i, 31);
}
for (u32 i = 0; i < static_cast<u32>(inputs.size()); i++) {
const auto exponent_dst = i == 3u ? 19u : 32u + i;
const auto mantissa_dst = i == 3u ? 21u : 48u + i;
if (i == 3u) {
code.push_back(0x7e267ef9u); // captured v_frexp_exp_i32_f32 v19, abs(v23)
code.push_back(0x00260617u);
} else {
code.push_back(EncodeVop1(0x3f, exponent_dst, Vgpr(i)));
}
AppendStoreVgpr(&code, exponent_dst, i);
if (i == 3u) {
code.push_back(0x7e2a80f9u); // captured v_frexp_mant_f32 v21, abs(v23)
code.push_back(0x00260617u);
} else {
code.push_back(EncodeVop1(0x40, mantissa_dst, Vgpr(i)));
}
AppendStoreVgpr(&code, mantissa_dst, inputs.size() + i);
}
AppendEnd(&code);
TestCase test;
test.name = "VectorFrexpF32Edges";
test.code = std::move(code);
test.initial = inputs;
test.initial.resize(inputs.size() * 2u);
test.expected = {
0u, 0u, 0u, 1u, 3u,
4u, 0xffffff83u, 0xffffff82u, 0xffffff6cu, 0u,
0u, 0u, 0u, 0x00000000u, 0x80000000u,
0x3f000000u, 0x3f000000u, 0x3f000000u, 0xbf000000u, 0x3f000000u,
0x3f7ffffeu, 0x3f000000u, 0x7f800000u, 0x7fc00000u, 0xff800000u,
0x7fa00001u,
};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_FREXP_EXP_I32_F32, O::V_FREXP_MANT_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_FREXP_EXP_I32_F32", inputs.size()},
{"V_FREXP_MANT_F32", inputs.size()}};
test.required_spirv = {"FindUMsb", "OpBitFieldUExtract",
"OpShiftLeftLogical"};
return test;
}
TestCase CvtF32ToIntSaturatesNaNAndOutOfRange() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x7fc00000u);
AppendVMovLiteral(&code, 1, 0xbf800000u);
AppendVMovLiteral(&code, 2, 0x7f800000u);
AppendVMovLiteral(&code, 3, 0x4f800000u);
AppendVMovLiteral(&code, 4, 0xff800000u);
AppendVMovLiteral(&code, 5, 0x4f000000u);
code.push_back(EncodeVop1(0x07, 10, Vgpr(0)));
code.push_back(EncodeVop1(0x07, 11, Vgpr(1)));
code.push_back(EncodeVop1(0x07, 12, Vgpr(2)));
code.push_back(EncodeVop1(0x07, 13, Vgpr(3)));
code.push_back(EncodeVop1(0x08, 14, Vgpr(0)));
code.push_back(EncodeVop1(0x08, 15, Vgpr(2)));
code.push_back(EncodeVop1(0x08, 16, Vgpr(4)));
code.push_back(EncodeVop1(0x08, 17, Vgpr(5)));
for (u32 i = 0; i < 8; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"CvtF32ToIntSaturatesNaNAndOutOfRange",
code,
{},
{0, 0, 0xffffffffu, 0xffffffffu, 0, 0x7fffffffu, 0x80000000u,
0x7fffffffu},
{O::V_MOV_B32, O::V_CVT_U32_F32, O::V_CVT_I32_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorSpecialF32FlushesDenormalInputs() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x00000001u);
code.push_back(EncodeVop1(0x27, 1, Vgpr(0)));
code.push_back(EncodeVop1(0x2a, 2, Vgpr(0)));
code.push_back(EncodeVop1(0x2e, 3, Vgpr(0)));
code.push_back(EncodeVop1(0x33, 4, Vgpr(0)));
AppendStoreVgpr(&code, 1, 0);
AppendStoreVgpr(&code, 2, 1);
AppendStoreVgpr(&code, 3, 2);
AppendStoreVgpr(&code, 4, 3);
AppendEnd(&code);
return {"VectorSpecialF32FlushesDenormalInputs",
code,
{},
{0xff800000u, 0x7f800000u, 0x7f800000u, 0x00000000u},
{O::V_MOV_B32, O::V_LOG_F32, O::V_RCP_F32, O::V_RSQ_F32,
O::V_SQRT_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorCompareInteger64Edges() {
using O = ShaderOpcode;
constexpr std::array<std::array<uint64_t, 2>, 10> pairs{{
{0, 0}, {0, 1}, {1, 0}, {0xffffffffull, 0x100000000ull},
{0x100000000ull, 0xffffffffull}, {0x8000000000000000ull, 0},
{0x7fffffffffffffffull, 0x8000000000000000ull},
{0xffffffffffffffffull, 0xffffffffffffffffull},
{0xffffffff80000000ull, 0xffffffff7fffffffull},
{0x1234567880000000ull, 0x123456787fffffffull}}};
TestCase test;
test.name = "VectorCompareInteger64Edges";
for (const auto& pair : pairs)
for (const auto bits : pair)
test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
test.expected = test.initial;
const auto store_mask = [&](u32 sgpr, bool value) {
AppendStoreSgprPair(&test.code, sgpr, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), {u32(value), 0});
};
for (u32 i = 0; i < pairs.size(); ++i) {
for (u32 word = 0; word < 4; ++word) {
AppendVMovU32(&test.code, 31, (i * 4 + word) * 4);
AppendBufferLoadDword(&test.code, 1 + word, 31);
}
test.code.push_back(EncodeVop1(0x02, 8, Vgpr(3)));
test.code.push_back(EncodeVop1(0x02, 9, Vgpr(4)));
const auto lhs = pairs[i][0], rhs = pairs[i][1];
const auto signed_lhs = std::bit_cast<int64_t>(lhs);
const auto signed_rhs = std::bit_cast<int64_t>(rhs);
for (const auto [opcode, value] : std::array<std::pair<u32, bool>, 9>{{
{0xa0, false}, {0xa1, signed_lhs < signed_rhs},
{0xa3, signed_lhs <= signed_rhs}, {0xa5, lhs != rhs}, {0xa7, true},
{0xe0, false}, {0xe3, lhs <= rhs}, {0xe6, lhs >= rhs}, {0xe7, true}}}) {
test.code.push_back(EncodeVopc(opcode, Vgpr(1), 3));
store_mask(106, value);
AppendVop3(&test.code, opcode, 20, Vgpr(1), 8);
store_mask(20, value);
}
// A negative literal expands by signedness, including equality comparisons.
for (const auto [opcode, value] : std::array<std::pair<u32, bool>, 5>{{
{0xa1, -1 < signed_rhs}, {0xa3, -1 <= signed_rhs},
{0xa5, -1 != signed_rhs},
{0xa2, -1 == signed_rhs}, {0xe3, 0xffffffffull <= rhs}}}) {
test.code.push_back(EncodeVopc(opcode, 255, 3));
test.code.push_back(0xffffffffu);
store_mask(106, value);
}
AppendVop3(&test.code, 0xa3, 20, Vgpr(1), 255);
test.code.push_back(0xffffffffu);
store_mask(20, signed_lhs <= -1);
test.code.push_back(EncodeSop1(0x04, 10, 126));
AppendVop3(&test.code, 0xb2, 126, 255, Vgpr(3)); // CMPX_EQ_I64 sign-extends -1.
test.code.push_back(0xffffffffu);
test.code.push_back(EncodeSop1(0x04, 20, 126));
test.code.push_back(EncodeSop1(0x04, 126, 10));
store_mask(20, -1 == signed_rhs);
// Captured 8395e43f382309df PC 0x20cc: s[2:3] = (s[0:1] != 0) & EXEC.
// Save the test buffer descriptor while its SGPRs hold the captured operands.
test.code.push_back(EncodeSop1(0x04, 32, 0));
test.code.push_back(EncodeSop1(0x04, 34, 2));
test.code.push_back(EncodeVop1(0x02, 0, Vgpr(1)));
test.code.push_back(EncodeVop1(0x02, 1, Vgpr(2)));
test.code.insert(test.code.end(), {0xd4a50002u, 0x00010000u});
test.code.push_back(EncodeSop1(0x04, 20, 2));
test.code.push_back(EncodeSop1(0x04, 0, 32));
test.code.push_back(EncodeSop1(0x04, 2, 34));
store_mask(20, lhs != 0);
}
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::V_CMP_F_I64, O::V_CMP_LT_I64, O::V_CMP_LE_I64, O::V_CMP_EQ_I64,
O::V_CMP_NE_I64, O::V_CMP_T_I64, O::V_CMPX_EQ_I64,
O::V_CMP_F_U64, O::V_CMP_LE_U64, O::V_CMP_GE_U64, O::V_CMP_T_U64,
O::S_MOV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpSLessThan", "OpULessThanEqual", "OpUGreaterThanEqual"};
return test;
}
TestCase VectorCompareExecWaveMasks(u32 wave_size) {
using O = ShaderOpcode;
// Rows exercise signed 16-bit truncation, full-width integer ordering, and
// ordered FP64 comparisons (including NaN, signed zero, and subnormals).
constexpr std::array<std::array<std::array<uint64_t, 2>, 8>, 3> inputs{{
{{{0x1111ffff, 0x22220000}, {0x33338000, 0x44447fff},
{0x55550000, 0x66660000}, {0x77777fff, 0x88888000},
{0x99990001, 0xaaaa0002}, {0xbbbb0002, 0xcccc0001},
{0xddddffff, 0xeeeeffff}, {0xffff8001, 0x11118000}}},
{{{0, 0}, {0, 1}, {1, 0}, {0xffffffffull, 0x100000000ull},
{0x100000000ull, 0xffffffffull}, {0x8000000000000000ull, 0},
{0xffffffffffffffffull, 0xffffffffffffffffull},
{0x1234567880000000ull, 0x123456787fffffffull}}},
{{{0, 0x8000000000000000ull}, {1, 0}, {0x8000000000000001ull, 0},
{0x7ff8000000000001ull, 0}, {0, 0x7ff0000000000001ull},
{0xc000000000000000ull, 0xbff0000000000000ull},
{0x7ff0000000000000ull, 0x7ff0000000000000ull},
{0x3ff0000000000001ull, 0x3ff0000000000000ull}}}}};
struct Compare { u32 encoding; O opcode; u32 group; };
constexpr std::array comparisons{
Compare{0x99, O::V_CMPX_LT_I16, 0}, Compare{0x9a, O::V_CMPX_EQ_I16, 0},
Compare{0x9b, O::V_CMPX_LE_I16, 0}, Compare{0x9c, O::V_CMPX_GT_I16, 0},
Compare{0x9d, O::V_CMPX_NE_I16, 0}, Compare{0x9e, O::V_CMPX_GE_I16, 0},
Compare{0xb0, O::V_CMPX_F_I64, 1}, Compare{0xb1, O::V_CMPX_LT_I64, 1},
Compare{0xb2, O::V_CMPX_EQ_I64, 1}, Compare{0xb3, O::V_CMPX_LE_I64, 1},
Compare{0xb7, O::V_CMPX_T_I64, 1}, Compare{0xf0, O::V_CMPX_F_U64, 1},
Compare{0xf1, O::V_CMPX_LT_U64, 1}, Compare{0xf2, O::V_CMPX_EQ_U64, 1},
Compare{0xf3, O::V_CMPX_LE_U64, 1}, Compare{0xf4, O::V_CMPX_GT_U64, 1},
Compare{0xf6, O::V_CMPX_GE_U64, 1}, Compare{0xf7, O::V_CMPX_T_U64, 1},
Compare{0x33, O::V_CMPX_LE_F64, 2}, Compare{0x36, O::V_CMPX_GE_F64, 2}};
constexpr u32 low_exec = 0xeeeeeeeeu, high_exec = 0x77777777u;
constexpr u32 vcc_lo = 0x12345678u, vcc_hi = 0x89abcdefu;
TestCase test;
test.name = wave_size == 32 ? "VectorCompareExecWave32Masks" : "VectorCompareExecWave64Masks";
test.initial.assign(12 * wave_size, 0);
for (u32 group = 0; group < inputs.size(); ++group)
for (u32 lane = 0; lane < wave_size; ++lane)
for (u32 word = 0; word < 4; ++word)
test.initial[(group * 4 + word) * wave_size + lane] =
u32(inputs[group][lane % 8][word / 2] >> (32 * (word % 2)));
test.expected = test.initial;
auto& code = test.code;
code.push_back(EncodeSop1(0x04, 10, 126));
AppendSMovLiteral(&code, 106, vcc_lo);
AppendSMovLiteral(&code, 107, vcc_hi);
for (const auto compare : comparisons) {
for (u32 word = 0; word < 4; ++word) {
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
AppendVMovU32(&code, 31, (compare.group * 4 + word) * wave_size * 4);
code.push_back(EncodeVop2(0x25, 31, Vgpr(31), 30));
AppendBufferLoadDword(&code, 1 + word, 31);
}
std::array<u32, 2> expected_exec{};
for (u32 lane = 0; lane < wave_size; ++lane) {
const auto pair = inputs[compare.group][lane % 8];
const auto a = std::bit_cast<int16_t>(uint16_t(pair[0]));
const auto b = std::bit_cast<int16_t>(uint16_t(pair[1]));
bool result = false;
switch (compare.encoding) {
case 0x99: result = a < b; break;
case 0x9a: result = a == b; break;
case 0x9b: result = a <= b; break;
case 0x9c: result = a > b; break;
case 0x9d: result = a != b; break;
case 0x9e: result = a >= b; break;
case 0xb0: case 0xf0: result = false; break;
case 0xb1: result = std::bit_cast<int64_t>(pair[0]) < std::bit_cast<int64_t>(pair[1]); break;
case 0xb2: case 0xf2: result = pair[0] == pair[1]; break;
case 0xb3: result = std::bit_cast<int64_t>(pair[0]) <= std::bit_cast<int64_t>(pair[1]); break;
case 0xb7: case 0xf7: result = true; break;
case 0xf1: result = pair[0] < pair[1]; break;
case 0xf3: result = pair[0] <= pair[1]; break;
case 0xf4: result = pair[0] > pair[1]; break;
case 0xf6: result = pair[0] >= pair[1]; break;
case 0x33: result = std::bit_cast<double>(pair[0]) <= std::bit_cast<double>(pair[1]); break;
case 0x36: result = std::bit_cast<double>(pair[0]) >= std::bit_cast<double>(pair[1]); break;
}
if (result) expected_exec[lane / 32] |= 1u << (lane % 32);
}
expected_exec[0] &= low_exec;
expected_exec[1] &= high_exec;
for (const bool vop3 : {false, true}) {
AppendSMovLiteral(&code, 126, low_exec);
if (wave_size == 64) AppendSMovLiteral(&code, 127, high_exec);
if (vop3) AppendVop3(&code, compare.encoding, 20, Vgpr(1), Vgpr(3));
else code.push_back(EncodeVopc(compare.encoding, Vgpr(1), 3));
code.push_back(EncodeSMovB32(20, 126));
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSop1(0x04, 126, 10));
for (const auto [reg, value] : std::array<std::pair<u32, u32>, 4>{{
{20, expected_exec[0]}, {21, expected_exec[1]},
{106, vcc_lo}, {107, vcc_hi}}}) {
AppendStoreSgprAtLaneDwordOffset(&code, reg, 0, u32(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, value);
}
}
test.opcodes.push_back(compare.opcode);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes.insert(test.opcodes.end(), {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM});
test.required_spirv = {"OpFOrdLessThanEqual", "OpFOrdGreaterThanEqual"};
return test;
}
TestCase VectorCompareF64Edges() {
using O = ShaderOpcode;
constexpr std::array<std::array<uint64_t, 2>, 15> pairs{{
{0xc000000000000000ull, 0xbff0000000000000ull},
{0x8000000000000001ull, 0}, {0x3fc45f306dc9c882ull, 0x3fc45f306dc9c882ull},
{0, 0x8000000000000000ull}, {1, 0}, {1, 1}, {1, 2},
{0x11d04c7b00ull, 0}, {0x3ff0000000000000ull, 0x3ff0000000000001ull},
{0xbff0000000000000ull, 0xbff0000000000000ull},
{0x7ff0000000000000ull, 0x7ff0000000000000ull},
{0x7ff0000000000000ull, 0xfff0000000000000ull},
{0x7ff8000000000001ull, 0x7ff8000000000001ull},
{0x7ff0000000000001ull, 0}, {0x7fefffffffffffffull, 0x7fefffffffffffffull}}};
TestCase test;
test.name = "VectorCompareF64Edges";
for (const auto& pair : pairs)
for (const auto bits : pair)
test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
test.expected = test.initial;
const auto store_mask = [&](u32 sgpr, bool value) {
AppendStoreSgprPair(&test.code, sgpr, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), {u32(value), 0});
};
for (u32 i = 0; i < pairs.size(); ++i) {
for (u32 word = 0; word < 4; ++word) {
AppendVMovU32(&test.code, 31, (i * 4 + word) * 4);
AppendBufferLoadDword(&test.code, 1 + word, 31);
}
const auto lhs = std::bit_cast<double>(pairs[i][0]);
const auto rhs = std::bit_cast<double>(pairs[i][1]);
test.code.push_back(EncodeVopc(0x22, Vgpr(1), 3));
store_mask(106, lhs == rhs);
test.code.push_back(EncodeVopc(0x23, Vgpr(1), 3));
store_mask(106, lhs <= rhs);
AppendVop3(&test.code, 0x23, 20, Vgpr(1), Vgpr(3), 0, 1, 0, false, 0, 1);
store_mask(20, -std::abs(lhs) <= rhs);
test.code.push_back(EncodeVopc(0x23, 255, 3));
test.code.push_back(0x3ff00000u); // FP64 literals supply the high word.
store_mask(106, 1.0 <= rhs);
AppendVop3(&test.code, 0x23, 20, 248, Vgpr(3));
store_mask(20, std::bit_cast<double>(uint64_t{0x3fc45f306dc9c882ull}) <= rhs);
AppendVop3(&test.code, 0x22, 20, Vgpr(1), Vgpr(3), 0, 1, 0, false, 0, 1);
store_mask(20, -std::abs(lhs) == rhs);
test.code.push_back(EncodeVop1(0x02, 100, Vgpr(3)));
test.code.push_back(EncodeVop1(0x02, 101, Vgpr(4)));
// Captured e6d76d24f59f8015 PC 0xe4: compare zero with s100:s101.
test.code.insert(test.code.end(), {0xd422006au, 0x0000c880u});
store_mask(106, 0.0 == rhs);
AppendVop3(&test.code, 0x23, 20, Vgpr(1), 100);
store_mask(20, lhs <= rhs);
}
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::V_CMP_EQ_F64, O::V_CMP_LE_F64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "OpFOrdEqual",
"OpFOrdLessThanEqual"};
return test;
}
TestCase VectorCompare64WaveMasks(u32 wave_size, bool integer_ne = false) {
using O = ShaderOpcode;
constexpr std::array<uint64_t, 4> values{0, 1, 0x8000000000000000ull,
0x7ff8000000000000ull};
constexpr u32 low_exec = 0xeeeeeeeeu, high_exec = 0x77777777u;
constexpr u32 vcc_hi = 0x89abcdefu;
constexpr u32 vcc_lo = 0x12345678u;
const u32 result_reg = integer_ne ? 20 : 106;
const u32 result_bits = integer_ne ? 0xeeeeeeeeu : 0x55555555u;
TestCase test;
test.name = integer_ne
? (wave_size == 32 ? "VectorCompareNeI64Wave32Masks" : "VectorCompareNeI64Wave64Masks")
: (wave_size == 32 ? "VectorCompareF64Wave32Masks" : "VectorCompareF64Wave64Masks");
test.initial.assign((integer_ne ? 8 : 6) * wave_size, 0);
for (u32 lane = 0; lane < wave_size; ++lane) {
test.initial[lane] = u32(values[lane % 4]);
test.initial[wave_size + lane] = u32(values[lane % 4] >> 32);
}
test.expected = test.initial;
for (u32 lane = 0; lane < wave_size; ++lane) {
test.expected[2 * wave_size + lane] = low_exec & result_bits;
test.expected[3 * wave_size + lane] = wave_size == 64 ? high_exec & result_bits : vcc_hi;
test.expected[4 * wave_size + lane] = low_exec;
test.expected[5 * wave_size + lane] = wave_size == 64 ? high_exec : 0;
if (integer_ne) {
test.expected[6 * wave_size + lane] = vcc_lo;
test.expected[7 * wave_size + lane] = vcc_hi;
}
}
auto& code = test.code;
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
AppendBufferLoadDword(&code, 1, 30);
AppendVMovU32(&code, 31, wave_size * 4);
code.push_back(EncodeVop2(0x25, 31, Vgpr(31), 30));
AppendBufferLoadDword(&code, 2, 31);
code.push_back(EncodeSop1(0x04, 10, 126));
AppendSMovLiteral(&code, result_reg + 1, vcc_hi);
if (integer_ne) {
AppendSMovLiteral(&code, 106, vcc_lo);
AppendSMovLiteral(&code, 107, vcc_hi);
}
AppendSMovLiteral(&code, 126, low_exec);
if (wave_size == 64) AppendSMovLiteral(&code, 127, high_exec);
AppendVop3(&code, integer_ne ? 0xa5 : 0x22, result_reg, InlineU32(0), Vgpr(1));
code.push_back(EncodeSMovB32(22, 126));
code.push_back(EncodeSMovB32(23, 127));
code.push_back(EncodeSop1(0x04, 126, 10));
for (const auto [reg, offset] : std::array<std::pair<u32, u32>, 4>{{
{result_reg, 2}, {result_reg + 1, 3}, {22, 4}, {23, 5}}})
AppendStoreSgprAtLaneDwordOffset(&code, reg, 0, offset * wave_size);
if (integer_ne) {
AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, 6 * wave_size);
AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, 7 * wave_size);
}
AppendEnd(&code);
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD,
integer_ne ? O::V_CMP_NE_I64 : O::V_CMP_EQ_F64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {integer_ne ? "OpINotEqual" : "OpFOrdEqual"};
return test;
}
TestCase VectorF64CapturedScreenSpaceShadows() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorF64CapturedScreenSpaceShadows";
const auto store_pair = [&](u32 reg, double value) {
const auto bits = std::bit_cast<uint64_t>(value);
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, reg + 1,
static_cast<u32>(test.expected.size() + 1));
test.expected.insert(test.expected.end(), {static_cast<u32>(bits),
static_cast<u32>(bits >> 32)});
};
for (const auto values :
{std::array<int32_t, 3>{63, 127, 101},
std::array<int32_t, 3>{16777217, -16777217, 123456789},
std::array<int32_t, 3>{INT32_MIN, INT32_MAX, -31}}) {
AppendVMovLiteral(&test.code, 7, static_cast<u32>(values[0]));
AppendVMovLiteral(&test.code, 1, static_cast<u32>(values[1]));
AppendVMovLiteral(&test.code, 9, static_cast<u32>(values[2]));
// Exact FP64 instruction words from b90e2024732c6111, PCs 0xc8 through
// 0x130.
test.code.insert(test.code.end(), {0x7e060907u, 0x7e0a0901u, 0x7e020909u});
store_pair(3, double(values[0]));
store_pair(5, double(values[1]));
store_pair(1, double(values[2]));
test.code.push_back(0x7e065f03u);
const double reciprocal = 1.0 / double(values[0]);
store_pair(3, reciprocal);
test.code.insert(test.code.end(), {0xd5650005u, 0x00020305u});
const double product = double(values[1]) * double(values[2]);
store_pair(5, product);
test.code.insert(test.code.end(), {0xd54c0001u, 0x84060705u});
const double result = std::fma(product, reciprocal, -double(values[2]));
store_pair(1, result);
test.code.push_back(0x7e041f01u);
AppendStoreVgpr(&test.code, 2, static_cast<u32>(test.expected.size()));
test.expected.push_back(std::bit_cast<u32>(static_cast<float>(result)));
}
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::V_CVT_F64_I32, O::V_RCP_F64,
O::V_MUL_F64, O::V_FMA_F64, O::V_CVT_F32_F64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpCapability Float64",
"OpTypeFloat 64",
"OpFDiv",
"Fma",
"SignedZeroInfNanPreserve 64"};
test.ir_counts = {{"ConvertF64S32", 9},
{"FPRecip64", 3},
{"FPMul64", 3},
{"FPFma64", 3},
{"ConvertF32F64", 3}};
ShaderComputeInputInfo info{};
std::vector<u32> normal_key, other_key;
BuildStageStaticKey(info, normal_key);
info.float_mode = 0xc4;
BuildStageStaticKey(info, other_key);
Require(test.name, "pipeline identity",
normal_key != other_key && normal_key.size() == other_key.size(),
"FP64 rounding modes must not share a compiled shader key");
return test;
}
TestCase VectorFractF64CapturedAndEdges() {
using O = ShaderOpcode;
constexpr std::array<std::array<uint64_t, 2>, 17> cases{{
{0x400a000000000000ull, 0x3fd0000000000000ull}, // 3.25 -> .25
{0xbff4000000000000ull, 0x3fe8000000000000ull}, // -1.25 -> .75
{0x3ff0000000000001ull, 0x3cb0000000000000ull}, // 1 + 2^-52
{0xbff0000000000001ull, 0x3feffffffffffffeull},
{0x4320000000000001ull, 0x3fe0000000000000ull}, // 2^51 + .5
{0, 0}, {0x8000000000000000ull, 0},
{0x401c000000000000ull, 0}, {0xc01c000000000000ull, 0},
{1, 1}, {0x8000000000000001ull, 0x3ff0000000000000ull},
{0x000fffffffffffffull, 0x000fffffffffffffull},
{0x7fefffffffffffffull, 0},
{0x7ff0000000000000ull, 0x7ff8000000000000ull},
{0xfff0000000000000ull, 0x7ff8000000000000ull},
{0x7ff8123456789abcull, 0x7ff8000000000000ull},
{0x7ff0000000000001ull, 0x7ff8000000000000ull}}};
TestCase test;
test.name = "VectorFractF64CapturedAndEdges";
const auto store_pair = [&](u32 reg, uint64_t expected) {
const auto index = static_cast<u32>(test.expected.size());
AppendStoreVgpr(&test.code, reg, index);
AppendStoreVgpr(&test.code, reg + 1, index + 1);
test.expected.insert(test.expected.end(), {u32(expected), u32(expected >> 32)});
};
for (u32 i = 0; i < cases.size(); i++) {
const auto [source, expected] = cases[i];
test.initial.insert(test.initial.end(), {u32(source), u32(source >> 32)});
for (u32 word = 0; word < 2; word++) {
AppendVMovU32(&test.code, 30, (i * 2 + word) * 4);
AppendBufferLoadDword(&test.code, 4 + word, 30);
}
test.code.push_back(0x7e087d04u); // Captured v_fract_f64 v[4:5], v[4:5].
if ((source & 0x7ff0000000000000ull) == 0x7ff0000000000000ull) {
// Classify quiet NaNs without requiring a particular payload or sign.
AppendVMovU32(&test.code, 4, 0);
AppendVMovLiteral(&test.code, 6, 0x7ff80000u);
test.code.push_back(EncodeVop2(0x1b, 5, Vgpr(6), 5));
}
store_pair(4, expected);
}
AppendVMovU32(&test.code, 1, 0);
AppendVMovLiteral(&test.code, 2, 0xbff40000u);
AppendVop3(&test.code, 0x1be, 1, Vgpr(1), 0, 0, 1, 0, false, 0, 1);
store_pair(1, 0x3fe8000000000000ull); // Odd pair; ABS precedes NEG.
AppendVMovU32(&test.code, 4, 1);
AppendVMovLiteral(&test.code, 5, 0x3ff00000u);
test.code.push_back(EncodeSop1(0x04, 12, 126));
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
test.code.push_back(0x7e087d04u);
test.code.push_back(EncodeSop1(0x04, 126, 12));
store_pair(4, 0x3ff0000000000001ull); // Both words survive inactive EXEC.
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FRACT_F64,
O::V_AND_B32, O::S_MOV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_FRACT_F64 v4, v4", cases.size() + 1}};
test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "Fract"};
return test;
}
TestCase VectorMinMaxF64CapturedAndEdges() {
using O = ShaderOpcode;
// MODE.IEEE=0: select the numeric operand, or the second NaN unchanged.
constexpr uint64_t positive_zero = 0, negative_zero = 0x8000000000000000ull;
constexpr uint64_t one = 0x3ff0000000000000ull, minus_two = 0xc000000000000000ull;
constexpr uint64_t qnan = 0x7ff8123456789abcull, snan = 0xfff0000000000042ull;
constexpr uint64_t infinity = 0x7ff0000000000000ull;
constexpr std::array<std::array<uint64_t, 4>, 20> cases{{
{one, minus_two, minus_two, one}, {minus_two, one, minus_two, one},
{one, one, one, one},
{positive_zero, negative_zero, negative_zero, positive_zero},
{negative_zero, positive_zero, negative_zero, positive_zero},
{negative_zero, negative_zero, negative_zero, negative_zero},
{positive_zero, positive_zero, positive_zero, positive_zero},
{qnan, one, one, one}, {one, qnan, one, one},
{snan, minus_two, minus_two, minus_two}, {minus_two, snan, minus_two, minus_two},
{qnan, snan, snan, snan}, {snan, qnan, qnan, qnan},
{infinity, one, one, infinity},
{infinity | negative_zero, one, infinity | negative_zero, one},
{0x3fe0000000000000ull, one, 0x3fe0000000000000ull, one},
{1, negative_zero | 1, negative_zero | 1, 1}, {1, 0, 0, 1},
{1, 2, 1, 2}, {2, 1, 1, 2}}};
TestCase test;
test.name = "VectorMinMaxF64CapturedAndEdges";
for (const auto &item : cases) {
for (u32 source = 0; source < 2; source++) {
test.initial.insert(test.initial.end(), {u32(item[source]), u32(item[source] >> 32)});
}
}
test.expected = test.initial;
const auto store_pair = [&](u32 reg, uint64_t bits) {
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, reg + 1, static_cast<u32>(test.expected.size() + 1));
test.expected.insert(test.expected.end(), {u32(bits), u32(bits >> 32)});
};
for (u32 i = 0; i < cases.size(); i++) {
for (u32 word = 0; word < 4; word++) {
AppendVMovU32(&test.code, 30, (i * 4 + word) * 4);
AppendBufferLoadDword(&test.code, 2 + word, 30);
}
AppendVop3(&test.code, 0x166, 7, Vgpr(2), Vgpr(4));
store_pair(7, cases[i][2]);
AppendVop3(&test.code, 0x167, 9, Vgpr(2), Vgpr(4));
store_pair(9, cases[i][3]);
// CS daced64c72398e80 PCs 0x10c/0x11c clamp the pair to [0, 1].
test.code.insert(test.code.end(), {0xd5670006u, 0x00020480u,
0xd5660006u, 0x00020cf2u});
const auto source = std::bit_cast<double>(cases[i][0]);
uint64_t clamped = 0;
if (source > 0) clamped = source < 1 ? cases[i][0] : one;
store_pair(6, clamped);
}
AppendSMovLiteral(&test.code, 8, 0);
AppendSMovLiteral(&test.code, 9, u32(minus_two >> 32));
// Scalar source and odd destination pairs; ABS must precede NEG.
AppendVop3(&test.code, 0x166, 1, 8, 243, 0, 1, 0, false, 0, 1);
store_pair(1, minus_two);
AppendVop3(&test.code, 0x167, 1, 8, 243, 0, 1, 0, false, 0, 1);
store_pair(1, 0xbff0000000000000ull);
test.code.push_back(EncodeSop1(0x04, 12, 126));
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
AppendVop3(&test.code, 0x166, 1, 242, 242);
AppendVop3(&test.code, 0x167, 1, 242, 242);
test.code.push_back(EncodeSop1(0x04, 126, 12));
store_pair(1, 0xbff0000000000000ull);
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_DWORD,
O::V_MIN_F64, O::V_MAX_F64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64",
"SignedZeroInfNanPreserve 64"};
return test;
}
TestCase VectorRoundF64EdgesAndModifiers() {
using O = ShaderOpcode;
constexpr std::array<uint64_t, 14> sources{
0x400a000000000000ull, 0xbff4000000000000ull,
0x3fd0000000000000ull, 0xbfd0000000000000ull,
0, 0x8000000000000000ull, 0x4330000000000001ull, 0xc330000000000001ull,
0x7ff0000000000000ull, 0xfff0000000000000ull,
0xfff8123456789abcull, 0x7ff0000000000001ull, 1, 0x8000000000000001ull};
TestCase test;
test.name = "VectorRoundF64EdgesAndModifiers";
for (const auto bits : sources) {
test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
}
test.expected = test.initial;
const auto rounded = [](u32 opcode, double value) {
switch (opcode) {
case 0x17: return std::trunc(value);
case 0x18: return std::ceil(value);
case 0x1a: return std::floor(value);
default: std::abort();
}
};
const auto store_pair = [&](u32 reg, uint64_t bits) {
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, reg + 1, static_cast<u32>(test.expected.size() + 1));
test.expected.insert(test.expected.end(), {u32(bits), u32(bits >> 32)});
};
for (const u32 opcode : {0x17u, 0x18u, 0x1au}) {
for (u32 i = 0; i < sources.size(); i++) {
for (u32 word = 0; word < 2; word++) {
AppendVMovU32(&test.code, 30, (i * 2 + word) * 4);
AppendBufferLoadDword(&test.code, 2 + word, 30);
}
test.code.push_back(EncodeVop1(opcode, 5, Vgpr(2)));
AppendVop3(&test.code, 0x180 + opcode, 7, Vgpr(2), 0);
const auto value = std::bit_cast<double>(sources[i]);
for (const u32 reg : {5u, 7u}) {
if (std::isnan(value)) {
// IEEE=false does not promise a particular NaN payload or quiet bit.
test.code.push_back(EncodeVopc(0x22, Vgpr(reg), reg));
AppendStoreSgprPair(&test.code, 106, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), {0, 0});
} else {
store_pair(reg, std::bit_cast<uint64_t>(rounded(opcode, value)));
}
}
}
AppendSMovLiteral(&test.code, 8, 0);
AppendSMovLiteral(&test.code, 9, 0xbff40000u);
AppendVop3(&test.code, 0x180 + opcode, 1, 8, 0, 0, 1, 0, false, 0, 1);
const auto expected = std::bit_cast<uint64_t>(rounded(opcode, -1.25));
store_pair(1, expected);
test.code.push_back(EncodeSop1(0x04, 12, 126));
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
test.code.push_back(EncodeVop1(opcode, 1, 242));
test.code.push_back(EncodeSop1(0x04, 126, 12));
store_pair(1, expected);
}
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_DWORD,
O::V_TRUNC_F64, O::V_CEIL_F64, O::V_FLOOR_F64, O::V_CMP_EQ_F64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "Trunc", "Ceil", "Floor"};
return test;
}
TestCase VectorFractF64CapturedChainRuntimeExec() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorFractF64CapturedChainRuntimeExec";
auto& code = test.code;
AppendSMovLiteral(&code, 60, 3);
AppendSMovLiteral(&code, 61, 3);
code.push_back(EncodeSop1(0x04, 126, 60)); // Lanes 0,1 and 32,33.
code.push_back(EncodeVop1(0x01, 20, Vgpr(0)));
code.push_back(EncodeVop2(0x1b, 30, InlineU32(1), 0));
code.push_back(EncodeVop2(0x1a, 30, InlineU32(4), 30));
AppendVMovU32(&code, 7, 192 * 4);
code.push_back(EncodeVop2(0x25, 30, Vgpr(7), 30));
constexpr u32 input_regs[] = {6, 4, 10, 11};
for (u32 word = 0; word < 4; ++word) {
code.push_back(EncodeMubuf0(0x0c, word * 4));
code.push_back(EncodeMubuf1(input_regs[word], 0, 30));
}
code.push_back(EncodeSmem0(0x08, 16, 0));
code.push_back(EncodeSmem1(200 * 4, 125));
constexpr u32 output_regs[] = {12, 22, 24};
for (const auto reg : output_regs) AppendVMovLiteral(&code, reg, 0xdeadbeefu);
code.push_back(EncodeVopc(0xc2, InlineU32(1), 6));
code.push_back(EncodeSop1(0x24, 8, 106));
code.push_back(EncodeSMovB32(106, 16));
AppendSMovLiteral(&code, 107, 0x3fea36a9u);
code.push_back(EncodeVop1(0x2a, 4, Vgpr(4))); // Runtime FP32 reciprocal with FP64 narrowing.
// Captured 4ee720fc4678fa19 PCs 0x1e8..0x20c, 0x23c..0x258, 0x260..0x270.
code.insert(code.end(), {0x7e04086au, 0xbeea03f4u, 0x7e082104u, 0xd54c0004u,
0x0410d502u, 0xbeea0b86u, 0xbeeb03ffu, 0x3fe3f9e1u,
0x7e087d04u, 0x7e181f04u});
code.push_back(EncodeVop1(0x01, 4, Vgpr(10))); // Second runtime image value.
code.insert(code.end(), {0x7e082104u, 0xd54c0004u, 0x0410d502u, 0xbeea0b87u,
0xbeeb03ffu, 0x3fc46bacu, 0x7e087d04u, 0x7e2c1f04u,
0x7e08210bu, 0xd54c0002u, 0x0410d502u, 0x7e047d02u,
0x7e301f02u});
code.push_back(EncodeSop1(0x04, 126, 8));
for (u32 component = 0; component < 3; ++component)
AppendStoreVgprAtLaneDwordOffset(&code, output_regs[component], 20, component * 64);
AppendEnd(&code);
test.initial.resize(192);
test.initial.insert(test.initial.end(), {1, 0x3e800000u, 0xbfa00000u, 0x40600000u,
0, 0x3f800000u, 0x40000000u, 0x40400000u, 3});
test.expected = test.initial;
constexpr u32 results[] = {0x3eea3fbcu, 0x3f1f6d21u, 0x3f7a860du};
for (u32 component = 0; component < 3; ++component) {
for (const u32 lane : {0u, 32u}) test.expected[component * 64 + lane] = results[component];
for (const u32 lane : {1u, 33u}) test.expected[component * 64 + lane] = 0xdeadbeefu;
}
test.compute_info.wave_size = 64;
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
test.required_spirv = {"OpCapability Float64", "OpFDiv", "Fma", "Fract"};
test.forbidden_spirv = {"RoundingModeRTE"};
test.ir_counts = {{"FPRecip32", 1}, {"ConvertF64S32", 1}, {"ConvertF64F32", 3},
{"FPFma64", 3}, {"FPFract64", 3}, {"ConvertF32F64", 3}};
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_MOV_B32, O::V_AND_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD,
O::S_BUFFER_LOAD_DWORD, O::V_CMP_EQ_U32, O::S_AND_SAVEEXEC_B64,
O::S_BREV_B32, O::V_CVT_F64_I32, O::V_CVT_F64_F32, O::V_FMA_F64,
O::V_FRACT_F64, O::V_RCP_F32, O::V_CVT_F32_F64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase VectorF64WideningConversions() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorF64WideningConversions";
const auto store_pair = [&](u32 reg, uint64_t bits) {
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, reg + 1,
static_cast<u32>(test.expected.size() + 1));
test.expected.insert(test.expected.end(), {static_cast<u32>(bits),
static_cast<u32>(bits >> 32)});
};
for (const u32 bits :
{0u, 0x80000000u, 0x3f800001u, 0xbf800001u, 0x00800000u, 0x7f7fffffu,
0xff7fffffu, 0x7f800000u, 0xff800000u, 1u, 0x80000001u}) {
AppendSMovLiteral(&test.code, 8, bits);
test.code.push_back(0x7e002008u);
const auto source = (bits & 0x7f800000u) == 0 ? bits & 0x80000000u : bits;
store_pair(0, std::bit_cast<uint64_t>(
static_cast<double>(std::bit_cast<float>(source))));
}
for (const u32 bits : {0x7fc12345u, 0x7f812345u}) {
AppendSMovLiteral(&test.code, 8, bits);
test.code.push_back(0x7e002008u);
AppendVMovLiteral(&test.code, 3, 0x7ff80000u);
test.code.push_back(EncodeVop2(0x1b, 1, Vgpr(3), 1));
AppendStoreVgpr(&test.code, 1, static_cast<u32>(test.expected.size()));
test.expected.push_back(0x7ff80000u);
}
AppendVMovLiteral(&test.code, 1, std::bit_cast<u32>(-3.5f));
AppendVop3(&test.code, 0x190, 1, Vgpr(1), 0, 0, 1, 0, false, 0, 1);
store_pair(1, std::bit_cast<uint64_t>(-3.5));
for (const u32 value :
{0u, 1u, 16777217u, 0x7fffffffu, 0x80000000u, 0xffffffffu}) {
AppendVMovLiteral(&test.code, 1, value);
test.code.push_back(EncodeVop1(0x16, 1, Vgpr(1)));
store_pair(1, std::bit_cast<uint64_t>(static_cast<double>(value)));
}
AppendSMovLiteral(&test.code, 8, 0xffffffffu);
AppendVop3(&test.code, 0x196, 1, 8u, 0);
store_pair(1, std::bit_cast<uint64_t>(4294967295.0));
test.code.push_back(EncodeSop1(0x04, 12, 126));
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
test.code.push_back(EncodeVop1(0x10, 1, InlineU32(0)));
test.code.push_back(EncodeVop1(0x16, 1, InlineU32(0)));
test.code.push_back(EncodeSop1(0x04, 126, 12));
store_pair(1, std::bit_cast<uint64_t>(4294967295.0));
AppendEnd(&test.code);
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_MOV_B32,
O::V_AND_B32, O::V_CVT_F64_F32, O::V_CVT_F64_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "OpFConvert",
"OpConvertUToF"};
std::vector<u32> vertex_code;
AppendSMovLiteral(&vertex_code, 8, std::bit_cast<u32>(2.0f));
vertex_code.insert(vertex_code.end(),
{0x7e002008u, 0x7e005f00u, 0x7e401f00u});
vertex_code.push_back(EncodeExp0(0x0c, 0xf));
vertex_code.push_back(EncodeExp1(32, 32, 32, 32));
AppendEnd(&vertex_code);
ShaderVertexInputInfo vertex{};
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Vertex;
options.input_info.vertex = &vertex;
auto translated = ShaderRecompiler::TranslateProgram(vertex_code, options);
auto result =
ShaderRecompiler::CompileProgram(std::move(translated), options, {});
ValidateSpirv(test.name, result.spirv);
Require(test.name, "captured vertex conversion chain",
result.ir_dump.find("ConvertF64F32") != std::string::npos &&
result.ir_dump.find("FPRecip64") != std::string::npos &&
result.ir_dump.find("ConvertF32F64") != std::string::npos,
"vertex compilation lost the captured widening/reciprocal/narrowing "
"sequence");
return test;
}
TestCase VectorF64ModesModifiersAndExec() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorF64ModesModifiersAndExec";
constexpr std::array<std::array<uint64_t, 2>, 8> add_pairs{{
{1, 1}, {1, 0x8000000000000001ull},
{0x8000000000000000ull, 0x8000000000000000ull},
{0x3ff0000000000000ull, 0x3ca0000000000000ull},
{0x3ff0000000000001ull, 0x3ca0000000000000ull},
{0x3ff0000000000000ull, 0xbff0000000000000ull},
{0x7ff0000000000000ull, 0x3ff0000000000000ull},
{0xfff0000000000000ull, 0xbff0000000000000ull}}};
for (const auto& pair : add_pairs)
for (const auto bits : pair)
test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
// Runtime F64 values immediately around even/odd F32 halfway points, both signs.
constexpr std::array<std::pair<uint64_t, u32>, 12> narrowing{{
{0x3ff000000fffffffull, 0x3f800000u}, {0x3ff0000010000000ull, 0x3f800000u},
{0x3ff0000010000001ull, 0x3f800001u}, {0x3ff000002fffffffull, 0x3f800001u},
{0x3ff0000030000000ull, 0x3f800002u}, {0x3ff0000030000001ull, 0x3f800002u},
{0xbff000000fffffffull, 0xbf800000u}, {0xbff0000010000000ull, 0xbf800000u},
{0xbff0000010000001ull, 0xbf800001u}, {0xbff000002fffffffull, 0xbf800001u},
{0xbff0000030000000ull, 0xbf800002u}, {0xbff0000030000001ull, 0xbf800002u}}};
for (const auto& item : narrowing)
test.initial.insert(test.initial.end(), {u32(item.first), u32(item.first >> 32)});
test.expected = test.initial;
const auto load_pair = [&](u32 reg, uint64_t bits) {
AppendVMovLiteral(&test.code, reg, static_cast<u32>(bits));
AppendVMovLiteral(&test.code, reg + 1, static_cast<u32>(bits >> 32));
};
const auto store_pair = [&](u32 reg, uint64_t bits) {
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, reg + 1,
static_cast<u32>(test.expected.size() + 1));
test.expected.insert(test.expected.end(), {static_cast<u32>(bits),
static_cast<u32>(bits >> 32)});
};
const auto store_word = [&](u32 reg, u32 bits) {
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
test.expected.push_back(bits);
};
for (const auto [source, expected] :
{std::pair<uint64_t, uint64_t>{0, 0x7ff0000000000000ull},
{0x8000000000000000ull, 0xfff0000000000000ull},
{0x7ff0000000000000ull, 0},
{0xfff0000000000000ull, 0x8000000000000000ull},
{0x0008000000000000ull, 0x7fe0000000000000ull},
{0x7fe0000000000000ull, 0x0008000000000000ull}}) {
load_pair(1, source);
test.code.push_back(EncodeVop1(0x2f, 1, Vgpr(1)));
store_pair(1, expected);
}
for (u32 i = 0; i < add_pairs.size(); ++i) {
const auto pair = add_pairs[i];
for (u32 word = 0; word < 4; ++word) {
AppendVMovU32(&test.code, 31, (i * 4 + word) * 4);
AppendBufferLoadDword(&test.code, 1 + word, 31);
}
AppendVop3(&test.code, 0x164, 1, Vgpr(1), Vgpr(3));
store_pair(1, std::bit_cast<uint64_t>(std::bit_cast<double>(pair[0]) +
std::bit_cast<double>(pair[1])));
}
// Scalar pair, exact inline 1/(2*pi), literal high word, and ABS before NEG.
AppendSMovLiteral(&test.code, 8, 0);
AppendSMovLiteral(&test.code, 9, 0xbff00000u);
AppendVop3(&test.code, 0x164, 1, 8, 244, 0, 1, 0, false, 0, 1);
store_pair(1, std::bit_cast<uint64_t>(1.0));
AppendVop3(&test.code, 0x164, 1, 255, 240);
test.code.push_back(0x40080000u);
store_pair(1, std::bit_cast<uint64_t>(3.5));
AppendVop3(&test.code, 0x164, 1, 248, InlineU32(0));
store_pair(1, 0x3fc45f306dc9c882ull);
for (const auto pair : std::array<std::array<uint64_t, 2>, 2>{{
{0x7ff0000000000000ull, 0xfff0000000000000ull},
{0x7ff0000000000001ull, 0}}}) {
load_pair(1, pair[0]);
load_pair(3, pair[1]);
AppendVop3(&test.code, 0x164, 1, Vgpr(1), Vgpr(3));
AppendVMovLiteral(&test.code, 3, 0x7ff80000u);
test.code.push_back(EncodeVop2(0x1b, 2, Vgpr(3), 2));
store_word(2, 0x7ff80000u);
}
// A fused result retains the residual that separate multiply/add would round
// to zero.
load_pair(1, std::bit_cast<uint64_t>(1.0 + 0x1p-27));
load_pair(3, std::bit_cast<uint64_t>(1.0 - 0x1p-27));
AppendVop3(&test.code, 0x14c, 1, Vgpr(1), Vgpr(3), 243u);
store_pair(1, std::bit_cast<uint64_t>(-0x1p-54));
// Unaligned pairs, SGPR pairs, inline floats, literal highword expansion, ABS
// then NEG.
test.code.push_back(EncodeSop1(0x03, 8, 255));
test.code.push_back(0u);
test.code.push_back(EncodeSop1(0x03, 9, 255));
test.code.push_back(0xbff00000u);
AppendVop3(&test.code, 0x165, 1, 8u, 244u, 0, 1, 0, false, 0, 1);
store_pair(1, std::bit_cast<uint64_t>(-2.0));
AppendVop3(&test.code, 0x165, 1, 255u, 242u);
test.code.push_back(0x40080000u);
store_pair(1, std::bit_cast<uint64_t>(3.0));
AppendVop3(&test.code, 0x165, 1, 248u, 242u);
store_pair(1, 0x3fc45f306dc9c882ull);
AppendVop3(&test.code, 0x184, 1, InlineU32(3), 0);
store_pair(1, std::bit_cast<uint64_t>(3.0));
AppendVop3(&test.code, 0x1af, 1, 244u, 0);
store_pair(1, std::bit_cast<uint64_t>(0.5));
// Preserve both destination words when EXEC is clear, restoring the original
// live lanes.
test.code.push_back(EncodeSop1(0x04, 12, 126));
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
test.code.push_back(EncodeVop1(0x04, 1, InlineU32(9)));
AppendVop3(&test.code, 0x164, 1, 242, 242);
test.code.push_back(EncodeSop1(0x04, 126, 12));
store_pair(1, std::bit_cast<uint64_t>(0.5));
for (const auto [source, expected] :
{std::pair<uint64_t, u32>{0x47f0000000000000ull, 0x7f7fffffu},
{0xc7f0000000000000ull, 0xff7fffffu},
{0x7ff0000000000000ull, 0x7f800000u},
{0xfff0000000000000ull, 0xff800000u},
{0x3800000000000000ull, 0},
{0xb800000000000000ull, 0x80000000u},
{0x3810000000000000ull, 0x00800000u},
{0x3ff0000010000000ull, 0x3f800000u}}) {
load_pair(1, source);
AppendVop3(&test.code, 0x18f, 1, Vgpr(1), 0);
store_word(1, expected);
}
for (u32 i = 0; i < narrowing.size(); ++i) {
for (u32 word = 0; word < 2; ++word) {
AppendVMovU32(&test.code, 31, (add_pairs.size() * 4 + i * 2 + word) * 4);
AppendBufferLoadDword(&test.code, 1 + word, 31);
}
AppendVop3(&test.code, 0x18f, 1, Vgpr(1), 0);
store_word(1, narrowing[i].second);
}
load_pair(1, 0x7ff8123456789abcull);
test.code.push_back(EncodeVop1(0x2f, 1, Vgpr(1)));
AppendVMovLiteral(&test.code, 3, 0x7ff80000u);
test.code.push_back(EncodeVop2(0x1b, 2, Vgpr(3), 2));
store_word(2, 0x7ff80000u);
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64,
O::V_CVT_F64_I32, O::V_RCP_F64, O::V_MUL_F64,
O::V_ADD_F64, O::V_FMA_F64, O::V_CVT_F32_F64, O::V_AND_B32,
O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.initial.resize(test.expected.size());
test.required_spirv = {"OpFAdd"};
return test;
}
TestCase VectorRcpIflagF32IntegerReciprocal() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 0);
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 2);
AppendVMovLiteral(&code, 3, 0xffffffffu);
for (u32 i = 0; i < 4; i++) {
code.push_back(EncodeVop1(0x06, 4 + i, Vgpr(i)));
code.push_back(EncodeVop1(0x2b, 8 + i, Vgpr(4 + i)));
}
AppendVMovLiteral(&code, 12, 0xc0000000u);
AppendVop3(&code, 0x1ab, 13, Vgpr(12), 0, 0, 1, 0, false, 1);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 8 + i, i);
}
AppendStoreVgpr(&code, 13, 4);
AppendEnd(&code);
TestCase test;
test.name = "VectorRcpIflagF32IntegerReciprocal";
test.code = std::move(code);
test.expected = {0x7f800000u, 0x3f800000u, 0x3f000000u, 0x2f800000u,
0x3f800000u};
test.opcodes = {O::V_MOV_B32, O::V_CVT_F32_U32, O::V_RCP_IFLAG_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_RCP_IFLAG_F32", 5}};
test.ir_counts = {{"FPRecipIFlag32", 5}};
test.required_spirv = {"OpConvertUToF", "OpFDiv"};
test.forbidden_spirv = {"8388608"};
return test;
}
TestCase VectorSinCosMaxFiniteSpecialCases() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xff7fffffu);
code.push_back(EncodeVop1(0x35, 1, Vgpr(0)));
code.push_back(EncodeVop1(0x36, 2, Vgpr(0)));
AppendStoreVgpr(&code, 1, 0);
AppendStoreVgpr(&code, 2, 1);
AppendEnd(&code);
return {"VectorSinCosMaxFiniteSpecialCases",
code,
{},
{0x00000000u, 0x3f800000u},
{O::V_MOV_B32, O::V_SIN_F32, O::V_COS_F32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorCompareF32ExposureGuard() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorCompareF32ExposureGuard";
test.initial = {0x00000001u, 0x80000001u, 0x007fffffu, 0x807fffffu,
0x00000000u, 0x80000000u, 0x3f800000u, 0x40000000u};
test.expected = test.initial;
AppendVMovLiteral(&test.code, 1, 0x3f800000u);
for (u32 i = 0; i < test.initial.size(); i++) {
AppendVMovU32(&test.code, 31, i * sizeof(u32));
AppendBufferLoadDword(&test.code, 0, 31);
// Captured exposure guard: replace zero with 1.0 before taking its reciprocal.
test.code.push_back(EncodeVopc(0x02, InlineU32(0), 0));
test.code.push_back(EncodeVop2(0x01, 2, Vgpr(0), 1));
test.code.push_back(EncodeVop1(0x2a, 3, Vgpr(2)));
AppendStoreVgpr(&test.code, 2, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, 3, static_cast<u32>(test.expected.size() + 1));
test.expected.push_back(i == 7 ? 0x40000000u : 0x3f800000u);
test.expected.push_back(i == 7 ? 0x3f000000u : 0x3f800000u);
}
test.initial.resize(test.expected.size());
AppendEnd(&test.code);
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_CMP_EQ_F32, O::V_CNDMASK_B32,
O::V_RCP_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase VectorCompareF32DenormalModes(u32 mode) {
using O = ShaderOpcode;
enum Relation : u32 { Less, Equal, Greater, Unordered };
struct Pair {
u32 lhs;
u32 rhs;
Relation relation;
};
constexpr Pair pairs[] = {
{0x00000001u, 0x00000000u, Equal},
{0x80000001u, 0x80000000u, Equal},
{0x00000000u, 0x00000001u, Equal},
{0x80000000u, 0x80000001u, Equal},
{0x007fffffu, 0x80000001u, Equal},
{0x00800000u, 0x007fffffu, Greater},
{0xbf800000u, 0x80000001u, Less},
{0xff800000u, 0xbf800000u, Less},
{0x7f800000u, 0x7f800000u, Equal},
{0x7fc12345u, 0x00000000u, Unordered},
{0x00000000u, 0x7fa12345u, Unordered},
{0x00000000u, 0x80000000u, Equal},
};
TestCase test;
constexpr const char* names[] = {"VectorCompareF32DenormalModeC0",
"VectorCompareF32DenormalModeE0"};
test.name = names[(mode >> 5u) & 1u];
test.compute_info.float_mode = static_cast<uint8_t>(mode);
test.has_compute_info = true;
for (const auto& pair : pairs) {
test.initial.insert(test.initial.end(), {pair.lhs, pair.rhs});
}
test.expected = test.initial;
AppendVMovU32(&test.code, 1, 1);
for (u32 i = 0; i < std::size(pairs); i++) {
AppendVMovU32(&test.code, 31, i * 2u * sizeof(u32));
AppendBufferLoadDword(&test.code, 2, 31);
AppendVMovU32(&test.code, 31, (i * 2u + 1u) * sizeof(u32));
AppendBufferLoadDword(&test.code, 3, 31);
// The sixteen architectural predicates select subsets of LT/EQ/GT/unordered.
for (u32 predicate = 0; predicate < 16; predicate++) {
test.code.push_back(EncodeVopc(predicate, Vgpr(2), 3));
test.code.push_back(EncodeVop2(0x01, 4, InlineU32(0), 1));
AppendStoreVgpr(&test.code, 4, static_cast<u32>(test.expected.size()));
test.expected.push_back((predicate >> pairs[i].relation) & 1u);
}
}
test.initial.resize(test.expected.size());
AppendEnd(&test.code);
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_CMP_EQ_F32, O::V_CMP_LE_F32,
O::V_CMP_GE_F32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase VectorCompareOps() {
using O = ShaderOpcode;
std::vector<u32> code;
std::vector<u32> results;
std::vector<u32> expected;
u32 dst = 10;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
AppendVMovLiteral(&code, 2, 0x40000000u);
AppendVMovLiteral(&code, 3, 0x40400000u);
code.push_back(EncodeVop1(0x01, 4, InlineU32(1)));
AppendVMovLiteral(&code, 5, 0xffffffffu);
auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) {
if (dst == 31u) {
dst++;
}
code.push_back(EncodeVopc(opcode, src0, src1));
code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1));
results.push_back(dst);
expected.push_back(value ? 1u : 0u);
dst++;
};
append_compare(0x00, Vgpr(2), 3, false);
append_compare(0x01, Vgpr(2), 3, true);
append_compare(0x02, Vgpr(2), 3, false);
append_compare(0x03, Vgpr(2), 3, true);
append_compare(0x04, Vgpr(2), 3, false);
append_compare(0x05, Vgpr(2), 3, true);
append_compare(0x06, Vgpr(2), 3, false);
append_compare(0x07, Vgpr(2), 3, true);
append_compare(0x08, Vgpr(2), 3, false);
append_compare(0x09, Vgpr(2), 3, true);
append_compare(0x0a, Vgpr(2), 3, false);
append_compare(0x0b, Vgpr(2), 3, true);
append_compare(0x0c, Vgpr(2), 3, false);
append_compare(0x0d, Vgpr(2), 3, true);
append_compare(0x0e, Vgpr(2), 3, false);
append_compare(0x0f, Vgpr(2), 3, true);
append_compare(0x80, Vgpr(5), 4, false);
append_compare(0x81, Vgpr(5), 4, true);
append_compare(0x82, Vgpr(5), 4, false);
append_compare(0x83, Vgpr(5), 4, true);
append_compare(0x84, Vgpr(5), 4, false);
append_compare(0x85, Vgpr(5), 4, true);
append_compare(0x86, Vgpr(5), 4, false);
append_compare(0x87, Vgpr(5), 4, true);
append_compare(0xc0, Vgpr(4), 5, false);
append_compare(0xc1, Vgpr(4), 5, true);
append_compare(0xc2, Vgpr(4), 5, false);
append_compare(0xc3, Vgpr(4), 5, true);
append_compare(0xc4, Vgpr(4), 5, false);
append_compare(0xc5, Vgpr(4), 5, true);
append_compare(0xc6, Vgpr(4), 5, false);
append_compare(0xc7, Vgpr(4), 5, true);
code.push_back(0x7c1d00f9u);
code.push_back(0x86069201u);
code.push_back(EncodeVopc(0x84, 249, 4));
code.push_back(EncodeVopcSdwa(5, 19, 1));
code.push_back(EncodeVopc(0x01, 250, 3));
code.push_back(EncodeVop2Dpp(2));
AppendVMovLiteral(&code, 6, 0xc0000000u);
AppendVMovLiteral(&code, 7, 0x3f800000u);
AppendVop3(&code, 0x04, 20, Vgpr(6), Vgpr(7), 0, 1);
for (u32 i = 0; i < static_cast<u32>(results.size()); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendStoreSgpr(&code, 18, static_cast<u32>(results.size()));
expected.push_back(1u);
AppendStoreSgpr(&code, 19, static_cast<u32>(results.size() + 1u));
expected.push_back(0u);
AppendStoreSgpr(&code, 106, static_cast<u32>(results.size() + 2u));
expected.push_back(1u);
AppendStoreSgpr(&code, 20, static_cast<u32>(results.size() + 3u));
expected.push_back(1u);
AppendEnd(&code);
return {"VectorCompareOps",
code,
{},
expected,
{O::V_MOV_B32, O::V_CNDMASK_B32, O::V_CMP_F_F32,
O::V_CMP_LT_F32, O::V_CMP_EQ_F32, O::V_CMP_LE_F32,
O::V_CMP_GT_F32, O::V_CMP_LG_F32, O::V_CMP_GE_F32,
O::V_CMP_O_F32, O::V_CMP_U_F32, O::V_CMP_NGE_F32,
O::V_CMP_NLG_F32, O::V_CMP_NGT_F32, O::V_CMP_NLE_F32,
O::V_CMP_NEQ_F32, O::V_CMP_NLT_F32, O::V_CMP_TRU_F32,
O::V_CMP_F_I32, O::V_CMP_LT_I32, O::V_CMP_EQ_I32,
O::V_CMP_LE_I32, O::V_CMP_GT_I32, O::V_CMP_NE_I32,
O::V_CMP_GE_I32, O::V_CMP_T_I32, O::V_CMP_F_U32,
O::V_CMP_LT_U32, O::V_CMP_EQ_U32, O::V_CMP_LE_U32,
O::V_CMP_GT_U32, O::V_CMP_NE_U32, O::V_CMP_GE_U32,
O::V_CMP_T_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVop3CompareEqU64OnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
code.push_back(EncodeSop1(0x04, 106, 126)); // s_mov_b64 vcc, exec
code.push_back(0xd4e2006au);
code.push_back(0x0000d47eu); // v_cmp_eq_u64 vcc, exec, vcc
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
AppendStoreVgpr(&code, 2, 0);
AppendSMovLiteral(&code, 107, 1); // high-dword mismatch
code.push_back(0xd4e2006au);
code.push_back(0x0000d47eu); // v_cmp_eq_u64 vcc, exec, vcc
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, 1);
AppendEnd(&code);
TestCase test{"VectorVop3CompareEqU64OnGpu",
code,
{},
{1, 0},
{O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_U64,
O::V_CNDMASK_B32, O::S_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
test.decoded_counts = {{"V_CMP_EQ_U64 vcc_lo, exec_lo, vcc_lo", 2}};
return test;
}
TestCase VectorVop3CompareNeU64OnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
code.push_back(EncodeSop1(0x04, 106, 126)); // s_mov_b64 vcc, exec
code.push_back(0xd4e5006au);
code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
AppendStoreVgpr(&code, 2, 0);
AppendSMovLiteral(&code, 106, 0);
AppendSMovLiteral(&code, 107, 0);
code.push_back(0xd4e5006au);
code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, 1);
AppendEnd(&code);
return {"VectorVop3CompareNeU64OnGpu",
code,
{},
{0, 1},
{O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_NE_U64, O::V_CNDMASK_B32,
O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVopcCmpxNeU64CapturedExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 7, 0);
AppendVMovU32(&code, 8, 0);
AppendVMovU32(&code, 31, 0);
code.push_back(0x7dea0e80u); // v_cmpx_ne_u64 exec, 0, v[7:8]
AppendBufferStoreDword(&code, 8, 31);
code.push_back(EncodeSMovB32(126, InlineU32(1)));
AppendVMovU32(&code, 8, 1);
AppendVMovU32(&code, 31, 4);
code.push_back(0x7dea0e80u); // v_cmpx_ne_u64 exec, 0, v[7:8]
AppendBufferStoreDword(&code, 8, 31);
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 3, 0);
AppendVMovU32(&code, 31, 8);
code.push_back(0x7dea0480u); // v_cmpx_ne_u64 exec, 0, v[2:3]
AppendBufferStoreDword(&code, 3, 31);
code.push_back(EncodeSMovB32(126, InlineU32(1)));
AppendVMovU32(&code, 3, 1);
AppendVMovU32(&code, 31, 12);
code.push_back(0x7dea0480u); // v_cmpx_ne_u64 exec, 0, v[2:3]
AppendBufferStoreDword(&code, 3, 31);
AppendEnd(&code);
TestCase test{"VectorVopcCmpxNeU64CapturedExecMask",
code,
{0x11223344u, 0x55667788u, 0xaabbccddu, 0xeeff0011u},
{0x11223344u, 1u, 0xaabbccddu, 1u},
{O::V_MOV_B32, O::V_CMPX_NE_U64, O::BUFFER_STORE_DWORD,
O::S_MOV_B32, O::S_ENDPGM}};
test.decoded_counts = {{"V_CMPX_NE_U64 exec_lo, 0, v7", 2},
{"V_CMPX_NE_U64 exec_lo, 0, v2", 2}};
return test;
}
TestCase VectorVop3CmpxNeI64CapturedExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 7, 0);
AppendVMovU32(&code, 8, 0);
AppendVMovU32(&code, 31, 0);
code.push_back(0xd4b5007eu);
code.push_back(0x00020e80u); // v_cmpx_ne_i64 exec, 0, v[7:8]
AppendBufferStoreDword(&code, 8, 31);
code.push_back(EncodeSMovB32(126, InlineU32(1)));
AppendVMovU32(&code, 8, 1);
AppendVMovU32(&code, 31, 4);
code.push_back(0xd4b5007eu);
code.push_back(0x00020e80u); // v_cmpx_ne_i64 exec, 0, v[7:8]
AppendBufferStoreDword(&code, 8, 31);
AppendEnd(&code);
TestCase test{"VectorVop3CmpxNeI64CapturedExecMask",
code,
{0x11223344u, 0x55667788u},
{0x11223344u, 1u},
{O::V_MOV_B32, O::V_CMPX_NE_I64, O::BUFFER_STORE_DWORD,
O::S_MOV_B32, O::S_ENDPGM}};
test.decoded_counts = {{"V_CMPX_NE_I64 exec_lo, 0, v7", 2}};
return test;
}
TestCase VectorVop3CompareGtU64OnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
AppendSMovLiteral(&code, 4, 0);
AppendSMovLiteral(&code, 5, 0x80000000u);
code.push_back(0xd4e4006au);
code.push_back(0x00010004u); // v_cmp_gt_u64 vcc, s[4:5], 0
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
AppendStoreVgpr(&code, 2, 0);
code.push_back(0xd4e4006au);
code.push_back(0x0001006au); // v_cmp_gt_u64 vcc, vcc, 0
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, 1);
AppendSMovLiteral(&code, 106, 0);
AppendSMovLiteral(&code, 107, 0);
code.push_back(0xd4e4006au);
code.push_back(0x0001006au); // v_cmp_gt_u64 vcc, vcc, 0
code.push_back(EncodeVop2(0x01, 4, InlineU32(0), 1));
AppendStoreVgpr(&code, 4, 2);
AppendEnd(&code);
TestCase test{"VectorVop3CompareGtU64OnGpu",
code,
{},
{1, 1, 0},
{O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_GT_U64, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorVopcCompareLtU64OnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 5);
AppendVMovU32(&code, 1, 0);
AppendVMovU32(&code, 2, 1);
AppendSMovLiteral(&code, 4, 4);
AppendSMovLiteral(&code, 5, 0);
code.push_back(EncodeVopc(0xe1, 4, 0));
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 2));
AppendStoreVgpr(&code, 3, 0);
AppendSMovLiteral(&code, 4, 6);
code.push_back(EncodeVopc(0xe1, 4, 0));
code.push_back(EncodeVop2(0x01, 4, InlineU32(0), 2));
AppendStoreVgpr(&code, 4, 1);
AppendEnd(&code);
TestCase test{"VectorVopcCompareLtU64OnGpu",
code,
{},
{1, 0},
{O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_LT_U64,
O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"V_CMP_LT_U64 vcc_lo, s4, v0", 2}};
return test;
}
TestCase VectorVop3CompareEqI64OnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
code.push_back(EncodeSop1(0x04, 4, 126)); // s_mov_b64 s[4:5], exec
code.push_back(0xd4a2006au);
code.push_back(0x0000087eu); // v_cmp_eq_i64 vcc, exec, s[4:5]
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
AppendStoreVgpr(&code, 2, 0);
AppendSMovLiteral(&code, 4, 0);
code.push_back(0xd4a2006au);
code.push_back(0x0000087eu); // low dword mismatch
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, 1);
AppendEnd(&code);
return {"VectorVop3CompareEqI64OnGpu",
code,
{},
{1, 0},
{O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_I64, O::V_CNDMASK_B32,
O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorCompareClassF32() {
using O = ShaderOpcode;
std::vector<u32> code;
std::vector<u32> expected;
AppendVMovU32(&code, 1, 1);
u32 out = 0;
auto append_class = [&](u32 bits, u32 class_mask, bool match) {
AppendVMovLiteral(&code, 0, bits);
AppendVMovU32(&code, 2, class_mask);
code.push_back(EncodeVopc(0x88, Vgpr(0), 2));
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, out++);
expected.push_back(match ? 1u : 0u);
};
append_class(0x7fc00000u, 1u << 1u, true);
append_class(0xff800000u, 1u << 2u, true);
append_class(0xbf800000u, 1u << 3u, true);
append_class(0x80000000u, 1u << 5u, true);
append_class(0x00000000u, 1u << 6u, true);
append_class(0x40000000u, 1u << 8u, true);
append_class(0x7f800000u, 1u << 9u, true);
append_class(0x40000000u, 1u << 3u, false);
append_class(0x00000001u, 1u << 7u, true);
append_class(0x80000001u, 1u << 4u, true);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovU32(&code, 2, 1u << 3u);
AppendVop3(&code, 0x88, 20, Vgpr(0), Vgpr(2), 0, 0, 0, false, 0, 1);
AppendStoreSgprPair(&code, 20, out);
expected.push_back(1u);
expected.push_back(0u);
AppendEnd(&code);
return {"VectorCompareClassF32",
code,
{},
expected,
{O::V_MOV_B32, O::V_CMP_CLASS_F32, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorCompareClassF32ImmediateAndDynamicMasks() {
using O = ShaderOpcode;
// One input for each architectural class bit, in bit order.
constexpr std::array<u32, 10> values{
0x7f800001u, 0x7fc00001u, 0xff800000u, 0xbf800000u, 0x80000001u,
0x80000000u, 0x00000000u, 0x00000001u, 0x3f800000u, 0x7f800000u};
constexpr std::array<u32, 8> masks{
0u, 0x3ffu, 3u, 7u, 0xfffffc00u, 0xffffffffu, 0xfffffc03u, 0xfffffc07u};
TestCase test;
test.name = "VectorCompareClassF32ImmediateAndDynamicMasks";
test.initial.assign(values.begin(), values.end());
test.initial.insert(test.initial.end(), masks.begin(), masks.end());
test.expected = test.initial;
auto &code = test.code;
AppendVMovU32(&code, 1, 1);
for (u32 index = 0; index < values.size(); ++index) {
AppendVMovU32(&code, 30, index * 4u);
AppendBufferLoadDword(&code, 0, 30);
for (u32 mask_index = 0; mask_index < masks.size(); ++mask_index) {
for (const bool dynamic : {false, true}) {
if (dynamic) {
AppendVMovU32(&code, 30, (values.size() + mask_index) * 4u);
AppendBufferLoadDword(&code, 2, 30);
code.push_back(EncodeVopc(0x88, Vgpr(0), 2));
} else {
AppendVop3(&code, 0x88, 106, Vgpr(0), 255);
code.push_back(masks[mask_index]);
}
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, static_cast<u32>(test.expected.size()));
test.expected.push_back((masks[mask_index] & (1u << index)) != 0u);
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_CMP_CLASS_F32,
O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpIsNan"};
return test;
}
TestCase VectorVopcSdwaCmpxClassF32CapturedExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 13, 0x7fc00000u); // Quiet NaN.
AppendVMovU32(&code, 3, 7);
AppendVMovU32(&code, 31, 0);
code.push_back(EncodeSMovB32(106, InlineU32(2))); // qNaN class mask.
code.insert(code.end(), {0x7d30d4f9u, 0x8606000du});
AppendBufferStoreDword(&code, 3, 31);
code.push_back(EncodeSMovB32(126, InlineU32(1))); // Restore lane-zero EXEC.
AppendVMovLiteral(&code, 13, 0x3f800000u); // Positive normal.
AppendVMovU32(&code, 3, 9);
AppendVMovU32(&code, 31, 4);
code.push_back(EncodeSMovB32(106, InlineU32(2))); // qNaN class mask.
code.insert(code.end(), {0x7d30d4f9u, 0x8606000du});
AppendBufferStoreDword(&code, 3, 31);
AppendEnd(&code);
TestCase test{
"VectorVopcSdwaCmpxClassF32CapturedExecMask",
code,
{0x11111111u, 0x22222222u},
{7u, 0x22222222u},
{O::S_MOV_B32, O::V_MOV_B32, O::V_CMPX_CLASS_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {
{"V_CMPX_CLASS_F32 exec_lo, v13, vcc_lo", 2}};
return test;
}
TestCase VectorCmpClassF16(u32 wave_size) {
using O = ShaderOpcode;
constexpr std::array<u32, 10> values{
0x7c01u, 0x7e01u, 0xfc00u, 0xbc00u, 0x8001u,
0x8000u, 0x0000u, 0x0001u, 0x3c00u, 0x7c00u};
constexpr u32 guard = 0x87654321u;
TestCase test;
test.name = wave_size == 64 ? "VectorCmpClassF16Wave64" : "VectorCmpClassF16Wave32";
for (const auto value : values) {
test.initial.insert(test.initial.end(), {0xa55a0000u | value, (value << 16u) | 0x7e01u});
}
test.expected = test.initial;
auto &code = test.code;
const auto compare = [&](std::initializer_list<u32> words, u32 expected,
u32 dst = 106u, u32 exec = 1u) {
AppendSMovLiteral(&code, 106, 0x12345678u);
AppendSMovLiteral(&code, 107, guard);
AppendSMovLiteral(&code, dst + 1u, guard);
code.push_back(EncodeSopc(0x06, InlineU32(expected), InlineU32(0)));
code.insert(code.end(), words);
const u32 sources[] = {dst, dst + 1u, 126, 127, 253};
for (u32 i = 0; i < std::size(sources); ++i) {
code.push_back(EncodeSMovB32(30u + i, sources[i]));
}
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 results[] = {expected, wave_size == 32 ? guard : 0u, exec, 0u, 1u - expected};
for (u32 i = 0; i < std::size(results); ++i) {
AppendStoreSgpr(&code, 30u + i, static_cast<u32>(test.expected.size()));
test.expected.push_back(results[i]);
}
if (dst != 106u) {
AppendStoreSgpr(&code, 106, static_cast<u32>(test.expected.size()));
AppendStoreSgpr(&code, 107, static_cast<u32>(test.expected.size()) + 1u);
test.expected.insert(test.expected.end(), {0x12345678u, guard});
}
};
for (u32 index = 0; index < values.size(); ++index) {
AppendVMovU32(&code, 30, index * 8u);
AppendBufferLoadDword(&code, 0, 30);
AppendVMovU32(&code, 30, index * 8u + 4u);
AppendBufferLoadDword(&code, 2, 30);
for (u32 bit = 0; bit < 12; ++bit) {
AppendVMovU32(&code, 1, bit == 11 ? 0 : 1u << bit);
const u32 expected = bit == index ? 1u : 0u;
compare({EncodeVopc(0x8f, Vgpr(0), 1)}, expected);
compare({EncodeVop3Word0(0x8f, 20), EncodeVop3Word1(Vgpr(0), Vgpr(1))},
expected, 20);
}
// SDWA selects the high half, applies -abs, and writes an explicit SGPR pair.
compare({EncodeVopc(0x8f, 249, InlineU32(56)),
EncodeVopcSdwa(2, 20, 1, 5, 6, 0, 0, 1, 1, 0, 0, 0, 1)},
index >= 3 && index <= 8 ? 1u : 0u, 20);
compare({EncodeVop3Word0(0x8f, 20, 1),
EncodeVop3Word1(Vgpr(0), InlineU32(56), 0, 0, 1)},
index >= 3 && index <= 8 ? 1u : 0u, 20);
}
AppendVMovU32(&code, 1, 1u << 8u);
compare({EncodeVopc(0x8f, 242, 1)}, 1); // Inline 1.0 becomes half 0x3c00.
compare({EncodeVopc(0x8f, 255, 1), 0xdead3c00u}, 1); // Literal low half.
compare({EncodeVop3Word0(0x8f, 20), EncodeVop3Word1(242, 248)}, 1, 20);
// Inline INV_2PI supplies half bits 0x3118, including the positive-normal class.
code.push_back(EncodeSMovB32(126, InlineU32(0)));
compare({EncodeVopc(0x8f, 242, 1)}, 0, 106, 0); // Inactive lanes clear the result.
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_CMP_EQ_U32, O::BUFFER_LOAD_DWORD,
O::V_CMP_CLASS_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMP_CLASS_F16", 264u}};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase VectorCmpxClassF16(u32 wave_size) {
using O = ShaderOpcode;
// One raw half value for each architectural class bit, in bit order.
constexpr std::array<u32, 10> values{
0x7c01u, 0x7e01u, 0xfc00u, 0xbc00u, 0x8001u,
0x8000u, 0x0000u, 0x0001u, 0x3c00u, 0x7c00u};
TestCase test;
test.name = wave_size == 64 ? "VectorCmpxClassF16Wave64" : "VectorCmpxClassF16Wave32";
for (const auto value : values) {
test.initial.insert(test.initial.end(), {0xa55a0000u | value, (value << 16u) | 0x7e01u});
}
test.expected = test.initial;
auto &code = test.code;
AppendSMovLiteral(&code, 106, 0x12345678u);
AppendSMovLiteral(&code, 107, 0x87654321u);
code.push_back(EncodeSMovB32(127, InlineU32(0)));
const auto compare = [&](std::initializer_list<u32> words, u32 expected) {
code.insert(code.end(), words);
code.push_back(EncodeSMovB32(20, 126));
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
AppendStoreSgpr(&code, 20, static_cast<u32>(test.expected.size()));
AppendStoreSgpr(&code, 21, static_cast<u32>(test.expected.size()) + 1u);
test.expected.insert(test.expected.end(), {expected, 0});
};
for (u32 index = 0; index < values.size(); ++index) {
AppendVMovU32(&code, 30, index * 8u);
AppendBufferLoadDword(&code, 0, 30);
AppendVMovU32(&code, 30, index * 8u + 4u);
AppendBufferLoadDword(&code, 2, 30);
for (u32 bit = 0; bit < 12; ++bit) {
AppendVMovU32(&code, 1, bit == 11 ? 0 : 1u << bit);
const u32 expected = bit == index ? 1 : 0;
if ((bit & 1u) == 0) {
compare({EncodeVopc(0x9f, Vgpr(0), 1)}, expected);
} else {
compare({EncodeVop3Word0(0x9f, 126), EncodeVop3Word1(Vgpr(0), Vgpr(1))}, expected);
}
}
const u32 finite = index >= 3 && index <= 8 ? 1 : 0;
compare({0x7d3f70f9u, 0x86360000u}, finite); // Captured -abs(v0), inline56.
compare({0x7d3f70f9u, 0x86350002u}, finite); // Captured -abs(v2.word1).
}
AppendVMovU32(&code, 1, 1u << 8u);
compare({EncodeVopc(0x9f, 242, 1)}, 1); // Inline1.0 encodes half0x3c00.
AppendVMovU32(&code, 1, 1u << 6u);
compare({EncodeVopc(0x9f, 249, 1),
EncodeVopcSdwa(242, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 1)}, 1);
AppendVMovU32(&code, 1, 1u << 7u);
compare({EncodeVopc(0x9f, 249, 1),
EncodeVopcSdwa(242, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 1)}, 1);
AppendVMovU32(&code, 30, 8u * 8u);
AppendBufferLoadDword(&code, 0, 30);
compare({EncodeVopc(0x9f, 249, 248),
EncodeVopcSdwa(0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 1)}, 1);
// INV_2PI as a half class mask is0x3118, including positive normal bit8.
AppendVMovU32(&code, 1, 1u << 8u);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
compare({EncodeVopc(0x9f, Vgpr(0), 1)}, 0);
AppendStoreSgpr(&code, 106, static_cast<u32>(test.expected.size()));
AppendStoreSgpr(&code, 107, static_cast<u32>(test.expected.size()) + 1u);
test.expected.insert(test.expected.end(), {0x12345678u, 0x87654321u});
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMPX_CLASS_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Wave32VccMasksPreserveOtherHalf() {
using O = ShaderOpcode;
TestCase test;
test.name = "Wave32VccMasksPreserveOtherHalf";
auto &code = test.code;
AppendSMovLiteral(&code, 106, 0x12345678u);
AppendSMovLiteral(&code, 107, 0x2468ace0u);
AppendVMovU32(&code, 1, 20);
u32 offset = 0;
const auto check_masks = [&](u32 low, u32 high) {
AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, offset);
AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, offset + 4);
AppendVop3(&code, 0x101, 2, InlineU32(10), Vgpr(1), 107);
code.push_back(EncodeVop2(0x01, 3, InlineU32(10), 1));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset + 8);
AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, offset + 12);
test.expected.insert(test.expected.end(), 4, low);
test.expected.insert(test.expected.end(), 4, high);
for (u32 mask : {high, low}) {
for (u32 lane = 0; lane < 4; ++lane) {
test.expected.push_back((mask & (1u << lane)) != 0 ? 20u : 10u);
}
}
offset += 16;
};
// PPSA10737 keeps a table offset in VCC_LO while comparing into VCC_HI.
AppendVop3(&code, 0xc2, 107, InlineU32(1), Vgpr(0));
check_masks(0x12345678u, 2u);
AppendVop3(&code, 0xc2, 106, InlineU32(2), Vgpr(0));
check_masks(4u, 2u);
code.push_back(EncodeSop2(0x10, 106, 106, 107)); // s_or_b32
check_masks(6u, 2u);
code.push_back(EncodeSop1(0x04, 106, 193u)); // Explicit s_mov_b64 writes both.
check_masks(0xffffffffu, 0xffffffffu);
AppendEnd(&code);
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_OR_B32, O::V_MOV_B32,
O::V_CMP_EQ_U32, O::V_CNDMASK_B32, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase VectorCompareF16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
std::vector<u32> expected;
AppendVMovU32(&code, 1, 1);
AppendVMovLiteral(&code, 2, 0x40003c00u); // low=1.0h, high=2.0h
AppendVMovLiteral(&code, 3, 0x44004200u); // low=3.0h, high=4.0h
AppendVMovLiteral(&code, 4, 0x00007e00u); // low=qNaN
u32 dst = 10;
u32 out = 0;
auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) {
code.push_back(EncodeVopc(opcode, src0, src1));
code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1));
AppendStoreVgpr(&code, dst, out++);
expected.push_back(value ? 1u : 0u);
dst++;
};
append_compare(0xc9, Vgpr(2), 3, true);
append_compare(0xca, Vgpr(2), 2, true);
append_compare(0xcb, Vgpr(2), 3, true);
append_compare(0xcc, Vgpr(3), 2, true);
append_compare(0xcd, Vgpr(2), 3, true);
append_compare(0xce, Vgpr(3), 2, true);
append_compare(0xe9, Vgpr(2), 3, true);
append_compare(0xe9, Vgpr(3), 2, false);
append_compare(0xe9, Vgpr(2), 2, false);
append_compare(0xe9, Vgpr(4), 2, true);
append_compare(0xe9, Vgpr(2), 4, true);
append_compare(0xed, Vgpr(4), 2, true);
auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) {
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeVopc(opcode, src0, src1));
AppendStoreVgpr(&code, 1, out++);
expected.push_back(1u);
};
append_cmpx(0xd9, Vgpr(2), 3);
append_cmpx(0xda, Vgpr(2), 2);
append_cmpx(0xdb, Vgpr(2), 3);
append_cmpx(0xdc, Vgpr(3), 2);
append_cmpx(0xde, Vgpr(3), 2);
append_cmpx(0xfd, Vgpr(4), 2);
append_cmpx(0xfe, Vgpr(2), 2);
AppendEnd(&code);
return {"VectorCompareF16Ops",
code,
{},
expected,
{O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_LT_F16, O::V_CMP_EQ_F16,
O::V_CMP_LE_F16, O::V_CMP_GT_F16, O::V_CMP_LG_F16, O::V_CMP_GE_F16,
O::V_CMP_NGE_F16, O::V_CMP_NEQ_F16, O::V_CMPX_LT_F16, O::V_CMPX_EQ_F16,
O::V_CMPX_LE_F16, O::V_CMPX_GT_F16, O::V_CMPX_GE_F16,
O::V_CMPX_NEQ_F16, O::V_CMPX_NLT_F16, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorCompareI16U16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
std::vector<u32> expected;
AppendVMovU32(&code, 1, 1);
AppendVMovLiteral(&code, 2, 0xabcd8000u);
AppendVMovLiteral(&code, 3, 0x00007fffu);
AppendVMovLiteral(&code, 4, 0x12348000u);
AppendVMovLiteral(&code, 5, 0xffff0001u);
AppendVMovLiteral(&code, 6, 0x00000001u);
u32 dst = 10;
u32 out = 0;
auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) {
code.push_back(EncodeVopc(opcode, src0, src1));
code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1));
AppendStoreVgpr(&code, dst, out++);
expected.push_back(value ? 1u : 0u);
dst++;
};
append_compare(0x8a, Vgpr(2), 4, true);
append_compare(0x8a, Vgpr(2), 3, false);
append_compare(0x8b, Vgpr(2), 3, true);
append_compare(0x8b, Vgpr(3), 2, false);
append_compare(0x8b, Vgpr(5), 6, true);
append_compare(0x8c, Vgpr(3), 2, true);
append_compare(0x8c, Vgpr(2), 3, false);
append_compare(0x8d, Vgpr(2), 3, true);
append_compare(0x8d, Vgpr(5), 6, false);
append_compare(0xaa, Vgpr(2), 4, true);
append_compare(0xaa, Vgpr(2), 3, false);
append_compare(0xab, Vgpr(3), 2, true);
append_compare(0xab, Vgpr(2), 3, false);
append_compare(0xac, Vgpr(2), 3, true);
append_compare(0xac, Vgpr(3), 2, false);
append_compare(0xad, Vgpr(2), 3, true);
append_compare(0xad, Vgpr(5), 6, false);
append_compare(0xae, Vgpr(2), 3, true);
append_compare(0xae, Vgpr(3), 2, false);
append_compare(0xae, Vgpr(5), 6, true);
AppendEnd(&code);
return {"VectorCompareI16U16Ops",
code,
{},
expected,
{O::V_MOV_B32, O::V_CMP_EQ_I16, O::V_CMP_LE_I16, O::V_CMP_GT_I16,
O::V_CMP_NE_I16, O::V_CMP_EQ_U16, O::V_CMP_LE_U16, O::V_CMP_GT_U16,
O::V_CMP_NE_U16, O::V_CMP_GE_U16, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop2SdwaCndmaskSourceModifier() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 6, 0x00000000u);
AppendVMovLiteral(&code, 47, 0x12345678u);
AppendVMovLiteral(&code, 53, 0x3f800000u);
code.push_back(EncodeVopc(0xc7, Vgpr(6), 6)); // v_cmp_t_u32
code.push_back(0x025e6af9u);
code.push_back(0x16060635u);
AppendStoreVgpr(&code, 47, 0);
AppendEnd(&code);
return {"Vop2SdwaCndmaskSourceModifier",
code,
{},
{0xbf800000u},
{O::V_MOV_B32, O::V_CMP_T_U32, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop2SdwaCndmaskCapturedByte3Source() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x55667788u);
AppendBufferLoadDword(&code, 3, 30);
code.push_back(EncodeVopc(0xc0, Vgpr(0), 0)); // v_cmp_f_u32
code.push_back(0x023a00f9u);
code.push_back(0x06030603u);
AppendStoreVgpr(&code, 29, 1);
code.push_back(EncodeVopc(0xc7, Vgpr(0), 0)); // v_cmp_t_u32
code.push_back(0x023a00f9u);
code.push_back(0x06030603u);
AppendStoreVgpr(&code, 29, 2);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaCndmaskCapturedByte3Source";
test.code = code;
test.initial = {0xabcd1234u, 0, 0};
test.expected = {0xabcd1234u, 0x000000abu, 0x55667788u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_CMP_F_U32,
O::V_CMP_T_U32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {{"V_CNDMASK_B32 v29, v3.sdwa(sel=3,sext=0), v0", 2}};
test.required_spirv = {"OpBitFieldUExtract", "OpSelect"};
return test;
}
TestCase Vop3CndmaskUsesSgprMaskLaneBits() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0), EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1), EncodeVopc(0xc2, InlineU32(0), 0),
EncodeSop1(0x04, 4, 106),
};
AppendVop3(&code, 0x101, 2, InlineU32(10), InlineU32(20), 4);
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 1));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "Vop3CndmaskUsesSgprMaskLaneBits";
test.code = code;
test.expected = {20, 10, 10, 10, 20, 10, 10, 10};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase Vop3CndmaskAllowsDataSourceModifier() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 7, 0x3f800000u);
AppendSMovLiteral(&code, 34, 1);
AppendSMovLiteral(&code, 35, 0);
code.push_back(0xd5010004u);
code.push_back(0x408a0f07u);
AppendStoreVgpr(&code, 4, 0);
AppendEnd(&code);
return {"Vop3CndmaskAllowsDataSourceModifier",
code,
{},
{0xbf800000u},
{O::V_MOV_B32, O::S_MOV_B32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorCompareExecOps() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
AppendVMovLiteral(&code, 2, 0x40000000u);
AppendVMovLiteral(&code, 3, 0x40400000u);
code.push_back(EncodeVop1(0x01, 4, InlineU32(1)));
AppendVMovLiteral(&code, 5, 0xffffffffu);
u32 out = 0;
auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) {
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeVopc(opcode, src0, src1));
AppendStoreVgpr(&code, 1, out++);
};
append_cmpx(0x11, Vgpr(2), 3);
append_cmpx(0x12, Vgpr(3), 3);
append_cmpx(0x13, Vgpr(2), 3);
append_cmpx(0x14, Vgpr(3), 2);
append_cmpx(0x15, Vgpr(2), 3);
append_cmpx(0x16, Vgpr(3), 3);
append_cmpx(0x19, Vgpr(2), 3);
append_cmpx(0x1a, Vgpr(3), 3);
append_cmpx(0x1b, Vgpr(2), 3);
append_cmpx(0x1c, Vgpr(3), 2);
append_cmpx(0x1d, Vgpr(2), 3);
append_cmpx(0x1e, Vgpr(3), 3);
append_cmpx(0x91, Vgpr(5), 4);
append_cmpx(0x92, Vgpr(4), 4);
append_cmpx(0x93, Vgpr(5), 4);
append_cmpx(0x94, Vgpr(4), 5);
append_cmpx(0x95, Vgpr(5), 4);
append_cmpx(0x96, Vgpr(4), 4);
append_cmpx(0xd1, Vgpr(4), 5);
append_cmpx(0xd2, Vgpr(4), 4);
append_cmpx(0xd3, Vgpr(4), 5);
append_cmpx(0xd4, Vgpr(5), 4);
append_cmpx(0xd5, Vgpr(4), 5);
append_cmpx(0xd6, Vgpr(4), 4);
AppendEnd(&code);
return {"VectorCompareExecOps",
code,
{},
std::vector<u32>(out, 1),
{O::S_MOV_B32, O::V_MOV_B32, O::V_CMPX_LT_F32,
O::V_CMPX_EQ_F32, O::V_CMPX_LE_F32, O::V_CMPX_GT_F32,
O::V_CMPX_LG_F32, O::V_CMPX_GE_F32, O::V_CMPX_NGE_F32,
O::V_CMPX_NLG_F32, O::V_CMPX_NGT_F32, O::V_CMPX_NLE_F32,
O::V_CMPX_NEQ_F32, O::V_CMPX_NLT_F32, O::V_CMPX_LT_I32,
O::V_CMPX_EQ_I32, O::V_CMPX_LE_I32, O::V_CMPX_GT_I32,
O::V_CMPX_NE_I32, O::V_CMPX_GE_I32, O::V_CMPX_LT_U32,
O::V_CMPX_EQ_U32, O::V_CMPX_LE_U32, O::V_CMPX_GT_U32,
O::V_CMPX_NE_U32, O::V_CMPX_GE_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorVopcCmpxOrderedCapturedExecMask() {
using O = ShaderOpcode;
struct CompareCase {
u32 lhs;
u32 rhs;
u32 initial_exec;
u32 expected_exec;
bool captured;
};
constexpr std::array<CompareCase, 8> cases{{
{0x3f800000u, 0x3f800000u, 1u, 1u, true}, // Finite value.
{0x7fc00000u, 0x7fc00000u, 1u, 0u, true}, // Quiet NaN.
{0x7f800001u, 0x7f800001u, 1u, 0u, true}, // Signaling NaN.
{0x7f800000u, 0x7f800000u, 1u, 1u, true}, // Infinity is ordered.
{0x80000000u, 0x80000000u, 1u, 1u, true}, // Signed zero is ordered.
{0x3f800000u, 0x3f800000u, 0u, 0u, true}, // Inactive lane.
{0x3f800000u, 0x7fc00000u, 1u, 0u, false}, // NaN in SRC1.
{0x7fc00000u, 0x3f800000u, 1u, 0u, false}, // NaN in SRC0.
}};
constexpr u32 vcc_lo = 0x13579bdfu;
constexpr u32 vcc_hi = 0x2468ace0u;
TestCase test;
test.name = "VectorVopcCmpxOrderedCapturedExecMask";
auto &code = test.code;
for (const auto &entry : cases) {
test.initial.push_back(entry.lhs);
test.initial.push_back(entry.rhs);
}
test.expected = test.initial;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 8u);
AppendBufferLoadDword(&code, 1, 30);
AppendVMovU32(&code, 30, i * 8u + 4u);
AppendBufferLoadDword(&code, 2, 30);
AppendSMovLiteral(&code, 106, vcc_lo);
AppendSMovLiteral(&code, 107, vcc_hi);
code.push_back(EncodeSMovB32(126, InlineU32(entry.initial_exec)));
code.push_back(entry.captured ? 0x7c2e0301u
: EncodeVopc(0x17u, Vgpr(1), 2));
code.push_back(EncodeSMovB32(20, 126));
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(22, 106));
code.push_back(EncodeSMovB32(23, 107));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = static_cast<u32>(cases.size()) * 2u + i * 4u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 22, out + 2u);
test.expected.insert(test.expected.end(),
{entry.expected_exec, 0u, vcc_lo, vcc_hi});
}
test.initial.resize(test.expected.size());
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMPX_O_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMPX_O_F32 exec_lo, v1, v1", 6u},
{"V_CMPX_O_F32 exec_lo, v1, v2", 2u}};
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase VectorVop3FloatCompareNegSourceModifier() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovU32(&code, 1, 0);
AppendVop3(&code, 0x01, 20, Vgpr(0), Vgpr(1), 0, 0, 0, false, 0,
0x1); // -1.0 < 0.0
AppendStoreSgprPair(&code, 20, 0);
AppendEnd(&code);
return {"VectorVop3FloatCompareNegSourceModifier",
code,
{},
{1, 0},
{O::V_MOV_B32, O::V_CMP_LT_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVop3CmpxWritesExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 2);
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 30, 0);
AppendVop3(&code, 0xd1, 5, Vgpr(0), Vgpr(1)); // v_cmpx_lt_u32, false
AppendVMovU32(&code, 2, 7);
AppendBufferStoreDword(&code, 2, 30);
AppendEnd(&code);
return {"VectorVop3CmpxWritesExecMask",
code,
{0},
{0},
{O::V_MOV_B32, O::V_CMPX_LT_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVopcSdwaCmpxWritesExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 2);
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 30, 0);
code.push_back(EncodeVopc(0xd1, 249, 1)); // v_cmpx_lt_u32.sdwa, false
code.push_back(EncodeVopcSdwa(0));
AppendVMovU32(&code, 2, 7);
AppendBufferStoreDword(&code, 2, 30);
AppendEnd(&code);
return {"VectorVopcSdwaCmpxWritesExecMask",
code,
{0},
{0},
{O::V_MOV_B32, O::V_CMPX_LT_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVopcCmpxGtU16CapturedSdwaExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 2, 0x00008000u);
AppendVMovU32(&code, 3, 7);
AppendVMovU32(&code, 31, 0);
code.insert(code.end(), {0x7d7900f9u, 0x86060002u});
AppendBufferStoreDword(&code, 3, 31);
AppendVMovU32(&code, 2, 0x00010000u);
AppendVMovU32(&code, 3, 9);
AppendVMovU32(&code, 31, 4);
code.insert(code.end(), {0x7d7900f9u, 0x86060002u});
AppendBufferStoreDword(&code, 3, 31);
AppendEnd(&code);
TestCase test{"VectorVopcCmpxGtU16CapturedSdwaExecMask",
code,
{0x11111111u, 0x22222222u},
{7, 0x22222222u},
{O::V_MOV_B32, O::V_CMPX_GT_U16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
test.decoded_counts = {{"V_CMPX_GT_U16", 2}};
return test;
}
TestCase VectorVopcCmpxLtU16CapturedSdwaExecMask() {
using O = ShaderOpcode;
struct CompareCase {
u32 lhs;
u32 rhs;
u32 incoming_exec;
u32 expected_exec;
u32 encoding = 0; // Captured SDWA, compact, VOP3, WORD_1 src0/src1.
};
const std::array<CompareCase, 12> cases{{
{0xffff0000u, 0x12340001u, 1, 1}, // Ignore high halves.
{0xabcd0001u, 0x12340001u, 1, 0}, // Strict comparison: equal is false.
{0x0000ffffu, 0x12340001u, 1, 0}, // Unsigned, not signed -1.
{0x00007fffu, 0x12348000u, 1, 1},
{0x00008000u, 0x12347fffu, 1, 0},
{0x12340000u, 0x56780000u, 1, 0},
{0x0000fffeu, 0x1234ffffu, 1, 1},
{0x00000000u, 0x12340001u, 0, 0}, // CMPX cannot reactivate an inactive lane.
{0x00008000u, 0x0000ffffu, 1, 1, 1},
{0x0000ffffu, 0x00008000u, 1, 0, 2},
{0x0001ffffu, 0x12340002u, 1, 1, 3},
{0x00000001u, 0x00020000u, 1, 1, 4},
}};
constexpr u32 vcc_hi = 0x89abcdefu;
TestCase test;
test.name = "VectorVopcCmpxLtU16CapturedSdwaExecMask";
for (const auto &entry : cases) {
test.initial.push_back(entry.lhs);
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 0, 30); // Runtime input prevents constant folding.
AppendVMovU32(&code, 1, entry.rhs);
AppendSMovLiteral(&code, 106, entry.rhs);
AppendSMovLiteral(&code, 107, vcc_hi);
code.push_back(EncodeSMovB32(126, InlineU32(entry.incoming_exec)));
switch (entry.encoding) {
case 1: code.push_back(0x7d720300u); break; // compact v0, v1
case 2: code.insert(code.end(), {0xd4b9007eu, 0x00020300u}); break;
case 3: code.insert(code.end(), {0x7d72d4f9u, 0x86050000u}); break;
case 4: code.insert(code.end(), {0x7d72d4f9u, 0x85060000u}); break;
default: code.insert(code.end(), {0x7d72d4f9u, 0x86060000u}); break;
}
code.push_back(EncodeSMovB32(20, 126)); // Snapshot EXEC before restoring it.
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = static_cast<u32>(cases.size()) + i * 4u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 106, out + 2u); // CMPX must not overwrite either VCC half.
test.expected.insert(test.expected.end(),
{entry.expected_exec, 0u, entry.rhs, vcc_hi});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMPX_LT_U16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMPX_LT_U16", cases.size()}};
test.required_spirv = {"OpULessThan"};
return test;
}
TestCase VectorVopcCmpxLtI16CapturedSdwaExecMask() {
using O = ShaderOpcode;
enum Encoding { Captured, Compact, Vop3, HighLhs, HighRhs, SdwaOperands };
struct CompareCase {
u32 lhs;
u32 rhs;
u32 expected_exec;
Encoding encoding = Captured;
u32 incoming_exec = 1;
u32 src0_sel = 6;
u32 src1_sel = 6;
u32 src0_sext = 0;
u32 src1_sext = 0;
};
const std::array<CompareCase, 30> cases{{
{0xffff000eu, 15u, 1}, // Ignore the high half: 14 < 15.
{0x8000000fu, 15u, 0}, // Equality is false despite a negative high half.
{0x00000010u, 15u, 0},
{0x7fff0000u, 15u, 1},
{0x0000ffffu, 15u, 1}, // Signed -1, not unsigned 65535.
{0x00008000u, 15u, 1}, // Minimum signed halfword.
{0x00007fffu, 15u, 0}, // Maximum signed halfword.
{0x0000fff0u, 15u, 1},
{0x0000ffffu, 15u, 0, Captured, 0}, // Cannot reactivate an inactive lane.
{0xffff000fu, 15u, 0, Captured, 0},
{0x1234ffffu, 0xabcd0000u, 1, Compact}, // -1 < 0.
{0xffff0000u, 0x0000ffffu, 0, Compact}, // 0 < -1 is false.
{0xabcd8000u, 0x12347fffu, 1, Vop3},
{0x12347fffu, 0xabcd8000u, 0, Vop3},
{0xffff0010u, 15u, 1, HighLhs}, // Select -1 in WORD_1.
{0x0010ffffu, 15u, 0, HighLhs}, // Select 16, not the negative low half.
{0x1234fffeu, 0xffff8000u, 1, HighRhs}, // -2 < selected -1.
{0x1234ffffu, 0xabcd0000u, 1, SdwaOperands, 1, 4, 4, 1, 1},
{0x12345680u, 0u, 1, SdwaOperands, 1, 0, 6, 1, 0}, // BYTE_0 SEXT: -128.
{0x1234567fu, 0u, 0, SdwaOperands, 1, 0, 6, 1, 0}, // +127.
{0x12345680u, 0u, 0, SdwaOperands, 1, 0, 6, 0, 0}, // Without SEXT: +128.
{0x12348000u, 0u, 1, SdwaOperands, 1, 1, 6, 1, 0},
{0x12803456u, 0u, 1, SdwaOperands, 1, 2, 6, 1, 0},
{0x80345678u, 0u, 1, SdwaOperands, 1, 3, 6, 1, 0},
{0x0000ffffu, 0x12345680u, 0, SdwaOperands, 1, 6, 0, 0, 1}, // -1 < -128 is false.
{0x0000ff7fu, 0x12345680u, 1, SdwaOperands, 1, 6, 0, 0, 1}, // -129 < -128.
{0x0000ffffu, 0x12347f00u, 1, SdwaOperands, 1, 6, 1, 0, 1},
{0u, 0x12803456u, 0, SdwaOperands, 1, 6, 2, 0, 1},
{0u, 0x80345678u, 0, SdwaOperands, 1, 6, 3, 0, 1},
{0xff000001u, 0x000100ffu, 1, SdwaOperands, 1, 3, 2, 1, 1},
}};
constexpr u32 vcc_lo = 0x76543210u;
constexpr u32 vcc_hi = 0x89abcdefu;
TestCase test;
test.name = "VectorVopcCmpxLtI16CapturedSdwaExecMask";
for (const auto &entry : cases) {
test.initial.push_back(entry.lhs);
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 1, 30); // Runtime input prevents constant folding.
AppendVMovU32(&code, 2, entry.rhs);
AppendSMovLiteral(&code, 106, vcc_lo);
AppendSMovLiteral(&code, 107, vcc_hi);
code.push_back(EncodeSMovB32(126, InlineU32(entry.incoming_exec)));
switch (entry.encoding) {
case Compact: code.push_back(0x7d320501u); break;
case Vop3: code.insert(code.end(), {0xd499007eu, 0x00020501u}); break;
case HighLhs: code.insert(code.end(), {0x7d331ef9u, 0x86050001u}); break;
case HighRhs:
code.push_back(EncodeVopc(0x99u, 249u, 2u));
code.push_back(EncodeVopcSdwa(1u, 0u, 0u, 6u, 5u));
break;
case SdwaOperands:
code.push_back(EncodeVopc(0x99u, 249u, 2u));
code.push_back(EncodeVopcSdwa(1u, 0u, 0u, entry.src0_sel, entry.src1_sel,
entry.src0_sext, entry.src1_sext));
break;
default: code.insert(code.end(), {0x7d331ef9u, 0x86060001u}); break;
}
code.push_back(EncodeSMovB32(20, 126)); // Snapshot EXEC before restoring it.
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = static_cast<u32>(cases.size()) + i * 4u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 106, out + 2u); // CMPX preserves both VCC halves.
test.expected.insert(test.expected.end(),
{entry.expected_exec, 0u, vcc_lo, vcc_hi});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMPX_LT_I16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMPX_LT_I16", cases.size()}};
test.required_spirv = {"OpSLessThan", "OpBitFieldSExtract"};
return test;
}
TestCase VectorVopcCmpxLtI16WaveMasks(u32 wave_size) {
using O = ShaderOpcode;
constexpr u32 marker = 0xfeedabcdu;
constexpr u32 vcc_lo = 0x01234567u;
constexpr u32 vcc_hi = 0x89abcdefu;
constexpr u32 expected_exec_lo = 0xaaaaaaaau;
const u32 expected_exec_hi = wave_size == 64u ? 0x33333333u : 0u;
TestCase test;
test.name = wave_size == 32u ? "VectorVopcCmpxLtI16Wave32Masks"
: "VectorVopcCmpxLtI16Wave64Masks";
test.initial.assign(6u * wave_size, 0xdeadbeefu);
constexpr u32 inputs[] = {0xffff000eu, 0x0000ffffu, 0x8000000fu, 0x12348000u};
for (u32 lane = 0; lane < wave_size; ++lane) {
test.initial[lane] = inputs[lane % 4u];
}
test.expected = test.initial;
for (u32 lane = 0; lane < wave_size; ++lane) {
test.expected[wave_size + lane] = expected_exec_lo;
test.expected[2u * wave_size + lane] = expected_exec_hi;
test.expected[3u * wave_size + lane] = vcc_lo;
test.expected[4u * wave_size + lane] = vcc_hi;
const u32 mask = lane < 32u ? expected_exec_lo : expected_exec_hi;
if ((mask & (1u << (lane % 32u))) != 0u) {
test.expected[5u * wave_size + lane] = marker;
}
}
auto &code = test.code;
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
AppendBufferLoadDword(&code, 1, 30);
AppendVMovLiteral(&code, 4, marker);
code.push_back(EncodeSop1(0x04, 10, 126)); // Save the full incoming EXEC.
AppendSMovLiteral(&code, 106, vcc_lo);
AppendSMovLiteral(&code, 107, vcc_hi);
AppendSMovLiteral(&code, 126, 0xeeeeeeeeu);
AppendSMovLiteral(&code, 127, 0x77777777u);
code.insert(code.end(), {0x7d331ef9u, 0x86060001u});
code.push_back(EncodeSMovB32(20, 126));
code.push_back(EncodeSMovB32(21, 127));
AppendStoreVgprAtLaneDwordOffset(&code, 4, 0, 5u * wave_size);
code.push_back(EncodeSop1(0x04, 126, 10)); // Restore lanes to inspect the raw masks.
AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, wave_size);
AppendStoreSgprAtLaneDwordOffset(&code, 21, 0, 2u * wave_size);
AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, 3u * wave_size);
AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, 4u * wave_size);
AppendEnd(&code);
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD, O::V_CMPX_LT_I16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpSLessThan", "OpBitFieldSExtract"};
return test;
}
TestCase VectorVopcCmpxU16SdwaCompactVop3ExecMask(bool not_equal) {
using O = ShaderOpcode;
struct CompareCase {
u32 lhs;
u32 rhs;
u32 incoming_exec;
u32 expected_exec;
u32 encoding = 0; // SDWA, compact, VOP3, sign-extended words/bytes, literal.
};
const std::array<CompareCase, 21> cases{{
{0x12340000u, 0x56780000u, 1, 1}, // Equal low half; ignore high half.
{0xffff0001u, 0xabcd0001u, 1, 1},
{0x00000000u, 0x12340000u, 1, 1},
{0xaaaa5555u, 0xbbbb5555u, 1, 1},
{0x00000000u, 0x00000001u, 1, 0},
{0x0000ffffu, 0x00000000u, 1, 0},
{0x7fff8000u, 0x00018000u, 1, 1}, // Equal low halves despite different high halves.
{0x00000000u, 0x00010000u, 1, 1},
{0x12340001u, 0x12340002u, 1, 0, 1},
{0x00008000u, 0x0000ffffu, 1, 0, 2},
{0x0001ffffu, 0x12340001u, 1, 0, 3}, // Sign-extended low halves differ.
{0x00000001u, 0x00010001u, 0, 0}, // CMPX cannot reactivate an inactive lane.
{0x12345680u, 0x0000ff80u, 1, 1, 4}, // SEXT BYTE_0 is unsigned U16 0xff80.
{0x123456ffu, 0x0000ffffu, 1, 1, 4},
{0x0000ff80u, 0x12348000u, 1, 1, 5}, // SEXT BYTE_1 on src1.
{0x1234ffffu, 0x5678ffffu, 1, 1, 6}, // SEXT WORD_0 versus DWORD low half.
{0xabcd0001u, 1u, 1, 1, 7}, // Literal source, equal low halves.
{0xabcd0000u, 1u, 1, 0, 7},
{0x00000000u, 1u, 0, 0}, // Neither EQ nor NE may reactivate an inactive lane.
{0x00010000u, 0xffff0001u, 1, 1, 8}, // Select the high word of src0.
{0xabcd0002u, 0x0002ffffu, 1, 1, 9}, // Select the high word of src1.
}};
const u32 opcode = not_equal ? 0xbdu : 0xbau;
constexpr u32 vcc_hi = 0x89abcdefu;
TestCase test;
test.name = not_equal ? "VectorVopcCmpxNeU16SdwaCompactVop3ExecMask"
: "VectorVopcCmpxEqU16SdwaCompactVop3ExecMask";
for (const auto &entry : cases) {
test.initial.push_back(entry.lhs);
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 0, 30); // Runtime input prevents constant folding.
AppendVMovU32(&code, 1, entry.rhs);
AppendSMovLiteral(&code, 106, entry.rhs);
AppendSMovLiteral(&code, 107, vcc_hi);
code.push_back(EncodeSMovB32(126, InlineU32(entry.incoming_exec)));
switch (entry.encoding) {
case 1: code.push_back(EncodeVopc(opcode, Vgpr(0), 1u)); break;
case 2: AppendVop3(&code, opcode, 126u, Vgpr(0), Vgpr(1)); break;
case 3:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 6u, 6u, 1u, 1u));
break;
case 4:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 0u, 6u, 1u, 0u));
break;
case 5:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 6u, 1u, 0u, 1u));
break;
case 6:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 4u, 6u, 1u, 0u));
break;
case 7:
code.push_back(EncodeVopc(opcode, 255u, 0u));
code.push_back(entry.rhs);
break;
case 8:
case 9:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u, 0u, 0u, entry.encoding == 8u ? 5u : 6u,
entry.encoding == 9u ? 5u : 6u));
break;
default:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u));
break;
}
code.push_back(EncodeSMovB32(20, 126)); // Snapshot EXEC before restoring it.
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(126, InlineU32(1u)));
const u32 out = static_cast<u32>(cases.size()) + i * 4u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 106, out + 2u); // CMPX must not overwrite either VCC half.
const u32 expected_exec = not_equal ? entry.incoming_exec & (entry.expected_exec ^ 1u)
: entry.expected_exec;
test.expected.insert(test.expected.end(), {expected_exec, 0u, entry.rhs, vcc_hi});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
not_equal ? O::V_CMPX_NE_U16 : O::V_CMPX_EQ_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{not_equal ? "V_CMPX_NE_U16" : "V_CMPX_EQ_U16", cases.size()}};
test.required_spirv = {not_equal ? "OpINotEqual" : "OpIEqual"};
return test;
}
TestCase VectorVopcCmpxEqU16SdwaCompactVop3ExecMask() {
return VectorVopcCmpxU16SdwaCompactVop3ExecMask(false);
}
TestCase VectorVopcCmpxNeU16SdwaCompactVop3ExecMask() {
return VectorVopcCmpxU16SdwaCompactVop3ExecMask(true);
}
TestCase VectorVopcCmpNgtF16CapturedSdwaAndEdges() {
using O = ShaderOpcode;
enum Encoding { Captured, Compact, Vop3NegAbs, HighWord, ScalarDst, SdwaNegAbs };
struct CompareCase {
u32 lhs;
u32 rhs;
u32 expected;
Encoding encoding = Captured;
u32 exec = 1;
};
const std::array<CompareCase, 17> cases{{
{0x7e003c00u, 0, 0}, // +1; ignore NaN in the upper half.
{0x3c00bc00u, 0, 1}, // -1
{0x3c000000u, 0, 1}, // +0
{0x3c008000u, 0, 1}, // -0
{0x3c007e00u, 0, 1}, // Quiet NaN: NGT differs from ordered LE.
{0x3c007c01u, 0, 1}, // Signaling NaN
{0x00007c00u, 0, 0}, // +infinity
{0x0000fc00u, 0, 1}, // -infinity
{0x0000bc00u, 0, 0, Captured, 0}, // Inactive lane contributes no VCC bit.
{0x00003c00u, 0x00007e00u, 1, Compact}, // NaN in src1.
{0x00004000u, 0x00003c00u, 0, Compact}, // +2 > +1
{0x0000bc00u, 0, 1, Vop3NegAbs}, // -abs(-1)
{0x00003c00u, 0, 1, Vop3NegAbs}, // -abs(+1)
{0xbc004000u, 0x3c00c000u, 1, HighWord}, // Select both upper halves.
{0x00004000u, 0x00003c00u, 0, ScalarDst},
{0x00003c00u, 0x00004000u, 1, ScalarDst},
{0x0000bc00u, 0, 1, SdwaNegAbs},
}};
constexpr u32 vcc_sentinel = 0x89abcdefu;
TestCase test;
test.name = "VectorVopcCmpNgtF16CapturedSdwaAndEdges";
for (const auto &entry : cases) {
test.initial.insert(test.initial.end(), {entry.lhs, entry.rhs});
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 8u);
AppendBufferLoadDword(&code, 21, 30); // Runtime inputs prevent constant folding.
AppendVMovU32(&code, 30, i * 8u + 4u);
AppendBufferLoadDword(&code, 1, 30);
AppendSMovLiteral(&code, 106, vcc_sentinel);
AppendSMovLiteral(&code, 107, vcc_sentinel);
code.push_back(EncodeSMovB32(126, InlineU32(entry.exec)));
switch (entry.encoding) {
case Captured: code.insert(code.end(), {0x7dd700f9u, 0x86060015u}); break;
case Compact: code.push_back(EncodeVopc(0xeb, Vgpr(21), 1)); break;
case Vop3NegAbs:
AppendVop3(&code, 0xeb, 106, Vgpr(21), Vgpr(1), 0, 1, 0, false, 0, 1);
break;
case HighWord:
code.push_back(EncodeVopc(0xeb, 249, 1));
code.push_back(EncodeVopcSdwa(21, 0, 0, 5, 5));
break;
case ScalarDst:
code.push_back(EncodeVopc(0xeb, 249, 1));
code.push_back(EncodeVopcSdwa(21, 22, 1));
break;
case SdwaNegAbs:
code.push_back(EncodeVopc(0xeb, 249, 1));
code.push_back(EncodeVopcSdwa(21, 0, 0, 6, 6, 0, 0, 1, 1));
break;
}
code.push_back(EncodeSMovB32(20, entry.encoding == ScalarDst ? 22 : 106));
code.push_back(EncodeSMovB32(21, 126)); // CMP must preserve EXEC even if false.
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = static_cast<u32>(test.initial.size()) + i * 4u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 106, out + 2u);
test.expected.insert(test.expected.end(),
{entry.expected, entry.exec,
entry.encoding == ScalarDst ? vcc_sentinel : entry.expected,
vcc_sentinel});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMP_NGT_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMP_NGT_F16", cases.size()}};
test.required_spirv = {"OpFUnordLessThanEqual"};
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase VectorVopcCmpNltF16CapturedSdwaAndEdges() {
using O = ShaderOpcode;
enum Encoding { Captured, Compact, Vop3NegAbs, SdwaNegAbs };
struct CompareCase {
u32 packed; // Captured instruction compares low half against high half.
u32 expected;
Encoding encoding = Captured;
u32 exec = 1;
};
const std::array<CompareCase, 16> cases{{
{0x40003c00u, 0}, // +1 < +2
{0x3c004000u, 1}, // +2 > +1
{0x3c003c00u, 1}, // Equal
{0x80000000u, 1}, // +0 versus -0
{0x00008000u, 1}, // -0 versus +0
{0x3c007e00u, 1}, // Quiet NaN in src0: unordered GE, not ordered GE.
{0x7e003c00u, 1}, // Quiet NaN in src1
{0x3c007c01u, 1}, // Signaling NaN in src0
{0x7c013c00u, 1}, // Signaling NaN in src1
{0xfc007c00u, 1}, // +infinity versus -infinity
{0x7c00fc00u, 0}, // -infinity versus +infinity
{0x00003c00u, 0, Captured, 0}, // Inactive lane contributes no mask bit.
{0x40003c00u, 0, Compact},
{0x7e003c00u, 1, Compact},
{0x0000bc00u, 0, Vop3NegAbs}, // -abs(-1) < 0
{0x0000bc00u, 0, SdwaNegAbs},
}};
constexpr u32 sentinel = 0x89abcdefu;
TestCase test;
test.name = "VectorVopcCmpNltF16CapturedSdwaAndEdges";
for (const auto &entry : cases) {
test.initial.push_back(entry.packed);
}
test.expected = test.initial;
auto &code = test.code;
code.push_back(EncodeSMovB32(22, 2)); // Preserve the input buffer descriptor.
code.push_back(EncodeSMovB32(23, 3));
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 3, 30); // Runtime input prevents constant folding.
AppendVMovU32(&code, 1, entry.packed >> 16u);
for (u32 reg : {3u, 106u, 107u}) {
AppendSMovLiteral(&code, reg, sentinel);
}
code.push_back(EncodeSMovB32(126, InlineU32(entry.exec)));
switch (entry.encoding) {
case Captured: code.insert(code.end(), {0x7ddc06f9u, 0x05068203u}); break;
case Compact: code.push_back(EncodeVopc(0xee, Vgpr(3), 1)); break;
case Vop3NegAbs:
AppendVop3(&code, 0xee, 2, Vgpr(3), Vgpr(1), 0, 1, 0, false, 0, 1);
break;
case SdwaNegAbs:
code.insert(code.end(), {0x7ddc06f9u, 0x05368203u});
break;
}
code.push_back(EncodeSMovB32(20, entry.encoding == Compact ? 106 : 2));
code.push_back(EncodeSMovB32(21, 3)); // Wave32 must preserve adjacent SGPR.
code.push_back(EncodeSMovB32(24, 126));
code.push_back(EncodeSMovB32(25, 127));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeSMovB32(2, 22));
code.push_back(EncodeSMovB32(3, 23));
const u32 out = static_cast<u32>(cases.size()) + i * 6u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 24, out + 2u);
AppendStoreSgprPair(&code, 106, out + 4u);
test.expected.insert(test.expected.end(),
{entry.expected, sentinel, entry.exec, 0u,
entry.encoding == Compact ? entry.expected : sentinel,
sentinel});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMP_NLT_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMP_NLT_F16", cases.size()}};
test.required_spirv = {"OpFUnordGreaterThanEqual"};
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase VectorVopcCmpxNgtF16CapturedSdwaExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 31, 0x7e003c00u);
AppendVMovU32(&code, 3, 7);
AppendVMovU32(&code, 30, 0);
code.insert(code.end(), {0x7df700f9u, 0x8605001fu});
AppendBufferStoreDword(&code, 3, 30);
AppendVMovLiteral(&code, 31, 0x3c007e00u);
AppendVMovU32(&code, 3, 9);
AppendVMovU32(&code, 30, 4);
code.insert(code.end(), {0x7df700f9u, 0x8605001fu});
AppendBufferStoreDword(&code, 3, 30);
AppendEnd(&code);
TestCase test{"VectorVopcCmpxNgtF16CapturedSdwaExecMask",
code,
{0x11111111u, 0x22222222u},
{7, 0x22222222u},
{O::V_MOV_B32, O::V_CMPX_NGT_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
test.decoded_counts = {{"V_CMPX_NGT_F16", 2}};
return test;
}
TestCase VectorVopcCmpxNleF16CapturedSdwaExecMask() {
using O = ShaderOpcode;
struct CompareCase { u32 value, expected, exec = 1; bool vop3 = false; };
const std::array<CompareCase, 10> cases{{
{0x3c007e00u, 1}, {0xbc003c00u, 0}, // High half +1/-1; opposite low halves.
{0x00003c00u, 0}, {0x80003c00u, 0}, // Signed zeros.
{0x7e003c00u, 1}, {0x7c013c00u, 1}, // Quiet/signaling NaN are unordered.
{0x7c003c00u, 1}, {0xfc003c00u, 0}, // Infinities.
{0x3c000000u, 0, 0}, // Inactive lane cannot enter EXEC.
{0x0000bc00u, 1, 1, true}, // VOP3 negates -1 before comparison.
}};
constexpr u32 sentinel = 0x89abcdefu;
TestCase test;
test.name = "VectorVopcCmpxNleF16CapturedSdwaExecMask";
for (const auto &entry : cases) test.initial.push_back(entry.value);
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 5, 30);
AppendSMovLiteral(&code, 106, sentinel);
AppendSMovLiteral(&code, 107, sentinel);
code.push_back(EncodeSMovB32(126, InlineU32(entry.exec)));
if (entry.vop3) {
AppendVop3(&code, 0xfc, 106, Vgpr(5), InlineU32(0), 0, 0, 0, false, 0, 1);
} else {
code.insert(code.end(), {0x7df900f9u, 0x86050005u});
}
code.push_back(EncodeSMovB32(8, 126));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = test.expected.size();
code.push_back(EncodeVop1(0x01, 0, 8));
AppendStoreVgpr(&code, 0, out);
AppendStoreSgprPair(&code, 106, out + 1u);
test.expected.insert(test.expected.end(), {entry.expected, sentinel, sentinel});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMPX_NLE_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMPX_NLE_F16", cases.size()}};
test.required_spirv = {"OpFUnordGreaterThan"};
return test;
}
TestCase VectorCompareInvertedMaskSelect() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x12345678u);
code.push_back(EncodeVopc(0xc7, Vgpr(0), 0)); // v_cmp_t_u32 v0, v0
code.push_back(EncodeSop1(0x08, 106, 106)); // s_not_b64 vcc, vcc
code.push_back(EncodeVop2(0x01, 1, Vgpr(0), 128)); // v_cndmask_b32 v1, v0, 0
AppendBufferStoreDword(&code, 1, 30);
AppendEnd(&code);
return {"VectorCompareInvertedMaskSelect",
code,
{},
{0x12345678u},
{O::V_MOV_B32, O::V_CMP_T_U32, O::S_NOT_B64, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BranchSelect() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSMovB32(0, InlineU32(0)),
EncodeSopc(0x06, 0, InlineU32(0)),
EncodeSopp(0x05, 2),
EncodeSMovB32(1, InlineU32(1)),
EncodeSopp(0x02, 1),
EncodeSMovB32(1, InlineU32(7)),
EncodeSop2(0x0a, 2, 1, InlineU32(2)),
EncodeVop1(0x01, 0, 2),
};
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"BranchSelect",
code,
{},
{7},
{O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CBRANCH_SCC1, O::S_BRANCH,
O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase SimpleLoop() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSMovB32(0, InlineU32(0)),
EncodeSopc(0x0a, 0, InlineU32(4)),
EncodeSopp(0x04, 2),
EncodeSop2(0x00, 0, 0, InlineU32(1)),
EncodeSopp(0x02, 0xfffcu),
EncodeVop1(0x01, 0, 0),
};
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"SimpleLoop",
code,
{},
{4},
{O::S_MOV_B32, O::S_CMP_LT_U32, O::S_CBRANCH_SCC0, O::S_ADD_U32,
O::S_BRANCH, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase SharedReturnKeepsSelectedValues() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 5, 4),
EncodeVop2(0x1a, 5, InlineU32(6), 5),
EncodeVop2(0x25, 5, Vgpr(0), 5),
EncodeSMovB32(8, InlineU32(10)),
EncodeSopc(0x06, 4, InlineU32(0)),
0,
EncodeSMovB32(8, InlineU32(20)),
EncodeSopc(0x06, 4, InlineU32(1)),
0,
EncodeSMovB32(8, InlineU32(30)),
};
AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
AppendEnd(&code);
const auto shared_return = static_cast<u32>(code.size());
AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
AppendEnd(&code);
code[5] = EncodeSopp(0x05, shared_return - 6u);
code[8] = EncodeSopp(0x05, shared_return - 9u);
TestCase test;
test.name = "SharedReturnKeepsSelectedValues";
test.code = code;
for (u32 group = 0; group < 3; group++) {
test.expected.insert(test.expected.end(), 64, (group + 1u) * 10u);
}
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CBRANCH_SCC1,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpSelectionMerge"};
test.forbidden_spirv = {"OpSwitch"};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 4;
test.has_compute_info = true;
test.dispatch_x = 3;
return test;
}
TestCase SiblingSharedExitKeepsCapturedConditions() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 5, 4),
EncodeVop2(0x1a, 5, InlineU32(6), 5),
EncodeVop2(0x25, 5, Vgpr(0), 5), // group * 64 + lane
EncodeSop2(0x0e, 6, 4, InlineU32(1)),
0, // choose left or right
EncodeSMovB32(8, InlineU32(20)),
EncodeSop2(0x0e, 6, 4, InlineU32(2)),
0, // left: skip nested exit
EncodeSop2(0x0e, 6, 4, InlineU32(4)),
0, // nested left: shared exit
0, // left -> normal return
EncodeSMovB32(8, InlineU32(30)),
EncodeSop2(0x0e, 6, 4, InlineU32(8)),
0, // first right shared exit
EncodeSMovB32(8, InlineU32(40)),
EncodeSop2(0x0e, 6, 4, InlineU32(16)),
0, // second right shared exit
EncodeSMovB32(8, InlineU32(50)),
EncodeSopc(0x06, InlineU32(0), InlineU32(1)), // overwrite SCC
};
AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
AppendEnd(&code);
const auto shared_exit = static_cast<u32>(code.size());
code.push_back(EncodeSop2(0x00, 8, 8, InlineU32(1))); // overwrite SCC again
AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
AppendEnd(&code);
code[4] = EncodeSopp(0x04, 11u - 5u);
code[7] = EncodeSopp(0x04, 10u - 8u);
code[9] = EncodeSopp(0x04, shared_exit - 10u);
code[10] = EncodeSopp(0x02, 17u - 11u);
code[13] = EncodeSopp(0x04, shared_exit - 14u);
code[16] = EncodeSopp(0x04, shared_exit - 17u);
TestCase test;
test.name = "SiblingSharedExitKeepsCapturedConditions";
test.code = std::move(code);
for (u32 group = 0; group < 32; ++group) {
u32 result = 50;
if (group & 1u) {
if ((group & 2u) && !(group & 4u)) result = 21;
} else if (!(group & 8u)) {
result = 31;
} else if (!(group & 16u)) {
result = 41;
}
test.expected.insert(test.expected.end(), 64, result);
}
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::S_AND_B32, O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_ADD_U32,
O::S_CBRANCH_SCC0, O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpSelectionMerge"};
test.forbidden_spirv = {"OpSwitch"};
test.compute_info.wave_size = 64;
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 4;
test.has_compute_info = true;
test.dispatch_x = 32;
return test;
}
TestCase BranchVccnzUsesWaveMask() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1),
EncodeVop2(0x1a, 3, InlineU32(2), 1),
EncodeVopc(0xc2, InlineU32(0), 0),
EncodeSopp(0x07, 2),
EncodeVop1(0x01, 2, InlineU32(11)),
EncodeSopp(0x02, 1),
};
AppendVMovU32(&code, 2, 42);
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "BranchVccnzUsesWaveMask";
test.code = code;
test.expected = std::vector<u32>(8, 42);
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_EQ_U32, O::S_CBRANCH_VCCNZ, O::S_BRANCH,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase ScalarBranchRestoresInactiveLanes(u32 wave_size, u32 threads) {
using O = ShaderOpcode;
TestCase test;
if (threads < wave_size) {
test.name = wave_size == 32 ? "ScalarBranchRestoresPartialWave32"
: "ScalarBranchRestoresPartialWave64";
} else {
test.name = wave_size == 32 ? "ScalarBranchRestoresInactiveWave32"
: "ScalarBranchRestoresInactiveWave64";
}
auto &code = test.code;
code = {EncodeVop1(0x01, 1, InlineU32(32)),
EncodeVop2(0x25, 1, Vgpr(0), 1),
EncodeSop1(0x04, 8, 126), // Save all participating lanes.
EncodeSMovB32(20, InlineU32(0)),
EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)),
EncodeSopp(0x08, 3), // EXEC is nonzero for the whole wave.
EncodeSop1(0x04, 126, 8)};
// The branch restores inactive lanes before reading the last lane's value.
AppendVop3(&code, 0x360, 20, Vgpr(1), InlineU32(threads - 1));
code.push_back(EncodeSop1(0x04, 126, 8));
AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 0);
AppendEnd(&code);
test.initial.assign(threads, 0);
test.expected.assign(threads, 32 + threads - 1);
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, O::S_MOV_B64,
O::S_CBRANCH_EXECZ, O::V_READLANE_B32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformBallot", "OpAll"};
test.compute_info.wave_size = wave_size;
test.compute_info.threads_num[0] = threads;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase ScalarMemRealtimeCapturedPlaceholder() {
using O = ShaderOpcode;
namespace D = ShaderRecompiler::Decoder;
std::vector<u32> code = {0xf4940300u, 0xfa000000u};
D::Instruction decoded;
D::DecodeInstruction(code, 0, decoded);
Require("ScalarMemRealtimeCapturedPlaceholder", "decode",
decoded.opcode == O::S_MEMREALTIME && decoded.word_count == 2 &&
decoded.dst.kind == D::OperandKind::Sgpr && decoded.dst.reg == 12 &&
decoded.data_dwords == 2 && decoded.src_count == 0 &&
decoded.src0.kind == D::OperandKind::Unknown &&
decoded.src1.kind == D::OperandKind::Unknown,
"captured clock instruction must write s12:s13 without memory operands");
AppendStoreSgprPair(&code, 12, 0);
code.insert(code.end(), {0xf4940300u, 0xfa000000u});
AppendStoreSgprPair(&code, 12, 2);
AppendEnd(&code);
return {"ScalarMemRealtimeCapturedPlaceholder",
code,
{},
{UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX},
{O::S_MEMREALTIME, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarMemoryLoadVariants() {
using O = ShaderOpcode;
struct Load {
u32 opcode;
u32 dst;
u32 byte_offset;
u32 dwords;
};
const Load s_loads[] = {
{0x00, 20, 0, 1}, {0x01, 22, 4, 2}, {0x02, 24, 12, 4},
{0x03, 28, 28, 8}, {0x04, 36, 60, 16},
};
const Load s_buffer_loads[] = {
{0x08, 52, 0, 1}, {0x09, 54, 4, 2}, {0x0a, 56, 12, 4},
{0x0b, 60, 28, 8}, {0x0c, 68, 60, 16},
};
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(0)));
for (const auto &load : s_loads) {
AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset);
}
for (const auto &load : s_buffer_loads) {
AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset);
}
u32 out = 0;
for (const auto &load : s_loads) {
for (u32 i = 0; i < load.dwords; i++) {
AppendStoreSgpr(&code, load.dst + i, out++);
}
}
for (const auto &load : s_buffer_loads) {
for (u32 i = 0; i < load.dwords; i++) {
AppendStoreSgpr(&code, load.dst + i, out++);
}
}
AppendEnd(&code);
std::vector<u32> initial;
for (u32 i = 0; i < 31u; i++) {
initial.push_back(0x10000000u + i);
}
std::vector<u32> expected = initial;
expected.insert(expected.end(), initial.begin(), initial.end());
return {"ScalarMemoryLoadVariants",
code,
initial,
expected,
{O::S_MOV_B32, O::S_LOAD_DWORD, O::S_LOAD_DWORDX2, O::S_LOAD_DWORDX4,
O::S_LOAD_DWORDX8, O::S_LOAD_DWORDX16, O::S_BUFFER_LOAD_DWORD,
O::S_BUFFER_LOAD_DWORDX2, O::S_BUFFER_LOAD_DWORDX4,
O::S_BUFFER_LOAD_DWORDX8, O::S_BUFFER_LOAD_DWORDX16, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarBufferOffsetAlignmentAndCarry() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 12, 0xfffffffcu);
code.push_back(EncodeSop2(0x00, 12, 12, 4));
code.push_back(EncodeSmem0(0x08, 20));
code.push_back(EncodeSmem1(4, 12));
AppendSMovLiteral(&code, 12, 3u);
code.push_back(EncodeSop2(0x00, 12, 12, 4));
code.push_back(EncodeSmem0(0x08, 21));
code.push_back(EncodeSmem1(1, 12));
AppendStoreSgpr(&code, 20, 0);
AppendStoreSgpr(&code, 21, 1);
AppendEnd(&code);
TestCase test{"ScalarBufferOffsetAlignmentAndCarry", code,
{0x11111111u, 0x22222222u}, {0u, 0x11111111u},
{O::S_MOV_B32, O::S_ADD_U32, O::S_BUFFER_LOAD_DWORD,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.forbidden_spirv = {"flattened_srt"};
test.compute_info.threads_num[0] = 1;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 4;
test.has_compute_info = true;
return test;
}
TestCase ScalarLoadSignedImmediateOffsetAddsSoffset() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(8)));
code.push_back(EncodeSMovB32(2, InlineU32(0)));
code.push_back(EncodeSMovB32(3, InlineU32(0)));
code.push_back(EncodeSmem0(0x00, 1, 1));
code.push_back(EncodeSmem1(0x1ffffcu, 0));
AppendStoreSgpr(&code, 1, 0);
AppendEnd(&code);
return {"ScalarLoadSignedImmediateOffsetAddsSoffset",
code,
{0x11111111u, 0x22222222u},
{0x22222222u, 0x22222222u},
{O::S_MOV_B32, O::S_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferLoadStore() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 0, 30);
code.push_back(EncodeVop1(0x01, 31, InlineU32(4)));
AppendBufferStoreDword(&code, 0, 31);
AppendEnd(&code);
return {
"BufferLoadStore",
code,
{0x11223344u, 0},
{0x11223344u, 0x11223344u},
{O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 8);
code.push_back(EncodeMubuf0(0x0cu, 0, true, true));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {
"BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset",
code,
{0x11111111u, 0x22222222u, 0x33333333u},
{0x33333333u, 0x22222222u, 0x33333333u},
{O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 12);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
code.push_back(EncodeMubuf0(0x1cu, 0, true, true));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
return {"BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset",
code,
{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u},
{0x11111111u, 0x22222222u, 0x33333333u, 0xabcdef01u},
{O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferLoadDwordNoAddressFlagsIgnoresVaddr() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 8);
code.push_back(EncodeMubuf0(0x0cu, 0, false, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 3);
AppendEnd(&code);
return {
"BufferLoadDwordNoAddressFlagsIgnoresVaddr",
code,
{0x11111111u, 0x22222222u, 0x33333333u, 0},
{0x11111111u, 0x22222222u, 0x33333333u, 0x11111111u},
{O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferLoadDwordIdxenUsesDescriptorStride() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
code.push_back(EncodeMubuf0(0x0cu, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadDwordIdxenUsesDescriptorStride";
test.code = code;
test.initial = {0x11111111u, 0x22222222u, 0x33333333u,
0x44444444u, 0x55555555u, 0x66666666u,
0x77777777u, 0x88888888u, 0x99aabbccu};
test.expected = {0x99aabbccu, 0x22222222u, 0x33333333u,
0x44444444u, 0x55555555u, 0x66666666u,
0x77777777u, 0x88888888u, 0x99aabbccu};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(16, 3);
test.has_user_data = true;
return test;
}
TestCase BufferStoreDwordIdxenUsesDescriptorStride() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
code.push_back(EncodeMubuf0(0x1cu, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreDwordIdxenUsesDescriptorStride";
test.code = code;
test.initial = {0x11111111u, 0x22222222u, 0x33333333u,
0x44444444u, 0x55555555u, 0x66666666u,
0x77777777u, 0x88888888u, 0x99aabbccu};
test.expected = {0x11111111u, 0x22222222u, 0x33333333u,
0x44444444u, 0x55555555u, 0x66666666u,
0x77777777u, 0x88888888u, 0xabcdef01u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(16, 3);
test.has_user_data = true;
return test;
}
TestCase BufferStoreDwordAppliesHostOffset() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xabcdef01u);
code.push_back(EncodeMubuf0(0x1cu, 2, false, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreDwordAppliesHostOffset";
test.code = code;
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0};
test.expected = {0x11111111u, 0xabcdef01u, 0x33333333u, 0};
test.storage_buffer_range_bytes = 8;
test.storage_buffer_offsets = {2};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8);
test.has_user_data = true;
return test;
}
TestCase BufferOffsetsUsePackedWords(bool storage) {
using O = ShaderOpcode;
constexpr std::array offsets{0u, 12u, 20u, 28u, 36u};
TestCase test;
test.name = storage ? "BufferOffsetsPackedWordsStorage" : "BufferOffsetsPackedWordsPush";
test.initial.resize(10);
test.expected = test.initial;
for (u32 i = 0; i < offsets.size(); ++i) {
const auto base = i * 4u;
AppendSMovLiteral(&test.code, base, 0x1000u + base * 0x1000u);
AppendSMovLiteral(&test.code, base + 1u, 4u << 16u);
AppendSMovLiteral(&test.code, base + 2u, 1u);
AppendSMovLiteral(&test.code, base + 3u, 1u << 24u);
const auto value = 0x12345678u + i;
AppendVMovLiteral(&test.code, i, value);
test.code.push_back(EncodeMubuf0(0x1cu, 0, false, false));
test.code.push_back(EncodeMubuf1(i, i, 20));
test.expected[offsets[i] / 4u] = value;
}
AppendEnd(&test.code);
test.storage_buffer_range_bytes = 4;
test.storage_buffer_offsets.assign(offsets.begin(), offsets.end());
test.expand_shader_data_storage = storage;
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase BufferLoadVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x08, 0, 20);
AppendVMovU32(&code, 20, 2);
AppendBufferLoadOpcode(&code, 0x09, 1, 20);
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x0a, 2, 20);
AppendVMovU32(&code, 20, 2);
AppendBufferLoadOpcode(&code, 0x0b, 3, 20);
AppendVMovU32(&code, 20, 8);
AppendBufferLoadOpcode(&code, 0x0d, 4, 20);
AppendBufferLoadOpcode(&code, 0x0f, 6, 20);
AppendBufferLoadOpcode(&code, 0x0e, 9, 20);
for (u32 i = 0; i < 13u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"BufferLoadVariants",
code,
{0x80ff7f01u, 0x00008001u, 0x11223344u, 0x55667788u, 0x99aabbccu,
0xddeeff00u},
{0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u, 0x55667788u,
0x11223344u, 0x55667788u, 0x99aabbccu, 0x11223344u, 0x55667788u,
0x99aabbccu, 0xddeeff00u},
{O::BUFFER_LOAD_UBYTE, O::BUFFER_LOAD_SBYTE, O::BUFFER_LOAD_USHORT,
O::BUFFER_LOAD_SSHORT, O::BUFFER_LOAD_DWORDX2,
O::BUFFER_LOAD_DWORDX3, O::BUFFER_LOAD_DWORDX4, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferSubwordLoadsAtHostOffset(u32 bytes, u32 component, bool sign) {
using O = ShaderOpcode;
TestCase test;
test.name = bytes == 1 ? (sign ? "BufferSbyteHostOffsetTail" : "BufferUbyteHostOffsetTail")
: sign ? "BufferSshortHostOffsetTail"
: component == 0 ? "BufferUshortHostOffsetFirst"
: component == 2 ? "BufferUshortHostOffsetCarry"
: "BufferUshortHostOffsetTail";
// The captured stride-six index buffer uses USHORT IDXEN offsets 4, 2, 0.
test.code = {EncodeMubuf0((bytes == 1 ? 0x08u : 0x0au) + sign,
component, true, false),
EncodeMubuf1(1, 0, 0),
EncodeMubuf0(0x1cu, 0, true, false), EncodeMubuf1(1, 12, 0)};
AppendEnd(&test.code);
test.initial = {0x800112abu, 0xfedc2345u, 0x80007fffu, 0xfaceffffu,
0xccccccccu, 0xccccccccu, 0xccccccccu};
test.expected = test.initial;
for (u32 record = 0; record < 2; ++record) {
u32 value = 0;
std::memcpy(&value, reinterpret_cast<const uint8_t *>(test.initial.data()) +
bytes + record * 6 + component, bytes);
if (sign && (value & (1u << (bytes * 8u - 1u))) != 0) {
value |= UINT32_MAX << (bytes * 8u);
}
test.expected[4 + record] = value;
}
test.expected[6] = 0;
test.storage_buffer_range_bytes = 12;
test.storage_buffer_offsets = {bytes, 16};
test.user_data = MakeStructuredStorageBufferData(6, 2);
test.user_data[0] = 0x1000u + bytes;
test.user_data[48] = 0x2000u;
test.user_data[49] = 4u << 16u;
test.user_data[50] = 3;
test.user_data[51] = 0x5204;
test.has_user_data = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.threads_num[0] = 3;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.has_compute_info = true;
test.opcodes = {bytes == 1 ? (sign ? O::BUFFER_LOAD_SBYTE : O::BUFFER_LOAD_UBYTE)
: (sign ? O::BUFFER_LOAD_SSHORT : O::BUFFER_LOAD_USHORT),
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase BufferSubwordHostOffsetOverflow() {
auto test = BufferSubwordLoadsAtHostOffset(2, 0, false);
test.name = "BufferSubwordHostOffsetOverflow";
test.code.clear();
AppendVMovLiteral(&test.code, 20, UINT32_MAX);
test.code.insert(test.code.end(),
{EncodeMubuf0(0x0au), EncodeMubuf1(1, 0, 20),
EncodeMubuf0(0x1cu, 0, true, false), EncodeMubuf1(1, 12, 0)});
AppendVMovLiteral(&test.code, 1, 0xbeef);
test.code.insert(test.code.end(), {EncodeMubuf0(0x1au), EncodeMubuf1(1, 0, 20)});
AppendEnd(&test.code);
test.expected[4] = test.expected[5] = 0;
test.opcodes.push_back(ShaderOpcode::V_MOV_B32);
test.opcodes.push_back(ShaderOpcode::BUFFER_STORE_SHORT);
return test;
}
TestCase BufferSubwordStoresAtHostOffset(u32 bytes) {
using O = ShaderOpcode;
TestCase test;
test.name = bytes == 1 ? "BufferByteStoreHostOffsetTail" : "BufferShortStoreHostOffsetTail";
AppendVMovLiteral(&test.code, 1, bytes == 1 ? 0xa5u : 0xcafeu);
test.code.push_back(EncodeMubuf0(bytes == 1 ? 0x18u : 0x1au,
6u - bytes, true, false));
test.code.push_back(EncodeMubuf1(1, 0, 0));
AppendEnd(&test.code);
test.initial = {0x800112abu, 0xfedc2345u, 0x80007fffu, 0xfaceffffu,
0x13579bdfu, 0x2468ace0u};
test.expected = bytes == 1
? std::vector<u32>{0x800112abu, 0xfea52345u, 0x80007fffu, 0xfaceffa5u,
0x13579bdfu, 0x2468ace0u}
: std::vector<u32>{0x800112abu, 0xcafe2345u, 0x80007fffu, 0xfacecafeu,
0x13579bdfu, 0x2468ace0u};
test.storage_buffer_range_bytes = 12;
test.storage_buffer_offsets = {bytes};
test.user_data = MakeStructuredStorageBufferData(6, 2);
test.user_data[0] = 0x1000u + bytes;
test.has_user_data = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.threads_num[0] = 3;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, bytes == 1 ? O::BUFFER_STORE_BYTE : O::BUFFER_STORE_SHORT,
O::S_ENDPGM};
return test;
}
TestCase BufferLoadDwordx4SnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMubuf0(0x0eu, 0, true, true));
code.push_back(EncodeMubuf1(21, 0, 21));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 21 + i, 4 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadDwordx4SnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
0, 0, 0, 0};
test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX4, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(16, 2);
test.has_user_data = true;
return test;
}
TestCase BufferLoadDwordx2SnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMubuf0(0x0du, 0, true, true));
code.push_back(EncodeMubuf1(21, 0, 21));
for (u32 i = 0; i < 2; i++) {
AppendStoreVgpr(&code, 21 + i, 2 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadDwordx2SnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0, 0};
test.expected = {0x11111111u, 0x22222222u, 0x11111111u, 0x22222222u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX2, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(8, 2);
test.has_user_data = true;
return test;
}
TestCase BufferLoadDwordx3SnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMubuf0(0x0fu, 0, true, true));
code.push_back(EncodeMubuf1(21, 0, 21));
for (u32 i = 0; i < 3; i++) {
AppendStoreVgpr(&code, 21 + i, 3 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadDwordx3SnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0, 0, 0};
test.expected = {0x11111111u, 0x22222222u, 0x33333333u,
0x11111111u, 0x22222222u, 0x33333333u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX3, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(12, 2);
test.has_user_data = true;
return test;
}
TestCase BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 8);
AppendBufferLoadOpcode(&code, 0x0e, 0, 20);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = {0x33333333u, 0x44444444u, 0u, 0u};
test.storage_buffer_range_bytes = 16;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX4, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase BufferLoadsGpuSelectedDescriptors(u32 width) {
const bool scalar = width == 1u;
const bool xyz = width == 3u;
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
struct DescriptorCase {
u32 stride, records, mode, soffset;
bool swizzle, bound;
std::array<u32, 6> expected;
};
const DescriptorCase cases[] = {
{24, 2, 0, 0, false, true, {7, 8, 9, 10, 11, 12}},
// PPSA01417 selects XYZ vertices from a table of stride-16 descriptors.
{16, 2, 0, 0, false, true, {5, 6, 7, 8, 0, 0}},
{12, 2, 0, 0, false, true, {4, 5, 6, 0, 0, 0}},
{12, 2, 1, 0, false, true, {4, 5, 6, 7, 8, 9}},
{12, 1, 2, 0, false, true, {4, 5, 6, 7, 8, 9}},
{12, 0, 2, 0, false, true, {}},
// OOB3 checks offset + soffset, independently of index * stride.
{12, 20, 3, 4, false, true, {5, 6, 7, 8, 0, 0}},
{12, 2, 0, 0, false, false, {}},
// RDNA2 swizzling uses four-byte elements; soffset is added afterward.
{12, 2, 0, 4, true, true, {3, 4, 5, 0, 0, 0}},
{12, 8, 3, 4, true, true, {3, 4, 5, 0, 0, 0}},
{12, 4, 3, 4, true, true, {}},
};
// OFFEN byte offset 5 aligns to DWORD; swizzled offsets use eight-index groups.
constexpr u32 scalar_expected[] = {2, 2, 2, 2, 2, 0, 3, 0, 10, 10, 0};
TestCase test;
test.name = scalar ? "BufferLoadDwordGpuSelectedDescriptors"
: xyz ? "BufferLoadDwordx3GpuSelectedDescriptors"
: "BufferLoadsGpuSelectedDescriptors";
test.initial.resize(2048);
for (u32 i = 0; i < std::size(cases); ++i) {
const auto &input = cases[i];
const u32 data_offset = 4096 + i * 128;
// 120-byte table entries with a descriptor at byte 8, as in PPSA04677.
const std::array<u32, 4> descriptor{
static_cast<u32>(GuestBase + data_offset),
(input.stride << 16u) | (input.swizzle ? 1u << 31u : 0u) | 1u,
input.records, (input.bound ? 0x5204u : 0x204u) | (input.mode << 28u)};
std::copy(descriptor.begin(), descriptor.end(), test.initial.begin() + 130 + i * 30);
for (u32 word = 0; word < 32; ++word) {
test.initial[data_offset / 4 + word] = i * 100 + word + 1;
}
// Reverse the table order through a GPU load and readfirstlane, not host constants.
const u32 selected = static_cast<u32>(std::size(cases)) - 1 - i;
test.initial[64 + i] = selected;
AppendVMovU32(&test.code, 30, (64 + i) * 4);
AppendBufferLoadDword(&test.code, 0, 30);
test.code.push_back(EncodeVop1(0x02, 20, Vgpr(0)));
test.code.push_back(EncodeSop2(0x26, 20, 20, 255)); // s_mul_i32 s20, s20, 120
test.code.push_back(120);
test.code.push_back(EncodeSmem0(0x0a, 8, 0));
test.code.push_back(EncodeSmem1(520, 20));
AppendSMovLiteral(&test.code, 22, cases[selected].soffset);
AppendVMovU32(&test.code, 21, scalar ? 5u : 1u);
test.code.push_back(EncodeMubuf0(scalar ? 0x0c : xyz ? 0x0f : 0x0e, 0, !scalar, scalar));
test.code.push_back(EncodeMubuf1(0, 2, 21, 22));
if (!scalar) {
test.code.push_back(EncodeMubuf0(xyz ? 0x0f : 0x0d, xyz ? 12 : 16, true, false));
test.code.push_back(EncodeMubuf1(xyz ? 3 : 4, 2, 21, 22));
}
const u32 outputs = scalar ? 1u : 6u;
for (u32 component = 0; component < outputs; ++component) {
AppendStoreVgpr(&test.code, component, i * outputs + component);
const u32 expected = scalar ? scalar_expected[selected] : cases[selected].expected[component];
test.expected.push_back(expected == 0 ? 0 : selected * 100 + expected);
}
}
AppendEnd(&test.code);
test.bda_mappings = {{GuestBase, 0}};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_READFIRSTLANE_B32, O::S_MUL_I32, O::S_BUFFER_LOAD_DWORDX4,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
if (scalar) {
test.decoded_counts = {{"BUFFER_LOAD_DWORD ", 2u * std::size(cases)}};
} else if (xyz) {
test.opcodes.push_back(O::BUFFER_LOAD_DWORDX3);
} else {
test.opcodes.insert(test.opcodes.end(), {O::BUFFER_LOAD_DWORDX4, O::BUFFER_LOAD_DWORDX2});
}
test.required_spirv = {"OpConvertUToPtr", "PhysicalStorageBuffer"};
return test;
}
TestCase BufferLoadsGpuSelectedDescriptors() {
return BufferLoadsGpuSelectedDescriptors(4u);
}
TestCase BufferLoadDwordx3GpuSelectedDescriptors() {
return BufferLoadsGpuSelectedDescriptors(3u);
}
TestCase BufferLoadDwordGpuSelectedDescriptors() {
return BufferLoadsGpuSelectedDescriptors(1u);
}
TestCase BufferLoadFormatXGpuSelectedDescriptors() {
using O = ShaderOpcode;
using F = Prospero::BufferFormat;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
struct DescriptorCase {
F format;
u32 selector, stride, records, mode, index, offset, soffset;
bool swizzle;
u32 expected;
};
constexpr DescriptorCase cases[] = {
{F::k8UInt, 4, 1, 4, 3, 1, 0, 0, false, 0x7f},
{F::k8UInt, 4, 0, 4, 3, 0, 3, 0, false, 0x80},
{F::k16UInt, 4, 2, 4, 0, 1, 0, 0, false, 0x80ff},
{F::k32UInt, 4, 4, 2, 0, 1, 0, 0, false, 0x8001ff00},
{F::k32_32_32_32Float, 7, 16, 1, 0, 0, 0, 0, false, 0x3fc00000},
{F::k8SInt, 4, 0, 16, 3, 0, 3, 0, false, 0xffffff80},
{F::k16SInt, 4, 0, 16, 3, 0, 2, 0, false, 0xffff80ff},
{F::k8UNorm, 4, 0, 16, 3, 0, 2, 0, false, 0x3f800000},
{F::k8SNorm, 4, 0, 16, 3, 0, 3, 0, false, 0xbf800000},
{F::k8UScaled, 4, 0, 16, 3, 0, 1, 0, false, 0x42fe0000},
{F::k8SScaled, 4, 0, 16, 3, 0, 3, 0, false, 0xc3000000},
{F::k16Float, 4, 0, 16, 3, 0, 8, 0, false, 0x3f800000},
{F::k11_11_10UInt, 5, 0, 16, 3, 0, 0, 0, false, 0x7ef},
{F::k10_10_10_2UInt, 7, 0, 16, 3, 0, 0, 0, false, 2},
{F::k8_8_8_8UInt, 6, 0, 16, 3, 0, 0, 0, false, 0xff},
// Align the whole format first, then advance components from one swizzled base.
{F::k16UInt, 4, 0, 16, 3, 0, 1, 0, false, 0x7f01},
{F::k16UInt, 4, 0, 4, 3, 0, 3, 0, false, 0},
{F::k32_32Float, 5, 8, 2, 0, 1, 0, 4, true, 0x3fc00000},
// FORMAT_X still transfers a complete RGBA32 record for range checking.
{F::k32_32_32_32Float, 4, 0, 16, 3, 0, 4, 0, false, 0},
{F::k32_32_32_32Float, 4, 8, 1, 0, 0, 0, 0, false, 0},
{F::k32_32_32_32Float, 1, 0, 16, 3, 0, 4, 0, false, 0x3f800000},
{F::k32UInt, 1, 4, 0, 0, 0, 0, 0, false, 1},
{F::k32UInt, 0, 4, 1, 0, 0, 0, 0, false, 0},
{F::kInvalid, 1, 4, 1, 0, 0, 0, 0, false, 0},
{F::k32UInt, 4, 4, 0, 2, 0, 0, 0, false, 0},
{F::k32UInt, 4, 0, 4, 3, 0, 0, 8, false, 0},
};
TestCase test;
test.name = "BufferLoadFormatXGpuSelectedDescriptors";
test.initial.resize(2048);
constexpr std::array payload{0x80ff7f01u, 0x8001ff00u, 0x40003c00u, 0x3fc00000u};
for (u32 i = 0; i < std::size(cases); ++i) {
const auto &input = cases[i];
const u32 data_offset = 4096 + i * 64;
const std::array<u32, 8> row{
static_cast<u32>(GuestBase + data_offset),
1u | (input.stride << 16u) | (input.swizzle ? 1u << 31u : 0u),
input.records, input.selector | (static_cast<u32>(input.format) << 12u) |
(input.mode << 28u),
input.index, input.offset, input.soffset, 0};
std::copy(row.begin(), row.end(), test.initial.begin() + 128 + i * 8);
std::copy(payload.begin(), payload.end(), test.initial.begin() + data_offset / 4);
// Read the selection on the GPU, as GTA's descriptor waterfall does.
test.initial[64 + i] = std::size(cases) - 1 - i;
test.expected.push_back(cases[std::size(cases) - 1 - i].expected);
}
auto &code = test.code;
AppendSMovLiteral(&code, 28, 0);
const auto loop = static_cast<u32>(code.size());
code.push_back(EncodeVop1(0x01, 30, 28));
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 30));
code.push_back(EncodeMubuf0(0x0c, 256));
code.push_back(EncodeMubuf1(2, 0, 30));
code.push_back(EncodeVop1(0x02, 20, Vgpr(2)));
code.push_back(EncodeSop2(0x1e, 20, 20, InlineU32(5)));
code.push_back(EncodeSmem0(0x0b, 8, 0));
code.push_back(EncodeSmem1(512, 20));
code.push_back(EncodeVop1(0x01, 21, 12));
code.push_back(EncodeVop1(0x01, 22, 13));
code.push_back(EncodeMubuf0(0x00, 0, true, true));
code.push_back(EncodeMubuf1(4, 2, 21, 14));
AppendBufferStoreDword(&code, 4, 30);
code.push_back(EncodeSop2(0x00, 28, 28, InlineU32(1)));
code.push_back(EncodeSopc(0x0a, 28, InlineU32(std::size(cases))));
code.push_back(EncodeSopp(0x05, loop - static_cast<u32>(code.size()) - 1u));
AppendEnd(&code);
test.bda_mappings = {{GuestBase, 0}};
test.required_spirv = {"PhysicalStorageBuffer", "OpSwitch", "OpLoopMerge"};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::V_LSHLREV_B32,
O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_LSHL_B32,
O::S_BUFFER_LOAD_DWORDX8, O::BUFFER_LOAD_FORMAT_X,
O::BUFFER_STORE_DWORD, O::S_ADD_U32, O::S_CMP_LT_U32,
O::S_CBRANCH_SCC1, O::S_ENDPGM};
return test;
}
TestCase BufferStoreDwordx4DropsOnlyOutOfBoundsTail() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
AppendVMovLiteral(&code, 1, 0xbbbbbbbbu);
AppendVMovLiteral(&code, 2, 0xccccccccu);
AppendVMovLiteral(&code, 3, 0xddddddddu);
AppendVMovU32(&code, 20, 8);
code.push_back(EncodeMubuf0(0x1eu));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreDwordx4DropsOnlyOutOfBoundsTail";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = {0x11111111u, 0x22222222u, 0xaaaaaaaau, 0xbbbbbbbbu};
test.storage_buffer_range_bytes = 16;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORDX4, O::S_ENDPGM};
return test;
}
TestCase BufferLoadFormatXyzwRejectsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 8);
AppendBufferLoadOpcode(&code, 0x03, 0, 20);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwRejectsPartialRecord";
test.code = std::move(code);
test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.expected = {0u, 0u, 0u, 0u};
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXyzwDropsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
AppendVMovLiteral(&code, 1, 0xbbbbbbbbu);
AppendVMovLiteral(&code, 2, 0xccccccccu);
AppendVMovLiteral(&code, 3, 0xddddddddu);
AppendVMovU32(&code, 20, 8);
code.push_back(EncodeMubuf0(0x07u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzwDropsPartialRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = test.initial;
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
return test;
}
TestCase BufferLoadFormatXRejectsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 12);
AppendBufferLoadOpcode(&code, 0x00, 0, 20);
// Bounds precede alignment: rounding offset 1 down must not make its full format fit.
AppendVMovU32(&code, 20, 1);
AppendBufferLoadOpcode(&code, 0x00, 1, 20);
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXRejectsPartialRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = {0u, 0u, 0x33333333u, 0x44444444u};
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXRejectsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
AppendVMovU32(&code, 20, 12);
code.push_back(EncodeMubuf0(0x04u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXRejectsPartialRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = test.initial;
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
return test;
}
TestCase BufferLoadFormatXyRejectsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 8);
AppendBufferLoadOpcode(&code, 0x01, 0, 20);
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyRejectsPartialRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = {0u, 0u, 0x33333333u, 0x44444444u};
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXyRejectsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
AppendVMovLiteral(&code, 1, 0xbbbbbbbbu);
AppendVMovU32(&code, 20, 8);
code.push_back(EncodeMubuf0(0x05u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyRejectsPartialRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = test.initial;
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XY, O::S_ENDPGM};
return test;
}
TestCase BufferStoreVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 0, 0xaa);
AppendBufferStoreOpcode(&code, 0x18, 0, 20);
AppendVMovU32(&code, 20, 2);
AppendVMovLiteral(&code, 1, 0x0000bbccu);
AppendBufferStoreOpcode(&code, 0x1a, 1, 20);
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
AppendBufferStoreOpcode(&code, 0x1d, 2, 20);
AppendVMovU32(&code, 20, 12);
AppendVMovLiteral(&code, 4, 0x33333333u);
AppendVMovLiteral(&code, 5, 0x44444444u);
AppendVMovLiteral(&code, 6, 0x55555555u);
AppendBufferStoreOpcode(&code, 0x1f, 4, 20);
AppendVMovU32(&code, 20, 24);
AppendVMovLiteral(&code, 7, 0x66666666u);
AppendVMovLiteral(&code, 8, 0x77777777u);
AppendVMovLiteral(&code, 9, 0x88888888u);
AppendVMovLiteral(&code, 10, 0x99999999u);
AppendBufferStoreOpcode(&code, 0x1e, 7, 20);
AppendEnd(&code);
return {"BufferStoreVariants",
code,
std::vector<u32>(10, 0),
{0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
0x55555555u, 0x66666666u, 0x77777777u, 0x88888888u, 0x99999999u},
{O::V_MOV_B32, O::BUFFER_STORE_BYTE, O::BUFFER_STORE_SHORT,
O::BUFFER_STORE_DWORDX2, O::BUFFER_STORE_DWORDX3,
O::BUFFER_STORE_DWORDX4, O::S_ENDPGM}};
}
TestCase BufferFormatVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x00, 0, 20);
AppendBufferLoadOpcode(&code, 0x01, 1, 20);
AppendBufferLoadOpcode(&code, 0x02, 3, 20);
AppendBufferLoadOpcode(&code, 0x03, 6, 20);
for (u32 i = 0; i < 10u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
TestCase load;
load.name = "BufferFormatLoadVariants";
load.code = code;
load.initial = {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u};
load.expected = {0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u,
0x11121314u, 0x21222324u, 0x01020304u, 0x11121314u,
0x21222324u, 0x31323334u};
load.opcodes = {O::BUFFER_LOAD_FORMAT_X,
O::BUFFER_LOAD_FORMAT_XY,
O::BUFFER_LOAD_FORMAT_XYZ,
O::BUFFER_LOAD_FORMAT_XYZW,
O::V_MOV_B32,
O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return load;
}
TestCase BufferLoadFormatXyzwSnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMubuf0(0x03u, 0, true, true));
code.push_back(EncodeMubuf1(21, 0, 21));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 21 + i, 4 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwSnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
0, 0, 0, 0};
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(
16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
return test;
}
TestCase BufferLoadFormatXyzwResource32_32FloatAppliesXy01() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x03, 0, 20);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwResource32_32FloatAppliesXy01";
test.code = std::move(code);
test.initial = {0x3f800000u, 0x40000000u, 0, 0};
test.expected = {0x3f800000u, 0x40000000u, 0, 0x3f800000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(
16, 1, false, BufferFormat(Prospero::BufferFormat::k32_32Float));
test.user_data[3] = (test.user_data[3] & ~0xfffu) | DstSel(4, 5, 0, 1);
test.has_user_data = true;
return test;
}
TestCase BufferLoadFormatXyzwResource323232FloatRepeatsSource() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x03, 0, 20);
for (u32 i = 0; i < 4; ++i) AppendStoreVgpr(&code, i, i + 4);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwResource323232FloatRepeatsSource";
test.code = std::move(code);
test.initial = {0x3e800000u, 0x3f000000u, 0x3f400000u, 0xdeadbeefu, 0, 0, 0, 0};
test.expected = {0x3e800000u, 0x3f000000u, 0x3f400000u, 0xdeadbeefu,
0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3e800000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(
0, 32, false, BufferFormat(Prospero::BufferFormat::k32_32_32Float));
test.has_user_data = true;
return test;
}
TestCase BufferLoadFormatXyzwResource8_8_8_8UintAppliesWzy1() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x03, 0, 20);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwResource8_8_8_8UintAppliesWzy1";
test.code = std::move(code);
test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u};
test.expected = {0x11u, 0x22u, 0x33u, 1u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(
16, 1, false, BufferFormat(Prospero::BufferFormat::k8_8_8_8UInt));
test.user_data[3] = (test.user_data[3] & ~0xfffu) | DstSel(7, 6, 5, 1);
test.has_user_data = true;
return test;
}
TestCase BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 21, 0x11111111u);
AppendVMovLiteral(&code, 22, 0x22222222u);
AppendVMovLiteral(&code, 23, 0x33333333u);
AppendVMovLiteral(&code, 24, 0x44444444u);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeMubuf0(0x03u, 0, true, true));
code.push_back(EncodeMubuf1(21, 0, 21));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeSMovB32(127, InlineU32(0)));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 21 + i, 4 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress";
test.code = std::move(code);
test.initial = {0xaaaaaaaa, 0xbbbbbbbb, 0xcccccccc, 0xdddddddd, 0, 0, 0, 0};
test.expected = {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu,
0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.opcodes = {
O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_FORMAT_XYZW,
O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(
16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
return test;
}
TestCase BufferFormatStoreVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x01020304u);
AppendBufferStoreOpcode(&code, 0x04, 0, 20);
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 1, 0x11121314u);
AppendVMovLiteral(&code, 2, 0x21222324u);
AppendBufferStoreOpcode(&code, 0x05, 1, 20);
AppendVMovU32(&code, 20, 12);
AppendVMovLiteral(&code, 3, 0x31323334u);
AppendVMovLiteral(&code, 4, 0x41424344u);
AppendVMovLiteral(&code, 5, 0x51525354u);
AppendBufferStoreOpcode(&code, 0x06, 3, 20);
AppendVMovU32(&code, 20, 24);
AppendVMovLiteral(&code, 6, 0x61626364u);
AppendVMovLiteral(&code, 7, 0x71727374u);
AppendVMovLiteral(&code, 8, 0x81828384u);
AppendVMovLiteral(&code, 9, 0x91929394u);
AppendBufferStoreOpcode(&code, 0x07, 6, 20);
AppendEnd(&code);
return {"BufferFormatStoreVariants",
code,
std::vector<u32>(10, 0),
{0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u,
0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u},
{O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::BUFFER_STORE_FORMAT_XY,
O::BUFFER_STORE_FORMAT_XYZ, O::BUFFER_STORE_FORMAT_XYZW,
O::S_ENDPGM}};
}
TestCase BufferLoadFormatXResource8UintZeroExtendsByte() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMubuf0(0x00u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 1);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXResource8UintZeroExtendsByte";
test.code = code;
test.initial = {0x11223344u, 0};
test.expected = {0x11223344u, 0x00000044u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXResource16UintWritesHalfword() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x0000aaaau);
code.push_back(EncodeMubuf0(0x04u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXResource16UintWritesHalfword";
test.code = code;
test.initial = {0x11223344u};
test.expected = {0x1122aaaau};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 4, false, 11);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXResource16UintPreservesAdjacentLanes() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x25, 1, InlineU32(1), 0));
code.push_back(EncodeMubuf0(0x04u, 0, true, false));
code.push_back(EncodeMubuf1(1, 0, 0));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXResource16UintPreservesAdjacentLanes";
test.code = code;
test.initial = std::vector<u32>(32, 0);
for (u32 index = 0; index < 32; index++) {
test.expected.push_back(((index * 2 + 2) << 16) | (index * 2 + 1));
}
test.opcodes = {O::V_ADD_NC_U32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = 64;
test.has_compute_info = true;
test.user_data = MakeStructuredStorageBufferData(2, 64, false, 11);
test.has_user_data = true;
test.forbidden_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange"};
return test;
}
TestCase BufferStoreFormatXyzwPackedUnormSkinningVectors() {
using O = ShaderOpcode;
// The captured skinning dispatch stores a compressed quaternion with W = 1.
constexpr std::array<float, 12> values = {
0.25f, 0.5f, 0.75f, 1.0f,
-1.0f, 0.0f, 2.0f, 1.0f,
0.5f, 0.5f, 0.5f, 0.5f};
std::vector<u32> code;
for (u32 record = 0; record < 3; record++) {
for (u32 component = 0; component < 4; component++) {
AppendVMovLiteral(&code, component,
std::bit_cast<u32>(values[record * 4 + component]));
}
AppendVMovU32(&code, 20, record);
code.push_back(EncodeMubuf0(0x07u, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
}
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzwPackedUnormSkinningVectors";
test.code = std::move(code);
test.initial = std::vector<u32>(4, 0xdeadbeefu);
test.expected = {0xeff80100u, 0xfff00000u, 0xa0080200u, 0xdeadbeefu};
test.user_data = MakeStructuredStorageBufferData(
4, 3, false, BufferFormat(Prospero::BufferFormat::k10_10_10_2UNorm));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXZeroFillsPackedRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, std::bit_cast<u32>(0.5f));
AppendVMovU32(&code, 20, 4);
code.push_back(EncodeMubuf0(0x04u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXZeroFillsPackedRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0xffffffffu, 0x33333333u};
test.expected = {0x11111111u, 0x00000200u, 0x33333333u};
test.user_data = MakeStructuredStorageBufferData(
0, 12, false, BufferFormat(Prospero::BufferFormat::k10_10_10_2UNorm));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXyzwFloat16ConvertsComponents() {
using O = ShaderOpcode;
// This finite F32 has low bits 0x7e63, which form an F16 NaN if truncated.
constexpr std::array<float, 8> values = {
std::bit_cast<float>(0x3f807e63u), -2.0f, 0.5f, 0.25f,
-4.0f, 3.0f, 0.0f, 2.0f};
std::vector<u32> code;
for (u32 record = 0; record < 2; record++) {
for (u32 component = 0; component < 4; component++) {
AppendVMovLiteral(&code, component,
std::bit_cast<u32>(values[record * 4 + component]));
}
AppendVMovU32(&code, 20, record);
code.push_back(EncodeMubuf0(0x07u, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
}
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzwFloat16ConvertsComponents";
test.code = std::move(code);
test.initial = std::vector<u32>(4, 0xdeadbeefu);
test.expected = {0xc0003c04u, 0x34003800u, 0x4200c400u, 0x40000000u};
test.user_data = MakeStructuredStorageBufferData(
8, 2, false, BufferFormat(Prospero::BufferFormat::k16_16_16_16Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
test.required_spirv = {"PackHalf2x16"};
return test;
}
TestCase BufferStoreFormatXyzwSnorm16CapturedSkinningVectors() {
using O = ShaderOpcode;
// Normal and tangent from the character skinning dispatch (eadfd178c07feebd).
constexpr std::array<u32, 8> values = {
0x3d98f7c5u, 0xbedae712u, 0xbf66a19eu, 0x3f800000u,
0x3f7e6baeu, 0xbd339a7du, 0x3dd0a2e2u, 0xbf800000u};
std::vector<u32> code;
for (u32 record = 0; record < 2; record++) {
for (u32 component = 0; component < 4; component++) {
AppendVMovLiteral(&code, component, values[record * 4 + component]);
}
AppendVMovU32(&code, 20, record);
code.push_back(EncodeMubuf0(0x07u, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
}
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzwSnorm16CapturedSkinningVectors";
test.code = std::move(code);
test.initial = std::vector<u32>(4, 0xdeadbeefu);
test.expected = {0xc947098fu, 0x7fff8cb0u, 0xfa637f35u, 0x80010d0au};
test.user_data = MakeStructuredStorageBufferData(
8, 2, false, BufferFormat(Prospero::BufferFormat::k16_16_16_16SNorm));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords() {
using O = ShaderOpcode;
constexpr std::array<float, 9> values = {
-2.0f, -1.0f, -0.75f, -0.25f, 0.0f, 0.25f, 0.75f, 1.0f, 2.0f};
std::vector<u32> code;
for (u32 i = 0; i < values.size(); i++) {
AppendVMovLiteral(&code, 0, std::bit_cast<u32>(values[i]));
AppendVMovU32(&code, 20, i * 2 + 2);
code.push_back(EncodeMubuf0(0x04u));
code.push_back(EncodeMubuf1(0, 0, 20));
}
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords";
test.code = std::move(code);
test.initial = std::vector<u32>(6, 0xdeadbeefu);
test.expected = {0x8001beefu, 0xa0018001u, 0x0000e000u,
0x5fff2000u, 0x7fff7fffu, 0xdeadbeefu};
test.user_data = MakeStructuredStorageBufferData(
0, 24, false, BufferFormat(Prospero::BufferFormat::k16SNorm));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
return test;
}
std::vector<TestCase> BufferLoadFormatD16XCases() {
using F = Prospero::BufferFormat;
using O = ShaderOpcode;
struct LoadCase {
const char *name;
F format;
u32 input;
u32 expected;
u32 selector = 4;
u32 index = 0;
bool active = true;
bool overlap = false;
};
constexpr LoadCase inputs[] = {
{"BufferLoadFormatD16XFloat16", F::k16Float, 0x7e63c000u, 0xabcdc000u},
{"BufferLoadFormatD16XFloat32", F::k32Float, 0x3f000000u, 0xabcd3800u},
{"BufferLoadFormatD16XUint32", F::k32UInt, 0x1234fffeu, 0xabcdffffu},
{"BufferLoadFormatD16XUintAtMax", F::k32UInt, 0x0000ffffu, 0xabcdffffu},
{"BufferLoadFormatD16XUintMax32", F::k32UInt, 0xffffffffu, 0xabcdffffu},
{"BufferLoadFormatD16XSint32", F::k32SInt, 0xffff8001u, 0xabcd8001u},
{"BufferLoadFormatD16XSintAboveMax", F::k32SInt, 0x00008000u, 0xabcd7fffu},
{"BufferLoadFormatD16XSintBelowMin", F::k32SInt, 0xffff7fffu, 0xabcd8000u},
{"BufferLoadFormatD16XSintAtMax", F::k32SInt, 0x00007fffu, 0xabcd7fffu},
{"BufferLoadFormatD16XSintAtMin", F::k32SInt, 0xffff8000u, 0xabcd8000u},
{"BufferLoadFormatD16XSintMax32", F::k32SInt, 0x7fffffffu, 0xabcd7fffu},
{"BufferLoadFormatD16XSintMin32", F::k32SInt, 0x80000000u, 0xabcd8000u},
{"BufferLoadFormatD16XUnorm8", F::k8UNorm, 0xffu, 0xabcd3c00u},
{"BufferLoadFormatD16XSnorm16", F::k16SNorm, 0x8001u, 0xabcdbc00u},
{"BufferLoadFormatD16XFloatOne", F::k32Float, 0u, 0xabcd3c00u, 1},
{"BufferLoadFormatD16XOobFloatOne", F::k32Float, 0u, 0xabcd3c00u, 1, 4},
{"BufferLoadFormatD16XOobUintOne", F::k32UInt, 0u, 0xabcd0001u, 1, 4},
{"BufferLoadFormatD16XOobZero", F::k32Float, 0u, 0xabcd0000u, 4, 4},
{"BufferLoadFormatD16XInactive", F::k32Float, 0u, 0xabcd1234u, 4, 0, false},
{"BufferLoadFormatD16XOverlappingAddress", F::k16UInt, 7u, 7u, 4, 0, true, true},
};
std::vector<TestCase> cases;
for (const auto &input : inputs) {
TestCase test;
test.name = input.name;
AppendVMovLiteral(&test.code, 0, input.overlap ? input.index : 0xabcd1234u);
AppendVMovU32(&test.code, 1, input.index);
if (!input.active) {
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
}
test.code.push_back(EncodeMubuf0(0x80u, 0, true, false));
test.code.push_back(EncodeMubuf1(0, 0, input.overlap ? 0 : 1));
if (!input.active) {
test.code.push_back(EncodeSMovB32(126, InlineU32(1)));
test.code.push_back(EncodeSMovB32(127, InlineU32(0)));
}
AppendStoreVgpr(&test.code, 0, 2);
AppendEnd(&test.code);
test.initial = {input.input, 0xdeadbeefu, 0u, 0xcafef00du};
test.expected = {input.input, 0xdeadbeefu, input.expected, 0xcafef00du};
test.user_data = MakeStructuredStorageBufferData(4, 4, false, BufferFormat(input.format));
test.user_data[3] = (test.user_data[3] & ~7u) | input.selector;
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_D16_X,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
cases.push_back(std::move(test));
}
return cases;
}
std::vector<TestCase> BufferStoreFormatD16XCases() {
using F = Prospero::BufferFormat;
using O = ShaderOpcode;
struct StoreCase {
const char *name;
F format;
u32 input;
u32 expected;
};
constexpr StoreCase inputs[] = {
{"BufferStoreFormatD16XFloat16", F::k16Float, 0x7e63c000u, 0xffffc000u},
{"BufferStoreFormatD16XFloat32", F::k32Float, 0x7e633800u, 0x3f000000u},
{"BufferStoreFormatD16XUint32", F::k32UInt, 0x7e63fffeu, 0x0000fffeu},
{"BufferStoreFormatD16XSint32", F::k32SInt, 0x7e63fffeu, 0xfffffffeu},
{"BufferStoreFormatD16XUnorm8", F::k8UNorm, 0x7e633800u, 0xffffff80u},
{"BufferStoreFormatD16XSnorm16", F::k16SNorm, 0x7e63b800u, 0xffffc000u},
{"BufferStoreFormatD16XZeroFill", F::k16_16Float, 0x7e633800u, 0x00003800u},
};
std::vector<TestCase> cases;
for (const auto &input : inputs) {
TestCase test;
test.name = input.name;
AppendVMovLiteral(&test.code, 66, input.input);
AppendVMovU32(&test.code, 65, 1);
// Match the captured CS824586f35db5c80d instruction at pc 0x484.
test.code.push_back(EncodeMubuf0(0x84u, 0, true, false));
test.code.push_back(EncodeMubuf1(66, 0, 65));
AppendEnd(&test.code);
test.initial = {0xdeadbeefu, 0xffffffffu, 0xcafef00du};
test.expected = {0xdeadbeefu, input.expected, 0xcafef00du};
test.user_data = MakeStructuredStorageBufferData(4, 3, false, BufferFormat(input.format));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_D16_X, O::S_ENDPGM};
cases.push_back(std::move(test));
}
return cases;
}
std::vector<TestCase> FormattedStoreConversionCases() {
using F = Prospero::BufferFormat;
using O = ShaderOpcode;
struct ConversionCase {
const char *name[2];
F format;
u32 components;
std::array<float, 4> values;
u32 expected;
};
constexpr float nan = std::bit_cast<float>(0x7fc12345u);
constexpr float inf = std::bit_cast<float>(0x7f800000u);
constexpr ConversionCase inputs[] = {
{{"BufferStoreSnorm8Range", "TBufferStoreSnorm8Range"},
F::k8_8_8_8SNorm, 4, {-2.0f, 0.25f, 0.75f, 2.0f}, 0x7f5f2081u},
{{"BufferStoreSnorm8Ties", "TBufferStoreSnorm8Ties"},
F::k8_8SNorm, 2, {0.5f, -0.5f}, 0xffffc040u},
{{"BufferStoreSnormPackedAlpha", "TBufferStoreSnormPackedAlpha"},
F::k10_10_10_2SNorm, 4, {-1.0f, -0.5f, 0.5f, 1.0f}, 0xd00c0201u},
{{"BufferStoreSnormPackedFirstChannel", "TBufferStoreSnormPackedFirstChannel"},
F::k2_10_10_10SNorm, 4, {1.0f, -1.0f, 0.5f, -0.5f}, 0xc0100807u},
{{"BufferStoreSnormPackedZeroFillX", "TBufferStoreSnormPackedZeroFillX"},
F::k10_10_10_2SNorm, 1, {-1.0f}, 0x00000201u},
{{"BufferStoreSnormPackedZeroFillXy", "TBufferStoreSnormPackedZeroFillXy"},
F::k10_10_10_2SNorm, 2, {-1.0f, 1.0f}, 0x0007fe01u},
{{"BufferStoreSnormPackedZeroFillXyz", "TBufferStoreSnormPackedZeroFillXyz"},
F::k10_10_10_2SNorm, 3, {-1.0f, 1.0f, -1.0f}, 0x2017fe01u},
{{"BufferStoreSnorm8ZeroFillX", "TBufferStoreSnorm8ZeroFillX"},
F::k8_8_8_8SNorm, 1, {-1.0f}, 0x00000081u},
{{"BufferStoreSnorm16ZeroFillX", "TBufferStoreSnorm16ZeroFillX"},
F::k16_16SNorm, 1, {-1.0f}, 0x00008001u},
{{"BufferStoreUscaledZeroFillXy", "TBufferStoreUscaledZeroFillXy"},
F::k8_8_8_8UScaled, 2, {42.0f, 99.0f}, 0x0000632au},
{{"BufferStoreSscaledZeroFillXyz", "TBufferStoreSscaledZeroFillXyz"},
F::k8_8_8_8SScaled, 3, {-42.0f, 127.0f, -128.0f}, 0x00807fd6u},
{{"BufferStoreUscaled8Fraction", "TBufferStoreUscaled8Fraction"},
F::k8UScaled, 1, {42.875f}, 0xffffff2au},
{{"BufferStoreSscaled8Fraction", "TBufferStoreSscaled8Fraction"},
F::k8SScaled, 1, {-42.875f}, 0xffffffd6u},
{{"BufferStoreUscaled8Range", "TBufferStoreUscaled8Range"},
F::k8_8_8_8UScaled, 4, {-1.0f, 255.875f, 256.0f, 0.0f}, 0x00ffff00u},
{{"BufferStoreSscaled8Range", "TBufferStoreSscaled8Range"},
F::k8_8_8_8SScaled, 4, {-129.0f, -128.75f, 127.875f, 128.0f}, 0x7f7f8080u},
{{"BufferStoreUscaled16Range", "TBufferStoreUscaled16Range"},
F::k16_16UScaled, 2, {65536.0f, 42.875f}, 0x002affffu},
{{"BufferStoreSscaled16Range", "TBufferStoreSscaled16Range"},
F::k16_16SScaled, 2, {-32769.0f, 32768.0f}, 0x7fff8000u},
{{"BufferStoreUscaledPacked", "TBufferStoreUscaledPacked"},
F::k10_10_10_2UScaled, 4, {1024.0f, 3.875f, -1.0f, 4.0f}, 0xc0000fffu},
{{"BufferStoreSscaledPackedAlpha", "TBufferStoreSscaledPackedAlpha"},
F::k10_10_10_2SScaled, 4, {-512.75f, 511.875f, -3.875f, 3.875f}, 0xffd7fe00u},
{{"BufferStoreSscaledPackedFirstChannel", "TBufferStoreSscaledPackedFirstChannel"},
F::k2_10_10_10SScaled, 4, {3.0f, -512.0f, 511.0f, -1.0f}, 0xffdff803u},
{{"BufferStoreSnorm8SpecialValues", "TBufferStoreSnorm8SpecialValues"},
F::k8_8_8_8SNorm, 4, {nan, inf, -inf, -0.0f}, 0x00817f00u},
{{"BufferStoreUscaled8SpecialValues", "TBufferStoreUscaled8SpecialValues"},
F::k8_8_8_8UScaled, 4, {nan, inf, -inf, -0.0f}, 0x0000ff00u},
{{"BufferStoreSscaled8SpecialValues", "TBufferStoreSscaled8SpecialValues"},
F::k8_8_8_8SScaled, 4, {nan, inf, -inf, -0.0f}, 0x00807f00u},
};
constexpr O store_opcodes[2][4] = {
{O::BUFFER_STORE_FORMAT_X, O::BUFFER_STORE_FORMAT_XY,
O::BUFFER_STORE_FORMAT_XYZ, O::BUFFER_STORE_FORMAT_XYZW},
{O::TBUFFER_STORE_FORMAT_X, O::TBUFFER_STORE_FORMAT_XY,
O::TBUFFER_STORE_FORMAT_XYZ, O::TBUFFER_STORE_FORMAT_XYZW},
};
std::vector<TestCase> cases;
for (const auto &input : inputs) {
for (const bool typed : {false, true}) {
TestCase test;
test.name = input.name[typed];
for (u32 component = 0; component < input.components; ++component) {
AppendVMovLiteral(&test.code, component,
std::bit_cast<u32>(input.values[component]));
}
AppendVMovU32(&test.code, 20, 4);
const u32 opcode = 3 + input.components;
if (typed) {
const u32 format = BufferFormat(input.format);
test.code.push_back(EncodeMtbuf0(opcode, format & 0xfu, format >> 4u));
test.code.push_back(EncodeMtbuf1(opcode, 0, 0, 20));
} else {
test.code.push_back(EncodeMubuf0(opcode));
test.code.push_back(EncodeMubuf1(0, 0, 20));
}
AppendEnd(&test.code);
test.initial = {0xdeadbeefu, 0xffffffffu, 0xcafef00du};
test.expected = {0xdeadbeefu, input.expected, 0xcafef00du};
test.user_data = MakeStructuredStorageBufferData(
0, 12, false, BufferFormat(input.format));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, store_opcodes[typed][input.components - 1],
O::S_ENDPGM};
cases.push_back(std::move(test));
}
}
return cases;
}
TestCase BufferLoadFormatXyResource88UintExtractsBytes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMubuf0(0x01u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 2);
AppendStoreVgpr(&code, 1, 3);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyResource8_8UintExtractsBytes";
test.code = code;
test.initial = {0x0000807fu, 0x55667788u, 0, 0};
test.expected = {0x0000807fu, 0x55667788u, 0x0000007fu, 0x00000080u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18);
test.has_user_data = true;
return test;
}
TestCase BufferLoadFormatXyResource8888UnormConvertsFirstTwoComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMubuf0(0x01u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
TestCase test;
test.name =
"BufferLoadFormatXyResource8_8_8_8UnormConvertsFirstTwoComponents";
test.code = code;
test.initial = {0x44332211u, 0xdeadbeefu};
test.expected = {0x3d888889u, 0x3e088889u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 56);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXyResource88UintWritesBytes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x000000aau);
AppendVMovLiteral(&code, 1, 0x000000bbu);
code.push_back(EncodeMubuf0(0x05u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyResource8_8UintWritesBytes";
test.code = code;
test.initial = {0x11223344u, 0x55667788u};
test.expected = {0x1122bbaau, 0x55667788u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XY, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXyResource32UintWritesOneDword() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
AppendVMovLiteral(&code, 1, 0x12345678u);
code.push_back(EncodeMubuf0(0x05u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyResource32UintWritesOneDword";
test.code = code;
test.initial = {0x11111111u, 0x22222222u};
test.expected = {0xabcdef01u, 0x22222222u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XY, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 20);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXyzResource3232UintWritesTwoDwords() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
AppendVMovLiteral(&code, 1, 0x12345678u);
AppendVMovLiteral(&code, 2, 0x0badc0deu);
code.push_back(EncodeMubuf0(0x06u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzResource32_32UintWritesTwoDwords";
test.code = code;
test.initial = {0x11111111u, 0x22222222u, 0x33333333u};
test.expected = {0xabcdef01u, 0x12345678u, 0x33333333u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZ, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 62);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXyzwResource323232UintWritesThreeDwords() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
AppendVMovLiteral(&code, 1, 0x12345678u);
AppendVMovLiteral(&code, 2, 0x0badc0deu);
AppendVMovLiteral(&code, 3, 0xfeedfaceu);
code.push_back(EncodeMubuf0(0x07u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzwResource32_32_32UintWritesThreeDwords";
test.code = code;
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = {0xabcdef01u, 0x12345678u, 0x0badc0deu, 0x44444444u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 72);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXyzResource8UintWritesOneByte() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x000000aau);
AppendVMovLiteral(&code, 1, 0x000000bbu);
AppendVMovLiteral(&code, 2, 0x000000ccu);
code.push_back(EncodeMubuf0(0x06u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzResource8UintWritesOneByte";
test.code = code;
test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu};
test.expected = {0x112233aau, 0x55667788u, 0x99aabbccu};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZ, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXAddTidUsesLaneIndex() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x12345678u);
code.push_back(EncodeMubuf0(0x04u, 0, false, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXAddTidUsesLaneIndex";
test.code = code;
test.initial = std::vector<u32>(64, 0);
test.expected = std::vector<u32>(64, 0);
std::fill_n(test.expected.begin(), 32, 0x12345678u);
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 0;
test.compute_info.wave_size = 32;
test.has_compute_info = true;
test.user_data = MakeStructuredStorageBufferData(4, 64, true);
test.has_user_data = true;
test.required_spirv = {"BuiltIn SubgroupLocalInvocationId"};
test.forbidden_spirv = {"BuiltIn LocalInvocationId",
"OpGroupNonUniformBallot"};
return test;
}
TestCase BufferStoreFormatXDropsOutOfRangeRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
code.push_back(EncodeMubuf0(0x04u, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXDropsOutOfRangeRecord";
test.code = code;
test.initial = {0x11111111u, 0x22222222u};
test.expected = {0x11111111u, 0x22222222u};
test.storage_buffer_range_bytes = 4;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(4, 1, false, 20);
test.has_user_data = true;
return test;
}
TestCase TBufferCapturedZeroStrideOob(bool raw_bounds = false, bool scalar = false) {
using O = ShaderOpcode;
TestCase test;
test.name = scalar ? "TBufferZeroStrideOobPreservesScalarRead"
: raw_bounds ? "TBufferZeroStrideRawBoundsStillLoads"
: "TBufferCapturedZeroStrideOob";
test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u,
0x12345678u, 0xdeadbeefu, 0xdeadbeefu};
test.expected = test.initial;
for (u32 i = 0; i < 4; i++) {
test.expected[1 + i] = raw_bounds ? test.initial[i] : 0u;
}
test.expected[5] = test.initial[4];
if (scalar) test.expected[6] = test.initial[0];
test.user_data = MakeStructuredStorageBufferData(4, test.initial.size());
std::copy_n(test.user_data.begin(), 4, test.user_data.begin() + 4);
test.user_data[4] = 0x2000u;
// Captured HITMAN descriptor in s[32:35]: mode 0, zero stride, arbitrary base.
test.user_data[32] = 0x4a398620u;
test.user_data[33] = 0;
test.user_data[34] = 0x2b5u;
test.user_data[35] = 0x0004d000u | (raw_bounds ? 3u << 28u : 0u);
test.has_user_data = true;
test.storage_buffer_offsets = {0, 0, 16};
test.expected_buffer_resources = raw_bounds || scalar
? std::vector<u32>{0, 1, 2}
: std::vector<u32>{0, 2};
auto &code = test.code;
AppendVMovLiteral(&code, 0, test.initial[0]);
AppendStoreVgpr(&code, 0, 0); // Keep resource zero before the removed middle slot.
AppendVMovU32(&code, 30, 0);
code.push_back(0xea6b2000u);
code.push_back(0x8008191eu); // Exact pc 0x47c: typed float4, s[32:35], v30.
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMubuf0(0x0cu));
code.push_back(EncodeMubuf1(33, 1, 20));
if (scalar) {
code.push_back(EncodeSmem0(0x08u, 40, 16));
code.push_back(EncodeSmem1(0));
code.push_back(EncodeVop1(0x01u, 32, 40));
}
for (u32 i = 0; i < 4; i++) AppendStoreVgpr(&code, 25 + i, 1 + i);
AppendStoreVgpr(&code, 33, 5);
if (scalar) AppendStoreVgpr(&code, 32, 6);
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
if (scalar) test.opcodes.push_back(O::S_BUFFER_LOAD_DWORD);
test.ir_counts = {{"LoadBufferU32x4", raw_bounds ? 1u : 0u}};
return test;
}
TestCase BufferZeroStrideOobFormatsAndWidths() {
using O = ShaderOpcode;
TestCase test;
test.name = "BufferZeroStrideOobFormatsAndWidths";
test.initial = std::vector<u32>(20, 0xdeadbeefu);
test.expected = std::vector<u32>(20, 0);
test.expected[10] = 0x3f800000u;
test.expected[12] = 0x3f800000u;
test.expected[14] = 1;
test.expected[16] = 1;
test.expected[18] = 0xabcdef01u;
test.expected[19] = 0xdeadbeefu;
test.user_data = MakeStructuredStorageBufferData(4, 20);
for (u32 base : {4u, 8u}) {
test.user_data[base] = 0x4a398620u;
test.user_data[base + 2] = 0x2b5u;
const u32 format = base == 4 ? 77u : 75u; // Four 32-bit float or unsigned integer components.
test.user_data[base + 3] = (format << 12u) | DstSel(1, 4, 1, 0);
}
test.has_user_data = true;
test.expected_buffer_resources = std::vector<u32>{0};
auto &code = test.code;
AppendVMovU32(&code, 0, 0);
AppendStoreVgpr(&code, 0, 0);
AppendVMovU32(&code, 20, 0);
u32 destination = 0;
for (const auto [opcode, width] :
{std::pair{0x08u, 1u}, {0x0au, 1u}, {0x0cu, 1u}, {0x0du, 2u}, {0x0fu, 3u}}) {
code.push_back(EncodeMubuf0(opcode));
code.push_back(EncodeMubuf1(destination, 1, 20));
destination += width;
}
AppendVMovU32(&code, 8, 0);
AppendVMovU32(&code, 9, 0);
for (u32 base : {1u, 2u}) {
code.push_back(EncodeMubuf0(0x03u));
code.push_back(EncodeMubuf1(base == 1 ? 10 : 14, base, 20));
}
AppendVMovLiteral(&code, 18, 0xabcdef01u);
code.push_back(EncodeSop1(0x04u, 60, 126u)); // Preserve the partial wave's active lanes.
code.push_back(EncodeSop1(0x04u, 126, InlineU32(0)));
code.push_back(EncodeMubuf0(0x00u));
code.push_back(EncodeMubuf1(18, 1, 20));
code.push_back(EncodeSop1(0x04u, 126, 60u));
for (u32 i = 0; i < 19; i++) AppendStoreVgpr(&code, i, i);
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_UBYTE,
O::BUFFER_LOAD_USHORT, O::BUFFER_LOAD_DWORD,
O::BUFFER_LOAD_DWORDX2, O::BUFFER_LOAD_DWORDX3,
O::BUFFER_LOAD_FORMAT_X, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.ir_counts = {{"LoadBuffer", 0}};
return test;
}
TestCase TBufferLoadVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadOpcode(&code, 0x00, 0, 20);
AppendTBufferLoadOpcode(&code, 0x01, 1, 20);
AppendTBufferLoadOpcode(&code, 0x02, 3, 20);
AppendTBufferLoadOpcode(&code, 0x03, 6, 20);
for (u32 i = 0; i < 10u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadVariants",
code,
{0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u},
{0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u, 0x11121314u,
0x21222324u, 0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u},
{O::TBUFFER_LOAD_FORMAT_X, O::TBUFFER_LOAD_FORMAT_XY,
O::TBUFFER_LOAD_FORMAT_XYZ, O::TBUFFER_LOAD_FORMAT_XYZW,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyzwSnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
constexpr auto format =
BufferFormat(Prospero::BufferFormat::k32_32_32_32Float);
code.push_back(
EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 21 + i, 4 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "TBufferLoadFormatXyzwSnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
0, 0, 0, 0};
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(16, 2);
test.has_user_data = true;
return test;
}
TestCase TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
constexpr auto format = BufferFormat(Prospero::BufferFormat::k8_8_8_8UInt);
code.push_back(
EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 21 + i, 4 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x44332211u, 0, 0, 0, 0, 0, 0, 0};
test.expected = {0x44332211u, 0, 0, 0, 0x11u, 0x22u, 0x33u, 0x44u};
test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(4, 8);
test.has_user_data = true;
return test;
}
TestCase TBufferStoreFormatX8UintWritesOneByte() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 0, 0xaa);
code.push_back(EncodeMtbuf0(0x04, 5, 0));
code.push_back(EncodeMtbuf1(0x04, 0, 0, 20));
AppendEnd(&code);
return {"TBufferStoreFormatX8UintWritesOneByte",
code,
{0x11223344u},
{0x112233aau},
{O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_X, O::S_ENDPGM}};
}
TestCase TBufferLoadFormatX8UintZeroExtendsByte() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x00, 5, 0));
code.push_back(EncodeMtbuf1(0x00, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"TBufferLoadFormatX8UintZeroExtendsByte",
code,
{0x11223344u, 0},
{0x00000044u, 0},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatX8888UintExtractsFirstByte() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20,
Prospero::BufferFormat::k8_8_8_8UInt);
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"TBufferLoadFormatX8_8_8_8UintExtractsFirstByte",
code,
{0x44332211u, 0},
{0x00000011u, 0},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXIdxenUsesDescriptorStride() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
code.push_back(EncodeMtbuf0(0x00, 5, 0, 0, true, false));
code.push_back(EncodeMtbuf1(0x00, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "TBufferLoadFormatXIdxenUsesDescriptorStride";
test.code = code;
test.initial = {0x01020304u, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu};
test.expected = {0x0000007eu, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu};
test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(16, 3);
test.has_user_data = true;
return test;
}
TestCase TBufferLoadFormatX16FloatConvertsToFloat() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20,
Prospero::BufferFormat::k16Float);
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"TBufferLoadFormatX16FloatConvertsToFloat",
code,
{0x00003c00u, 0},
{0x3f800000u, 0},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXSintSignExtendsSubDword() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x00, 6, 0));
code.push_back(EncodeMtbuf1(0x00, 0, 0, 20));
AppendVMovU32(&code, 20, 4);
code.push_back(EncodeMtbuf0(0x00, 12, 0));
code.push_back(EncodeMtbuf1(0x00, 1, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
return {"TBufferLoadFormatXSintSignExtendsSubDword",
code,
{0x00000080u, 0x00008001u},
{0xffffff80u, 0xffff8001u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferStoreFormatXSintWritesSubDword() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0xffffff80u);
code.push_back(EncodeMtbuf0(0x04, 6, 0));
code.push_back(EncodeMtbuf1(0x04, 0, 0, 20));
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 1, 0xffff8001u);
code.push_back(EncodeMtbuf0(0x04, 12, 0));
code.push_back(EncodeMtbuf1(0x04, 1, 0, 20));
AppendEnd(&code);
return {"TBufferStoreFormatXSintWritesSubDword",
code,
{0x11223344u, 0x55667788u},
{0x11223380u, 0x55668001u},
{O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_X, O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXy88IntegerComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x01, 2, 1));
code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
AppendVMovU32(&code, 20, 4);
code.push_back(EncodeMtbuf0(0x01, 3, 1));
code.push_back(EncodeMtbuf1(0x01, 2, 0, 20));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXy8_8IntegerComponents",
code,
{0x0000807fu, 0x00007f80u, 0, 0},
{0x0000007fu, 0x00000080u, 0xffffff80u, 0x0000007fu},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferStoreFormatXy88IntegerComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 0, 0xaa);
AppendVMovU32(&code, 1, 0xbb);
code.push_back(EncodeMtbuf0(0x05, 2, 1));
code.push_back(EncodeMtbuf1(0x05, 0, 0, 20));
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 2, 0xffffff80u);
AppendVMovU32(&code, 3, 0x7f);
code.push_back(EncodeMtbuf0(0x05, 3, 1));
code.push_back(EncodeMtbuf1(0x05, 2, 0, 20));
AppendEnd(&code);
return {"TBufferStoreFormatXy8_8IntegerComponents",
code,
{0x11223344u, 0x55667788u},
{0x1122bbaau, 0x55667f80u},
{O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_XY, O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXy1616IntegerComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x01, 11, 1));
code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
AppendVMovU32(&code, 20, 4);
code.push_back(EncodeMtbuf0(0x01, 12, 1));
code.push_back(EncodeMtbuf1(0x01, 2, 0, 20));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXy16_16IntegerComponents",
code,
{0x80017fffu, 0x7fff8000u, 0, 0},
{0x00007fffu, 0x00008001u, 0xffff8000u, 0x00007fffu},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferStoreFormatXy1616IntegerComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 0, 0xaaaa);
AppendVMovU32(&code, 1, 0xbbbb);
code.push_back(EncodeMtbuf0(0x05, 11, 1));
code.push_back(EncodeMtbuf1(0x05, 0, 0, 20));
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 2, 0xffff8000u);
AppendVMovU32(&code, 3, 0x7fff);
code.push_back(EncodeMtbuf0(0x05, 12, 1));
code.push_back(EncodeMtbuf1(0x05, 2, 0, 20));
AppendEnd(&code);
return {"TBufferStoreFormatXy16_16IntegerComponents",
code,
{0x11223344u, 0x55667788u},
{0xbbbbaaaau, 0x7fff8000u},
{O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_XY, O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyz16161616UintLoadsHalfwords() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20,
Prospero::BufferFormat::k16_16_16_16UInt);
for (u32 i = 0; i < 3; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXyz16_16_16_16UintLoadsHalfwords",
code,
{0x22221111u, 0x44443333u, 0},
{0x00001111u, 0x00002222u, 0x00003333u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZ, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXy88UnormConvertsToFloat() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x01, 14, 0));
code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
return {"TBufferLoadFormatXy8_8UnormConvertsToFloat",
code,
{0x0000ff80u, 0},
{0x3f008081u, 0x3f800000u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXy88SnormConvertsToFloat() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x01, 0, 20,
Prospero::BufferFormat::k8_8SNorm);
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
return {"TBufferLoadFormatXy8_8SnormConvertsToFloat",
code,
{0x00007f80u, 0},
{0xbf800000u, 0x3f800000u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXy1616UnormConvertsToFloat() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x01, 7, 1));
code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
return {"TBufferLoadFormatXy16_16UnormConvertsToFloat",
code,
{0xffff8000u, 0},
{0x3f000080u, 0x3f800000u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x01, 8, 3));
code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
return {"TBufferLoadFormatXy8_8_8_8UnormConvertsFirstTwoComponents",
code,
{0x44332211u, 0xdeadbeefu},
{0x3d888889u, 0x3e088889u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyzw8888UintExtractsBytes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20,
Prospero::BufferFormat::k8_8_8_8UInt);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXyzw8_8_8_8UintExtractsBytes",
code,
{0x44332211u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu},
{0x00000011u, 0x00000022u, 0x00000033u, 0x00000044u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyzw1010102SnormConvertsToFloat() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20,
Prospero::BufferFormat::k10_10_10_2SNorm);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXyzw10_10_10_2SnormConvertsToFloat",
code,
{0x800801ffu, 0, 0, 0},
{0x3f800000u, 0xbf800000u, 0, 0xbf800000u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyz111110FloatUnpacks() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20,
Prospero::BufferFormat::k11_11_10Float);
for (u32 i = 0; i < 3; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXyz11_11_10FloatUnpacks",
code,
{0x781e03c0u, 0, 0},
{0x3f800000u, 0x3f800000u, 0x3f800000u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZ, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyzw3232FloatZerosMissingComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x03, 0, 4));
code.push_back(EncodeMtbuf1(0x03, 0, 0, 20));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXyzw32_32FloatZerosMissingComponents",
code,
{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u},
{0x11111111u, 0x22222222u, 0, 0},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x40000000u);
AppendVMovLiteral(&code, 2, 0x40400000u);
AppendVMovLiteral(&code, 3, 0x40800000u);
code.push_back(EncodeMtbuf0(0x07, 0, 4));
code.push_back(EncodeMtbuf1(0x07, 0, 0, 20));
AppendEnd(&code);
return {"TBufferStoreFormatXyzw32_32FloatWritesOnlyPresentComponents",
code,
{0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu},
{0x3f800000u, 0x40000000u, 0xccccccccu, 0xddddddddu},
{O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM}};
}
TestCase TBufferStoreVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x01020304u);
AppendTBufferStoreOpcode(&code, 0x04, 0, 20);
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 1, 0x11121314u);
AppendVMovLiteral(&code, 2, 0x21222324u);
AppendTBufferStoreOpcode(&code, 0x05, 1, 20);
AppendVMovU32(&code, 20, 12);
AppendVMovLiteral(&code, 3, 0x31323334u);
AppendVMovLiteral(&code, 4, 0x41424344u);
AppendVMovLiteral(&code, 5, 0x51525354u);
AppendTBufferStoreOpcode(&code, 0x06, 3, 20);
AppendVMovU32(&code, 20, 24);
AppendVMovLiteral(&code, 6, 0x61626364u);
AppendVMovLiteral(&code, 7, 0x71727374u);
AppendVMovLiteral(&code, 8, 0x81828384u);
AppendVMovLiteral(&code, 9, 0x91929394u);
AppendTBufferStoreOpcode(&code, 0x07, 6, 20);
AppendEnd(&code);
return {"TBufferStoreVariants",
code,
std::vector<u32>(10, 0),
{0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u,
0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u},
{O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_X, O::TBUFFER_STORE_FORMAT_XY,
O::TBUFFER_STORE_FORMAT_XYZ, O::TBUFFER_STORE_FORMAT_XYZW,
O::S_ENDPGM}};
}
TestCase FlatLoadDlcCaptured(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 64 ? "FlatLoadDlcCapturedWave64" : "FlatLoadDlcCapturedWave32";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
AppendVMovU32(&code, 4, 0);
AppendVMovU32(&code, 5, 31);
// PPSA04288 CS 05b58beb842bacbf:0x638, FLAT_LOAD_DWORDX2 v[0:1], v[4:5] DLC.
code.insert(code.end(), {0xdc341000u, 0x007d0004u});
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
// The descriptor store aliases the physical address read by the next load.
AppendStoreVgpr(&code, 1, 8);
code.push_back(0xbbfd0000u); // S_WAITCNT_VSCNT null, 0.
code.insert(code.end(), {0xdc341000u, 0x007d0004u});
AppendStoreVgpr(&code, 0, 2);
AppendStoreVgpr(&code, 1, 3);
AppendEnd(&code);
test.initial = {0, 0, 0, 0, 0, 0, 0, 0, 0x11223344u, 0x55667788u};
test.expected = {0x11223344u, 0x55667788u, 0x55667788u, 0x55667788u,
0, 0, 0, 0, 0x55667788u, 0x55667788u};
test.bda_mappings = {{0x1f00000000ull, 32}};
test.opcodes = {O::S_MOV_B64, O::V_MOV_B32, O::FLAT_LOAD_DWORDX2,
O::BUFFER_STORE_DWORD, O::S_WAITCNT_VSCNT, O::S_ENDPGM};
test.required_spirv = {"OpDecorate %buffers Coherent", " 0 Coherent", "Volatile|Aligned 4"};
test.decoded_counts = {{"dlc=1", 2}};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase FlatLoadVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeFlat0(0x08, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x09, 0, 2));
code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0a, 0, 0));
code.push_back(EncodeFlat1(2, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0b, 0, 2));
code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0c, 0, 8));
code.push_back(EncodeFlat1(13, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0d, 0, 8));
code.push_back(EncodeFlat1(4, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0f, 0, 8));
code.push_back(EncodeFlat1(6, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0e, 0, 8));
code.push_back(EncodeFlat1(9, 0x7d, 0, 20));
for (u32 i = 0; i < 14u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
const std::vector<u32> guest_memory{0x80ff7f01u, 0x00008001u, 0x11223344u,
0x55667788u, 0x99aabbccu, 0xddeeff00u};
constexpr size_t byte_offset = 8;
std::vector<u32> host_memory(
(byte_offset + guest_memory.size() * sizeof(u32) + sizeof(u32) - 1u) /
sizeof(u32));
std::memcpy(reinterpret_cast<uint8_t *>(host_memory.data()) + byte_offset,
guest_memory.data(), guest_memory.size() * sizeof(u32));
TestCase test{"FlatLoadVariants",
code,
std::move(host_memory),
{0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u,
0x55667788u, 0x11223344u, 0x55667788u, 0x99aabbccu,
0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u,
0x11223344u},
{O::V_MOV_B32, O::FLAT_LOAD_UBYTE, O::FLAT_LOAD_SBYTE,
O::FLAT_LOAD_USHORT, O::FLAT_LOAD_SSHORT, O::FLAT_LOAD_DWORD,
O::FLAT_LOAD_DWORDX2, O::FLAT_LOAD_DWORDX3,
O::FLAT_LOAD_DWORDX4, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.bda_mappings = {{0, static_cast<u32>(byte_offset)}};
return test;
}
TestCase FlatSubdwordLoadsApplyByteOffset() {
using O = ShaderOpcode;
constexpr size_t byte_offset = 3;
std::vector<u32> code;
AppendVMovU32(&code, 20, static_cast<u32>(byte_offset));
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeFlat0(0x08, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0a, 0, 1));
code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
constexpr u32 guest_word = 0x80ff7f01u;
std::vector<u32> host_memory(
(byte_offset + sizeof(guest_word) + sizeof(u32) - 1u) / sizeof(u32));
std::memcpy(reinterpret_cast<uint8_t *>(host_memory.data()) + byte_offset,
&guest_word, sizeof(guest_word));
TestCase test;
test.name = "FlatSubdwordLoadsApplyByteOffset";
test.code = std::move(code);
test.initial = std::move(host_memory);
test.expected = {0x01u, 0x0000ff7fu};
test.bda_mappings = {{0, 0}};
test.opcodes = {O::V_MOV_B32, O::FLAT_LOAD_UBYTE, O::FLAT_LOAD_USHORT,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase GlobalLoadShortD16Captured(u32 wave_size) {
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
constexpr u32 Marker = 0xa5a50000u;
constexpr uint64_t ScalarBase = GuestBase - Marker - 0x70u;
constexpr uint64_t ExecMask = 0xaaaaaaaa55555555ull;
constexpr u32 Values[] = {0u, 0x7fffu, 0x8000u, 0xffffu};
TestCase test;
test.name = wave_size == 32u ? "GlobalLoadShortD16CapturedWave32"
: "GlobalLoadShortD16CapturedWave64";
test.initial.resize(2u * wave_size + wave_size / 2u);
test.expected.resize(2u * wave_size);
for (u32 lane = 0; lane < wave_size; ++lane) {
const u32 value = Values[lane % std::size(Values)];
test.initial[2u * wave_size + lane / 2u] |= value << (16u * (lane % 2u));
test.expected[lane] = Marker | value;
test.expected[wave_size + lane] = Marker |
((ExecMask & (uint64_t{1} << lane)) != 0u ? value : lane * 2u);
}
auto &code = test.code;
AppendSMovLiteral(&code, 2, static_cast<u32>(ScalarBase));
AppendSMovLiteral(&code, 3, static_cast<u32>(ScalarBase >> 32u));
code.push_back(EncodeVop2(0x1a, 6, InlineU32(1), 0));
AppendVMovLiteral(&code, 5, Marker);
code.push_back(EncodeVop2(0x25, 6, Vgpr(5), 6));
for (bool masked : {false, true}) {
code.push_back(EncodeVop1(0x01, 5, Vgpr(6)));
if (masked) {
code.push_back(EncodeSop1(0x04, 8, 126));
AppendSMovLiteral(&code, 126, static_cast<u32>(ExecMask));
AppendSMovLiteral(&code, 127, static_cast<u32>(ExecMask >> 32u));
}
// Captured GLOBAL_LOAD_SHORT_D16: v5 supplies the byte offset and destination.
code.insert(code.end(), {0xdc908070u, 0x05020005u});
if (masked) code.push_back(EncodeSop1(0x04, 126, 8));
AppendStoreVgprAtLaneDwordOffset(&code, 5, 0, masked ? wave_size : 0u);
}
AppendEnd(&code);
test.bda_mappings = {{GuestBase, 8u * wave_size}};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_MOV_B32, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::FLAT_LOAD_SHORT_D16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"FLAT_LOAD_SHORT_D16", 2}};
return test;
}
TestCase FlatLoadShortD16AddressSegments() {
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, static_cast<u32>(GuestBase) + 2u);
AppendVMovLiteral(&code, 21, static_cast<u32>(GuestBase >> 32u));
for (u32 segment : {0u, 2u}) {
AppendVMovLiteral(&code, 3, 0x12345678u);
code.push_back(EncodeFlat0(0x24, segment, segment == 0u ? 0u : 0xffeu));
code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
AppendStoreVgpr(&code, 3, segment == 0u ? 0u : 1u);
}
AppendVMovU32(&code, 20, 0u);
AppendVMovLiteral(&code, 3, 0xbeef5678u);
code.push_back(EncodeFlat0(0x1c, 1, 0));
code.push_back(EncodeFlat1(0, 0x7d, 3, 20));
AppendVMovLiteral(&code, 3, 0x98761234u);
code.push_back(EncodeFlat0(0x24, 1, 2));
code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
AppendStoreVgpr(&code, 3, 2);
AppendEnd(&code);
TestCase test{"FlatLoadShortD16AddressSegments", code,
{0, 0, 0, 0, 0x80017fffu},
{0x12348001u, 0x12347fffu, 0x9876beefu},
{O::V_MOV_B32, O::FLAT_LOAD_SHORT_D16, O::FLAT_STORE_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.bda_mappings = {{GuestBase, 16}};
test.compute_info.scratch_size_dwords = 1;
return test;
}
TestCase BranchVccnzUsesCarryProducedWaveMask() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0), EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1), EncodeVop2(0x1a, 3, InlineU32(2), 1),
EncodeVopc(0xc0, Vgpr(0), 0),
};
AppendVMovLiteral(&code, 2, 0xffffffffu);
code.push_back(EncodeVop2(0x28, 5, Vgpr(2), 1));
code.push_back(EncodeSopp(0x07, 2));
code.push_back(EncodeVop1(0x01, 2, InlineU32(11)));
code.push_back(EncodeSopp(0x02, 1));
AppendVMovU32(&code, 2, 42);
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "BranchVccnzUsesCarryProducedWaveMask";
test.code = code;
test.expected = std::vector<u32>(8, 42);
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_F_U32, O::V_ADDC_U32, O::S_CBRANCH_VCCNZ,
O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase ScalarLoadAlignsComponentsAndMasksAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 1u);
AppendSMovLiteral(&code, 2, 3u);
AppendSMovLiteral(&code, 3, 0xffff0000u);
code.push_back(EncodeSmem0(0x00, 1, 1));
code.push_back(EncodeSmem1(3, 0));
AppendStoreSgpr(&code, 1, 0);
AppendEnd(&code);
return {"ScalarLoadAlignsComponentsAndMasksAddress",
code,
{0x11111111u, 0x22222222u},
{0x11111111u, 0x22222222u},
{O::S_MOV_B32, O::S_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarLoadAlignsDynamicBase() {
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x10000;
std::vector<u32> code;
AppendSMovLiteral(&code, 8, GuestBase);
AppendSMovLiteral(&code, 9, 0);
code.push_back(EncodeSop2(0x00, 8, 8, 4)); // group ID supplies each base low-bit pattern
AppendSMovLiteral(&code, 10, 7);
code.push_back(EncodeSmem0(0x01, 12, 4));
code.push_back(EncodeSmem1(7, 10)); // all three address components align independently
code.push_back(EncodeVop1(0x01, 0, 12));
code.push_back(EncodeVop1(0x01, 1, 13));
code.push_back(EncodeVop1(0x01, 2, 4));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(3), 2));
AppendBufferStoreDword(&code, 0, 3);
code.push_back(EncodeVop2(0x25, 3, InlineU32(4), 3));
AppendBufferStoreDword(&code, 1, 3);
AppendEnd(&code);
TestCase test;
test.name = "ScalarLoadAlignsDynamicBase";
test.code = std::move(code);
test.initial.resize(20);
test.initial[18] = 0x12345678;
test.initial[19] = 0x9abcdef0;
test.expected = test.initial;
for (u32 group = 0; group < 4; ++group) {
test.expected[group * 2] = test.initial[18];
test.expected[group * 2 + 1] = test.initial[19];
}
test.bda_mappings = {{GuestBase, 64}};
test.required_spirv = {"get_bda_pointer"};
test.forbidden_spirv = {"flattened_srt"};
test.opcodes = {O::S_MOV_B32, O::S_ADD_U32, O::S_LOAD_DWORDX2, O::V_MOV_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.wave_size = 32;
test.compute_info.threads_num[0] = 1;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 4;
test.has_compute_info = true;
test.dispatch_x = 4;
return test;
}
TestCase ScalarBufferFromLoopReadlane(u32 wave_size) {
using O = ShaderOpcode;
constexpr u32 GuestBase = 0x10000;
constexpr u32 DescriptorCount = 5;
constexpr u32 PayloadBase = GuestBase + DescriptorCount * 32;
std::vector<u32> code;
code.push_back(EncodeVop2(0x1a, 1, InlineU32(2), 0));
AppendBufferLoadDword(&code, 2, 1);
AppendSMovLiteral(&code, 8, GuestBase + 3);
AppendSMovLiteral(&code, 9, 0);
AppendSMovLiteral(&code, 28, 0);
const auto loop = static_cast<u32>(code.size());
code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); // SMEM ignores EXEC.
code.push_back(EncodeSmem0(0x03, 16, 4)); // descriptor and runtime soffset
code.push_back(EncodeSmem1(3, 125)); // null keeps this a flattening candidate.
code.push_back(EncodeSmem0(0x0a, 32, 8));
code.push_back(EncodeSmem1(7, 20));
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
code.push_back(EncodeVop1(0x01, 3, 28));
code.push_back(EncodeVop2(0x1a, 31, InlineU32(4), 3));
for (u32 component = 0; component < 4; ++component) {
code.push_back(EncodeVop1(0x01, 4, 32 + component));
AppendBufferStoreDword(&code, 4, 31);
code.push_back(EncodeVop2(0x25, 31, InlineU32(4), 31));
}
code.push_back(EncodeSop1(0x04, 126, 193));
code.push_back(EncodeSop2(0x00, 29, 28, InlineU32(wave_size == 64 ? 32 : 0)));
AppendVop3(&code, 0x360, 8, Vgpr(2), 29);
code.push_back(EncodeSop2(0x00, 28, 28, InlineU32(1)));
code.push_back(EncodeSopc(0x0a, 28, InlineU32(DescriptorCount)));
code.push_back(EncodeSopp(0x05, loop - static_cast<u32>(code.size()) - 1u));
AppendEnd(&code);
TestCase test;
test.name = wave_size == 32 ? "ScalarBufferFromLoopReadlane32"
: "ScalarBufferFromLoopReadlane64";
test.code = std::move(code);
test.initial.resize(128 + DescriptorCount * 8 + 8);
constexpr u32 records[] = {20, 5, 3, 0x40000005u, 20};
for (u32 i = 0; i < DescriptorCount; ++i) {
test.initial[(wave_size == 64 ? 32 : 0) + i] =
GuestBase + ((i + 1) % DescriptorCount) * 32 + 3;
const auto row = 128 + i * 8;
test.initial[row] = PayloadBase + 3;
test.initial[row + 1] = (i == 1 || i == 3 ? 4u << 16u : 0u) |
(i == 1 ? 1u << 31u : 0u);
test.initial[row + 2] = records[i];
// Scalar addressing ignores FORMAT, swizzle, ADD_TID, INDEX_STRIDE and OOB_SELECT.
test.initial[row + 3] = i == 1 ? (3u << 21u) | (1u << 23u) | (3u << 28u) : 0u;
test.initial[row + 4] = i == 4 ? 0xffffffffu : 11u;
}
for (u32 i = 0; i < 8; ++i)
test.initial[128 + DescriptorCount * 8 + i] = 0x11111111u * (i + 1u);
test.expected = {0x44444444u, 0x55555555u, 0, 0,
0x44444444u, 0x55555555u, 0, 0,
0, 0, 0, 0,
0x44444444u, 0x55555555u, 0x66666666u, 0x77777777u,
0, 0, 0, 0};
test.bda_mappings = {{GuestBase, 512}};
test.required_spirv = {"get_bda_pointer", "OpLoopMerge", "OpGroupNonUniformShuffle"};
test.forbidden_spirv = {"flattened_srt"};
test.opcodes = {O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD, O::S_MOV_B32,
O::S_MOV_B64, O::S_LOAD_DWORDX8, O::S_BUFFER_LOAD_DWORDX4,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::V_ADD_NC_U32,
O::S_ADD_U32, O::V_READLANE_B32, O::S_CMP_LT_U32,
O::S_CBRANCH_SCC1, O::S_ENDPGM};
test.compute_info.wave_size = wave_size;
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase FlatVirtualAddressRebasesGuestAllocation() {
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
constexpr uint64_t ExtendedBase = Libs::LibKernel::Memory::kExtendedMemoryBase + GuestBase;
static_assert(BufferCache::PageIndex(ExtendedBase) ==
((Libs::Graphics::LOWER_ADDRESS_SIZE + GuestBase) >> BufferCache::CACHING_PAGEBITS));
static_assert(BufferCache::GuestAddress(BufferCache::PageIndex(ExtendedBase) <<
BufferCache::CACHING_PAGEBITS) == ExtendedBase);
std::vector<u32> code;
AppendVMovLiteral(&code, 20, static_cast<u32>(GuestBase + 4u));
AppendVMovLiteral(&code, 21, static_cast<u32>(GuestBase >> 32u));
code.push_back(EncodeFlat0(0x0c, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0c, 2, 0));
code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
AppendVMovLiteral(&code, 21, static_cast<u32>((GuestBase >> 32u) + 1u));
code.push_back(EncodeFlat0(0x0c, 0, 0));
code.push_back(EncodeFlat1(2, 0x7d, 0, 20));
AppendVMovLiteral(&code, 20, static_cast<u32>(ExtendedBase + 4u));
AppendVMovLiteral(&code, 21, static_cast<u32>(ExtendedBase >> 32u));
code.push_back(EncodeFlat0(0x0c, 0, 0));
code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendStoreVgpr(&code, 2, 2);
AppendStoreVgpr(&code, 3, 3);
AppendEnd(&code);
TestCase test;
test.name = "FlatVirtualAddressRebasesGuestAllocation";
test.code = std::move(code);
test.initial = {0xfeedfaceu, 0xcafebabeu, 0, 0x12345678u};
test.expected = {0x12345678u, 0x12345678u, 0, 0xcafebabeu};
test.bda_mappings = {{GuestBase, 8}, {ExtendedBase, 0}};
test.opcodes = {O::V_MOV_B32, O::FLAT_LOAD_DWORD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase GlobalSignedImmediateRebasesBeforeSaddr() {
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, static_cast<u32>(GuestBase));
AppendSMovLiteral(&code, 1, static_cast<u32>(GuestBase >> 32u));
AppendVMovU32(&code, 20, 8);
code.push_back(EncodeFlat0(0x0c, 2, 0xffcu));
code.push_back(EncodeFlat1(0, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "GlobalSignedImmediateRebasesBeforeSaddr";
test.code = std::move(code);
test.initial = {0xfeedfaceu, 0xcafebabeu, 0x11111111u, 0x12345678u};
test.expected = {0x12345678u};
test.bda_mappings = {{GuestBase, 8}};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase FlatSegmentIgnoresSaddrAndMasksOffsetMsb() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(4)));
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeFlat0(0x0c, 0, 4));
code.push_back(EncodeFlat1(0, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeFlat0(0x0c, 0, 0x804));
code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
std::vector<u32> initial(514, 0);
initial[1] = 0x11111111u;
initial[2] = 0x22222222u;
initial[513] = 0x33333333u;
TestCase test;
test.name = "FlatSegmentIgnoresSaddrAndMasksOffsetMsb";
test.code = code;
test.initial = initial;
test.expected = {0x11111111u, 0x11111111u};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.bda_mappings = {{0, 0}};
return test;
}
TestCase ScratchIsPrivatePerInvocation() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(20, InlineU32(0)));
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeFlat0(0x1c, 1, 4));
code.push_back(EncodeFlat1(0, 20, 0, 21));
code.push_back(EncodeFlat0(0x0c, 1, 4));
code.push_back(EncodeFlat1(5, 20, 0, 21));
AppendStoreVgprAtLaneDwordOffset(&code, 5, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ScratchIsPrivatePerInvocation";
test.code = std::move(code);
for (u32 lane = 0; lane < 64; lane++) {
test.expected.push_back(lane);
}
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32,
O::FLAT_STORE_DWORD, O::FLAT_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.compute_info.scratch_size_dwords = 2;
test.has_compute_info = true;
return test;
}
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
TestCase FlatStoreVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 0, 0xaa);
code.push_back(EncodeFlat0(0x18, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
AppendVMovU32(&code, 20, 2);
AppendVMovLiteral(&code, 1, 0x0000bbccu);
code.push_back(EncodeFlat0(0x1a, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 1, 20));
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
code.push_back(EncodeFlat0(0x1d, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 2, 20));
AppendVMovU32(&code, 20, 12);
AppendVMovLiteral(&code, 4, 0x33333333u);
AppendVMovLiteral(&code, 5, 0x44444444u);
AppendVMovLiteral(&code, 6, 0x55555555u);
code.push_back(EncodeFlat0(0x1f, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 4, 20));
AppendVMovU32(&code, 20, 24);
AppendVMovLiteral(&code, 7, 0x66666666u);
AppendVMovLiteral(&code, 8, 0x77777777u);
AppendVMovLiteral(&code, 9, 0x88888888u);
AppendVMovLiteral(&code, 10, 0x99999999u);
code.push_back(EncodeFlat0(0x1e, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 7, 20));
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeFlat0(0x1c, 0, 40));
code.push_back(EncodeFlat1(0, 0x7d, 2, 20));
AppendEnd(&code);
TestCase test{"FlatStoreVariants",
code,
std::vector<u32>(12, 0),
{0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u,
0x44444444u, 0x55555555u, 0x66666666u, 0x77777777u,
0x88888888u, 0x99999999u, 0x11111111u, 0},
{O::V_MOV_B32, O::FLAT_STORE_BYTE, O::FLAT_STORE_SHORT,
O::FLAT_STORE_DWORD, O::FLAT_STORE_DWORDX2,
O::FLAT_STORE_DWORDX3, O::FLAT_STORE_DWORDX4, O::S_ENDPGM}};
return test;
}
#endif
TestCase DsReadWriteVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x11223344u);
code.push_back(EncodeDs0(0x0d, 0));
code.push_back(EncodeDs1(0, 2, 1));
code.push_back(EncodeDs0(0x36, 0));
code.push_back(EncodeDs1(3, 0, 1));
AppendVMovU32(&code, 4, 0xaa);
code.push_back(EncodeDs0(0x1e, 4));
code.push_back(EncodeDs1(0, 4, 1));
code.push_back(EncodeDs0(0x3a, 4));
code.push_back(EncodeDs1(5, 0, 1));
code.push_back(EncodeDs0(0x39, 4));
code.push_back(EncodeDs1(6, 0, 1));
AppendVMovLiteral(&code, 7, 0x000080ffu);
code.push_back(EncodeDs0(0x1f, 8));
code.push_back(EncodeDs1(0, 7, 1));
code.push_back(EncodeDs0(0x3c, 8));
code.push_back(EncodeDs1(8, 0, 1));
code.push_back(EncodeDs0(0x3b, 8));
code.push_back(EncodeDs1(9, 0, 1));
AppendVMovLiteral(&code, 10, 0x10101010u);
AppendVMovLiteral(&code, 11, 0x11111111u);
code.push_back(EncodeDs0(0x4d, 12));
code.push_back(EncodeDs1(0, 10, 1));
code.push_back(EncodeDs0(0x76, 12));
code.push_back(EncodeDs1(14, 0, 1));
AppendVMovLiteral(&code, 16, 0x20202020u);
AppendVMovLiteral(&code, 17, 0x21212121u);
AppendVMovLiteral(&code, 18, 0x22222222u);
code.push_back(EncodeDs0(0xde, 20));
code.push_back(EncodeDs1(0, 16, 1));
code.push_back(EncodeDs0(0xfe, 20));
code.push_back(EncodeDs1(19, 0, 1));
AppendVMovLiteral(&code, 22, 0x30303030u);
AppendVMovLiteral(&code, 23, 0x31313131u);
AppendVMovLiteral(&code, 24, 0x32323232u);
AppendVMovLiteral(&code, 25, 0x33333333u);
code.push_back(EncodeDs0(0xdf, 32));
code.push_back(EncodeDs1(0, 22, 1));
code.push_back(EncodeDs0(0xff, 32));
code.push_back(EncodeDs1(26, 0, 1));
const u32 results[] = {3, 5, 6, 8, 9, 14, 15, 19, 20, 21, 26, 27, 28, 29};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"DsReadWriteVariants",
code,
std::vector<u32>(14, 0),
{0x11223344u, 0xaau, 0xffffffaau, 0x80ffu, 0xffff80ffu, 0x10101010u,
0x11111111u, 0x20202020u, 0x21212121u, 0x22222222u, 0x30303030u,
0x31313131u, 0x32323232u, 0x33333333u},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_B32, O::DS_WRITE_B8,
O::DS_READ_U8, O::DS_READ_I8, O::DS_WRITE_B16, O::DS_READ_U16,
O::DS_READ_I16, O::DS_WRITE_B64, O::DS_READ_B64, O::DS_WRITE_B96,
O::DS_READ_B96, O::DS_WRITE_B128, O::DS_READ_B128,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsReadU16D16CapturedPreservesHighHalf() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x89abcdefu);
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 2, 1)); // ds_write_b32 v2, v1
AppendVMovLiteral(&code, 3, 0x12345678u);
code.push_back(EncodeDs0(0xa6));
code.push_back(EncodeDs1(3, 0, 1)); // ds_read_u16_d16 v3, v1
AppendVMovU32(&code, 7, 0);
code.push_back(0xda980000u);
code.push_back(0x07000007u); // captured ds_read_u16_d16 v7, v7
AppendStoreVgpr(&code, 3, 0);
AppendStoreVgpr(&code, 7, 1);
AppendEnd(&code);
TestCase test{"DsReadU16D16CapturedPreservesHighHalf",
code,
std::vector<u32>(2, 0),
{0x1234cdefu, 0x0000cdefu},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_U16_D16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"DS_READ_U16_D16", 2}};
return test;
}
TestCase DsWriteB16D16HiCapturedUsesHighHalf() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 3, 0xdeadbeefu);
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 3, 1)); // ds_write_b32 v1, v3
AppendVMovU32(&code, 20, 2);
AppendVMovLiteral(&code, 2, 0x1234abcdu);
AppendVMovLiteral(&code, 0, 0x87654321u);
code.push_back(0xda840000u);
code.push_back(0x00000214u); // captured ds_write_b16_d16_hi v20, v2
code.push_back(EncodeDs0(0x36));
code.push_back(EncodeDs1(4, 0, 1)); // ds_read_b32 v4, v1
AppendStoreVgpr(&code, 4, 0);
AppendStoreVgpr(&code, 0, 1);
AppendEnd(&code);
TestCase test{"DsWriteB16D16HiCapturedUsesHighHalf",
code,
std::vector<u32>(2, 0),
{0x1234beefu, 0x87654321u},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_WRITE_B16_D16_HI,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"DS_WRITE_B16_D16_HI", 1}};
return test;
}
TestCase DsWriteB8D16HiWritesByteTwo() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 3, 0xdeadbeefu);
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 3, 1)); // ds_write_b32 v1, v3
// Every byte of the source differs, so selecting any byte but 23:16 fails.
AppendVMovU32(&code, 20, 1);
AppendVMovLiteral(&code, 2, 0x11a22233u);
AppendVMovLiteral(&code, 0, 0x87654321u);
code.push_back(EncodeDs0(0xa0));
code.push_back(EncodeDs1(0, 2, 20)); // ds_write_b8_d16_hi v20, v2
code.push_back(EncodeDs0(0x36));
code.push_back(EncodeDs1(4, 0, 1)); // ds_read_b32 v4, v1
AppendStoreVgpr(&code, 4, 0);
AppendStoreVgpr(&code, 0, 1);
AppendEnd(&code);
TestCase test{"DsWriteB8D16HiWritesByteTwo",
code,
std::vector<u32>(2, 0),
{0xdeada2efu, 0x87654321u},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_WRITE_B8_D16_HI,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"DS_WRITE_B8_D16_HI", 1}};
return test;
}
TestCase DsReadU16D16HiCapturedPreservesLowHalf() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x89abcdefu);
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 2, 1)); // ds_write_b32 v2, v1
AppendVMovLiteral(&code, 5, 0x12345678u);
AppendVMovU32(&code, 6, 0);
code.push_back(0xda9c0000u);
code.push_back(0x05000006u); // captured ds_read_u16_d16_hi v5, v6
AppendStoreVgpr(&code, 5, 0);
AppendEnd(&code);
TestCase test{"DsReadU16D16HiCapturedPreservesLowHalf",
code,
std::vector<u32>(1, 0),
{0xcdef5678u},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_U16_D16_HI,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"DS_READ_U16_D16_HI", 1}};
return test;
}
TestCase DsReadWrite2Variants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
code.push_back(EncodeDs0(0x0e, (3u << 8u) | 1u));
code.push_back(EncodeDs1Ex(0, 3, 2, 1));
code.push_back(EncodeDs0(0x37, (3u << 8u) | 1u));
code.push_back(EncodeDs1Ex(4, 0, 0, 1));
AppendVMovLiteral(&code, 6, 0x33333333u);
AppendVMovLiteral(&code, 7, 0x44444444u);
AppendVMovLiteral(&code, 8, 0x55555555u);
AppendVMovLiteral(&code, 9, 0x66666666u);
code.push_back(EncodeDs0(0x4d, 32));
code.push_back(EncodeDs1(0, 6, 1));
code.push_back(EncodeDs0(0x4d, 48));
code.push_back(EncodeDs1(0, 8, 1));
code.push_back(EncodeDs0(0x77, (6u << 8u) | 4u));
code.push_back(EncodeDs1Ex(10, 0, 0, 1));
AppendVMovLiteral(&code, 14, 0x77777777u);
AppendVMovLiteral(&code, 15, 0x88888888u);
code.push_back(EncodeDs0(0x0f, (2u << 8u) | 1u));
code.push_back(EncodeDs1Ex(0, 15, 14, 1));
code.push_back(EncodeDs0(0x38, (2u << 8u) | 1u));
code.push_back(EncodeDs1Ex(16, 0, 0, 1));
AppendVMovLiteral(&code, 18, 0x99999999u);
AppendVMovLiteral(&code, 19, 0xaaaaaaaau);
AppendVMovLiteral(&code, 20, 0xbbbbbbbbu);
AppendVMovLiteral(&code, 21, 0xccccccccu);
code.push_back(EncodeDs0(0x4e, (10u << 8u) | 8u));
code.push_back(EncodeDs1Ex(0, 20, 18, 1));
code.push_back(EncodeDs0(0x77, (10u << 8u) | 8u));
code.push_back(EncodeDs1Ex(22, 0, 0, 1));
AppendVMovLiteral(&code, 26, 0xddddddddu);
AppendVMovLiteral(&code, 27, 0xeeeeeeeeu);
AppendVMovLiteral(&code, 28, 0xf0f0f0f0u);
AppendVMovLiteral(&code, 29, 0x12345678u);
code.push_back(EncodeDs0(0x4f, (2u << 8u) | 1u));
code.push_back(EncodeDs1Ex(0, 28, 26, 1));
code.push_back(EncodeDs0(0x78, (2u << 8u) | 1u));
code.push_back(EncodeDs1Ex(34, 0, 0, 1));
const u32 results[] = {4, 5, 10, 11, 12, 13, 22, 23,
24, 25, 16, 17, 34, 35, 36, 37};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"DsReadWrite2Variants",
code,
std::vector<u32>(16, 0),
{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u,
0x66666666u, 0x99999999u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu,
0x77777777u, 0x88888888u, 0xddddddddu, 0xeeeeeeeeu, 0xf0f0f0f0u,
0x12345678u},
{O::V_MOV_B32, O::DS_WRITE2_B32, O::DS_READ2_B32, O::DS_WRITE_B64,
O::DS_READ2_B64, O::DS_WRITE2_B64, O::DS_WRITE2ST64_B32,
O::DS_READ2ST64_B32, O::DS_WRITE2ST64_B64, O::DS_READ2ST64_B64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsWideReadSnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 6, 0x11111111u);
AppendVMovLiteral(&code, 7, 0x22222222u);
code.push_back(EncodeDs0(0x4d, 0));
code.push_back(EncodeDs1(0, 6, 1));
AppendVMovLiteral(&code, 8, 0x33333333u);
AppendVMovLiteral(&code, 9, 0x44444444u);
code.push_back(EncodeDs0(0x4d, 16));
code.push_back(EncodeDs1(0, 8, 1));
code.push_back(EncodeDs0(0x77, 2u << 8u));
code.push_back(EncodeDs1Ex(1, 0, 0, 1));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 1u + i, i);
}
AppendEnd(&code);
return {"DsWideReadSnapshotsOverlappingAddress",
code,
std::vector<u32>(4, 0),
{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u},
{O::V_MOV_B32, O::DS_WRITE_B64, O::DS_READ2_B64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsReadWrite2EqualOffsetsUseData0() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
code.push_back(EncodeDs0(0x0e, (1u << 8u) | 1u));
code.push_back(EncodeDs1Ex(0, 3, 2, 1));
AppendVMovLiteral(&code, 4, 0x33333333u);
AppendVMovLiteral(&code, 5, 0x44444444u);
AppendVMovLiteral(&code, 6, 0x55555555u);
AppendVMovLiteral(&code, 7, 0x66666666u);
code.push_back(EncodeDs0(0x4e, (2u << 8u) | 2u));
code.push_back(EncodeDs1Ex(0, 6, 4, 1));
code.push_back(EncodeDs0(0x37, (1u << 8u) | 1u));
code.push_back(EncodeDs1Ex(8, 0, 0, 1));
code.push_back(EncodeDs0(0x77, (2u << 8u) | 2u));
code.push_back(EncodeDs1Ex(10, 0, 0, 1));
for (u32 i = 0; i < 6; i++) {
AppendStoreVgpr(&code, 8u + i, i);
}
AppendEnd(&code);
return {"DsReadWrite2EqualOffsetsUseData0",
code,
std::vector<u32>(6, 0),
{0x11111111u, 0x11111111u, 0x33333333u, 0x44444444u, 0x33333333u,
0x44444444u},
{O::V_MOV_B32, O::DS_WRITE2_B32, O::DS_WRITE2_B64, O::DS_READ2_B32,
O::DS_READ2_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsWideLdsPartialBounds() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
AppendVMovLiteral(&code, 4, 0x33333333u);
AppendVMovLiteral(&code, 5, 0x44444444u);
code.push_back(EncodeDs0(0xdf));
code.push_back(EncodeDs1(0, 2, 1));
code.push_back(EncodeDs0(0xff));
code.push_back(EncodeDs1(6, 0, 1));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 6u + i, i);
}
AppendEnd(&code);
TestCase test{"DsWideLdsPartialBounds",
code,
std::vector<u32>(4, 0),
{0x11111111u, 0x22222222u, 0x33333333u, 0},
{O::V_MOV_B32, O::DS_WRITE_B128, O::DS_READ_B128,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.compute_info.lds_size_dwords = 3;
return test;
}
TestCase DsWideGdsPartialBounds() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
AppendVMovLiteral(&code, 4, 0x33333333u);
AppendVMovLiteral(&code, 5, 0x44444444u);
code.push_back(EncodeDs0(0xdf, 0, true));
code.push_back(EncodeDs1(0, 2, 1));
code.push_back(EncodeDs0(0xff, 0, true));
code.push_back(EncodeDs1(6, 0, 1));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 6u + i, i);
}
AppendEnd(&code);
TestCase test{"DsWideGdsPartialBounds",
code,
std::vector<u32>(4, 0),
{0x11111111u, 0x22222222u, 0x33333333u, 0},
{O::V_MOV_B32, O::DS_WRITE_B128, O::DS_READ_B128,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.gds_initial = {0, 0, 0};
test.expected_gds = {0x11111111u, 0x22222222u, 0x33333333u};
return test;
}
TestCase DsAtomicNoReturnVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
const u32 initial[] = {10, 10, 2, 1, 0, 3, 9, 0xfffffff0u, 0xfffffff0u,
10, 10, 0xf0f0u, 0xf000u, 0xf00fu};
const u32 values[] = {5, 3, 2, 5, 2, 5, 5, 5, 5,
5, 20, 0x0ff0u, 0x0f00u, 0x00ffu};
const u32 ops[] = {0x00, 0x01, 0x03, 0x03, 0x04, 0x04, 0x04,
0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b};
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovLiteral(&code, 2, initial[i]);
code.push_back(EncodeDs0(0x0d, i * 4u));
code.push_back(EncodeDs1(0, 2, 1));
AppendVMovLiteral(&code, 3, values[i]);
code.push_back(EncodeDs0(ops[i], i * 4u));
code.push_back(EncodeDs1(0, 3, 1));
code.push_back(EncodeDs0(0x36, i * 4u));
code.push_back(EncodeDs1(10u + i, 0, 1));
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, 10u + i, i);
}
AppendEnd(&code);
return {"DsAtomicNoReturnVariants",
code,
std::vector<u32>(14, 0),
{15, 7, 0, 2, 2, 2, 5, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u,
0xf0f0u},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_ADD_U32, O::DS_SUB_U32,
O::DS_INC_U32, O::DS_DEC_U32, O::DS_MIN_I32, O::DS_MAX_I32,
O::DS_MIN_U32, O::DS_MAX_U32, O::DS_AND_B32, O::DS_OR_B32,
O::DS_XOR_B32, O::DS_READ_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase DsMaskedOrBounds(bool gds) {
using O = ShaderOpcode;
TestCase test;
test.name = gds ? "DsMaskedOrGdsBounds" : "DsMaskedOrLdsBounds";
auto &code = test.code;
constexpr u32 initial[] = {0xf0f00ff0u, 0x12345678u, 0x87654321u};
constexpr u32 masks[] = {0x00ff00ffu, 0u, 0xffffffffu};
constexpr u32 values[] = {0x33000300u, 0x80000001u, 0xabcdef01u};
AppendVMovU32(&code, 0, 0x1234);
for (u32 i = 0; i < std::size(initial); ++i) {
AppendVMovU32(&code, 2, i * 4u);
AppendVMovLiteral(&code, 8, initial[i]);
code.push_back(EncodeDs0(0x0d, 0x144, gds));
code.push_back(EncodeDs1(0, 8, 2));
// Captured mask v8 and data v10 are distinct; LDS wraps and aligns the address.
AppendVMovU32(&code, 2, (gds ? 0u : 0x10000u) + i * 4u + 3u);
AppendVMovLiteral(&code, 8, masks[i]);
AppendVMovLiteral(&code, 10, values[i]);
code.insert(code.end(), {0xd8300144u | (gds ? 1u << 17u : 0u), 0x000a0802u});
}
AppendVMovLiteral(&code, 8, 0xffffffffu);
AppendVMovU32(&code, 10, 0);
AppendVMovU32(&code, 2, 12); // First DWORD beyond the allocation.
code.insert(code.end(), {EncodeDs0(0x0c, 0x144, gds), 0x000a0802u});
AppendVMovU32(&code, 2, 0);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.insert(code.end(), {EncodeDs0(0x0c, 0x144, gds), 0x000a0802u});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
for (u32 i = 0; i < std::size(initial); ++i) {
code.push_back(EncodeDs0(0x36, 0x144u + i * 4u, gds));
code.push_back(EncodeDs1(11, 0, 2));
AppendStoreVgpr(&code, 11, i);
test.expected.push_back((initial[i] & ~masks[i]) | values[i]);
}
AppendStoreVgpr(&code, 0, 3); // The encoded VDst must stay untouched.
test.expected.push_back(0x1234);
AppendEnd(&code);
test.compute_info.lds_size_dwords = 0x144 / 4 + 3;
if (gds) {
test.gds_initial.resize(0x144 / 4 + 3);
test.expected_gds = test.gds_initial;
std::copy_n(test.expected.begin(), 3, test.expected_gds.begin() + 0x144 / 4);
}
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, O::DS_MSKOR_B32,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicCompareExchange"};
return test;
}
TestCase DsMaskedOrContention(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 32 ? "DsMaskedOrContentionWave32" : "DsMaskedOrContentionWave64";
auto &code = test.code;
AppendVMovU32(&code, 2, 0);
AppendVMovLiteral(&code, 8, 0xffffffffu);
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 8, 2));
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
AppendVMovU32(&code, 8, 1);
code.push_back(EncodeVop2(0x1a, 8, Vgpr(0), 8));
AppendVMovU32(&code, 9, 0);
AppendSMovLiteral(&code, 126, 0x55555555u);
AppendSMovLiteral(&code, 127, 0xaaaaaaaau);
code.push_back(EncodeDs0(0x0c));
code.push_back(EncodeDs1Ex(0, 9, 8, 2));
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
code.push_back(EncodeDs0(0x36));
code.push_back(EncodeDs1(11, 0, 2));
AppendStoreVgprAtLaneDwordOffset(&code, 11, 0, 0);
AppendEnd(&code);
test.expected.assign(128, wave_size == 32 ? 0xaaaaaaaau : 0u);
test.compute_info.threads_num[0] = 128;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.compute_info.lds_size_dwords = 1;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_U32,
O::V_LSHLREV_B32, O::S_BARRIER, O::DS_WRITE_B32, O::DS_MSKOR_B32,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicCompareExchange"};
return test;
}
TestCase DsAtomicReturnVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
const u32 initial[] = {10, 10, 0xfffffff0u, 0xfffffff0u, 10,
10, 0xf0f0u, 0xf000u, 0xf00fu, 10};
const u32 values[] = {5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu, 99};
const u32 ops[] = {0x20, 0x21, 0x25, 0x26, 0x27,
0x28, 0x29, 0x2a, 0x2b, 0x2d};
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovLiteral(&code, 2, initial[i]);
code.push_back(EncodeDs0(0x0d, i * 4u));
code.push_back(EncodeDs1(0, 2, 1));
AppendVMovLiteral(&code, 3, values[i]);
code.push_back(EncodeDs0(ops[i], i * 4u));
code.push_back(EncodeDs1(10u + i, 3, 1));
code.push_back(EncodeDs0(0x36, i * 4u));
code.push_back(EncodeDs1(20u + i, 0, 1));
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, 10u + i, i);
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, 20u + i, i + 10u);
}
AppendEnd(&code);
return {
"DsAtomicReturnVariants",
code,
std::vector<u32>(20, 0),
{10, 10, 0xfffffff0u, 0xfffffff0u, 10, 10, 0xf0f0u, 0xf000u, 0xf00fu, 10,
15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u, 99},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_ADD_RTN_U32, O::DS_SUB_RTN_U32,
O::DS_MIN_RTN_I32, O::DS_MAX_RTN_I32, O::DS_MIN_RTN_U32,
O::DS_MAX_RTN_U32, O::DS_AND_RTN_B32, O::DS_OR_RTN_B32,
O::DS_XOR_RTN_B32, O::DS_WRXCHG_RTN_B32, O::DS_READ_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsAtomic64Bounds(bool add) {
using O = ShaderOpcode;
TestCase test;
test.name = add ? "DsAddU64Bounds" : "DsOrB64Bounds";
const u32 atomic_word = add ? 0xd9000000u : 0xd9280000u;
auto &code = test.code;
AppendVMovU32(&code, 0, 0x1234);
AppendVMovU32(&code, 7, 0);
const std::array<u32, 5> initial = add
? std::array<u32, 5>{0xffffffffu, 1, 0xffffffffu, 0xfffffffeu, 0x13579bdf}
: std::array<u32, 5>{0x100, 0x200, 0x400, 0x800, 0x13579bdf};
for (u32 i = 0; i < std::size(initial); ++i) {
AppendVMovLiteral(&code, 3, initial[i]);
code.push_back(EncodeDs0(0x0d, i * 4u));
code.push_back(EncodeDs1(0, 3, 7));
}
AppendVMovU32(&code, 63, 1);
AppendVMovU32(&code, 64, 2);
AppendVMovU32(&code, 7, 3); // The 64-bit address ignores the low three bits.
code.insert(code.end(), {atomic_word, 0x00003f07u});
AppendVMovLiteral(&code, 7, 0x10007);
code.push_back(EncodeDs0(add ? 0x40 : 0x4a, 2)); // Offset before wrap/alignment -> 8.
code.push_back(EncodeDs1(0, 63, 7));
AppendVMovLiteral(&code, 63, 0xffffffffu);
AppendVMovLiteral(&code, 64, 0xffffffffu);
AppendVMovU32(&code, 7, 16); // Only one DWORD remains: ignore the entire atomic.
code.insert(code.end(), {atomic_word, 0x00003f07u});
AppendVMovU32(&code, 7, 0);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.insert(code.end(), {atomic_word, 0x00003f07u});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
for (u32 i = 0; i < std::size(initial); ++i) {
code.push_back(EncodeDs0(0x36, i * 4u));
code.push_back(EncodeDs1(8, 0, 7));
AppendStoreVgpr(&code, 8, i);
}
AppendStoreVgpr(&code, 0, 5); // No-return DS must not overwrite the encoded VDst.
AppendEnd(&code);
test.expected = add ? std::vector<u32>{0, 4, 0, 1, 0x13579bdf, 0x1234}
: std::vector<u32>{0x101, 0x202, 0x401, 0x802, 0x13579bdf, 0x1234};
test.compute_info.lds_size_dwords = 5;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32,
add ? O::DS_ADD_U64 : O::DS_OR_B64,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {add ? "OpAtomicIAdd %ulong" : "OpAtomicOr %ulong",
"WorkgroupMemoryExplicitLayoutKHR", "Aliased"};
return test;
}
TestCase DsAtomic64Contention(bool add, u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = add ? (wave_size == 32 ? "DsAddU64ContentionWave32" : "DsAddU64ContentionWave64")
: (wave_size == 32 ? "DsOrB64ContentionWave32" : "DsOrB64ContentionWave64");
auto &code = test.code;
AppendVMovU32(&code, 7, 0);
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
code.push_back(EncodeSop1(0x04, 126, 106));
for (u32 offset : {0, 4}) {
code.push_back(EncodeDs0(0x0d, offset));
code.push_back(EncodeDs1(0, 7, 7));
}
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
if (add) {
AppendVMovLiteral(&code, 3, 0xffffffffu);
AppendVMovU32(&code, 4, 1);
} else {
AppendVMovU32(&code, 5, 1);
AppendVMovU32(&code, 6, 2);
code.push_back(EncodeVop2(0x1a, 3, Vgpr(0), 5));
code.push_back(EncodeVop2(0x1a, 4, Vgpr(0), 6));
}
AppendSMovLiteral(&code, 126, 0x55555555u);
AppendSMovLiteral(&code, 127, 0xaaaaaaaau);
code.insert(code.end(), {add ? 0xd9000000u : 0xd9280000u, 0x00000307u});
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
for (u32 half = 0; half < 2; ++half) {
code.push_back(EncodeDs0(0x36, half * 4));
code.push_back(EncodeDs1(8, 0, 7));
AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, half * 128);
}
AppendEnd(&code);
test.expected.assign(128, add ? 0xffffffc0u : wave_size == 32 ? 0x55555555u : 0xffffffffu);
test.expected.resize(256, add ? 0x7fu : wave_size == 32 ? 0xaaaaaaaau : 0xfffffffeu);
test.compute_info.threads_num[0] = 128;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.compute_info.lds_size_dwords = 2;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::S_BARRIER,
O::V_CMP_EQ_U32, O::V_ADD_NC_U32,
O::DS_WRITE_B32, add ? O::DS_ADD_U64 : O::DS_OR_B64, O::DS_READ_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
if (!add) {
test.opcodes.push_back(O::V_LSHLREV_B32);
}
test.required_spirv = {add ? "OpAtomicIAdd %ulong" : "OpAtomicOr %ulong",
"WorkgroupMemoryExplicitLayoutKHR", "Aliased"};
return test;
}
TestCase DsBoundedAtomicBoundaries(bool decrement, bool gds) {
using O = ShaderOpcode;
constexpr u32 guard = 0x13579bdfu;
struct Boundary {
u32 before, limit, incremented, decremented;
};
constexpr Boundary boundaries[] = {
{4, 5, 5, 3},
{5, 5, 0, 4},
{6, 5, 0, 5},
{0, 5, 1, 5},
{0, 0, 0, 0},
{1, 0, 0, 0},
{0xffffffffu, 0xffffffffu, 0, 0xfffffffeu},
{0xfffffffeu, 0xffffffffu, 0xffffffffu, 0xfffffffdu},
{0, 0xffffffffu, 1, 0xffffffffu},
{0x80000000u, 0x7fffffffu, 0, 0x7fffffffu},
{0x7fffffffu, 0x80000000u, 0x80000000u, 0x7ffffffeu},
{0xffffffffu, 0x80000000u, 0, 0x80000000u}};
constexpr u32 count = static_cast<u32>(std::size(boundaries));
const u32 opcode = decrement ? 0x24u : 0x23u;
TestCase test;
test.name =
decrement
? (gds ? "DsDecReturnGdsBoundaries" : "DsDecReturnLdsBoundaries")
: (gds ? "DsIncReturnGdsBoundaries" : "DsIncReturnLdsBoundaries");
auto &code = test.code;
AppendVMovU32(&code, 2, 0);
std::vector<u32> memory(68, guard);
for (u32 i = 0; i < count; ++i)
memory[i + 1] = boundaries[i].before;
if (!gds) {
for (u32 i = 0; i < memory.size(); ++i) {
AppendVMovLiteral(&code, 3, memory[i]);
code.push_back(EncodeDs0(0x0d, i * 4));
code.push_back(EncodeDs1(0, 3, 2));
}
}
auto StoreResult = [&](u32 vgpr, u32 expected) {
AppendStoreVgpr(&code, vgpr, static_cast<u32>(test.expected.size()));
test.expected.push_back(expected);
};
for (u32 i = 0; i < count; ++i) {
AppendVMovU32(&code, 2, i * 4);
AppendVMovLiteral(&code, 3, boundaries[i].limit);
// Preserve the captured DATA0/VDST alias and exact increment words in GDS.
code.push_back(decrement ? EncodeDs0(opcode, 4, gds)
: (gds ? 0xd88e0004u : 0xd88c0004u));
code.push_back(0x03000302u);
StoreResult(3, boundaries[i].before);
code.push_back(EncodeDs0(0x36, 4, gds));
code.push_back(EncodeDs1(4, 0, 2));
StoreResult(4, decrement ? boundaries[i].decremented
: boundaries[i].incremented);
}
// EXEC=0 preserves memory and the aliased destination register.
AppendVMovU32(&code, 2, 0);
AppendVMovLiteral(&code, 3, 0x12345678u);
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeDs0(opcode, 4, gds));
code.push_back(0x03000302u);
code.push_back(EncodeSop1(0x04, 126, 10));
StoreResult(3, 0x12345678u);
// Use both offset bytes and alias VDST with ADDR instead of DATA0.
AppendVMovU32(&code, 2, 4);
AppendVMovLiteral(&code, 3, 0xffffffffu);
code.push_back(EncodeDs0(opcode, 0x104, gds));
code.push_back(EncodeDs1(2, 3, 2));
StoreResult(2, guard);
// Overwriting an unused return must not let DCE discard the atomic update.
AppendVMovU32(&code, 2, 0);
code.push_back(EncodeDs0(opcode, 0x108, gds));
code.push_back(EncodeDs1(4, 3, 2));
AppendVMovU32(&code, 4, 0);
// An active access just outside the resource returns zero.
AppendVMovU32(&code, 2, static_cast<u32>(memory.size() * 4 - 4));
code.push_back(EncodeDs0(opcode, 4, gds));
code.push_back(0x03000302u);
StoreResult(3, 0);
// Read back all words, including guards around both address ranges.
AppendVMovU32(&code, 2, 0);
for (u32 i = 0; i < memory.size(); ++i) {
u32 expected = memory[i];
if (i >= 1 && i <= count) {
expected = decrement ? boundaries[i - 1].decremented
: boundaries[i - 1].incremented;
} else if (i == 66) {
expected = decrement ? guard - 2 : guard + 2;
}
code.push_back(EncodeDs0(0x36, i * 4, gds));
code.push_back(EncodeDs1(4, 0, 2));
StoreResult(4, expected);
if (gds)
test.expected_gds.push_back(expected);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.compute_info.lds_size_dwords = static_cast<u32>(memory.size());
test.opcodes = {
O::S_MOV_B64, O::V_MOV_B32,
O::DS_READ_B32, decrement ? O::DS_DEC_RTN_U32 : O::DS_INC_RTN_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange",
"OpLoopMerge"};
if (gds)
test.gds_initial = memory;
else
test.opcodes.push_back(O::DS_WRITE_B32);
return test;
}
TestCase DsBoundedAtomicContention(bool decrement, bool gds, u32 wave_size) {
using O = ShaderOpcode;
constexpr u32 guard = 0x13579bdfu;
constexpr u32 limit = 15;
TestCase test;
if (wave_size == 32) {
test.name = decrement ? (gds ? "DsDecReturnGdsContentionWave32"
: "DsDecReturnLdsContentionWave32")
: (gds ? "DsIncReturnGdsContentionWave32"
: "DsIncReturnLdsContentionWave32");
} else {
test.name = decrement ? (gds ? "DsDecReturnGdsContentionWave64"
: "DsDecReturnLdsContentionWave64")
: (gds ? "DsIncReturnGdsContentionWave64"
: "DsIncReturnLdsContentionWave64");
}
auto &code = test.code;
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 3, limit);
AppendVMovU32(&code, 5, 1);
AppendVMovLiteral(&code, 6, guard);
if (!gds) {
// One lane initializes LDS before every wave in the workgroup uses it.
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(EncodeDs0(0x0d, 4));
code.push_back(EncodeDs1(0, 2, 2));
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
}
// Distinct masks in the wave64 halves also check inactive return
// preservation.
code.push_back(EncodeSop1(0x04, 10, 126));
AppendSMovLiteral(&code, 126, 0x55555555u);
AppendSMovLiteral(&code, 127, 0xaaaaaaaau);
code.push_back(EncodeDs0(decrement ? 0x24 : 0x23, 4, gds));
code.push_back(EncodeDs1(6, 3, 2));
// Returned old values form a histogram; no lane ordering is assumed.
code.push_back(EncodeVop2(0x1a, 7, InlineU32(2), 6));
AppendBufferStoreOpcode(&code, 0x32, 5, 7);
code.push_back(EncodeSop1(0x04, 126, 10));
AppendVMovLiteral(&code, 7, guard);
code.push_back(EncodeVopc(0xc2, Vgpr(7), 6));
code.push_back(EncodeVop2(0x01, 8, InlineU32(0), 5));
AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, 16);
code.push_back(EncodeSopp(0x0a, 0));
// S_BARRIER orders LDS; use atomic add-zero to read GDS consistently.
code.push_back(EncodeDs0(gds ? 0x20 : 0x36, 4, gds));
code.push_back(EncodeDs1(9, 2, 2));
AppendStoreVgprAtLaneDwordOffset(&code, 9, 0, 144);
AppendEnd(&code);
test.expected.assign(16, 4); // 64 active lanes, four complete wraps.
for (u32 lane = 0; lane < 128; ++lane) {
const uint64_t mask = 0xaaaaaaaa55555555ull;
test.expected.push_back(
(mask & (uint64_t{1} << (lane % wave_size))) == 0 ? 1 : 0);
}
test.expected.resize(272, 0);
test.initial.resize(test.expected.size());
test.compute_info.threads_num[0] = 128;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.compute_info.lds_size_dwords = 3;
test.has_compute_info = true;
test.opcodes = {O::S_MOV_B32,
O::S_MOV_B64,
O::S_BARRIER,
O::V_MOV_B32,
O::V_LSHLREV_B32,
O::V_ADD_NC_U32,
O::V_CMP_EQ_U32,
O::V_CNDMASK_B32,
decrement ? O::DS_DEC_RTN_U32 : O::DS_INC_RTN_U32,
gds ? O::DS_ADD_RTN_U32 : O::DS_READ_B32,
O::BUFFER_ATOMIC_ADD,
O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange",
"OpLoopMerge"};
if (gds) {
test.gds_initial = {guard, 0, guard};
test.expected_gds = test.gds_initial;
} else
test.opcodes.push_back(O::DS_WRITE_B32);
return test;
}
TestCase DsMiscVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 0);
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x40800000u);
code.push_back(EncodeDs0(0x0d, 0));
code.push_back(EncodeDs1(0, 2, 1));
AppendVMovLiteral(&code, 3, 0x3f800000u);
code.push_back(EncodeDs0(0x0d, 4));
code.push_back(EncodeDs1(0, 3, 1));
AppendVMovLiteral(&code, 4, 0x40000000u);
code.push_back(EncodeDs0(0x12, 0));
code.push_back(EncodeDs1(0, 4, 1));
AppendVMovLiteral(&code, 5, 0x40400000u);
code.push_back(EncodeDs0(0x13, 4));
code.push_back(EncodeDs1(0, 5, 1));
code.push_back(EncodeDs0(0x36, 0));
code.push_back(EncodeDs1(6, 0, 1));
code.push_back(EncodeDs0(0x36, 4));
code.push_back(EncodeDs1(7, 0, 1));
AppendVMovLiteral(&code, 8, 0x12345678u);
code.push_back(EncodeDs0(0x35, 0x001f));
code.push_back(EncodeDs1(9, 0, 8));
code.push_back(EncodeSMovB32(124, InlineU32(0)));
AppendVMovLiteral(&code, 10, 0xabcdef01u);
code.push_back(EncodeDs0(0xb0, 8));
code.push_back(EncodeDs1(0, 10, 0));
code.push_back(EncodeDs0(0xb1, 8));
code.push_back(EncodeDs1(11, 0, 0));
const u32 results[] = {6, 7, 9, 11};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
TestCase test;
test.name = "DsMiscVariants";
test.code = code;
test.initial = std::vector<u32>(4, 0);
test.expected = {0x40000000u, 0x40400000u, 0x12345678u, 0xabcdef01u};
test.opcodes = {O::V_MOV_B32,
O::S_MOV_B32,
O::DS_WRITE_B32,
O::DS_MIN_F32,
O::DS_MAX_F32,
O::DS_READ_B32,
O::DS_SWIZZLE_B32,
O::DS_WRITE_ADDTID_B32,
O::DS_READ_ADDTID_B32,
O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 1;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.has_compute_info = true;
return test;
}
TestCase DsFloatMinMaxIgnoresData1() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x40800000u);
code.push_back(EncodeDs0(0x0d, 0));
code.push_back(EncodeDs1(0, 2, 1));
AppendVMovLiteral(&code, 3, 0x40800000u);
code.push_back(EncodeDs0(0x0d, 4));
code.push_back(EncodeDs1(0, 3, 1));
AppendVMovLiteral(&code, 4, 0x41100000u);
AppendVMovLiteral(&code, 5, 0x40000000u);
code.push_back(EncodeDs0(0x12, 0));
code.push_back(EncodeDs1Ex(0, 5, 4, 1));
AppendVMovLiteral(&code, 6, 0x3f800000u);
AppendVMovLiteral(&code, 7, 0x40400000u);
code.push_back(EncodeDs0(0x13, 4));
code.push_back(EncodeDs1Ex(0, 7, 6, 1));
code.push_back(EncodeDs0(0x36, 0));
code.push_back(EncodeDs1(8, 0, 1));
code.push_back(EncodeDs0(0x36, 4));
code.push_back(EncodeDs1(9, 0, 1));
AppendStoreVgpr(&code, 8, 0);
AppendStoreVgpr(&code, 9, 1);
AppendEnd(&code);
TestCase test;
test.name = "DsFloatMinMaxIgnoresData1";
test.code = code;
test.initial = std::vector<u32>(2, 0);
test.expected = {0x40800000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_MIN_F32,
O::DS_MAX_F32, O::DS_READ_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 1;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.has_compute_info = true;
return test;
}
TestCase DsFloatMinMaxClasses(bool gds) {
using O = ShaderOpcode;
// Rows are memory, columns source: -inf, -normal, -denorm, -0, +0,
// +denorm, +normal, +inf, qNaN, sNaN. O/N select old/new; Q/R quiet them.
constexpr std::string_view minimum[] = {
"OOOOOOOOOR", "NNOOOOOOOR", "NNNNOOOOOR", "NNNOOOOOOR", "NNNNOOOOOR",
"NNNNNNOOOR", "NNNNNNOOOR", "NNNNNNNOOR", "NNNNNNNNOR", "QQQQQQQQQQ"};
constexpr std::string_view maximum[] = {
"ONNNNNNNOR", "OONNNNNNOR", "OOOONNNNOR", "OOOONNNNOR", "OOOOONNNOR",
"OOOOOONNOR", "OOOOOONNOR", "OOOOOOOOOR", "NNNNNNNNOR", "QQQQQQQQQQ"};
constexpr u32 old_values[] = {
0xff800000u, 0xc0000000u, 0x80000100u, 0x80000000u, 0u,
0x00000100u, 0x40000000u, 0x7f800000u, 0x7fc12345u, 0x7f812345u};
constexpr u32 sources[] = {
0xff800000u, 0xc0400000u, 0x80000001u, 0x80000000u, 0u,
0x00000001u, 0x40400000u, 0x7f800000u, 0xffc54321u, 0xff854321u};
const auto select = [](char choice, u32 old, u32 source) {
const u32 bits = choice == 'O' || choice == 'Q' ? old : source;
return choice == 'Q' || choice == 'R' ? bits | 0x00400000u : bits;
};
TestCase test;
test.name = gds ? "DsFloatMinMaxGdsClasses" : "DsFloatMinMaxLdsClasses";
for (u32 row = 0; row < std::size(old_values); ++row) {
for (u32 column = 0; column < std::size(sources); ++column) {
const auto old = old_values[row];
const auto source = sources[column];
test.initial.insert(test.initial.end(), {old, source, 0u, 0u});
test.expected.insert(test.expected.end(),
{old, source, select(minimum[row][column], old, source),
select(maximum[row][column], old, source)});
}
}
// Reversing positive denorm magnitudes still selects source for min and old for max.
test.initial.insert(test.initial.end(), {1u, 0x100u, 0u, 0u});
test.expected.insert(test.expected.end(), {1u, 0x100u, 0x100u, 1u});
const auto count = static_cast<u32>(test.initial.size() / 4u);
auto &code = test.code;
code.push_back(EncodeVop2(0x1a, 1, InlineU32(3), 0));
code.push_back(EncodeVop2(0x1a, 8, InlineU32(4), 0));
code.push_back(EncodeMubuf0(0x0d));
code.push_back(EncodeMubuf1(2, 0, 8));
for (u32 offset : {0u, 4u}) {
code.push_back(EncodeDs0(0x0d, offset, gds));
code.push_back(EncodeDs1(0, 2, 1));
}
code.push_back(EncodeDs0(0x12, 0, gds));
code.push_back(EncodeDs1Ex(0, 250, 3, 1)); // DATA1 is unused and uninitialized.
code.push_back(EncodeDs0(0x13, 4, gds));
code.push_back(EncodeDs1Ex(0, 251, 3, 1));
AppendVMovLiteral(&code, 6, 0xff800000u);
AppendVMovLiteral(&code, 7, 0x7f800000u);
code.push_back(EncodeSop1(0x04, 4, 126));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
// Inactive updates must leave the table results untouched.
code.push_back(EncodeDs0(0x12, 0, gds));
code.push_back(EncodeDs1(0, 6, 1));
code.push_back(EncodeDs0(0x13, 4, gds));
code.push_back(EncodeDs1(0, 7, 1));
code.push_back(EncodeSop1(0x04, 126, 4));
// Active accesses immediately beyond the allocation must also be discarded.
AppendVMovU32(&code, 9, count * 8u);
code.push_back(EncodeDs0(0x12, 0, gds));
code.push_back(EncodeDs1(0, 6, 9));
code.push_back(EncodeDs0(0x13, 0, gds));
code.push_back(EncodeDs1(0, 7, 9));
for (u32 component = 0; component < 2u; ++component) {
code.push_back(EncodeDs0(0x36, component * 4u, gds));
code.push_back(EncodeDs1(4u + component, 0, 1));
}
code.push_back(EncodeMubuf0(0x1d, 8));
code.push_back(EncodeMubuf1(4, 12, 8));
AppendEnd(&code);
test.compute_info.threads_num[0] = count;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.lds_size_dwords = count * 2u;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::S_MOV_B64,
O::BUFFER_LOAD_DWORDX2, O::DS_WRITE_B32, O::DS_MIN_F32,
O::DS_MAX_F32, O::DS_READ_B32, O::BUFFER_STORE_DWORDX2,
O::S_ENDPGM};
test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange", "OpLoopMerge"};
test.forbidden_spirv = {"OpFOrdLessThan", "OpFOrdGreaterThan"};
if (gds) {
test.gds_initial.resize(count * 2u);
for (u32 i = 0; i < count; ++i) {
test.expected_gds.push_back(test.expected[i * 4u + 2u]);
test.expected_gds.push_back(test.expected[i * 4u + 3u]);
}
}
return test;
}
TestCase DsSwizzleInvalidSourceLaneZero() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeDs0(0x35, 0x00e0));
code.push_back(EncodeDs1(2, 0, 1));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "DsSwizzleInvalidSourceLaneZero";
test.code = code;
test.expected = {0, 0, 0, 0};
test.opcodes = {O::V_MOV_B32, O::DS_SWIZZLE_B32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase DsPermuteCapturedExecOffsetAndWrap() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
AppendBufferLoadDword(&code, 10, 30);
AppendVMovLiteral(&code, 1, 0xdeadbeefu);
AppendVMovU32(&code, 9, 100);
code.push_back(EncodeVop2(0x25, 9, Vgpr(0), 9));
code.push_back(EncodeSop1(0x04, 4, 126));
code.push_back(EncodeVop2(0x1b, 18, InlineU32(1), 0));
code.push_back(EncodeVopc(0xc2, InlineU32(1), 18));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(0xdac80000u);
code.push_back(0x0100090au); // Captured SH2 instruction: ds_permute_b32 v1, v10, v9
code.push_back(EncodeSop1(0x04, 126, 4));
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 4);
code.push_back(0xdac80000u);
code.push_back(0x0100090au);
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 8);
AppendVMovU32(&code, 10, 127);
code.push_back(EncodeDs0(0xb2, 1));
code.push_back(EncodeDs1(1, 9, 10));
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 12);
AppendEnd(&code);
TestCase test;
test.name = "DsPermuteCapturedExecOffsetAndWrap";
test.code = code;
test.initial = {0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
test.expected = {0, 0, 12, 4, 0xdeadbeefu, 103, 0xdeadbeefu, 0,
101, 103, 0, 102, 103, 0, 0, 0};
test.opcodes = {O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD, O::V_MOV_B32,
O::V_ADD_NC_U32, O::V_AND_B32, O::V_CMP_EQ_U32, O::S_MOV_B64,
O::DS_PERMUTE_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformShuffle", "OpGroupNonUniformBallot"};
test.forbidden_spirv = {" Workgroup", "OpControlBarrier", "OpMemoryBarrier"};
test.decoded_counts = {{"DS_PERMUTE_B32", 3}};
test.ir_counts = {{"PermuteU32", 3}};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase DsPermuteWave64UsesIndependentHalves() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 9, 100);
code.push_back(EncodeVop2(0x25, 9, Vgpr(0), 9));
code.push_back(EncodeVop2(0x1d, 10, InlineU32(31), 0));
code.push_back(EncodeVop2(0x1a, 10, InlineU32(2), 10));
code.push_back(0xdac80000u);
code.push_back(0x0100090au);
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "DsPermuteWave64UsesIndependentHalves";
test.code = code;
test.initial = std::vector<u32>(64, 0);
for (u32 lane = 0; lane < 64; ++lane) {
test.expected.push_back(100 + (lane ^ 31));
}
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_XOR_B32, O::V_LSHLREV_B32,
O::DS_PERMUTE_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformShuffle", "OpGroupNonUniformBallot"};
test.forbidden_spirv = {" Workgroup", "OpControlBarrier", "OpMemoryBarrier"};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 64;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase DsBpermuteCapturedExecOffsetAndWrap() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
AppendBufferLoadDword(&code, 17, 30);
AppendVMovLiteral(&code, 1, 0xdeadbeefu);
AppendVMovU32(&code, 3, 100);
code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3));
code.push_back(EncodeSop1(0x04, 4, 126));
code.push_back(EncodeVop2(0x1b, 18, InlineU32(1), 0));
code.push_back(EncodeVopc(0xc2, InlineU32(1), 18));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(0xdacc0000u);
code.push_back(0x01000311u); // ds_bpermute_b32 v1, v17, v3
code.push_back(EncodeSop1(0x04, 126, 4));
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 4);
AppendVMovU32(&code, 17, 124);
code.push_back(EncodeDs0(0xb3, 4));
code.push_back(EncodeDs1(2, 3, 17));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 8);
AppendEnd(&code);
TestCase test;
test.name = "DsBpermuteCapturedExecOffsetAndWrap";
test.code = code;
test.initial = {0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0};
test.expected = {0, 0, 12, 4,
0xdeadbeefu, 0, 0xdeadbeefu, 101,
100, 100, 100, 100};
test.opcodes = {O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD,
O::V_MOV_B32, O::V_ADD_NC_U32,
O::V_AND_B32, O::V_CMP_EQ_U32,
O::S_MOV_B64,
O::DS_BPERMUTE_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformShuffle",
"OpGroupNonUniformBallot"};
test.forbidden_spirv = {" Workgroup", "OpControlBarrier",
"OpMemoryBarrier"};
test.decoded_counts = {{"DS_BPERMUTE_B32", 2}};
test.ir_counts = {{"BpermuteU32", 2}};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase DsBpermuteWave64UsesIndependentHalves() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 3, 100);
code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3));
AppendVMovU32(&code, 17, 128);
code.push_back(0xdacc0000u);
code.push_back(0x01000311u); // ds_bpermute_b32 v1, v17, v3
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "DsBpermuteWave64UsesIndependentHalves";
test.code = code;
test.initial = std::vector<u32>(64, 0);
test.expected = std::vector<u32>(32, 100);
test.expected.insert(test.expected.end(), 32, 132);
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32,
O::DS_BPERMUTE_B32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformShuffle"};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 64;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase Wave64CrossHalfLaneAndLds() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeVop2(0x25, 1, Vgpr(0), 1));
AppendVop3(&code, 0x360, 8, Vgpr(1), InlineU32(47));
AppendStoreSgprAtLaneDwordOffset(&code, 8, 0, 0);
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeVop2(0x1b, 2, InlineU32(63), 0));
code.push_back(EncodeVopc(0xc2, InlineU32(40), 2));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(EncodeVop1(0x02, 9, Vgpr(1)));
code.push_back(EncodeSop1(0x04, 126, 10));
AppendStoreSgprAtLaneDwordOffset(&code, 9, 0, 128);
AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, 256);
AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, 384);
code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0));
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 1, 2));
code.push_back(EncodeSopp(0x0a, 0));
code.push_back(EncodeVop2(0x1d, 3, InlineU32(64), 0));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 3));
code.push_back(EncodeDs0(0x36));
code.push_back(EncodeDs1(4, 0, 3));
AppendStoreVgprAtLaneDwordOffset(&code, 4, 0, 512);
AppendEnd(&code);
TestCase test;
test.name = "Wave64CrossHalfLaneAndLds";
test.code = std::move(code);
test.initial.resize(640);
test.expected.insert(test.expected.end(), 64, 147);
test.expected.insert(test.expected.end(), 64, 211);
test.expected.insert(test.expected.end(), 64, 140);
test.expected.insert(test.expected.end(), 64, 204);
test.expected.insert(test.expected.end(), 128, 0);
test.expected.insert(test.expected.end(), 128, 256);
for (u32 lane = 0; lane < 128; lane++) {
test.expected.push_back(100 + (lane ^ 64));
}
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_READLANE_B32,
O::V_READFIRSTLANE_B32, O::V_CMP_EQ_U32, O::S_MOV_B64,
O::V_LSHLREV_B32, O::V_AND_B32, O::V_XOR_B32, O::DS_WRITE_B32,
O::S_BARRIER, O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformBallot", "OpGroupNonUniformShuffle",
"OpControlBarrier"};
test.ir_counts = {{"Barrier", 1}};
test.compute_info.threads_num[0] = 128;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
void CheckWave64WholeWaveResults() {
const auto test = Wave64CrossHalfLaneAndLds();
const auto native = CompileCase(test, 64);
const auto emulated = CompileCase(test, 32);
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string native_text, emulated_text;
Require(test.name, "whole-wave disassembly",
tools.Disassemble(native.spirv, &native_text) &&
tools.Disassemble(emulated.spirv, &emulated_text),
"failed to disassemble whole-wave operations");
const auto ballots = CountText(native_text, "OpGroupNonUniformBallot ");
Require(test.name, "whole-wave ballot reuse",
ballots != 0 &&
CountText(emulated_text, "OpGroupNonUniformBallot ") == ballots * 2,
"emulated halves repeated a whole-wave ballot");
// READLANE still emits two shuffles per half; READFIRSTLANE needs one shared pair.
Require(test.name, "whole-wave first-lane reuse",
CountText(emulated_text, "OpGroupNonUniformShuffle ") == 6,
"emulated halves repeated the first-active-lane broadcast");
}
TestCase Wave64RawMasksAndScalarBranch() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 7);
AppendSMovLiteral(&code, 8, 33u << 16u);
code.push_back(EncodeSop2(0x29, 126, 193u, 8)); // captured ES BFE EXEC,-1,width
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
code.push_back(EncodeSop2(0x1f, 126, InlineU32(1), InlineU32(32)));
const auto exec_branch = code.size();
code.push_back(0);
code.push_back(EncodeSopp(0x0a, 0));
code.push_back(EncodeSop1(0x24, 10, 193u)); // s_and_saveexec_b64 s[10:11],-1
const auto scc_branch = code.size();
code.push_back(0);
AppendVMovU32(&code, 1, 9);
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 64);
const auto restore = code.size();
code[exec_branch] = EncodeSopp(0x08, restore - exec_branch - 1);
code[scc_branch] = EncodeSopp(0x04, restore - scc_branch - 1);
code.push_back(EncodeSop1(0x04, 126, 193u));
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 12, 106));
code.push_back(EncodeSMovB32(13, InlineU32(1)));
AppendVop3(&code, 0x101, 2, InlineU32(0), InlineU32(11), 12);
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 128);
code.push_back(EncodeSMovB32(127, InlineU32(1)));
AppendVMovU32(&code, 3, 13);
AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, 192);
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, 256);
code.push_back(EncodeSMovB32(107, InlineU32(2)));
code.push_back(EncodeSMovB32(127, 107));
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 320);
AppendEnd(&code);
TestCase test;
test.name = "Wave64RawMasksAndScalarBranch";
test.code = std::move(code);
test.initial.assign(384, 0xdeadbeef);
test.expected = test.initial;
for (u32 lane = 0; lane < 64; lane++) {
if (lane < 33) {
test.expected[lane] = 7;
test.expected[192 + lane] = 13;
}
test.expected[128 + lane] = lane < 33 ? 11 : 0;
}
test.expected[64 + 32] = 9;
test.expected[256] = 13;
test.expected[320] = 7;
test.expected[320 + 33] = 7;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_BFE_U64, O::S_LSHL_B64,
O::S_CBRANCH_EXECZ, O::S_AND_SAVEEXEC_B64, O::S_CBRANCH_SCC0,
O::S_BARRIER, O::S_MOV_B64, O::V_CMP_EQ_U32, O::V_CNDMASK_B32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase Wave64PartialMultidimensionalWorkgroup() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x25, 3, Vgpr(2), 2));
code.push_back(EncodeVop2(0x25, 3, Vgpr(2), 3));
code.push_back(EncodeVop2(0x25, 3, Vgpr(1), 3));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(3), 3));
code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3));
code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 3));
AppendVMovU32(&code, 5, 1);
AppendBufferStoreOpcode(&code, 0x32, 5, 4, false);
AppendEnd(&code);
TestCase test;
test.name = "Wave64PartialMultidimensionalWorkgroup";
test.code = std::move(code);
test.initial.resize(128);
test.expected.assign(96, 1);
test.expected.resize(128);
test.opcodes = {O::V_ADD_NC_U32, O::V_LSHLREV_B32, O::V_MOV_B32,
O::BUFFER_ATOMIC_ADD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 3;
test.compute_info.threads_num[2] = 4;
test.compute_info.thread_ids_num = 3;
test.has_compute_info = true;
return test;
}
TestCase Wave64AppendConsumeHighHalf() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 124, 4);
code.push_back(EncodeSop2(0x1f, 126, 193u, InlineU32(32)));
code.push_back(EncodeDs0(0x3e, 0, true));
code.push_back(EncodeDs1(1, 0, 0));
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
code.push_back(EncodeDs0(0x3d, 0, true));
code.push_back(EncodeDs1(2, 0, 0));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 64);
AppendEnd(&code);
TestCase test;
test.name = "Wave64AppendConsumeHighHalf";
test.code = std::move(code);
test.initial.assign(128, 0xdeadbeef);
test.expected = test.initial;
for (u32 lane = 32; lane < 64; lane++) {
test.expected[lane] = 10;
test.expected[64 + lane] = 42;
}
test.gds_initial = {10};
test.expected_gds = {10};
test.opcodes = {O::S_MOV_B32, O::S_LSHL_B64, O::DS_APPEND, O::DS_CONSUME,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false,
bool store_completion = false) {
using O = ShaderOpcode;
constexpr u32 groups = 31;
TestCase test;
test.name = store_completion ? (wave_size == 64 ? "Wave64StoreCompletionPublication"
: "Wave32StoreCompletionPublication")
: dlc_only ? "BufferWorkgroupPublicationDlc"
: wave_size == 64 ? "BufferWorkgroupPublicationWave64"
: "BufferWorkgroupPublicationWave32";
auto &code = test.code;
// Allocate work in execution order so every polled predecessor has started.
// Only one lane per workgroup owns a ticket and publishes its prefix total.
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 20, 0);
AppendBufferStoreOpcode(&code, 0x32, 1, 20, true);
code.push_back(EncodeVop1(0x02, 20, Vgpr(1)));
code.push_back(EncodeVop2(0x1a, 20, InlineU32(2), 1));
AppendVMovU32(&code, 2, 0);
code.push_back(EncodeSMovB32(21, InlineU32(0)));
AppendSMovLiteral(&code, 22, 1000000);
if (store_completion) AppendSMovLiteral(&code, 26, groups * 4);
code.push_back(EncodeSopc(0x06, 20, InlineU32(0)));
const auto first_ticket = code.size();
code.push_back(0);
const auto poll = code.size();
if (store_completion) {
code.push_back(EncodeVop2(0x25, 4, 26, 20));
AppendVMovU32(&code, 3, 0);
AppendBufferStoreOpcode(&code, 0x32, 3, 4, true);
}
code.push_back(EncodeMubuf0(0x0c, 0, false, true, !dlc_only) | (1u << 15u));
code.push_back(EncodeMubuf1(2, 12, 20));
code.push_back(EncodeSopp(0x0c, 0)); // S_WAITCNT vmcnt(0).
code.push_back(EncodeVop1(0x02, 23, Vgpr(store_completion ? 3 : 2)));
code.push_back(EncodeSopc(0x07, 23, InlineU32(0)));
const auto ready = code.size();
code.push_back(0);
code.push_back(EncodeSop2(0x00, 21, 21, InlineU32(1)));
code.push_back(EncodeSopc(0x0a, 21, 22));
code.push_back(EncodeSopp(0x05, static_cast<u32>(poll - code.size() - 1)));
// A broken poll reports an incorrect result instead of hanging the test GPU.
AppendVMovLiteral(&code, 2, 0x80000000u);
const auto publish = code.size();
code[first_ticket] = EncodeSopp(0x05, publish - first_ticket - 1);
code[ready] = EncodeSopp(0x05, publish - ready - 1);
code.push_back(EncodeVop2(0x25, 2, Vgpr(1), 2));
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2));
code.push_back(EncodeVop2(0x25, 20, InlineU32(4), 20));
if (store_completion) {
// The last lane writes the payload; lane zero publishes readiness after the
// native wave-wide store completion, including both emulated wave64 halves.
code.push_back(EncodeVop1(0x02, 24, Vgpr(2)));
code.push_back(EncodeVop1(0x02, 25, Vgpr(20)));
code.push_back(EncodeSop1(0x04, 126, 193u));
code.push_back(EncodeVop1(0x01, 2, 24));
code.push_back(EncodeVop1(0x01, 20, 25));
code.push_back(EncodeSop2(0x1f, 126, InlineU32(1), InlineU32(wave_size - 1)));
}
// RDNA2 stores publish to L2 even without GLC/DLC on the producer.
AppendBufferStoreDword(&code, 2, 20);
if (store_completion) {
code.push_back(0xbbfd0000u); // Captured S_WAITCNT_VSCNT null, 0 at 5b10:0x2464.
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
code.push_back(EncodeVop2(0x25, 4, 26, 20));
AppendVMovU32(&code, 3, 1);
AppendBufferStoreOpcode(&code, 0x32, 3, 4, true);
}
AppendEnd(&code);
test.initial.assign(groups * (store_completion ? 2 : 1) + 1, 0);
test.expected = {groups};
for (u32 ticket = 0; ticket < groups; ++ticket) {
test.expected.push_back((ticket + 1) * (ticket + 2) / 2);
}
if (store_completion) test.expected.resize(groups * 2 + 1, 1);
test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::V_MOV_B32,
O::BUFFER_ATOMIC_ADD, O::V_READFIRSTLANE_B32,
O::V_LSHLREV_B32, O::S_CMP_EQ_U32, O::S_CMP_LG_U32,
O::S_CMP_LT_U32, O::S_CBRANCH_SCC1, O::S_ADD_U32,
O::S_WAITCNT, O::BUFFER_LOAD_DWORD, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"Coherent", "Volatile", "OpAtomicIAdd", "OpLoopMerge"};
if (store_completion) {
test.opcodes.push_back(O::S_WAITCNT_VSCNT);
test.opcodes.push_back(O::S_LSHL_B64);
test.required_spirv.push_back("OpControlBarrier %uint_3 %uint_1 %uint_2116");
}
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.dispatch_x = groups;
// DLC alone does not bypass RDNA2's L0, so only check its translated policy.
test.compile_only = dlc_only;
return test;
}
TestCase BufferAtomicVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
const u32 values[] = {100, 5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu};
const u32 ops[] = {0x30, 0x32, 0x33, 0x35, 0x36,
0x37, 0x38, 0x39, 0x3a, 0x3b};
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovU32(&code, 20, i * 4u);
AppendVMovU32(&code, i, values[i]);
AppendBufferStoreOpcode(&code, ops[i], i, 20, true);
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, i, i + 10u);
}
AppendEnd(&code);
return {
"BufferAtomicVariants",
code,
{10, 10, 10, 0xfffffff0u, 10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
{100, 15, 7, 0xfffffff0u, 5, 5, 20,
0x00f0u, 0xff00u, 0xf0f0u, 10, 10, 10, 0xfffffff0u,
10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu},
{O::V_MOV_B32, O::BUFFER_ATOMIC_SWAP, O::BUFFER_ATOMIC_ADD,
O::BUFFER_ATOMIC_SUB, O::BUFFER_ATOMIC_SMIN, O::BUFFER_ATOMIC_UMIN,
O::BUFFER_ATOMIC_SMAX, O::BUFFER_ATOMIC_UMAX, O::BUFFER_ATOMIC_AND,
O::BUFFER_ATOMIC_OR, O::BUFFER_ATOMIC_XOR, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase BufferAtomicAndX2GlcAndExec() {
using O = ShaderOpcode;
TestCase test;
test.name = "BufferAtomicAndX2GlcAndExec";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 12, 126));
AppendVMovU32(&code, 20, 0);
const u32 values[][2] = {{0x0ff00ff0u, 0xf00ff00fu},
{0xff00ff00u, 0x00ff00ffu},
{0, 0}};
for (u32 i = 0; i < std::size(values); ++i) {
AppendVMovLiteral(&code, 2, values[i][0]);
AppendVMovLiteral(&code, 3, values[i][1]);
if (i == 2) {
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
}
AppendBufferStoreOpcode(&code, 0x59, 2, 20, i != 1);
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 2, 2 + i * 2);
AppendStoreVgpr(&code, 3, 3 + i * 2);
}
AppendEnd(&code);
test.initial = {0x12345678u, 0x87654321u, 0, 0, 0, 0, 0, 0};
test.expected = {0x02000600u, 0x00050001u, 0x12345678u, 0x87654321u,
0xff00ff00u, 0x00ff00ffu, 0, 0};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_ATOMIC_AND_X2,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicAnd %ulong", "Int64Atomics"};
return test;
}
TestCase BufferAtomicCmpSwapExactRaw() {
using O = ShaderOpcode;
std::vector<u32> code;
const auto Store = [&](u32 value, u32 index) {
AppendVMovU32(&code, 31, index * sizeof(u32));
code.push_back(EncodeMubuf0(0x1cu));
code.push_back(EncodeMubuf1(value, 8, 31));
};
AppendVMovU32(&code, 4, 1);
AppendVMovLiteral(&code, 1, 0x22222222u);
AppendVMovLiteral(&code, 2, 0x11111111u);
code.push_back(0xe0c46000u);
code.push_back(0x80080104u);
Store(1, 2);
AppendVMovLiteral(&code, 1, 0x33333333u);
AppendVMovLiteral(&code, 2, 0x44444444u);
code.push_back(0xe0c46000u);
code.push_back(0x80080104u);
Store(1, 3);
Store(2, 4);
AppendVMovLiteral(&code, 1, 0x55555555u);
AppendVMovLiteral(&code, 2, 0x22222222u);
code.push_back(EncodeMubuf0(0x31u, 0, true, false));
code.push_back(0x80080104u);
Store(1, 5);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicCmpSwapExactRaw";
test.code = std::move(code);
test.initial = {0xaaaaaaaau, 0x11111111u, 0, 0, 0, 0};
test.expected = {0xaaaaaaaau, 0x55555555u, 0x11111111u,
0x22222222u, 0x44444444u, 0x55555555u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_ATOMIC_CMPSWAP,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicCompareExchange"};
const auto descriptor = MakeStructuredStorageBufferData(
sizeof(u32), static_cast<u32>(test.initial.size()));
std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 32);
test.has_user_data = true;
return test;
}
TestCase BufferAtomicGlc0DoesNotReturnOldValue() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 0, 5);
AppendBufferStoreOpcode(&code, 0x32, 0, 20);
AppendStoreVgpr(&code, 0, 1);
AppendEnd(&code);
return {
"BufferAtomicGlc0DoesNotReturnOldValue",
code,
{10, 0},
{15, 5},
{O::V_MOV_B32, O::BUFFER_ATOMIC_ADD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferAtomicFMinExactRawGlcModes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x40000000u); // 2.0
code.push_back(0xe0fc0000u);
code.push_back(0x80010000u); // exact failing buffer_atomic_fmin, GLC=0
AppendStoreVgpr(&code, 0, 1);
AppendVMovLiteral(&code, 0, 0x3f800000u); // 1.0
code.push_back(0xe0fc4000u);
code.push_back(0x80010000u); // same instruction with GLC=1
AppendStoreVgpr(&code, 0, 2);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicFMinExactRawGlcModes";
test.code = code;
test.initial = {0x40800000u, 0, 0}; // 4.0
test.expected = {0x3f800000u, 0x40000000u, 0x40000000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
const auto descriptor = MakeStructuredStorageBufferData(
0, static_cast<u32>(test.initial.size() * sizeof(u32)));
std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4);
test.user_data[50] = 1u << 20u;
test.has_user_data = true;
return test;
}
TestCase BufferAtomicFMinSpecialValues() {
using O = ShaderOpcode;
const u32 values[] = {
0x40000000u, // 2.0
0x7f800000u, // +infinity
0xff800000u, // -infinity
0x3f800000u, // 1.0
0x7fc00000u, // quiet NaN
0x00000000u, // +0.0
0x80000000u, // -0.0
0x00000000u, // +0.0
0x80000001u, // smallest negative denorm
};
std::vector<u32> code;
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovU32(&code, 20, i * 4u);
AppendVMovLiteral(&code, i, values[i]);
AppendBufferStoreOpcode(&code, 0x3f, i, 20, true);
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, i, i + static_cast<u32>(std::size(values)));
}
AppendEnd(&code);
return {"BufferAtomicFMinSpecialValues",
code,
{0x40800000u, 0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u,
0x80000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0, 0, 0, 0, 0, 0,
0, 0, 0},
{0x40000000u, 0xbf800000u, 0xff800000u, 0x7fc00000u, 0x3f800000u,
0x80000000u, 0x00000000u, 0x00000000u, 0x80000001u, 0x40800000u,
0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u, 0x80000000u,
0x00000000u, 0x00000001u, 0x00000000u},
{O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase BufferAtomicFMinContendedWorkgroup() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // v_cvt_f32_u32 v1, thread_id.x
AppendVMovU32(&code, 20, 0);
AppendBufferStoreOpcode(&code, 0x3f, 1, 20);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicFMinContendedWorkgroup";
test.code = code;
test.initial = {0x42c80000u}; // 100.0
test.expected = {0x00000000u}; // min(100.0, 0.0 .. 63.0)
test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase BufferAtomicFMaxExactRawGlcModes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 3, 0x40000000u); // 2.0
code.push_back(0xe100000cu);
code.push_back(
0x80010300u); // exact failing buffer_atomic_fmax v3, offset 12, GLC=0
AppendStoreVgpr(&code, 3, 4);
AppendVMovLiteral(&code, 0, 0x40800000u); // 4.0
code.push_back(0xe100400cu);
code.push_back(0x80010000u); // same address with GLC=1
AppendStoreVgpr(&code, 0, 5);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicFMaxExactRawGlcModes";
test.code = code;
test.initial = {0, 0, 0, 0x3f800000u, 0, 0}; // memory[3] = 1.0
test.expected = {0, 0, 0, 0x40800000u, 0x40000000u, 0x40000000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
const auto descriptor = MakeStructuredStorageBufferData(
0, static_cast<u32>(test.initial.size() * sizeof(u32)));
std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4);
test.user_data[50] = 1u << 20u;
test.has_user_data = true;
return test;
}
TestCase BufferAtomicFMaxSpecialValues() {
using O = ShaderOpcode;
const u32 values[] = {
0x40000000u, // 2.0, finite update
0x40000000u, // 2.0, finite no-update
0x7f800000u, // incoming +infinity
0xff800000u, // incoming -infinity
0x3f800000u, // finite against old +infinity
0x3f800000u, // finite against old -infinity
0x3f800000u, // finite against old quiet NaN
0x7fcabcdeu, // incoming quiet NaN
0x3f800000u, // finite against old signaling NaN
0x7faabcdeu, // incoming signaling NaN
0x00000000u, // +0.0 against old -0.0
0x80000000u, // -0.0 against old +0.0
0x80000000u, // -0.0 against a negative denorm
0x80000001u, // negative denorm against -0.0
0x00000001u, // positive denorm against +0.0
0x00000000u, // +0.0 against a positive denorm
};
std::vector<u32> code;
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovU32(&code, 20, i * 4u);
AppendVMovLiteral(&code, i, values[i]);
AppendBufferStoreOpcode(&code, 0x40, i, 20, true);
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, i, i + static_cast<u32>(std::size(values)));
}
AppendEnd(&code);
return {"BufferAtomicFMaxSpecialValues",
code,
{0x3f800000u, 0x40800000u, 0xbf800000u, 0x3f800000u,
0x7f800000u, 0xff800000u, 0x7fc12345u, 0x3f800000u,
0x7fa54321u, 0x3f800000u, 0x80000000u, 0x00000000u,
0x80000001u, 0x80000000u, 0x00000000u, 0x00000001u,
0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0},
{0x40000000u, 0x40800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u,
0x3f800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u, 0x3f800000u,
0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000001u,
0x00000001u, 0x3f800000u, 0x40800000u, 0xbf800000u, 0x3f800000u,
0x7f800000u, 0xff800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u,
0x3f800000u, 0x80000000u, 0x00000000u, 0x80000001u, 0x80000000u,
0x00000000u, 0x00000001u},
{O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase BufferAtomicFMaxContendedWorkgroup() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // v_cvt_f32_u32 v1, thread_id.x
AppendVMovU32(&code, 20, 0);
AppendBufferStoreOpcode(&code, 0x40, 1, 20);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicFMaxContendedWorkgroup";
test.code = code;
test.initial = {0xc2c80000u}; // -100.0
test.expected = {0x427c0000u}; // max(-100.0, 0.0 .. 63.0)
test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
std::vector<u32> MakeRgbaImage(u32 width, u32 height, u32 value = 0) {
return std::vector<u32>(static_cast<size_t>(width) * height * 4u, value);
}
void SetRgbaPixel(std::vector<u32> *image, u32 width, u32 x, u32 y, u32 r,
u32 g, u32 b, u32 a) {
const auto base = static_cast<size_t>((y * width + x) * 4u);
(*image)[base + 0] = r;
(*image)[base + 1] = g;
(*image)[base + 2] = b;
(*image)[base + 3] = a;
}
TestCase ImageLoadVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 23, 0);
code.push_back(EncodeMimg0(0x00, 0xf));
code.push_back(EncodeMimg1(0, 20));
code.push_back(EncodeMimg0(0x01, 0xf));
code.push_back(EncodeMimg1(4, 20));
AppendVMovU32(&code, 24, 0);
code.push_back(EncodeMimg0(0x0e, 0x1));
code.push_back(EncodeMimg1(8, 24));
for (u32 i = 0; i < 9u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
TestCase test;
test.name = "ImageLoadVariants";
test.code = code;
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u, 0x3f800000u, 0x40000000u,
0x40400000u, 0x40800000u, 4};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD,
O::IMAGE_LOAD_MIP, O::IMAGE_GET_RESINFO,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.sampled_image_rgba = image;
return test;
}
TestCase DsAppendConsumeUsesEncodedLdsSelector() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 124, 0xffff0000u);
AppendVMovU32(&code, 1, 0);
AppendVMovU32(&code, 2, 10);
code.push_back(EncodeDs0(0x0d, 0));
code.push_back(EncodeDs1(0, 2, 1));
code.push_back(EncodeDs0(0x3e, 1));
code.push_back(EncodeDs1(3, 0, 0));
code.push_back(EncodeDs0(0x3d, 2));
code.push_back(EncodeDs1(4, 0, 0));
AppendSMovLiteral(&code, 124, 0);
code.push_back(EncodeDs0(0x3e, 3));
code.push_back(EncodeDs1(5, 0, 0));
code.push_back(EncodeDs0(0x36, 0));
code.push_back(EncodeDs1(6, 0, 1));
AppendStoreVgpr(&code, 3, 0);
AppendStoreVgpr(&code, 4, 1);
AppendStoreVgpr(&code, 5, 2);
AppendStoreVgpr(&code, 6, 3);
AppendEnd(&code);
return {"DsAppendConsumeLdsSelector",
code,
{},
{10, 11, 10, 11},
{O::S_MOV_B32, O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_B32,
O::DS_APPEND, O::DS_CONSUME, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsAppendAllocatesAcrossWaves(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 64 ? "DsAppendAllocatesAcrossWaves64"
: "DsAppendAllocatesAcrossWaves32";
auto &code = test.code;
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 5, 1);
// Only workgroup lane zero initializes the shared allocation counter.
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(EncodeDs0(0x0d, 0));
code.push_back(EncodeDs1(0, 2, 2));
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
for (u32 batch = 0; batch < 2; ++batch) {
// Captured CS ea0aceac sets M0=0 and uses OFFSET=1. LDS ignores M0
// and aligns OFFSET before allocating once for every active wave.
AppendSMovLiteral(&code, 124, batch == 0 ? 0 : 0xffff0000u);
code.push_back(EncodeDs0(0x3e, batch == 0 ? 1 : 3));
code.push_back(EncodeDs1(3, 0, 0));
code.push_back(EncodeVop2(0x24, 6, 127, 3));
code.push_back(EncodeVop2(0x23, 6, 126, 6));
code.push_back(EncodeVop2(0x1a, 7, InlineU32(2), 6));
// A histogram detects duplicate allocations without assuming wave order.
AppendBufferStoreOpcode(&code, 0x32, 5, 7);
}
code.push_back(EncodeSopp(0x0a, 0));
code.push_back(EncodeDs0(0x36, 0));
code.push_back(EncodeDs1(8, 0, 2));
AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, 512);
AppendEnd(&code);
test.expected.assign(512, 1);
test.expected.resize(768, 512);
test.initial.resize(test.expected.size());
test.compute_info.threads_num[0] = 256;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.compute_info.lds_size_dwords = 1;
test.has_compute_info = true;
test.forbidden_spirv = {"\"gds\""};
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_BARRIER,
O::V_MOV_B32, O::V_CMP_EQ_U32, O::DS_WRITE_B32,
O::DS_APPEND, O::V_MBCNT_HI_U32_B32,
O::V_MBCNT_LO_U32_B32, O::V_LSHLREV_B32,
O::BUFFER_ATOMIC_ADD, O::DS_READ_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase DsAppendUsesEncodedGdsSelector() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 124, 0x00000008u);
code.push_back(EncodeDs0(0x3e, 0, true));
code.push_back(EncodeDs1(0, 0, 0));
code.push_back(EncodeDs0(0x3d, 0, true));
code.push_back(EncodeDs1(1, 0, 0));
code.push_back(EncodeDs0(0x3e, 4, true));
code.push_back(EncodeDs1(2, 0, 0));
code.push_back(EncodeDs0(0x3d, 4, true));
code.push_back(EncodeDs1(3, 0, 0));
AppendSMovLiteral(&code, 124, 0x00080008u);
code.push_back(EncodeDs0(0x3e, 4, true));
code.push_back(EncodeDs1(4, 0, 0));
code.push_back(EncodeDs0(0x3d, 4, true));
code.push_back(EncodeDs1(5, 0, 0));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendStoreVgpr(&code, 2, 2);
AppendStoreVgpr(&code, 3, 3);
AppendStoreVgpr(&code, 4, 4);
AppendStoreVgpr(&code, 5, 5);
AppendEnd(&code);
TestCase test{"DsAppendGdsSelector",
code,
{},
{10, 11, 20, 21, 40, 41},
{O::S_MOV_B32, O::DS_APPEND, O::DS_CONSUME,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.gds_initial = {10, 20, 30, 40};
test.expected_gds = {10, 20, 30, 40};
return test;
}
TestCase DsOrderedCountAddressAndExec(u32 wave_size, bool pixel_counter) {
using O = ShaderOpcode;
TestCase test;
test.name = pixel_counter
? (wave_size == 64 ? "DsOrderedCountPixelAddressWave64"
: "DsOrderedCountPixelAddressWave32")
: (wave_size == 64 ? "DsOrderedCountAddressAndExecWave64"
: "DsOrderedCountAddressAndExecWave32");
auto &code = test.code;
// Both the base and OFFSET0 are aligned independently; M0's low bits are
// a wave index, or a pixel packer ID, rather than a byte address or size.
AppendSMovLiteral(&code, 124, 0x01030003u);
code.push_back(EncodeSop1(0x04, 10, 126)); // Save the full EXEC mask.
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0)); // ADDR value = lane + 1.
AppendVMovU32(&code, 9, 99); // DATA0 must not supply the count.
const u32 selected_lane = wave_size == 64 ? 47u : 7u;
const std::array<uint64_t, 3> masks = {
wave_size == 64 ? UINT64_MAX : uint64_t{UINT32_MAX},
uint64_t{1} << selected_lane, 0};
u32 counter = 1000;
for (bool exchange : {false, true}) {
for (size_t mode = 0; mode < masks.size(); ++mode) {
AppendVMovU32(&code, 3, 0xdeadbeefu);
AppendSMovLiteral(&code, 126, static_cast<u32>(masks[mode]));
AppendSMovLiteral(&code, 127, static_cast<u32>(masks[mode] >> 32u));
const u32 release_done = exchange && mode == 2 ? 3u : 0u;
const u32 offset1 = release_done | (pixel_counter ? 4u : 0u) |
(exchange ? 0x10u : 0u);
code.push_back(EncodeDs0(0x3f, (offset1 << 8u) | 7u, true));
code.push_back(EncodeDs1(3, 9, 2));
code.push_back(EncodeSopp(0x0c, 0)); // Wait for the returned GDS value.
code.push_back(EncodeSop1(0x04, 126, 10)); // Restore EXEC before reading all lanes.
AppendStoreVgprAtLaneDwordOffset(&code, 3, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, counter);
if (mode != 2) {
const u32 value = mode == 0 ? 1u : selected_lane + 1u;
counter = exchange ? value : counter + value;
}
}
}
AppendEnd(&code);
test.initial.assign(test.expected.size(), 0xdeadbeefu);
const u32 counter_address = pixel_counter ? 0x110u : 0x104u;
test.gds_initial.assign(0x120u / sizeof(u32), 100);
test.gds_initial[counter_address / sizeof(u32)] = 1000;
test.expected_gds = test.gds_initial;
test.expected_gds[counter_address / sizeof(u32)] = counter;
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_ADD_NC_U32, O::V_MOV_B32,
O::DS_ORDERED_COUNT, O::S_WAITCNT, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase DsAppendConsumeGdsRegionBounds() {
using O = ShaderOpcode;
struct Access {
u32 m0, offset;
bool valid;
};
constexpr Access accesses[] = {
{0x00000004u, 3, true},
{0x00000004u, 4, false},
{0x00000003u, 0, false},
{0xbffc0004u, 3, true},
{0xbffc0008u, 0, false},
{0xc0000004u, 0, false},
{0x0000c000u, 0xbfff, true},
{0x0000c000u, 0xc000, false},
};
TestCase test;
test.name = "DsAppendConsumeGdsRegionBounds";
auto &code = test.code;
for (const auto &access : accesses) {
AppendSMovLiteral(&code, 124, access.m0);
code.push_back(EncodeDs0(0x3e, access.offset, true));
code.push_back(EncodeDs1(0, 0, 0));
code.push_back(EncodeDs0(0x3d, access.offset, true));
code.push_back(EncodeDs1(1, 0, 0));
AppendStoreVgpr(&code, 0, static_cast<u32>(test.expected.size()));
test.expected.push_back(access.valid ? 10 : 0);
AppendStoreVgpr(&code, 1, static_cast<u32>(test.expected.size()));
test.expected.push_back(access.valid ? 11 : 0);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
// Backing beyond the guest partition must not make an invalid region valid.
test.gds_initial.assign(0x10000u / sizeof(u32), 10);
test.expected_gds = test.gds_initial;
test.opcodes = {O::S_MOV_B32, O::DS_APPEND, O::DS_CONSUME,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase DsGdsSubdwordAndAtomicWrites() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, Vgpr(0)));
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0));
code.push_back(EncodeDs0(0x1e, 0, true));
code.push_back(EncodeDs1(0, 2, 1));
code.push_back(EncodeVop2(0x1a, 8, InlineU32(1), 0));
code.push_back(EncodeVop2(0x25, 8, InlineU32(12), 8));
code.push_back(EncodeDs0(0x1f, 0, true));
code.push_back(EncodeDs1(0, 2, 8));
AppendVMovU32(&code, 3, 4);
AppendVMovU32(&code, 4, 1);
code.push_back(EncodeDs0(0x00, 0, true));
code.push_back(EncodeDs1(0, 4, 3));
AppendVMovU32(&code, 5, 8);
AppendVMovLiteral(&code, 6, 0x40a00000u);
AppendVMovLiteral(&code, 7, 0x40a00000u);
code.push_back(EncodeDs0(0x12, 0, true));
code.push_back(EncodeDs1Ex(0, 7, 6, 5));
AppendEnd(&code);
TestCase test;
test.name = "DsGdsSubdwordAndAtomics";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::DS_WRITE_B8, O::DS_WRITE_B16, O::DS_ADD_U32,
O::DS_MIN_F32, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
test.gds_initial = {0, 0, 0x42c80000u, 0, 0};
test.expected_gds = {0x04030201u, 4, 0x40a00000u, 0x00020001u, 0x00040003u};
return test;
}
TestCase ImageLoadR32UintUsesIntegerSampledImage() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
code.push_back(EncodeMimg0(0x00, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageLoadR32UintUsesIntegerSampledImage";
test.code = code;
test.expected = {0xdeadbeefu};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba.resize(16);
test.sampled_image_rgba[6] = 0xdeadbeefu;
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.required_spirv = {"image_10", "OpTypeImage %uint",
"OpImageFetch %v4uint"};
return test;
}
TestCase ImageLoadFmaskUsesNativeSampleMapping() {
auto test = ImageLoadR32UintUsesIntegerSampledImage();
test.name = "ImageLoadFmaskUsesNativeSampleMapping";
test.expected = {0x76543210u};
test.user_data = {0x303ac300u, 0xca100000u, 0x021bc3bfu, 0x91800004u,
0u, 0x00700000u, 0u, 0u};
test.image_descriptor_swizzle = DstSel(4, 0, 0, 0);
test.sampled_image_rgba.clear();
test.required_spirv = {"OpConstant %uint 1985229328"};
test.forbidden_spirv = {"OpTypeImage", "OpImageFetch"};
return test;
}
TestCase ImageLoadR32SintUsesSignedSampledImage() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
code.push_back(EncodeMimg0(0x00, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageLoadR32SintUsesSignedSampledImage";
test.code = code;
test.expected = {0xffffff80u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba.resize(16);
test.sampled_image_rgba[6] = 0xffffff80u;
test.sampled_image_format = vk::Format::eR32Sint;
test.sampled_image_dwords_per_pixel = 1;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32SInt);
test.has_user_data = true;
test.required_spirv = {"image_17", "OpTypeImage %int", "OpImageFetch %v4int",
"OpBitcast %uint"};
return test;
}
TestCase ImageLoadPackedUintUnpacksAndSwizzles() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
code.push_back(EncodeMimg0(0x00, 0xf));
code.push_back(EncodeMimg1(0, 20));
for (u32 component = 0; component < 4; component++) {
AppendStoreVgpr(&code, component, component);
}
AppendEnd(&code);
TestCase test;
test.name = "ImageLoadPackedUintUnpacksAndSwizzles";
test.code = std::move(code);
test.expected = {0x2abu, 0x456u, 0x321u, 0x321u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba.resize(16);
test.sampled_image_rgba[6] = 0xaae2b321u;
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.user_data =
MakeSampledTextureData(Prospero::BufferFormat::k11_11_10UInt);
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(6, 5, 4, 7);
test.required_spirv = {"OpImageFetch %v4uint", "OpBitFieldUExtract"};
return test;
}
TestCase ImageSamplePackedUintConvertsSampleAndGather() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x3f200000u);
AppendVMovLiteral(&code, 21, 0x3ec00000u);
code.push_back(EncodeMimg0(0x20, 0xf));
code.push_back(EncodeMimg1(0, 20));
code.push_back(EncodeMimg0(0x47, 0x2));
code.push_back(EncodeMimg1(4, 20));
for (u32 component = 0; component < 8; component++) {
AppendStoreVgpr(&code, component, component);
}
AppendEnd(&code);
TestCase test;
test.name = "ImageSamplePackedUintConvertsSampleAndGather";
test.code = std::move(code);
test.expected = {0x2abu, 0x456u, 0x321u, 0x321u,
0x456u, 0x456u, 0x456u, 0x456u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::IMAGE_GATHER4_LZ,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.sampled_image_rgba = std::vector<u32>(16, 0xaae2b321u);
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.user_data =
MakeSampledTextureData(Prospero::BufferFormat::k11_11_10UInt);
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(6, 5, 4, 7);
test.expected_force_point_sampler = true;
test.required_spirv = {"OpImageSampleExplicitLod", "OpImageGather",
"OpBitFieldUExtract"};
return test;
}
template <bool rg> TestCase ImageSampleUScaled8() {
using O = ShaderOpcode;
TestCase test;
test.name = rg ? "ImageSampleRG8UScaled" : "ImageSampleR8UScaled";
AppendVMovLiteral(&test.code, 20, std::bit_cast<u32>(0.375f));
AppendVMovLiteral(&test.code, 21, std::bit_cast<u32>(0.375f));
test.code.push_back(EncodeMimg0(0x20, 0xf));
test.code.push_back(EncodeMimg1(0, 20));
test.code.push_back(EncodeMimg0(0x47, 0x1));
test.code.push_back(EncodeMimg1(4, 20));
test.code.push_back(EncodeMimg0(0x47, 0x8));
test.code.push_back(EncodeMimg1(8, 20));
for (u32 component = 0; component < 12; ++component) {
AppendStoreVgpr(&test.code, component, component);
}
AppendEnd(&test.code);
// R varies horizontally; G varies vertically in the opposite direction.
// Bilinear samples must retain fractions despite host UNorm filter precision.
test.image_width = test.image_height = 2;
test.sampled_image_rgba = rg ? std::vector<u32>{0xffffff00u, 0x00ff0000u}
: std::vector<u32>{0xff00ff00u};
test.sampled_image_format = rg ? vk::Format::eR8G8Unorm : vk::Format::eR8Unorm;
test.sampled_image_dwords_per_pixel = 1;
test.sampler_filter = vk::Filter::eLinear;
test.expected_float_tolerance = 0.01f;
test.user_data = MakeSampledTextureData(rg ? Prospero::BufferFormat::k8_8UScaled
: Prospero::BufferFormat::k8UScaled);
test.user_data[1] |= 1u << 30u;
test.user_data[2] = 1u << 14u;
test.user_data[50] = 12u * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(rg ? 5 : 4, 4, 0, 1);
for (float value : {rg ? 191.25f : 63.75f, 63.75f, 0.0f, 1.0f,
0.0f, rg ? 0.0f : 255.0f, 255.0f, rg ? 255.0f : 0.0f,
1.0f, 1.0f, 1.0f, 1.0f}) {
test.expected.push_back(std::bit_cast<u32>(value));
}
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::IMAGE_GATHER4_LZ,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase ImageLoadR128IgnoresAdjacentMaskSgprs() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
code.push_back(EncodeSop1(0x04, 4, 126)); // s_mov_b64 s[4:5], exec
code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 1, true));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageLoadR128IgnoresAdjacentMaskSgprs";
test.code = code;
test.expected = {0xdeadbeefu};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::IMAGE_LOAD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.sampled_image_rgba.resize(16);
test.sampled_image_rgba[6] = 0xdeadbeefu;
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.decoded_counts = {{"r128=1", 1}};
return test;
}
TestCase ImageLoad1DUsesScalarCoordinate() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 0));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageLoad1DUsesScalarCoordinate";
test.code = code;
test.expected = {0xdeadbeefu};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.image_width = 4;
test.image_height = 1;
test.sampled_image_rgba = {0, 0, 0xdeadbeefu, 0};
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.sampled_image_type = vk::ImageType::e1D;
test.sampled_image_view_type = vk::ImageViewType::e1D;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1D)
<< 28u;
test.has_user_data = true;
test.required_spirv = {"image_8"};
return test;
}
TestCase ImageGather2DInstructionWith1DDescriptor() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x3f000000u);
AppendVMovLiteral(&code, 21, 0x3f000000u);
code.push_back(EncodeMimg0(0x47, 0x1));
code.push_back(EncodeMimg1(0, 20));
for (u32 i = 0; i < 4u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 1);
SetRgbaPixel(&image, 4, 1, 0, 0x40000000u, 0, 0, 0);
SetRgbaPixel(&image, 4, 2, 0, 0x40400000u, 0, 0, 0);
TestCase test;
test.name = "ImageGather2DInstructionWith1DDescriptor";
test.code = code;
test.expected = {0x40000000u, 0x40400000u, 0x40400000u, 0x40000000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_GATHER4_LZ, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.image_width = 4;
test.image_height = 1;
test.sampled_image_rgba = image;
test.sampled_image_type = vk::ImageType::e1D;
test.sampled_image_view_type = vk::ImageViewType::e1D;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32Float);
test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1D)
<< 28u;
test.has_user_data = true;
test.required_spirv = {"image_1", "OpImageQuerySizeLod",
"OpImageSampleExplicitLod", "Floor"};
test.forbidden_spirv = {"OpImageGather"};
return test;
}
TestCase ImageLoad1DArrayUsesLayerCoordinate() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 4));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageLoad1DArrayUsesLayerCoordinate";
test.code = code;
test.expected = {0xcafebabeu};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.image_width = 4;
test.image_height = 1;
test.sampled_image_rgba = {0, 0, 0, 0, 0, 0, 0xcafebabeu, 0};
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.sampled_image_type = vk::ImageType::e1D;
test.sampled_image_view_type = vk::ImageViewType::e1DArray;
test.sampled_image_layers = 2;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1DArray)
<< 28u;
test.has_user_data = true;
test.required_spirv = {"image_9"};
return test;
}
TestCase ImageLoad1DArrayDescriptorUsesSelectedLayer() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 0));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageLoad1DArrayDescriptorUsesSelectedLayer";
test.code = code;
test.expected = {0xcafebabeu};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.image_width = 4;
test.image_height = 1;
test.sampled_image_rgba = {
0, 0, 0xdeadbeefu, 0, 0, 0, 0xcafebabeu, 0,
};
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.sampled_image_type = vk::ImageType::e1D;
test.sampled_image_view_type = vk::ImageViewType::e1D;
test.sampled_image_layers = 2;
test.sampled_image_view_base_layer = 1;
test.sampled_image_view_layers = 1;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1DArray)
<< 28u;
test.user_data[4] = 1u | (1u << 16u);
test.has_user_data = true;
test.required_spirv = {"image_8"};
return test;
}
TestCase ImageLoadMipUsesVaddr2Lod2D() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 1);
AppendVMovU32(&code, 23, 0);
code.push_back(EncodeMimg0(0x01, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
auto base = MakeRgbaImage(4, 4);
SetRgbaPixel(&base, 4, 1, 1, 0x3f800000u, 0, 0, 0);
auto mip1 = MakeRgbaImage(2, 2);
SetRgbaPixel(&mip1, 2, 1, 1, 0x40000000u, 0, 0, 0);
TestCase test;
test.name = "ImageLoadMipUsesVaddr2Lod2D";
test.code = code;
test.expected = {0x40000000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD_MIP, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba_mips = {base, mip1};
return test;
}
TestCase ImageLoadMipNsaUsesSelectedAddressVgprs() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 30, 1);
AppendVMovU32(&code, 31, 1);
code.push_back(EncodeMimg0(0x01, 0x1, 1));
code.push_back(EncodeMimg1(0, 20));
code.push_back((30u << 0u) | (31u << 8u));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
auto base = MakeRgbaImage(4, 4);
SetRgbaPixel(&base, 4, 1, 0, 0x3f800000u, 0, 0, 0);
auto mip1 = MakeRgbaImage(2, 2);
SetRgbaPixel(&mip1, 2, 1, 1, 0x40a00000u, 0, 0, 0);
TestCase test;
test.name = "ImageLoadMipNsaUsesSelectedAddressVgprs";
test.code = code;
test.expected = {0x40a00000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD_MIP, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba_mips = {base, mip1};
return test;
}
TestCase ImageLoadA16UintCoordsOnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x00010002u); // x=2, y=1 packed as u16.
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeMimg0(0x00, 0xf));
code.push_back(EncodeMimg1(0, 20, 0, 0, true));
for (u32 i = 0; i < 4u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
TestCase test;
test.name = "ImageLoadA16UintCoordsOnGpu";
test.code = code;
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba = image;
test.required_spirv = {"OpShiftRightLogical", "OpBitwiseAnd"};
test.forbidden_spirv = {"UnpackHalf2x16"};
return test;
}
TestCase ImageGetResinfoDmaskWidthHeight() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 24, 0);
code.push_back(EncodeMimg0(0x0e, 0x3));
code.push_back(EncodeMimg1(0, 24));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
TestCase test;
test.name = "ImageGetResinfoDmaskWidthHeight";
test.code = code;
test.expected = {4, 2};
test.opcodes = {O::V_MOV_B32, O::IMAGE_GET_RESINFO, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.image_width = 4;
test.image_height = 2;
test.sampled_image_rgba = MakeRgbaImage(test.image_width, test.image_height);
return test;
}
TestCase ImageGetResinfoDmaskMipLevels() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 24, 0);
code.push_back(EncodeMimg0(0x0e, 0x8));
code.push_back(EncodeMimg1(0, 24));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageGetResinfoDmaskMipLevels";
test.code = code;
test.expected = {2};
test.opcodes = {O::V_MOV_B32, O::IMAGE_GET_RESINFO, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
auto base = MakeRgbaImage(4, 4);
auto mip1 = MakeRgbaImage(2, 2);
test.sampled_image_rgba_mips = {base, mip1};
return test;
}
TestCase ImageSampleAndGather() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x3f200000u);
AppendVMovLiteral(&code, 21, 0x3ec00000u);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMimg0(0x20, 0xf));
code.push_back(EncodeMimg1(0, 20));
code.push_back(EncodeMimg0(0x47, 0x1));
code.push_back(EncodeMimg1(4, 20));
AppendVMovU32(&code, 24, 0);
AppendVMovLiteral(&code, 25, 0x3f200000u);
AppendVMovLiteral(&code, 26, 0x3ec00000u);
AppendVMovU32(&code, 27, 0);
code.push_back(EncodeMimg0(0x57, 0x1));
code.push_back(EncodeMimg1(8, 24));
code.push_back(EncodeMimg0(0x60, 0x1));
code.push_back(EncodeMimg1(12, 20));
for (u32 i = 0; i < 13u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
for (u32 y = 0; y < 4u; y++) {
for (u32 x = 0; x < 4u; x++) {
SetRgbaPixel(&image, 4, x, y, 0x3f800000u, 0, 0, 0);
}
}
SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
TestCase test;
test.name = "ImageSampleAndGather";
test.code = code;
test.expected = {0x3f800000u,
0x40000000u,
0x40400000u,
0x40800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0};
test.opcodes = {
O::V_MOV_B32, O::IMAGE_SAMPLE, O::IMAGE_GET_LOD,
O::IMAGE_GATHER4_LZ, O::IMAGE_GATHER4_LZ_O, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba = image;
return test;
}
TestCase ImageCubeGradientsPreserveDerivatives() {
using O = ShaderOpcode;
TestCase test;
test.name = "ImageCubeGradientsPreserveDerivatives";
for (u32 component = 0; component < 7u; component++) {
constexpr std::array values{0.125f, 0.0f, 0.0f, 0.125f,
1.375f, 1.625f, 10.0f};
AppendVMovLiteral(&test.code, 20u + component,
std::bit_cast<u32>(values[component]));
}
test.code.push_back(EncodeMimg0(0x22, 0x1, 0, false, 3));
test.code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&test.code, 0, 0);
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.user_data[1] |= 3u << 30u;
test.user_data[2] = 3u << 14u;
test.user_data[3] =
(static_cast<u32>(Prospero::ImageType::kCube) << 28u) | (2u << 16u);
test.user_data[4] = 11u;
test.user_data[5] = 2u << 4u;
test.user_data[50] = sizeof(u32);
test.has_user_data = true;
test.sampled_image_view_type = vk::ImageViewType::e2DArray;
test.sampled_image_layers = 12;
for (u32 mip = 0; mip < 3u; mip++) {
const auto layer_dwords = (4u >> mip) * (4u >> mip) * 4u;
auto &pixels = test.sampled_image_rgba_mips.emplace_back(layer_dwords * 12u);
for (u32 layer = 0; layer < 12u; layer++) {
std::fill_n(pixels.begin() + layer * layer_dwords, layer_dwords,
std::bit_cast<u32>(static_cast<float>(1u + layer + 100u * mip)));
}
}
// Guest face 10 is host layer 8. Unmodified gradients select mip 0;
// applying the cube coordinate bias to gradients instead selects mip 2.
test.expected = {std::bit_cast<u32>(9.0f)};
test.required_spirv = {"OpImageSampleExplicitLod", "Grad"};
return test;
}
TestCase ImageGatherExplicitLod() {
using O = ShaderOpcode;
std::vector<u32> code;
// PPSA29343 uses NSA coordinates v81/v84 with its explicit LOD in v78.
AppendVMovLiteral(&code, 81, 0x3f000000u);
AppendVMovLiteral(&code, 84, 0x3f000000u);
AppendVMovLiteral(&code, 78, 0x3f800000u);
code.push_back(EncodeMimg0(0x44, 0x1, 1));
code.push_back(EncodeMimg1(9, 81, 0, 2));
code.push_back(84u | (78u << 8u));
code.push_back(EncodeMimg0(0x24, 0x1, 1));
code.push_back(EncodeMimg1(13, 81, 0, 2));
code.push_back(84u | (78u << 8u));
for (u32 i = 0; i < 5u; i++) {
AppendStoreVgpr(&code, i + 9, i);
}
AppendEnd(&code);
TestCase test;
test.name = "ImageGatherExplicitLod";
test.code = std::move(code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_GATHER4_L, O::IMAGE_SAMPLE,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
// A gather's single-mip descriptors must not replace the ordinary sample's
// full mip chain when both instructions use the same guest descriptor.
test.expected = std::vector<u32>(5, 0x41000000u);
test.sampled_image_rgba_mips = {std::vector<u32>(4 * 4 * 4, 0x3f800000u),
std::vector<u32>(2 * 2 * 4, 0x41000000u)};
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.user_data[1] |= 3u << 30u;
test.user_data[2] = 3u << 14u;
test.user_data[3] |= 1u << 16u;
test.user_data[8] = 0x92u;
test.user_data[9] = 0xfff000u;
test.user_data[10] = 0x05000000u; // Captured point sampler, preclamp disabled.
test.user_data[50] = test.expected.size() * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.has_user_data = true;
test.expected_mip_descriptors = 2;
test.required_spirv = {"OpImageGather", "OpImageSampleExplicitLod"};
return test;
}
template <u32 mode> TestCase ImageGatherLodPerLane() {
using O = ShaderOpcode;
constexpr std::array names{
"ImageGatherLodPerLane", "ImageGatherLodBaseMip",
"ImageGatherLodSamplerBounds", "ImageGatherLodTextureMinimum",
"ImageGatherLodMipNone", "ImageGatherLodPointPostclampMaximum",
"ImageGatherLodPointPreclampMaximum", "ImageGatherLodPointPostclampMinimum",
"ImageGatherLodPointPreclampMinimum"};
constexpr std::array lods = mode < 5
? std::array{-1.0f, 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 9.0f}
: std::array{-1.0f, 0.0f, 0.25f, 0.75f, 1.25f, 1.75f, 2.75f, 9.0f};
constexpr std::array<std::array<u32, 8>, 9> selected_mips{{
{0, 0, 1, 2, 3, 4, 4, 4},
{1, 1, 2, 3, 4, 4, 4, 4},
{2, 2, 2, 3, 3, 3, 3, 3},
{3, 3, 3, 3, 3, 3, 3, 3},
{1, 1, 1, 1, 1, 1, 1, 1},
{0, 0, 0, 1, 1, 2, 2, 2},
{0, 0, 0, 1, 1, 1, 1, 1},
{2, 2, 2, 2, 2, 2, 3, 4},
{1, 1, 1, 1, 1, 2, 3, 4},
}};
constexpr u32 lanes = lods.size();
constexpr u32 sampler_min = mode == 2 ? 256 : mode >= 7 ? 448 : 0;
constexpr u32 sampler_max = mode == 2 ? 512 : mode == 3 ? 256
: mode == 5 || mode == 6 ? 448 : 1024;
TestCase test;
test.name = names[mode];
test.image_width = test.image_height = 16;
test.sampled_image_view_base_mip = mode > 0 && mode < 5 ? 1 : 0;
for (u32 mip = 0; mip < 5; mip++) {
test.sampled_image_rgba_mips.emplace_back(
(16u >> mip) * (16u >> mip) * 4u,
std::bit_cast<u32>(static_cast<float>(mip + 1)));
}
for (const auto lod : lods) {
test.initial.push_back(std::bit_cast<u32>(lod));
}
test.expected = test.initial;
for (u32 component = 0; component < 4; component++) {
for (const auto mip : selected_mips[mode]) {
test.expected.push_back(std::bit_cast<u32>(static_cast<float>(mip + 1)));
}
}
auto &code = test.code;
code.push_back(EncodeVop1(0x01, 16, Vgpr(0)));
code.push_back(EncodeVop2(0x1a, 31, InlineU32(2), 16));
code.push_back(EncodeMubuf0(0x0c));
code.push_back(EncodeMubuf1(22, 4, 31));
AppendVMovLiteral(&code, 20, 0x3f000000u);
AppendVMovLiteral(&code, 21, 0x3f000000u);
code.push_back(EncodeMimg0(0x44, 0x1));
code.push_back(EncodeMimg1(0, 20, 0, 2));
for (u32 component = 0; component < 4; component++) {
AppendStoreVgprAtLaneDwordOffset(&code, component, 16, lanes * (component + 1));
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD,
O::IMAGE_GATHER4_L, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.user_data[1] |= (3u << 30u) | (mode == 3 ? 768u << 8u : 0u);
test.user_data[2] = 3u | (15u << 14u);
test.user_data[3] |= (test.sampled_image_view_base_mip << 12u) | (4u << 16u);
test.user_data[8] = 0x92u;
test.user_data[9] = sampler_min | (sampler_max << 12u);
// Point preclamp selects whether the half-level rounding adjustment precedes
// or follows the clamps. The streaming T# minimum follows the sampler bounds.
test.user_data[10] = mode == 4 ? 0x4001u : (1u << 26u);
if constexpr (mode == 6 || mode == 8) {
test.user_data[10] |= 1u << 28u;
}
test.user_data[18] = lanes * sizeof(u32);
test.user_data[19] = 3u << 28u;
test.user_data[50] = test.expected.size() * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.has_user_data = true;
test.compute_info.threads_num[0] = lanes;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
test.expected_mip_descriptors = 5 - test.sampled_image_view_base_mip;
test.required_spirv = {"OpImageGather"};
return test;
}
TestCase ImageD16GatherPacksHalfPairs() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x3f000000u);
AppendVMovLiteral(&code, 21, 0x3f000000u);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMimg0(0x47, 0x1));
code.push_back(EncodeMimg1(0, 20, 0, 0, false, true));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
for (u32 y = 0; y < 4u; y++) {
for (u32 x = 0; x < 4u; x++) {
SetRgbaPixel(&image, 4, x, y, 0x3f000000u, 0, 0, 0);
}
}
TestCase test;
test.name = "ImageD16GatherPacksHalfPairs";
test.code = code;
test.expected = {0x38003800u, 0x38003800u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_GATHER4_LZ, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba = image;
test.decoded_counts = {{"d16=1", 1}};
test.required_spirv = {"OpImageGather", "PackHalf2x16"};
return test;
}
TestCase ImageSampleA16SamplerCoordsOnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x36003900u); // x=0.625, y=0.375 packed as f16.
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeMimg0(0x20, 0xf));
code.push_back(EncodeMimg1(0, 20, 0, 0, true));
for (u32 i = 0; i < 4u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
TestCase test;
test.name = "ImageSampleA16SamplerCoordsOnGpu";
test.code = code;
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba = image;
test.required_spirv = {"UnpackHalf2x16"};
return test;
}
TestCase ImageSampleOpcodeAliasUsesNormalCoords() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 2, 0x3f000000u);
AppendVMovLiteral(&code, 3, 0x3f000000u);
code.push_back(0xf0800109u); // observed image_sample_a v6, v2, s0, s24
code.push_back(0x00c00602u);
AppendStoreVgpr(&code, 6, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageSampleOpcodeAliasUsesNormalCoords";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpImageSampleExplicitLod"};
test.forbidden_spirv = {"UnpackHalf2x16"};
test.compile_only = true;
return test;
}
TestCase ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20,
1); // Non-constant +1 X offset is not a SPIR-V ConstOffset.
AppendVMovLiteral(&code, 21, 0x36003900u); // x=0.625, y=0.375 packed as f16.
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMimg0(0x30, 0xf));
code.push_back(EncodeMimg1(0, 20, 0, 0, true));
for (u32 i = 0; i < 4u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
TestCase test;
test.name = "ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu";
test.code = code;
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba = image;
test.required_spirv = {"UnpackHalf2x16"};
test.forbidden_spirv = {"OpBitFieldSExtract"};
return test;
}
TestCase ImageSampleA16CompareBiasRdna2AddressOrder() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x00003800u); // bias=0.5 as low f16.
AppendVMovLiteral(&code, 21, 0x3f000000u); // PCF reference stays 32-bit.
AppendVMovLiteral(&code, 22, 0x36003900u); // x=0.625, y=0.375 packed as f16.
code.push_back(EncodeMimg0(0x2d, 0x1));
code.push_back(EncodeMimg1(0, 20, 0, 0, true));
AppendStoreVgpr(&code, 0, 0);
// Ordinary and comparison reads of one sharp require distinct SPIR-V image types.
code.push_back(EncodeMimg0(0x27, 0x1));
code.push_back(EncodeMimg1(1, 22, 0, 0, true));
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
TestCase test;
test.name = "ImageSampleA16CompareBiasRdna2AddressOrder";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpTypeImage %float 2D 0", "OpTypeImage %float 2D 1",
"OpImageSampleDrefExplicitLod", "OpImageSampleExplicitLod",
"UnpackHalf2x16"};
test.compile_only = true;
return test;
}
TestCase ImageGatherCompareOpcodes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 21, 0x3f000000u);
AppendVMovLiteral(&code, 22, 0x3f200000u);
AppendVMovLiteral(&code, 23, 0x3ec00000u);
AppendVMovLiteral(&code, 24, 0x3f000000u);
AppendVMovLiteral(&code, 25, 0x3f200000u);
AppendVMovLiteral(&code, 26, 0x3ec00000u);
AppendVMovLiteral(&code, 28, 0x3f000000u);
AppendVMovLiteral(&code, 29, 0x3f200000u);
AppendVMovLiteral(&code, 30, 0x3ec00000u);
AppendVMovU32(&code, 32, 0);
AppendVMovLiteral(&code, 33, 0x3f000000u);
AppendVMovLiteral(&code, 34, 0x3f200000u);
AppendVMovLiteral(&code, 35, 0x3ec00000u);
code.push_back(EncodeMimg0(0x48, 0x1));
code.push_back(EncodeMimg1(4, 24));
code.push_back(EncodeMimg0(0x4f, 0x1));
code.push_back(EncodeMimg1(8, 28));
code.push_back(EncodeMimg0(0x58, 0x1));
code.push_back(EncodeMimg1(0, 20));
code.push_back(EncodeMimg0(0x5f, 0x1));
code.push_back(EncodeMimg1(12, 32));
AppendStoreVgpr(&code, 4, 0);
AppendStoreVgpr(&code, 8, 1);
AppendStoreVgpr(&code, 0, 2);
AppendStoreVgpr(&code, 12, 3);
AppendEnd(&code);
TestCase test;
test.name = "ImageGatherCompareOpcodes";
test.code = code;
test.opcodes = {
O::V_MOV_B32, O::IMAGE_GATHER4_C, O::IMAGE_GATHER4_C_LZ,
O::IMAGE_GATHER4_C_O, O::IMAGE_GATHER4_C_LZ_O, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpImageDrefGather", "OpBitFieldSExtract"};
test.compile_only = true;
return test;
}
TestCase ImageStoreVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 2);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x40000000u);
AppendVMovLiteral(&code, 2, 0x40400000u);
AppendVMovLiteral(&code, 3, 0x40800000u);
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendVMovU32(&code, 24, 3);
AppendVMovU32(&code, 25, 2);
AppendVMovU32(&code, 26, 0);
AppendVMovU32(&code, 27, 0);
AppendVMovLiteral(&code, 4, 0x40a00000u);
AppendVMovLiteral(&code, 5, 0x40c00000u);
AppendVMovLiteral(&code, 6, 0x40e00000u);
AppendVMovLiteral(&code, 7, 0x41000000u);
code.push_back(EncodeMimg0(0x09, 0xf));
code.push_back(EncodeMimg1(4, 24));
AppendVMovLiteral(&code, 8, 0x12345678u);
AppendStoreVgpr(&code, 8, 0);
AppendEnd(&code);
auto expected_image = MakeRgbaImage(4, 4);
SetRgbaPixel(&expected_image, 4, 1, 2, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
SetRgbaPixel(&expected_image, 4, 3, 2, 0x40a00000u, 0x40c00000u, 0x40e00000u,
0x41000000u);
TestCase test;
test.name = "ImageStoreVariants";
test.code = code;
test.expected = {0x12345678u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_STORE_MIP,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_rgba = expected_image;
return test;
}
TestCase ImageD16StoreUnpacksHalfPairs() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 2);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x38003400u);
AppendVMovLiteral(&code, 1, 0x3c003a00u);
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20, 0, 0, false, true));
AppendEnd(&code);
auto expected_image = MakeRgbaImage(4, 4);
SetRgbaPixel(&expected_image, 4, 1, 2, 0x3e800000u, 0x3f000000u, 0x3f400000u,
0x3f800000u);
TestCase test;
test.name = "ImageD16StoreUnpacksHalfPairs";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.expected_storage_image_rgba = expected_image;
test.decoded_counts = {{"d16=1", 1}};
test.required_spirv = {"UnpackHalf2x16", "OpImageWrite"};
return test;
}
void CheckComputeThreadDimensions(VulkanHarness &vulkan) {
using O = ShaderOpcode;
constexpr u32 grid_size = 16 * 8 * 4;
const std::array extents = {std::array{9u, 5u, 3u}, std::array{13u, 7u, 4u},
std::array{3u, 2u, 1u}, std::array{16u, 8u, 4u}};
for (const u32 wave_size : {32u, 64u}) {
for (const bool thread_dimensions : {false, true}) {
TestCase test;
test.name = thread_dimensions ? (wave_size == 32 ? "ThreadDimensions32" : "ThreadDimensions64")
: (wave_size == 32 ? "GroupDimensions32" : "GroupDimensions64");
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 20, 126)); // Save the SPI-provided initial EXEC.
AppendSMovLiteral(&code, 24, grid_size * 4);
for (u32 axis = 0; axis < 3; ++axis) {
code.push_back(EncodeVop1(0x01, 3 + axis, 4 + axis));
code.push_back(EncodeVop2(0x1a, 3 + axis, InlineU32(3 - axis), 3 + axis));
code.push_back(EncodeVop2(0x25, 3 + axis, Vgpr(axis), 3 + axis));
}
code.push_back(EncodeVop2(0x1a, 4, InlineU32(4), 4));
code.push_back(EncodeVop2(0x1a, 5, InlineU32(7), 5));
code.push_back(EncodeVop2(0x25, 3, Vgpr(4), 3));
code.push_back(EncodeVop2(0x25, 3, Vgpr(5), 3));
code.push_back(EncodeVop2(0x25, 20, InlineU32(1), 3));
code.push_back(EncodeVop2(0x1a, 20, InlineU32(2), 20));
AppendVMovU32(&code, 6, 1);
AppendVMovU32(&code, 7, 0);
AppendBufferStoreDword(&code, 6, 20);
AppendBufferStoreOpcode(&code, 0x32, 6, 7);
code.push_back(EncodeSop2(0x0f, 126, 20, InlineU32(1)));
AppendVMovU32(&code, 6, 2); // Divergence changes only lane zero of each wave.
code.push_back(EncodeSop1(0x04, 126, 20));
code.push_back(EncodeVop2(0x25, 20, 24, 20));
AppendBufferStoreDword(&code, 6, 20);
AppendVMovU32(&code, 8, 1);
AppendBufferStoreOpcode(&code, 0x32, 8, 7);
AppendEnd(&code);
test.initial.resize(1 + 2 * grid_size);
test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::S_AND_B64, O::V_MOV_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
O::BUFFER_ATOMIC_ADD, O::S_ENDPGM};
test.compute_info.wave_size = wave_size;
test.compute_info.host_subgroup_size = vulkan.SubgroupSize();
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 4;
test.compute_info.threads_num[2] = 2;
std::fill_n(test.compute_info.group_id, 3, true);
test.compute_info.thread_ids_num = 3;
test.compute_info.workgroup_register = 4;
test.compute_info.dispatch_thread_dimensions = thread_dimensions;
test.has_compute_info = true;
test.expand_shader_data_storage = thread_dimensions && wave_size == 32;
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
std::vector<u32> compiled_key;
BuildStageStaticKey(test.compute_info, compiled_key);
for (const auto &extent : extents) {
std::copy(extent.begin(), extent.end(), test.compute_info.dispatch_threads_num);
std::vector<u32> runtime_key;
BuildStageStaticKey(test.compute_info, runtime_key);
Require(test.name, "runtime shader reuse", compiled_key == runtime_key,
"dispatch thread counts changed the compiled shader cache key");
test.dispatch_x = thread_dimensions ? (extent[0] + 7) / 8 : 2;
test.dispatch_y = thread_dimensions ? (extent[1] + 3) / 4 : 2;
test.dispatch_z = thread_dimensions ? (extent[2] + 1) / 2 : 2;
test.expected.assign(test.initial.size(), 0);
for (u32 z = 0; z < 4; ++z) {
for (u32 y = 0; y < 8; ++y) {
for (u32 x = 0; x < 16; ++x) {
if (thread_dimensions && (x >= extent[0] || y >= extent[1] || z >= extent[2])) continue;
const auto index = x + 16 * y + 128 * z;
const auto local_index = x % 8 + 8 * (y % 4) + 32 * (z % 2);
test.expected[0] += 2;
test.expected[1 + index] = 1;
test.expected[1 + grid_size + index] = local_index % wave_size == 0 ? 2 : 1;
}
}
}
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output);
const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
vulkan.DestroyBuffer(&output);
CompareWords(test, "initial EXEC, restored EXEC and padded-lane atomics", test.expected, actual);
if (!thread_dimensions) break;
}
std::printf("[compute] %-32s ok\n", test.name);
}
}
}
void CheckRuntimeBufferRecords(VulkanHarness &vulkan) {
using namespace ShaderRecompiler::IR;
TestCase test;
test.name = "RuntimeBufferRecords";
// Captured b71ef7c579835a0d: read an input count, divide it into chunks,
// construct a structured descriptor, then clear each live record's last DWORD.
test.code = {
0xbfa00003u, 0xd7460000u, 0x04010c0eu, 0x8805ff0du, 0x00040000u, 0xbe84030cu,
0xbe860381u, 0xbe8703ffu, 0x00016204u, 0xb06a0100u, 0xe0300000u, 0x80010400u,
0x92049281u, 0x7e0a026au, 0x996a9480u, 0xb0062000u, 0xb00a0080u, 0x380212f9u,
0x8686066au, 0x7e040208u, 0x7e0602ffu, 0x00016204u, 0xbf8c3f70u, 0x7d8808ffu,
0x00000200u, 0x7d8808f9u, 0x06868404u, 0x7d8808f9u, 0x06868606u, 0x020d80f9u,
0x8686060au, 0xd5010005u, 0x00120affu, 0x00000200u, 0xd5010006u, 0x001a0d05u,
0x7e0a0d06u, 0x7e0c0506u, 0xd76d0004u, 0x041a08c1u, 0x7e0a5705u, 0x81ea0680u,
0x7e080504u, 0x100a0affu, 0x4f7ffffeu, 0x7e0a0f05u, 0x7ed60505u, 0x936a6b6au,
0x9aea6b6au, 0x816a6a6bu, 0x9a87046au, 0x936a0607u, 0x4e08086au, 0x4e0a0806u,
0x7d8608f9u, 0x06868406u, 0x7d860a06u, 0x87ea046au, 0x020902f9u, 0x86860680u,
0xd5286a04u, 0x00120807u, 0xbeea047eu, 0x7da40080u, 0xbf880003u, 0x7e0a0280u,
0xe0700000u, 0x80000500u, 0xbefe046au, 0x7e0a0280u, 0x7e000502u, 0x7e020501u,
0x7e040504u, 0x7e060503u, 0xe0702010u, 0x80000500u, 0xbf810000u,
};
constexpr u32 count_address = 4;
test.user_data[0] = 16;
test.user_data[1] = 4u << 16;
test.user_data[2] = 1;
test.user_data[3] = 0x5204;
test.user_data[8] = 20;
test.user_data[12] = count_address;
test.has_user_data = true;
test.storage_buffer_offsets = {count_address, 16, 20};
test.initial.assign(5 + 5 * 130, 0xdeadbeefu);
test.initial[1] = 72;
test.has_compute_info = true;
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.lds_size_dwords = 0;
test.compute_info.wave_size = 64;
test.compute_info.float_mode = 0xc0;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 14;
test.compute_info.dispatch_thread_dimensions = true;
test.compute_info.dispatch_threads_num[0] = 2;
test.compute_info.dispatch_threads_num[1] = test.compute_info.dispatch_threads_num[2] = 1;
test.forbidden_spirv = {"get_bda_pointer", "flattened_srt", "OpFDiv", "OpUMulExtended"};
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
Require(test.name, "runtime record count",
compiled.program.info.buffers.size() == 3 &&
compiled.resources.buffers[2].dwords[2] == 2 &&
compiled.resources.specialization_reads ==
std::vector<std::pair<uint64_t, uint64_t>>{{count_address, 4}},
"descriptor capacity lost its exact clean input read");
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Compute;
options.wave_size = 64;
options.user_data = test.user_data;
options.input_info.compute = &test.compute_info;
auto translated = ShaderRecompiler::TranslateProgram(test.code, options);
auto plan = ExtractResourcePlan(translated.program);
struct CountMemory { u32 value = 72; u32 reads = 0; bool clean = true; } memory;
SrtRuntime runtime{.user_data = test.user_data, .userdata = &memory};
ResourceSnapshot snapshot;
ResourceSpecialization specialization;
Require(test.name, "strict reader required",
!MaterializeResources(plan, runtime, snapshot, specialization),
"vector descriptor input bypassed the clean-memory reader");
runtime.read_specialization_memory = [](void *data, uint64_t address, std::span<u32> words) {
auto &memory = *static_cast<CountMemory *>(data);
++memory.reads;
if (!memory.clean || address != count_address || words.size() != 1) return false;
words[0] = memory.value;
return true;
};
memory.clean = false;
Require(test.name, "dirty input rejected",
!MaterializeResources(plan, runtime, snapshot, specialization),
"GPU-dirty count was accepted for host descriptor evaluation");
memory.clean = true;
ResourceSpecialization initial_specialization;
// Reuse the same resource plan and compiled shader as the count and coverage change.
for (const u32 count : {0u, 72u, 512u, 8192u, 262144u}) {
memory.value = count;
memory.reads = 0;
const u32 chunk = count < 512 ? 64 : count < 8192 ? 128 : count < 262144 ? 256 : 512;
const u32 records = (count + chunk - 1) / chunk;
Require(test.name, "cached descriptor refresh",
MaterializeResources(plan, runtime, snapshot, specialization) && memory.reads == 1 &&
snapshot.buffers[2].dwords[2] == records &&
snapshot.specialization_reads ==
std::vector<std::pair<uint64_t, uint64_t>>{{count_address, 4}},
"count refresh repeated a memory read or retained an old descriptor capacity");
if (count == 0) initial_specialization = specialization;
Require(test.name, "capacity is not specialization",
specialization == initial_specialization,
"changing record capacity would recompile the cached shader");
test.initial.assign(5 + 5 * records, 0xdeadbeefu);
for (const u32 threads : {2u, 3u, 65u}) {
test.initial[1] = count;
test.compute_info.dispatch_threads_num[0] = threads;
test.dispatch_x = (threads + 63) / 64;
test.expected = test.initial;
test.expected[4] = 0;
for (u32 index = 0; index < std::min(threads, records); ++index) {
test.expected[9 + index * 5] = 0;
}
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output);
const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
vulkan.DestroyBuffer(&output);
CompareWords(test, "record capacity and padded invocation bounds", test.expected, actual);
}
}
std::printf("[compute] %-32s ok\n", test.name);
}
void CheckIndirectBufferStore(VulkanHarness &vulkan) {
using namespace ShaderRecompiler::IR;
TestCase test;
test.name = "IndirectBufferStore";
test.initial.assign(64, 0xdeadbeefu);
test.storage_buffer_range_bytes = 64;
test.storage_buffer_offsets = {12, 108};
test.required_spirv = {"OpSwitch"};
test.forbidden_spirv = {"PhysicalStorageBuffer"};
CompiledShader compiled;
auto &program = compiled.program;
program.stage = ShaderType::Compute;
program.wave_size = 32;
program.srt_plan_complete = true;
program.resource_tracking_complete = true;
program.shader_info_complete = true;
program.block_storage.push_back(std::make_unique<Block>());
auto *block = program.block_storage.back().get();
program.blocks.push_back(block);
auto &lane = block->AppendNewInst(ValueOpcode::LaneId);
auto &selector = block->AppendNewInst(ValueOpcode::BitwiseAnd32,
{Value(&lane), Value(7u)});
auto &key = block->AppendNewInst(ValueOpcode::IAdd32,
{Value(&selector), Value(8u)});
auto &handle = block->AppendNewInst(
ValueOpcode::GetBufferResource,
{Value(&key), Value(0u), Value(0u), Value(0u)});
handle.SetFlags<uint32_t>(0u);
auto &offset = block->AppendNewInst(ValueOpcode::IMul32,
{Value(&lane), Value(4u)});
auto &data = block->AppendNewInst(ValueOpcode::IAdd32,
{Value(&lane), Value(0x1000u)});
auto &exec = block->AppendNewInst(ValueOpcode::INotEqual32,
{Value(&lane), Value(4u)});
block->AppendNewInst(ValueOpcode::StoreBufferU32,
{Value(&handle), Value(0u), Value(&offset), Value(0u),
Value(&data), Value(&exec)});
program.memory_info.push_back({.kind = ResourceKind::Buffer, .offen = true});
BufferResource root;
root.packed_stride = 1;
root.written = true;
root.indirect_root = 0;
root.indirect_mapping_offset = 4;
root.indirect_search_iterations = std::bit_width(3u);
root.indirect_resources = {0u, 1u};
auto candidate = root;
candidate.indirect_search_iterations = 0;
candidate.indirect_resources.clear();
program.descriptor_sources.resize(1);
program.descriptor_sources[0].indirect_descriptor =
DescriptorSource::IndirectDescriptor{.table_stride = 16};
ShaderComputeInputInfo compute;
compute.wave_size = 32;
compute.host_subgroup_size = vulkan.SubgroupSize();
compute.threads_num[0] = 32;
compute.threads_num[1] = compute.threads_num[2] = 1;
for (const u32 null_buffers : {0u, 1u, 2u}) {
program.info.buffers = {root, candidate};
if (null_buffers != 0u) program.info.buffers[1].packed_stride = 0;
if (null_buffers == 2u) {
// Materialization deduplicates all null destinations into one unmapped root.
program.info.buffers.resize(1);
program.info.buffers[0].packed_stride = 0;
program.info.buffers[0].indirect_root = BufferResource::NoIndirectBuffer;
program.info.buffers[0].indirect_search_iterations = 0;
program.info.buffers[0].indirect_resources.clear();
}
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
Require(test.name, "candidate bindings",
program.bindings.descriptor_counts[0] == (null_buffers == 2u ? 1u : 2u) &&
program.info.buffers[0].descriptor_index == 0u &&
(null_buffers == 2u || program.info.buffers[1].descriptor_index == 1u),
"indirect store lost a buffer binding");
compiled.packed_user_data.resize(program.bindings.ShaderDataDwords());
compiled.packed_user_data[program.bindings.memory_offset_dword] = 12u | (108u << 8u);
ValidateSpirv(test.name, compiled.spirv);
if (null_buffers == 2u) {
test.required_spirv.clear();
test.forbidden_spirv.push_back("OpSwitch");
}
CheckSpirvText(test, compiled.spirv);
for (const bool swapped : {false, true}) {
if (null_buffers != 0u && swapped) continue;
// Repeated keys select two distinct logical buffers within one native allocation.
// Keys below, between and above the live keys must not select either buffer.
compiled.resources.flattened_srt = {0u, 0u, 0u, 0u, 3u,
10u, swapped ? 0u : 1u,
12u, swapped ? 1u : 0u,
14u, swapped ? 0u : 1u};
test.expected = test.initial;
for (const u32 lane : {2u, 6u, 10u, 12u, 14u}) {
const u32 resource = (lane & 7u) == 4u ? swapped : !swapped;
if (null_buffers == 2u || (null_buffers == 1u && resource == 1u)) continue;
test.expected[test.storage_buffer_offsets[resource] / 4u + lane] = 0x1000u + lane;
}
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output);
const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
vulkan.DestroyBuffer(&output);
CompareWords(test, "key selection, EXEC, bounds and null descriptors", test.expected, actual);
}
}
std::printf("[compute] %-32s ok\n", test.name);
}
void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) {
constexpr const char *name = "IndirectImageKeySwitch";
constexpr uint32_t mapping_capacity = 1793u;
using namespace ShaderRecompiler::IR;
Program program{};
program.stage = ShaderType::Compute;
program.wave_size = 32;
program.srt_plan_complete = true;
program.resource_tracking_complete = true;
program.shader_info_complete = true;
program.block_storage.push_back(std::make_unique<Block>());
auto *block = program.block_storage.back().get();
program.blocks.push_back(block);
auto &key = block->AppendNewInst(ValueOpcode::LaneId);
auto &image =
block->AppendNewInst(ValueOpcode::GetImageResource,
{Value(&key), Value(0u), Value(0u), Value(0u),
Value(0u), Value(0u), Value(0u), Value(0u)});
image.SetFlags<uint32_t>(0u);
auto &sampler =
block->AppendNewInst(ValueOpcode::GetSamplerResource,
{Value(0u), Value(0u), Value(0u), Value(0u)});
sampler.SetFlags<uint32_t>(0u);
auto &address = block->AppendNewInst(
ValueOpcode::MakeImageAddress,
{Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
Value(0u)});
MemoryInfo memory{};
memory.kind = ResourceKind::Image;
memory.resource = 0;
memory.sampler = 0;
memory.dmask = 0xf;
memory.image_sample_flags = ShaderRecompiler::Decoder::ImageSampleFlagLod;
memory.image_dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
memory.image_address_components = 3;
program.memory_info.push_back(memory);
const MemoryFlags memory_flags{0u, 0x10f0u};
uint64_t memory_flag_bits = 0;
std::memcpy(&memory_flag_bits, &memory_flags, sizeof(memory_flags));
auto &sample = block->AppendNewInst(
ValueOpcode::ImageSampleRaw,
{Value(&image), Value(&sampler), Value(&address)}, memory_flag_bits);
auto &sample_x = block->AppendNewInst(ValueOpcode::CompositeExtractU32x4,
{Value(&sample), Value(0u)});
block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(&sample_x)});
program.descriptor_sources.resize(2);
program.descriptor_sources[0].dword_count = 8;
program.descriptor_sources[0].indirect_descriptor =
DescriptorSource::IndirectDescriptor{
.selector = DescriptorSource::IndirectDescriptor::SelectorRead{
.source = 0, .stride = 224, .offset = 12},
.table_source = 0, .table_stride = 32};
program.descriptor_sources[1].dword_count = 4;
ImageResource root{};
root.source = 0;
root.first_use_pc = 0x10f0u;
root.resource_class = ImageResourceClass::Sampled;
root.numeric_class = Prospero::TextureNumericClass::Float;
root.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
root.read = true;
root.indirect_root = 0;
root.indirect_mapping_offset = 0;
root.indirect_search_iterations = std::bit_width(mapping_capacity);
root.indirect_resources = {0u, 1u};
auto candidate = root;
candidate.indirect_search_iterations = 0;
candidate.indirect_resources.clear();
program.info.images = {root, candidate};
program.info.samplers.push_back({1u, 0x10f0u});
program.info.sampled_pairs.push_back({0u, 0u, 0x10f0u});
ShaderComputeInputInfo compute{};
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
auto spirv = ShaderRecompiler::Spirv::EmitProgram(program,
{.compute = &compute});
ValidateSpirv(name, spirv);
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string text;
Require(name, "SPIR-V disassembly", tools.Disassemble(spirv, &text),
"failed to disassemble indirect image shader");
Require(name, "homogeneous image array",
text.find("OpSwitch") == std::string::npos &&
CountText(text, "OpImageSampleExplicitLod") == 1 &&
text.find("SampledImageArrayNonUniformIndexing") != std::string::npos,
"homogeneous image keys did not select one descriptor array sample");
u32 sampled_image = 0;
std::vector<u32> nonuniform;
for (size_t offset = 5; offset < spirv.size();) {
const auto words = std::span<const u32>(spirv).subspan(offset, spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform)
nonuniform.push_back(words[1]);
if (opcode == spv::OpImageSampleExplicitLod) sampled_image = words[3];
offset += words.size();
}
Require(name, "nonuniform sampled image operand",
sampled_image != 0u && std::ranges::find(nonuniform, sampled_image) != nonuniform.end(),
"the sample operand lacks its nonuniform decoration");
program.info.images[0].indirect_search_iterations = 0;
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
ValidateSpirv("DenseIndirectImageTable", spirv);
Require(name, "dense image map disassembly", tools.Disassemble(spirv, &text),
"failed to disassemble dense image table shader");
Require(name, "bounded dense image selection",
CountText(text, "OpULessThan") == 1 &&
CountText(text, "OpImageSampleExplicitLod") == 1 &&
text.find("SampledImageArrayNonUniformIndexing") != std::string::npos,
"dense image selection lost its bounds check or native nonuniform array");
// Materialized children follow all native roots, including unrelated images.
root.indirect_resources = {0u};
program.info.images = {root, candidate};
program.info.images[1].indirect_root = ImageResource::NoIndirectImage;
for (u32 ordinal = 1; ordinal < 189u; ++ordinal) {
root.indirect_resources.push_back(static_cast<u32>(program.info.images.size()));
program.info.images.push_back(candidate);
}
program.info.images[0] = root;
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
ValidateSpirv(name, spirv);
Require(name, "large image array disassembly", tools.Disassemble(spirv, &text),
"failed to disassemble the materialized image array");
Require(name, "large homogeneous image array",
text.find("OpSwitch") == std::string::npos &&
CountText(text, "OpImageSampleExplicitLod") == 1,
"materialized images expanded into per-candidate samples");
program.info.images[root.indirect_resources[103]].dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
program.info.images[root.indirect_resources[103]].cube = true;
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
ValidateSpirv(name, spirv);
Require(name, "mixed contiguous image runs", tools.Disassemble(spirv, &text) &&
CountText(text, "OpImageSampleExplicitLod") == 3,
"one cube split the native arrays into per-image samples");
program.memory_info[0].image_dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
program.memory_info[0].image_address_components = 4;
for (u32 component = 0; component < 4u; component++) {
constexpr std::array values{1.375f, 1.625f, 10.0f, 2.0f};
address.SetArg(component, Value(std::bit_cast<u32>(values[component])));
}
root.indirect_resources = {0u, 1u, 2u};
program.info.images = {root, candidate, candidate};
for (const bool cube_first : {true, false}) {
for (u32 resource = 0; resource < 3u; resource++) {
auto &image_resource = program.info.images[resource];
image_resource.cube = resource == (cube_first ? 0u : 1u);
image_resource.dimension =
resource == (cube_first ? 1u : 0u)
? ShaderRecompiler::Decoder::ImageDimension::Dim2D
: ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
}
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
ValidateSpirv(name, spirv);
Require(name, "mixed SPIR-V disassembly", tools.Disassemble(spirv, &text),
"failed to disassemble mixed indirect image shader");
Require(name, "mixed key switch",
CountText(text, "OpImageSampleExplicitLod") == 3 &&
CountText(text, "OpFSub") == 2,
"mixed candidates lost a sample or shared cube coordinate conversion");
std::vector<std::span<const u32>> definitions(spirv[3]);
u32 samples = 0;
for (size_t offset = 5; offset < spirv.size();) {
const auto words = std::span<const u32>(spirv).subspan(offset, spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpTypeVector) {
definitions[words[1]] = words;
} else if (opcode == spv::OpCompositeConstruct || opcode == spv::OpBitcast ||
opcode == spv::OpConstant) {
definitions[words[2]] = words;
} else if (opcode == spv::OpImageSampleExplicitLod) {
const auto coord = definitions[words[4]];
const auto lod = definitions[words[6]];
const auto components = samples == (cube_first ? 1u : 0u) ? 2u : 3u;
Require(name, "mixed coordinate and LOD layout",
!coord.empty() && definitions[coord[1]][3] == components &&
words[5] == spv::ImageOperandsLodMask && lod.size() == 4u &&
definitions[lod[3]].size() == 4u &&
definitions[lod[3]][3] == std::bit_cast<u32>(2.0f),
"candidate coordinates changed the instruction's LOD operand");
samples++;
}
offset += words.size();
}
Require(name, "mixed sample count", samples == 3u,
"mixed candidate switch did not retain every image");
}
// Keep an unrelated native root between the null image and appended children,
// then select both sides of a cube boundary and the unmapped suffix on the GPU.
root.indirect_resources = {0u, 2u, 3u, 4u, 5u, 6u};
root.indirect_search_iterations = 0;
program.info.images.assign(7, candidate);
program.info.images[0] = root;
program.info.images[1].indirect_root = ImageResource::NoIndirectImage;
program.info.images[4].dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
program.info.images[4].cube = true;
for (u32 component = 0; component < 4u; ++component) {
constexpr std::array values{1.5f, 1.5f, 0.0f, 0.0f};
address.SetArg(component, Value(std::bit_cast<u32>(values[component])));
}
auto &byte_offset = *block->PrependNewInst(
std::next(block->begin()), ValueOpcode::IMul32, {Value(&key), Value(4u)});
image.SetArg(0, Value(&byte_offset));
auto &output = block->AppendNewInst(ValueOpcode::GetBufferResource,
{Value(0u), Value(0u), Value(0u), Value(0u)});
output.SetFlags<u32>(0u);
program.memory_info.push_back({.kind = ResourceKind::Buffer, .offen = true});
auto &store = block->AppendNewInst(
ValueOpcode::StoreBufferU32,
{Value(&output), Value(0u), Value(&byte_offset), Value(0u), Value(&sample_x), Value(true)});
store.SetFlags(MemoryFlags{1u, 0u});
program.info.buffers.push_back({.packed_stride = 1, .written = true});
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
TestCase test;
test.name = "IndirectImageContiguousRuns";
CompiledShader compiled;
compiled.program = std::move(program);
constexpr std::array ordinals{0u, 1u, 2u, 3u, 4u, 5u, 2u, 0u};
compiled.resources.flattened_srt.push_back(ordinals.size());
compiled.resources.flattened_srt.insert(compiled.resources.flattened_srt.end(),
ordinals.begin(), ordinals.end());
std::vector<VulkanHarness::Image> textures;
for (u32 resource = 0; resource < 7u; ++resource) {
const auto layers = resource == 4u ? 6u : 1u;
std::vector<u32> pixels(layers * 4u, std::bit_cast<u32>(float(resource)));
textures.push_back(vulkan.CreateImageMips(
test.name, 1, 1, vk::Format::eR32G32B32A32Sfloat,
vk::ImageUsageFlagBits::eSampled, {pixels}, 4u,
vk::ImageLayout::eShaderReadOnlyOptimal, vk::ImageType::e2D,
resource == 4u ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D, layers));
}
const auto native_sampler = vulkan.CreateSampler(test.name);
for (const u32 wave_size : {32u, 64u}) {
compute.wave_size = wave_size;
compute.host_subgroup_size = vulkan.SubgroupSize();
compute.threads_num[0] = wave_size;
compute.threads_num[1] = compute.threads_num[2] = 1u;
compiled.program.wave_size = wave_size;
compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(compiled.program, {.compute = &compute});
compiled.packed_user_data.resize(compiled.program.bindings.ShaderDataDwords());
ValidateSpirv(test.name, compiled.spirv);
const auto halves = wave_size > compute.host_subgroup_size ? 2u : 1u;
Require(test.name, "native array run samples", tools.Disassemble(compiled.spirv, &text) &&
CountText(text, "OpImageSampleExplicitLod") == 3u * halves,
"mixed runs expanded into per-image samples");
test.initial.assign(wave_size, 0xdeadbeefu);
test.expected.assign(wave_size, 0u);
for (u32 lane = 0; lane < ordinals.size(); ++lane) {
test.expected[lane] = std::bit_cast<u32>(float(root.indirect_resources[ordinals[lane]]));
}
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr,
native_sampler, textures);
const auto actual = vulkan.ReadBuffer(test.name, buffer, test.expected.size());
vulkan.DestroyBuffer(&buffer);
CompareWords(test, "root, native slots, cube and mapping bounds", test.expected, actual);
}
vulkan.Device().destroySampler(native_sampler);
for (auto &texture : textures) vulkan.DestroyImage(&texture);
std::printf("[compute] %-32s ok\n", test.name);
}
TestCase ImageStoreMipSelectsPpsa01340Descriptor() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 2);
AppendVMovU32(&code, 23, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x40000000u);
AppendVMovLiteral(&code, 2, 0x40400000u);
AppendVMovLiteral(&code, 3, 0x40800000u);
code.push_back(EncodeMimg0(0x09, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
TestCase test;
test.name = "ImageStoreMipSelectsPpsa01340Descriptor";
test.code = std::move(code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE_MIP, O::S_ENDPGM};
const std::array<u32, 8> descriptor{0x0294dc00u, 0xc4700000u, 0x00b3c13fu,
0x91b31facu, 0x00000000u, 0x00700030u,
0xb07b0000u, 0x0002ac3cu};
std::copy(descriptor.begin(), descriptor.end(), test.user_data.begin());
test.has_user_data = true;
test.compile_only = true;
test.expected_mip_descriptors = 3;
test.required_spirv = {"OpSwitch", "OpImageWrite"};
return test;
}
template <u32 mode> TestCase ImageStoreSingleMipBounds() {
using O = ShaderOpcode;
constexpr std::array names{"ImageStoreSingleMipValid", "ImageStoreSingleMipBeyondEnd",
"ImageStoreSingleMipNegative"};
constexpr std::array lods{0u, 1u, UINT32_MAX};
TestCase test;
test.name = names[mode];
AppendVMovU32(&test.code, 20, 0);
AppendVMovU32(&test.code, 21, 0);
AppendVMovU32(&test.code, 22, lods[mode]);
for (u32 component = 0; component < 4; component++) {
AppendVMovLiteral(&test.code, component, 0x40000000u);
}
test.code.push_back(EncodeMimg0(0x09, 0xf));
test.code.push_back(EncodeMimg1(0, 20));
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE_MIP, O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.has_user_data = true;
test.storage_image_rgba = std::vector<u32>(4 * 4 * 4, 0x3f800000u);
test.expected_storage_image_rgba = test.storage_image_rgba;
if constexpr (mode == 0) {
std::fill_n(test.expected_storage_image_rgba.begin(), 4, 0x40000000u);
}
test.expected_mip_descriptors = 1;
if constexpr (mode == 0) {
test.required_spirv = {"OpImageWrite"};
}
return test;
}
TestCase ImageStoreRgbOneUsesInverseSwizzle() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 2);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x3f008081u);
AppendVMovLiteral(&code, 2, 0x3e808081u);
AppendVMovLiteral(&code, 3, 0x3f40c0c1u);
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
TestCase test;
test.name = "ImageStoreRgbOneUsesInverseSwizzle";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.has_user_data = true;
test.user_data[0] = 0x1000u;
test.user_data[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u;
test.image_descriptor_swizzle = DstSel(4, 5, 6, 1);
test.storage_image_format = vk::Format::eR8G8B8A8Unorm;
test.storage_image_dwords_per_pixel = 1;
test.storage_image_rgba = std::vector<u32>(16, 0);
test.expected_storage_image_rgba = std::vector<u32>(16, 0);
test.expected_storage_image_rgba[2 * 4 + 1] = 0x004080ffu;
return test;
}
TestCase ImageStoreDuplicateSelectorUsesInverseSwizzle() {
auto test = ImageStoreRgbOneUsesInverseSwizzle();
test.name = "ImageStoreDuplicateSelectorUsesInverseSwizzle";
test.image_descriptor_swizzle = DstSel(4, 4, 6, 7);
test.expected_storage_image_rgba[2 * 4 + 1] = 0xc04000ffu;
return test;
}
TestCase ImageStoreBgraUsesInverseSwizzle() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 2);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x3f008081u); // 128/255
AppendVMovLiteral(&code, 2, 0x3e808081u); // 64/255
AppendVMovLiteral(&code, 3, 0x3f3fbfc0u); // 191/255
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
TestCase test;
test.name = "ImageStoreBgraUsesInverseSwizzle";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.has_user_data = true;
test.user_data[0] = 0x1000u;
test.user_data[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u;
test.image_descriptor_swizzle = DstSel(6, 5, 4, 7);
test.storage_image_format = vk::Format::eR8G8B8A8Unorm;
test.storage_image_dwords_per_pixel = 1;
test.storage_image_rgba = std::vector<u32>(16, 0);
test.expected_storage_image_rgba = std::vector<u32>(16, 0);
test.expected_storage_image_rgba[2 * 4 + 1] = 0xbfff8040u;
return test;
}
TestCase ImageStoreYzwxUsesInverseSwizzle() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 2);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x3f000000u);
AppendVMovLiteral(&code, 2, 0x3e800000u);
AppendVMovLiteral(&code, 3, 0x3f400000u);
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
TestCase test;
test.name = "ImageStoreYzwxUsesInverseSwizzle";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.user_data =
MakeStorageTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(5, 6, 7, 4);
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.expected_storage_image_rgba = MakeRgbaImage(4, 4);
SetRgbaPixel(&test.expected_storage_image_rgba, 4, 1, 2, 0x3f400000u,
0x3f800000u, 0x3f000000u, 0x3e800000u);
return test;
}
TestCase ImageStoreR32FloatUsesFormatlessStorageImage() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 0x3f800000u;
TestCase test;
test.name = "ImageStoreR32FloatUsesFormatlessStorageImage";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.storage_image_format = vk::Format::eR32Sfloat;
test.storage_image_dwords_per_pixel = 1;
test.storage_image_rgba = std::vector<u32>(16, 0);
test.expected_storage_image_rgba = expected_image;
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat"};
test.forbidden_spirv = {"OpCapability StorageImageReadWithoutFormat",
"Rgba32f", "OpTypeSampler", "OpTypeSampledImage"};
return test;
}
TestCase ImageStoreR32SintUsesRawUintView() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x80000001u);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 0x80000001u;
TestCase test;
test.name = "ImageStoreR32SintUsesRawUintView";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32SInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = expected_image;
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false)};
test.forbidden_spirv = {"R32ui"};
return test;
}
TestCase ImageStoreR32UintUsesFormatlessStorageImage() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x12345678u);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 0x12345678u;
TestCase test;
test.name = "ImageStoreR32UintUsesFormatlessStorageImage";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = expected_image;
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false)};
test.forbidden_spirv = {"R32ui"};
return test;
}
TestCase ImageStorePackedUintSaturatesChannels() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 0x800u);
AppendVMovLiteral(&code, 1, UINT32_MAX);
AppendVMovU32(&code, 2, 0x400u);
AppendVMovU32(&code, 3, 0x55u);
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = UINT32_MAX;
TestCase test;
test.name = "ImageStorePackedUintSaturatesChannels";
test.code = std::move(code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.user_data =
MakeStorageTextureData(Prospero::BufferFormat::k11_11_10UInt);
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(4, 5, 6, 0);
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = std::move(expected_image);
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false), "OpULessThan",
"OpSelect"};
test.forbidden_spirv = {"R32ui"};
return test;
}
TestCase ImageStorePackedUintHonorsSparseDmask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 0x321u);
AppendVMovU32(&code, 1, 0x2abu);
code.push_back(EncodeMimg0(0x08, 0x5));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 0xaac00321u;
TestCase test;
test.name = "ImageStorePackedUintHonorsSparseDmask";
test.code = std::move(code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.user_data =
MakeStorageTextureData(Prospero::BufferFormat::k11_11_10UInt);
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(4, 5, 6, 0);
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = std::move(expected_image);
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false)};
test.forbidden_spirv = {"R32ui"};
return test;
}
TestCase ComputeTgSizeSgprUsesWaveMetadata() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendStoreSgpr(&code, 2, 0);
AppendEnd(&code);
TestCase test;
test.name = "ComputeTgSizeSgprUsesWaveMetadata";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 4;
test.compute_info.threads_num[2] = 16;
test.compute_info.group_id[0] = true;
test.compute_info.group_id[1] = true;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 3;
test.compute_info.workgroup_register = 0;
test.compute_info.tg_size_en = true;
test.has_compute_info = true;
test.compile_only = true;
test.required_spirv = {"OpUDiv", "OpShiftLeftLogical", "2147483648"};
return test;
}
TestCase ImageStoreAndAtomicShareTypedBinding() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 5);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendVMovU32(&code, 0, 3);
code.push_back(EncodeMimg0(0x11, 0x1, 0, true));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 8;
TestCase test;
test.name = "ImageStoreAndAtomicShareTypedBinding";
test.code = std::move(code);
test.expected = {5};
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_ATOMIC_ADD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = std::move(expected_image);
test.required_spirv = {"OpImageWrite", "OpImageTexelPointer", "R32ui",
StorageUint2DImageBindingName(true)};
test.forbidden_spirv = {"StorageImageReadWithoutFormat",
"StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false)};
return test;
}
// RDNA 2 MIMG opcode 0x10: replacement at VDATA, comparison at VDATA+1.
TestCase ImageAtomicCompareSwapGlcAndExec() {
TestCase test;
test.name = "ImageAtomicCompareSwapGlcAndExec";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve EXEC.
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
const auto exchange = [&](u32 replacement, u32 comparison, bool glc, u32 output) {
AppendVMovLiteral(&code, 0, replacement);
AppendVMovLiteral(&code, 1, comparison);
code.push_back(EncodeMimg0(0x10, 3, 0, glc));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, output);
AppendStoreVgpr(&code, 1, output + 1);
};
exchange(0x12345678u, 0xdeadbeefu, true, 0); // Match; return original bits.
exchange(0x87654321u, 0xdeadbeefu, true, 2); // Mismatch; preserve texel.
exchange(0xabcdef01u, 0x12345678u, false, 4); // GLC=0 preserves VDATA.
AppendVMovLiteral(&code, 0, 0x11111111u);
AppendVMovLiteral(&code, 1, 0xabcdef01u);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); // Disable EXEC.
code.push_back(EncodeMimg0(0x10, 3, 0, true));
code.push_back(EncodeMimg1(0, 20));
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 0, 6);
AppendStoreVgpr(&code, 1, 7);
AppendEnd(&code);
test.expected = {0xdeadbeefu, 0xdeadbeefu, 0x12345678u, 0xdeadbeefu,
0xabcdef01u, 0x12345678u, 0x11111111u, 0xabcdef01u};
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
ShaderOpcode::IMAGE_ATOMIC_CMPSWAP, ShaderOpcode::BUFFER_STORE_DWORD,
ShaderOpcode::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.storage_image_r32ui[6] = 0xdeadbeefu;
test.expected_storage_image_r32ui = test.storage_image_r32ui;
test.expected_storage_image_r32ui[6] = 0xabcdef01u;
// Device scope, AcquireRelease | ImageMemory (0x808).
test.required_spirv = {"OpAtomicCompareExchange", "OpImageTexelPointer",
"OpMemoryBarrier %uint_1 %uint_2056"};
return test;
}
TestCase ImageAtomicSwapReturnsPreviousTexel() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 5);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendVMovU32(&code, 0, 3);
code.push_back(EncodeMimg0(0x0f, 0x1, 0, true));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 3;
TestCase test;
test.name = "ImageAtomicSwapReturnsPreviousTexel";
test.code = std::move(code);
test.expected = {5};
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_ATOMIC_SWAP,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = std::move(expected_image);
test.required_spirv = {"OpAtomicExchange", "OpImageTexelPointer"};
return test;
}
TestCase ImageStoreAndAtomicUseSeparateBindings() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 5);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20, 0));
AppendVMovU32(&code, 0, 3);
code.push_back(EncodeMimg0(0x11, 0x1, 0, true));
code.push_back(EncodeMimg1(0, 20, 2));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
auto user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
std::copy_n(user_data.begin(), 8, user_data.begin() + 8);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 8;
TestCase test;
test.name = "ImageStoreAndAtomicUseSeparateBindings";
test.code = std::move(code);
test.expected = {5};
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_ATOMIC_ADD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = user_data;
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = std::move(expected_image);
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
"OpImageWrite",
"OpImageTexelPointer",
"R32ui",
StorageUint2DImageBindingName(false),
StorageUint2DImageBindingName(true)};
return test;
}
TestCase ImageAtomicVariants() {
using O = ShaderOpcode;
const u32 initial[] = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu};
const u32 values[] = {5, 5, 0x0ff0u, 0x0f00u, 0x00ffu};
const u32 ops[] = {0x11, 0x15, 0x18, 0x19, 0x1a};
std::vector<u32> code;
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovU32(&code, 20, i & 3u);
AppendVMovU32(&code, 21, i >> 2u);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, values[i]);
code.push_back(EncodeMimg0(ops[i], 0x1, 0, true));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, i);
}
AppendEnd(&code);
TestCase test;
test.name = "ImageAtomicVariants";
test.code = code;
test.expected = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu};
test.opcodes = {O::V_MOV_B32, O::IMAGE_ATOMIC_ADD,
O::IMAGE_ATOMIC_UMIN, O::IMAGE_ATOMIC_AND,
O::IMAGE_ATOMIC_OR, O::IMAGE_ATOMIC_XOR,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpImageTexelPointer", "R32ui",
StorageUint2DImageBindingName(true)};
test.forbidden_spirv = {"OpTypeSampler", "OpTypeSampledImage",
"StorageImageReadWithoutFormat",
"StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false)};
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.storage_image_r32ui = std::vector<u32>(16, 0);
for (u32 i = 0; i < static_cast<u32>(std::size(initial)); i++) {
test.storage_image_r32ui[i] = initial[i];
}
test.expected_storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui[0] = 15;
test.expected_storage_image_r32ui[1] = 5;
test.expected_storage_image_r32ui[2] = 0x00f0u;
test.expected_storage_image_r32ui[3] = 0xff00u;
test.expected_storage_image_r32ui[4] = 0xf0f0u;
return test;
}
// Input and old texel cover high-DWORD ordering, carry/wrap, the unsigned sign
// boundary, and equal-high-DWORD ordering through the RG32_UINT -> R64_UINT view.
constexpr uint64_t ImageAtomicInputs[][2] = {
{0x0000000100000001ull, 0x00000000ffffffffull},
{0x00000000ffffffffull, 0x0000000100000001ull},
{0x8000000000000000ull, 0x7fffffffffffffffull},
{0x12345678ffffffffull, 0x1234567800000001ull},
{0xffffffffffffffffull, 0x8000000000000000ull},
};
struct ImageAtomicIntegerCase {
u32 encoding;
ShaderOpcode opcode;
const char *name32;
const char *name64;
const char *spirv;
uint64_t expected64[std::size(ImageAtomicInputs)];
u32 expected32[std::size(ImageAtomicInputs)];
};
constexpr ImageAtomicIntegerCase ImageAtomicIntegerCases[] = {
{0x0fu, ShaderOpcode::IMAGE_ATOMIC_SWAP,
"ImageAtomicSwap32GlcAndExec", "ImageAtomicSwap64GlcAndExec", "OpAtomicExchange",
{0x0000000100000001ull, 0x00000000ffffffffull, 0x8000000000000000ull,
0x12345678ffffffffull, 0xffffffffffffffffull},
{0x00000001u, 0xffffffffu, 0x00000000u, 0xffffffffu, 0xffffffffu}},
{0x11u, ShaderOpcode::IMAGE_ATOMIC_ADD,
"ImageAtomicAdd32GlcAndExec", "ImageAtomicAdd64GlcAndExec", "OpAtomicIAdd",
{0x0000000200000000ull, 0x0000000200000000ull, 0xffffffffffffffffull,
0x2468acf100000000ull, 0x7fffffffffffffffull},
{0x00000000u, 0x00000000u, 0xffffffffu, 0x00000000u, 0xffffffffu}},
{0x15u, ShaderOpcode::IMAGE_ATOMIC_UMIN,
"ImageAtomicUMin32GlcAndExec", "ImageAtomicUMin64GlcAndExec", "OpAtomicUMin",
{0x00000000ffffffffull, 0x00000000ffffffffull, 0x7fffffffffffffffull,
0x1234567800000001ull, 0x8000000000000000ull},
{0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}},
{0x17u, ShaderOpcode::IMAGE_ATOMIC_UMAX,
"ImageAtomicUMax32GlcAndExec", "ImageAtomicUMax64GlcAndExec", "OpAtomicUMax",
{0x0000000100000001ull, 0x0000000100000001ull, 0x8000000000000000ull,
0x12345678ffffffffull, 0xffffffffffffffffull},
{0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}},
{0x18u, ShaderOpcode::IMAGE_ATOMIC_AND,
"ImageAtomicAnd32GlcAndExec", "ImageAtomicAnd64GlcAndExec", "OpAtomicAnd",
{0x0000000000000001ull, 0x0000000000000001ull, 0x0000000000000000ull,
0x1234567800000001ull, 0x8000000000000000ull},
{0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}},
{0x19u, ShaderOpcode::IMAGE_ATOMIC_OR,
"ImageAtomicOr32GlcAndExec", "ImageAtomicOr64GlcAndExec", "OpAtomicOr",
{0x00000001ffffffffull, 0x00000001ffffffffull, 0xffffffffffffffffull,
0x12345678ffffffffull, 0xffffffffffffffffull},
{0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}},
{0x1au, ShaderOpcode::IMAGE_ATOMIC_XOR,
"ImageAtomicXor32GlcAndExec", "ImageAtomicXor64GlcAndExec", "OpAtomicXor",
{0x00000001fffffffeull, 0x00000001fffffffeull, 0xffffffffffffffffull,
0x00000000fffffffeull, 0x7fffffffffffffffull},
{0xfffffffeu, 0xfffffffeu, 0xffffffffu, 0xfffffffeu, 0xffffffffu}},
};
TestCase ImageAtomicIntegerGlcAndExec(const ImageAtomicIntegerCase &atomic, bool wide) {
const u32 texel_dwords = wide ? 2 : 1;
TestCase test;
test.name = wide ? atomic.name64 : atomic.name32;
test.user_data = MakeStorageTextureData(wide ? Prospero::BufferFormat::k32_32UInt
: Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui.assign(16 * texel_dwords, 0);
test.expected_storage_image_r32ui = test.storage_image_r32ui;
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve EXEC.
for (u32 i = 0; i < std::size(ImageAtomicInputs); ++i) {
const auto input = ImageAtomicInputs[i][0];
const auto old = ImageAtomicInputs[i][1];
const bool glc = i != 0 && i != 4;
AppendVMovU32(&code, 0, i & 3u);
AppendVMovU32(&code, 1, i >> 2u);
AppendVMovLiteral(&code, 2, static_cast<u32>(input));
AppendVMovLiteral(&code, 3, static_cast<u32>(input >> 32u));
code.push_back(EncodeMimg0(atomic.encoding, wide ? 3u : 1u, 0, glc));
code.push_back(EncodeMimg1(2, 0));
AppendStoreVgpr(&code, 2, i * 2);
AppendStoreVgpr(&code, 3, i * 2 + 1);
test.expected.push_back(static_cast<u32>(glc ? old : input));
test.expected.push_back(static_cast<u32>((wide && glc ? old : input) >> 32u));
const uint64_t result = wide ? atomic.expected64[i] : atomic.expected32[i];
for (u32 word = 0; word < texel_dwords; ++word) {
test.storage_image_r32ui[i * texel_dwords + word] =
static_cast<u32>(old >> (word * 32u));
test.expected_storage_image_r32ui[i * texel_dwords + word] =
static_cast<u32>(result >> (word * 32u));
}
}
// A leaked update must change the inactive texel for every operation.
const bool clear_bits = atomic.opcode == ShaderOpcode::IMAGE_ATOMIC_AND ||
atomic.opcode == ShaderOpcode::IMAGE_ATOMIC_UMIN;
const u32 inactive = clear_bits ? 0xffffffffu : 0u;
for (u32 word = 0; word < texel_dwords; ++word) {
test.storage_image_r32ui[5u * texel_dwords + word] = inactive;
test.expected_storage_image_r32ui[5u * texel_dwords + word] = inactive;
}
AppendVMovU32(&code, 0, 1);
AppendVMovU32(&code, 1, 1);
AppendVMovLiteral(&code, 2, 0xabcdef01u);
AppendVMovLiteral(&code, 3, 0x87654321u);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeMimg0(atomic.encoding, wide ? 3u : 1u, 0, true));
code.push_back(EncodeMimg1(2, 0));
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 2, std::size(ImageAtomicInputs) * 2);
AppendStoreVgpr(&code, 3, std::size(ImageAtomicInputs) * 2 + 1);
test.expected.insert(test.expected.end(), {0xabcdef01u, 0x87654321u});
AppendEnd(&code);
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
atomic.opcode, ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
test.required_spirv = {atomic.spirv, "OpImageTexelPointer", wide ? "R64ui" : "R32ui"};
if (wide) {
test.required_spirv.push_back("Int64ImageEXT");
test.required_spirv.push_back("SPV_EXT_shader_image_int64");
}
return test;
}
template <bool max_value> TestCase ImageAtomicSignedMinMax() {
using O = ShaderOpcode;
// Input, old value, signed minimum, signed maximum.
constexpr u32 values[][4] = {
{0xfffffffbu, 7, 0xfffffffbu, 7},
{7, 0xfffffffbu, 0xfffffffbu, 7},
{0x80000000u, 0x7fffffffu, 0x80000000u, 0x7fffffffu},
{0x7fffffffu, 0x80000000u, 0x80000000u, 0x7fffffffu},
{0xfffffff6u, 0xfffffffbu, 0xfffffff6u, 0xfffffffbu},
{0xfffffffbu, 0xfffffff6u, 0xfffffff6u, 0xfffffffbu},
{2, 8, 2, 8}, {8, 2, 2, 8},
};
TestCase test;
test.name = max_value ? "ImageAtomicSMaxGlcAndExec" : "ImageAtomicSMinGlcAndExec";
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui.assign(16, 0);
test.expected_storage_image_r32ui = test.storage_image_r32ui;
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 12, 126));
for (u32 i = 0; i < std::size(values); ++i) {
AppendVMovU32(&code, 20, i & 3u);
AppendVMovU32(&code, 21, i >> 2u);
AppendVMovLiteral(&code, 2, values[i][0]);
const bool glc = (i & 1u) == 0;
code.push_back(EncodeMimg0(max_value ? 0x16 : 0x14, 1, 0, glc));
code.push_back(EncodeMimg1(2, 20));
AppendStoreVgpr(&code, 2, i);
test.expected.push_back(values[i][glc ? 1 : 0]);
test.storage_image_r32ui[i] = values[i][1];
test.expected_storage_image_r32ui[i] = values[i][max_value ? 3 : 2];
}
AppendVMovLiteral(&code, 2, max_value ? 0x7fffffffu : 0x80000000u);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeMimg0(max_value ? 0x16 : 0x14, 1, 0, true));
code.push_back(EncodeMimg1(2, 20));
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 2, std::size(values));
test.expected.push_back(max_value ? 0x7fffffffu : 0x80000000u);
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64,
max_value ? O::IMAGE_ATOMIC_SMAX : O::IMAGE_ATOMIC_SMIN,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {max_value ? "OpAtomicSMax" : "OpAtomicSMin", "R32ui"};
return test;
}
template <bool max_value> TestCase ImageAtomicFloatGlcAndExec() {
using O = ShaderOpcode;
std::vector<u32> code;
constexpr u32 atomic_word = max_value ? 0xf07c0108u : 0xf0780108u;
code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve active workgroup lanes.
AppendVMovU32(&code, 8, 2);
AppendVMovU32(&code, 9, 1);
AppendVMovLiteral(&code, 5, 0x40000000u); // 2.0
// FMAX uses the captured GLC=0 instruction word.
code.insert(code.end(), {atomic_word, 0x00010508u});
AppendStoreVgpr(&code, 5, 0);
AppendVMovLiteral(&code, 5, max_value ? 0x40800000u : 0x3f800000u);
code.insert(code.end(), {atomic_word | 0x2000u, 0x00010508u}); // GLC=1.
AppendStoreVgpr(&code, 5, 1);
AppendVMovLiteral(&code, 5,
max_value ? 0x41000000u : 0x3f000000u); // Inactive lane.
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.insert(code.end(), {atomic_word | 0x2000u, 0x00010508u});
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 5, 2);
AppendEnd(&code);
TestCase test;
test.name = max_value ? "ImageAtomicFMaxCapturedGlcAndExec"
: "ImageAtomicFMinGlcAndExec";
test.code = std::move(code);
test.expected = {0x40000000u, 0x40000000u,
max_value ? 0x41000000u : 0x3f000000u};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64,
max_value ? O::IMAGE_ATOMIC_FMAX : O::IMAGE_ATOMIC_FMIN,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
const std::array<u32, 8> descriptor = {
0x504a7c00u, 0xc1600000u, 0x0086c0efu, 0x91b80924u,
0x00000000u, 0x00700080u, 0x00000000u, 0x00000000u};
std::copy_n(descriptor.begin(), 8, test.user_data.begin() + 4);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.storage_image_r32ui[6] = max_value ? 0x3f800000u : 0x40800000u;
test.expected_storage_image_r32ui = test.storage_image_r32ui;
test.expected_storage_image_r32ui[6] = max_value ? 0x40800000u : 0x3f800000u;
test.required_spirv = {"OpImageTexelPointer", "OpAtomicCompareExchange", "R32ui"};
return test;
}
template <bool max_value> TestCase ImageAtomicFloatSpecialValues() {
using O = ShaderOpcode;
// source, previous texel, expected minimum, expected maximum.
const u32 cases[][4] = {
{0x40000000u, 0x3f800000u, 0x3f800000u, 0x40000000u},
{0xc0000000u, 0xbf800000u, 0xc0000000u, 0xbf800000u},
{0x7f800000u, 0x3f800000u, 0x3f800000u, 0x7f800000u},
{0xff800000u, 0x3f800000u, 0xff800000u, 0x3f800000u},
{0x3f800000u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u},
{0x7fcabcdeu, 0x3f800000u, 0x3f800000u, 0x3f800000u},
{0x3f800000u, 0x7fa54321u, 0x7fa54321u, 0x7fa54321u},
{0x7faabcdeu, 0x3f800000u, 0x3f800000u, 0x3f800000u},
{0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u},
{0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u},
{0x80000000u, 0x80000001u, 0x80000001u, 0x80000000u},
{0x80000001u, 0x80000000u, 0x80000001u, 0x80000000u},
{0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u},
{0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u},
};
TestCase test;
test.name = max_value ? "ImageAtomicFMaxSpecialValues"
: "ImageAtomicFMinSpecialValues";
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = test.storage_image_r32ui;
for (u32 i = 0; i < std::size(cases); i++) {
AppendVMovU32(&test.code, 20, i & 3u);
AppendVMovU32(&test.code, 21, i >> 2u);
AppendVMovLiteral(&test.code, 0, cases[i][0]);
test.code.push_back(EncodeMimg0(max_value ? 0x1f : 0x1e, 1, 0, true));
test.code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&test.code, 0, i);
test.expected.push_back(cases[i][1]);
test.storage_image_r32ui[i] = cases[i][1];
test.expected_storage_image_r32ui[i] = cases[i][max_value ? 3 : 2];
}
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32,
max_value ? O::IMAGE_ATOMIC_FMAX : O::IMAGE_ATOMIC_FMIN,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
template <bool max_value> TestCase ImageAtomicFloatContendedWorkgroup() {
using O = ShaderOpcode;
TestCase test;
test.name = max_value ? "ImageAtomicFMaxContendedWorkgroup"
: "ImageAtomicFMinContendedWorkgroup";
test.code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // Float thread_id.x.
AppendVMovU32(&test.code, 20, 0);
AppendVMovU32(&test.code, 21, 0);
test.code.push_back(EncodeMimg0(max_value ? 0x1f : 0x1e, 1));
test.code.push_back(EncodeMimg1(1, 20));
AppendEnd(&test.code);
test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32,
max_value ? O::IMAGE_ATOMIC_FMAX : O::IMAGE_ATOMIC_FMIN,
O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.storage_image_r32ui[0] =
max_value ? 0xc2c80000u : 0x42c80000u; // -/+100.0
test.expected_storage_image_r32ui = test.storage_image_r32ui;
test.expected_storage_image_r32ui[0] =
max_value ? 0x427c0000u : 0u; // 63.0 / 0.0
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase ImageAtomicGlc0DoesNotReturnOldValue() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 5);
code.push_back(EncodeMimg0(0x11, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageAtomicGlc0DoesNotReturnOldValue";
test.code = code;
test.expected = {5};
test.opcodes = {O::V_MOV_B32, O::IMAGE_ATOMIC_ADD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.storage_image_r32ui[0] = 10;
test.expected_storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui[0] = 15;
return test;
}
GraphicsCase GraphicsInterpolationExport() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVintrp(0x00, 0, 0, 0, 0));
code.push_back(EncodeVintrp(0x01, 0, 0, 0, 0));
code.push_back(EncodeVintrp(0x02, 1, 0, 1, 2));
AppendVMovLiteral(&code, 2, 0x3f400000u);
AppendVMovLiteral(&code, 3, 0x3f800000u);
code.push_back(EncodeExp0(0x00, 0xf));
code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&code);
return {"GraphicsInterpolationExport",
code,
{0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u},
{O::V_INTERP_P1_F32, O::V_INTERP_P2_F32, O::V_INTERP_MOV_F32,
O::V_MOV_B32, O::EXP, O::S_ENDPGM}};
}
GraphicsCase GraphicsPositionWExport() {
GraphicsCase test;
test.name = "GraphicsPositionWExport";
test.pixel_position_w = true;
test.vertex_clip_w = 4.0f;
// POS_W occupies v2; the rasterizer supplies FragCoord.w=1/4 for this triangle.
test.fragment_code = {EncodeExp0(0x00, 0xf), EncodeExp1(2, 2, 2, 2)};
AppendEnd(&test.fragment_code);
test.expected_pixel = {0x40800000u, 0x40800000u, 0x40800000u, 0x40800000u};
test.opcodes = {ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
return test;
}
GraphicsCase GraphicsPackedHalfCentroid() {
GraphicsCase test;
test.name = "GraphicsPackedHalfCentroid";
test.pixel_perspective_centroid_vgpr = 0;
test.pixel_custom_interpolation_mask = 1;
test.pixel_interpolator_settings = {0x420u};
// Captured ab810715011baef3 interpolation, with attr3.x remapped to attr0.x.
test.fragment_code = {
0xc8120002u, 0xc80e0000u, // raw vertex0/vertex1 packed halves
0xcc204007u, 0x9c1206f2u, // low(vertex1) - low(vertex0)
0xcc207005u, 0x9c1206f2u, // high(vertex1) - high(vertex0)
0xc80a0001u, // raw vertex2
0xcc204006u, 0x9c1204f2u,
0xcc207003u, 0x9c1204f2u,
0xcc20400eu, 0x04120f00u, // low = I * delta10 + vertex0
0xcc20600fu, 0x04120b00u, // high = I * delta10 + vertex0
0x3e1c0306u, 0x3e1e0303u, // += J * delta20
EncodeExp0(0x00, 0xf), EncodeExp1(14, 15, 14, 15)};
AppendEnd(&test.fragment_code);
// At the pixel center the vertex weights are (5/8, 1/8, 1/4).
// Packed values are (0,1), (1,2), (3,3): expected result (0.875, 1.625).
test.vertices = {
0xbf800000u, 0xbf800000u, 0x3c000000u, 0, 0, 0,
0x40e00000u, 0xbf800000u, 0x40003c00u, 0, 0, 0,
0xbf800000u, 0x40400000u, 0x42004200u, 0, 0, 0};
test.expected_pixel = {0x3f600000u, 0x3fd00000u, 0x3f600000u, 0x3fd00000u};
test.opcodes = {ShaderOpcode::V_INTERP_MOV_F32, ShaderOpcode::V_FMA_F32, ShaderOpcode::V_MAC_F32,
ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
return test;
}
GraphicsCase GraphicsPerspectiveSample(bool multisample) {
GraphicsCase test;
test.name = multisample ? "GraphicsPerspectiveSample4x"
: "GraphicsPerspectiveSample1x";
test.pixel_perspective_sample_vgpr = 0;
test.samples = multisample ? vk::SampleCountFlagBits::e4
: vk::SampleCountFlagBits::e1;
test.fragment_code = {
EncodeVop2(0x08, 2, Vgpr(0), 0), // I squared
EncodeVop2(0x08, 3, Vgpr(1), 1), // J squared
EncodeExp0(0x00, 0xf), EncodeExp1(2, 3, 0, 1)};
AppendEnd(&test.fragment_code);
// The triangle gives I=x/2, J=y/2. Squaring before the 4x resolve
// distinguishes sample interpolation (21/256) from pixel-center (1/16).
const auto square = std::bit_cast<u32>(multisample ? 21.0f / 256.0f : 1.0f / 16.0f);
test.expected_pixel = {square, square, 0x3e800000u, 0x3e800000u};
test.opcodes = {ShaderOpcode::V_MUL_F32, ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
return test;
}
GraphicsCase GraphicsPackedHalfInputAlias(bool second_weights) {
auto test = GraphicsPackedHalfCentroid();
test.name = second_weights ? "GraphicsPackedHalfAliasSecondWeights"
: "GraphicsPackedHalfInputAlias";
test.pixel_custom_interpolation_mask = 2u;
test.pixel_interpolator_settings = {0u, 0x420u};
// The head shader reads one exported vec4 through two logical inputs:
// ordinary interpolation and raw per-vertex packed-half interpolation.
// Move the captured raw attr0.x loads to logical attr1.x, still mapped to 0.
for (const auto word : {0u, 1u, 6u}) {
test.fragment_code[word] |= 1u << 10u;
}
test.fragment_code.resize(test.fragment_code.size() - 3u);
test.fragment_code.push_back(EncodeVintrp(0x00, 16, 0, 1, 0));
test.fragment_code.push_back(EncodeVintrp(0x01, 16, 0, 1, 1));
AppendVMovLiteral(&test.fragment_code, 17, 0x3f800000u);
test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
test.fragment_code.push_back(EncodeExp1(14, 15, 16, 17));
AppendEnd(&test.fragment_code);
// The smooth component (2,4,8) must vary independently of packed UVs.
test.vertices[3] = 0x40000000u;
test.vertices[9] = 0x40800000u;
test.vertices[15] = 0x41000000u;
if (second_weights) {
// Change the right vertex from x=7 to x=3: weights become (1/2,1/4,1/4).
test.vertices[6] = 0x40400000u;
test.expected_pixel = {0x3f800000u, 0x3fe00000u, 0x40800000u, 0x3f800000u};
} else {
test.expected_pixel = {0x3f600000u, 0x3fd00000u, 0x40700000u, 0x3f800000u};
}
test.opcodes.insert(test.opcodes.end(),
{ShaderOpcode::V_INTERP_P1_F32, ShaderOpcode::V_INTERP_P2_F32,
ShaderOpcode::V_MOV_B32});
return test;
}
GraphicsCase GraphicsSmoothRawInputAlias() {
auto test = GraphicsPackedHalfInputAlias(false);
test.name = "GraphicsSmoothRawInputAlias";
test.pixel_perspective_centroid_vgpr = UINT32_MAX;
// No explicit barycentric builtin: alias promotion must discover that the
// ordinary input needs weights while the raw alias returns vertex zero.
test.fragment_code = {EncodeVintrp(0x00, 0, 0, 1, 0),
EncodeVintrp(0x01, 0, 0, 1, 1),
EncodeVintrp(0x02, 1, 1, 1, 2),
EncodeExp0(0x00, 0xf), EncodeExp1(0, 1, 0, 1)};
AppendEnd(&test.fragment_code);
test.expected_pixel = {0x40700000u, 0x40000000u, 0x40700000u, 0x40000000u};
test.opcodes = {ShaderOpcode::V_INTERP_P1_F32, ShaderOpcode::V_INTERP_P2_F32,
ShaderOpcode::V_INTERP_MOV_F32, ShaderOpcode::EXP,
ShaderOpcode::S_ENDPGM};
return test;
}
GraphicsCase GraphicsSmoothFlatInputAlias(bool flat_first, bool provoking_last = false) {
auto test = GraphicsSmoothRawInputAlias();
test.name = provoking_last
? (flat_first ? "GraphicsFlatSmoothAliasLast" : "GraphicsSmoothFlatAliasLast")
: (flat_first ? "GraphicsFlatSmoothInputAlias" : "GraphicsSmoothFlatInputAlias");
test.pixel_custom_interpolation_mask = 0;
test.provoking_vtx_last = provoking_last;
test.pixel_interpolator_settings = flat_first ? std::vector<u32>{0x400u, 0u}
: std::vector<u32>{0u, 0x400u};
const u32 smooth = flat_first ? 1u : 0u;
const u32 flat = smooth ^ 1u;
// One export carries smooth values (2,4,8) and packed light IDs (5,7,11).
// Flat P0 preserves the ID bits; flat P1/P2 uses the same provoking vertex.
test.vertices[4] = 0x3f800005u;
test.vertices[10] = 0x3f800007u;
test.vertices[16] = 0x3f80000bu;
test.fragment_code = {EncodeVintrp(0x00, 0, smooth, 1, 0),
EncodeVintrp(0x01, 0, smooth, 1, 1),
EncodeVintrp(0x02, 1, flat, 2, 2),
EncodeVop2(0x1b, 1, InlineU32(63), 1),
EncodeVop1(0x06, 1, Vgpr(1)),
EncodeVintrp(0x00, 2, flat, 1, 0),
EncodeVintrp(0x01, 2, flat, 1, 1),
EncodeExp0(0x00, 0xf), EncodeExp1(0, 1, 2, 0)};
AppendEnd(&test.fragment_code);
test.expected_pixel = {0x40700000u, provoking_last ? 0x41300000u : 0x40a00000u,
provoking_last ? 0x41000000u : 0x40000000u, 0x40700000u};
test.opcodes.insert(test.opcodes.end(),
{ShaderOpcode::V_AND_B32, ShaderOpcode::V_CVT_F32_U32});
return test;
}
GraphicsCase GraphicsAncillaryLayer(bool front_face) {
GraphicsCase test;
test.name = front_face ? "GraphicsAncillaryAfterFrontFace" : "GraphicsAncillaryLayer";
test.pixel_ancillary = true;
test.pixel_front_face = front_face;
test.layers = 8;
const auto ancillary = Vgpr(front_face ? 3 : 2);
test.fragment_code.push_back(EncodeSop1(0x04, 8, 126));
test.fragment_code.push_back(EncodeSop1(0x0a, 126, 126));
AppendVop3(&test.fragment_code, 0x148, 4, ancillary, InlineU32(16), InlineU32(11));
AppendVop3(&test.fragment_code, 0x149, 5, ancillary, InlineU32(16), InlineU32(3));
test.fragment_code.push_back(EncodeSop1(0x04, 126, 8));
AppendVMovU32(&test.fragment_code, 6, test.layers);
test.fragment_code.push_back(EncodeVopc(0xc1, Vgpr(4), 6)); // Dynamic layer < count.
test.fragment_code.push_back(EncodeSop1(0x04, 126, 106));
test.fragment_code.push_back(EncodeVop1(0x06, 0, Vgpr(4))); // float(unsigned layer)
test.fragment_code.push_back(EncodeVop1(0x05, 1, Vgpr(5))); // float(signed low 3 bits)
AppendVMovLiteral(&test.fragment_code, 2, 0);
AppendVMovLiteral(&test.fragment_code, 3, 0x3f800000u);
test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
test.fragment_code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&test.fragment_code);
test.opcodes = {ShaderOpcode::V_BFE_U32, ShaderOpcode::V_BFE_I32,
ShaderOpcode::V_CVT_F32_U32, ShaderOpcode::V_CVT_F32_I32,
ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
ShaderOpcode::S_WQM_B64, ShaderOpcode::V_CMP_LT_U32,
ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
for (u32 layer = 0; layer < test.layers; ++layer) {
const auto signed_bits = layer < 4 ? static_cast<int>(layer) : static_cast<int>(layer) - 8;
test.expected_pixel.insert(test.expected_pixel.end(),
{std::bit_cast<u32>(static_cast<float>(layer)),
std::bit_cast<u32>(static_cast<float>(signed_bits)), 0, 0x3f800000u});
}
return test;
}
GraphicsCase GraphicsAncillarySampleId() {
GraphicsCase test;
test.name = "GraphicsAncillarySampleId";
test.pixel_ancillary = true;
test.samples = vk::SampleCountFlagBits::e4;
AppendVop3(&test.fragment_code, 0x148, 4, Vgpr(2), InlineU32(8), InlineU32(4));
AppendVop3(&test.fragment_code, 0x149, 5, Vgpr(2), InlineU32(8), InlineU32(2));
AppendVop3(&test.fragment_code, 0x148, 6, Vgpr(2), InlineU32(9), InlineU32(1));
test.fragment_code.push_back(EncodeVop1(0x06, 0, Vgpr(4)));
test.fragment_code.push_back(EncodeVop1(0x05, 1, Vgpr(5)));
test.fragment_code.push_back(EncodeVop1(0x06, 2, Vgpr(6)));
AppendVMovLiteral(&test.fragment_code, 3, 0x3f800000u);
test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
test.fragment_code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&test.fragment_code);
test.opcodes = {ShaderOpcode::V_BFE_U32, ShaderOpcode::V_BFE_I32,
ShaderOpcode::V_CVT_F32_U32, ShaderOpcode::V_CVT_F32_I32,
ShaderOpcode::V_MOV_B32, ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
// Average samples 0..3, their signed low two bits, and their second bit.
test.expected_pixel = {0x3fc00000u, 0xbf000000u, 0x3f000000u, 0x3f800000u};
return test;
}
GraphicsCase GraphicsFlatInterpolatorExport() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVintrp(0x02, 0, 0, 0, 2));
AppendVMovLiteral(&code, 1, 0x00000000u);
AppendVMovLiteral(&code, 2, 0x00000000u);
AppendVMovLiteral(&code, 3, 0x3f800000u);
code.push_back(EncodeExp0(0x00, 0xf));
code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&code);
GraphicsCase test;
test.name = "GraphicsFlatInterpolatorExport";
test.fragment_code = code;
test.expected_pixel = {0x3e800000u, 0x00000000u, 0x00000000u, 0x3f800000u};
test.opcodes = {O::V_INTERP_MOV_F32, O::V_MOV_B32, O::EXP, O::S_ENDPGM};
test.pixel_interpolator_settings = {0x00000400u};
test.vertices = {
0xbf800000u, 0xbf800000u, 0x3e800000u, 0x00000000u, 0x00000000u,
0x3f800000u, 0x40400000u, 0xbf800000u, 0x3f400000u, 0x00000000u,
0x00000000u, 0x3f800000u, 0xbf800000u, 0x40400000u, 0x3e800000u,
0x00000000u, 0x00000000u, 0x3f800000u,
};
return test;
}
GraphicsCase GraphicsDsAddtidScratchExport() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(124, InlineU32(0)));
AppendVMovLiteral(&code, 0, 0x3f000000u);
code.push_back(EncodeDs0(0xb0, 0));
code.push_back(EncodeDs1(0, 0, 0));
code.push_back(EncodeDs0(0xb1, 0));
code.push_back(EncodeDs1(4, 0, 0));
code.push_back(EncodeExp0(0x00, 0xf));
code.push_back(EncodeExp1(4, 4, 4, 4));
AppendEnd(&code);
return {"GraphicsDsAddtidScratchExport",
code,
{0x3f000000u, 0x3f000000u, 0x3f000000u, 0x3f000000u},
{O::S_MOV_B32, O::V_MOV_B32, O::DS_WRITE_ADDTID_B32,
O::DS_READ_ADDTID_B32, O::EXP, O::S_ENDPGM}};
}
GraphicsCase GraphicsDirectSgprPushConstantExport() {
using O = ShaderOpcode;
const std::vector<u32> values = {0x3e800000u, 0x3f000000u, 0x3f400000u,
0x3f800000u};
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 0, 0));
code.push_back(EncodeVop1(0x01, 1, 1));
code.push_back(EncodeVop1(0x01, 2, 2));
code.push_back(EncodeVop1(0x01, 3, 3));
code.push_back(EncodeExp0(0x00, 0xf));
code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&code);
GraphicsCase test;
test.name = "GraphicsDirectSgprPushConstantExport";
test.fragment_code = code;
test.expected_pixel = values;
test.opcodes = {O::V_MOV_B32, O::EXP, O::S_ENDPGM};
for (size_t i = 0; i < values.size(); i++) {
test.user_data[i] = values[i];
}
test.has_user_data = true;
test.push_constants = values;
return test;
}
GraphicsCase GraphicsInlineSrtScalarPromotionExport() {
using O = ShaderOpcode;
const std::vector<u32> values = {0x00000000u, 0x00000000u, 0x00000000u,
0x3f800000u};
std::vector<u32> code;
AppendVop3(&code, 0x12fu, 0, 0, 1);
AppendVop3(&code, 0x12fu, 1, 2, 3);
code.push_back(EncodeExp0(0x00, 0xf, true, true, true));
code.push_back(EncodeExp1(0, 1, 0, 0));
AppendEnd(&code);
GraphicsCase test;
test.name = "GraphicsInlineSrtScalarPromotionExport";
test.fragment_code = code;
test.expected_pixel = values;
test.opcodes = {O::V_CVT_PKRTZ_F16_F32, O::EXP, O::S_ENDPGM};
for (size_t i = 0; i < values.size(); i++) {
test.user_data[i] = values[i];
}
test.has_user_data = true;
test.push_constants = values;
return test;
}
GraphicsCase GraphicsNullVmExportDiscardsInactiveExec() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f000000u);
AppendVMovLiteral(&code, 1, 0x3f400000u);
AppendVMovLiteral(&code, 2, 0x3f800000u);
AppendVMovLiteral(&code, 3, 0x40000000u);
code.push_back(EncodeExp0(0x00, 0xf, false));
code.push_back(EncodeExp1(0, 1, 2, 3));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeExp0(0x09, 0x0, true, false, true));
code.push_back(EncodeExp1(0, 0, 0, 0));
AppendEnd(&code);
return {"GraphicsNullVmExportDiscardsInactiveExec",
code,
{0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
{O::V_MOV_B32, O::EXP, O::S_MOV_B64, O::S_ENDPGM}};
}
GraphicsCase GraphicsMrt0OffVmExportDiscardsInactiveExec() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f000000u);
AppendVMovLiteral(&code, 1, 0x3f400000u);
AppendVMovLiteral(&code, 2, 0x3f800000u);
AppendVMovLiteral(&code, 3, 0x40000000u);
code.push_back(EncodeExp0(0x00, 0xf, false));
code.push_back(EncodeExp1(0, 1, 2, 3));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeExp0(0x00, 0x0, true, true, true));
code.push_back(EncodeExp1(0, 0, 0, 0));
AppendEnd(&code);
return {"GraphicsMrt0OffVmExportDiscardsInactiveExec",
code,
{0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
{O::V_MOV_B32, O::EXP, O::S_MOV_B64, O::S_ENDPGM}};
}
GraphicsCase GraphicsFinalVmExportSupersedesEarlierVmMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f000000u);
AppendVMovLiteral(&code, 1, 0x3f400000u);
AppendVMovLiteral(&code, 2, 0x3f800000u);
AppendVMovLiteral(&code, 3, 0x40000000u);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeExp0(0x00, 0xf, false, false, true));
code.push_back(EncodeExp1(0, 1, 2, 3));
code.push_back(EncodeSMovB32(126, 193u));
code.push_back(EncodeSMovB32(127, 193u));
AppendVMovLiteral(&code, 4, 0x40400000u);
AppendVMovLiteral(&code, 5, 0x40800000u);
AppendVMovLiteral(&code, 6, 0x40a00000u);
AppendVMovLiteral(&code, 7, 0x40c00000u);
code.push_back(EncodeExp0(0x00, 0xf, true, false, true));
code.push_back(EncodeExp1(4, 5, 6, 7));
AppendEnd(&code);
return {"GraphicsFinalVmExportSupersedesEarlierVmMask",
code,
{0x40400000u, 0x40800000u, 0x40a00000u, 0x40c00000u},
{O::V_MOV_B32, O::S_MOV_B64, O::EXP, O::S_MOV_B32, O::S_ENDPGM}};
}
GraphicsCase GraphicsBranchPathFinalVmExportDiscardsInactiveExec() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f000000u);
AppendVMovLiteral(&code, 1, 0x3f400000u);
AppendVMovLiteral(&code, 2, 0x3f800000u);
AppendVMovLiteral(&code, 3, 0x40000000u);
code.push_back(EncodeExp0(0x00, 0xf, false));
code.push_back(EncodeExp1(0, 1, 2, 3));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeExp0(0x09, 0x0, true, false, true));
code.push_back(EncodeExp1(0, 0, 0, 0));
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(1)));
const auto branch_index = code.size();
code.push_back(0);
AppendEnd(&code);
const auto later_export_index = code.size();
code[branch_index] = EncodeSopp(
0x05, static_cast<u32>(later_export_index - branch_index - 1u));
code.push_back(EncodeSMovB32(126, 193u));
code.push_back(EncodeSMovB32(127, 193u));
AppendVMovLiteral(&code, 4, 0x40400000u);
AppendVMovLiteral(&code, 5, 0x40800000u);
AppendVMovLiteral(&code, 6, 0x40a00000u);
AppendVMovLiteral(&code, 7, 0x40c00000u);
code.push_back(EncodeExp0(0x00, 0xf, true, false, true));
code.push_back(EncodeExp1(4, 5, 6, 7));
AppendEnd(&code);
return {"GraphicsBranchPathFinalVmExportDiscardsInactiveExec",
code,
{0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
{O::V_MOV_B32, O::EXP, O::S_MOV_B64, O::S_CMP_EQ_U32,
O::S_CBRANCH_SCC1, O::S_MOV_B32, O::S_ENDPGM}};
}
TestCase MultipleWorkitemsGlobalId() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1),
EncodeVop2(0x25, 2, InlineU32(64), 1),
EncodeVop2(0x1a, 3, InlineU32(2), 1),
};
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "MultipleWorkitemsGlobalId";
test.code = code;
test.expected = {64, 65, 66, 67, 68, 69, 70, 71};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase DispatcherIrreducibleControlFlow() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSopp(0x05, 5)); // entry -> B, fallthrough A
AppendVMovU32(&code, 0, 7);
AppendStoreVgpr(&code, 0, 0);
code.push_back(EncodeSopp(0x04, 1)); // A exits while SCC is initially zero
code.push_back(EncodeSopp(0x02, 0xfffau)); // B -> A
AppendEnd(&code);
TestCase test;
test.name = "DispatcherIrreducibleControlFlow";
test.code = code;
test.expected = {7};
test.opcodes = {O::S_CBRANCH_SCC1, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_CBRANCH_SCC0, O::S_BRANCH, O::S_ENDPGM};
return test;
}
#include "ShaderRayTracingGpuTests.inc"
std::vector<TestCase> MakeCases() {
std::vector<TestCase> cases;
cases.reserve(128);
auto AddCase = [&cases](TestCase (*factory)()) {
cases.push_back(factory());
};
cases.push_back(BufferWorkgroupPublication(32));
cases.push_back(BufferWorkgroupPublication(64));
cases.push_back(BufferWorkgroupPublication(32, true));
cases.push_back(BufferWorkgroupPublication(32, false, true));
cases.push_back(BufferWorkgroupPublication(64, false, true));
AddCase(IntegerAddSubMul);
AddCase(BitwiseOps);
AddCase(Shifts);
AddCase(ScalarZeroShiftWithRuntimeCount);
AddCase(ExactPushConstantExtent);
AddCase(ScalarShiftCountsMaskLowBits);
AddCase(Rdna2ScalarOpcodes);
AddCase(ScalarTrapDisabledFallsThroughExactRaw);
AddCase(ScalarWaitcntDepctrCapturedVmVsrc);
AddCase(ScalarExtendedArithmetic);
AddCase(ScalarArithmeticSccCarryBorrowOverflow);
AddCase(ScalarMinMaxSccComparisonEdges);
AddCase(ScalarAbsI32UpdatesScc);
AddCase(ScalarAbsdiffBitset64AndSetprio);
AddCase(ScalarShiftLeftAddSccCarryEdges);
AddCase(ScalarCompareOps);
AddCase(ScalarShiftAddAndMaskOps);
AddCase(ScalarNotB64UpdatesScc);
AddCase(ScalarQuadmaskB64);
AddCase(ScalarFlbitI32B64Gpu);
AddCase(ScalarFlbitI32B32Gpu);
AddCase(ScalarSaveExecOps);
AddCase(ScalarOrn2SaveexecUsesSourceOrNotExec);
cases.push_back(ScalarOrn2SaveexecB32(32, 32));
cases.push_back(ScalarOrn2SaveexecB32(64, 64));
cases.push_back(ScalarOrn2SaveexecB32(32, 4));
cases.push_back(ScalarSubvectorLoops(32));
cases.push_back(ScalarSubvectorLoops(64));
AddCase(ScalarGetpcWritesNextInstructionPc);
AddCase(ScalarBitfieldPack);
AddCase(ScalarBitcmpB64DynamicOperands);
AddCase(ScalarBitcmpB64IntegerConstants);
AddCase(ScalarBrevB32PreservesScc);
for (const u32 bits : {8u, 16u}) {
cases.push_back(ScalarSextMasks(bits, 32));
cases.push_back(ScalarSextMasks(bits, 64));
}
AddCase(ScalarSextConstantsAndAliases);
cases.push_back(ScalarBrevB64OperandsAndMasks(32));
cases.push_back(ScalarBrevB64OperandsAndMasks(64));
AddCase(ScalarBfeI32CapturedRawSignExtends);
AddCase(BitfieldExtractWidthPastEndEdges);
AddCase(Scalar64BitOps);
AddCase(ScalarDynamic64BitOps);
cases.push_back(ScalarAshrI64Edges(false));
cases.push_back(ScalarAshrI64Edges(true));
AddCase(ScalarAshrI64OperandsAndAliases);
cases.push_back(ScalarAshrI64Masks(32));
cases.push_back(ScalarAshrI64Masks(64));
AddCase(VectorDynamicU64ShiftEdges);
AddCase(ScalarConditionalMoveB64);
AddCase(ScalarConditionalMoveB64PreservesMasks);
AddCase(ScalarAndn2B64SccUsesMaskShadow);
AddCase(ScalarMaskHighWriteInvalidatesProvenance);
cases.push_back(ScalarSelectB64PreservesMaskProvenance(32));
cases.push_back(ScalarSelectB64PreservesMaskProvenance(64));
AddCase(ScalarWqmB64SelectsSccDomain);
AddCase(ScalarWqmB64PreservesPartialMasks);
cases.push_back(ScalarWqmB32Masks(32));
cases.push_back(ScalarWqmB32Masks(64));
AddCase(ScalarMaskProvenanceOverlapAndMixedBinary);
cases.push_back(ScalarB32MaskDataflow(32));
cases.push_back(ScalarB32MaskDataflow(64));
AddCase(ScalarLiteral);
AddCase(VectorMoves);
AddCase(VectorVop3MoveAppliesFloatSourceModifiers);
AddCase(VectorIntegerOps);
AddCase(VectorIntegerExtremaRuntimeEdges);
AddCase(VectorFfbhI32NativeAndVop3OnGpu);
AddCase(Vop1SdwaBfrevSourceSelectors);
AddCase(Vop1SdwaFfbhCapturedScalarLowWordSource);
AddCase(Vop1SdwaFfblCapturedHighWordSource);
AddCase(Vop1SdwaNotCapturedByte0Source);
AddCase(Vop1SdwaNotPreservesHighWordDestination);
AddCase(Vop1SdwaNotPartialSourcesAndDestinations);
cases.push_back(Vop2SdwaBitwiseByteDestinations(32));
cases.push_back(Vop2SdwaBitwiseByteDestinations(64));
AddCase(Vop1SdwaMovByteDestinations);
cases.push_back(Vop1SdwaMovByteSources(32));
cases.push_back(Vop1SdwaMovByteSources(64));
AddCase(Vop2SdwaSubNcExactByte2Destination);
AddCase(Vop2SdwaAddNcCapturedHighWordDestination);
AddCase(Vop2SdwaAshrrevCapturedWord0SignExtends);
cases.push_back(Vop2SdwaMaxI32CapturedHighWord(32));
cases.push_back(Vop2SdwaMaxI32CapturedHighWord(64));
cases.push_back(Vop2SdwaMulI24Destinations(32));
cases.push_back(Vop2SdwaMulI24Destinations(64));
cases.push_back(Vop2SdwaMulU24Destinations(32));
cases.push_back(Vop2SdwaMulU24Destinations(64));
AddCase(Vop2SdwaLshrrevCapturedByte1Source);
AddCase(Vop2SdwaSubNcPreservesByteAndWordDestinations);
AddCase(Vop3CvtPkI16I32Captured);
AddCase(Vop3MulLoU16CapturedAndSelectors);
AddCase(Vop3MadI16CapturedSelectorsAndSaturation);
AddCase(Vop3Min3U16CapturedAndSelectors);
AddCase(Vop3Med3I16Captured);
AddCase(Vop2SdwaMinU32PreservesWordDestination);
AddCase(VectorShiftCountsMaskLowBits);
AddCase(VectorVop3LshrrevB64Captured);
AddCase(VectorVop3LshlrevB64Captured);
AddCase(VectorVop3IntegerOps);
AddCase(VectorBfeI32SignExtendsField);
AddCase(VectorAlignByteUsesTwoBitByteOffset);
AddCase(VectorCarryAndBitCountOps);
AddCase(VectorMbcntUsesThreadMask);
AddCase(VectorAddcWritesPerLaneCarryOut);
AddCase(VectorAddcUsesPerLaneCarryIn);
AddCase(VectorSubCoCiU32CompactAndVop3);
AddCase(VectorSubrevCoCiU32ExactRawOnGpu);
AddCase(VectorVop3BSubrevCoCiUsesEncodedMasks);
AddCase(VectorVop3BCarryOutWritesSgprMask);
AddCase(VectorVop3BCarryOutUsesEncodedSdst);
AddCase(VectorVop3BSubCoU32UsesRdna2Opcode310);
AddCase(VectorMadU64U32UnsignedCarryOut);
AddCase(VectorLaneAndPackedOps);
AddCase(Vop2PkFmacF16AccumulatesPackedHalvesIndependently);
AddCase(Vop2PkFmacF16DppNegatesBothPackedHalves);
AddCase(Vop3pOpselHiUsesArchitecturalSourceBits);
AddCase(Vop3pIntegerNegationCapturedAndSelectedHalves);
AddCase(Vop3pInt16Ops);
AddCase(CvtPkU8F32PacksSelectedByte);
AddCase(CvtPkrtzF16F32SubnormalRoundsTowardZero);
AddCase(CvtPkrtzF16F32SdwaAndOutputModifiers);
AddCase(PackedMinMaxF16NanAndSignedZeroEdges);
AddCase(VectorMinMaxF16Ops);
AddCase(VectorCvtF16U16ByteSource);
AddCase(VectorCvtU16F16Sdwa);
AddCase(NativeAndSdwa16BitDestinationWrites);
AddCase(VectorMinMaxMed3F16Ops);
AddCase(VectorSpecialF16Ops);
AddCase(VectorFractF16CapturedAndEdges);
AddCase(VectorFractF16Modifiers);
AddCase(VectorCosF16CapturedSdwaAndEdges);
AddCase(VectorSinF16SdwaAndEdges);
AddCase(VectorWritelaneIgnoresExecMask);
AddCase(VectorReadlaneFromInactiveWrittenLane);
AddCase(VectorLaneWave32RuntimeSelectorWraps);
AddCase(VectorReadlaneSelectsTwoKeysWithinWave);
AddCase(VectorPermlanex16);
AddCase(VectorPermlane16FetchInactiveZero);
AddCase(VectorPermlane16FetchInactiveFi);
cases.push_back(VectorDpp8Captured(false));
cases.push_back(VectorDpp8Captured(true));
AddCase(VectorDppQuadPermuteReverse);
AddCase(VectorDppRowXmask);
AddCase(VectorDppBankMaskPreservesDestination);
AddCase(VectorDppBoundsControlZeroPreservesDestination);
AddCase(Vop3FmacF32NegatedSourceAccumulates);
AddCase(VectorMadF32RoundsProduct);
AddCase(VectorMadF32FlushesDenorms);
AddCase(Vop3LdexpSourceModifier);
AddCase(Vop1MoveRelSource);
AddCase(Vop1MoveRelDestination);
AddCase(VectorFloatSpecialOps);
AddCase(CubeIdCapturedNegationAndOutputScale);
AddCase(MadMixF16LiteralHalfSourceUsesOpsel);
AddCase(MadMixF16NegHiIsAbsAndNegIsIndependent);
AddCase(VectorVop3FmaF16UsesRdna2Opcode34b);
AddCase(FloatInlineConstantF16UsesNumericValue);
AddCase(Vop3Int16Ops);
AddCase(VectorFloatControlContractPreservesInfNan);
AddCase(VectorFloatArithmeticOps);
AddCase(VectorMinMaxF32NanAndSignedZeroEdges);
AddCase(VectorMed3F32NanUsesMin3Path);
AddCase(VectorFloatConversionOps);
AddCase(VectorFrexpF32Edges);
AddCase(CvtF32ToIntSaturatesNaNAndOutOfRange);
AddCase(VectorSpecialF32FlushesDenormalInputs);
AddCase(VectorRcpIflagF32IntegerReciprocal);
AddCase(VectorCompareF64Edges);
AddCase(VectorCompareInteger64Edges);
cases.push_back(VectorCompareExecWaveMasks(32));
cases.push_back(VectorCompareExecWaveMasks(64));
cases.push_back(VectorCompare64WaveMasks(32));
cases.push_back(VectorCompare64WaveMasks(64));
cases.push_back(VectorCompare64WaveMasks(32, true));
cases.push_back(VectorCompare64WaveMasks(64, true));
AddCase(VectorF64CapturedScreenSpaceShadows);
AddCase(VectorFractF64CapturedAndEdges);
AddCase(VectorMinMaxF64CapturedAndEdges);
AddCase(VectorRoundF64EdgesAndModifiers);
AddCase(VectorFractF64CapturedChainRuntimeExec);
AddCase(VectorF64ModesModifiersAndExec);
AddCase(VectorF64WideningConversions);
AddCase(VectorSinCosMaxFiniteSpecialCases);
AddCase(VectorCompareF32ExposureGuard);
for (const auto mode : {0xc0u, 0xe0u}) {
cases.push_back(VectorCompareF32DenormalModes(mode));
}
AddCase(VectorCompareOps);
AddCase(VectorVop3CompareEqI64OnGpu);
AddCase(VectorVop3CompareEqU64OnGpu);
AddCase(VectorVop3CompareGtU64OnGpu);
AddCase(VectorVopcCompareLtU64OnGpu);
AddCase(VectorVop3CompareNeU64OnGpu);
AddCase(VectorVopcCmpxNeU64CapturedExecMask);
AddCase(VectorVop3CmpxNeI64CapturedExecMask);
AddCase(VectorCompareClassF32);
AddCase(VectorCompareClassF32ImmediateAndDynamicMasks);
AddCase(VectorVopcSdwaCmpxClassF32CapturedExecMask);
cases.push_back(VectorCmpClassF16(32));
cases.push_back(VectorCmpClassF16(64));
cases.push_back(VectorCmpxClassF16(32));
cases.push_back(VectorCmpxClassF16(64));
AddCase(VectorCompareF16Ops);
AddCase(VectorCompareI16U16Ops);
AddCase(Wave32VccMasksPreserveOtherHalf);
AddCase(Vop2SdwaCndmaskSourceModifier);
AddCase(Vop2SdwaCndmaskCapturedByte3Source);
AddCase(Vop3CndmaskUsesSgprMaskLaneBits);
AddCase(Vop3CndmaskAllowsDataSourceModifier);
AddCase(VectorCompareExecOps);
AddCase(VectorVopcCmpxOrderedCapturedExecMask);
AddCase(VectorVop3FloatCompareNegSourceModifier);
AddCase(VectorVop3CmpxWritesExecMask);
AddCase(VectorVopcSdwaCmpxWritesExecMask);
AddCase(VectorVopcCmpxGtU16CapturedSdwaExecMask);
AddCase(VectorVopcCmpxLtU16CapturedSdwaExecMask);
AddCase(VectorVopcCmpxLtI16CapturedSdwaExecMask);
for (u32 wave_size : {32u, 64u}) {
cases.push_back(VectorVopcCmpxLtI16WaveMasks(wave_size));
}
AddCase(VectorVopcCmpxEqU16SdwaCompactVop3ExecMask);
AddCase(VectorVopcCmpxNeU16SdwaCompactVop3ExecMask);
AddCase(VectorVopcCmpNgtF16CapturedSdwaAndEdges);
AddCase(VectorVopcCmpNltF16CapturedSdwaAndEdges);
AddCase(VectorVopcCmpxNgtF16CapturedSdwaExecMask);
AddCase(VectorVopcCmpxNleF16CapturedSdwaExecMask);
AddCase(VectorCompareInvertedMaskSelect);
AddCase(BranchSelect);
AddCase(SimpleLoop);
AddCase(SharedReturnKeepsSelectedValues);
AddCase(SiblingSharedExitKeepsCapturedConditions);
AddCase(BranchVccnzUsesWaveMask);
AddCase([] { return ScalarBranchRestoresInactiveLanes(32, 32); });
AddCase([] { return ScalarBranchRestoresInactiveLanes(64, 64); });
AddCase([] { return ScalarBranchRestoresInactiveLanes(32, 4); });
AddCase([] { return ScalarBranchRestoresInactiveLanes(64, 4); });
AddCase(BranchVccnzUsesCarryProducedWaveMask);
AddCase(ScalarMemRealtimeCapturedPlaceholder);
AddCase(ScalarMemoryLoadVariants);
AddCase(ScalarBufferOffsetAlignmentAndCarry);
AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset);
AddCase(ScalarLoadAlignsComponentsAndMasksAddress);
AddCase(ScalarLoadAlignsDynamicBase);
cases.push_back(ScalarBufferFromLoopReadlane(32));
cases.push_back(ScalarBufferFromLoopReadlane(64));
AddCase(BufferLoadStore);
AddCase(BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset);
AddCase(BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset);
AddCase(BufferLoadDwordNoAddressFlagsIgnoresVaddr);
AddCase(BufferLoadDwordIdxenUsesDescriptorStride);
AddCase(BufferStoreDwordIdxenUsesDescriptorStride);
AddCase(BufferStoreDwordAppliesHostOffset);
cases.push_back(BufferOffsetsUsePackedWords(false));
cases.push_back(BufferOffsetsUsePackedWords(true));
AddCase(BufferLoadVariants);
for (const u32 component : {4u, 2u, 0u}) {
cases.push_back(BufferSubwordLoadsAtHostOffset(2, component, false));
}
cases.push_back(BufferSubwordLoadsAtHostOffset(2, 4, true));
cases.push_back(BufferSubwordLoadsAtHostOffset(1, 5, false));
cases.push_back(BufferSubwordLoadsAtHostOffset(1, 5, true));
AddCase(BufferSubwordHostOffsetOverflow);
cases.push_back(BufferSubwordStoresAtHostOffset(2));
cases.push_back(BufferSubwordStoresAtHostOffset(1));
AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress);
AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress);
AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress);
AddCase(BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail);
AddCase(BufferLoadsGpuSelectedDescriptors);
AddCase(BufferLoadDwordGpuSelectedDescriptors);
AddCase(BufferLoadDwordx3GpuSelectedDescriptors);
AddCase(BufferLoadFormatXGpuSelectedDescriptors);
AddCase(BufferStoreDwordx4DropsOnlyOutOfBoundsTail);
AddCase(BufferLoadFormatXyzwRejectsPartialRecord);
AddCase(BufferStoreFormatXyzwDropsPartialRecord);
AddCase(BufferLoadFormatXRejectsPartialRecord);
AddCase(BufferStoreFormatXRejectsPartialRecord);
AddCase(BufferLoadFormatXyRejectsPartialRecord);
AddCase(BufferStoreFormatXyRejectsPartialRecord);
AddCase(BufferStoreVariants);
AddCase(BufferFormatVariants);
AddCase(BufferLoadFormatXyzwSnapshotsOverlappingAddress);
AddCase(BufferLoadFormatXyzwResource32_32FloatAppliesXy01);
AddCase(BufferLoadFormatXyzwResource323232FloatRepeatsSource);
AddCase(BufferLoadFormatXyzwResource8_8_8_8UintAppliesWzy1);
AddCase(BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress);
AddCase(BufferFormatStoreVariants);
AddCase(BufferStoreFormatXResource16UintWritesHalfword);
AddCase(BufferStoreFormatXResource16UintPreservesAdjacentLanes);
AddCase(BufferStoreFormatXyzwPackedUnormSkinningVectors);
AddCase(BufferStoreFormatXZeroFillsPackedRecord);
AddCase(BufferStoreFormatXyzwFloat16ConvertsComponents);
AddCase(BufferStoreFormatXyzwSnorm16CapturedSkinningVectors);
AddCase(BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords);
for (auto &test : BufferLoadFormatD16XCases()) {
cases.push_back(std::move(test));
}
for (auto &test : BufferStoreFormatD16XCases()) {
cases.push_back(std::move(test));
}
for (auto &test : FormattedStoreConversionCases()) {
cases.push_back(std::move(test));
}
AddCase(BufferLoadFormatXResource8UintZeroExtendsByte);
AddCase(BufferLoadFormatXyResource88UintExtractsBytes);
AddCase(BufferLoadFormatXyResource8888UnormConvertsFirstTwoComponents);
AddCase(BufferStoreFormatXyResource88UintWritesBytes);
AddCase(BufferStoreFormatXyzResource3232UintWritesTwoDwords);
AddCase(BufferStoreFormatXyzwResource323232UintWritesThreeDwords);
AddCase(BufferStoreFormatXyResource32UintWritesOneDword);
AddCase(BufferStoreFormatXyzResource8UintWritesOneByte);
AddCase(BufferStoreFormatXAddTidUsesLaneIndex);
AddCase(BufferStoreFormatXDropsOutOfRangeRecord);
AddCase(TBufferLoadVariants);
AddCase([] { return TBufferCapturedZeroStrideOob(); });
AddCase([] { return TBufferCapturedZeroStrideOob(true); });
AddCase([] { return TBufferCapturedZeroStrideOob(false, true); });
AddCase(BufferZeroStrideOobFormatsAndWidths);
AddCase(TBufferLoadFormatXyzwSnapshotsOverlappingAddress);
AddCase(TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress);
AddCase(TBufferLoadFormatX8UintZeroExtendsByte);
AddCase(TBufferLoadFormatX8888UintExtractsFirstByte);
AddCase(TBufferLoadFormatXIdxenUsesDescriptorStride);
AddCase(TBufferLoadFormatX16FloatConvertsToFloat);
AddCase(TBufferStoreFormatX8UintWritesOneByte);
AddCase(TBufferStoreFormatXSintWritesSubDword);
AddCase(TBufferLoadFormatXSintSignExtendsSubDword);
AddCase(TBufferLoadFormatXy1616IntegerComponents);
AddCase(TBufferStoreFormatXy1616IntegerComponents);
AddCase(TBufferLoadFormatXyz16161616UintLoadsHalfwords);
AddCase(TBufferLoadFormatXy1616UnormConvertsToFloat);
AddCase(TBufferLoadFormatXy88UnormConvertsToFloat);
AddCase(TBufferLoadFormatXy88SnormConvertsToFloat);
AddCase(TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents);
AddCase(TBufferLoadFormatXyzw8888UintExtractsBytes);
AddCase(TBufferLoadFormatXyzw1010102SnormConvertsToFloat);
AddCase(TBufferLoadFormatXyz111110FloatUnpacks);
AddCase(TBufferLoadFormatXyzw3232FloatZerosMissingComponents);
AddCase(TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents);
AddCase(TBufferStoreFormatXy88IntegerComponents);
AddCase(TBufferLoadFormatXy88IntegerComponents);
AddCase(TBufferStoreVariants);
cases.push_back(FlatLoadDlcCaptured(32));
cases.push_back(FlatLoadDlcCaptured(64));
AddCase(FlatLoadVariants);
AddCase(FlatSubdwordLoadsApplyByteOffset);
cases.push_back(GlobalLoadShortD16Captured(32));
cases.push_back(GlobalLoadShortD16Captured(64));
AddCase(FlatLoadShortD16AddressSegments);
AddCase(FlatVirtualAddressRebasesGuestAllocation);
AddCase(GlobalSignedImmediateRebasesBeforeSaddr);
AddCase(FlatSegmentIgnoresSaddrAndMasksOffsetMsb);
AddCase(ScratchIsPrivatePerInvocation);
cases.push_back(FlatStackApertures(32));
cases.push_back(FlatStackApertures(64));
cases.push_back(BvhIntersections(true, true, 1));
cases.push_back(BvhIntersections(true, true, 0, true));
for (bool barycentrics : {false, true}) {
for (bool sorted : {false, true}) {
cases.push_back(BvhIntersections(barycentrics, sorted));
}
}
AddCase(DsReadWriteVariants);
AddCase(DsWriteB16D16HiCapturedUsesHighHalf);
AddCase(DsWriteB8D16HiWritesByteTwo);
AddCase(DsReadU16D16CapturedPreservesHighHalf);
AddCase(DsReadU16D16HiCapturedPreservesLowHalf);
AddCase(DsAppendConsumeUsesEncodedLdsSelector);
cases.push_back(DsAppendAllocatesAcrossWaves(32));
cases.push_back(DsAppendAllocatesAcrossWaves(64));
AddCase(DsAppendUsesEncodedGdsSelector);
for (u32 wave_size : {32, 64}) {
cases.push_back(DsOrderedCountAddressAndExec(wave_size, false));
cases.push_back(DsOrderedCountAddressAndExec(wave_size, true));
}
AddCase(DsAppendConsumeGdsRegionBounds);
AddCase(DsGdsSubdwordAndAtomicWrites);
AddCase(DsReadWrite2Variants);
AddCase(DsWideReadSnapshotsOverlappingAddress);
AddCase(DsReadWrite2EqualOffsetsUseData0);
AddCase(DsWideLdsPartialBounds);
AddCase(DsWideGdsPartialBounds);
AddCase(DsAtomicNoReturnVariants);
for (bool gds : {false, true}) {
cases.push_back(DsMaskedOrBounds(gds));
}
for (u32 wave_size : {32, 64}) {
cases.push_back(DsMaskedOrContention(wave_size));
}
AddCase(DsAtomicReturnVariants);
for (bool add : {false, true}) {
cases.push_back(DsAtomic64Bounds(add));
for (u32 wave_size : {32, 64}) {
cases.push_back(DsAtomic64Contention(add, wave_size));
}
}
AddCase(BufferAtomicAndX2GlcAndExec);
AddCase(ImageAtomicSignedMinMax<false>);
AddCase(ImageAtomicSignedMinMax<true>);
for (bool decrement : {false, true}) {
for (bool gds : {false, true}) {
cases.push_back(DsBoundedAtomicBoundaries(decrement, gds));
for (u32 wave_size : {32u, 64u}) {
cases.push_back(DsBoundedAtomicContention(decrement, gds, wave_size));
}
}
}
AddCase(DsMiscVariants);
AddCase(DsFloatMinMaxIgnoresData1);
AddCase([] { return DsFloatMinMaxClasses(false); });
AddCase([] { return DsFloatMinMaxClasses(true); });
AddCase(DsSwizzleInvalidSourceLaneZero);
AddCase(DsPermuteCapturedExecOffsetAndWrap);
AddCase(DsPermuteWave64UsesIndependentHalves);
AddCase(DsBpermuteCapturedExecOffsetAndWrap);
AddCase(DsBpermuteWave64UsesIndependentHalves);
AddCase(Wave64CrossHalfLaneAndLds);
AddCase(Wave64RawMasksAndScalarBranch);
AddCase(Wave64PartialMultidimensionalWorkgroup);
AddCase(Wave64AppendConsumeHighHalf);
AddCase(BufferAtomicVariants);
AddCase(BufferAtomicCmpSwapExactRaw);
AddCase(BufferAtomicGlc0DoesNotReturnOldValue);
AddCase(BufferAtomicFMinExactRawGlcModes);
AddCase(BufferAtomicFMinSpecialValues);
AddCase(BufferAtomicFMinContendedWorkgroup);
AddCase(BufferAtomicFMaxExactRawGlcModes);
AddCase(BufferAtomicFMaxSpecialValues);
AddCase(BufferAtomicFMaxContendedWorkgroup);
AddCase(ImageLoadVariants);
AddCase(ImageLoadR32UintUsesIntegerSampledImage);
AddCase(ImageLoadFmaskUsesNativeSampleMapping);
AddCase(ImageLoadR32SintUsesSignedSampledImage);
AddCase(ImageLoadPackedUintUnpacksAndSwizzles);
AddCase(ImageSamplePackedUintConvertsSampleAndGather);
AddCase(ImageSampleUScaled8<false>);
AddCase(ImageSampleUScaled8<true>);
AddCase(ImageLoadR128IgnoresAdjacentMaskSgprs);
AddCase(ImageLoad1DUsesScalarCoordinate);
AddCase(ImageGather2DInstructionWith1DDescriptor);
AddCase(ImageLoad1DArrayUsesLayerCoordinate);
AddCase(ImageLoad1DArrayDescriptorUsesSelectedLayer);
AddCase(ImageLoadMipUsesVaddr2Lod2D);
AddCase(ImageLoadMipNsaUsesSelectedAddressVgprs);
AddCase(ImageLoadA16UintCoordsOnGpu);
AddCase(ImageGetResinfoDmaskWidthHeight);
AddCase(ImageGetResinfoDmaskMipLevels);
AddCase(ImageSampleAndGather);
AddCase(ImageCubeGradientsPreserveDerivatives);
AddCase(ImageGatherExplicitLod);
AddCase(ImageGatherLodPerLane<0>);
AddCase(ImageGatherLodPerLane<1>);
AddCase(ImageGatherLodPerLane<2>);
AddCase(ImageGatherLodPerLane<3>);
AddCase(ImageGatherLodPerLane<4>);
AddCase(ImageGatherLodPerLane<5>);
AddCase(ImageGatherLodPerLane<6>);
AddCase(ImageGatherLodPerLane<7>);
AddCase(ImageGatherLodPerLane<8>);
AddCase(ImageD16GatherPacksHalfPairs);
AddCase(ImageSampleA16SamplerCoordsOnGpu);
AddCase(ImageSampleOpcodeAliasUsesNormalCoords);
AddCase(ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu);
AddCase(ImageSampleA16CompareBiasRdna2AddressOrder);
AddCase(ImageGatherCompareOpcodes);
AddCase(ImageStoreVariants);
AddCase(ImageD16StoreUnpacksHalfPairs);
AddCase(ImageStoreMipSelectsPpsa01340Descriptor);
AddCase(ImageStoreSingleMipBounds<0>);
AddCase(ImageStoreSingleMipBounds<1>);
AddCase(ImageStoreSingleMipBounds<2>);
AddCase(ImageStoreRgbOneUsesInverseSwizzle);
AddCase(ImageStoreDuplicateSelectorUsesInverseSwizzle);
AddCase(ImageStoreBgraUsesInverseSwizzle);
AddCase(ImageStoreYzwxUsesInverseSwizzle);
AddCase(ImageStoreR32FloatUsesFormatlessStorageImage);
AddCase(ImageStoreR32SintUsesRawUintView);
AddCase(ImageStoreR32UintUsesFormatlessStorageImage);
AddCase(ImageStorePackedUintSaturatesChannels);
AddCase(ImageStorePackedUintHonorsSparseDmask);
AddCase(ComputeTgSizeSgprUsesWaveMetadata);
AddCase(ImageStoreAndAtomicShareTypedBinding);
AddCase(ImageAtomicCompareSwapGlcAndExec);
AddCase(ImageAtomicSwapReturnsPreviousTexel);
AddCase(ImageStoreAndAtomicUseSeparateBindings);
AddCase(ImageAtomicVariants);
for (const auto &atomic : ImageAtomicIntegerCases) {
cases.push_back(ImageAtomicIntegerGlcAndExec(atomic, false));
cases.push_back(ImageAtomicIntegerGlcAndExec(atomic, true));
}
AddCase(ImageAtomicFloatGlcAndExec<true>);
AddCase(ImageAtomicFloatGlcAndExec<false>);
AddCase(ImageAtomicFloatSpecialValues<true>);
AddCase(ImageAtomicFloatSpecialValues<false>);
AddCase(ImageAtomicFloatContendedWorkgroup<true>);
AddCase(ImageAtomicFloatContendedWorkgroup<false>);
AddCase(ImageAtomicGlc0DoesNotReturnOldValue);
AddCase(MultipleWorkitemsGlobalId);
AddCase(DispatcherIrreducibleControlFlow);
return cases;
}
std::vector<GraphicsCase> MakeGraphicsCases() {
return {
GraphicsInterpolationExport(),
GraphicsPositionWExport(),
GraphicsPackedHalfCentroid(),
GraphicsPerspectiveSample(false),
GraphicsPerspectiveSample(true),
GraphicsPackedHalfInputAlias(false),
GraphicsPackedHalfInputAlias(true),
GraphicsSmoothRawInputAlias(),
GraphicsSmoothFlatInputAlias(false),
GraphicsSmoothFlatInputAlias(true),
GraphicsSmoothFlatInputAlias(false, true),
GraphicsSmoothFlatInputAlias(true, true),
GraphicsAncillaryLayer(false),
GraphicsAncillaryLayer(true),
GraphicsAncillarySampleId(),
GraphicsFlatInterpolatorExport(),
GraphicsDsAddtidScratchExport(),
GraphicsDirectSgprPushConstantExport(),
GraphicsInlineSrtScalarPromotionExport(),
GraphicsNullVmExportDiscardsInactiveExec(),
GraphicsMrt0OffVmExportDiscardsInactiveExec(),
GraphicsFinalVmExportSupersedesEarlierVmMask(),
GraphicsBranchPathFinalVmExportDiscardsInactiveExec(),
};
}
void CheckComputeLdsLimit(VulkanHarness &vulkan) {
constexpr const char *name = "ComputeLdsLimit";
std::vector<u32> code;
AppendVMovU32(&code, 1, 7);
code.push_back(EncodeDs0(0x0d)); // ds_write_b32 v0, v1
code.push_back(EncodeDs1(0, 1, 0));
AppendEnd(&code);
ShaderMapUserData(reinterpret_cast<uint64_t>(code.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(code.size() * sizeof(u32))});
GraphicContext graphics;
graphics.device = vulkan.Device();
graphics.physical_device_properties =
vulkan.RuntimeContext().GetPhysicalDeviceProperties();
auto &limit = graphics.physical_device_properties.limits.maxComputeSharedMemorySize;
// Reproduce a 48-KiB host, or a smaller host limit when the test device requires it.
limit = std::min(limit, 48u * 1024u) & ~511u;
const auto limit_units = static_cast<uint16_t>(limit / 512u);
HW::ComputeShaderInfo regs{};
regs.cs_regs.data_addr = reinterpret_cast<uint64_t>(code.data());
regs.cs_regs.num_thread_x = 1;
regs.cs_regs.num_thread_y = 1;
regs.cs_regs.num_thread_z = 1;
regs.cs_regs.wave_size = 32;
PipelineCache cache(graphics);
ShaderProgram previous;
uint16_t previous_units = 0;
for (const auto units : {static_cast<uint16_t>(limit_units - 1u), limit_units,
uint16_t{112}, uint16_t{128}, uint16_t{128}}) {
regs.cs_regs.lds_size = units;
ShaderComputeInputInfo input{};
const auto program = cache.GetComputeProgram(regs, {}, input);
Require(name, "guest allocation and device backing",
input.lds_size_dwords == units * 128u &&
input.lds_storage == (units > limit_units),
"compute LDS was truncated or chose backing that exceeds the device limit");
Require(name, "pipeline creation",
cache.GetComputePipeline(input, program).pipeline != nullptr,
"full-size LDS shader did not create a Vulkan compute pipeline");
if (previous.id != 0) {
Require(name, "guest-size cache identity",
(program.id == previous.id) == (units == previous_units),
"different LDS sizes shared a shader, or repeating a size recompiled it");
}
previous = program;
previous_units = units;
}
std::printf("[gpu] %-32s ok\n", name);
}
void CheckPs5GameExampleImageClearRuntimeShape() {
const auto MakeCode = [] {
std::vector<u32> code;
AppendVop3(&code, 0x346u, 4, 8, InlineU32(6), Vgpr(0));
for (u32 i = 0; i < 4; i++) {
code.push_back(EncodeVop1(0x01u, i, i + 4u));
}
code.push_back(EncodeMubuf0(0x07u, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 4));
AppendEnd(&code);
return code;
};
std::array<u32, 8> user_data{};
user_data[0] = 0x00010000u;
user_data[1] = 16u << 16u;
user_data[2] = 64;
user_data[3] =
(static_cast<uint32_t>(Prospero::BufferFormat::k32_32_32_32UInt) << 12u) |
4u | (5u << 3u) | (6u << 6u) | (7u << 9u);
std::fill(user_data.begin() + 4, user_data.end(), 0xff000000u);
ShaderComputeInputInfo compute{};
compute.threads_num[0] = 64;
compute.threads_num[1] = 1;
compute.threads_num[2] = 1;
compute.dispatch_threads_num[0] = 64;
compute.dispatch_threads_num[1] = 1;
compute.dispatch_threads_num[2] = 1;
compute.group_id[0] = true;
compute.dispatch_thread_dimensions = true;
compute.wave_size = 64;
compute.thread_ids_num = 1;
compute.workgroup_register = 8;
u32 scalar_clear = 0x40404040u;
bool clean_scalar = false;
const auto Compile = [&](const std::vector<u32> &code) {
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Compute;
options.wave_size = 64;
options.user_data_base = 0;
options.user_data = user_data;
options.input_info.compute = &compute;
options.dump_ir = false;
auto translated = ShaderRecompiler::TranslateProgram(code, options);
auto resource_plan =
ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot resources;
ShaderRecompiler::IR::ResourceSpecialization specialization;
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = options.user_data,
.shader_base = reinterpret_cast<uint64_t>(code.data()),
.userdata = &scalar_clear,
.read_specialization_memory = clean_scalar
? +[](void *data, uint64_t address, std::span<uint32_t> words) {
if (address != reinterpret_cast<uint64_t>(data) || words.size() != 1u) return false;
words[0] = *static_cast<uint32_t *>(data);
return true;
} : nullptr,
};
Require("Ps5GameExampleImageClear", "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
resource_plan, runtime, resources, specialization),
"runtime clear resources could not be materialized");
auto result = ShaderRecompiler::CompileProgram(
std::move(translated), options, specialization);
ValidateSpirv("Ps5GameExampleImageClear", result.spirv);
return resources;
};
const auto code = MakeCode();
auto positive = Compile(code);
auto runtime_resources = positive;
compute.stage.resources = &runtime_resources;
ShaderBufferResource descriptor{};
u32 packed_clear = 0;
uint64_t size = 0;
Require("Ps5GameExampleImageClear", "runtime shape",
ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
packed_clear, size) &&
descriptor.Base48() == 0x10000u && size == 64u * 16u &&
packed_clear == 0xff000000u,
"exact Prospero runtime binding did not resolve to a complete clear");
auto narrow_exec = code;
narrow_exec.insert(narrow_exec.end() - 3,
EncodeSop1(0x24u, 10, InlineU32(1))); // S_AND_SAVEEXEC_B64.
runtime_resources = Compile(narrow_exec);
Require("Ps5GameExampleImageClear", "guest EXEC narrowing",
!ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
packed_clear, size),
"a guest predicate narrowing the store was accepted as a complete clear");
auto wrong_index = code;
wrong_index[0] = (wrong_index[0] & ~(0x3ffu << 16u)) | (0x347u << 16u);
runtime_resources = Compile(wrong_index);
Require("Ps5GameExampleImageClear", "add-before-shift address",
!ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
packed_clear, size),
"V_ADD_LSHL was mistaken for V_LSHL_ADD contiguous coverage");
user_data[7] ^= 1u;
runtime_resources = Compile(code);
Require("Ps5GameExampleImageClear", "non-repeated clear",
!ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
packed_clear, size),
"non-uniform uint4 data was replaced with a color clear");
runtime_resources = positive;
compute.dispatch_threads_num[0] = 63;
Require("Ps5GameExampleImageClear", "mismatched requested extent",
!ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
packed_clear, size),
"a dispatch exceeding its requested extent was classified as a complete clear");
runtime_resources.buffers[0].dwords[2] = 65;
compute.dispatch_threads_num[0] = 65;
Require("Ps5GameExampleImageClear", "partial final workgroup",
!ResolveComputeBufferFill(compute, 65, 1, 1, 0x61u, descriptor,
packed_clear, size),
"a nondivisible thread dispatch was classified as a complete clear");
runtime_resources = positive;
compute.dispatch_threads_num[0] = 32;
Require("Ps5GameExampleImageClear", "partial dispatch",
!ResolveComputeBufferFill(compute, 32, 1, 1, 0x61u, descriptor,
packed_clear, size),
"partial buffer coverage was classified as a complete clear");
// Same descriptor/dispatch as GTA3, with the stored value encoded in the
// shader.
user_data = {0x00200000u, 4u << 16u, 64u, 0x14204u};
compute.workgroup_register = 4;
compute.dispatch_thread_dimensions = false;
std::vector<u32> scalar_code;
AppendVop3(&scalar_code, 0x346u, 0, 4, InlineU32(6), Vgpr(0));
scalar_code.push_back(EncodeVop1(0x01u, 1, InlineU32(0)));
scalar_code.push_back(EncodeMubuf0(0x04u, 0, true, false));
scalar_code.push_back(EncodeMubuf1(1, 0, 0));
AppendEnd(&scalar_code);
auto scalar = Compile(scalar_code);
runtime_resources = scalar;
Require("Ps5GameExampleImageClear", "inline scalar fill",
ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size) &&
descriptor.Base48() == 0x200000u && size == 256u &&
packed_clear == 0u,
"GTA3's actual scalar store was not recognized from IR");
Require("Ps5GameExampleImageClear", "excess dispatch",
!ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
packed_clear, size),
"excess invocation coverage was accepted as a fill");
auto alias = scalar.buffers.front();
runtime_resources.buffers.push_back(alias);
Require("Ps5GameExampleImageClear", "aliased scalar input",
!ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a scalar read aliasing the destination was accepted as uniform");
runtime_resources = scalar;
runtime_resources.buffers[0].dwords[3] =
(static_cast<u32>(Prospero::BufferFormat::k32Float) << 12u) | 0x204u;
Require("Ps5GameExampleImageClear", "wrong store format",
!ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a format that changes the stored value was accepted as a uint fill");
// A body change preserving every descriptor and dispatch field must change
// the result.
scalar_code[2] = EncodeVop1(0x01u, 1, InlineU32(7));
runtime_resources = Compile(scalar_code);
Require("Ps5GameExampleImageClear", "changed store value",
ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size) &&
packed_clear == 7u,
"clear recognition ignored the shader's actual stored value");
scalar_code[2] = EncodeVop1(0x01u, 1, Vgpr(0));
runtime_resources = Compile(scalar_code);
Require("Ps5GameExampleImageClear", "varying store value",
!ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a varying store with identical resources was treated as a fill");
const auto scalar_address = reinterpret_cast<uint64_t>(&scalar_clear);
user_data[4] = static_cast<u32>(scalar_address);
user_data[5] = static_cast<u32>(scalar_address >> 32u) | (16u << 16u);
user_data[6] = 1;
user_data[7] = 0x4dfacu;
compute.workgroup_register = 8;
scalar_code.clear();
AppendVop3(&scalar_code, 0x346u, 0, 8, InlineU32(6), Vgpr(0));
scalar_code.push_back(EncodeSmem0(0x08u, 106, 2));
scalar_code.push_back(EncodeSmem1(0, 125));
scalar_code.push_back(EncodeVop1(0x01u, 1, 106));
scalar_code.push_back(EncodeMubuf0(0x04u, 0, true, false));
scalar_code.push_back(EncodeMubuf1(1, 0, 0));
AppendEnd(&scalar_code);
clean_scalar = true;
auto loaded = Compile(scalar_code);
runtime_resources = loaded;
Require("Ps5GameExampleImageClear", "scalar-loaded fill",
ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size) &&
packed_clear == scalar_clear && descriptor.Base48() == 0x200000u,
"GTA3's scalar input was confused with its destination");
clean_scalar = false;
runtime_resources = Compile(scalar_code);
Require("Ps5GameExampleImageClear", "unavailable clean scalar",
!ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a scalar without a clean reader was replaced by a fill");
std::printf("[host] %-32s ok\n", "Ps5GameExampleImageClear");
}
void CheckTessellationProgram(const char *name, u32 ls_stride, u32 hs_stride) {
using namespace ShaderRecompiler;
const bool multiplied_stride = hs_stride == 112;
const u32 local_address = multiplied_stride ? 18 : 21;
const u32 control_point = multiplied_stride ? 30 : 34;
const u32 relative_patch = multiplied_stride ? 28 : 32;
const u32 output_address = multiplied_stride ? 31 : 35;
const u32 input_address = multiplied_stride ? 25 : 29;
std::array<std::vector<u32>, 3> code;
auto &local = code[0];
// The fused LS addresses one control point using its v3 ordinal.
local.push_back(EncodeVop2(0x0b, local_address, 255u, 3));
local.push_back(ls_stride);
local.push_back(EncodeDs0(0x0e, (3u << 8u) | 1u));
local.push_back(EncodeDs1Ex(0, 5, 2, local_address));
local.push_back(EncodeSop1(0x20, 0, 6));
local.push_back(0xffffffffu); // Shader footer is not executable after LS handoff.
auto &control = code[1];
AppendVop3(&control, 0x365, 0, 193u, InlineU32(0));
AppendVop3(&control, 0x366, 0, 193u, Vgpr(0));
control.push_back(EncodeSopp(0x0a));
control.push_back(EncodeVopc(0xd1, 249u, 3));
control.push_back(EncodeVopcSdwa(0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 1));
AppendVop3(&control, 0x148, control_point, Vgpr(1), InlineU32(8),
InlineU32(5));
AppendVop3(&control, 0x148, relative_patch, Vgpr(1), InlineU32(0),
InlineU32(8));
if (multiplied_stride) {
// The later patch writes 112-byte outputs but reads 108-byte LS records.
// The second multiply overwrites packed v1; only memory consumers identify
// which of the two independently computed addresses supplies each stride.
control.push_back(EncodeVop2(0x0b, 0, 255u, control_point));
control.push_back(hs_stride);
control.push_back(EncodeVop2(0x0b, 1, 255u, control_point));
control.push_back(ls_stride);
AppendVop3(&control, 0x143, output_address, 255u, Vgpr(relative_patch),
Vgpr(0));
control.push_back(3 * hs_stride);
AppendVop3(&control, 0x143, input_address, 255u, Vgpr(relative_patch),
Vgpr(1));
control.push_back(3 * ls_stride);
} else {
control.push_back(EncodeSopk(0x00, 106, 3 * hs_stride));
control.push_back(EncodeVop2(0x0b, 2, 249u, 1));
control.push_back(EncodeVop2Sdwa(106, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1));
AppendVop3(&control, 0x346, output_address, Vgpr(control_point), InlineU32(7),
Vgpr(2));
control.push_back(EncodeVop2(0x0b, input_address, 255u, control_point));
control.push_back(ls_stride);
}
control.push_back(EncodeDs0(0x37, (3u << 8u) | 1u));
control.push_back(EncodeDs1Ex(10, 0, 0, input_address));
control.push_back(EncodeMubuf0(0x1d, 28));
control.push_back(EncodeMubuf1(10, 2, output_address, 2));
// Keep the control-point-dependent value and the last written dword live.
control.push_back(EncodeMubuf0(0x1c, hs_stride - 8));
control.push_back(EncodeMubuf1(0, 2, output_address, 2));
// The game reuses previously written data VGPRs for CP0-only ring addresses.
AppendVMovLiteral(&control, 4, 0x42280000u);
AppendVMovLiteral(&control, 5, 0x422c0000u);
control.push_back(EncodeVopc(0xd4, InlineU32(1), control_point));
control.push_back(EncodeVop2(0x1a, 4, InlineU32(4), relative_patch));
control.push_back(EncodeVop2(0x1a, 6, InlineU32(6), relative_patch));
control.push_back(EncodeVop2(0x26, 5, 255u, 6));
control.push_back(0x7fc0);
for (u32 i = 0; i < 4; i++) {
AppendVMovLiteral(&control, i, std::bit_cast<u32>(static_cast<float>(i + 1)));
}
control.push_back(EncodeMubuf0(0x1e));
control.push_back(EncodeMubuf1(0, 2, 4, 4));
for (u32 i = 0; i < 4; i++) {
control.push_back(EncodeMubuf0(0x1c, i * 16));
control.push_back(EncodeMubuf1(i, 2, 5, 2));
}
AppendEnd(&control);
auto &evaluation = code[2];
AppendVop3(&evaluation, 0x169, 24, 255u, Vgpr(7));
evaluation.push_back(3 * hs_stride);
evaluation.push_back(EncodeMubuf0(0x0d, 28));
evaluation.push_back(EncodeMubuf1(10, 2, 24, 4));
evaluation.push_back(EncodeMubuf0(0x0d, 2 * hs_stride + 28));
evaluation.push_back(EncodeMubuf1(12, 2, 24, 4));
evaluation.push_back(EncodeExp0(0x0c, 0xf));
evaluation.push_back(EncodeExp1(5, 6, 10, 12));
AppendEnd(&evaluation);
ShaderTessellationInputInfo tess{.input_control_points = 3,
.output_control_points = 3,
.domain = 1,
.partitioning = 2,
.output_topology = 2};
AnalyzeTessellationPrograms(local, control, tess);
Require(name, "decoded interface",
tess.ls_stride == ls_stride && tess.hs_stride == hs_stride,
"captured LS and HS address arithmetic must produce distinct strides");
constexpr std::array stages{ShaderType::Local, ShaderType::TessellationControl,
ShaderType::TessellationEvaluation};
for (u32 stage = 0; stage < stages.size(); stage++) {
ShaderVertexInputInfo vertex;
vertex.logical_stage = stages[stage];
vertex.tess = tess;
CompileOptions options;
options.stage = stages[stage];
options.input_info.vertex = &vertex;
auto translated = TranslateProgram(code[stage], options);
std::set<u32> local_offsets, evaluation_offsets, factor_offsets, patch_offsets;
for (const auto *block : translated.program.blocks) {
for (const auto &inst : *block) {
if (inst.GetOpcode() != IR::ValueOpcode::GetTessellationAttribute &&
inst.GetOpcode() != IR::ValueOpcode::SetTessellationAttribute) {
continue;
}
if (!inst.Arg(1).IsImmediate()) {
continue;
}
switch (static_cast<IR::TessellationAttribute>(inst.Arg(0).U32())) {
case IR::TessellationAttribute::LocalOutput:
local_offsets.insert(inst.Arg(1).U32());
break;
case IR::TessellationAttribute::EvaluationInput:
evaluation_offsets.insert(inst.Arg(1).U32());
break;
case IR::TessellationAttribute::Factor:
factor_offsets.insert(inst.Arg(1).U32());
break;
case IR::TessellationAttribute::PatchOutput:
patch_offsets.insert(inst.Arg(1).U32());
break;
default:
break;
}
}
}
Require(name, "separate DS offsets", stage != 0 || local_offsets == std::set<u32>{4, 12},
"DS_WRITE2 must preserve both independently encoded offsets");
Require(name, "offchip component addressing",
stage != 2 ||
evaluation_offsets == std::set<u32>{28, 2 * hs_stride + 28},
"TES control point and component offsets must survive lowering");
Require(name, "triangle factor layout",
stage != 1 || factor_offsets == std::set<u32>{0, 4, 8, 12},
"the four packed guest factors must retain their order");
Require(name, "predicated patch addresses",
stage != 1 || patch_offsets == std::set<u32>{0x7fc0, 0x7fd0, 0x7fe0, 0x7ff0},
"CP0 patch stores must use the address assigned on their active predicate");
auto result = CompileProgram(std::move(translated), options, {});
ValidateSpirv(name, result.spirv);
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string assembly;
Require(name, "disassembly", tools.Disassemble(result.spirv, &assembly),
"failed to disassemble tessellation shader");
Require(name, "native local inputs",
stage != 0 || (assembly.find("VertexIndex") != std::string::npos &&
assembly.find("InstanceIndex") != std::string::npos),
"LS v2/v5 must read native vertex/instance indices");
Require(name, "control point lane",
stage != 1 || (assembly.find("InvocationId") != std::string::npos &&
assembly.find("SubgroupLocalInvocationId") == std::string::npos),
"HS lane masks must use control point invocation IDs");
Require(name, "control point barrier",
stage != 1 || (assembly.find("OpMemoryModel Logical GLSL450") != std::string::npos &&
assembly.find("OpControlBarrier %uint_2 %uint_4 %uint_0") != std::string::npos),
"HS barriers must match GLSL450 control point synchronization semantics");
Require(name, "factor builtins",
stage != 1 || (assembly.find("TessLevelOuter") != std::string::npos &&
assembly.find("TessLevelInner") != std::string::npos),
"triangle factors require outer[4] and inner[2] Vulkan builtins");
Require(name, "native domain and winding",
stage != 2 || (assembly.find("TessCoord") != std::string::npos &&
assembly.find("Triangles") != std::string::npos &&
assembly.find("SpacingFractionalOdd") != std::string::npos &&
assembly.find("VertexOrderCw") != std::string::npos),
"TES must preserve guest triangle domain, partitioning, and winding");
}
std::printf("[host] %-32s ok\n", name);
}
void CheckTessellationPrograms() {
CheckTessellationProgram("TessellationShiftedStride", 124, 128);
CheckTessellationProgram("TessellationMultipliedStride", 108, 112);
}
void CheckEmbeddedFetchVertexOffset() {
const auto MakeFetch = [](std::initializer_list<std::pair<u32, u32>> adds,
std::optional<std::pair<u32, u32>> late_add = {},
u32 accumulator_vgpr = 0, bool ngg_sad = false,
bool local = false) {
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(0)));
code.push_back(EncodeSmem0(0x02u, 20, 4));
code.push_back(EncodeSmem1(0));
const auto AppendOffsets = [&]() {
for (const auto [sgpr, index_vgpr] : adds) {
if (ngg_sad) {
AppendVop3(&code, 0x15du, accumulator_vgpr, sgpr, InlineU32(0),
Vgpr(index_vgpr));
} else {
AppendVop3B(&code, 0x30fu, accumulator_vgpr, 0, sgpr,
Vgpr(index_vgpr));
}
}
};
if (ngg_sad) {
AppendOffsets();
}
code.push_back(EncodeVop2(0x01u, 0, Vgpr(local ? 5 : 8), local ? 2 : 5));
if (!ngg_sad) {
AppendOffsets();
}
code.push_back(EncodeMubuf0(0x03u, 0, true));
code.push_back(EncodeMubuf1(9, 5, 0));
if (late_add.has_value()) {
AppendVop3B(&code, 0x30fu, 0, 0, late_add->first, Vgpr(late_add->second));
}
if (local) {
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeDs0(0x0e, 1u << 8u));
code.push_back(EncodeDs1Ex(0, 10, 9, 20));
code.push_back(EncodeDs0(0x0e, (3u << 8u) | 2u));
code.push_back(EncodeDs1Ex(0, 12, 11, 20));
code.push_back(EncodeSop1(0x20, 0, 6));
code.push_back(0xffffffffu);
} else {
AppendEnd(&code);
}
return code;
};
const auto Compile = [&](const char *name, const std::vector<u32> &code,
u32 slot10, ShaderType stage = ShaderType::Vertex,
Prospero::BufferFormat format = Prospero::BufferFormat::k32_32_32_32Float) {
std::array<u32, 11> user_data{};
user_data[10] = slot10;
ShaderVertexInputInfo vertex;
vertex.logical_stage = stage;
vertex.tess = {.input_control_points = 3, .output_control_points = 3,
.ls_stride = 124, .hs_stride = 128, .domain = 1,
.partitioning = 2, .output_topology = 2};
vertex.fetch_embedded = true;
vertex.fetch_buffer_reg = 0;
vertex.fetch_attrib_reg = 2;
vertex.resources_num = 1;
vertex.resources[0].fields[3] =
BufferFormat(format) << 12u | DstSel(4, 5, 6, 7);
vertex.resources_dst[0].attr_id = 0;
vertex.resources_dst[0].registers_num = 4;
ShaderRecompiler::CompileOptions options;
options.stage = stage;
options.user_data_base = 8;
options.user_data = user_data;
options.input_info.vertex = &vertex;
auto translated = ShaderRecompiler::TranslateProgram(code, options);
auto resource_plan =
ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot resources;
ShaderRecompiler::IR::ResourceSpecialization specialization;
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = options.user_data,
.shader_base = reinterpret_cast<uint64_t>(code.data()),
};
Require(name, "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
resource_plan, runtime, resources, specialization),
"embedded fetch resources could not be materialized");
auto result = ShaderRecompiler::CompileProgram(
std::move(translated), options, specialization);
Require(name, "fetch rewrite",
result.program.info.vertex_fetch_components[0] == 4,
"encoded fetch sequence was not recognized and rewritten");
return CompiledShader{std::move(result.spirv), std::move(result.program),
std::move(resources), {}};
};
const auto Resolve = [](const CompiledShader &result,
u32 index_offset) {
ShaderVertexInputInfo vertex;
vertex.fetch_embedded = true;
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.user_data_base = result.program.user_data_base;
program.info.vertex_offset_sgpr = result.program.info.vertex_offset_sgpr;
vertex.stage.program = &program;
vertex.stage.resources = &result.resources;
return ResolveDrawOffsets(index_offset, vertex).first;
};
const auto ResolveInstance = [](const CompiledShader &result) {
ShaderVertexInputInfo vertex;
vertex.fetch_embedded = true;
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.user_data_base = result.program.user_data_base;
program.info.instance_offset_sgpr =
result.program.info.instance_offset_sgpr;
vertex.stage.program = &program;
vertex.stage.resources = &result.resources;
return ResolveDrawOffsets(0, vertex).second;
};
for (const auto format : {Prospero::BufferFormat::k32_32_32_32Float,
Prospero::BufferFormat::k32_32_32_32UInt}) {
const auto local = Compile("LocalEmbeddedFetch", MakeFetch({}, {}, 0, false, true),
0, ShaderType::Local, format);
ValidateSpirv("LocalEmbeddedFetch", local.spirv);
Require("LocalEmbeddedFetch", "live vertex attributes",
std::ranges::any_of(local.program.info.inputs, [](const auto &input) {
return input.kind == ShaderRecompiler::IR::StageInputKind::Parameter &&
input.location == 0;
}), "LS indexed fetch must consume native vertex attributes");
if (format == Prospero::BufferFormat::k32_32_32_32UInt) {
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string assembly;
Require("LocalEmbeddedFetch", "disassembly", tools.Disassemble(local.spirv, &assembly),
"failed to disassemble unsigned LS fetch");
Require("LocalEmbeddedFetch", "unsigned input type",
assembly.find("OpTypePointer Input %v4uint") != std::string::npos &&
assembly.find("OpConvertUToF") == std::string::npos,
"unsigned LS attributes must preserve their native integer bits");
}
}
const auto valid =
Compile("EmbeddedFetchVertexOffset", MakeFetch({{18, 0}}), 7);
Require(
"EmbeddedFetchVertexOffset", "parse",
valid.program.info.vertex_offset_sgpr == 18 && Resolve(valid, 0) == 7 &&
Resolve(valid, 5) == 5,
"canonical fetch offset or register index-offset precedence is wrong");
const auto ngg_code = MakeFetch({{18, 5}}, {}, 5, true);
Require("EmbeddedFetchNggVertexOffset", "encoding",
ngg_code[3] == 0xd55d0005u && ngg_code[4] == 0x04150012u,
"test does not encode the PS5 V_SAD_U32 vertex-offset prolog");
const auto ngg = Compile("EmbeddedFetchNggVertexOffset", ngg_code, 8);
Require("EmbeddedFetchNggVertexOffset", "parse",
ngg.program.info.vertex_offset_sgpr == 18 && Resolve(ngg, 0) == 8 &&
Resolve(ngg, 5) == 5,
"PS5 NGG vertex-index offset or register index-offset precedence is "
"wrong");
const auto ngg_instance =
Compile("EmbeddedFetchNggInstanceOffset",
MakeFetch({{18, 8}}, {}, 8, true), 81);
Require("EmbeddedFetchNggInstanceOffset", "parse",
ngg_instance.program.info.vertex_offset_sgpr == -1 &&
ngg_instance.program.info.instance_offset_sgpr == 18 &&
ResolveInstance(ngg_instance) == 81,
"PS5 NGG instance-index offset was not detected or resolved");
const auto instance_conflict = Compile(
"EmbeddedFetchConflictingInstanceOffset",
MakeFetch({{17, 8}, {18, 8}}, {}, 8, true), 81);
const auto instance_malformed =
Compile("EmbeddedFetchMalformedInstanceOffset",
MakeFetch({{18, 7}}, {}, 7, true), 81);
const auto instance_outside =
Compile("EmbeddedFetchOutsideInstanceOffset",
MakeFetch({{19, 8}}, {}, 8, true), 81);
for (const auto *result :
{&instance_conflict, &instance_malformed, &instance_outside}) {
Require("EmbeddedFetchNggInstanceOffset", "fail closed",
result->program.info.instance_offset_sgpr == -1 &&
ResolveInstance(*result) == 0,
"conflicting, malformed, or out-of-window instance add was "
"classified");
}
const auto pointer = 0x5b7c5100u;
const auto late = Compile("EmbeddedFetchLateOffset",
MakeFetch({}, std::pair<u32, u32>{18, 0}), pointer);
const auto conflict = Compile("EmbeddedFetchConflictingOffset",
MakeFetch({{17, 0}, {18, 0}}), pointer);
const auto malformed =
Compile("EmbeddedFetchMalformedOffset", MakeFetch({{18, 1}}), pointer);
const auto outside =
Compile("EmbeddedFetchOutsideOffset", MakeFetch({{19, 0}}), pointer);
for (const auto *result : {&late, &conflict, &malformed, &outside}) {
Require("EmbeddedFetchVertexOffset", "fail closed",
result->program.info.vertex_offset_sgpr == -1 &&
Resolve(*result, 0) == 0,
"non-prolog, conflicting, malformed, or out-of-window add was "
"classified");
}
std::printf("[host] %-32s ok\n", "EmbeddedFetchVertexOffset");
}
[[noreturn]] void RunReverseRenderTargetDeathCase() {
(void)TextureGetRenderTargetFormat(Prospero::ChannelLayout::k16_16_16_16,
Prospero::ChannelType::kSrgb,
Prospero::ChannelOrder::kStandard);
std::_Exit(0x7f);
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckRenderTargetFormatContract() {
const auto r8_uint = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k8, Prospero::ChannelType::kUInt,
Prospero::ChannelOrder::kStandard);
Require("RenderTargetFormat", "R8 uint",
r8_uint.format == vk::Format::eR8Uint &&
r8_uint.bytes_per_element == 1u &&
r8_uint.export_mapping.IsIdentity(),
"R8 uint render-target tuple was rejected");
const auto rgb565 = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k5_6_5, Prospero::ChannelType::kUNorm,
Prospero::ChannelOrder::kStandard);
Require("RenderTargetFormat", "RGB565 UNorm",
rgb565.format == vk::Format::eB5G6R5UnormPack16 &&
rgb565.bytes_per_element == 2u &&
rgb565.export_mapping.IsIdentity(),
"RGB565 UNorm render-target tuple was rejected");
const auto uint_format = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kUInt,
Prospero::ChannelOrder::kStandard);
Require("RenderTargetFormat", "RGBA16 uint",
uint_format.format == vk::Format::eR16G16B16A16Uint &&
uint_format.bytes_per_element == 8u &&
uint_format.export_mapping.IsIdentity(),
"RGBA16 uint render-target tuple was rejected");
const auto format = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat,
Prospero::ChannelOrder::kReversed);
Require("ReverseRenderTarget", "exact format",
format.format == vk::Format::eR16G16B16A16Sfloat &&
format.bytes_per_element == 8u &&
format.export_mapping == Prospero::ColorMappingAbgr,
"exact reverse RGBA16F render-target tuple was rejected");
Require("ReverseRenderTarget", "write masks",
format.export_mapping.ApplyMask(0x1u) == 0x8u &&
format.export_mapping.ApplyMask(0x2u) == 0x4u &&
format.export_mapping.ApplyMask(0x4u) == 0x2u &&
format.export_mapping.ApplyMask(0x8u) == 0x1u &&
format.export_mapping.ApplyMask(0xfu) == 0xfu,
"reverse RGBA16F component mask was not mapped exactly once");
const auto rg32 = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k32_32, Prospero::ChannelType::kFloat,
Prospero::ChannelOrder::kReversed);
Require(
"ReverseRenderTarget", "GR32 float",
rg32.format == vk::Format::eR32G32Sfloat &&
rg32.bytes_per_element == 8u &&
rg32.export_mapping == Prospero::ColorMappingGr &&
rg32.export_mapping.ApplyMask(0x1u) == 0x2u &&
rg32.export_mapping.ApplyMask(0x2u) == 0x1u &&
rg32.export_mapping.ApplyMask(0x3u) == 0x3u &&
rg32.export_mapping.ApplyMask(0xfu) == 0xfu,
"reverse GR32F render-target export or write-mask mapping is incorrect");
const auto argb = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat,
Prospero::ChannelOrder::kAltReversed);
Require(
"ReverseRenderTarget", "ARGB float",
argb.format == vk::Format::eR16G16B16A16Sfloat &&
argb.export_mapping == Prospero::ColorMappingArgb &&
argb.export_mapping.ApplyMask(0x1u) == 0x2u &&
argb.export_mapping.ApplyMask(0x2u) == 0x4u &&
argb.export_mapping.ApplyMask(0x4u) == 0x8u &&
argb.export_mapping.ApplyMask(0x8u) == 0x1u,
"alternate-reversed render-target mapping did not invert its ARGB cycle");
char path[MAX_PATH]{};
Require("ReverseRenderTarget", "host",
GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
"GetModuleFileName failed");
std::string command = std::string("\"") + path + "\" --reverse-rt-death";
std::vector<char> mutable_command(command.begin(), command.end());
mutable_command.push_back('\0');
STARTUPINFOA startup{sizeof(startup)};
PROCESS_INFORMATION process{};
Require("ReverseRenderTarget", "host",
CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr,
FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup,
&process) != 0,
"CreateProcess failed");
Require("ReverseRenderTarget", "host",
WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0,
"unsupported adjacent render-target tuple timed out");
DWORD exit_code = 0;
const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0;
CloseHandle(process.hThread);
CloseHandle(process.hProcess);
Require("ReverseRenderTarget", "hard failure", exited && exit_code == 321,
"invalid render-target tuple did not retain the fatal guard");
std::printf("[host] %-32s ok\n", "RenderTargetFormat");
}
#endif
[[noreturn]] void RunImageViewDeathCase(const char *kind) {
if (std::strcmp(kind, "sampled-invalid-selector") == 0) {
(void)SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
DstSel(4, 5, 6, 2));
} else if (std::strcmp(kind, "sampled-incompatible-format") == 0) {
(void)SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR16Unorm,
DstSel(4, 0, 0, 1));
} else if (std::strcmp(kind, "sampled-invalid-high") == 0) {
(void)SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
DstSel(7, 6, 5, 3));
} else if (std::strcmp(kind, "storage-incompatible-format") == 0) {
ValidateStorageColorView(vk::Format::eR8G8B8A8Srgb,
vk::Format::eR16G16B16A16Unorm,
DstSel(4, 5, 6, 7));
} else if (std::strcmp(kind, "volume-mip-count") == 0 ||
std::strcmp(kind, "volume-slice-range") == 0) {
VulkanHarness vulkan;
auto &graphics = vulkan.RuntimeContext();
CommandScheduler scheduler(vulkan.RuntimeRenderer(), graphics);
ImageInfo volume_info{};
volume_info.pixel_format = vk::Format::eR8G8B8A8Unorm;
volume_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm;
volume_info.type = Prospero::ImageType::kColor3D;
volume_info.extent = {8, 8, 4};
volume_info.resources = {2, 1};
volume_info.pitch = 8;
volume_info.bytes_per_block = 4;
volume_info.samples = 1;
volume_info.tile_mode = Prospero::TileMode::kLinear;
Libs::Graphics::Image volume(graphics, scheduler, volume_info);
ImageViewInfo view{};
view.format = volume_info.pixel_format;
view.type = vk::ImageViewType::e2DArray;
view.aspect = vk::ImageAspectFlagBits::eColor;
view.base_level = 1;
view.level_count = 1;
view.layer_count = 1;
view.usage = vk::ImageUsageFlagBits::eSampled;
if (std::strcmp(kind, "volume-mip-count") == 0) {
view.base_level = 0;
view.level_count = 2;
} else {
view.base_layer = 1;
view.layer_count = 2;
}
(void)volume.FindView(view);
} else {
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.written = true;
if (std::strcmp(kind, "storage-kind") == 0) {
resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
} else if (std::strcmp(kind, "storage-no-write") == 0) {
resource.written = false;
} else if (std::strcmp(kind, "storage-nonuint-atomic") == 0) {
resource.atomic = true;
} else if (std::strcmp(kind, "storage-compare") == 0) {
resource.depth_compare = true;
} else if (std::strcmp(kind, "storage-dimension") == 0) {
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Unknown;
} else {
std::_Exit(0x7e);
}
ValidateStorageImageResource(resource);
}
std::_Exit(0x7f);
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckSampledColorViews() {
Require("SampledColorViews", "identity",
SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7),
"RGBA did not select the identity view");
uint32_t valid_swizzles = 0;
for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) {
bool expected = true;
for (uint32_t channel = 0; channel < 4; channel++) {
const auto selector = GetDstSel(swizzle, channel);
if (selector == 2 || selector == 3) {
expected = false;
}
}
const bool valid = IsValidImageSwizzle(swizzle);
const bool supported = IsSupportedSampledColorView(
vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, swizzle);
Require("SampledColorViews", "exhaustive read swizzle domain",
valid == expected && supported == expected,
"sampled swizzle validator disagreed with the PS5 selector domain");
valid_swizzles += valid;
}
Require("SampledColorViews", "all PS5 read swizzles",
valid_swizzles == 1296 && !IsValidImageSwizzle(DstSel(4, 5, 6, 2)) &&
!IsValidImageSwizzle(DstSel(4, 5, 6, 3)) &&
!IsValidImageSwizzle(0x1000),
"valid PS5 sampled mappings were rejected or reserved selectors were "
"admitted");
const auto arbitrary = DstSel(5, 1, 7, 0);
const auto components = TextureGetComponentMapping(arbitrary, {});
Require("SampledColorViews", "generic Vulkan component mapping",
SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
arbitrary) == arbitrary &&
components.r == vk::ComponentSwizzle::eG &&
components.g == vk::ComponentSwizzle::eOne &&
components.b == vk::ComponentSwizzle::eA &&
components.a == vk::ComponentSwizzle::eZero,
"arbitrary valid sampled mapping did not use the generic view path");
Require("SampledColorViews", "R8 R001",
SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR8Unorm,
DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1),
"R8 did not select its R001 view");
Require("SampledColorViews", "R8 000R",
SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR8Unorm,
DstSel(0, 0, 0, 4)) == DstSel(0, 0, 0, 4),
"R8 did not select its 000R component-mapped view");
Require("SampledColorViews", "mutable R8 uint/unorm 000R",
SelectSampledColorView(vk::Format::eR8Uint, vk::Format::eR8Unorm,
DstSel(0, 0, 0, 4)) == DstSel(0, 0, 0, 4),
"compatible R8 integer target sampled view was rejected");
Require("SampledColorViews", "R16G16 RG01",
SelectSampledColorView(vk::Format::eR16G16Sfloat,
vk::Format::eR16G16Sfloat,
DstSel(4, 5, 0, 1)) == DstSel(4, 5, 0, 1),
"R16G16 did not select its RG01 view");
Require("SampledColorViews", "alpha one",
SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
DstSel(4, 5, 6, 1)) == DstSel(4, 5, 6, 1),
"RGB1 did not select the alpha-one view");
Require("SampledColorViews", "mutable BGRA target",
SelectSampledColorView(vk::Format::eB8G8R8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
"BGRA target did not select the exact RGBA/BGRA mutable view");
Require(
"SampledColorViews", "mutable sRGB view of UNORM BGRA target",
SelectSampledColorView(vk::Format::eB8G8R8A8Unorm,
vk::Format::eR8G8B8A8Srgb,
DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
"UNORM BGRA target did not select its compatible sRGB RGBA sampled view");
Require("SampledColorViews", "mutable sRGB BGRA target",
SelectSampledColorView(vk::Format::eB8G8R8A8Srgb,
vk::Format::eR8G8B8A8Srgb,
DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
"sRGB BGRA target did not select its matching mutable RGBA view");
constexpr auto colorspace_swizzle = DstSel(5, 1, 7, 0);
Require("SampledColorViews", "mutable sRGB/UNORM views",
SelectSampledColorView(vk::Format::eR8G8B8A8Srgb,
vk::Format::eR8G8B8A8Unorm,
DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) &&
SelectSampledColorView(vk::Format::eB8G8R8A8Unorm,
vk::Format::eB8G8R8A8Srgb,
colorspace_swizzle) == colorspace_swizzle,
"same-order mutable sRGB/UNORM sampled views were rejected");
Require("SampledColorViews", "mutable packed RGB10 view",
SelectSampledColorView(vk::Format::eA2R10G10B10UnormPack32,
vk::Format::eA2B10G10R10UnormPack32,
DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
"packed RGB10 target did not select its matching mutable "
"channel-order view");
Require(
"SampledColorViews", "reverse RGBA16F sampled view",
SelectSampledColorView(vk::Format::eR16G16B16A16Sfloat,
vk::Format::eR16G16B16A16Sfloat,
DstSel(7, 6, 5, 4)) == DstSel(7, 6, 5, 4),
"reverse RGBA16F target did not select its reciprocal ABGR sampled view");
Require("SampledColorViews", "mutable integer-class views",
SelectSampledColorView(vk::Format::eR16G16B16A16Sfloat,
vk::Format::eR16G16B16A16Uint,
DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) &&
SelectSampledColorView(
vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Uint,
DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) &&
SelectSampledColorView(vk::Format::eR8G8B8A8Uint,
vk::Format::eR8G8B8A8Unorm,
DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7),
"compatible integer render-target sampled view was rejected");
Require("SampledColorViews", "invalid depth format",
!IsSupportedSampledDepthView(vk::Format::eD24UnormS8Uint,
vk::Format::eR8Unorm,
DstSel(4, 4, 4, 4)),
"incompatible sampled depth format was accepted");
Require("SampledColorViews", "invalid depth swizzle",
!IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
vk::Format::eR32Sfloat,
DstSel(4, 5, 6, 7)),
"incompatible sampled depth swizzle was accepted");
Require("SampledColorViews", "D32 depth target",
IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
vk::Format::eR32Sfloat,
DstSel(4, 4, 4, 4)),
"D32 depth target did not select its depth-aspect view");
Require("SampledColorViews", "D16 R000 depth target",
IsSupportedSampledDepthView(
vk::Format::eD16Unorm, vk::Format::eR16Unorm, DstSel(4, 0, 0, 0)),
"D16 depth target did not select its R000 depth-aspect view");
Require("SampledColorViews", "D16 RRR1 depth target",
IsSupportedSampledDepthView(
vk::Format::eD16Unorm, vk::Format::eR16Unorm, DstSel(4, 4, 4, 1)),
"D16 depth target rejected replicated depth with constant alpha");
Require("SampledColorViews", "D16S8 R001 depth target",
IsSupportedSampledDepthView(vk::Format::eD16UnormS8Uint,
vk::Format::eR16Unorm,
DstSel(4, 0, 0, 1)),
"D16S8 depth target did not select its R001 depth-aspect view");
Require("SampledColorViews", "promoted D24S8 R001 depth target",
IsSupportedSampledDepthView(vk::Format::eD24UnormS8Uint,
vk::Format::eR16Unorm,
DstSel(4, 0, 0, 1)),
"D24S8 host fallback did not preserve the guest R16 depth view");
Require("SampledColorViews", "promoted D32S8 R001 depth target",
IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
vk::Format::eR16Unorm,
DstSel(4, 0, 0, 1)),
"D32S8 host fallback did not preserve the guest R16 depth view");
Require("SampledColorViews", "D32S8 R001 depth target",
IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
vk::Format::eR32Sfloat,
DstSel(4, 0, 0, 1)),
"D32S8 depth target did not select its R001 depth-aspect view");
ShaderRecompiler::IR::ImageResource storage_resource{};
storage_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Storage;
storage_resource.numeric_class = Prospero::TextureNumericClass::Float;
storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
storage_resource.written = true;
Require("SampledColorViews", "storage resource",
IsSupportedStorageImageResource(storage_resource),
"exact storage resource contract was rejected");
storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
storage_resource.read = true;
Require("SampledColorViews", "read-write 3D storage resource",
IsSupportedStorageImageResource(storage_resource),
"basic read-write 3D storage resource was rejected");
storage_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
storage_resource.read = false;
Require("SampledColorViews", "write-only 2D-array storage resource",
IsSupportedStorageImageResource(storage_resource),
"basic write-only 2D-array storage resource was rejected");
storage_resource.numeric_class = Prospero::TextureNumericClass::Uint;
Require("SampledColorViews", "write-only uint 2D-array storage resource",
IsSupportedStorageImageResource(storage_resource),
"basic write-only uint 2D-array storage resource was rejected");
storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
storage_resource.read = true;
storage_resource.atomic = true;
Require("SampledColorViews", "atomic uint 2D storage resource",
IsSupportedStorageImageResource(storage_resource),
"atomic uint storage resource was rejected");
storage_resource.atomic = false;
storage_resource.mip_mode =
ShaderRecompiler::IR::ImageMipMode::Dynamic;
storage_resource.mip_count = 3;
Require("SampledColorViews", "dynamic storage mip resource",
IsSupportedStorageImageResource(storage_resource),
"dynamic storage mip resource was rejected");
char path[MAX_PATH]{};
Require("SampledColorViews", "host",
GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
"GetModuleFileName failed");
for (const char *kind :
{"sampled-invalid-selector", "sampled-incompatible-format",
"sampled-invalid-high", "storage-incompatible-format", "storage-kind",
"storage-no-write", "storage-nonuint-atomic", "storage-compare",
"storage-dimension", "volume-mip-count", "volume-slice-range"}) {
std::string command =
std::string("\"") + path + "\" --image-view-death " + kind;
std::vector<char> mutable_command(command.begin(), command.end());
mutable_command.push_back('\0');
STARTUPINFOA startup{sizeof(startup)};
PROCESS_INFORMATION process{};
Require("SampledColorViews", "host",
CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr,
FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup,
&process) != 0,
"CreateProcess failed");
Require("SampledColorViews", "host",
WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0,
"unsupported view death case timed out");
DWORD exit_code = 0;
const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0;
CloseHandle(process.hThread);
CloseHandle(process.hProcess);
Require("SampledColorViews", "host", exited && exit_code == 321,
std::string(kind) +
" component mapping did not report a fatal error");
}
std::printf("[host] %-32s ok\n", "SampledColorRenderTargetViews");
}
#endif
void CheckSampledDepthResource() {
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = true;
resource.depth_compare = true;
Require("SampledDepthResource", "comparison read",
IsSupportedSampledDepthResource(resource),
"basic comparison-sampled depth resource was rejected");
resource.depth_compare = false;
Require("SampledDepthResource", "ordinary read",
IsSupportedSampledDepthResource(resource),
"basic non-comparison depth read was rejected");
const auto basic = resource;
resource.read = false;
Require("SampledDepthResource", "read required",
!IsSupportedSampledDepthResource(resource),
"non-reading depth resource was accepted");
resource = basic;
resource.written = true;
Require("SampledDepthResource", "write rejected",
!IsSupportedSampledDepthResource(resource),
"writable depth resource was accepted");
resource = basic;
resource.atomic = true;
Require("SampledDepthResource", "atomic rejected",
!IsSupportedSampledDepthResource(resource),
"atomic depth resource was accepted");
resource = basic;
resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::Dynamic;
resource.mip_count = 3;
Require("SampledDepthResource", "dynamic mip read",
IsSupportedSampledDepthResource(resource),
"dynamic sampled depth mip resource was rejected");
resource.depth_compare = true;
Require("SampledDepthResource", "dynamic comparison mip read",
IsSupportedSampledDepthResource(resource),
"dynamic comparison depth mip resource was rejected");
resource = basic;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
Require("SampledDepthResource", "singleton array accepted",
IsSupportedSampledDepthResource(resource),
"array depth resource was rejected");
resource = basic;
resource.numeric_class = Prospero::TextureNumericClass::Uint;
Require("SampledDepthResource", "integer read",
IsSupportedSampledDepthResource(resource),
"read-only uint depth resource was rejected");
resource.depth_compare = true;
Require("SampledDepthResource", "integer comparison rejected",
!IsSupportedSampledDepthResource(resource),
"uint depth resource accepted comparison sampling");
Require("SampledDepthResource", "R32 uint depth view",
IsSupportedSampledDepthFormat(vk::Format::eD32Sfloat,
vk::Format::eR32Uint) &&
!IsSupportedSampledDepthFormat(vk::Format::eR32Uint,
vk::Format::eR32Uint),
"R32 uint depth-view compatibility diverged from its backing role");
std::printf("[host] %-32s ok\n", "SampledDepthResource");
}
void CheckImageTransitionState(RenderContext &renderer) {
constexpr const char *name = "ImageTransitionState";
auto &context = renderer.GetGraphics();
CommandScheduler scheduler(renderer, context);
const auto MakeInfo = [](vk::Format format, uint32_t levels,
uint32_t layers) {
ImageInfo info{};
info.pixel_format = format;
info.guest_format = Prospero::BufferFormat::k8UNorm;
info.type = Prospero::ImageType::kColor2D;
info.extent = {4, 4, 1};
info.resources = {levels, layers};
info.pitch = 4;
info.bytes_per_block = 1;
info.samples = 1;
info.tile_mode = Prospero::TileMode::kLinear;
return info;
};
const auto graphics_stage = vk::PipelineStageFlagBits2::eAllGraphics |
vk::PipelineStageFlagBits2::eComputeShader;
constexpr uint64_t copy_capacity = 128ull << 20;
Require(
name, "buffered-copy capacity bands",
ImageTestAccess::CopyRows(32ull << 10, 4096, copy_capacity) == 4096 &&
ImageTestAccess::CopyRows(32ull << 10, 4097, copy_capacity) == 4096 &&
ImageTestAccess::CopyRows(copy_capacity + 4, 1, copy_capacity) == 0,
"buffered image copy does not split at the fixed scratch capacity");
Image image(context, scheduler, MakeInfo(vk::Format::eR8Unorm, 2, 3));
auto barriers =
image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, graphics_stage, {});
Require(
name, "initial full state",
barriers.size() == 1 &&
barriers[0].srcStageMask ==
vk::PipelineStageFlagBits2::eAllCommands &&
barriers[0].srcAccessMask == vk::AccessFlagBits2::eNone &&
barriers[0].dstStageMask == graphics_stage &&
barriers[0].dstAccessMask == vk::AccessFlagBits2::eShaderRead &&
barriers[0].oldLayout == vk::ImageLayout::eUndefined &&
barriers[0].newLayout == vk::ImageLayout::eShaderReadOnlyOptimal &&
barriers[0].subresourceRange.aspectMask ==
vk::ImageAspectFlagBits::eColor &&
barriers[0].subresourceRange.levelCount == VK_REMAINING_MIP_LEVELS &&
barriers[0].subresourceRange.layerCount ==
VK_REMAINING_ARRAY_LAYERS &&
image.backing.state.layout ==
vk::ImageLayout::eShaderReadOnlyOptimal &&
image.backing.state.access_mask == vk::AccessFlagBits2::eShaderRead &&
image.backing.state.pl_stage == graphics_stage,
"initial transition did not record or retain pinned state");
Require(name, "repeat read elision",
image
.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, graphics_stage, {})
.empty(),
"identical read-only transition emitted a barrier");
barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eTransferRead,
graphics_stage, {});
Require(name, "access-only transition",
barriers.size() == 1 &&
barriers[0].oldLayout == barriers[0].newLayout &&
barriers[0].srcAccessMask == vk::AccessFlagBits2::eShaderRead,
"same-layout access change was lost");
const auto CheckRepeatedWrite = [&](vk::AccessFlagBits2 access,
const char *label) {
image.backing.state = {};
image.backing.subresource_states.clear();
(void)image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage,
{});
Require(
name, label,
image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage, {})
.size() == 1,
"pinned repeated-write dependency was elided");
};
CheckRepeatedWrite(vk::AccessFlagBits2::eTransferWrite,
"repeat transfer write");
CheckRepeatedWrite(vk::AccessFlagBits2::eShaderWrite, "repeat shader write");
CheckRepeatedWrite(vk::AccessFlagBits2::eMemoryWrite, "repeat memory write");
image.backing.state = {};
image.backing.subresource_states.clear();
const auto attachment_access = vk::AccessFlagBits2::eColorAttachmentRead |
vk::AccessFlagBits2::eColorAttachmentWrite;
(void)image.GetBarriers(vk::ImageLayout::eColorAttachmentOptimal,
attachment_access, graphics_stage, {});
Require(name, "pinned attachment-write quirk",
image
.GetBarriers(vk::ImageLayout::eColorAttachmentOptimal,
attachment_access, graphics_stage, {})
.empty(),
"attachment writes diverged from the pinned repeated-write set");
image.backing.state = {};
image.backing.subresource_states.clear();
(void)image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, graphics_stage, {});
constexpr ImageSubresourceRange mip1_layer0{1, 1, 0, 1};
barriers = image.GetBarriers(vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderWrite,
graphics_stage, mip1_layer0);
Require(name, "first partial",
barriers.size() == 1 &&
barriers[0].subresourceRange.baseMipLevel == 1 &&
barriers[0].subresourceRange.baseArrayLayer == 0 &&
image.backing.subresource_states.size() == 6 &&
image.backing.subresource_states[3].layout ==
vk::ImageLayout::eGeneral &&
image.backing.subresource_states[3].access_mask ==
vk::AccessFlagBits2::eShaderWrite &&
image.backing.subresource_states[1].layout ==
vk::ImageLayout::eShaderReadOnlyOptimal,
"partial state did not use mip-major indexing");
constexpr ImageSubresourceRange mip0_layer1{0, 1, 1, 1};
barriers = image.GetBarriers(
vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
vk::PipelineStageFlagBits2::eTransfer, mip0_layer1);
Require(name, "second partial",
barriers.size() == 1 &&
barriers[0].subresourceRange.baseMipLevel == 0 &&
barriers[0].subresourceRange.baseArrayLayer == 1 &&
image.backing.subresource_states[1].layout ==
vk::ImageLayout::eTransferDstOptimal &&
image.backing.subresource_states[1].access_mask ==
vk::AccessFlagBits2::eTransferWrite &&
image.backing.subresource_states[3].layout ==
vk::ImageLayout::eGeneral,
"second partial transition changed the wrong cell");
const ImageSubresourceRange explicit_full{0, 2, 0, 3};
barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, graphics_stage,
explicit_full);
bool saw_mip0_layer1 = false;
bool saw_mip1_layer0 = false;
for (const auto &barrier : barriers) {
saw_mip0_layer1 |=
barrier.subresourceRange.baseMipLevel == 0 &&
barrier.subresourceRange.baseArrayLayer == 1 &&
barrier.oldLayout == vk::ImageLayout::eTransferDstOptimal &&
barrier.srcStageMask == vk::PipelineStageFlagBits2::eTransfer;
saw_mip1_layer0 |=
barrier.subresourceRange.baseMipLevel == 1 &&
barrier.subresourceRange.baseArrayLayer == 0 &&
barrier.oldLayout == vk::ImageLayout::eGeneral &&
barrier.srcAccessMask == vk::AccessFlagBits2::eShaderWrite;
}
Require(name, "partial normalization",
barriers.size() == 2 && saw_mip0_layer1 && saw_mip1_layer0 &&
image.backing.subresource_states.empty() &&
image.backing.state.layout ==
vk::ImageLayout::eShaderReadOnlyOptimal &&
image.backing.state.access_mask ==
vk::AccessFlagBits2::eShaderRead &&
image.backing.state.pl_stage == graphics_stage,
"full transition did not reconcile and collapse split state");
barriers = image.GetBarriers(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead,
vk::PipelineStageFlagBits2::eTransfer, {});
Require(name, "normalized full transition",
barriers.size() == 1 &&
barriers[0].subresourceRange.levelCount ==
VK_REMAINING_MIP_LEVELS &&
barriers[0].subresourceRange.layerCount ==
VK_REMAINING_ARRAY_LAYERS &&
barriers[0].srcStageMask == graphics_stage &&
barriers[0].dstStageMask == vk::PipelineStageFlagBits2::eTransfer,
"normalized state did not return to one full barrier");
auto depth_info = MakeInfo(vk::Format::eD32SfloatS8Uint, 1, 1);
depth_info.guest_format = Prospero::BufferFormat::k32Float;
depth_info.bytes_per_block = 4;
depth_info.tile_mode = Prospero::TileMode::kDepth;
Image depth(context, scheduler, depth_info);
barriers =
depth.GetBarriers(vk::ImageLayout::eDepthStencilReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, graphics_stage, {});
Require(name, "depth stencil aspects",
barriers.size() == 1 && barriers[0].subresourceRange.aspectMask ==
(vk::ImageAspectFlagBits::eDepth |
vk::ImageAspectFlagBits::eStencil),
"depth/stencil transition did not use the full image aspect");
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
image.backing.state = {};
image.backing.subresource_states.clear();
image.Transit(vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eTransferRead,
{}, scheduler.Current().Handle());
Require(name, "mixed access stages",
image.backing.state.pl_stage ==
(graphics_stage | vk::PipelineStageFlagBits2::eTransfer),
"mixed shader/transfer access omitted a required pipeline stage");
auto source_info = depth_info;
source_info.extent = {2, 2, 1};
source_info.pitch = 2;
source_info.stencil = {0x1000, 4};
auto destination_info = depth_info;
destination_info.resources.levels = 2;
destination_info.stencil = {0x2000, 16};
Image source(context, scheduler, source_info);
Image destination(context, scheduler, destination_info);
StreamBuffer upload(context, scheduler, MemoryUsage::Upload, 4096);
StreamBuffer download(context, scheduler, MemoryUsage::Download, 4096);
const auto [upload_data, upload_offset] = upload.Map(128, 4);
Require(name, "depth/stencil upload map", upload_data != nullptr,
"combined depth/stencil upload allocation failed");
constexpr std::array<uint32_t, 4> expected_depth{0x3e800000u, 0x3f000000u,
0x3f400000u, 0x3f800000u};
constexpr std::array<uint8_t, 4> expected_stencil{0x12, 0x34, 0x56, 0x78};
std::memcpy(upload_data, expected_depth.data(), sizeof(expected_depth));
std::memcpy(upload_data + 64, expected_stencil.data(),
sizeof(expected_stencil));
upload.Commit();
std::array<vk::BufferImageCopy, 2> source_copies{};
source_copies[0].bufferOffset = upload_offset;
source_copies[0].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0,
1};
source_copies[0].imageExtent = {2, 2, 1};
source_copies[1].bufferOffset = upload_offset + 64;
source_copies[1].imageSubresource = {vk::ImageAspectFlagBits::eStencil, 0, 0,
1};
source_copies[1].imageExtent = {2, 2, 1};
source.Upload(source_copies, upload.Handle(), upload_offset, 128);
destination.CopyMip(source, 1, 0);
const auto [download_data, download_offset] = download.Map(128, 4);
Require(name, "depth/stencil download map", download_data != nullptr,
"combined depth/stencil download allocation failed");
download.Commit();
auto destination_copies = source_copies;
destination_copies[0].bufferOffset = download_offset;
destination_copies[0].imageSubresource.mipLevel = 1;
destination_copies[1].bufferOffset = download_offset + 64;
destination_copies[1].imageSubresource.mipLevel = 1;
destination.Download(destination_copies, download.Handle(), download_offset,
128);
auto buffered_info = MakeInfo(vk::Format::eR8Unorm, 1, 1);
buffered_info.extent = {8, 4, 1};
buffered_info.pitch = 8;
buffered_info.mip_layout[0] = {0, 32, 8, 4};
Image buffered_source(context, scheduler, buffered_info);
Image buffered_destination(context, scheduler, buffered_info);
constexpr std::array<uint8_t, 32> buffered_expected{
0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa,
0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x21, 0x32, 0x43, 0x54, 0x65,
0x76, 0x87, 0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f};
const auto [buffered_upload_data, buffered_upload_offset] =
upload.Map(buffered_expected.size(), 4);
Require(name, "buffered-copy upload map", buffered_upload_data != nullptr,
"buffered-copy source allocation failed");
std::memcpy(buffered_upload_data, buffered_expected.data(),
buffered_expected.size());
upload.Commit();
vk::BufferImageCopy buffered_region{};
buffered_region.bufferOffset = buffered_upload_offset;
buffered_region.bufferRowLength = buffered_info.pitch;
buffered_region.bufferImageHeight = buffered_info.extent.height;
buffered_region.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1};
buffered_region.imageExtent = buffered_info.extent;
buffered_source.Upload(std::span{&buffered_region, 1}, upload.Handle(),
buffered_upload_offset, buffered_expected.size());
Buffer copy_scratch(context, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
16);
StreamBuffer copy_parameters(context, scheduler, MemoryUsage::Stream, 4096);
TileManager copy_tiler(context, scheduler, copy_parameters);
buffered_destination.CopyImageWithBuffer(buffered_source, copy_scratch, copy_tiler);
const auto [buffered_download_data, buffered_download_offset] =
download.Map(buffered_expected.size(), 4);
Require(name, "buffered-copy download map", buffered_download_data != nullptr,
"buffered-copy destination allocation failed");
download.Commit();
buffered_region.bufferOffset = buffered_download_offset;
buffered_destination.Download(std::span{&buffered_region, 1},
download.Handle(), buffered_download_offset,
buffered_expected.size());
scheduler.Finish();
download.Invalidate(download_offset, 128);
download.Invalidate(buffered_download_offset, buffered_expected.size());
Require(name, "depth/stencil mip contents",
std::memcmp(download_data, expected_depth.data(),
sizeof(expected_depth)) == 0 &&
std::memcmp(download_data + 64, expected_stencil.data(),
sizeof(expected_stencil)) == 0,
"combined depth/stencil mip copy lost an aspect");
Require(name, "buffered-copy chunk contents",
std::memcmp(buffered_download_data, buffered_expected.data(),
buffered_expected.size()) == 0,
"real buffered image copy lost a row across its scratch boundary");
std::printf("[host] %-32s ok\n", name);
}
void CheckDepthTextureEncoding() {
const ShaderTextureResource disabled_sampler_tweaks{{
0x05135600u,
0xc1600000u,
0x010dc1dfu,
0x91800924u,
0x00000000u,
0x00000000u,
0x00000000u,
0x00000000u,
}};
Require("DepthTextureEncoding", "disabled sampler tweaks",
disabled_sampler_tweaks.PerfMod5() == 0 &&
IsSupportedDepthTextureEncoding(disabled_sampler_tweaks),
"valid sampler modulation factor zero was rejected");
const ShaderTextureResource uncompressed_msaa{{
0x00705d00u,
0xc1600000u,
0x010dc1dfu,
0xe1810924u,
0x00000000u,
0x00700010u,
0x00000000u,
0x00000000u,
}};
Require("DepthTextureEncoding", "uncompressed 2x MSAA depth",
IsSupportedDepthTextureEncoding(uncompressed_msaa),
"valid uncompressed MSAA depth descriptor required an HTILE "
"compatibility flag");
auto r128_msaa = uncompressed_msaa;
std::fill(r128_msaa.fields + 4, r128_msaa.fields + 8, 0u);
Require("DepthTextureEncoding", "R128 uncompressed 2x MSAA depth",
IsSupportedDepthTextureEncoding(r128_msaa, true),
"valid R128 MSAA depth descriptor required omitted dwords");
const ShaderTextureResource compressed_descriptor{{
0x0104d500u,
0xc1600000u,
0x021bc3bfu,
0x91800924u,
0x00000000u,
0x00700000u,
0x80280000u,
0x000106d4u,
}};
Require("DepthTextureEncoding", "compressed HTILE descriptor",
IsSupportedDepthTextureEncoding(compressed_descriptor),
"valid compressed depth descriptor required a prior depth target");
const ShaderTextureResource disabled_metadata{{
0x20018100u, 0xc1600000u, 0x021bc1dfu, 0x91800204u,
0x00000000u, 0x00700000u, 0x00000000u, 0x0020037fu,
}};
auto retained_metadata_address = compressed_descriptor;
retained_metadata_address.fields[6] &= 0xff000000u;
Require("DepthTextureEncoding", "disabled metadata with retained address",
IsSupportedDepthTextureEncoding(disabled_metadata) &&
IsSupportedDepthTextureEncoding(retained_metadata_address),
"uncompressed depth rejected inactive metadata address bits");
const ShaderTextureResource first_use_depth{{
0x0225fc00u, 0x01600000u, 0x00000000u, 0x91800924u,
0x00000000u, 0x00700000u, 0x80280000u, 0x000225fdu,
}};
Require("DepthTextureEncoding", "PPSA30490 first-use 1x1 depth",
IsSupportedDepthTextureEncoding(first_use_depth),
"captured comparison texture required tracked HTILE metadata");
auto missing_metadata = compressed_descriptor;
missing_metadata.fields[6] &= 0x00ffffffu;
missing_metadata.fields[7] = 0;
auto out_of_range_metadata = compressed_descriptor;
out_of_range_metadata.fields[7] = TRACKER_ADDRESS_SIZE >> 16u;
auto unaligned_metadata = compressed_descriptor;
unaligned_metadata.fields[6] ^= 1u << 24u;
auto dcc_only_control = compressed_descriptor;
dcc_only_control.fields[6] |= 1u << 22u;
auto non_msaa_iterate_256 = compressed_descriptor;
non_msaa_iterate_256.fields[6] |= 1u << 10u;
Require(
"DepthTextureEncoding", "invalid HTILE encoding",
!IsSupportedDepthTextureEncoding(missing_metadata) &&
!IsSupportedDepthTextureEncoding(out_of_range_metadata) &&
!IsSupportedDepthTextureEncoding(unaligned_metadata) &&
!IsSupportedDepthTextureEncoding(dcc_only_control) &&
!IsSupportedDepthTextureEncoding(non_msaa_iterate_256),
"compressed sampled depth accepted an unsupported address or control");
std::printf("[host] %-32s ok\n", "DepthTextureEncoding");
}
ShaderRecompiler::IR::ImageResource BasicStorageTextureResource() {
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
resource.read = true;
resource.written = true;
return resource;
}
ShaderTextureResource BasicStorageTextureDescriptor() {
return {{0x00785d00u, 0x04700000u, 0x00080008u, 0xa1b00facu, 0x00000020u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource BasicLinearStorageTextureResource() {
auto resource = BasicStorageTextureResource();
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = false;
return resource;
}
ShaderTextureResource BasicLinearStorageTextureDescriptor() {
return {{0x04bcc401u, 0xc3800000u, 0x021bc3bfu, 0x900003acu, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource BasicBgraStorageTextureResource() {
auto resource = BasicStorageTextureResource();
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = false;
return resource;
}
ShaderTextureResource BasicBgraStorageTextureDescriptor() {
return {{0x007c6500u, 0xc3800000u, 0x010dc1dfu, 0x91b00f2eu, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa06228R11G11B10StorageTextureDescriptor() {
return {{0x10c6b500u, 0xc2400000u, 0x010dc1dfu, 0x91b003acu, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa01530MaxMipStorageTextureDescriptor() {
return {{0x04a42900u, 0xc3e00000u, 0x000fc00fu, 0x91b0022cu, 0x00000000u,
0x00700050u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa01340MipRangeStorageTextureDescriptor() {
return {{0x0294dc00u, 0xc4700000u, 0x00b3c13fu, 0x91b31facu, 0x00000000u,
0x00700030u, 0xb07b0000u, 0x0002ac3cu}};
}
ShaderTextureResource Ppsa01340OverwideMipStorageTextureDescriptor() {
return {{0x0346c000u, 0xc4b00000u, 0x007fc07fu, 0x90990facu, 0x00000000u,
0x00700080u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource Ppsa01530MaxMipStorageTextureResource() {
auto resource = BasicBgraStorageTextureResource();
resource.numeric_class = Prospero::TextureNumericClass::Uint;
return resource;
}
ShaderTextureResource Ppsa02527R16FloatStorageTextureDescriptor() {
return {{0x00ce3500u, 0xc0d00000u, 0x010dc1dfu, 0x91b00204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa02527R32FloatStorageTextureDescriptor() {
return {{0x00cea900u, 0xc1600000u, 0x0086c0efu, 0x91b00204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa02527R8UnormStorageTextureDescriptor() {
return {{0x00c7d500u, 0xc0100000u, 0x0086c0efu, 0x91b00204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource BasicYzwxStorageTextureDescriptor() {
return {{0x00627801u, 0xc4d00000u, 0x0001c001u, 0x900009f5u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource BasicArrayStorageTextureResource() {
auto resource = BasicLinearStorageTextureResource();
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
return resource;
}
ShaderTextureResource BasicArrayStorageTextureDescriptor() {
return {{0x20179000u, 0x03800000u, 0x00000000u, 0xd1b00f2eu, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource BasicUintArrayStorageTextureResource() {
auto resource = BasicArrayStorageTextureResource();
resource.numeric_class = Prospero::TextureNumericClass::Uint;
return resource;
}
ShaderTextureResource BasicUintArrayStorageTextureDescriptor() {
return {{0x20179200u, 0x01400000u, 0x00000000u, 0xd1b00204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Standard4KBUintArrayStorageTextureDescriptor() {
return {{0x006c6600u, 0x01400000u, 0x00000000u, 0xd0500204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Standard64KBStorageTextureDescriptor() {
return {{0x011fab00u, 0xc3800000u, 0x0003c003u, 0xd0900facu, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource Ppsa14053DepthTileStorageTextureResource() {
return BasicUintArrayStorageTextureResource();
}
ShaderTextureResource Ppsa14053DepthTileStorageTextureDescriptor() {
return {{0x20144c00u, 0x00500000u, 0x00000000u, 0xd1800204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa10112D16StorageTextureDescriptor() {
return {{0x205b9000u, 0xc0700000u, 0x0021803bu, 0xd1800204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource BasicUintVolumeStorageTextureResource() {
auto resource = BasicStorageTextureResource();
resource.numeric_class = Prospero::TextureNumericClass::Uint;
resource.read = false;
return resource;
}
ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() {
return {{0x20180600u, 0xc0b00000u, 0x0003c003u, 0xa0000004u, 0x0000000fu,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource AtomicStorageTextureResource() {
auto resource = BasicLinearStorageTextureResource();
resource.numeric_class = Prospero::TextureNumericClass::Uint;
resource.read = true;
resource.atomic = true;
return resource;
}
ShaderTextureResource AtomicStorageTextureDescriptor() {
return {{0x304bb700u, 0xc1400000u, 0x0000001fu, 0x91b00204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
[[noreturn]] void RunStorageTextureDescriptorDeathCase(const char *kind) {
auto resource = BasicStorageTextureResource();
auto descriptor = BasicStorageTextureDescriptor();
if (std::strcmp(kind, "linear-rgb1-read") == 0) {
resource = BasicLinearStorageTextureResource();
descriptor = BasicLinearStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "bgra-read") == 0) {
resource = BasicBgraStorageTextureResource();
descriptor = BasicBgraStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "r16-float-read") == 0) {
resource = BasicBgraStorageTextureResource();
descriptor = Ppsa02527R16FloatStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "r8-unorm-read") == 0) {
resource = BasicBgraStorageTextureResource();
descriptor = Ppsa02527R8UnormStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "yzwx-read") == 0) {
resource = BasicLinearStorageTextureResource();
descriptor = BasicYzwxStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "reserved-swizzle") == 0) {
resource = BasicLinearStorageTextureResource();
descriptor = BasicLinearStorageTextureDescriptor();
descriptor.fields[3] =
(descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 2);
} else if (std::strcmp(kind, "resource") == 0) {
resource.written = false;
} else if (std::strcmp(kind, "type") == 0) {
descriptor.fields[3] =
(descriptor.fields[3] & 0x0fffffffu) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
} else if (std::strcmp(kind, "standard256b-volume") == 0) {
descriptor.fields[3] =
(descriptor.fields[3] & ~(0x1fu << 20u)) |
(static_cast<uint32_t>(Prospero::TileMode::kStandard256B) << 20u);
} else if (std::strcmp(kind, "inverted-mip-range") == 0) {
descriptor.fields[3] |= 1u << 12u;
descriptor.fields[5] |= 1u << 4u;
} else if (std::strcmp(kind, "swizzle") == 0) {
descriptor.fields[3] =
(descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 1);
} else if (std::strcmp(kind, "array-base-out-of-range") == 0) {
resource = BasicArrayStorageTextureResource();
descriptor = BasicArrayStorageTextureDescriptor();
descriptor.fields[4] |= 1u << 16u;
} else if (std::strcmp(kind, "reserved") == 0) {
descriptor.fields[1] |= 1u << 29u;
} else if (std::strcmp(kind, "uint-format") == 0) {
descriptor.fields[1] =
(descriptor.fields[1] & ~0x1ff00000u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8UInt) << 20u);
} else if (std::strcmp(kind, "uint-resource-float-format") == 0) {
resource = BasicUintArrayStorageTextureResource();
descriptor = BasicArrayStorageTextureDescriptor();
} else if (std::strcmp(kind, "atomic-format") == 0) {
resource = AtomicStorageTextureResource();
descriptor = AtomicStorageTextureDescriptor();
descriptor.fields[1] =
(descriptor.fields[1] & ~0x1ff00000u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k8UInt) << 20u);
} else if (std::strcmp(kind, "depth-tile-read") == 0) {
resource = Ppsa14053DepthTileStorageTextureResource();
descriptor = Ppsa14053DepthTileStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "depth-tile-extent") == 0) {
resource = Ppsa14053DepthTileStorageTextureResource();
descriptor = Ppsa14053DepthTileStorageTextureDescriptor();
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
descriptor.fields[3] =
(descriptor.fields[3] & 0x0fffffffu) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
descriptor.fields[4] |= 1u;
} else if (std::strcmp(kind, "depth-tile-fmask") == 0) {
resource = BasicArrayStorageTextureResource();
descriptor = Ppsa10112D16StorageTextureDescriptor();
descriptor.fields[1] =
(descriptor.fields[1] & ~(0x1ffu << 20u)) |
(static_cast<uint32_t>(Prospero::BufferFormat::kFmask8_S4_F4) << 20u);
} else {
std::_Exit(0x7e);
}
ValidateStorageTexture(resource, descriptor, 0x10000);
std::_Exit(0x7f);
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckBasicStorageTextureDescriptor() {
const auto descriptor = BasicStorageTextureDescriptor();
Require("BasicStorageTexture", "descriptor",
descriptor.Base40() == 0x785d0000ull &&
descriptor.Width5() + 1u == 33 &&
descriptor.Height5() + 1u == 33 && descriptor.Depth() + 1u == 33,
"basic 3D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicStorageTextureResource(), descriptor, 0x10000);
const ShaderTextureResource extended{{0x204aca00u, 0xc4700000u, 0x000fc00fu,
0xa1b00facu, 0x0000003fu, 0x00700000u,
0x006b0000u, 0x00204b0au}};
auto extended_resource = BasicStorageTextureResource();
extended_resource.read = false;
Require("BasicStorageTexture", "extended descriptor",
extended.fields[6] != 0 && extended.fields[7] != 0,
"extended 3D storage descriptor fixture is malformed");
ValidateStorageTexture(extended_resource, extended, 0x400000);
const auto linear = BasicLinearStorageTextureDescriptor();
Require("BasicStorageTexture", "linear descriptor",
linear.Base40() == 0x4bcc40100ull && linear.Width5() + 1u == 3840 &&
linear.Height5() + 1u == 2160 && linear.Depth() + 1u == 1 &&
linear.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
linear.TileMode() == Prospero::TileMode::kLinear,
"PPSA07429 linear 2D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicLinearStorageTextureResource(), linear,
0x1fa4000);
const auto bgra = BasicBgraStorageTextureDescriptor();
Require("BasicStorageTexture", "BGRA descriptor",
bgra.Base40() == 0x7c650000ull && bgra.Width5() + 1u == 1920 &&
bgra.Height5() + 1u == 1080 && bgra.Depth() + 1u == 1 &&
bgra.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
bgra.TileMode() == Prospero::TileMode::kRenderTarget &&
bgra.DstSelXYZW() == DstSel(6, 5, 4, 7),
"PPSA02604 BGRA 2D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), bgra, 0x870000);
const ShaderTextureResource r128{
{0x0202e500u, 0xc8200000u, 0x010dc1dfu, 0x91b00facu, 0, 0, 0, 0}};
auto r128_resource = BasicBgraStorageTextureResource();
r128_resource.r128 = true;
Require("BasicStorageTexture", "PPSA01736 R128 descriptor",
r128.Base40() == 0x202e50000ull && r128.Width5() + 1u == 1920 &&
r128.Height5() + 1u == 1080 && r128.Depth() + 1u == 1 &&
r128.Format() == Prospero::BufferFormat::k8_8_8_8Srgb &&
r128.TileMode() == Prospero::TileMode::kRenderTarget &&
r128.DstSelXYZW() == DstSel(4, 5, 6, 7),
"PPSA01736 R128 storage descriptor fixture is malformed");
ValidateStorageTexture(r128_resource, r128, 0x870000);
const auto r11g11b10 = Ppsa06228R11G11B10StorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA06228 R11G11B10 descriptor",
r11g11b10.Base40() == 0x10c6b50000ull &&
r11g11b10.Width5() + 1u == 1920 &&
r11g11b10.Height5() + 1u == 1080 && r11g11b10.Depth() + 1u == 1 &&
r11g11b10.Format() == Prospero::BufferFormat::k11_11_10Float &&
r11g11b10.TileMode() == Prospero::TileMode::kRenderTarget &&
r11g11b10.DstSelXYZW() == DstSel(4, 5, 6, 1),
"PPSA06228 R11G11B10 storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), r11g11b10,
0x870000);
ValidateStorageColorView(vk::Format::eB8G8R8A8Unorm,
vk::Format::eB10G11R11UfloatPack32,
r11g11b10.DstSelXYZW());
const auto max_mip = Ppsa01530MaxMipStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA01530 max-mip descriptor",
max_mip.Base40() == 0x4a4290000ull && max_mip.Width5() + 1u == 64 &&
max_mip.Height5() + 1u == 64 && max_mip.Depth() + 1u == 1 &&
max_mip.BaseLevel() == 0 && max_mip.LastLevel() == 0 &&
max_mip.MaxMip() == 5 &&
max_mip.Format() == Prospero::BufferFormat::k32_32UInt &&
max_mip.TileMode() == Prospero::TileMode::kRenderTarget &&
max_mip.DstSelXYZW() == DstSel(4, 5, 0, 1),
"PPSA01530 max-mip storage descriptor fixture is malformed");
ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), max_mip,
0x20000);
auto mip_one = max_mip;
mip_one.fields[3] |= (1u << 12u) | (1u << 16u);
Require("BasicStorageTexture", "PPSA01530 mip-one descriptor",
mip_one.BaseLevel() == 1 && mip_one.LastLevel() == 1 &&
mip_one.MaxMip() == 5,
"PPSA01530 mip-one storage descriptor fixture is malformed");
ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), mip_one,
0x20000);
const auto mip_range = Ppsa01340MipRangeStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA01340 mip-range descriptor",
mip_range.Base40() == 0x294dc0000ull &&
mip_range.Width5() + 1u == 1280 &&
mip_range.Height5() + 1u == 720 && mip_range.Depth() + 1u == 1 &&
mip_range.BaseLevel() == 1 && mip_range.LastLevel() == 3 &&
mip_range.MaxMip() == 3 &&
mip_range.Format() == Prospero::BufferFormat::k16_16_16_16Float &&
mip_range.TileMode() == Prospero::TileMode::kRenderTarget &&
mip_range.DstSelXYZW() == DstSel(4, 5, 6, 7),
"PPSA01340 mip-range storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), mip_range,
0xa30000);
const auto overwide_mip = Ppsa01340OverwideMipStorageTextureDescriptor();
Require(
"BasicStorageTexture", "PPSA01340 over-wide mip view",
overwide_mip.Base40() == 0x346c00000ull &&
overwide_mip.Width5() + 1u == 512 &&
overwide_mip.Height5() + 1u == 512 && overwide_mip.BaseLevel() == 0 &&
overwide_mip.LastLevel() == 9 && overwide_mip.MaxMip() == 8 &&
overwide_mip.Format() == Prospero::BufferFormat::k32_32_32_32UInt &&
overwide_mip.TileMode() == Prospero::TileMode::kStandard64KB,
"captured over-wide storage mip view is malformed");
TileSizeAlign overwide_size{};
TileGetTextureTotalSize(overwide_mip.Format(), overwide_mip.Width5() + 1u,
overwide_mip.Height5() + 1u, 1,
overwide_mip.MaxMip() + 1u, overwide_mip.TileMode(),
false, overwide_size);
Require("BasicStorageTexture", "PPSA01340 over-wide mip footprint",
overwide_size.size == 0x560000 && overwide_size.align == 0x10000,
"captured nine-level Standard64KB footprint changed");
ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), overwide_mip,
overwide_size.size);
const auto r16_float = Ppsa02527R16FloatStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA02527 R16F descriptor",
r16_float.Base40() == 0xce350000ull &&
r16_float.Width5() + 1u == 1920 &&
r16_float.Height5() + 1u == 1080 && r16_float.Depth() + 1u == 1 &&
r16_float.Format() == Prospero::BufferFormat::k16Float &&
r16_float.TileMode() == Prospero::TileMode::kRenderTarget &&
r16_float.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA02527 R16F 2D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), r16_float,
0x480000);
const auto r32_float = Ppsa02527R32FloatStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA02527 R32F descriptor",
r32_float.Base40() == 0xcea90000ull &&
r32_float.Width5() + 1u == 960 &&
r32_float.Height5() + 1u == 540 && r32_float.Depth() + 1u == 1 &&
r32_float.Format() == Prospero::BufferFormat::k32Float &&
r32_float.TileMode() == Prospero::TileMode::kRenderTarget &&
r32_float.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA02527 R32F 2D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), r32_float,
0x280000);
const auto r8_unorm = Ppsa02527R8UnormStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA02527 R8 UNORM descriptor",
r8_unorm.Base40() == 0xc7d50000ull && r8_unorm.Width5() + 1u == 960 &&
r8_unorm.Height5() + 1u == 540 && r8_unorm.Depth() + 1u == 1 &&
r8_unorm.Format() == Prospero::BufferFormat::k8UNorm &&
r8_unorm.TileMode() == Prospero::TileMode::kRenderTarget &&
r8_unorm.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA02527 R8 UNORM 2D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), r8_unorm, 0xc0000);
const auto yzwx = BasicYzwxStorageTextureDescriptor();
Require("BasicStorageTexture", "YZWX descriptor",
yzwx.Base40() == 0x62780100ull && yzwx.Width5() + 1u == 8 &&
yzwx.Height5() + 1u == 8 && yzwx.Depth() + 1u == 1 &&
yzwx.Format() == Prospero::BufferFormat::k32_32_32_32Float &&
yzwx.TileMode() == Prospero::TileMode::kLinear &&
yzwx.DstSelXYZW() == DstSel(5, 6, 7, 4),
"PPSA04181 linear YZWX storage descriptor fixture is malformed");
ValidateStorageTexture(BasicLinearStorageTextureResource(), yzwx, 0x800);
auto all_swizzles = yzwx;
all_swizzles.fields[1] =
(all_swizzles.fields[1] & ~0x1ff00000u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k16_16_16_16Float) << 20u);
uint32_t valid_storage_swizzles = 0;
for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) {
if (!IsValidImageSwizzle(swizzle)) {
continue;
}
all_swizzles.fields[3] = (all_swizzles.fields[3] & ~0xfffu) | swizzle;
ValidateStorageTexture(BasicLinearStorageTextureResource(), all_swizzles,
0x800);
ValidateStorageColorView(vk::Format::eR16G16B16A16Sfloat,
vk::Format::eR16G16B16A16Sfloat, swizzle);
ValidateStorageColorView(vk::Format::eR8G8B8A8Srgb,
vk::Format::eR8G8B8A8Unorm, swizzle);
ValidateStorageColorView(vk::Format::eB8G8R8A8Srgb,
vk::Format::eR8G8B8A8Unorm, swizzle);
valid_storage_swizzles++;
}
Require("BasicStorageTexture", "all write swizzles",
valid_storage_swizzles == 1296,
"valid write-only storage image mappings were rejected");
const ShaderTextureResource cube{{0x025bca00u, 0xc4700000u, 0x003fc03fu,
0xb1b00facu, 0x00000005u, 0x00700080u,
0x00000000u, 0x00000000u}};
auto cube_resource = BasicArrayStorageTextureResource();
cube_resource.cube = true;
Require("BasicStorageTexture", "PPSA07429 cube descriptor",
cube.Base40() == 0x25bca0000ull && cube.Width5() + 1u == 256 &&
cube.Height5() + 1u == 256 && cube.Depth() + 1u == 6 &&
cube.BaseArray5() == 0 && cube.MaxMip() == 8 &&
cube.Type() == Prospero::ImageType::kCube &&
cube.Format() == Prospero::BufferFormat::k16_16_16_16Float &&
cube.TileMode() == Prospero::TileMode::kRenderTarget &&
cube.DstSelXYZW() == DstSel(4, 5, 6, 7),
"PPSA07429 cube storage descriptor fixture is malformed");
ValidateStorageTexture(cube_resource, cube, 0x420000);
const auto array = BasicArrayStorageTextureDescriptor();
Require("BasicStorageTexture", "2D-array descriptor",
array.Base40() == 0x2017900000ull && array.Width5() + 1u == 1 &&
array.Height5() + 1u == 1 && array.Depth() + 1u == 1 &&
array.BaseArray5() == 0 &&
array.Type() == Prospero::ImageType::kColor2DArray &&
array.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
array.TileMode() == Prospero::TileMode::kRenderTarget &&
array.DstSelXYZW() == DstSel(6, 5, 4, 7),
"PPSA21268 2D-array storage descriptor fixture is malformed");
ValidateStorageTexture(BasicArrayStorageTextureResource(), array, 0x10000);
const ShaderTextureResource mip_array{{0x20268d00u, 0xc4700000u, 0x001fc01fu,
0xd1b11facu, 0x00000000u, 0x00700070u,
0x00000000u, 0x00000000u}};
Require("BasicStorageTexture", "PPSA14457 mip-one 2D-array descriptor",
mip_array.BaseLevel() == 1 && mip_array.LastLevel() == 1 &&
mip_array.MaxMip() == 7 &&
mip_array.Type() == Prospero::ImageType::kColor2DArray &&
mip_array.Depth() == 0 && mip_array.BaseArray5() == 0,
"PPSA14457 mip-one 2D-array storage descriptor fixture is malformed");
ValidateStorageTexture(BasicArrayStorageTextureResource(), mip_array,
0x30000);
const auto uint_array = BasicUintArrayStorageTextureDescriptor();
Require("BasicStorageTexture", "uint 2D-array descriptor",
uint_array.Base40() == 0x2017920000ull &&
uint_array.Width5() + 1u == 1 && uint_array.Height5() + 1u == 1 &&
uint_array.Depth() + 1u == 1 && uint_array.BaseArray5() == 0 &&
uint_array.Type() == Prospero::ImageType::kColor2DArray &&
uint_array.Format() == Prospero::BufferFormat::k32UInt &&
uint_array.TileMode() == Prospero::TileMode::kRenderTarget &&
uint_array.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA21268 uint 2D-array storage descriptor fixture is malformed");
ValidateStorageTexture(BasicUintArrayStorageTextureResource(), uint_array,
0x10000);
const ShaderTextureResource standard256b{{
0x204e4900u,
0x43c00000u,
0x00004000u,
0x90100facu,
0x00000000u,
0x00700000u,
0x00000000u,
0x00000000u,
}};
auto standard256b_resource = BasicBgraStorageTextureResource();
standard256b_resource.numeric_class = Prospero::TextureNumericClass::Uint;
TileSizeAlign standard256b_size{};
TileGetTextureTotalSize(standard256b.Format(), standard256b.Width5() + 1u,
standard256b.Height5() + 1u,
standard256b.Depth() + 1u, 1, standard256b.TileMode(),
false, standard256b_size);
Require("BasicStorageTexture", "Standard256B uint 2D descriptor",
standard256b.Width5() + 1u == 2 && standard256b.Height5() + 1u == 2 &&
standard256b.Depth() + 1u == 1 &&
standard256b.Type() == Prospero::ImageType::kColor2D &&
standard256b.Format() == Prospero::BufferFormat::k8_8_8_8UInt &&
standard256b.TileMode() == Prospero::TileMode::kStandard256B &&
standard256b.DstSelXYZW() == DstSel(4, 5, 6, 7) &&
standard256b_size.size == 0x100 &&
standard256b_size.align == 0x100,
"captured PPSA08511 Standard256B storage descriptor is malformed");
ValidateStorageTexture(standard256b_resource, standard256b,
standard256b_size.size);
const auto standard4kb_array = Standard4KBUintArrayStorageTextureDescriptor();
TileSizeAlign standard4kb_size{};
TileGetTextureTotalSize(standard4kb_array.Format(), 1, 1, 1, 1,
standard4kb_array.TileMode(), false,
standard4kb_size);
Require(
"BasicStorageTexture", "Standard4KB uint 2D-array descriptor",
standard4kb_array.Type() == Prospero::ImageType::kColor2DArray &&
standard4kb_array.Format() == Prospero::BufferFormat::k32UInt &&
standard4kb_array.TileMode() == Prospero::TileMode::kStandard4KB &&
standard4kb_array.DstSelXYZW() == DstSel(4, 0, 0, 1) &&
standard4kb_size.size == 0x1000 && standard4kb_size.align == 0x1000,
"captured Standard4KB uint 2D-array descriptor is malformed");
ValidateStorageTexture(BasicUintArrayStorageTextureResource(),
standard4kb_array, standard4kb_size.size);
auto based_standard4kb_array = standard4kb_array;
based_standard4kb_array.fields[0] = 0x006c6800u;
based_standard4kb_array.fields[4] = 0x00010001u;
TileSizeAlign based_standard4kb_size{};
TileGetTextureTotalSize(based_standard4kb_array.Format(), 1, 1,
based_standard4kb_array.Depth() + 1u, 1,
based_standard4kb_array.TileMode(), false,
based_standard4kb_size);
Require("BasicStorageTexture", "based Standard4KB array view",
based_standard4kb_array.Base40() == 0x6c680000ull &&
based_standard4kb_array.BaseArray5() == 1 &&
based_standard4kb_array.Depth() == 1 &&
based_standard4kb_size.size == 0x2000 &&
based_standard4kb_size.align == 0x1000,
"captured based Standard4KB array view is malformed");
ValidateStorageTexture(BasicUintArrayStorageTextureResource(),
based_standard4kb_array, based_standard4kb_size.size);
const auto standard64kb = Standard64KBStorageTextureDescriptor();
TileSizeAlign standard64kb_size{};
TileGetTextureTotalSize(standard64kb.Format(), standard64kb.Width5() + 1u,
standard64kb.Height5() + 1u,
standard64kb.Depth() + 1u, 1, standard64kb.TileMode(),
false, standard64kb_size);
Require("BasicStorageTexture", "Standard64KB 2D descriptor",
standard64kb.Type() == Prospero::ImageType::kColor2DArray &&
standard64kb.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
standard64kb.TileMode() == Prospero::TileMode::kStandard64KB &&
standard64kb.DstSelXYZW() == DstSel(4, 5, 6, 7) &&
standard64kb_size.size == 0x10000 &&
standard64kb_size.align == 0x10000,
"captured Standard64KB storage descriptor is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), standard64kb,
standard64kb_size.size);
const auto uint_volume = BasicUintVolumeStorageTextureDescriptor();
Require("BasicStorageTexture", "uint 3D descriptor",
uint_volume.Base40() == 0x2018060000ull &&
uint_volume.Width5() + 1u == 16 &&
uint_volume.Height5() + 1u == 16 &&
uint_volume.Depth() + 1u == 16 && uint_volume.BaseArray5() == 0 &&
uint_volume.Type() == Prospero::ImageType::kColor3D &&
uint_volume.Format() == Prospero::BufferFormat::k16UInt &&
uint_volume.TileMode() == Prospero::TileMode::kLinear &&
uint_volume.DstSelXYZW() == DstSel(4, 0, 0, 0),
"PPSA21268 uint 3D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicUintVolumeStorageTextureResource(), uint_volume,
0x10000);
const ShaderTextureResource tiled_uint_volume{
{0x1ac0e530u, 0xc0b00000u, 0x0003c003u, 0xa0500004u, 0x0000000fu,
0x00700000u, 0x00000000u, 0x00000000u}};
Require("BasicStorageTexture", "tiled uint 3D descriptor",
tiled_uint_volume.Width5() + 1u == 16 &&
tiled_uint_volume.Height5() + 1u == 16 &&
tiled_uint_volume.Depth() + 1u == 16 &&
tiled_uint_volume.Type() == Prospero::ImageType::kColor3D &&
tiled_uint_volume.TileMode() == Prospero::TileMode::kStandard4KB,
"captured tiled uint 3D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicUintVolumeStorageTextureResource(),
tiled_uint_volume, 0x2000);
const auto depth_tile = Ppsa14053DepthTileStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA14053 depth-tile descriptor",
depth_tile.Base40() == 0x20144c0000ull &&
depth_tile.Width5() + 1u == 1 && depth_tile.Height5() + 1u == 1 &&
depth_tile.Depth() + 1u == 1 && depth_tile.BaseArray5() == 0 &&
depth_tile.Type() == Prospero::ImageType::kColor2DArray &&
depth_tile.Format() == Prospero::BufferFormat::k8UInt &&
depth_tile.TileMode() == Prospero::TileMode::kDepth &&
depth_tile.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA14053 write-only depth-tile storage descriptor fixture is "
"malformed");
ValidateStorageTexture(Ppsa14053DepthTileStorageTextureResource(), depth_tile,
0x10000);
const auto d16_depth_tile = Ppsa10112D16StorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA10112 D16 depth-tile descriptor",
d16_depth_tile.Base40() == 0x205b900000ull &&
d16_depth_tile.Width5() + 1u == 240 &&
d16_depth_tile.Height5() + 1u == 135 &&
d16_depth_tile.Depth() + 1u == 1 &&
d16_depth_tile.Type() == Prospero::ImageType::kColor2DArray &&
d16_depth_tile.Format() == Prospero::BufferFormat::k16UNorm &&
d16_depth_tile.TileMode() == Prospero::TileMode::kDepth &&
d16_depth_tile.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA10112 writable D16 depth-plane descriptor fixture is malformed");
const auto d16_pitch =
TileGetTexturePitch(d16_depth_tile.Format(), d16_depth_tile.Width5() + 1u,
d16_depth_tile.TileMode());
TileSizeAlign d16_size{};
TileGetTextureTotalSize(d16_depth_tile.Format(), d16_depth_tile.Width5() + 1u,
d16_depth_tile.Height5() + 1u,
d16_depth_tile.Depth() + 1u, 1,
d16_depth_tile.TileMode(), false, d16_size);
Require("BasicStorageTexture", "PPSA10112 D16 depth-tile footprint",
d16_pitch == 256 && d16_size.size == 0x20000 &&
d16_size.align == 0x10000,
"PPSA10112 writable D16 depth-plane footprint is incorrect");
ValidateStorageTexture(BasicArrayStorageTextureResource(), d16_depth_tile,
d16_size.size);
auto depth_tile_r32 = depth_tile;
depth_tile_r32.fields[1] =
(depth_tile_r32.fields[1] & ~(0x1ffu << 20u)) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u);
depth_tile_r32.fields[3] =
(depth_tile_r32.fields[3] & ~(0xfu << 28u)) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
auto depth_tile_r32_resource = Ppsa14053DepthTileStorageTextureResource();
depth_tile_r32_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
ValidateStorageTexture(depth_tile_r32_resource, depth_tile_r32, 0x10000);
Require("BasicStorageTexture", "R32_UINT replicated write mapping",
IsValidImageSwizzle(DstSel(4, 4, 4, 4)),
"single-channel replicated destination selection was rejected");
const auto atomic = AtomicStorageTextureDescriptor();
Require("BasicStorageTexture", "atomic R32_UINT descriptor",
atomic.Width5() + 1u == 128 && atomic.Height5() + 1u == 1 &&
atomic.Depth() + 1u == 1 &&
atomic.Type() == Prospero::ImageType::kColor2D &&
atomic.Format() == Prospero::BufferFormat::k32UInt &&
atomic.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA22102 image-atomic descriptor fixture is malformed");
ValidateStorageTexture(AtomicStorageTextureResource(), atomic, 0x10000);
char path[MAX_PATH]{};
Require("BasicStorageTexture", "host",
GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
"GetModuleFileName failed");
for (const char *kind : {"resource",
"type",
"standard256b-volume",
"inverted-mip-range",
"swizzle",
"linear-rgb1-read",
"bgra-read",
"r16-float-read",
"r8-unorm-read",
"yzwx-read",
"reserved-swizzle",
"array-base-out-of-range",
"reserved",
"uint-format",
"uint-resource-float-format",
"atomic-format",
"depth-tile-read",
"depth-tile-extent",
"depth-tile-fmask"}) {
std::string command = std::string("\"") + path +
"\" --storage-texture-descriptor-death " + kind;
std::vector<char> mutable_command(command.begin(), command.end());
mutable_command.push_back('\0');
STARTUPINFOA startup{sizeof(startup)};
PROCESS_INFORMATION process{};
Require("BasicStorageTexture", "host",
CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr,
FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup,
&process) != 0,
"CreateProcess failed");
Require("BasicStorageTexture", "host",
WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0,
"descriptor death case timed out");
DWORD exit_code = 0;
const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0;
CloseHandle(process.hThread);
CloseHandle(process.hProcess);
Require("BasicStorageTexture", "host", exited && exit_code == 321,
std::string(kind) +
" storage descriptor did not report a fatal error");
}
std::printf("[host] %-32s ok\n", "BasicStorageTextureDescriptor");
}
#endif
void CheckStorageTextureLinearUploadLayout() {
constexpr auto format = Prospero::BufferFormat::k8_8_8_8UNorm;
constexpr uint32_t width = 3840;
constexpr uint32_t height = 2160;
constexpr uint32_t depth = 1;
constexpr auto tile = Prospero::TileMode::kLinear;
const auto pitch = TileGetTexturePitch(format, width, tile);
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total);
const auto layout =
TextureCalcUploadLayout(format, width, height, 1, depth, tile, total.size,
false, "StorageTextureLinearTest");
const auto regions = TextureBuildImageCopies(layout);
Require("StorageTextureLinearUpload", "layout",
pitch == width && total.size == 0x1fa4000 && total.align == 256 &&
layout.surface.description.tile_mode == tile &&
layout.pitch == width && layout.slice_stride == total.size &&
regions.size() == 1 && regions[0].bufferOffset == 0 &&
regions[0].imageExtent.width == width &&
regions[0].imageExtent.height == height &&
regions[0].bufferRowLength == width,
"linear RGBA8 storage upload lost Prospero pitch or allocation size");
std::printf("[host] %-32s ok\n", "StorageTextureLinearUpload");
}
void CheckStorageTextureDepthTileUploadLayout() {
constexpr auto format = Prospero::BufferFormat::k8UInt;
constexpr uint32_t width = 1;
constexpr uint32_t height = 1;
constexpr uint32_t depth = 1;
constexpr auto tile = Prospero::TileMode::kDepth;
const auto pitch = TileGetTexturePitch(format, width, tile);
TileSizeAlign slice{};
TileSizeAlign total{};
TileSizeOffset level{};
TilePaddedSize padded{};
TileGetTextureSize(format, width, height, 1, tile, &slice, &level, &padded);
TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total);
const auto layout =
TextureCalcUploadLayout(format, width, height, 1, depth, tile, total.size,
false, "StorageTextureDepthTileTest");
const auto regions = TextureBuildImageCopies(layout);
Require("StorageTextureDepthTileUpload", "PPSA14053 layout",
pitch == 256 && padded.width == 256 && padded.height == 256 &&
slice.size == 0x10000 && slice.align == 0x10000 &&
level.size == slice.size && level.offset == 0 &&
total.size == slice.size && total.align == slice.align &&
layout.surface.description.tile_mode == tile &&
layout.surface.texture.block.family ==
TileBlockFamily::Depth64KB &&
layout.pitch == pitch && layout.slice_stride == pitch &&
layout.source_slice_stride == total.size &&
layout.mips[0].size == pitch &&
layout.surface.mips[0].size == total.size &&
regions.size() == 1 && regions[0].bufferOffset == 0 &&
regions[0].imageExtent.width == width &&
regions[0].imageExtent.height == height &&
regions[0].bufferRowLength == pitch,
"1x1 R8_UINT depth tile lost its 64 KiB source footprint");
std::printf("[host] %-32s ok\n", "StorageTextureDepthTileUpload");
}
void CheckImageSamplerSpecialization() {
ShaderSamplerResource filtered_sampler{};
filtered_sampler.fields[2] =
(1u << 20u) | (3u << 22u) | (2u << 24u) | (2u << 26u);
filtered_sampler.SetPointFiltering();
Require("ImageSamplerSpecialization", "forced point sampler",
filtered_sampler.XyMagFilter() == 0u &&
filtered_sampler.XyMinFilter() == 0u &&
filtered_sampler.ZFilter() == 1u &&
filtered_sampler.MipFilter() == 1u,
"point-only image format retained a filtered sampler mode");
ShaderSamplerResource base_mip_sampler{};
base_mip_sampler.fields[2] = (3u << 20u) | (1u << 22u) | (2u << 24u);
base_mip_sampler.SetPointFiltering();
Require("ImageSamplerSpecialization", "base-mip point sampler",
base_mip_sampler.XyMagFilter() == 0u &&
base_mip_sampler.XyMinFilter() == 0u &&
base_mip_sampler.ZFilter() == 1u &&
base_mip_sampler.MipFilter() == 0u,
"forcing point filtering enabled mipmapping on a base-mip sampler");
ShaderRecompiler::IR::Program mixed_sampler_program;
mixed_sampler_program.resource_tracking_complete = true;
ShaderRecompiler::IR::ImageResource sampled_image;
sampled_image.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
sampled_image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
sampled_image.read = true;
mixed_sampler_program.info.images = {sampled_image, sampled_image,
sampled_image, sampled_image};
mixed_sampler_program.info.samplers.push_back({0u, 4u});
mixed_sampler_program.info.sampled_pairs = {
{0u, 0u, 8u}, {1u, 0u, 12u}, {2u, 0u, 16u}, {3u, 0u, 20u}};
ShaderRecompiler::IR::MemoryInfo signed_memory;
signed_memory.kind = ShaderRecompiler::IR::ResourceKind::Image;
signed_memory.resource = 2u;
signed_memory.sampler = 0u;
mixed_sampler_program.memory_info.push_back(signed_memory);
mixed_sampler_program.block_storage.push_back(
std::make_unique<ShaderRecompiler::IR::Block>());
auto *mixed_sampler_block = mixed_sampler_program.block_storage.back().get();
mixed_sampler_program.blocks.push_back(mixed_sampler_block);
using ShaderRecompiler::IR::Value;
using ShaderRecompiler::IR::ValueOpcode;
auto &image_handle = mixed_sampler_block->AppendNewInst(
ValueOpcode::GetImageResource,
{Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
Value(0u), Value(0u)});
auto &sampler_handle = mixed_sampler_block->AppendNewInst(
ValueOpcode::GetSamplerResource,
{Value(0u), Value(0u), Value(0u), Value(0u)});
auto &image_address = mixed_sampler_block->AppendNewInst(
ValueOpcode::MakeImageAddress,
{Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
Value(0u)});
const ShaderRecompiler::IR::MemoryFlags signed_memory_flags{0u, 16u};
uint64_t signed_memory_flag_bits = 0;
std::memcpy(&signed_memory_flag_bits, &signed_memory_flags,
sizeof(signed_memory_flags));
mixed_sampler_block->AppendNewInst(
ValueOpcode::ImageSampleRaw,
{Value(&image_handle), Value(&sampler_handle), Value(&image_address)},
signed_memory_flag_bits);
ShaderRecompiler::IR::DescriptorValue native_image_descriptor{};
native_image_descriptor.dword_count = 8;
native_image_descriptor.dwords[0] = 0x1000u;
native_image_descriptor.dwords[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u;
native_image_descriptor.dwords[2] = 3u | (3u << 14u);
native_image_descriptor.dwords[3] =
DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
auto packed_image_descriptor = native_image_descriptor;
packed_image_descriptor.dwords[0] = 0x2000u;
packed_image_descriptor.dwords[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k11_11_10UInt) << 20u;
auto signed_image_descriptor = native_image_descriptor;
signed_image_descriptor.dwords[0] = 0x3000u;
signed_image_descriptor.dwords[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k32SInt) << 20u;
auto unsigned_image_descriptor = native_image_descriptor;
unsigned_image_descriptor.dwords[0] = 0x4000u;
unsigned_image_descriptor.dwords[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u;
ShaderRecompiler::IR::DescriptorValue sampler_descriptor{};
sampler_descriptor.dword_count = 4;
const std::array descriptors{native_image_descriptor, packed_image_descriptor,
signed_image_descriptor, unsigned_image_descriptor,
sampler_descriptor};
mixed_sampler_program.descriptor_sources.resize(descriptors.size());
for (u32 source = 0; source < descriptors.size(); source++) {
auto &destination = mixed_sampler_program.descriptor_sources[source];
destination.dword_count = descriptors[source].dword_count;
for (u32 dword = 0; dword < destination.dword_count; dword++) {
destination.dwords[dword] = Value(descriptors[source].dwords[dword]);
}
}
for (u32 image = 0; image < mixed_sampler_program.info.images.size(); image++) {
mixed_sampler_program.info.images[image].source = image;
}
mixed_sampler_program.info.samplers[0].source = 4;
mixed_sampler_program.srt_plan_complete = true;
auto mixed_sampler_plan =
ShaderRecompiler::IR::ExtractResourcePlan(mixed_sampler_program);
ShaderRecompiler::IR::ResourceSnapshot mixed_sampler_snapshot;
ShaderRecompiler::IR::ResourceSpecialization mixed_sampler_specialization;
Require("ImageSamplerSpecialization", "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
mixed_sampler_plan, {}, mixed_sampler_snapshot,
mixed_sampler_specialization),
"mixed sampler descriptors could not be materialized");
ShaderRecompiler::IR::ApplyResourceSpecialization(
mixed_sampler_program, mixed_sampler_specialization);
Require("ImageSamplerSpecialization", "mixed sampler variant",
mixed_sampler_program.info.samplers.size() == 3u &&
!mixed_sampler_program.info.samplers[0].force_point_filtering &&
!mixed_sampler_program.info.samplers[0].integer_border &&
!mixed_sampler_program.info.samplers[1].force_point_filtering &&
mixed_sampler_program.info.samplers[1].integer_border &&
mixed_sampler_program.info.samplers[2].force_point_filtering &&
mixed_sampler_program.info.samplers[2].integer_border &&
mixed_sampler_program.info.sampled_pairs[0].sampler == 0u &&
mixed_sampler_program.info.sampled_pairs[1].sampler == 2u &&
mixed_sampler_program.info.sampled_pairs[2].sampler == 2u &&
mixed_sampler_program.info.sampled_pairs[3].sampler == 1u &&
mixed_sampler_program.memory_info[0].sampler == 2u &&
mixed_sampler_snapshot.samplers.size() == 1u &&
std::ranges::all_of(mixed_sampler_program.info.samplers,
[](const auto& sampler) {
return sampler.source == 4u && sampler.snapshot_index == 0u;
}),
"a shared float/unsigned/signed sampler lost its border or filtering "
"class in the compiled bindings, instruction or descriptor snapshot");
for (const bool scalar : {false, true}) {
using namespace ShaderRecompiler::IR;
const char *name = scalar ? "ScalarMaterialImageDomain" : "GpuMaterialImageDomain";
constexpr u32 count = 70, table_base = 0x1000, material_base = 0x2000;
std::vector<u32> memory(4096);
for (u32 i = 0; i < count; ++i) {
auto descriptor = native_image_descriptor;
descriptor.dwords[0] += i * 0x100u;
std::copy(descriptor.dwords.begin(), descriptor.dwords.end(),
memory.begin() + table_base / 4u + i * 8u);
memory[material_base / 4u + i] = (i << 4u) | 3u;
}
memory[material_base / 4u + count] = (7u << 4u) | 15u; // Duplicate after shift.
memory[material_base / 4u + count + 1u] = 0x80000010u; // key<<5 wraps to entry1.
memory[material_base / 4u + count + 2u] = count << 4u; // Scalar table OOB is null.
std::array<u32, 64> user_data{};
const std::array<u32, 8> buffers{table_base, 32u << 16u, count, 0x5204u,
material_base, 4u << 16u, count + 3u, 0x5204u};
std::copy(buffers.begin(), buffers.end(), user_data.begin());
const auto output = MakeStructuredStorageBufferData(
0u, 64u, false, static_cast<u32>(Prospero::BufferFormat::k32UInt));
std::copy_n(output.begin(), 4, user_data.begin() + 48);
user_data[48] = 0x3000u;
if (scalar) user_data[30] = 1u;
std::vector<u32> code{EncodeVopc(0xc5, InlineU32(0), 0),
EncodeSop1(0x24, 24, 106), EncodeSopp(0x08, 0)};
const auto append_scalar_key = [&](bool numeric_use) {
code.insert(code.end(), {EncodeVop1(0x02, 20, Vgpr(0)),
EncodeSop2(0x00, 20, 20, 21),
EncodeSop2(0x26, 20, 20, InlineU32(3)),
EncodeSop2(0x1e, 20, 20, InlineU32(2)),
EncodeSop2(0x00, 20, 20, InlineU32(8)),
EncodeSmem0(0x08, 20, 2), EncodeSmem1(0, 20)});
if (numeric_use) code.push_back(EncodeVop1(0x01, 17, 20));
code.push_back(EncodeSop2(0x20, 20, 20, InlineU32(4)));
};
if (scalar) {
append_scalar_key(true);
} else {
code.insert(code.end(), {EncodeSop1(0x04, 26, 126), EncodeVop1(0x01, 16, InlineU32(1))});
// An earlier descriptor waterfall separates the EXECZ witness from the material load.
const auto previous_loop = code.size();
code.insert(code.end(), {EncodeSop1(0x04, 28, 126),
EncodeVop1(0x02, 22, Vgpr(0)), EncodeVopc(0xd2, 22, 0),
EncodeVop1(0x01, 16, InlineU32(0)),
EncodeSop1(0x04, 126, 28), EncodeVopc(0xd5, InlineU32(0), 16)});
code.push_back(EncodeSopp(0x09, static_cast<int16_t>(previous_loop - code.size() - 1u)));
code.insert(code.end(), {EncodeSop1(0x04, 126, 26),
EncodeMubuf0(0x0c, 0, true, false), EncodeMubuf1(15, 1, 0),
EncodeVop2(0x16, 15, InlineU32(4), 15),
EncodeVop1(0x01, 16, InlineU32(1))});
}
const auto image_loop = code.size();
if (!scalar)
code.insert(code.end(), {EncodeSop1(0x04, 28, 126),
EncodeVop1(0x02, 20, Vgpr(15)), EncodeVopc(0xd2, 20, 15)});
code.insert(code.end(), {EncodeSop2(0x1e, 20, 20, InlineU32(5)),
EncodeSmem0(0x0b, 12), EncodeSmem1(0, 20)});
const auto table_read_end = code.size();
code.insert(code.end(), {EncodeMimg0(0x24, 1), EncodeMimg1(4, 0, 3, 2)});
AppendBufferStoreDword(&code, 4, 0);
if (!scalar) {
code.insert(code.end(), {EncodeVop1(0x01, 16, InlineU32(0)),
EncodeSop1(0x04, 126, 28), EncodeVopc(0xd5, InlineU32(0), 16)});
code.push_back(EncodeSopp(0x09, static_cast<int16_t>(image_loop - code.size() - 1u)));
} else {
// Crossed exits introduce stale s16:19 Phi arms before the later s16:23 samples.
code.push_back(EncodeVop1(0x01, 5, InlineU32(0)));
code.push_back(EncodeSopc(0x06, 30, InlineU32(0)));
const auto skip_load = code.size();
code.push_back(EncodeSopp(0x05, 0));
append_scalar_key(false);
code.insert(code.end(), {EncodeSop2(0x1e, 20, 20, InlineU32(5)),
EncodeSmem0(0x0b, 16), EncodeSmem1(0, 20),
EncodeSopc(0x06, 31, InlineU32(0))});
const auto later_sample = code.size();
code.push_back(EncodeSopp(0x05, 0));
code.insert(code.end(), {EncodeMimg0(0x24, 1), EncodeMimg1(4, 0, 4, 2)});
const auto early_exit = code.size();
code.push_back(EncodeSopp(0x02, 0));
code[later_sample] = EncodeSopp(0x05, static_cast<int16_t>(code.size() - later_sample - 1u));
code.insert(code.end(), {EncodeMimg0(0x24, 1), EncodeMimg1(4, 0, 4, 2),
EncodeMimg0(0x24, 1), EncodeMimg1(5, 0, 4, 2)});
const auto leave_samples = code.size();
code.push_back(EncodeSopp(0x02, 0));
code[skip_load] = EncodeSopp(0x05, static_cast<int16_t>(code.size() - skip_load - 1u));
code.push_back(EncodeVop1(0x01, 4, InlineU32(0)));
code[early_exit] = EncodeSopp(0x02, static_cast<int16_t>(code.size() - early_exit - 1u));
code[leave_samples] = EncodeSopp(0x02, static_cast<int16_t>(code.size() - leave_samples - 1u));
AppendBufferStoreDword(&code, 4, 0);
AppendBufferStoreDword(&code, 5, 0);
AppendBufferStoreDword(&code, 17, 0);
}
code[2] = EncodeSopp(0x08, static_cast<int16_t>(code.size() - 3u));
AppendEnd(&code);
ShaderComputeInputInfo compute{};
compute.wave_size = 32;
compute.threads_num[0] = 32;
compute.thread_ids_num = 1;
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Compute;
options.wave_size = 32;
options.user_data = user_data;
options.input_info.compute = &compute;
auto translated = ShaderRecompiler::TranslateProgram(code, options);
auto plan = ExtractResourcePlan(translated.program);
ResourceSnapshot snapshot;
ResourceSpecialization specialization;
const SrtRuntime runtime{.user_data = user_data, .read_memory = ReadTestMemory,
.userdata = &memory, .read_specialization_memory = ReadTestMemory};
Require(name, "bounded native material plan",
plan.capture_specialization_reads && !translated.program.has_address_writes &&
MaterializeResources(plan, runtime, snapshot, specialization) &&
snapshot.images.size() == count + 1u,
"GPU-selected material descriptors did not expand beyond native root capacity");
const auto offset = specialization.images[0].indirect_mapping_offset;
const auto ordinal = [&](u32 byte_offset) {
const auto word = byte_offset >> 2u;
return word < snapshot.flattened_srt[offset]
? snapshot.flattened_srt[offset + 1u + word] : 0u;
};
Require(name, "dense table map and coherent snapshot",
specialization.images[0].indirect_search_iterations == 0u &&
snapshot.flattened_srt[offset] == (count - 1u) * 8u + 1u &&
ordinal(32u) != 0u &&
ordinal(count * 32u) == 0u && snapshot.images[0].dwords[0] == 0u &&
snapshot.specialization_reads ==
std::vector<std::pair<uint64_t, uint64_t>>{{table_base, count * 32u}},
"image table lost its dense offsets, explicit null or single coherent read");
const auto previous = specialization;
memory[table_base / 4u + 8u] += 0x10000u;
Require(name, "runtime descriptor refresh",
MaterializeResources(plan, runtime, snapshot, specialization) &&
specialization == previous && snapshot.images[ordinal(32u)].dwords[0] == 0x11100u,
"descriptor addresses were frozen or entered the shader specialization");
auto compiled = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization);
ValidateSpirv(name, compiled.spirv);
if (!scalar) {
user_data[4] += 1u;
Require(name, "GPU material reads remain on GPU",
MaterializeResources(plan, runtime, snapshot, specialization) &&
specialization == previous && snapshot.specialization_reads ==
std::vector<std::pair<uint64_t, uint64_t>>{{table_base, count * 32u}},
"GPU material addressing reentered host descriptor selection");
user_data[4] -= 1u;
}
auto dirty = runtime;
dirty.read_specialization_memory = +[](void*, uint64_t, std::span<u32>) { return false; };
Require(name, "unreadable image table rejection",
!MaterializeResources(plan, dirty, snapshot, specialization),
"GPU-selected texture materialization bypassed strict read provenance");
if (scalar) {
code.insert(code.begin() + table_read_end, EncodeVop1(0x01, 18, 12));
code.pop_back();
AppendBufferStoreDword(&code, 18, 0);
code[2] = EncodeSopp(0x08, static_cast<int16_t>(code.size() - 3u));
AppendEnd(&code);
} else {
code[2] = EncodeSopp(0x00, 0); // GPU selectors do not need a host nonempty-EXEC witness.
}
auto retained = ShaderRecompiler::TranslateProgram(code, options);
auto retained_plan = ExtractResourcePlan(retained.program);
Require(name, "retained descriptor consumers",
MaterializeResources(retained_plan, runtime, snapshot, specialization),
"retained GPU descriptor reads could not use the shared image table");
compiled = ShaderRecompiler::CompileProgram(std::move(retained), options, specialization);
ValidateSpirv(name, compiled.spirv);
}
std::printf("[host] %-32s ok\n", "ImageSpecializationPipelineId");
}
void CheckResourcePlanHandoff() {
constexpr const char *name = "ResourcePlanHandoff";
using namespace ShaderRecompiler::IR;
for (const bool numeric_read : {false, true}) {
std::vector<u32> code{EncodeSmem0(0x02, 8), EncodeSmem1(0, 125),
EncodeSmem0(0x08, 12, 4), EncodeSmem1(0, 125)};
if (numeric_read) {
code.insert(code.end(), {EncodeSmem0(0x00, 13), EncodeSmem1(16, 125)});
AppendVop3(&code, 0x125u, 0, 12, 13); // v_add_nc_u32 v0, s12, s13
} else {
code.push_back(EncodeVop1(0x01, 0, 12));
}
code.insert(code.end(), {EncodeMubuf0(0x1c, 0, false, false),
EncodeMubuf1(0, 2, 0)});
AppendEnd(&code);
std::array<u32, 2> user_data{};
std::vector<u32> memory{0x1000u, 0u, 16u, 0x30000000u | DstSel(4, 5, 6, 7), 7u};
ShaderComputeInputInfo compute{};
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Compute;
options.user_data = user_data;
options.input_info.compute = &compute;
auto translated = ShaderRecompiler::TranslateProgram(code, options);
auto plan = ExtractResourcePlan(translated.program);
ResourceSnapshot snapshot;
ResourceSpecialization specialization;
const SrtRuntime runtime{.user_data = user_data,
.read_memory = ReadTestMemory,
.userdata = &memory,
.read_specialization_memory = ReadTestMemory};
Require(name, "initial descriptors",
MaterializeResources(plan, runtime, snapshot, specialization) &&
snapshot.buffers.size() == 1 && snapshot.buffers[0].dwords[0] == 0x1000u,
"SRT buffer descriptor was not materialized");
auto compiled = ShaderRecompiler::CompileProgram(std::move(translated), options,
specialization);
ValidateSpirv(name, compiled.spirv);
const auto &bindings = compiled.program.bindings;
Require(name, "GPU data requirements",
compiled.program.info.user_data_registers.empty() && bindings.descriptor_counts[0] == 1 &&
(bindings.descriptor_counts[static_cast<size_t>(DescriptorBindingKind::FlattenedSrt)] != 0) ==
numeric_read,
"descriptor-only shader retained SRT uploads or live scalar data was removed");
memory[0] = 0x2000u;
memory[4] = 13u;
Require(name, "independent host plan",
MaterializeResources(plan, runtime, snapshot, specialization) &&
snapshot.buffers[0].dwords[0] == 0x2000u &&
(!numeric_read || snapshot.flattened_srt.back() == 13u),
"compiled shader cleanup invalidated or froze the host resource plan");
}
std::printf("[host] %-32s ok\n", name);
}
void CheckNativeImageDescriptorTypes() {
using ShaderRecompiler::IR::DescriptorBindingKind;
using ShaderRecompiler::IR::FirstImageBinding;
using ShaderRecompiler::IR::FirstStorageImageBinding;
for (uint32_t value = FirstImageBinding;
value < static_cast<uint32_t>(DescriptorBindingKind::Samplers);
value++) {
const auto expected = value < FirstStorageImageBinding
? vk::DescriptorType::eSampledImage
: vk::DescriptorType::eStorageImage;
Require("NativeImageDescriptorTypes", "image binding",
NativeDescriptorType(static_cast<DescriptorBindingKind>(value)) ==
expected,
"generated image binding has the wrong Vulkan descriptor type");
}
Require("NativeImageDescriptorTypes", "non-image bindings",
NativeDescriptorType(DescriptorBindingKind::Samplers) ==
vk::DescriptorType::eSampler &&
NativeDescriptorType(DescriptorBindingKind::Buffers) ==
vk::DescriptorType::eStorageBuffer &&
NativeDescriptorType(DescriptorBindingKind::Gds) ==
vk::DescriptorType::eStorageBuffer &&
NativeDescriptorType(DescriptorBindingKind::BdaPagetable) ==
vk::DescriptorType::eStorageBuffer &&
NativeDescriptorType(DescriptorBindingKind::FaultBuffer) ==
vk::DescriptorType::eStorageBuffer &&
NativeDescriptorType(DescriptorBindingKind::FlattenedSrt) ==
vk::DescriptorType::eStorageBuffer &&
NativeDescriptorType(DescriptorBindingKind::ShaderData) ==
vk::DescriptorType::eStorageBuffer,
"non-image binding has the wrong Vulkan descriptor type");
std::printf("[host] %-32s ok\n", "NativeImageDescriptorTypes");
}
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
void CheckShaderRecompilerFatalContracts() {
ExpectFatal("InvalidDescriptorBindingRejection", [] {
(void)NativeDescriptorType(
ShaderRecompiler::IR::DescriptorBindingKind::Count);
});
ExpectFatal("WritableFlatStoreRejection", [] {
const auto test = FlatStoreVariants();
(void)CompileCase(test);
});
ExpectFatal("GdsOrB64Rejection", [] {
TestCase test;
test.name = "GdsOrB64Rejection";
test.code = {EncodeDs0(0x4a, 0, true), EncodeDs1(0, 3, 7)};
AppendEnd(&test.code);
test.opcodes = {ShaderOpcode::DS_OR_B64, ShaderOpcode::S_ENDPGM};
(void)CompileCase(test);
});
{
ShaderRecompiler::IR::Program program;
program.resource_tracking_complete = true;
ShaderRecompiler::IR::ImageResource image;
image.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
image.numeric_class = Prospero::TextureNumericClass::Uint;
image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
image.read = true;
image.written = true;
image.atomic = true;
program.info.images.push_back(image);
ShaderRecompiler::IR::DescriptorValue descriptor{};
descriptor.dword_count = 8;
descriptor.dwords[0] = 0x2000u;
descriptor.dwords[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k11_11_10UInt) << 20u;
descriptor.dwords[2] = 3u | (3u << 14u);
descriptor.dwords[3] =
DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
image.source = 0;
program.info.images[0].source = 0;
program.descriptor_sources.resize(1);
program.descriptor_sources[0].dword_count = descriptor.dword_count;
for (u32 dword = 0; dword < descriptor.dword_count; dword++) {
program.descriptor_sources[0].dwords[dword] =
ShaderRecompiler::IR::Value(descriptor.dwords[dword]);
}
program.srt_plan_complete = true;
auto plan = ShaderRecompiler::IR::ExtractResourcePlan(program);
ShaderRecompiler::IR::ResourceSnapshot snapshot;
ShaderRecompiler::IR::ResourceSpecialization specialization;
Require("PackedAtomicImageRejection", "resource materialization",
!ShaderRecompiler::IR::MaterializeResources(
plan, {}, snapshot, specialization),
"packed atomic image descriptor was accepted");
}
}
#endif
void CheckStorageTextureVolumeUploadLayout() {
constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float;
constexpr uint32_t width = 33;
constexpr uint32_t height = 33;
constexpr uint32_t depth = 33;
const auto pitch =
TileGetTexturePitch(format, width, Prospero::TileMode::kRenderTarget);
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, depth, 1,
Prospero::TileMode::kRenderTarget, true, total);
const auto layout = TextureCalcUploadLayout(
format, width, height, 1, depth, Prospero::TileMode::kRenderTarget,
total.size, true, "StorageTextureVolumeTest");
const auto regions = TextureBuildImageCopies(layout);
Require(
"StorageTextureVolumeUpload", "layout",
pitch == 128 && total.size == 0x210000 && layout.slice_stride == 0x2208 &&
layout.source_slice_stride == 0 && layout.mips[0].size == 0x2208 &&
layout.surface.mips[0].size == 0x10000 && regions.size() == depth,
"3D render-target upload did not preserve its compact linear layout");
std::vector<GpuTileInfo> infos;
Require("StorageTextureVolumeUpload", "GPU records",
TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos) &&
infos.size() == depth,
"3D render-target GPU records were not built");
for (const uint32_t z : {0u, 1u, depth - 1u}) {
Require("StorageTextureVolumeUpload", "slice offsets",
infos[z].linear_offset == static_cast<uint64_t>(z) * 0x2208 &&
infos[z].tiled_offset == static_cast<uint64_t>(z) * 0x10000 &&
infos[z].surface_z == z && infos[z].pitch == width,
"volume slice lost its linear stride, block slice, or Z swizzle");
}
std::printf("[host] %-32s ok\n", "StorageTextureVolumeUpload");
}
void CheckStorageTextureVolumeMipRegions() {
constexpr auto format = Prospero::BufferFormat::k8_8_8_8UNorm;
constexpr uint32_t width = 8;
constexpr uint32_t height = 4;
constexpr uint32_t depth = 5;
constexpr uint32_t levels = 3;
constexpr auto tile = Prospero::TileMode::kLinear;
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, depth, levels, tile, true,
total);
const auto layout = TextureCalcUploadLayout(
format, width, height, levels, depth, tile, total.size, true,
"StorageTextureVolumeMipTest");
const auto copies = TextureBuildImageCopies(layout);
bool valid = copies.size() == 8;
size_t index = 0;
for (uint32_t level = 0; level < levels && valid; level++) {
const uint32_t mip_depth = std::max(depth >> level, 1u);
const uint32_t mip_width = std::max(width >> level, 1u);
const uint32_t mip_height = std::max(height >> level, 1u);
for (uint32_t z = 0; z < mip_depth; z++, index++) {
const auto &copy = copies[index];
valid &= copy.imageSubresource.mipLevel == level &&
copy.imageOffset.z == static_cast<int>(z) &&
copy.imageExtent.width == mip_width &&
copy.imageExtent.height == mip_height &&
copy.bufferOffset ==
layout.mips[level].offset + z * layout.slice_stride &&
copy.bufferRowLength == layout.mips[level].row_length &&
copy.bufferImageHeight == layout.mips[level].image_height;
}
}
valid &= index == copies.size();
Require("StorageTextureVolumeMipRegions", "per-mip depth", valid,
"direction-neutral image copies did not shrink depth or preserve "
"Vulkan Z coordinates");
std::printf("[host] %-32s ok\n", "StorageTextureVolumeMipRegions");
}
void CheckStandard64RenderTargetTileRoundTrip() {
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr auto tile = Prospero::TileMode::kStandard64KB;
constexpr uint32_t observed_width = 3840;
constexpr uint32_t observed_height = 2160;
const auto observed_pitch = TileGetTexturePitch(format, observed_width, tile);
TileSizeAlign observed{};
TileGetTextureSize(format, observed_width, observed_height, 1, tile,
&observed, nullptr, nullptr);
Require("Standard64RenderTarget", "observed layout",
observed_pitch == 3840 && observed.size == 0x1fe0000 &&
observed.align == 0x10000,
"PPSA02721 Standard64KB render-target footprint changed");
constexpr uint32_t width = 257;
constexpr uint32_t height = 131;
const auto pitch = TileGetTexturePitch(format, width, tile);
TileSizeAlign storage{};
TileGetTextureSize(format, width, height, 1, tile, &storage, nullptr,
nullptr);
Require("Standard64RenderTarget", "partial layout",
pitch == 384 && storage.size == 0x60000 && storage.align == 0x10000,
"partial Standard64KB footprint was not padded in 128x128 blocks");
std::printf("[host] %-32s ok\n", "Standard64RenderTarget");
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckStorageTextureGpuOwnedRebindState() {
constexpr uintptr_t base = 0x0000000200200000ull;
constexpr uint64_t size = 0x10000;
const auto guest_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
base, size, Common::VirtualMemory::Mode::ReadWrite,
"storage_texture_gpu_owned_rebind", true);
auto *memory = reinterpret_cast<uint8_t *>(guest_memory);
Require("StorageTextureGpuOwnedRebind", "allocation", guest_memory == base,
"fixed guest-owner allocation failed");
PageManager page_manager;
MemoryTracker tracker(page_manager);
tracker.ForEachUploadRange(
base, size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
uint64_t readable = 0;
uint64_t mapped = 0;
MEMORY_BASIC_INFORMATION protection{};
Require(
"StorageTextureGpuOwnedRebind", "owned",
tracker.IsRegionGpuModified(base, size) &&
(!HostMemoryQueryReadable(base, size, readable) || readable < size) &&
HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, mapped) &&
mapped == size &&
VirtualQuery(memory, &protection, sizeof(protection)) != 0 &&
protection.Protect == PAGE_NOACCESS,
"GPU-owned storage pages remained host-readable or lost tracker "
"identity");
tracker.UnmarkRegionAsGpuModified(base, size);
readable = 0;
Require("StorageTextureGpuOwnedRebind", "clean readback",
!tracker.IsRegionGpuModified(base, size) &&
!tracker.IsRegionCpuModified(base, size) &&
HostMemoryQueryReadable(base, size, readable) && readable == size,
"clean storage readback did not publish readable coherent backing");
tracker.MarkRegionAsGpuModified(base, size);
readable = 0;
Require(
"StorageTextureGpuOwnedRebind", "clean reclaim",
tracker.IsRegionGpuModified(base, size) &&
!tracker.IsRegionCpuModified(base, size) &&
(!HostMemoryQueryReadable(base, size, readable) || readable < size),
"clean storage rebind did not reclaim GPU ownership without an upload");
tracker.UnmarkRegionAsGpuModified(base, size);
tracker.MarkRegionAsCpuModified(base, size);
uint32_t dirty_ranges = 0;
bool upload_called = false;
tracker.ForEachUploadRange(
base, size, true, [&](uint64_t, uint64_t) noexcept { dirty_ranges++; },
[&]() noexcept { upload_called = true; });
readable = 0;
Require(
"StorageTextureGpuOwnedRebind", "dirty refresh",
dirty_ranges == 1 && upload_called &&
tracker.IsRegionGpuModified(base, size) &&
!tracker.IsRegionCpuModified(base, size) &&
(!HostMemoryQueryReadable(base, size, readable) || readable < size),
"CPU-dirty storage rebind did not refresh once and reclaim GPU "
"ownership");
tracker.UnmarkRegionAsGpuModified(base, size);
tracker.UntrackMemory(base, size);
Require("StorageTextureGpuOwnedRebind", "free",
Libs::LibKernel::Memory::FreeGuestMemory(base, size),
"guest-owner free failed");
std::printf("[host] %-32s ok\n", "StorageTextureGpuOwnedRebind");
}
#endif
void CheckNativeMsaaState() {
Require("NativeMsaaState", "sample encoding",
render_sample_count(0) == 1 && render_sample_count(1) == 2 &&
render_sample_count(2) == 4 && render_sample_count(3) == 8 &&
render_sample_count(4) == 0,
"PS5 sample encodings were not mapped exactly");
Require("NativeMsaaState", "Vulkan sample mapping",
vulkan_sample_count(1) == vk::SampleCountFlagBits::e1 &&
vulkan_sample_count(2) == vk::SampleCountFlagBits::e2 &&
vulkan_sample_count(4) == vk::SampleCountFlagBits::e4 &&
vulkan_sample_count(8) == vk::SampleCountFlagBits::e8 &&
vulkan_sample_count(3) == vk::SampleCountFlagBits{},
"native sample counts were not mapped exactly to Vulkan");
TileSizeAlign color{};
const auto color_pitch = TileGetRenderTargetPitch(1920, 8, 3);
Require("NativeMsaaState", "8x color footprint",
color_pitch == 1920 &&
TileGetRenderTargetSize(1920, 1080, color_pitch, 8, color, 3) &&
color.align == 0x10000 && color.size == 0x07f80000,
"8x R16G16B16A16 color footprint was not preserved");
TileSizeAlign depth{};
TileSizeAlign stencil{};
TileSizeAlign htile{};
Require("NativeMsaaState", "8x depth/stencil footprint",
TileGetDepthPitch(1920, 4, 3) == 1920 &&
TileGetDepthSize(1920, 1080, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth, 3) &&
depth.align == 0x10000 && depth.size == 0x03fc0000 &&
stencil.align == 0x10000 && stencil.size == 0x010e0000 &&
htile.align == 0x8000 && htile.size == 0x00030000,
"8x depth/stencil or fragment-independent HTile footprint regressed");
std::printf("[host] %-32s ok\n", "NativeMsaaState");
}
void CheckDepthHtileStencilCompatibility() {
Require("DepthHtileStencilCompatibility", "inactive stencil plane",
depth_htile_stencil_acceleration_compatible(false, true, false),
"inactive Hi-Stencil state was treated as an attachment");
Require("DepthHtileStencilCompatibility", "disabled acceleration",
depth_htile_stencil_acceleration_compatible(true, false, true),
"disabled Hi-Stencil state was rejected");
Require("DepthHtileStencilCompatibility", "PS5 stencil plus HTile",
depth_htile_stencil_acceleration_compatible(true, true, false),
"valid PS5 Hi-Stencil attachment was rejected");
Require("DepthHtileStencilCompatibility", "missing HTile metadata",
!depth_htile_stencil_acceleration_compatible(true, false, false),
"active Hi-Stencil without HTile metadata was admitted");
std::printf("[host] %-32s ok\n", "DepthHtileStencilCompatibility");
}
void CheckPs5DepthRegisterDecoding() {
constexpr uint32_t captured_z = 0x22900803u;
constexpr uint32_t captured_stencil = 0x00100801u;
const auto z = HW::DepthZInfo::Decode(captured_z);
const auto stencil = HW::DepthStencilInfo::Decode(captured_stencil);
Require(
"Ps5DepthRegisterDecoding", "captured texture-compatible state",
z.format == Prospero::DepthFormat::kZ32F && z.num_samples == 0 &&
z.htile_acceleration && !z.expclear_enabled &&
!z.partially_resident && z.max_mip_level == 0 &&
z.z_compare_base == Prospero::ZCompareBase::kZMin &&
z.texture_compatibility ==
Prospero::TextureCompatiblePlaneCompression::kEnable &&
z.HasValidTextureCompatibility() &&
stencil.format == Prospero::StencilFormat::k8UInt &&
!stencil.htile_stencil_disabled && !stencil.expclear_enabled &&
!stencil.partially_resident &&
stencil.texture_compatibility ==
Prospero::TextureCompatibleStencil::kEnable &&
stencil.HasValidTextureCompatibility(),
"valid PS5 depth/stencil aggregate encodings were not decoded exactly");
const auto malformed_z = HW::DepthZInfo::Decode(0x00000803u);
const auto malformed_stencil = HW::DepthStencilInfo::Decode(0x00100001u);
Require("Ps5DepthRegisterDecoding", "malformed aggregate state",
!malformed_z.HasValidTextureCompatibility() &&
!malformed_stencil.HasValidTextureCompatibility(),
"partial texture-compatible aggregate encodings were accepted");
std::printf("[host] %-32s ok\n", "Ps5DepthRegisterDecoding");
}
void CheckDepthAttachmentWrites() {
RenderDepthInfo target{};
target.desc.view_info.format = vk::Format::eD32SfloatS8Uint;
Require("DepthAttachmentWrites", "disabled depth test",
!target.AttachmentWriteAspects(),
"disabled depth testing claimed a depth write");
target.depth_test_enable = true;
target.depth_write_enable = true;
target.depth_compare_op = vk::CompareOp::eLess;
Require("DepthAttachmentWrites", "depth write",
target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth,
"write-enabled depth testing did not claim depth");
target.depth_test_enable = false;
target.depth_write_enable = false;
target.depth_load_clear_enable = true;
Require("DepthAttachmentWrites", "depth clear",
target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth,
"depth attachment clear did not claim depth");
target.depth_load_clear_enable = false;
target.stencil_test_enable = true;
target.stencil_front = {vk::StencilOp::eKeep, vk::StencilOp::eKeep,
vk::StencilOp::eKeep, vk::CompareOp::eAlways, 0xff, 0xff, 0};
target.stencil_back = target.stencil_front;
Require("DepthAttachmentWrites", "stencil keep",
!target.AttachmentWriteAspects(),
"KEEP-only stencil state claimed a stencil write");
target.stencil_front.failOp = vk::StencilOp::eZero;
Require("DepthAttachmentWrites", "unreachable stencil fail",
!target.AttachmentWriteAspects(),
"ALWAYS comparison claimed an unreachable fail operation");
target.stencil_front.failOp = vk::StencilOp::eKeep;
target.stencil_front.passOp = vk::StencilOp::eReplace;
Require("DepthAttachmentWrites", "stencil pass write",
target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil,
"write-capable stencil pass did not claim stencil");
target.stencil_front.writeMask = 0;
Require("DepthAttachmentWrites", "back-face keep",
!target.AttachmentWriteAspects(),
"masked front write or KEEP-only back face claimed stencil");
target.stencil_test_enable = false;
target.stencil_clear_enable = true;
Require("DepthAttachmentWrites", "stencil clear",
target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil,
"stencil attachment clear did not claim stencil");
target.desc.view_info.format = vk::Format::eD32Sfloat;
Require("DepthAttachmentWrites", "missing stencil aspect",
!target.AttachmentWriteAspects(),
"depth-only format claimed a stencil write");
std::printf("[host] %-32s ok\n", "DepthAttachmentWrites");
}
void CheckDepthFeedbackAspects() {
RenderDepthInfo target{};
target.desc.view_info.format = vk::Format::eD32SfloatS8Uint;
target.desc.view_info.base_level = 1;
target.desc.view_info.level_count = 2;
target.desc.view_info.base_layer = 3;
target.desc.view_info.layer_count = 2;
target.depth_test_enable = true;
target.depth_write_enable = true;
target.depth_compare_op = vk::CompareOp::eAlways;
target.stencil_test_enable = true;
target.stencil_front = {vk::StencilOp::eKeep, vk::StencilOp::eReplace,
vk::StencilOp::eKeep, vk::CompareOp::eAlways, 0xff, 0xff, 0};
target.stencil_back = target.stencil_front;
auto sampled = target.desc.view_info;
const auto feedback = [&](const ImageViewInfo& view) {
return DepthFeedbackAspects(target.AttachmentWriteAspects(),
target.desc.view_info, view);
};
sampled.aspect = vk::ImageAspectFlagBits::eDepth;
Require("DepthFeedbackAspects", "sampled writable depth",
feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
"overlapping depth sampling did not request depth feedback");
sampled.aspect = vk::ImageAspectFlagBits::eStencil;
Require("DepthFeedbackAspects", "sampled writable stencil",
feedback(sampled) == vk::ImageAspectFlagBits::eStencil,
"overlapping stencil sampling did not request stencil feedback");
sampled.aspect = vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
Require("DepthFeedbackAspects", "both writable aspects",
feedback(sampled) == sampled.aspect,
"combined depth/stencil sampling lost a writable aspect");
target.depth_write_enable = false;
Require("DepthFeedbackAspects", "read-only depth",
feedback(sampled) == vk::ImageAspectFlagBits::eStencil,
"read-only depth was included in stencil feedback");
target.depth_write_enable = true;
target.stencil_test_enable = false;
Require("DepthFeedbackAspects", "read-only stencil",
feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
"read-only stencil was included in depth feedback");
sampled.aspect = vk::ImageAspectFlagBits::eStencil;
Require("DepthFeedbackAspects", "sampled read-only aspect",
!feedback(sampled),
"sampling a read-only stencil requested feedback for depth writes");
sampled.aspect = vk::ImageAspectFlagBits::eDepth;
sampled.base_level = 3;
Require("DepthFeedbackAspects", "disjoint mip",
!feedback(sampled),
"disjoint mip ranges requested depth feedback");
sampled.base_level = 2;
sampled.base_layer = 5;
Require("DepthFeedbackAspects", "disjoint layer",
!feedback(sampled),
"disjoint layer ranges requested depth feedback");
sampled.base_layer = 4;
Require("DepthFeedbackAspects", "overlapping range boundary",
feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
"overlapping mip and layer boundaries missed depth feedback");
target.depth_write_enable = false;
target.depth_load_clear_enable = true;
Require("DepthFeedbackAspects", "depth clear only",
feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
"a sampled depth attachment clear missed feedback");
target.depth_load_clear_enable = false;
target.stencil_clear_enable = true;
sampled.aspect = vk::ImageAspectFlagBits::eStencil;
Require("DepthFeedbackAspects", "stencil clear only",
feedback(sampled) == vk::ImageAspectFlagBits::eStencil,
"a sampled stencil attachment clear missed feedback");
std::printf("[host] %-32s ok\n", "DepthFeedbackAspects");
}
void CheckDynamicRenderingState() {
RenderState first{};
first.width = 64;
first.height = 32;
first.num_layers = 2;
first.num_color_attachments = 1;
first.color_attachments[0].image_layout =
vk::ImageLayout::eColorAttachmentOptimal;
first.color_attachments[0].clear_value = {1, 2, 3, 4};
first.color_attachments[0].is_clear = true;
auto changed = first;
Require("DynamicRenderingState", "identical state", first == changed,
"fieldwise RenderState identity rejected an identical scope");
changed.color_attachments[0].is_clear = false;
Require("DynamicRenderingState", "clear identity", first != changed,
"clear/load state did not participate in rendering identity");
changed = first;
changed.num_layers = 1;
Require("DynamicRenderingState", "layer identity", first != changed,
"layer count did not participate in rendering identity");
changed = first;
changed.color_attachments[0].image_layout = vk::ImageLayout::eGeneral;
Require("DynamicRenderingState", "layout identity", first != changed,
"attachment layout did not participate in rendering identity");
PipelineRenderingState rgba{};
rgba.color_count = 1;
rgba.color_formats[0] = vk::Format::eR8G8B8A8Unorm;
auto uint_color = rgba;
uint_color.color_formats[0] = vk::Format::eR8G8B8A8Uint;
Require("DynamicRenderingState", "pipeline color format identity",
rgba != uint_color,
"distinct dynamic-rendering formats can alias one pipeline key");
auto depth = rgba;
depth.depth_format = vk::Format::eD32SfloatS8Uint;
depth.stencil_format = vk::Format::eD32SfloatS8Uint;
Require("DynamicRenderingState", "pipeline depth/stencil identity",
rgba != depth,
"depth/stencil formats did not participate in pipeline identity");
RenderDepthInfo attachment{};
attachment.desc.view_info.format = vk::Format::eD32SfloatS8Uint;
Require("DynamicRenderingState", "read-only depth/stencil layout",
depth_attachment_layout(attachment) ==
vk::ImageLayout::eDepthStencilReadOnlyOptimal,
"fully read-only depth/stencil used a writable layout");
attachment.depth_load_clear_enable = true;
Require("DynamicRenderingState", "deferred depth clear layout",
depth_attachment_layout(attachment) ==
vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal,
"a depth load clear used a read-only attachment layout");
attachment.depth_test_enable = true;
attachment.depth_write_enable = true;
Require("DynamicRenderingState", "depth-write stencil-read layout",
depth_attachment_layout(attachment) ==
vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal,
"depth-only writes did not retain read-only stencil");
attachment.depth_load_clear_enable = false;
attachment.depth_write_enable = false;
attachment.stencil_clear_enable = true;
Require("DynamicRenderingState", "depth-read stencil-write layout",
depth_attachment_layout(attachment) ==
vk::ImageLayout::eDepthReadOnlyStencilAttachmentOptimal,
"stencil-only writes did not retain read-only depth");
attachment.depth_write_enable = true;
Require("DynamicRenderingState", "writable depth/stencil layout",
depth_attachment_layout(attachment) ==
vk::ImageLayout::eDepthStencilAttachmentOptimal,
"combined depth/stencil writes did not use the writable layout");
std::printf("[host] %-32s ok\n", "DynamicRenderingState");
}
void CheckDepthTargetFootprints() {
TileSizeAlign stencil{};
TileSizeAlign htile{};
TileSizeAlign depth{};
struct AttachmentFormatCase {
const char *name;
Prospero::DepthFormat depth_format;
Prospero::StencilFormat stencil_format;
vk::Format expected;
};
constexpr AttachmentFormatCase attachment_cases[] = {
{"Z16", Prospero::DepthFormat::kZ16, Prospero::StencilFormat::kInvalid,
vk::Format::eD16Unorm},
{"Z16S8", Prospero::DepthFormat::kZ16, Prospero::StencilFormat::k8UInt,
vk::Format::eD16UnormS8Uint},
{"Z32", Prospero::DepthFormat::kZ32F, Prospero::StencilFormat::kInvalid,
vk::Format::eD32Sfloat},
{"Z32S8", Prospero::DepthFormat::kZ32F, Prospero::StencilFormat::k8UInt,
vk::Format::eD32SfloatS8Uint},
{"invalid depth", static_cast<Prospero::DepthFormat>(2u),
Prospero::StencilFormat::k8UInt, vk::Format::eUndefined},
{"invalid stencil", Prospero::DepthFormat::kZ16,
static_cast<Prospero::StencilFormat>(2u), vk::Format::eUndefined},
};
for (const auto &test : attachment_cases) {
Require("DepthTargetFootprints", test.name,
DepthAttachmentFormat(test.depth_format, test.stencil_format) ==
test.expected,
"PS5 depth/stencil attachment mapping changed");
}
Require("DepthTargetFootprints", "1920x1080 Z16S8 without HTile",
TileGetDepthSize(1920, 1080, 0, Prospero::DepthFormat::kZ16,
Prospero::StencilFormat::k8UInt, false, stencil,
htile, depth) &&
depth.size == 0x480000 && depth.align == 0x10000 &&
stencil.size == 0x280000 && stencil.align == 0x10000 &&
htile.size == 0,
"captured Z16S8 footprint disagrees with Prospero block rules");
struct TargetFormatCase {
const char *name;
vk::Format host_format;
Prospero::BufferFormat guest_format;
uint32_t bytes_per_element;
bool stencil;
bool supported;
bool readback;
};
constexpr TargetFormatCase target_cases[] = {
{"D16", vk::Format::eD16Unorm, Prospero::BufferFormat::k16UNorm, 2, false,
true, true},
{"D16S8", vk::Format::eD16UnormS8Uint, Prospero::BufferFormat::k16UNorm,
2, true, true, true},
{"D16 via D24S8", vk::Format::eD24UnormS8Uint,
Prospero::BufferFormat::k16UNorm, 2, true, true, true},
{"D16 via D32S8", vk::Format::eD32SfloatS8Uint,
Prospero::BufferFormat::k16UNorm, 2, true, true, true},
{"D32", vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float, 4,
false, true, true},
{"D32S8", vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k32Float,
4, true, true, true},
{"D16 plus stencil mismatch", vk::Format::eD16Unorm,
Prospero::BufferFormat::k16UNorm, 2, true, false, false},
{"fallback without stencil", vk::Format::eD24UnormS8Uint,
Prospero::BufferFormat::k16UNorm, 2, false, false, false},
{"D32 guest via D24", vk::Format::eD24UnormS8Uint,
Prospero::BufferFormat::k32Float, 4, true, false, false},
{"D16 byte mismatch", vk::Format::eD16Unorm,
Prospero::BufferFormat::k16UNorm, 4, false, false, false},
};
for (const auto &test : target_cases) {
ImageInfo target{};
target.pixel_format = test.host_format;
target.guest_format = test.guest_format;
target.bytes_per_block = test.bytes_per_element;
target.stencil = test.stencil ? GuestRange{0x10000, 0x10000} : GuestRange{};
Require("DepthTargetFootprints", test.name,
IsSupportedDepthTargetFormat(target) == test.supported &&
IsSupportedDepthPlaneReadback(target) == test.readback,
"host/guest depth format or exact readback policy changed");
}
ImageInfo compressed_stencil{};
compressed_stencil.pixel_format = vk::Format::eD32SfloatS8Uint;
compressed_stencil.guest_format = Prospero::BufferFormat::k32Float;
compressed_stencil.bytes_per_block = 4;
compressed_stencil.stencil = {0x10000, 0x10000};
compressed_stencil.metadata.stencil_compressed = true;
Require("DepthTargetFootprints", "compressed stencil depth-plane readback",
IsSupportedDepthTargetFormat(compressed_stencil) &&
IsSupportedDepthPlaneReadback(compressed_stencil),
"stencil metadata incorrectly blocked independent depth-plane "
"readback");
Require("DepthTargetFootprints", "lexicographic subresource comparison",
ImageSubresources{2, 1} > ImageSubresources{1, 4} &&
ImageSubresources{1, 4} < ImageSubresources{2, 1},
"subresource ordering diverged from required overlap semantics");
struct TransferPlaneCase {
const char *name;
vk::Format attachment;
vk::Format transfer;
uint32_t bytes;
};
constexpr TransferPlaneCase transfer_cases[] = {
{"D16S8 transfer", vk::Format::eD16UnormS8Uint, vk::Format::eD16Unorm, 2},
{"D24S8 transfer", vk::Format::eD24UnormS8Uint,
vk::Format::eX8D24UnormPack32, 4},
{"D32S8 transfer", vk::Format::eD32SfloatS8Uint, vk::Format::eD32Sfloat,
4},
{"invalid transfer", vk::Format::eR16Unorm, vk::Format::eUndefined, 0},
};
for (const auto &test : transfer_cases) {
Require("DepthTargetFootprints", test.name,
DepthAspectTransferFormat(test.attachment) == test.transfer &&
DepthAspectTransferBytes(test.attachment) == test.bytes,
"combined depth transfer-plane layout changed");
}
struct PromotionCase {
const char *name;
uint16_t source;
uint32_t d24;
uint32_t d32;
};
constexpr PromotionCase promotion_cases[] = {
{"zero", 0, 0, 0},
{"midpoint", 0x8000, 0x00800080, 0x3f000080},
{"maximum", 0xffff, 0x00ffffff, 0x3f800000},
};
for (const auto &test : promotion_cases) {
Require("DepthTargetFootprints", test.name,
EncodeD16AsD24(test.source) == test.d24 &&
EncodeD16AsD32(test.source) == test.d32,
"D16 host promotion changed the represented depth value");
}
Require("DepthTargetFootprints", "640x360 Z32S8 without HTile",
TileGetDepthSize(640, 360, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, false, stencil,
htile, depth),
"valid non-HTile depth/stencil footprint was rejected");
Require(
"DepthTargetFootprints", "640x360 Prospero block sizes",
depth.size == 0xf0000 && depth.align == 0x10000 &&
stencil.size == 0x60000 && stencil.align == 0x10000 &&
htile.size == 0 && htile.align == 0,
"non-HTile depth/stencil footprint disagrees with Prospero block rules");
const auto depth_pitch = TileGetTexturePitch(Prospero::BufferFormat::k32Float,
640, Prospero::TileMode::kDepth);
TileSizeAlign texture_depth{};
TileGetTextureTotalSize(Prospero::BufferFormat::k32Float, 640, 360, 1, 1,
Prospero::TileMode::kDepth, false, texture_depth);
Require("DepthTargetFootprints", "640x360 generic depth tile",
depth_pitch == 640 && texture_depth.size == 0xf0000 &&
texture_depth.align == 0x10000,
"generic depth texture sizing bypassed 64 KiB block padding");
Require(
"DepthTargetFootprints", "960x540 Z32S8 with HTile",
TileGetDepthSize(960, 540, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil, htile,
depth) &&
depth.size == 0x280000 && depth.align == 0x10000 &&
stencil.size == 0xc0000 && stencil.align == 0x10000 &&
htile.size == 0x10000 && htile.align == 0x8000,
"generic Prospero HTile block calculation rejected the title footprint");
Require("DepthTargetFootprints", "known HTile extent",
TileGetDepthSize(1280, 720, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth) &&
depth.size == 0x3c0000 && stencil.size == 0xf0000 &&
htile.size == 0x20000,
"validated 1280x720 HTile footprint regressed");
Require("DepthTargetFootprints", "PPSA06228 3840x2160 Z32S8 with HTile",
TileGetDepthSize(3840, 2160, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth) &&
depth.size == 0x1fe0000 && depth.align == 0x10000 &&
stencil.size == 0x870000 && stencil.align == 0x10000 &&
htile.size == 0xa0000 && htile.align == 0x8000,
"captured 3840x2160 depth/stencil/HTile footprint disagrees with "
"Prospero rules");
Require("DepthTargetFootprints", "invalid depth format",
!TileGetDepthSize(960, 540, 0, static_cast<Prospero::DepthFormat>(2u),
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth),
"unsupported depth format was silently admitted");
Require("DepthTargetFootprints", "invalid stencil format",
!TileGetDepthSize(960, 540, 0, Prospero::DepthFormat::kZ32F,
static_cast<Prospero::StencilFormat>(2u), true,
stencil, htile, depth),
"unsupported stencil format was silently admitted");
Require("DepthTargetFootprints", "invalid extent",
!TileGetDepthSize(0, 540, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth) &&
!TileGetDepthSize(16385, 540, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth),
"invalid HTile extent was silently admitted");
std::printf("[host] %-32s ok\n", "DepthTargetFootprints");
}
struct SlotLifetimeProbe {
explicit SlotLifetimeProbe(bool *destroyed) : destroyed(destroyed) {
if (destroyed == nullptr || *destroyed) {
EXIT("fence-lifetime probe has invalid construction state\n");
}
}
~SlotLifetimeProbe() { *destroyed = true; }
bool *destroyed = nullptr;
};
void CheckSlotVectorLifetime() {
bool destroyed = false;
Common::SlotVector<SlotLifetimeProbe> slots;
const auto first = slots.insert(&destroyed);
Require("SlotVectorLifetime", "inserted",
slots.is_allocated(first) && slots.try_get(first) != nullptr &&
!destroyed,
"slot insertion did not expose its cache-owned object");
slots.erase(first);
const bool first_destroyed = destroyed;
destroyed = false;
const auto second = slots.insert(&destroyed);
Require("SlotVectorLifetime", "generation",
first_destroyed && !destroyed && first.index == second.index &&
first != second && !slots.is_allocated(first) &&
slots.is_allocated(second),
"reused slot accepted a stale cache resource id");
std::printf("[host] %-32s ok\n", "SlotVectorLifetime");
}
void CheckEmbeddedFetchLaneSpill() {
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(0)));
code.push_back(EncodeSmem0(0x02u, 20, 4));
code.push_back(EncodeSmem1(0)); // s_load_dwordx4 s[20:23], s[8:9]
AppendVop3(&code, 0x361u, 6, 20, InlineU32(0)); // v_writelane_b32 v6, s20, 0
code.push_back(EncodeSMovB32(20, InlineU32(0)));
AppendVop3(&code, 0x360u, 20, Vgpr(6),
InlineU32(0)); // v_readlane_b32 s20, v6, 0
code.push_back(EncodeVop2(0x01u, 0, Vgpr(8), 5));
code.push_back(EncodeMubuf0(0x03u, 0, true));
code.push_back(EncodeMubuf1(9, 5, 0));
AppendEnd(&code);
std::array<u32, 11> user_data{};
ShaderVertexInputInfo vertex;
vertex.fetch_embedded = true;
vertex.fetch_buffer_reg = 0;
vertex.resources_num = 1;
vertex.resources[0].fields[3] =
BufferFormat(Prospero::BufferFormat::k32_32_32_32Float) << 12u | DstSel(4, 5, 6, 7);
vertex.resources_dst[0].attr_id = 0;
vertex.resources_dst[0].registers_num = 4;
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Vertex;
options.user_data_base = 8;
options.user_data = user_data;
options.input_info.vertex = &vertex;
auto translated = ShaderRecompiler::TranslateProgram(code, options);
auto resource_plan =
ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot resources;
ShaderRecompiler::IR::ResourceSpecialization specialization;
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = options.user_data,
.shader_base = reinterpret_cast<uint64_t>(code.data()),
};
Require("EmbeddedFetchLaneSpill", "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
resource_plan, runtime, resources, specialization),
"lane-spilled fetch resources could not be materialized");
auto result = ShaderRecompiler::CompileProgram(
std::move(translated), options, specialization);
Require("EmbeddedFetchLaneSpill", "fetch rewrite",
result.program.info.vertex_fetch_components[0] == 4,
"lane-spilled fetch descriptor was not recognized and rewritten");
std::printf("[host] %-32s ok\n", "EmbeddedFetchLaneSpill");
}
void CheckReferenceClockScale() {
uint64_t value = 0;
Require("ReferenceClockScale", "zero",
Sync::ScaleReferenceClock(0, 3000000000ull, value) && value == 0,
"zero host tick did not produce a zero GPU clock");
Require("ReferenceClockScale", "fractional second",
Sync::ScaleReferenceClock(1500000000ull, 3000000000ull, value) &&
value == 50000000ull,
"host half-second did not scale to 50,000,000 ticks");
Require("ReferenceClockScale", "whole and fractional",
Sync::ScaleReferenceClock(3750000000ull, 3000000000ull, value) &&
value == 125000000ull,
"host 1.25 seconds did not scale to 125,000,000 ticks");
Require("ReferenceClockScale", "monotonic floor",
Sync::ScaleReferenceClock(3750000001ull, 3000000000ull, value) &&
value == 125000000ull,
"sub-reference-tick increment did not use a monotonic floor");
Require("ReferenceClockScale", "guards",
!Sync::ScaleReferenceClock(1, 0, value) &&
!Sync::ScaleReferenceClock(UINT64_MAX, 1, value),
"invalid frequency or overflow was accepted");
std::printf("[host] %-32s ok\n", "ReferenceClockScale");
}
void CheckClipControlDepthClipState() {
HW::ClipControl clip;
Require("ClipControlDepthClipState", "default", clip.IsZClipEnabled(),
"default paired Z clipping was not enabled");
clip.min_z_clip_disable = true;
Require("ClipControlDepthClipState", "asymmetric near",
!clip.IsZClipEnabled(),
"near-plane disable did not disable paired host Z clipping");
clip.min_z_clip_disable = false;
clip.max_z_clip_disable = true;
Require("ClipControlDepthClipState", "asymmetric far", !clip.IsZClipEnabled(),
"far-plane disable did not disable paired host Z clipping");
clip.min_z_clip_disable = true;
Require("ClipControlDepthClipState", "both disabled", !clip.IsZClipEnabled(),
"paired Z-clip disable was not represented");
std::printf("[host] %-32s ok\n", "ClipControlDepthClipState");
}
void CheckVulkan13FeatureRequirements() {
const auto features = WindowContext::RequiredVulkan13Features();
Require(
"Vulkan13FeatureRequirements", "production requirements",
features.sType == vk::StructureType::ePhysicalDeviceVulkan13Features &&
features.pNext == nullptr && features.dynamicRendering == VK_TRUE &&
features.synchronization2 == VK_TRUE,
"production device creation did not require dynamic rendering and "
"synchronization2 from one feature declaration");
std::printf("[host] %-32s ok\n", "Vulkan13FeatureRequirements");
}
void CheckPm4AcquireMemNoOp(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
const std::array<uint32_t, 6> standard_payload{0x00400000u, 1u, 0u,
0u, 0u, 10u};
const std::array<uint32_t, 7> custom_payload{
0xdeadbeefu, 0xffffffffu, 0x80000000u, 0x13579bdfu,
0x2468ace0u, 0xaaaaaaaau, 0x55555555u};
const auto standard_before = standard_payload;
const auto custom_before = custom_payload;
Require("Pm4AcquireMemNoOp", "recognized packets",
CpOpAcquireMem(processor, 0xC0055800u, standard_payload.data(), 0,
0) == 6 &&
CpOpAcquireMem(processor, 0xc0061050u, custom_payload.data(), 0,
0) == 7 &&
standard_payload == standard_before &&
custom_payload == custom_before,
"ACQUIRE_MEM did not consume both pinned packet forms as "
"side-effect-free "
"no-ops");
std::printf("[host] %-32s ok\n", "Pm4AcquireMemNoOp");
}
void CheckPm4SyntheticOcclusionCounterDump(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
constexpr uint64_t untouched = 0x1122334455667788ull;
constexpr uint64_t ready_bit = 1ull << 63u;
alignas(16) std::array<std::array<uint64_t, 2>, 16> results;
for (auto &pair : results) {
pair.fill(untouched);
}
const auto dump = [&](uint64_t *address) {
const auto raw_address = reinterpret_cast<uint64_t>(address);
std::array<uint32_t, 4> packet{
KYTY_PM4(4, Pm4::IT_EVENT_WRITE, Pm4::R_ZERO), 0x00000139u,
static_cast<uint32_t>(raw_address),
static_cast<uint32_t>(raw_address >> 32u)};
Pm4Execution execution;
return processor.Process(execution, packet);
};
const auto begin_result = dump(&results[0][0]);
bool begin_written = begin_result == Pm4ProcessResult::Complete;
for (const auto &pair : results) {
begin_written &= pair[0] == ready_bit && pair[1] == untouched;
}
const auto end_result = dump(&results[0][1]);
bool end_written = end_result == Pm4ProcessResult::Complete;
uint64_t visible_samples = 0;
for (const auto &pair : results) {
end_written &= pair[0] == ready_bit && pair[1] == (ready_bit | 1u);
visible_samples += pair[1] - pair[0];
}
Require("Pm4SyntheticOcclusionCounterDump", "always-visible result",
begin_written && end_written && visible_samples == results.size(),
"EVENT_WRITE did not publish valid nonzero begin/end counters to "
"every PS5 DB");
std::printf("[host] %-32s ok\n", "Pm4SyntheticOcclusionCounterDump");
}
void CheckPm4Predication(RenderContext &renderer) {
constexpr const char *name = "Pm4Predication";
constexpr uint64_t ready = 1ull << 63u;
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
processor.BufferInit();
alignas(16) std::array<uint64_t, 32> query{};
alignas(16) uint64_t boolean = 0;
uint32_t predicated = 0;
uint32_t unconditional = 0;
const auto commands = [&](uint32_t op, uint32_t condition, uint32_t wait,
const void *source) {
const auto address = reinterpret_cast<uint64_t>(source);
const auto predicated_address = reinterpret_cast<uint64_t>(&predicated);
const auto unconditional_address = reinterpret_cast<uint64_t>(&unconditional);
return std::array<uint32_t, 14>{
// Captured packet form: c0022000 00010100 <query low> <query high>.
0xc0022000u, (op << 16u) | (wait << 12u) | (condition << 8u),
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0) | 1u, 0,
static_cast<uint32_t>(predicated_address),
static_cast<uint32_t>(predicated_address >> 32u), 11,
KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0), 0,
static_cast<uint32_t>(unconditional_address),
static_cast<uint32_t>(unconditional_address >> 32u), 22};
};
const auto check = [&](const char *stage, uint32_t op, uint32_t condition,
uint32_t wait, const void *source, bool skip) {
predicated = unconditional = 0;
const auto packet = commands(op, condition, wait, source);
const auto tick = renderer.GetCommandScheduler().CurrentTick();
Pm4Execution execution;
Require(name, stage,
processor.Process(execution, packet) == Pm4ProcessResult::Complete &&
processor.ShouldSkipPredicatedPackets() == skip &&
predicated == (skip ? 0u : 11u) && unconditional == 22,
"predicate polarity, tagged packet execution, or packet consumption is wrong");
Require(name, stage, renderer.GetCommandScheduler().CurrentTick() == tick,
"predication unnecessarily submitted and drained the host queue");
};
struct Case {
uint64_t delta;
uint32_t condition;
bool skip;
};
constexpr Case cases[]{{1, 1, false}, {1, 0, true},
{0, 1, true}, {0, 0, false}};
for (const auto &test : cases) {
query.fill(ready | 0x344u);
for (size_t db = 0; db < 16; db++) {
query[db * 2 + 1] += test.delta;
}
boolean = test.delta;
for (uint32_t wait : {0u, 1u}) {
check("ready Z-pass", 1, test.condition, wait, query.data(), test.skip);
check("bool preservation", 3, test.condition, wait, &boolean, test.skip);
}
}
query.fill(ready | 0x344u);
query[31]++;
check("only last DB visible", 1, 1, 0, query.data(), false);
check("inverse last DB visibility", 1, 0, 0, query.data(), true);
check("clear", 0, 0, 0, nullptr, false);
// Every begin AND end counter must be ready; no-wait must discard old skip state.
for (size_t missing = 0; missing < query.size(); missing++) {
query.fill(ready | 0x344u);
query[missing] &= ~ready;
for (uint32_t condition : {0u, 1u}) {
boolean = 0;
processor.SetPredication(1, 3, 0, &boolean, 0);
check("unavailable no-wait", 1, condition, 1, query.data(), false);
}
}
// Retry the same packet when its last DB's begin or end arrives later.
for (size_t missing : {30u, 31u}) {
query.fill(ready | 0x344u);
query[31]++;
query[missing] &= ~ready;
boolean = 0;
processor.SetPredication(1, 3, 0, &boolean, 0);
predicated = unconditional = 0;
const auto packet = commands(1, 1, 0, query.data());
Pm4Execution execution;
for (int retry = 0; retry < 2; retry++) {
Require(name, "pending wait suspends",
processor.Process(execution, packet) == Pm4ProcessResult::Blocked &&
!execution.MadeProgress() && predicated == 0 && unconditional == 0 &&
processor.ShouldSkipPredicatedPackets(),
"pending query advanced its packet or changed the prior predicate");
}
query[missing] |= ready;
Require(name, "ready wait resumes",
processor.Process(execution, packet) == Pm4ProcessResult::Complete &&
execution.MadeProgress() && predicated == 11 && unconditional == 22 &&
!processor.ShouldSkipPredicatedPackets(),
"ready query did not reevaluate and execute the suspended packet's suffix");
}
processor.BufferWait();
std::printf("[host] %-32s ok\n", name);
}
void CheckPm4StencilInfoValueLane(RenderContext &renderer) {
CommandProcessor processor(renderer, 0);
constexpr std::array<uint32_t, 2> payload{0x00100801u, 0x28000000u};
const auto consumed = HwCtxSetStencilInfo(
processor, 0xC0016900u, Pm4::DB_STENCIL_INFO, payload.data(), 1);
const auto &stencil = processor.GetCtx().GetDepthStencilInfo();
Require("Pm4StencilInfoValueLane", "standalone register value",
consumed == 1 && stencil.format == Prospero::StencilFormat::k8UInt &&
stencil.texture_compatibility ==
Prospero::TextureCompatibleStencil::kEnable &&
!stencil.expclear_enabled && !stencil.htile_stencil_disabled,
"standalone DB_STENCIL_INFO did not decode its sole payload value");
std::printf("[host] %-32s ok\n", "Pm4StencilInfoValueLane");
}
void CheckPm4NativeTargetGeometryRegisters(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor packet(renderer, 0);
CommandProcessor indirect(renderer, 0);
const auto write_context = [](CommandProcessor &processor, uint32_t offset,
uint32_t value) {
std::array<uint32_t, 3> command{
KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO), offset, value};
Pm4Execution execution;
return processor.Process(execution, command) == Pm4ProcessResult::Complete;
};
constexpr uint32_t color_slot = 7;
constexpr uint32_t width_minus_one = 1919;
constexpr uint32_t height_minus_one = 1079;
constexpr uint32_t depth_minus_one = 5;
constexpr uint32_t max_mip = 3;
constexpr uint32_t dimension_3d = 2;
constexpr uint32_t attrib2_value =
height_minus_one | (width_minus_one << 14u) | (max_mip << 28u);
constexpr uint32_t attrib3_value =
depth_minus_one |
(static_cast<uint32_t>(Prospero::TileMode::kStandard64KB) << 14u) |
(dimension_3d << 24u) | (1u << 26u) | (1u << 30u);
const bool color_packets_complete =
write_context(packet, Pm4::CB_COLOR7_ATTRIB2, attrib2_value) &&
write_context(packet, Pm4::CB_COLOR7_ATTRIB3, attrib3_value);
g_hw_ctx_indirect_func[Pm4::CB_COLOR7_ATTRIB2](
indirect, Pm4::CB_COLOR7_ATTRIB2, attrib2_value);
g_hw_ctx_indirect_func[Pm4::CB_COLOR7_ATTRIB3](
indirect, Pm4::CB_COLOR7_ATTRIB3, attrib3_value);
const auto color_matches = [](const HW::RenderTarget &target) {
return target.attrib2.width == width_minus_one &&
target.attrib2.height == height_minus_one &&
target.attrib2.num_mip_levels == max_mip &&
target.attrib3.depth == depth_minus_one &&
target.attrib3.tile_mode == Prospero::TileMode::kStandard64KB &&
target.attrib3.dimension == dimension_3d &&
target.attrib3.metadata_pipe_aligned;
};
Require("Pm4NativeTargetGeometry", "sparse color geometry",
color_packets_complete &&
color_matches(packet.GetCtx().GetRenderTarget(color_slot)) &&
color_matches(indirect.GetCtx().GetRenderTarget(color_slot)) &&
!color_matches(packet.GetCtx().GetRenderTarget(0)) &&
!color_matches(indirect.GetCtx().GetRenderTarget(0)),
"ATTRIB2/ATTRIB3 did not provide native color extent and layout");
constexpr uint32_t depth_size_value =
width_minus_one | (height_minus_one << 16u);
const bool depth_packet_complete =
write_context(packet, Pm4::DB_DEPTH_SIZE_XY, depth_size_value);
g_hw_ctx_indirect_func[Pm4::DB_DEPTH_SIZE_XY](indirect, Pm4::DB_DEPTH_SIZE_XY,
depth_size_value);
const auto depth_matches = [](const HW::DepthRenderTarget &target) {
return target.size.valid && target.size.x_max == width_minus_one &&
target.size.y_max == height_minus_one;
};
Require("Pm4NativeTargetGeometry", "native depth geometry",
depth_packet_complete &&
depth_matches(packet.GetCtx().GetDepthRenderTarget()) &&
depth_matches(indirect.GetCtx().GetDepthRenderTarget()),
"DB_DEPTH_SIZE_XY did not provide the native depth extent");
bool legacy_slots_are_unhandled = true;
for (uint32_t slot = 0; slot < 8; slot++) {
const auto stride = slot * 15u;
const std::array legacy_color_slots{
Pm4::CB_COLOR0_BASE + stride + 1u,
Pm4::CB_COLOR0_BASE + stride + 2u,
Pm4::CB_COLOR0_CMASK + stride + 1u,
Pm4::CB_COLOR0_FMASK + stride + 1u,
};
for (const auto offset : legacy_color_slots) {
legacy_slots_are_unhandled &= g_hw_ctx_func[offset] == nullptr &&
g_hw_ctx_indirect_func[offset] == nullptr;
}
}
// These offsets are the removed GCN DB_DEPTH_INFO, DB_DEPTH_SIZE, and
// DB_DEPTH_SLICE holes. Keep them numeric so deleting their legacy PM4 names
// does not weaken this dispatch-table regression check.
constexpr std::array<uint32_t, 3> legacy_depth_slots{0x000fu, 0x0016u,
0x0017u};
for (const auto offset : legacy_depth_slots) {
legacy_slots_are_unhandled &= g_hw_ctx_func[offset] == nullptr &&
g_hw_ctx_indirect_func[offset] == nullptr;
}
// Removed GCN PA_SC_NGG_MODE_CNTL.
legacy_slots_are_unhandled &= g_hw_ctx_func[0x0314u] == nullptr &&
g_hw_ctx_indirect_func[0x0314u] == nullptr;
// Removed GCN shader resource/checksum/queue registers. Numeric offsets keep
// this check independent of the deleted legacy names.
constexpr std::array<uint32_t, 17> legacy_shader_slots{
0x000u, 0x001u, 0x030u, 0x0b0u, 0x0bcu, 0x130u,
0x14au, 0x14bu, 0x20eu, 0x20fu, 0x210u, 0x211u, 0x216u, 0x217u,
0x219u, 0x21au, 0x27du,
};
for (const auto offset : legacy_shader_slots) {
legacy_slots_are_unhandled &= g_hw_sh_func[offset] == nullptr &&
g_hw_sh_indirect_func[offset] == nullptr;
}
// Restoring shader state after Toolkit rendering still emits the compiler's
// GS-front allocation metadata at 0xca.
for (uint32_t offset = 0x0cbu; offset <= 0x0ebu; offset++) {
legacy_slots_are_unhandled &= g_hw_sh_func[offset] == nullptr &&
g_hw_sh_indirect_func[offset] == nullptr;
}
std::array<uint32_t, 3> index_size_packet{
KYTY_PM4(3, Pm4::IT_SET_UCONFIG_REG_INDEX, Pm4::R_ZERO),
0x20000000u | Pm4::VGT_INDEX_TYPE,
0x441u,
};
Pm4Execution index_size_execution;
const bool native_index_size_packet_complete =
packet.Process(index_size_execution, index_size_packet) ==
Pm4ProcessResult::Complete &&
g_hw_uc_func[Pm4::VGT_INDEX_TYPE] != nullptr &&
g_hw_uc_indirect_func[Pm4::VGT_INDEX_TYPE] != nullptr &&
g_hw_uc_func[Pm4::IA_MULTI_VGT_PARAM] != nullptr &&
g_hw_uc_indirect_func[Pm4::IA_MULTI_VGT_PARAM] != nullptr;
const bool native_ps_address_slots_are_handled =
g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS] != nullptr &&
g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_PS] != nullptr &&
g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS] != nullptr &&
g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_PS] != nullptr;
const bool native_pace_slots_are_handled =
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PACE_ID_PS] != nullptr &&
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PACE_ID_GS] != nullptr &&
g_hw_sh_indirect_func[Pm4::COMPUTE_PACE_ID] != nullptr;
Require("Pm4NativeTargetGeometry", "legacy register holes",
legacy_slots_are_unhandled && native_pace_slots_are_handled &&
native_ps_address_slots_are_handled && native_index_size_packet_complete,
"a removed GCN register has a handler or a native packet is not handled");
std::printf("[host] %-32s ok\n", "Pm4NativeTargetGeometry");
}
void CheckPm4IndirectShaderRegisters(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
std::array<uint32_t, 24> registers{
Pm4::SPI_SHADER_PGM_RSRC4_GS, 0x0badc0deu,
Pm4::SPI_SHADER_PGM_CHKSUM_HS, 0x12345678u,
Pm4::SPI_SHADER_PGM_RSRC4_HS, 0x87654321u,
Pm4::SPI_SHADER_PGM_LO_ES, 0x050025e5u,
Pm4::SPI_SHADER_PGM_HI_ES, 0u,
Pm4::SPI_SHADER_PGM_RSRC1_GS, 0x60000002u,
Pm4::SPI_SHADER_PGM_RSRC2_GS, 0x00030008u,
0x000000cau, 0x03000002u,
Pm4::SPI_SHADER_PGM_LO_PS, 0x05000104u,
Pm4::SPI_SHADER_PGM_HI_PS, 0u,
Pm4::SPI_SHADER_PGM_RSRC1_PS, 0x022c018du,
Pm4::SPI_SHADER_PGM_RSRC2_PS, 0x0000003cu,
};
const auto address = reinterpret_cast<uint64_t>(registers.data());
std::array<uint32_t, 5> command{
KYTY_PM4(5, Pm4::IT_SET_SH_REG_INDIRECT, Pm4::R_ZERO),
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
0x80000000u, static_cast<uint32_t>(registers.size() / 2u),
};
Pm4Execution execution;
const bool handlers_present =
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_GS] != nullptr &&
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_CHKSUM_HS] != nullptr &&
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_HS] != nullptr;
Require("Pm4IndirectShaderRegisters", "indirect shader register stream",
handlers_present &&
processor.Process(execution, command) == Pm4ProcessResult::Complete,
"shader registers were rejected by SET_SH_REG_INDIRECT");
const auto &vs = processor.GetShCtx().GetVs();
const auto &ps = processor.GetShCtx().GetPs().ps_regs;
Require("Pm4IndirectShaderRegisters", "native state around front metadata",
vs.es_regs.data_addr == 0x50025e500ull &&
vs.gs_regs.rsrc1.vgprs == 2u &&
vs.gs_regs.rsrc1.gs_vgpr_component_count == 3u &&
vs.gs_regs.rsrc2.es_vgpr_component_count == 3u &&
vs.gs_regs.rsrc2.user_sgpr == 4u &&
ps.data_addr == 0x500010400ull && ps.rsrc1.vgprs == 13u &&
ps.rsrc2.user_sgpr == 30u,
"GS-front metadata changed native GS state or prevented later PS writes");
std::printf("[host] %-32s ok\n", "Pm4IndirectShaderRegisters");
}
void CheckPm4IndirectUconfigRegisters(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
std::array<uint32_t, 2> registers{
Pm4::UC_PARAMETER_OVERSUBSCRIPTION, 0x12345678u,
};
const auto address = reinterpret_cast<uint64_t>(registers.data());
std::array<uint32_t, 5> command{
KYTY_PM4(5, Pm4::IT_SET_UCONFIG_REG_INDIRECT, Pm4::R_ZERO),
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
0x80000000u, 1u,
};
Pm4Execution execution;
const bool handler_present =
g_hw_uc_indirect_func[Pm4::UC_PARAMETER_OVERSUBSCRIPTION] != nullptr;
Require("Pm4IndirectUconfigRegisters", "indirect UCONFIG register stream",
handler_present &&
processor.Process(execution, command) == Pm4ProcessResult::Complete,
"UCONFIG register was rejected by SET_UCONFIG_REG_INDIRECT");
std::printf("[host] %-32s ok\n", "Pm4IndirectUconfigRegisters");
}
void CheckPm4DirectShaderRegisterFallback(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
constexpr uint64_t shader_address = 0x0000123456789a00ull;
std::array<uint32_t, 4> packet{
KYTY_PM4(4, Pm4::IT_SET_SH_REG, Pm4::R_ZERO),
Pm4::SPI_SHADER_PGM_LO_ES,
static_cast<uint32_t>(shader_address >> 8u),
static_cast<uint32_t>(shader_address >> 40u),
};
Pm4Execution execution;
Require("Pm4DirectShaderRegisterFallback", "ES program base",
processor.Process(execution, packet) == Pm4ProcessResult::Complete &&
processor.GetShCtx().GetVs().es_regs.data_addr == shader_address,
"direct SET_SH_REG did not route its ES base pair through the "
"register handlers");
std::printf("[host] %-32s ok\n", "Pm4DirectShaderRegisterFallback");
}
void CheckPm4GuardBandRegisterRanges(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
processor.GetCtx().SetGuardBands(2.0f, 3.0f, 4.0f, 5.0f);
std::array<uint32_t, 3> single{
KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::PA_CL_GB_VERT_CLIP_ADJ,
std::bit_cast<uint32_t>(6.0f),
};
Pm4Execution single_execution;
const auto single_result = processor.Process(single_execution, single);
const auto &single_viewport = processor.GetCtx().GetScreenViewport();
Require("Pm4GuardBandRegisterRanges", "single register",
single_result == Pm4ProcessResult::Complete &&
single_viewport.guard_band_horz_clip == 2.0f &&
single_viewport.guard_band_vert_clip == 6.0f &&
single_viewport.guard_band_horz_discard == 4.0f &&
single_viewport.guard_band_vert_discard == 5.0f,
"single guard-band write changed the wrong context state");
std::array<uint32_t, 6> range{
KYTY_PM4(6, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::PA_CL_GB_VERT_CLIP_ADJ,
std::bit_cast<uint32_t>(7.0f),
std::bit_cast<uint32_t>(8.0f),
std::bit_cast<uint32_t>(9.0f),
std::bit_cast<uint32_t>(10.0f),
};
Pm4Execution range_execution;
const auto range_result = processor.Process(range_execution, range);
const auto &range_viewport = processor.GetCtx().GetScreenViewport();
Require("Pm4GuardBandRegisterRanges", "full range",
range_result == Pm4ProcessResult::Complete &&
range_viewport.guard_band_horz_clip == 9.0f &&
range_viewport.guard_band_vert_clip == 7.0f &&
range_viewport.guard_band_horz_discard == 10.0f &&
range_viewport.guard_band_vert_discard == 8.0f,
"full guard-band range did not preserve hardware register order");
std::printf("[host] %-32s ok\n", "Pm4GuardBandRegisterRanges");
}
void CheckPm4BlendColorRegisterRanges(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
processor.GetCtx().SetBlendColor({1.0f, 2.0f, 3.0f, 4.0f});
constexpr std::array<std::pair<uint32_t, float>, 4> singleton_values{{
{Pm4::CB_BLEND_RED, 5.0f},
{Pm4::CB_BLEND_BLUE, 7.0f},
{Pm4::CB_BLEND_GREEN, 6.0f},
{Pm4::CB_BLEND_ALPHA, 8.0f},
}};
bool singletons_complete = true;
for (const auto &[offset, value] : singleton_values) {
std::array<uint32_t, 3> packet{
KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
offset,
std::bit_cast<uint32_t>(value),
};
Pm4Execution execution;
singletons_complete &=
processor.Process(execution, packet) == Pm4ProcessResult::Complete;
}
const auto &singletons = processor.GetCtx().GetBlendColor();
Require("Pm4BlendColorRegisterRanges", "single registers",
singletons_complete && singletons.red == 5.0f &&
singletons.green == 6.0f && singletons.blue == 7.0f &&
singletons.alpha == 8.0f,
"single blend-color writes did not follow native PS5 register order");
std::array<uint32_t, 4> partial{
KYTY_PM4(4, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::CB_BLEND_BLUE,
std::bit_cast<uint32_t>(9.0f),
std::bit_cast<uint32_t>(10.0f),
};
Pm4Execution partial_execution;
const auto partial_result = processor.Process(partial_execution, partial);
const auto &partial_color = processor.GetCtx().GetBlendColor();
Require("Pm4BlendColorRegisterRanges", "partial range",
partial_result == Pm4ProcessResult::Complete &&
partial_color.red == 5.0f && partial_color.green == 10.0f &&
partial_color.blue == 9.0f && partial_color.alpha == 8.0f,
"partial blend-color range changed the wrong channels");
std::array<uint32_t, 6> full{
KYTY_PM4(6, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::CB_BLEND_RED,
std::bit_cast<uint32_t>(11.0f),
std::bit_cast<uint32_t>(13.0f),
std::bit_cast<uint32_t>(12.0f),
std::bit_cast<uint32_t>(14.0f),
};
Pm4Execution full_execution;
const auto full_result = processor.Process(full_execution, full);
const auto &full_color = processor.GetCtx().GetBlendColor();
Require("Pm4BlendColorRegisterRanges", "full range",
full_result == Pm4ProcessResult::Complete &&
full_color.red == 11.0f && full_color.green == 12.0f &&
full_color.blue == 13.0f && full_color.alpha == 14.0f,
"full blend-color range did not preserve native PS5 register order");
std::printf("[host] %-32s ok\n", "Pm4BlendColorRegisterRanges");
}
void CheckPm4PolygonOffsetRegisters(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
const auto &defaults = processor.GetCtx().GetPolyOffset();
Require("Pm4PolygonOffset", "format defaults",
defaults.neg_num_db_bits == -23 && defaults.db_is_float_fmt,
"polygon offset Z format did not default to D32F");
constexpr std::array<uint32_t, 6> payload{0x000000f0u,
std::bit_cast<uint32_t>(2.5f),
std::bit_cast<uint32_t>(16.0f),
std::bit_cast<uint32_t>(-3.25f),
std::bit_cast<uint32_t>(32.0f),
std::bit_cast<uint32_t>(4.5f)};
const auto consumed = HwCtxSetPolyOffsetRegisters(
processor, KYTY_PM4(8, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL, payload.data(), 0);
const auto &direct = processor.GetCtx().GetPolyOffset();
Require("Pm4PolygonOffset", "direct registers",
consumed == payload.size() && direct.neg_num_db_bits == -16 &&
!direct.db_is_float_fmt && direct.clamp == 2.5f &&
direct.front_scale == 16.0f && direct.front_offset == -3.25f &&
direct.back_scale == 32.0f && direct.back_offset == 4.5f,
"direct polygon offset register packet was decoded incorrectly");
g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL](
processor, Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL, 0x000001e9u);
g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE](
processor, Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE,
std::bit_cast<uint32_t>(24.0f));
const auto &indirect = processor.GetCtx().GetPolyOffset();
Require("Pm4PolygonOffset", "indirect registers",
indirect.neg_num_db_bits == -23 && indirect.db_is_float_fmt &&
indirect.front_scale == 24.0f && indirect.front_offset == -3.25f,
"indirect polygon offset register write was decoded incorrectly");
std::printf("[host] %-32s ok\n", "Pm4PolygonOffset");
}
void CheckPm4DepthControlHighBits(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
std::array<uint32_t, 3> direct{
KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::DB_DEPTH_CONTROL,
0x80000072u,
};
Pm4Execution direct_execution;
const bool direct_complete =
processor.Process(direct_execution, direct) == Pm4ProcessResult::Complete;
const auto &direct_control = processor.GetCtx().GetDepthControl();
const bool direct_decoded =
!direct_control.stencil_enable && direct_control.z_enable &&
!direct_control.z_write_enable && !direct_control.depth_bounds_enable &&
direct_control.zfunc == 7u && !direct_control.backface_enable &&
direct_control.stencilfunc == 0u && direct_control.stencilfunc_bf == 0u;
std::array<uint32_t, 2> registers{
Pm4::DB_DEPTH_CONTROL,
0x4070078du,
};
const auto address = reinterpret_cast<uint64_t>(registers.data());
std::array<uint32_t, 5> indirect{
KYTY_PM4(5, Pm4::IT_SET_CONTEXT_REG_INDIRECT, Pm4::R_ZERO),
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
0x80000000u, 1u,
};
Pm4Execution indirect_execution;
const bool indirect_complete =
processor.Process(indirect_execution, indirect) ==
Pm4ProcessResult::Complete;
const auto &indirect_control = processor.GetCtx().GetDepthControl();
const bool indirect_decoded =
indirect_control.stencil_enable && !indirect_control.z_enable &&
indirect_control.z_write_enable && indirect_control.depth_bounds_enable &&
indirect_control.zfunc == 0u && indirect_control.backface_enable &&
indirect_control.stencilfunc == 7u &&
indirect_control.stencilfunc_bf == 7u;
Require("Pm4DepthControlHighBits", "preserved high bits",
direct_complete && direct_decoded && indirect_complete &&
indirect_decoded,
"DB_DEPTH_CONTROL high bits rejected or changed decoded PS5 state");
std::printf("[host] %-32s ok\n", "Pm4DepthControlHighBits");
}
void CheckPm4DepthRenderOverride(RenderContext &renderer) {
constexpr const char *name = "Pm4DepthRenderOverride";
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
struct Case {
uint32_t value;
HW::DepthRenderOverride expected;
};
constexpr std::array cases{Case{0x28000000u, {true, true, false, false}},
Case{0x50000000u, {false, false, true, true}},
Case{0x18000000u, {false, true, false, true}},
Case{0x60000000u, {true, false, true, false}},
Case{0xf8000000u, {true, true, true, true}},
Case{0x82001a0fu, {false, false, false, false}}};
for (const bool indirect : {false, true}) {
for (const auto &test : cases) {
std::array<uint32_t, 2> registers{Pm4::DB_RENDER_OVERRIDE, test.value};
const auto address = reinterpret_cast<uint64_t>(registers.data());
const std::array<uint32_t, 3> direct_packet{
KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::DB_RENDER_OVERRIDE, test.value};
const std::array<uint32_t, 5> indirect_packet{
KYTY_PM4(5, Pm4::IT_SET_CONTEXT_REG_INDIRECT, Pm4::R_ZERO),
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
0x80000000u, 1u};
Pm4Execution execution;
const auto result = processor.Process(
execution, indirect ? std::span<const uint32_t>{indirect_packet}
: std::span<const uint32_t>{direct_packet});
const auto &actual = processor.GetCtx().GetDepthRenderOverride();
Require(
name, indirect ? "indirect register" : "direct register",
result == Pm4ProcessResult::Complete &&
actual.force_z_valid == test.expected.force_z_valid &&
actual.force_z_dirty == test.expected.force_z_dirty &&
actual.force_stencil_valid == test.expected.force_stencil_valid &&
actual.force_stencil_dirty == test.expected.force_stencil_dirty,
"DB_RENDER_OVERRIDE lost, exchanged, or retained force flags");
}
}
std::printf("[host] %-32s ok\n", name);
}
void CheckShaderFusion() {
struct Case {
bool gs;
uint32_t front1, front2, back1, back2, fused1, fused2, older2;
};
// Expected register results for both shader-fusion variants.
constexpr Case cases[]{
{true, 0x600c0007, 0x00030010, 0x600c0007, 0x008b0000,
0x600c0007, 0x008b0010, 0x008b0010}, // Captured volume ES + GS.
{true, 0x20ac0003, 0xf805004a, 0x40ec0007, 0x028b003e,
0x40ec0007, 0x1a8f000a, 0xfa8f000a},
{false, 0x20ac0003, 0xf805004a, 0x10ec0007, 0x028b003e,
0x20ec0007, 0x1a8b000a, 0xfa8b000a},
};
ShaderUserData user_data{};
ShaderSpecialRegs specials{};
for (const auto &c : cases) {
const auto checksum = c.gs ? Pm4::SPI_SHADER_PGM_CHKSUM_GS : Pm4::SPI_SHADER_PGM_CHKSUM_HS;
const auto rsrc1 = c.gs ? Pm4::SPI_SHADER_PGM_RSRC1_GS : Pm4::SPI_SHADER_PGM_RSRC1_HS;
const auto rsrc2 = c.gs ? Pm4::SPI_SHADER_PGM_RSRC2_GS : Pm4::SPI_SHADER_PGM_RSRC2_HS;
const auto lo = c.gs ? Pm4::SPI_SHADER_PGM_LO_ES : Pm4::SPI_SHADER_PGM_LO_LS;
std::array<ShaderRegister, 6> front_regs{{
{checksum, 0x1234}, {checksum, 0x5678}, {rsrc1, c.front1},
{rsrc2, c.front2}, {lo, 0}, {lo + 1u, 0}}};
const std::array<ShaderRegister, 6> back_regs{{
{checksum, 0}, {checksum, 0}, {rsrc1, c.back1},
{rsrc2, c.back2}, {lo, 0}, {lo + 1u, 0xabcdef00}}};
Shader front{};
front.user_data = &user_data;
front.code = reinterpret_cast<const void *>(0x12580e91c00ull);
front.sh_registers = front_regs.data();
front.specials = &specials;
front.type = static_cast<uint8_t>(c.gs ? Prospero::ShaderBinaryType::kGsFront
: Prospero::ShaderBinaryType::kHsFront);
front.num_sh_registers = front_regs.size();
for (const bool older : {false, true}) {
for (const bool in_place : {false, true}) {
auto registers = back_regs;
std::array<ShaderRegister, 6> scratch{};
Shader back = front;
back.sh_registers = registers.data();
back.type = static_cast<uint8_t>(c.gs ? Prospero::ShaderBinaryType::kGsBack
: Prospero::ShaderBinaryType::kHsBack);
Shader fused{};
const auto fuse = older ? Gen5::AgcUnknownNApJjpKNBl4 : Gen5::AgcUnknownFuseShaderHalves;
const auto result = fuse(&fused, &front, &back, in_place ? nullptr : scratch.data());
const auto *regs = fused.sh_registers;
Require("ShaderFusion", "native fused registers",
result == 0 && regs == (in_place ? registers.data() : scratch.data()) &&
regs[0].value == 0x1234 && regs[1].value == 0x5678 &&
regs[2].value == c.fused1 && regs[3].value == (older ? c.older2 : c.fused2) &&
regs[4].value == 0x2580e91c && regs[5].value == 0xabcdef01 &&
fused.type == static_cast<uint8_t>(c.gs ? Prospero::ShaderBinaryType::kGs
: Prospero::ShaderBinaryType::kHs) &&
fused.user_data == (older ? &user_data : nullptr) &&
(in_place || std::memcmp(registers.data(), back_regs.data(), sizeof(registers)) == 0),
"shader fusion lost front resources or changed the export's allocation/metadata behavior");
}
}
}
std::printf("[host] %-32s ok\n", "ShaderFusion");
}
struct CommandBufferLayout {
using Callback = KYTY_SYSV_ABI bool (*)(Gen5::CommandBuffer *, uint32_t,
void *);
uint32_t *bottom;
uint32_t *top;
uint32_t *cursor_up;
uint32_t *cursor_down;
Callback callback;
void *user_data;
uint32_t reserved_dw;
};
static_assert(offsetof(CommandBufferLayout, cursor_up) == 0x10);
static_assert(offsetof(CommandBufferLayout, reserved_dw) == 0x30);
void CheckPm4WaitPackets(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
constexpr auto packet_mismatch = static_cast<int>(0x8a6c000cu);
const auto execute = [&](uint32_t *packet, uint32_t size_dw) {
Pm4Execution execution;
return processor.Process(execution, {packet, size_dw}) ==
Pm4ProcessResult::Complete;
};
alignas(8) uint32_t label32 = 0x11223344u;
std::array<uint32_t, 16> packet32{};
CommandBufferLayout dcb32{packet32.data(),
packet32.data() + packet32.size(),
packet32.data(),
packet32.data() + packet32.size(),
nullptr,
nullptr,
0};
const auto address32 = reinterpret_cast<uint64_t>(&label32);
auto *emitted32 =
Gen5::AgcDcbWaitRegMem(reinterpret_cast<Gen5::CommandBuffer *>(&dcb32), 0,
3, 0, 0, &label32, label32, UINT32_MAX, 400);
Require("Pm4WaitPackets", "native 32-bit packet",
Gen5::AgcDcbWaitOnAddressGetSize(0) == 56 &&
emitted32 == packet32.data() &&
dcb32.cursor_up == packet32.data() + 14 &&
packet32[0] == 0xc0027904u && packet32[1] == 0x342u &&
packet32[2] ==
(0xc8010000u |
(static_cast<uint32_t>(address32 >> 32u) & 0xffffu)) &&
packet32[3] == static_cast<uint32_t>(address32) &&
packet32[4] == 0xc0053c00u && packet32[5] == 0x13u &&
packet32[6] == (static_cast<uint32_t>(address32) & ~0x3u) &&
packet32[7] ==
(static_cast<uint32_t>(address32 >> 32u) & 0x3ffffu) &&
packet32[8] == label32 && packet32[9] == UINT32_MAX &&
packet32[10] == 25u && packet32[11] == 0xc0017904u &&
packet32[12] == 0x342u && packet32[13] == 0xc8000000u &&
execute(packet32.data(), 14),
"32-bit wait packet differs from the expected stream");
alignas(8) uint32_t patched_label32 = 0xaabbccddu;
const auto patched_address32 = reinterpret_cast<uint64_t>(&patched_label32);
packet32[6] |= 0x2u;
packet32[7] |= 0xa5a40000u;
Require(
"Pm4WaitPackets", "native 32-bit patch",
Gen5::AgcWaitRegMemPatchAddress(packet32.data(), &patched_label32) == 0 &&
Gen5::AgcWaitRegMemPatchReference(packet32.data(), patched_label32) ==
0 &&
packet32[2] ==
(0xc8010000u |
(static_cast<uint32_t>(patched_address32 >> 32u) & 0xffffu)) &&
packet32[3] == static_cast<uint32_t>(patched_address32) &&
packet32[6] ==
((static_cast<uint32_t>(patched_address32) & ~0x3u) | 0x2u) &&
packet32[7] ==
((static_cast<uint32_t>(patched_address32 >> 32u) & 0x3ffffu) |
0xa5a40000u) &&
packet32[8] == patched_label32 && execute(packet32.data(), 14),
"32-bit native wait patch did not update metadata and hardware packet");
alignas(8) uint64_t label64 = 0x1122334455667788ull;
std::array<uint32_t, 16> packet64{};
CommandBufferLayout dcb64{packet64.data(),
packet64.data() + packet64.size(),
packet64.data(),
packet64.data() + packet64.size(),
nullptr,
nullptr,
0};
const auto address64 = reinterpret_cast<uint64_t>(&label64);
auto *emitted64 =
Gen5::AgcDcbWaitRegMem(reinterpret_cast<Gen5::CommandBuffer *>(&dcb64), 1,
3, 0, 0, &label64, label64, UINT64_MAX, 400);
Require("Pm4WaitPackets", "native 64-bit packet",
Gen5::AgcDcbWaitOnAddressGetSize(1) == 64 &&
Gen5::AgcDcbWaitOnAddressGetSize(2) == 0 &&
emitted64 == packet64.data() &&
dcb64.cursor_up == packet64.data() + packet64.size() &&
packet64[0] == 0xc0027904u && packet64[1] == 0x342u &&
packet64[2] ==
(0xc8020000u |
(static_cast<uint32_t>(address64 >> 32u) & 0xffffu)) &&
packet64[3] == static_cast<uint32_t>(address64) &&
packet64[4] == 0xc0079300u && packet64[5] == 0x13u &&
packet64[6] == (static_cast<uint32_t>(address64) & ~0x7u) &&
packet64[7] ==
(static_cast<uint32_t>(address64 >> 32u) & 0x3ffffu) &&
packet64[8] == static_cast<uint32_t>(label64) &&
packet64[9] == static_cast<uint32_t>(label64 >> 32u) &&
packet64[10] == UINT32_MAX && packet64[11] == UINT32_MAX &&
packet64[12] == 25u && packet64[13] == 0xc0017904u &&
packet64[14] == 0x342u && packet64[15] == 0xc8000000u &&
execute(packet64.data(), packet64.size()),
"64-bit wait packet differs from the expected stream");
alignas(8) uint64_t patched_label64 = 0x11223344aabbccddull;
const auto patched_address64 = reinterpret_cast<uint64_t>(&patched_label64);
packet64[6] |= 0x5u;
packet64[7] |= 0x5a580000u;
Require(
"Pm4WaitPackets", "native 64-bit patch",
Gen5::AgcWaitRegMemPatchAddress(packet64.data(), &patched_label64) == 0 &&
Gen5::AgcWaitRegMemPatchReference(packet64.data(),
0xdeadbeefaabbccddull) == 0 &&
packet64[2] ==
(0xc8020000u |
(static_cast<uint32_t>(patched_address64 >> 32u) & 0xffffu)) &&
packet64[3] == static_cast<uint32_t>(patched_address64) &&
packet64[6] ==
((static_cast<uint32_t>(patched_address64) & ~0x7u) | 0x5u) &&
packet64[7] ==
((static_cast<uint32_t>(patched_address64 >> 32u) & 0x3ffffu) |
0x5a580000u) &&
packet64[8] == 0xaabbccddu && packet64[9] == 0x11223344u &&
execute(packet64.data(), packet64.size()),
"64-bit native wait patch did not preserve the native high reference "
"DWORD");
std::array<uint32_t, 16> invalid{};
const uint32_t short_payload = 0x342u;
Require(
"Pm4WaitPackets", "invalid packet",
Gen5::AgcWaitRegMemPatchAddress(invalid.data(), &label32) ==
packet_mismatch &&
Gen5::AgcWaitRegMemPatchReference(invalid.data(), label32) ==
packet_mismatch &&
!Gen5::AgcIsInternalDataPacket(
KYTY_PM4(2, Pm4::IT_SET_UCONFIG_REG, 1), &short_payload),
"wait patch accepted a packet without the metadata prefix");
std::printf("[host] %-32s ok\n", "Pm4WaitPackets");
}
void CheckPm4DrawIndirectMultiPacket(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
const auto execute = [&](uint32_t *packet, uint32_t size_dw) {
Pm4Execution execution;
return processor.Process(execution, {packet, size_dw}) ==
Pm4ProcessResult::Complete;
};
constexpr uint64_t modifier = 0x0000000069380b1dull;
constexpr auto packet_mismatch = static_cast<int>(0x8a6c000cu);
const auto *count_address = reinterpret_cast<const volatile void *>(
static_cast<uintptr_t>(0x123456789abcdef3ull));
constexpr std::array<uint32_t, 16> expected{
0xc0017904u, 0x00000342u, 0xc6000008u, 0xc0082c00u,
0x11223344u, 0x00000111u, 0x00000113u, 0xc8000115u,
0x55667788u, 0x9abcdef0u, 0x12345678u, 0xaabbccddu,
0x00000022u, 0xc0017904u, 0x00000342u, 0xc6000000u};
std::array<uint32_t, 16> packet{};
CommandBufferLayout dcb{packet.data(),
packet.data() + packet.size(),
packet.data(),
packet.data() + packet.size(),
nullptr,
nullptr,
0};
auto *emitted = Gen5::AgcDcbDrawIndirectMulti(
reinterpret_cast<Gen5::CommandBuffer *>(&dcb), 0x11223344u, 1u,
0x55667788u, count_address, 0xaabbccddu, modifier);
const uint32_t invalid_payload[]{0x342u, 0xc6000010u};
Require("Pm4DrawIndirectMulti", "native packet",
emitted == packet.data() &&
dcb.cursor_up == packet.data() + packet.size() &&
packet == expected && execute(packet.data(), 3u) &&
execute(packet.data() + 13u, 3u) &&
Gen5::AgcWaitRegMemPatchAddress(packet.data(), count_address) ==
packet_mismatch &&
Gen5::AgcWaitRegMemPatchReference(packet.data(), 0x12345678u) ==
packet_mismatch &&
!Gen5::AgcIsInternalDataPacket(0xc0017904u, invalid_payload),
"indirect multi-draw packet differs from the expected stream");
constexpr uint32_t data_offset = 0x5c08;
constexpr uint32_t stride = 32;
std::array<uint32_t, data_offset / 4 + stride / 4 + 5> arguments{};
const std::array<uint32_t, 5> first{0, 1, 3, 0xfffffffeu, 9};
const std::array<uint32_t, 5> second{0, 2, 5, 7, 11};
std::copy(first.begin(), first.end(), arguments.begin() + data_offset / 4);
std::copy(second.begin(), second.end(), arguments.begin() + (data_offset + stride) / 4);
std::array<uint32_t, 8> indices{};
processor.BufferInit();
processor.SetDrawIndirectArgsBaseAddress(reinterpret_cast<uint64_t>(arguments.data()));
processor.SetIndexBaseAddress(reinterpret_cast<uint64_t>(indices.data()));
processor.SetIndexType(0);
std::array<uint32_t, 5> captured{0xc0032500u, data_offset, 0x94, 0x95, 0};
Require("Pm4DrawIndirectMulti", "captured indexed single execution",
execute(captured.data(), captured.size()),
"indexed indirect draw rejected native source-select zero");
// Zero primitive counts still traverse CP argument decoding and renderer entry,
// while avoiding unrelated shader and attachment requirements.
for (const bool indexed : {false, true}) {
for (const bool multi : {false, true}) {
for (const uint64_t draw_modifier : {0ull, 0x100ull, 1ull << 32u, (1ull << 32u) | 0x100ull}) {
packet.fill(0);
dcb.cursor_up = packet.data();
auto *buffer = reinterpret_cast<Gen5::CommandBuffer *>(&dcb);
alignas(4) uint32_t draw_count = 3;
if (multi) {
emitted = indexed ? Gen5::AgcDcbDrawIndexIndirectMulti(buffer, data_offset, 1, 2,
&draw_count, stride, draw_modifier)
: Gen5::AgcDcbDrawIndirectMulti(buffer, data_offset, 1, 2,
&draw_count, stride, draw_modifier);
} else {
emitted = indexed ? Gen5::AgcDcbDrawIndexIndirect(buffer, data_offset, draw_modifier)
: Gen5::AgcDcbDrawIndirect(buffer, data_offset, draw_modifier);
}
const uint32_t begin = multi && !indexed ? 3 : 0;
const uint32_t words = multi ? (indexed ? 10 : 16) : 5;
const uint32_t opcode = multi ? (indexed ? 0xc0083800u : 0xc0082c00u)
: (indexed ? 0xc0032500u : 0xc0032400u);
const uint32_t initiator = (indexed ? 0 : 2) |
(draw_modifier == 0x100 ? 0x20 : 0);
Require("Pm4DrawIndirectMulti", "indexed/auto initiator emission",
emitted == packet.data() && dcb.cursor_up == packet.data() + words &&
packet[begin] == opcode && packet[begin + 1] == data_offset &&
packet[begin + (multi ? 9 : 4)] == initiator,
"indirect opcode, source select, or modifier bits differ from the native packet");
Require("Pm4DrawIndirectMulti", "single/multi argument execution",
execute(packet.data(), words),
"indirect draw failed shared argument decoding or padded/count-limited traversal");
if (multi) {
draw_count = 0;
Require("Pm4DrawIndirectMulti", "zero indirect count execution",
execute(packet.data(), words), "zero indirect draw count did not complete");
}
}
}
}
renderer.GetCommandScheduler().Finish();
processor.BufferInit();
auto &shaders = processor.GetShCtx();
const auto &gs = shaders.GetVs().gs_user_sgpr.value;
const auto &hs = shaders.GetVs().hs_user_sgpr.value;
const auto check_offsets = [&](auto &commands, const auto &sgprs,
std::initializer_list<uint32_t> expected_offsets,
const char *label) {
Require("Pm4DrawIndirectMulti", label,
execute(commands.data(), commands.size()) &&
std::equal(expected_offsets.begin(), expected_offsets.end(), sgprs + 9),
"indirect draw did not write the expected user SGPR offsets");
};
// These locations match the captured NGG prolog: user SGPR9/10 become s17/s18.
shaders.SetGsUserSgpr(9, 17821, HW::UserSgprType::Unknown);
shaders.SetGsUserSgpr(10, 0x898f3f24u, HW::UserSgprType::Unknown);
arguments = {0, 1};
std::array<uint32_t, 5> indexed_packet{
KYTY_PM4(5, Pm4::IT_DRAW_INDEX_INDIRECT, 0), 0,
Pm4::SPI_SHADER_USER_DATA_GS_0 + 9,
Pm4::SPI_SHADER_USER_DATA_GS_0 + 10, 0};
check_offsets(indexed_packet, gs, {0, 0}, "clear stale offsets");
arguments = {0, 1, 5, static_cast<uint32_t>(-17), 3};
indexed_packet[2] |= (Pm4::SPI_SHADER_USER_DATA_GS_0 + 11) << 16u;
// Native sceAgcDcbDrawIndexIndirect (0x5a00) emits first-index enable in word3 bit28.
indexed_packet[3] |= 1u << 28u;
check_offsets(indexed_packet, gs, {static_cast<uint32_t>(-17), 3, 5}, "indexed offsets");
indexed_packet[3] ^= (1u << 28u) | (1u << 27u);
arguments[2] = 6;
check_offsets(indexed_packet, gs, {static_cast<uint32_t>(-17), 3, 5},
"index destination disabled");
arguments = {0, 1, 31, 5};
std::array<uint32_t, 5> auto_packet{
KYTY_PM4(5, Pm4::IT_DRAW_INDIRECT, 0), 0,
Pm4::SPI_SHADER_USER_DATA_HS_0 + 9,
Pm4::SPI_SHADER_USER_DATA_HS_0 + 10, 2};
check_offsets(auto_packet, hs, {31, 5}, "non-indexed offsets");
Require("Pm4DrawIndirectMulti", "shader stage", gs[9] == static_cast<uint32_t>(-17),
"non-indexed draw patched the wrong shader stage");
arguments = {0, 1, 6, 41, 7, 0, 1, 8, 43, 9, 0, 1, 10, 45, 11};
uint32_t count = 2;
const auto multi_count_address = reinterpret_cast<uint64_t>(&count);
// Native sceAgcDcbDrawIndexIndirectMulti (0x5800) puts this enable in word4 bit28.
std::array<uint32_t, 10> multi_packet{
KYTY_PM4(10, Pm4::IT_DRAW_INDEX_INDIRECT_MULTI, 0), 0,
indexed_packet[2], Pm4::SPI_SHADER_USER_DATA_GS_0 + 10,
(1u << 30u) | (1u << 28u) | Pm4::SH_NOP, 3,
static_cast<uint32_t>(multi_count_address),
static_cast<uint32_t>(multi_count_address >> 32u), 20, 0};
check_offsets(multi_packet, gs, {43, 9, 8}, "counted multi-draw offsets");
count = 0;
check_offsets(multi_packet, gs, {43, 9, 8}, "empty multi-draw");
count = 3;
multi_packet[5] = 1;
multi_packet[3] |= 1u << 28u;
multi_packet[4] &= ~(1u << 28u);
check_offsets(multi_packet, gs, {41, 7, 8}, "multi-draw enable word and count clamp");
arguments = {0, 1, 51, 13, 0, 1, 53, 15};
std::array<uint32_t, 10> auto_multi_packet{
KYTY_PM4(10, Pm4::IT_DRAW_INDIRECT_MULTI, 0), 0,
auto_packet[2], auto_packet[3], Pm4::SH_NOP, 2, 0, 0, 16, 2};
check_offsets(auto_multi_packet, hs, {53, 15}, "non-indexed multi-draw offsets");
arguments = {0, 1, 99, 101};
auto_packet[2] = Pm4::SH_NOP;
check_offsets(auto_packet, hs, {53, 101}, "independent destinations");
auto_packet[3] = Pm4::SH_NOP;
arguments[3] = 103;
check_offsets(auto_packet, hs, {53, 101}, "disabled destinations");
processor.BufferFlush();
std::printf("[host] %-32s ok\n", "Pm4DrawIndirectMulti");
}
void CheckPm4ContextStateOperations(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
constexpr auto primitive_type = static_cast<Prospero::PrimitiveType>(0x35u);
HW::RenderControl original_control{};
original_control.depth_clear_enable = true;
original_control.stencil_clear_enable = true;
processor.GetCtx().SetRenderControl(original_control);
processor.GetCtx().SetRenderTargetMask(0x1234abcd);
processor.GetUcfg().SetPrimitiveType(primitive_type);
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_PS](
processor, Pm4::SPI_SHADER_PGM_LO_PS, 0x00123456);
const auto shader_address = processor.GetShCtx().GetPs().ps_regs.data_addr;
const auto invoke = [&](ContextStateOperation operation) {
std::array<uint32_t, 32> packet{};
std::array<uint32_t, 6> segment_sizes{};
CommandBufferLayout dcb{packet.data(),
packet.data() + packet.size(),
packet.data(),
packet.data() + packet.size(),
nullptr,
nullptr,
0};
const auto operation_value = static_cast<uint32_t>(operation);
size_t segment_count = 0;
switch (operation) {
case ContextStateOperation::Clear:
segment_sizes = {5};
segment_count = 1;
break;
case ContextStateOperation::Push:
segment_sizes = {5, 8, 9, 3, 2};
segment_count = 5;
break;
case ContextStateOperation::Pop:
segment_sizes = {3, 5, 8, 9, 2};
segment_count = 5;
break;
case ContextStateOperation::PushClear:
segment_sizes = {5, 8, 9, 3, 2, 5};
segment_count = 6;
break;
}
const auto size_dw = static_cast<size_t>(
Gen5::AgcDcbContextStateOpGetSize(operation_value) / 4);
auto *emitted = Gen5::AgcDcbContextStateOp(
reinterpret_cast<Gen5::CommandBuffer *>(&dcb), operation_value);
bool packet_matches = true;
size_t segment_offset = 0;
for (size_t i = 0; i < segment_count; i++) {
const auto segment_size = segment_sizes[i];
const auto selector = (i == 0 ? Pm4::R_CONTEXT_STATE : Pm4::R_ZERO);
packet_matches &= packet[segment_offset] ==
KYTY_PM4(segment_size, Pm4::IT_NOP, selector);
for (size_t j = 1; j < segment_size; j++) {
packet_matches &= packet[segment_offset + j] ==
(i == 0 && j == 1 ? operation_value : 0u);
}
segment_offset += segment_size;
}
Require("Pm4ContextState", "HLE packet",
emitted == packet.data() &&
dcb.cursor_up == packet.data() + size_dw &&
segment_offset == size_dw && packet_matches,
"context-state HLE packet does not match its real allocation size");
Pm4Execution execution;
return processor.Process(execution, {packet.data(), size_dw});
};
Require("Pm4ContextState", "push-clear",
invoke(ContextStateOperation::PushClear) ==
Pm4ProcessResult::Complete &&
!processor.GetCtx().GetRenderControl().depth_clear_enable &&
!processor.GetCtx().GetRenderControl().stencil_clear_enable &&
processor.GetCtx().GetRenderTargetMask() ==
HW::Context{}.GetRenderTargetMask() &&
processor.GetUcfg().GetPrimType() == primitive_type &&
processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address,
"push-clear did not save and clear only Cx state");
processor.GetCtx().SetRenderTargetMask(0x55aa55aa);
Require("Pm4ContextState", "pop",
invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderControl().depth_clear_enable &&
processor.GetCtx().GetRenderControl().stencil_clear_enable &&
processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
"pop did not restore the complete saved Cx state");
Require("Pm4ContextState", "push",
invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
"push changed live Cx state");
processor.GetCtx().SetRenderTargetMask(0x01020304);
Require("Pm4ContextState", "push-pop restore",
invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
"ordinary push/pop did not restore Cx state");
Require("Pm4ContextState", "clear",
invoke(ContextStateOperation::Clear) == Pm4ProcessResult::Complete &&
!processor.GetCtx().GetRenderControl().depth_clear_enable &&
processor.GetCtx().GetRenderTargetMask() ==
HW::Context{}.GetRenderTargetMask() &&
processor.GetUcfg().GetPrimType() == primitive_type &&
processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address,
"clear reset state outside the Cx register domain");
processor.GetCtx().SetRenderTargetMask(0x0badc0de);
Require("Pm4ContextState", "push before clear-state packet",
invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete,
"push before CLEAR_STATE failed");
processor.GetCtx().SetRenderTargetMask(0xfeedface);
std::array<uint32_t, 2> clear_state_packet{0xc0001200, 0};
Pm4Execution clear_state_execution;
Require("Pm4ContextState", "clear-state packet",
processor.Process(clear_state_execution, clear_state_packet) ==
Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderTargetMask() ==
HW::Context{}.GetRenderTargetMask(),
"CLEAR_STATE did not clear the current Cx state");
Require("Pm4ContextState", "pop after clear-state packet",
invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderTargetMask() == 0x0badc0de,
"CLEAR_STATE discarded the pushed Cx state");
// CLEAR_STATE restores the viewport, scissor, and guard-band register defaults.
processor.GetCtx().SetScreenScissor(1, 2, 3, 4);
processor.GetCtx().SetGuardBands(2.0f, 3.0f, 4.0f, 5.0f);
Pm4Execution scissor_reset;
Require("Pm4ContextState", "scissor reset packet",
processor.Process(scissor_reset, clear_state_packet) == Pm4ProcessResult::Complete,
"CLEAR_STATE could not reset the rasterization state");
const auto &viewport = processor.GetCtx().GetScreenViewport();
const auto defaults = calc_final_scissor(viewport, {}, {20000, 20000}, 0);
Require("Pm4ContextState", "Rasterization defaults",
defaults.left == 0 && defaults.top == 0 && defaults.right == 16384 &&
defaults.bottom == 16384 && viewport.guard_band_horz_clip == 1.0f &&
viewport.guard_band_vert_clip == 1.0f &&
viewport.guard_band_horz_discard == 1.0f &&
viewport.guard_band_vert_discard == 1.0f &&
std::ranges::all_of(viewport.viewports, [](const auto &slot) {
return slot.xscale == 1.0f && slot.yscale == 1.0f && slot.zscale == 1.0f &&
slot.viewport_scissor_right == 16384 &&
slot.viewport_scissor_bottom == 16384 &&
!slot.viewport_scissor_window_offset_enable;
}),
"CLEAR_STATE did not restore scissor and guard-band defaults");
const std::array scissor_registers{
std::pair{Pm4::PA_SC_SCREEN_SCISSOR_TL, Pm4::PA_SC_SCREEN_SCISSOR_BR},
std::pair{Pm4::PA_SC_WINDOW_SCISSOR_TL, Pm4::PA_SC_WINDOW_SCISSOR_BR},
std::pair{Pm4::PA_SC_GENERIC_SCISSOR_TL, Pm4::PA_SC_GENERIC_SCISSOR_BR},
std::pair{Pm4::PA_SC_VPORT_SCISSOR_0_TL, Pm4::PA_SC_VPORT_SCISSOR_0_BR}};
for (const auto &[tl, br] : scissor_registers) {
processor.GetCtx().Reset();
g_hw_ctx_indirect_func[tl](processor, tl, 0);
g_hw_ctx_indirect_func[br](processor, br, 0);
HW::ScanModeControl scan{};
scan.vport_scissor_enable = tl == Pm4::PA_SC_VPORT_SCISSOR_0_TL;
const auto empty = calc_final_scissor(processor.GetCtx().GetScreenViewport(),
scan, {3840, 2160}, 0);
Require("Pm4ContextState", "explicit empty scissor",
empty.right == empty.left && empty.bottom == empty.top,
"an explicitly empty scissor was expanded or ignored");
}
processor.GetCtx().SetRenderTargetMask(0xabcdef01);
Require("Pm4ContextState", "push before processor reset",
invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete,
"push before reset failed");
processor.Reset();
processor.GetCtx().SetRenderTargetMask(0x76543210);
Require("Pm4ContextState", "processor reset discards push",
invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete &&
invoke(ContextStateOperation::Pop) ==
Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderTargetMask() == 0x76543210,
"processor reset retained an invalid saved Cx state");
Require("Pm4ContextState", "HLE packet size",
Gen5::AgcDcbContextStateOpGetSize(0) == 20 &&
Gen5::AgcDcbContextStateOpGetSize(1) == 108 &&
Gen5::AgcDcbContextStateOpGetSize(2) == 108 &&
Gen5::AgcDcbContextStateOpGetSize(3) == 128 &&
Gen5::AgcDcbContextStateOpGetSize(4) == 0,
"context-state HLE sizes do not match the expected packet layout");
std::printf("[host] %-32s ok\n", "Pm4ContextState");
}
void CheckPm4IndirectControlFlow(RenderContext &renderer) {
constexpr const char *name = "Pm4IndirectControlFlow";
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
uint32_t selected = 0;
uint32_t returned = 0;
const auto address = [](const void *value) {
return reinterpret_cast<uint64_t>(value);
};
const auto write = [&](uint32_t *destination, uint32_t value) {
return std::array<uint32_t, 5>{
KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0), 0,
static_cast<uint32_t>(address(destination)),
static_cast<uint32_t>(address(destination) >> 32u), value};
};
const auto indirect = [&](std::span<const uint32_t> target, bool chain) {
uint32_t control = 0x0f200000u | static_cast<uint32_t>(target.size());
if (chain) {
control |= 1u << 20u;
}
return std::array<uint32_t, 4>{
KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0),
static_cast<uint32_t>(address(target.data())),
static_cast<uint32_t>(address(target.data()) >> 32u), control};
};
const auto then_commands = write(&selected, 11);
const auto else_commands = write(&selected, 33);
const auto branch_suffix = write(&selected, 44);
const auto caller_suffix = write(&returned, 22);
alignas(8) uint64_t condition = 0;
std::array<uint32_t, 19> branch{
KYTY_PM4(14, Pm4::IT_INDIRECT_BUFFER, 0), 0,
static_cast<uint32_t>(address(&condition)),
static_cast<uint32_t>(address(&condition) >> 32u), UINT32_MAX,
UINT32_MAX, 1, 0,
static_cast<uint32_t>(address(then_commands.data())),
static_cast<uint32_t>(address(then_commands.data()) >> 32u),
static_cast<uint32_t>(then_commands.size()),
static_cast<uint32_t>(address(else_commands.data())),
static_cast<uint32_t>(address(else_commands.data()) >> 32u),
static_cast<uint32_t>(else_commands.size())};
std::copy(branch_suffix.begin(), branch_suffix.end(), branch.begin() + 14);
std::array<uint32_t, 9> caller{};
const auto call_branch = indirect(branch, false);
std::copy(call_branch.begin(), call_branch.end(), caller.begin());
std::copy(caller_suffix.begin(), caller_suffix.end(), caller.begin() + 4);
for (uint32_t mode : {1u, 2u}) {
branch[1] = mode | (3u << 8u);
for (uint64_t value : {0ull, 1ull}) {
condition = value;
selected = returned = 0;
uint32_t expected = 44;
if (value == 1) {
expected = 11;
} else if (mode == 2) {
expected = 33;
}
Pm4Execution execution;
Require(name, "conditional fetcher replacement",
processor.Process(execution, caller) == Pm4ProcessResult::Complete &&
selected == expected && returned == 22,
"taken branch resumed its discarded suffix or lost the caller's return");
}
}
// This stream exceeds native recursion capacity, while chains need one fetcher cursor.
std::vector<std::array<uint32_t, 4>> links(65536);
std::span<const uint32_t> target = then_commands;
for (auto &link : links) {
link = indirect(target, true);
target = link;
}
const auto call_chain = indirect(target, false);
std::copy(call_chain.begin(), call_chain.end(), caller.begin());
selected = returned = 0;
Pm4Execution execution;
Require(name, "long chain and call return",
processor.Process(execution, caller) == Pm4ProcessResult::Complete &&
selected == 11 && returned == 22,
"long chain did not complete and return to the original caller");
std::printf("[host] %-32s ok\n", name);
}
void CheckPm4WaitResume(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
uint32_t label = 0;
uint32_t prefix = 0;
uint32_t child_observation = UINT32_MAX;
uint32_t suffix = 0;
const auto address = [](const void *value) {
return reinterpret_cast<uint64_t>(value);
};
std::array<uint32_t, 12> child{};
child[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
child[1] = 0;
child[2] = static_cast<uint32_t>(address(&child_observation));
child[3] = static_cast<uint32_t>(address(&child_observation) >> 32u);
child[4] = 0;
child[5] = KYTY_PM4(7, Pm4::IT_WAIT_REG_MEM, 0);
child[6] = 0x10u | 3u;
child[7] = static_cast<uint32_t>(address(&label));
child[8] = static_cast<uint32_t>(address(&label) >> 32u);
child[9] = 1;
child[10] = UINT32_MAX;
std::array<uint32_t, 4> nested{};
nested[0] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
nested[1] = static_cast<uint32_t>(address(child.data()));
nested[2] = static_cast<uint32_t>(address(child.data()) >> 32u);
nested[3] = 0x0f300000u | static_cast<uint32_t>(child.size());
std::array<uint32_t, 14> commands{};
commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
commands[1] = 0;
commands[2] = static_cast<uint32_t>(address(&prefix));
commands[3] = static_cast<uint32_t>(address(&prefix) >> 32u);
commands[4] = 11;
commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
commands[6] = static_cast<uint32_t>(address(nested.data()));
commands[7] = static_cast<uint32_t>(address(nested.data()) >> 32u);
commands[8] = 0x0f200000u | static_cast<uint32_t>(nested.size());
commands[9] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
commands[10] = 0;
commands[11] = static_cast<uint32_t>(address(&suffix));
commands[12] = static_cast<uint32_t>(address(&suffix) >> 32u);
commands[13] = 22;
Pm4Execution execution;
Require("Pm4WaitResume", "suspend",
processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
prefix == 11 && child_observation == 0 && suffix == 0,
"blocked indirect wait did not preserve its command position");
Require("Pm4WaitResume", "still blocked",
processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
!execution.MadeProgress() && suffix == 0,
"blocked wait advanced or replayed its caller");
label = 1;
child[4] = 1;
Require(
"Pm4WaitResume", "resume",
processor.Process(execution, commands) == Pm4ProcessResult::Complete &&
prefix == 11 && child_observation == 0 && suffix == 22,
"resumed indirect wait replayed a child or did not finish its parent");
std::printf("[host] %-32s ok\n", "Pm4WaitResume");
}
void CheckPm4RewindResume(RenderContext &renderer) {
constexpr const char *name = "Pm4RewindResume";
Loader::SymbolDatabase symbols;
Libs::InitAgcDriver_1(&symbols);
const auto *emit_symbol = symbols.FindByNid("DwICrVxerkY", Loader::SymbolType::Func);
const auto *patch_symbol = symbols.FindByNid("eWaWyFegzgQ", Loader::SymbolType::Func);
Require(name, "async exports",
emit_symbol != nullptr && patch_symbol != nullptr &&
emit_symbol->vaddr == reinterpret_cast<uint64_t>(&Gen5::AgcAcbRewind) &&
patch_symbol->vaddr ==
reinterpret_cast<uint64_t>(&Gen5::AgcRewindPatchSetRewindState),
"async rewind imports did not resolve to the emitter and state patch");
const auto async_patch = reinterpret_cast<decltype(&Gen5::AgcRewindPatchSetRewindState)>(
patch_symbol->vaddr);
constexpr uint32_t sentinel = 0x12345678u;
std::array<uint32_t, 4> packet{sentinel, sentinel, sentinel, sentinel};
CommandBufferLayout acb{packet.data(), packet.data() + packet.size(),
packet.data() + 1, packet.data() + 3, nullptr, nullptr, 0};
auto *buffer = reinterpret_cast<Gen5::CommandBuffer *>(&acb);
for (const uint8_t state : {0, 1}) {
for (const uint8_t offload : {0, 1}) {
acb.cursor_up = packet.data() + 1;
Require(name, "async packet",
Gen5::AgcAcbRewind(buffer, state, offload) == packet.data() + 1 &&
acb.cursor_up == packet.data() + 3 &&
packet[0] == sentinel && packet[3] == sentinel &&
packet[1] == 0xc0005900u &&
packet[2] == ((uint32_t{state} << 31u) | (uint32_t{offload} << 24u)),
"async rewind packet or two-DWORD allocation differs from the native command");
}
}
const auto before = packet;
acb.cursor_up = packet.data() + 1;
acb.reserved_dw = 1;
Require(name, "bounded allocation",
Gen5::AgcAcbRewind(buffer, 1, 1) == nullptr &&
Gen5::AgcAcbRewind(nullptr, 1, 1) == nullptr &&
acb.cursor_up == packet.data() + 1 && packet == before,
"failed rewind allocation changed the cursor or command storage");
// Native patchers inspect only the opcode, preserving the header and low 31 bits.
std::array<uint32_t, 2> command{0xa5ff591fu, 0x0155aa55u};
Require(name, "patch preserves fields",
async_patch(command.data(), 1) == 0 && command[0] == 0xa5ff591fu &&
command[1] == 0x8155aa55u && async_patch(command.data(), 0) == 0 &&
command[0] == 0xa5ff591fu && command[1] == 0x0155aa55u,
"rewind patch changed the offload bit, header, or unrelated payload bits");
command[0] = 0xc0001000u;
const auto invalid = command;
Require(name, "wrong opcode",
async_patch(command.data(), 1) == static_cast<int>(0x8a6c000cu) &&
command == invalid,
"rewind patch accepted or modified a packet with the wrong opcode");
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
for (const bool async : {false, true}) {
for (const uint8_t offload : {0, 1}) {
if (!async && offload != 0) {
continue;
}
uint32_t suffix = 0;
const auto address = reinterpret_cast<uint64_t>(&suffix);
std::array<uint32_t, 7> commands{
0, 0, KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0), 0,
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u), 33};
CommandBufferLayout cb{commands.data(), commands.data() + 2,
commands.data(), commands.data() + 2, nullptr, nullptr, 0};
auto *command_buffer = reinterpret_cast<Gen5::CommandBuffer *>(&cb);
const auto *emitted = async ? Gen5::AgcAcbRewind(command_buffer, 0, offload)
: Gen5::AgcDcbRewind(command_buffer, 0);
const auto patch = async ? async_patch : &Gen5::AgcRewindPatchSetRewindState;
Pm4Execution execution;
Require(name, "pending",
emitted == commands.data() &&
processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
suffix == 0,
"pending rewind did not preserve its command position");
Require(name, "still pending",
processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
!execution.MadeProgress() && suffix == 0,
"pending rewind advanced before its state was patched");
Require(name, "patch and resume",
patch(commands.data(), 1) == 0 &&
commands[1] == (0x80000000u | (uint32_t{offload} << 24u)) &&
processor.Process(execution, commands) == Pm4ProcessResult::Complete &&
suffix == 33,
"patched rewind lost the offload bit or failed to resume");
}
}
std::printf("[host] %-32s ok\n", name);
}
void CheckPm4CeCompletion(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
uint32_t suffix = 0;
const auto address = reinterpret_cast<uint64_t>(&suffix);
std::array<uint32_t, 9> commands{};
commands[0] = 0xc0008600u;
commands[1] = 1;
commands[2] = 0xc0008500u;
commands[3] = 0;
commands[4] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
commands[5] = 0;
commands[6] = static_cast<uint32_t>(address);
commands[7] = static_cast<uint32_t>(address >> 32u);
commands[8] = 33;
processor.ResetDeCe();
Pm4Execution execution;
Require("Pm4CeCompletion", "wait",
processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
suffix == 0,
"DE did not wait for an active CE stream");
std::array<uint32_t, 2> ce_increment{0xc0008400u, 1};
Pm4Execution ce_execution;
Require("Pm4CeCompletion", "CE increment",
processor.Process(ce_execution, ce_increment) ==
Pm4ProcessResult::Complete,
"CE counter increment did not complete");
Require(
"Pm4CeCompletion", "counter gate",
processor.Process(execution, commands) == Pm4ProcessResult::Complete &&
suffix == 33,
"DE did not resume after CE advanced or failed to increment its counter");
suffix = 0;
Pm4Execution balanced_execution;
Require("Pm4CeCompletion", "balanced wait",
processor.Process(balanced_execution, commands) ==
Pm4ProcessResult::Blocked &&
suffix == 0,
"balanced CE/DE counters did not gate the next DE packet");
processor.SetCeComplete(true);
Require("Pm4CeCompletion", "stream complete",
processor.Process(balanced_execution, commands) ==
Pm4ProcessResult::Complete &&
suffix == 33,
"DE remained blocked after the CE stream completed");
std::printf("[host] %-32s ok\n", "Pm4CeCompletion");
}
} // namespace
} // namespace Libs::Graphics
int main(int argc, char **argv) {
using namespace Libs::Graphics;
std::setvbuf(stdout, nullptr, _IONBF, 0);
EnsureConfigInitialized();
CheckLeastRecentlyUsedCacheOrdering();
if (argc == 2 && std::strcmp(argv[1], "--thread-dimensions-only") == 0) {
VulkanHarness vulkan;
CheckComputeThreadDimensions(vulkan);
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--lds-limit-only") == 0) {
VulkanHarness vulkan;
CheckComputeLdsLimit(vulkan);
RunCase(&vulkan, DsWideLdsPartialBounds());
RunCase(&vulkan, DsAtomic64Bounds(false));
RunCase(&vulkan, DsAtomic64Bounds(true));
for (auto test : {DsWideLdsPartialBounds(), DsAtomic64Bounds(false), DsAtomic64Bounds(true)}) {
test.compute_info.lds_storage = true;
std::erase(test.required_spirv, "WorkgroupMemoryExplicitLayoutKHR");
test.required_spirv.push_back("StorageBuffer");
RunCase(&vulkan, test);
}
vulkan.CheckNativeIndirectDispatch();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--integer64-compare-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCompareInteger64Edges());
RunCase(&vulkan, VectorCompareExecWaveMasks(32));
RunCase(&vulkan, VectorCompareExecWaveMasks(64));
RunCase(&vulkan, VectorVopcCmpxNeU64CapturedExecMask());
RunCase(&vulkan, VectorVop3CmpxNeI64CapturedExecMask());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--new-opcodes-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCompareInteger64Edges());
RunCase(&vulkan, VectorCompare64WaveMasks(32, true));
RunCase(&vulkan, VectorCompare64WaveMasks(64, true));
RunCase(&vulkan, VectorCompareExecWaveMasks(32));
RunCase(&vulkan, VectorCompareExecWaveMasks(64));
RunCase(&vulkan, VectorCompareF64Edges());
RunCase(&vulkan, VectorF64ModesModifiersAndExec());
RunCase(&vulkan, VectorCompareF16Ops());
RunCase(&vulkan, ScalarBrevB64OperandsAndMasks(32));
RunCase(&vulkan, ScalarBrevB64OperandsAndMasks(64));
for (bool add : {false, true}) {
RunCase(&vulkan, DsAtomic64Bounds(add));
for (u32 wave_size : {32, 64}) {
RunCase(&vulkan, DsAtomic64Contention(add, wave_size));
}
}
RunCase(&vulkan, BufferAtomicAndX2GlcAndExec());
RunCase(&vulkan, ImageAtomicSignedMinMax<false>());
RunCase(&vulkan, ImageAtomicSignedMinMax<true>());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--s-memrealtime-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarMemRealtimeCapturedPlaceholder());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--packed-integer-neg-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop3pIntegerNegationCapturedAndSelectedHalves());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--dpp-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorDpp8Captured(false));
RunCase(&vulkan, VectorDpp8Captured(true));
RunCase(&vulkan, VectorDppQuadPermuteReverse());
RunCase(&vulkan, VectorDppRowXmask());
RunCase(&vulkan, VectorDppBankMaskPreservesDestination());
RunCase(&vulkan, VectorDppBoundsControlZeroPreservesDestination());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-o-f32-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpxOrderedCapturedExecMask());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--fp32-compare-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCompareF32ExposureGuard());
for (const auto mode : {0xc0u, 0xe0u}) {
RunCase(&vulkan, VectorCompareF32DenormalModes(mode));
}
RunCase(&vulkan, VectorCompareClassF32());
RunCase(&vulkan, VectorCompareClassF32ImmediateAndDynamicMasks());
RunCase(&vulkan, VectorCompareF16Ops());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--int16-alu-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCompareI16U16Ops());
RunCase(&vulkan, Vop3Int16Ops());
RunCase(&vulkan, Vop3pInt16Ops());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-format-store-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, BufferStoreFormatXyzwPackedUnormSkinningVectors());
RunCase(&vulkan, BufferStoreFormatXZeroFillsPackedRecord());
RunCase(&vulkan, BufferStoreFormatXyzwFloat16ConvertsComponents());
RunCase(&vulkan, BufferStoreFormatXyzwSnorm16CapturedSkinningVectors());
RunCase(&vulkan, BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords());
for (const auto &test : BufferLoadFormatD16XCases()) {
RunCase(&vulkan, test);
}
for (const auto &test : BufferStoreFormatD16XCases()) {
RunCase(&vulkan, test);
}
for (const auto &test : FormattedStoreConversionCases()) {
RunCase(&vulkan, test);
}
RunCase(&vulkan, BufferStoreFormatXResource16UintWritesHalfword());
RunCase(&vulkan, BufferStoreFormatXResource16UintPreservesAdjacentLanes());
RunCase(&vulkan, BufferStoreFormatXyResource88UintWritesBytes());
RunCase(&vulkan, BufferStoreFormatXyzwDropsPartialRecord());
RunCase(&vulkan, BufferStoreFormatXRejectsPartialRecord());
RunCase(&vulkan, BufferStoreFormatXyRejectsPartialRecord());
RunCase(&vulkan, BufferFormatStoreVariants());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-float16-store-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, BufferStoreFormatXyzwFloat16ConvertsComponents());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--s-ashr-i64-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarAshrI64Edges(false));
RunCase(&vulkan, ScalarAshrI64Edges(true));
RunCase(&vulkan, ScalarAshrI64OperandsAndAliases());
RunCase(&vulkan, ScalarAshrI64Masks(32));
RunCase(&vulkan, ScalarAshrI64Masks(64));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--tessellation-only") == 0) {
CheckTessellationPrograms();
CheckEmbeddedFetchVertexOffset();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-eq-u16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpxEqU16SdwaCompactVop3ExecMask());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-ne-u16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpxNeU16SdwaCompactVop3ExecMask());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-lt-i16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpxLtI16CapturedSdwaExecMask());
RunCase(&vulkan, VectorVopcCmpxLtI16WaveMasks(32));
RunCase(&vulkan, VectorVopcCmpxLtI16WaveMasks(64));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-lt-u16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpxLtU16CapturedSdwaExecMask());
RunCase(&vulkan, VectorVopcCmpxGtU16CapturedSdwaExecMask());
RunCase(&vulkan, VectorVopcSdwaCmpxWritesExecMask());
RunCase(&vulkan, VectorVop3CmpxWritesExecMask());
RunCase(&vulkan, VectorVopcSdwaCmpxClassF32CapturedExecMask());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--class-f16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCmpClassF16(32));
RunCase(&vulkan, VectorCmpClassF16(64));
RunCase(&vulkan, VectorCmpxClassF16(32));
RunCase(&vulkan, VectorCmpxClassF16(64));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-class-f16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCmpxClassF16(32));
RunCase(&vulkan, VectorCmpxClassF16(64));
RunCase(&vulkan, VectorCompareClassF32());
RunCase(&vulkan, VectorVopcSdwaCmpxClassF32CapturedExecMask());
RunCase(&vulkan, FloatInlineConstantF16UsesNumericValue());
RunCase(&vulkan, VectorCompareF16Ops());
RunCase(&vulkan, VectorMinMaxF16Ops());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmp-eq-f64-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCompareF64Edges());
RunCase(&vulkan, VectorCompare64WaveMasks(32));
RunCase(&vulkan, VectorCompare64WaveMasks(64));
RunCase(&vulkan, VectorF64CapturedScreenSpaceShadows());
RunCase(&vulkan, VectorFractF64CapturedAndEdges());
RunCase(&vulkan, VectorMinMaxF64CapturedAndEdges());
RunCase(&vulkan, VectorRoundF64EdgesAndModifiers());
RunCase(&vulkan, VectorFractF64CapturedChainRuntimeExec());
RunCase(&vulkan, VectorF64ModesModifiersAndExec());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmp-ngt-f16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpNgtF16CapturedSdwaAndEdges());
RunCase(&vulkan, VectorVopcCmpxNgtF16CapturedSdwaExecMask());
RunCase(&vulkan, VectorVopcCmpxNleF16CapturedSdwaExecMask());
RunCase(&vulkan, VectorCompareF16Ops());
RunCase(&vulkan, Wave32VccMasksPreserveOtherHalf());
RunCase(&vulkan, VectorVop3FloatCompareNegSourceModifier());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmp-nlt-f16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpNltF16CapturedSdwaAndEdges());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--fract-f16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorFractF16CapturedAndEdges());
RunCase(&vulkan, VectorFractF16Modifiers());
RunCase(&vulkan, NativeAndSdwa16BitDestinationWrites());
RunCase(&vulkan, VectorSpecialF16Ops());
RunCase(&vulkan, VectorCosF16CapturedSdwaAndEdges());
RunCase(&vulkan, VectorSinF16SdwaAndEdges());
RunCase(&vulkan, VectorFloatConversionOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--scaled-texture-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ImageSampleUScaled8<false>());
RunCase(&vulkan, ImageSampleUScaled8<true>());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--packed-texture-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckPackedTextureComponents();
vulkan.CheckBgra16Readback();
RunCase(&vulkan, ImageLoadPackedUintUnpacksAndSwizzles());
RunCase(&vulkan, ImageSamplePackedUintConvertsSampleAndGather());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--system-dialog-only") == 0) {
CheckErrorDialogLifecycle();
CheckSigninDialogLifecycle();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--fmask-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ImageLoadFmaskUsesNativeSampleMapping());
RunCase(&vulkan, ImageLoadR32UintUsesIntegerSampledImage());
return 0;
}
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
if (argc == 2 && std::strcmp(argv[1], "--shader-fatal-only") == 0) {
CheckShaderRecompilerFatalContracts();
return 0;
}
#endif
if (argc == 2 && std::strcmp(argv[1], "--ray-tracing-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, FlatStackApertures(32));
RunCase(&vulkan, FlatStackApertures(64));
RunCase(&vulkan, BvhIntersections(true, true, 1));
RunCase(&vulkan, BvhIntersections(true, true, 0, true));
for (bool barycentrics : {false, true}) {
for (bool sorted : {false, true}) {
RunCase(&vulkan, BvhIntersections(barycentrics, sorted));
}
}
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--scalar-sext-only") == 0) {
VulkanHarness vulkan;
for (const u32 bits : {8u, 16u}) {
RunCase(&vulkan, ScalarSextMasks(bits, 32));
RunCase(&vulkan, ScalarSextMasks(bits, 64));
}
RunCase(&vulkan, ScalarSextConstantsAndAliases());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--flat-d16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, FlatLoadDlcCaptured(32));
RunCase(&vulkan, FlatLoadDlcCaptured(64));
RunCase(&vulkan, GlobalLoadShortD16Captured(32));
RunCase(&vulkan, GlobalLoadShortD16Captured(64));
RunCase(&vulkan, FlatLoadShortD16AddressSegments());
RunCase(&vulkan, FlatSubdwordLoadsApplyByteOffset());
RunCase(&vulkan, FlatVirtualAddressRebasesGuestAllocation());
RunCase(&vulkan, GlobalSignedImmediateRebasesBeforeSaddr());
RunCase(&vulkan, FlatSegmentIgnoresSaddrAndMasksOffsetMsb());
RunCase(&vulkan, ScratchIsPrivatePerInvocation());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--indirect-buffer-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarBufferFromLoopReadlane(32));
RunCase(&vulkan, ScalarBufferFromLoopReadlane(64));
CheckIndirectBufferStore(vulkan);
CheckRuntimeBufferRecords(vulkan);
RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors());
RunCase(&vulkan, BufferLoadDwordGpuSelectedDescriptors());
RunCase(&vulkan, BufferLoadDwordx3GpuSelectedDescriptors());
RunCase(&vulkan, BufferLoadFormatXGpuSelectedDescriptors());
RunCase(&vulkan, BufferLoadFormatXRejectsPartialRecord());
RunCase(&vulkan, BufferLoadFormatXyRejectsPartialRecord());
RunCase(&vulkan, BufferLoadDwordx4SnapshotsOverlappingAddress());
RunCase(&vulkan, BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail());
RunCase(&vulkan, BufferLoadDwordIdxenUsesDescriptorStride());
RunCase(&vulkan, BufferStoreFormatXAddTidUsesLaneIndex());
RunCase(&vulkan, FlatVirtualAddressRebasesGuestAllocation());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-gather-lod-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ImageGatherExplicitLod());
RunCase(&vulkan, ImageGatherLodPerLane<0>());
RunCase(&vulkan, ImageGatherLodPerLane<1>());
RunCase(&vulkan, ImageGatherLodPerLane<2>());
RunCase(&vulkan, ImageGatherLodPerLane<3>());
RunCase(&vulkan, ImageGatherLodPerLane<4>());
RunCase(&vulkan, ImageGatherLodPerLane<5>());
RunCase(&vulkan, ImageGatherLodPerLane<6>());
RunCase(&vulkan, ImageGatherLodPerLane<7>());
RunCase(&vulkan, ImageGatherLodPerLane<8>());
RunCase(&vulkan, ImageStoreSingleMipBounds<0>());
RunCase(&vulkan, ImageStoreSingleMipBounds<1>());
RunCase(&vulkan, ImageStoreSingleMipBounds<2>());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--wave64-only") == 0) {
CheckWave64WholeWaveResults();
VulkanHarness vulkan;
RunCase(&vulkan, Wave32VccMasksPreserveOtherHalf());
RunCase(&vulkan, DsPermuteWave64UsesIndependentHalves());
RunCase(&vulkan, DsBpermuteWave64UsesIndependentHalves());
RunCase(&vulkan, Wave64CrossHalfLaneAndLds());
RunCase(&vulkan, Wave64RawMasksAndScalarBranch());
RunCase(&vulkan, Wave64PartialMultidimensionalWorkgroup());
RunCase(&vulkan, Wave64AppendConsumeHighHalf());
RunCase(&vulkan, VectorAddcWritesPerLaneCarryOut());
RunCase(&vulkan, VectorVop3BCarryOutWritesSgprMask());
RunCase(&vulkan, VectorVop3BCarryOutUsesEncodedSdst());
RunCase(&vulkan, DispatcherIrreducibleControlFlow());
RunCase(&vulkan, ScalarSaveExecOps());
RunCase(&vulkan, ScalarOrn2SaveexecUsesSourceOrNotExec());
RunCase(&vulkan, ScalarOrn2SaveexecB32(32, 32));
RunCase(&vulkan, ScalarOrn2SaveexecB32(64, 64));
RunCase(&vulkan, ScalarOrn2SaveexecB32(32, 4));
RunCase(&vulkan, ScalarSubvectorLoops(32));
RunCase(&vulkan, ScalarSubvectorLoops(64));
RunCase(&vulkan, ScalarNotB64UpdatesScc());
RunCase(&vulkan, ScalarSelectB64PreservesMaskProvenance(32));
RunCase(&vulkan, ScalarSelectB64PreservesMaskProvenance(64));
RunCase(&vulkan, ScalarConditionalMoveB64());
RunCase(&vulkan, ScalarConditionalMoveB64PreservesMasks());
RunCase(&vulkan, ScalarWqmB64SelectsSccDomain());
RunCase(&vulkan, ScalarWqmB64PreservesPartialMasks());
RunCase(&vulkan, ScalarWqmB32Masks(32));
RunCase(&vulkan, ScalarWqmB32Masks(64));
RunCase(&vulkan, ScalarMaskProvenanceOverlapAndMixedBinary());
RunCase(&vulkan, ScalarB32MaskDataflow(32));
RunCase(&vulkan, ScalarB32MaskDataflow(64));
RunCase(&vulkan, ScratchIsPrivatePerInvocation());
RunCase(&vulkan, Vop1MoveRelDestination());
RunCase(&vulkan, BranchVccnzUsesWaveMask());
RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(32, 32));
RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(64, 64));
RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(32, 4));
RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(64, 4));
RunCase(&vulkan, BranchVccnzUsesCarryProducedWaveMask());
RunCase(&vulkan, ImageSampleAndGather());
RunCase(&vulkan, ImageGatherExplicitLod());
RunCase(&vulkan, SharedReturnKeepsSelectedValues());
RunCase(&vulkan, SiblingSharedExitKeepsCapturedConditions());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--position-w-only") == 0) {
VulkanHarness vulkan;
RunGraphicsCase(&vulkan, GraphicsPositionWExport());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--centroid-only") == 0) {
VulkanHarness vulkan;
RunGraphicsCase(&vulkan, GraphicsPackedHalfCentroid());
RunGraphicsCase(&vulkan, GraphicsPerspectiveSample(false));
RunGraphicsCase(&vulkan, GraphicsPerspectiveSample(true));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--pixel-alias-only") == 0) {
CheckPixelParameterAliases();
CheckRectListShaders();
VulkanHarness vulkan;
RunGraphicsCase(&vulkan, GraphicsPackedHalfCentroid());
RunGraphicsCase(&vulkan, GraphicsPackedHalfInputAlias(false));
RunGraphicsCase(&vulkan, GraphicsPackedHalfInputAlias(true));
RunGraphicsCase(&vulkan, GraphicsSmoothRawInputAlias());
RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(false));
RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(true));
RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(false, true));
RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(true, true));
RunGraphicsCase(&vulkan, GraphicsFlatInterpolatorExport());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--clip-control-only") == 0) {
CheckClipControlDepthClipState();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--reference-clock-only") == 0) {
CheckReferenceClockScale();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--scheduler-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckSchedulerTimeline();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--host-image-allocation-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckHostImageAllocation();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--occlusion-dump-only") == 0) {
VulkanHarness vulkan;
CheckPm4SyntheticOcclusionCounterDump(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--predication-only") == 0) {
VulkanHarness vulkan;
CheckPm4Predication(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--descriptor-heap-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckDescriptorHeapLargeSet();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--push-constant-bank-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGraphicsPushConstantBank();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--shader-data-storage-only") == 0) {
CheckResourcePlanHandoff();
VulkanHarness vulkan;
RunCase(&vulkan, BufferOffsetsUsePackedWords(false));
RunCase(&vulkan, BufferOffsetsUsePackedWords(true));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-publication-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, BufferWorkgroupPublication(32));
RunCase(&vulkan, BufferWorkgroupPublication(64));
RunCase(&vulkan, BufferWorkgroupPublication(32, true));
RunCase(&vulkan, BufferWorkgroupPublication(32, false, true));
RunCase(&vulkan, BufferWorkgroupPublication(64, false, true));
RunCase(&vulkan, BufferAtomicVariants());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-cmpswap-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, BufferAtomicCmpSwapExactRaw());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-cmpswap-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ImageAtomicCompareSwapGlcAndExec());
RunCase(&vulkan, ImageAtomicSwapReturnsPreviousTexel());
RunCase(&vulkan, ImageAtomicVariants());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-atomic64-only") == 0) {
VulkanHarness vulkan;
for (const auto &atomic : ImageAtomicIntegerCases) {
RunCase(&vulkan, ImageAtomicIntegerGlcAndExec(atomic, false));
RunCase(&vulkan, ImageAtomicIntegerGlcAndExec(atomic, true));
}
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--ds-ordered-count-only") == 0) {
VulkanHarness vulkan;
for (u32 wave_size : {32, 64}) {
RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, false));
RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, true));
}
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--ds-atomics-only") == 0) {
VulkanHarness vulkan;
for (const auto &test : MakeCases()) {
if (std::string_view(test.name).starts_with("Ds") ||
std::string_view(test.name).starts_with("BufferAtomic")) {
RunCase(&vulkan, test);
}
}
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGpuMappedRangeLifecycle();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--stream-buffer-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckStreamBufferRing();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--gpu-tiler-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGpuTilerCpuParity();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGpuCommandLane();
CheckPm4IndirectControlFlow(vulkan.RuntimeRenderer());
CheckPm4WaitResume(vulkan.RuntimeRenderer());
CheckPm4RewindResume(vulkan.RuntimeRenderer());
CheckPm4CeCompletion(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--suspend-point-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGpuSuspendPoint();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--native64-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarDynamic64BitOps());
RunCase(&vulkan, VectorDynamicU64ShiftEdges());
RunCase(&vulkan, ScalarAshrI64Edges(false));
RunCase(&vulkan, ScalarAshrI64Edges(true));
RunCase(&vulkan, VectorCompareInteger64Edges());
RunCase(&vulkan, VectorMadU64U32UnsignedCarryOut());
RunCase(&vulkan, ScalarWqmB64SelectsSccDomain());
RunCase(&vulkan, ScalarWqmB64PreservesPartialMasks());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--alignbyte-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorAlignByteUsesTwoBitByteOffset());
RunCase(&vulkan, VectorVop3IntegerOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--mad-f32-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorMadF32RoundsProduct());
RunCase(&vulkan, VectorMadF32FlushesDenorms());
RunCase(&vulkan, Vop3FmacF32NegatedSourceAccumulates());
RunCase(&vulkan, VectorFloatArithmeticOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--zero-shift-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarZeroShiftWithRuntimeCount());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-bitwise-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop2SdwaBitwiseByteDestinations(32));
RunCase(&vulkan, Vop2SdwaBitwiseByteDestinations(64));
RunCase(&vulkan, Vop1SdwaNotPartialSourcesAndDestinations());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-mov-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop1SdwaMovByteDestinations());
RunCase(&vulkan, Vop1SdwaMovByteSources(32));
RunCase(&vulkan, Vop1SdwaMovByteSources(64));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-ffbh-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop1SdwaBfrevSourceSelectors());
RunCase(&vulkan, Vop1SdwaFfbhCapturedScalarLowWordSource());
RunCase(&vulkan, VectorFfbhI32NativeAndVop3OnGpu());
RunCase(&vulkan, Vop1SdwaFfblCapturedHighWordSource());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cubeid-omod-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, CubeIdCapturedNegationAndOutputScale());
RunCase(&vulkan, VectorFloatSpecialOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--zero-stride-oob-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, TBufferCapturedZeroStrideOob());
RunCase(&vulkan, TBufferCapturedZeroStrideOob(true));
RunCase(&vulkan, TBufferCapturedZeroStrideOob(false, true));
RunCase(&vulkan, BufferZeroStrideOobFormatsAndWidths());
RunCase(&vulkan, TBufferLoadFormatXIdxenUsesDescriptorStride());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-ashr-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop2SdwaAshrrevCapturedWord0SignExtends());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-max-i32-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop2SdwaMaxI32CapturedHighWord(32));
RunCase(&vulkan, Vop2SdwaMaxI32CapturedHighWord(64));
RunCase(&vulkan, VectorIntegerOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-addc-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorAddcWritesPerLaneCarryOut());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-mul-i24-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop2SdwaMulI24Destinations(32));
RunCase(&vulkan, Vop2SdwaMulI24Destinations(64));
RunCase(&vulkan, VectorIntegerOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-mul-u24-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop2SdwaMulU24Destinations(32));
RunCase(&vulkan, Vop2SdwaMulU24Destinations(64));
RunCase(&vulkan, Vop2SdwaMulI24Destinations(32));
RunCase(&vulkan, Vop2SdwaMulI24Destinations(64));
RunCase(&vulkan, VectorIntegerOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cvt-pk-i16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop3CvtPkI16I32Captured());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--med3-i16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop3Min3U16CapturedAndSelectors());
RunCase(&vulkan, Vop3Med3I16Captured());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--mul-lo-u16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop3MulLoU16CapturedAndSelectors());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--mad-i16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop3MadI16CapturedSelectorsAndSaturation());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--waitcnt-depctr-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarWaitcntDepctrCapturedVmVsrc());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sff1-b64-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarBitfieldPack());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--context-state-only") == 0) {
VulkanHarness vulkan;
CheckShaderFusion();
CheckPm4NativeTargetGeometryRegisters(vulkan.RuntimeRenderer());
CheckPm4IndirectShaderRegisters(vulkan.RuntimeRenderer());
CheckPm4IndirectUconfigRegisters(vulkan.RuntimeRenderer());
CheckPm4BlendColorRegisterRanges(vulkan.RuntimeRenderer());
CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer());
CheckPm4DepthControlHighBits(vulkan.RuntimeRenderer());
CheckPm4DepthRenderOverride(vulkan.RuntimeRenderer());
CheckPm4ContextStateOperations(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--pm4-wait-only") == 0) {
VulkanHarness vulkan;
CheckPm4WaitPackets(vulkan.RuntimeRenderer());
CheckPm4WaitResume(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--pm4-draw-multi-only") == 0) {
VulkanHarness vulkan;
CheckPm4DrawIndirectMultiPacket(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--rewind-only") == 0) {
VulkanHarness vulkan;
CheckPm4RewindResume(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-overlap-only") == 0) {
CheckDepthAttachmentWrites();
CheckDepthFeedbackAspects();
CheckDynamicRenderingState();
VulkanHarness vulkan;
vulkan.CheckRasterization(false);
vulkan.CheckRenderExecutorColorDiscovery();
vulkan.CheckRenderExecutorColor1DArrayDiscovery();
vulkan.CheckRenderExecutorColorVolumeDiscovery();
vulkan.CheckRenderExecutorColorMetadataClear();
vulkan.CheckSampledDccClear();
vulkan.CheckRenderExecutorColorDepthTileDiscovery();
vulkan.CheckRenderExecutorStencilBindingDiscovery();
vulkan.CheckUnifiedTextureCacheFlow();
vulkan.CheckBgra16Readback();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--draw-offset-only") == 0) {
CheckEmbeddedFetchVertexOffset();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--depth-feedback-only") == 0) {
CheckDepthAttachmentWrites();
CheckDepthFeedbackAspects();
CheckDynamicRenderingState();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--polygon-mode-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRasterization(false);
vulkan.CheckRasterization(false, Prospero::BufferFormat::k11_11_10Float);
vulkan.CheckRasterization(false, Prospero::BufferFormat::k10_10_10_2UScaled);
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--htile-clear-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckUnifiedTextureCacheFlow();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--dcc-clear-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRenderExecutorColorMetadataClear();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--compute-meta-clear-only") == 0) {
CheckDynamicRenderingState();
VulkanHarness vulkan;
vulkan.CheckComputeMetaClearClassification();
vulkan.CheckNativeIndirectDispatch();
vulkan.CheckRenderExecutorColorVolumeDiscovery();
vulkan.CheckRenderExecutorColorMetadataClear();
vulkan.CheckSampledDccClear();
vulkan.CheckRenderExecutorStencilBindingDiscovery();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--native-indirect-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckNativeIndirectDispatch();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--layered-image-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckUnifiedImageViewCache();
vulkan.CheckCubeFaceStorageExpansion();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-view-cache-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckUnifiedImageViewCache();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--storage-sampled-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckUnifiedImageViewCache();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--depth-readback-only") == 0) {
CheckDepthTargetFootprints();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--bda-page-table-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckBdaPageTableUploads();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckUnifiedTextureCacheFlow();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-gc-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckBufferCacheDirtyGarbageCollection();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sampler-border-only") == 0) {
CheckImageSamplerSpecialization();
VulkanHarness vulkan;
vulkan.CheckSamplerBorderColors();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sampled-depth-resource-only") == 0) {
VulkanHarness vulkan;
CheckSampledDepthResource();
CheckDepthTextureEncoding();
vulkan.CheckComparisonDepthTexture();
vulkan.CheckRasterization(true);
RunCase(nullptr, ImageSampleA16CompareBiasRdna2AddressOrder());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--indirect-image-only") == 0) {
CheckImageSamplerSpecialization();
VulkanHarness vulkan;
CheckIndirectImageKeySwitch(vulkan);
RunCase(&vulkan, ImageCubeGradientsPreserveDerivatives());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--readlane-key-guard-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorReadlaneSelectsTwoKeysWithinWave());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--rt1d-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRenderExecutorColorDiscovery();
vulkan.CheckRenderExecutorColor1DArrayDiscovery();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-transition-only") == 0) {
VulkanHarness vulkan;
CheckImageTransitionState(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-rebind-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRenderExecutorStencilBindingDiscovery();
return 0;
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-death") == 0) {
RunReverseRenderTargetDeathCase();
}
if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-only") == 0) {
CheckRenderTargetFormatContract();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--standard64-rt-only") == 0) {
CheckStandard64RenderTargetTileRoundTrip();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--standard-tile-rt-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRenderExecutorColorStandardTileDiscovery();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-view-only") == 0) {
VulkanHarness vulkan;
CheckSampledColorViews();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--storage-bgra-only") == 0) {
CheckSampledColorViews();
CheckBasicStorageTextureDescriptor();
VulkanHarness vulkan;
vulkan.CheckUnifiedImageViewCache();
RunCase(&vulkan, ImageStoreBgraUsesInverseSwizzle());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--storage-mip-host-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRenderExecutorStencilBindingDiscovery();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--storage-mip-only") == 0) {
RunCase(nullptr, ImageStoreMipSelectsPpsa01340Descriptor());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--storage-yzwx-only") == 0) {
CheckSampledColorViews();
CheckBasicStorageTextureDescriptor();
CheckStorageTextureGpuOwnedRebindState();
VulkanHarness vulkan;
RunCase(&vulkan, ImageStoreYzwxUsesInverseSwizzle());
return 0;
}
if (argc == 3 && std::strcmp(argv[1], "--image-view-death") == 0) {
RunImageViewDeathCase(argv[2]);
}
if (argc == 3 &&
std::strcmp(argv[1], "--storage-texture-descriptor-death") == 0) {
RunStorageTextureDescriptorDeathCase(argv[2]);
}
if (argc != 1) {
std::fprintf(stderr, "unknown test selector: %s\n", argv[1]);
return 2;
}
VulkanHarness vulkan;
// Keep compute and host checks running when rasterization features are missing.
const bool rasterization = vulkan.RasterizationSupported();
bool skipped_device_checks = false;
CheckRenderTargetFormatContract();
CheckSampledColorViews();
CheckImageTransitionState(vulkan.RuntimeRenderer());
CheckSampledDepthResource();
CheckDepthTextureEncoding();
vulkan.CheckSamplerBorderColors();
if (rasterization) {
vulkan.CheckComparisonDepthTexture();
vulkan.CheckRasterization(true);
} else {
skipped_device_checks = true;
}
CheckBasicStorageTextureDescriptor();
CheckStorageTextureLinearUploadLayout();
CheckStorageTextureDepthTileUploadLayout();
CheckStandard64RenderTargetTileRoundTrip();
CheckStorageTextureVolumeUploadLayout();
CheckStorageTextureVolumeMipRegions();
CheckStorageTextureGpuOwnedRebindState();
CheckNativeMsaaState();
CheckPs5DepthRegisterDecoding();
CheckDepthHtileStencilCompatibility();
CheckDepthAttachmentWrites();
CheckDepthFeedbackAspects();
CheckDynamicRenderingState();
CheckDepthTargetFootprints();
CheckSlotVectorLifetime();
#else
if (argc != 1) {
std::fprintf(stderr, "unknown test selector: %s\n", argv[1]);
return 2;
}
CheckShaderRecompilerFatalContracts();
CheckDepthFeedbackAspects();
VulkanHarness vulkan;
const bool rasterization = vulkan.RasterizationSupported();
bool skipped_device_checks = false;
#endif
CheckImageSamplerSpecialization();
CheckResourcePlanHandoff();
CheckNativeImageDescriptorTypes();
CheckClipControlDepthClipState();
CheckReferenceClockScale();
CheckErrorDialogLifecycle();
CheckSigninDialogLifecycle();
CheckVulkan13FeatureRequirements();
CheckPm4AcquireMemNoOp(vulkan.RuntimeRenderer());
CheckPm4SyntheticOcclusionCounterDump(vulkan.RuntimeRenderer());
CheckPm4Predication(vulkan.RuntimeRenderer());
CheckPm4StencilInfoValueLane(vulkan.RuntimeRenderer());
CheckPm4NativeTargetGeometryRegisters(vulkan.RuntimeRenderer());
CheckPm4IndirectShaderRegisters(vulkan.RuntimeRenderer());
CheckPm4IndirectUconfigRegisters(vulkan.RuntimeRenderer());
CheckPm4DirectShaderRegisterFallback(vulkan.RuntimeRenderer());
CheckPm4GuardBandRegisterRanges(vulkan.RuntimeRenderer());
CheckPm4BlendColorRegisterRanges(vulkan.RuntimeRenderer());
CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer());
CheckPm4DepthControlHighBits(vulkan.RuntimeRenderer());
CheckPm4DepthRenderOverride(vulkan.RuntimeRenderer());
CheckShaderFusion();
CheckPm4WaitPackets(vulkan.RuntimeRenderer());
CheckPm4DrawIndirectMultiPacket(vulkan.RuntimeRenderer());
CheckPm4ContextStateOperations(vulkan.RuntimeRenderer());
CheckPm4IndirectControlFlow(vulkan.RuntimeRenderer());
CheckPm4WaitResume(vulkan.RuntimeRenderer());
CheckPm4RewindResume(vulkan.RuntimeRenderer());
CheckPm4CeCompletion(vulkan.RuntimeRenderer());
CheckEmbeddedFetchVertexOffset();
CheckEmbeddedFetchLaneSpill();
CheckTessellationPrograms();
CheckPixelParameterAliases();
CheckRectListShaders();
CheckIndirectBufferStore(vulkan);
CheckRuntimeBufferRecords(vulkan);
CheckComputeThreadDimensions(vulkan);
CheckIndirectImageKeySwitch(vulkan);
CheckWave64WholeWaveResults();
CheckPs5GameExampleImageClearRuntimeShape();
vulkan.CheckSchedulerTimeline();
vulkan.CheckHostImageAllocation();
vulkan.CheckDescriptorHeapLargeSet();
vulkan.CheckGpuMappedRangeLifecycle();
vulkan.CheckStreamBufferRing();
vulkan.CheckGpuTilerCpuParity();
vulkan.CheckNativeIndirectDispatch();
CheckComputeLdsLimit(vulkan);
vulkan.CheckBdaPageTableUploads();
vulkan.CheckUnifiedTextureCacheFlow();
vulkan.CheckUnifiedImageViewCache();
vulkan.CheckPackedTextureComponents();
vulkan.CheckCubeFaceStorageExpansion();
if (rasterization) {
vulkan.CheckGraphicsPushConstantBank();
vulkan.CheckRenderExecutorColorDiscovery();
vulkan.CheckRenderExecutorColor1DArrayDiscovery();
vulkan.CheckRenderExecutorColorVolumeDiscovery();
vulkan.CheckRenderExecutorColorMetadataClear();
vulkan.CheckSampledDccClear();
vulkan.CheckRenderExecutorColorStandardTileDiscovery();
vulkan.CheckRenderExecutorColorDepthTileDiscovery();
vulkan.CheckRenderExecutorStencilBindingDiscovery();
vulkan.CheckBgra16Readback();
vulkan.CheckRasterization(false);
vulkan.CheckRasterization(false, Prospero::BufferFormat::k11_11_10Float);
vulkan.CheckRasterization(false, Prospero::BufferFormat::k10_10_10_2UScaled);
vulkan.CheckBufferCacheDirtyGarbageCollection();
} else {
skipped_device_checks = true;
}
const auto tests = MakeCases();
const auto graphics_tests = MakeGraphicsCases();
CheckOpcodeCoverage(tests, graphics_tests);
for (const auto &test : tests) {
RunCase(&vulkan, test);
}
for (const auto &test : graphics_tests) {
RunGraphicsCase(&vulkan, test);
}
vulkan.CheckGpuSuspendPoint();
vulkan.CheckGpuCommandLane();
if (skipped_device_checks) {
std::printf(
"ShaderRecompilerComputeTests: device rasterization checks skipped, this "
"device does not support the production rasterization features\n");
}
if (vulkan.SkippedCaseCount() > 0) {
std::printf("ShaderRecompilerComputeTests: %u graphics cases skipped\n",
vulkan.SkippedCaseCount());
}
std::printf("ShaderRecompilerComputeTests: all cases passed\n");
return 0;
}
Reference in New Issue View Git Blame Copy Permalink