shifathosn
/
KytyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

graphics: use per-invocation shader masks

This commit is contained in:
nmzik 2026-08-21 08:02:21 +02:00
parent 79443aa104
commit 49ea1680f8
30 changed files with 897 additions and 1140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp
View File

@ -1,58 +0,0 @@
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
namespace Libs::Graphics {
ShaderSubgroupCapabilities::ShaderSubgroupCapabilities(const GraphicContext& graphics)
: subgroup_size(graphics.subgroup_size), min_subgroup_size(graphics.min_subgroup_size),
max_subgroup_size(graphics.max_subgroup_size),
required_subgroup_size_stages(graphics.required_subgroup_size_stages),
subgroup_size_control_enabled(graphics.subgroup_size_control_enabled) {}
ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size) {
// A 64-wide guest wave can't assume the host GPU's lane layout matches
// GCN's. AMD RDNA also reports a 64-wide subgroup but uses a different
// lane/pixel layout, which corrupted rendering under "native wave" mode.
// Per-invocation emulation works regardless of the host's lane layout.
if (guest_wave_size == 64u) {
return ShaderLaneMaskMode::PerInvocation;
}
return ShaderLaneMaskMode::NativeWave;
}
ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities,
vk::ShaderStageFlagBits stage,
const ShaderRecompiler::IR::Program& program) {
const auto guest_wave_size = program.wave_size;
if ((guest_wave_size != 32u && guest_wave_size != 64u) ||
!program.spirv_requirements.has_value()) {
return {};
}
if (stage != vk::ShaderStageFlagBits::eCompute) {
const auto expected = SelectGraphicsLaneMaskMode(guest_wave_size);
if (program.lane_mask_mode != expected || (capabilities.subgroup_size != guest_wave_size &&
expected != ShaderLaneMaskMode::PerInvocation)) {
return {};
}
return {expected == ShaderLaneMaskMode::NativeWave
? ShaderSubgroupMode::Natural
: ShaderSubgroupMode::PerInvocationGraphics,
0};
}
if (program.lane_mask_mode != ShaderLaneMaskMode::NativeWave) {
return {};
}
if (capabilities.subgroup_size == guest_wave_size) {
return {ShaderSubgroupMode::Natural, 0};
}
if (capabilities.subgroup_size_control_enabled &&
(capabilities.required_subgroup_size_stages & stage) &&
guest_wave_size >= capabilities.min_subgroup_size &&
guest_wave_size <= capabilities.max_subgroup_size) {
return {ShaderSubgroupMode::Controlled, guest_wave_size};
}
return {ShaderSubgroupMode::FlattenedMasks, 0};
}
} // namespace Libs::Graphics

40
src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.h
View File

@ -1,40 +0,0 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
namespace Libs::Graphics {
enum class ShaderSubgroupMode {
Natural,
Controlled,
PerInvocationGraphics,
FlattenedMasks,
Unsupported
};
struct ShaderSubgroupConfiguration {
ShaderSubgroupMode mode = ShaderSubgroupMode::Unsupported;
uint32_t required_size = 0;
};
struct ShaderSubgroupCapabilities {
explicit ShaderSubgroupCapabilities(const GraphicContext& graphics);
uint32_t subgroup_size = 0;
uint32_t min_subgroup_size = 0;
uint32_t max_subgroup_size = 0;
vk::ShaderStageFlags required_subgroup_size_stages;
bool subgroup_size_control_enabled = false;
};
ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size);
ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities,
vk::ShaderStageFlagBits stage,
const ShaderRecompiler::IR::Program& program);
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_

70
src/graphics/host_gpu/renderer/pipeline/shaders.cpp
View File

@ -8,7 +8,6 @@
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
@ -415,48 +414,6 @@ static void CreateLayout(DescriptorCache& descriptor_cache,
}
}
static void ConfigureSubgroupSize(const GraphicContext& graphics, vk::ShaderStageFlagBits vk_stage,
const ShaderRecompiler::IR::Program& program,
vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo& required,
vk::PipelineShaderStageCreateInfo& stage) {
const auto config =
ConfigureShaderSubgroup(ShaderSubgroupCapabilities {graphics}, vk_stage, program);
switch (config.mode) {
case ShaderSubgroupMode::Natural: return;
case ShaderSubgroupMode::PerInvocationGraphics: {
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
LOGF("Vulkan running %u-lane graphics shader stage 0x%08x with per-invocation "
"EXEC/VCC on the host-native subgroup\n",
program.wave_size, static_cast<uint32_t>(vk_stage));
}
return;
}
case ShaderSubgroupMode::FlattenedMasks: {
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
LOGF("Vulkan running %u-lane shader stage 0x%08x with flattened per-invocation "
"EXEC/VCC\n",
program.wave_size, static_cast<uint32_t>(vk_stage));
}
return;
}
case ShaderSubgroupMode::Controlled: break;
case ShaderSubgroupMode::Unsupported:
default:
EXIT("Vulkan cannot run %u-lane shader stage 0x%08x: default=%u min=%u max=%u "
"controlled_stages=0x%08x\n",
program.wave_size, static_cast<uint32_t>(vk_stage), graphics.subgroup_size,
graphics.min_subgroup_size, graphics.max_subgroup_size,
static_cast<vk::ShaderStageFlags::MaskType>(
graphics.required_subgroup_size_stages));
}
required.sType = vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo;
required.requiredSubgroupSize = config.required_size;
stage.pNext = &required;
}
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
void CreatePipelineInternal(
GraphicContext& graphics, DescriptorCache& descriptor_cache,
@ -528,8 +485,7 @@ void CreatePipelineInternal(
rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr));
EXIT_NOT_IMPLEMENTED(ps_active && frag_shader_module == nullptr);
vk::PipelineShaderStageCreateInfo vert_shader_stage_info {};
vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo vert_subgroup_size {};
vk::PipelineShaderStageCreateInfo vert_shader_stage_info {};
vert_shader_stage_info.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
vert_shader_stage_info.pNext = nullptr;
@ -538,12 +494,8 @@ void CreatePipelineInternal(
vert_shader_stage_info.module = vert_shader_module;
vert_shader_stage_info.pName = "main";
vert_shader_stage_info.pSpecializationInfo = nullptr;
EXIT_IF(!vs_input_info.stage);
ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eVertex, *vs_input_info.stage.program,
vert_subgroup_size, vert_shader_stage_info);
vk::PipelineShaderStageCreateInfo frag_shader_stage_info {};
vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo frag_subgroup_size {};
vk::PipelineShaderStageCreateInfo frag_shader_stage_info {};
frag_shader_stage_info.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
frag_shader_stage_info.pNext = nullptr;
frag_shader_stage_info.flags = {};
@ -551,12 +503,6 @@ void CreatePipelineInternal(
frag_shader_stage_info.module = frag_shader_module;
frag_shader_stage_info.pName = "main";
frag_shader_stage_info.pSpecializationInfo = nullptr;
if (ps_active) {
EXIT_IF(!ps_input_info->stage);
ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eFragment,
*ps_input_info->stage.program, frag_subgroup_size,
frag_shader_stage_info);
}
vk::PipelineShaderStageCreateInfo tess_control_shader_stage_info {};
tess_control_shader_stage_info.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
@ -1074,8 +1020,15 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
comp_shader_stage_info.pName = "main";
comp_shader_stage_info.pSpecializationInfo = nullptr;
EXIT_IF(!input_info.stage);
ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eCompute, *input_info.stage.program,
comp_subgroup_size, comp_shader_stage_info);
const auto wave_size = input_info.stage.program->wave_size;
if (graphics.subgroup_size_control_enabled &&
(graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) &&
wave_size >= graphics.min_subgroup_size && wave_size <= graphics.max_subgroup_size) {
comp_subgroup_size.sType =
vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo;
comp_subgroup_size.requiredSubgroupSize = wave_size;
comp_shader_stage_info.pNext = &comp_subgroup_size;
}
vk::DescriptorSetLayout set_layouts[1] = {};
uint32_t set_layouts_num = 0;
@ -1083,7 +1036,6 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
vk::PushConstantRange push_constant_info[1];
uint32_t push_constant_info_num = 0;
EXIT_IF(!input_info.stage);
CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
push_constant_info_num, *input_info.stage.program,
vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute);

1
src/graphics/host_gpu/renderer/renderCompute.cpp
View File

@ -15,7 +15,6 @@
#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/vulkanCommon.h"

7
src/graphics/host_gpu/renderer/renderDraw.cpp
View File

@ -18,7 +18,6 @@
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/vulkanCommon.h"
@ -1061,12 +1060,10 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw
for (uint32_t i = 0; i < state.color_count; i++) {
target_export_mapping[state.color_info[i].target_slot] = state.color_info[i].export_mapping;
}
const auto lane_mask_mode = SelectGraphicsLaneMaskMode(64u);
if (log_phases) {
LogDrawPhase(draw.name, "ShaderCompileInfoVS");
}
if (!ShaderCompileInfoVS(vertex_shader_info, shader_regs, lane_mask_mode, state.vs_input_info,
if (!ShaderCompileInfoVS(vertex_shader_info, shader_regs, state.vs_input_info,
state.vs_shader)) {
EXIT("ShaderCompileInfoVS failed for draw %s\n", draw.name);
}
@ -1077,7 +1074,7 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw
if (log_phases) {
LogDrawPhase(draw.name, "ShaderCompileInfoPS");
}
if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, lane_mask_mode, state.vs_input_info,
if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, state.vs_input_info,
target_export_mapping, state.ps_input_info, state.ps_shader)) {
EXIT("ShaderCompileInfoPS failed for draw %s\n", draw.name);
}

25
src/graphics/presentation/window/vulkanWindow.cpp
View File

@ -514,10 +514,6 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
features12.pNext = &depth_clip_control;
features12.samplerMirrorClampToEdge = VK_TRUE;
vk::PhysicalDeviceSubgroupSizeControlFeatures subgroup_size_control {};
subgroup_size_control.sType = vk::StructureType::ePhysicalDeviceSubgroupSizeControlFeatures;
subgroup_size_control.pNext = &features12;
vk::PhysicalDeviceVulkan13Features supported_features13 {};
supported_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
@ -578,28 +574,23 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
device_features.shaderInt64 = supported_features2.features.shaderInt64;
}
if (graphics.subgroup_size_control_enabled &&
supported_features13.subgroupSizeControl == VK_TRUE) {
subgroup_size_control.subgroupSizeControl = VK_TRUE;
}
const auto* base_feature_chain = (subgroup_size_control.subgroupSizeControl == VK_TRUE
? static_cast<const void*>(&subgroup_size_control)
: static_cast<const void*>(&features12));
vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {};
robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
robustness2.pNext = const_cast<void*>(base_feature_chain);
robustness2.pNext = &features12;
if (robustness2_ext_enabled) {
robustness2.robustBufferAccess2 = supported_robustness2.robustBufferAccess2;
robustness2.robustImageAccess2 = supported_robustness2.robustImageAccess2;
robustness2.nullDescriptor = supported_robustness2.nullDescriptor;
}
auto features13 = required_features13;
features13.pNext =
robustness2_ext_enabled ? &robustness2 : const_cast<void*>(base_feature_chain);
const bool subgroup_size_control_enabled =
graphics.subgroup_size_control_enabled && supported_features13.subgroupSizeControl == VK_TRUE;
auto features13 = required_features13;
features13.pNext = robustness2_ext_enabled ? static_cast<void*>(&robustness2)
: static_cast<void*>(&features12);
features13.robustImageAccess = supported_features13.robustImageAccess;
features13.subgroupSizeControl = subgroup_size_control_enabled ? VK_TRUE : VK_FALSE;
LOGF("Vulkan robustness: robustImageAccess=%s robustImageAccess2=%s\n",
features13.robustImageAccess == VK_TRUE ? "true" : "false",

1
src/graphics/shader/recompiler/ShaderRecompiler.cpp
View File

@ -683,7 +683,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, output, error)) {
return false;
}
output.lane_mask_mode = options.lane_mask_mode;
output.shader_hash = options.shader_hash;
output.user_data_base = options.user_data_base;
output.user_data_count = options.user_data_count;

1
src/graphics/shader/recompiler/ShaderRecompiler.h
View File

@ -14,7 +14,6 @@ namespace Libs::Graphics::ShaderRecompiler {
struct CompileOptions {
ShaderType stage = ShaderType::Compute;
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave;
uint32_t wave_size = 64;
uint32_t user_data_base = 0;
uint32_t user_data_count = 64;

23
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
View File

@ -198,10 +198,7 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
program.spirv_requirements.emplace(requirements);
return true;
}
const auto MarkExactSubgroup = [&] {
requirements.requires_exact_subgroup = true;
requirements.subgroup_ballot = true;
};
const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (IR::AddressOpcodeInfoOf(inst.GetOpcode()).access != IR::AddressAccess::None) {
@ -230,7 +227,6 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
if (program.stage != ShaderType::Compute) {
return Fail(error, "buffer ADD_TID is only valid for compute shaders");
}
requirements.requires_exact_subgroup = true;
requirements.subgroup_local_invocation_id = true;
}
}
@ -250,17 +246,17 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
}
if (shared_access == IR::SharedAccess::Append ||
shared_access == IR::SharedAccess::Consume) {
MarkExactSubgroup();
MarkBallot();
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
}
}
switch (inst.GetOpcode()) {
case IR::ValueOpcode::Ballot: MarkExactSubgroup(); break;
case IR::ValueOpcode::Ballot: MarkBallot(); break;
case IR::ValueOpcode::DppMoveU32:
case IR::ValueOpcode::ReadFirstLane:
case IR::ValueOpcode::ReadLane: {
MarkExactSubgroup();
MarkBallot();
requirements.subgroup_shuffle = true;
if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) {
requirements.subgroup_local_invocation_id = true;
@ -270,18 +266,18 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
case IR::ValueOpcode::DppUpdateU32:
case IR::ValueOpcode::WqmMask:
case IR::ValueOpcode::WriteLane: {
MarkExactSubgroup();
MarkBallot();
requirements.subgroup_local_invocation_id = true;
break;
}
case IR::ValueOpcode::Permlane16U32: {
MarkExactSubgroup();
MarkBallot();
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
break;
}
case IR::ValueOpcode::SwizzleU32: {
MarkExactSubgroup();
MarkBallot();
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
break;
@ -346,9 +342,8 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
return false;
}
EmitterState state(program, resources, input_info);
state.stage = program.stage;
state.wave_size = program.wave_size;
state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
state.stage = program.stage;
state.wave_size = program.wave_size;
state.inputs.reserve(program.info.inputs.size());
state.outputs.reserve(program.info.outputs.size());
state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size());

23
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
View File

@ -58,13 +58,6 @@ uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
return result;
}
uint32_t BoolAsU32(ValueEmitContext& ctx, uint32_t value) {
const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), result, value,
ConstantU32(ctx.state, 1), ConstantU32(ctx.state, 0)});
return result;
}
uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
auto& state = ctx.state;
if (kind == IR::StageInputKind::LocalInvocationIndex) {
@ -222,8 +215,8 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
bool uint_output) {
auto& state = ctx.state;
if (exp.compr && !uint_output) {
const auto unpack = MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16
: GlslUnpackHalf2x16;
const auto unpack =
MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16 : GlslUnpackHalf2x16;
uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
ConstantF32(state, 0x3f800000u)};
for (uint32_t pair = 0; pair < 2u; pair++) {
@ -441,16 +434,8 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
return true;
case IR::ValueOpcode::Ballot:
if (state.per_invocation_masks) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), result,
BoolAsU32(ctx, ctx.Arg(inst, 0)), ConstantU32(state, 0),
ConstantU32(state, 0), ConstantU32(state, 0)});
ctx.Define(inst, result);
} else {
ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
}
ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
return true;
case IR::ValueOpcode::ReadFirstLane: {
const auto ballot = state.builder.AllocateId();

107
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
View File

@ -251,30 +251,30 @@ enum : uint32_t {
};
enum : uint32_t {
GlslRoundEven = 2,
GlslTrunc = 3,
GlslFAbs = 4,
GlslFloor = 8,
GlslCeil = 9,
GlslFract = 10,
GlslSin = 13,
GlslCos = 14,
GlslExp2 = 29,
GlslLog2 = 30,
GlslSqrt = 31,
GlslInverseSqrt = 32,
GlslFMin = 37,
GlslFMax = 40,
GlslFClamp = 43,
GlslLdexp = 53,
GlslFma = 50,
GlslPackSnorm2x16 = 56,
GlslPackUnorm2x16 = 57,
GlslPackHalf2x16 = 58,
GlslRoundEven = 2,
GlslTrunc = 3,
GlslFAbs = 4,
GlslFloor = 8,
GlslCeil = 9,
GlslFract = 10,
GlslSin = 13,
GlslCos = 14,
GlslExp2 = 29,
GlslLog2 = 30,
GlslSqrt = 31,
GlslInverseSqrt = 32,
GlslFMin = 37,
GlslFMax = 40,
GlslFClamp = 43,
GlslLdexp = 53,
GlslFma = 50,
GlslPackSnorm2x16 = 56,
GlslPackUnorm2x16 = 57,
GlslPackHalf2x16 = 58,
GlslUnpackUnorm2x16 = 61,
GlslUnpackHalf2x16 = 62,
GlslFindILsb = 73,
GlslFindUMsb = 75,
GlslUnpackHalf2x16 = 62,
GlslFindILsb = 73,
GlslFindUMsb = 75,
};
struct InputBinding {
@ -325,41 +325,40 @@ struct EmitterState {
: program(program_), resources(resources_), input_info(input_info_),
requirements(*program_.spirv_requirements) {}
Builder builder;
const IR::Program& program;
const IR::ResourceSnapshot& resources;
ShaderStageInputInfo input_info;
const IR::SpirvRequirements& requirements;
ShaderType stage = ShaderType::Unknown;
uint32_t wave_size = 64;
bool per_invocation_masks = false;
uint32_t storage_buffer_variable = 0;
Builder builder;
const IR::Program& program;
const IR::ResourceSnapshot& resources;
ShaderStageInputInfo input_info;
const IR::SpirvRequirements& requirements;
ShaderType stage = ShaderType::Unknown;
uint32_t wave_size = 64;
uint32_t storage_buffer_variable = 0;
std::array<uint32_t, IR::ShaderInfo::MaxBuffers + IR::ShaderInfo::MaxAddresses>
memory_byte_offsets {};
uint32_t address_memory_variable = 0;
uint32_t gds_variable = 0;
uint32_t gds_length = 0;
uint32_t push_constant_variable = 0;
uint32_t vsharp_storage_variable = 0;
uint32_t flattened_srt_variable = 0;
uint32_t lds_variable = 0;
uint32_t scratch_variable = 0;
memory_byte_offsets {};
uint32_t address_memory_variable = 0;
uint32_t gds_variable = 0;
uint32_t gds_length = 0;
uint32_t push_constant_variable = 0;
uint32_t vsharp_storage_variable = 0;
uint32_t flattened_srt_variable = 0;
uint32_t lds_variable = 0;
uint32_t scratch_variable = 0;
std::array<uint32_t, 2u * SampledImageViewKindCount> sampled_image_variables {};
std::array<uint32_t,
static_cast<uint32_t>(StorageImageClass::Count) * StorageImageViewKindCount>
storage_image_variables {};
uint32_t sampler_variable = 0;
uint32_t main_func = 0;
uint32_t entry_label = 0;
uint32_t current_label = 0;
uint32_t pixel_valid_mask_variable = 0;
uint32_t subgroup_local_invocation_id_variable = 0;
uint32_t per_vertex_variable = 0;
uint32_t depth_variable = 0;
uint32_t sample_mask_variable = 0;
std::vector<InputBinding> inputs;
std::vector<OutputBinding> outputs;
std::vector<uint32_t> interface_variables;
storage_image_variables {};
uint32_t sampler_variable = 0;
uint32_t main_func = 0;
uint32_t entry_label = 0;
uint32_t current_label = 0;
uint32_t pixel_valid_mask_variable = 0;
uint32_t subgroup_local_invocation_id_variable = 0;
uint32_t per_vertex_variable = 0;
uint32_t depth_variable = 0;
uint32_t sample_mask_variable = 0;
std::vector<InputBinding> inputs;
std::vector<OutputBinding> outputs;
std::vector<uint32_t> interface_variables;
};
uint32_t TypeVoid(EmitterState& state);

6
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
View File

@ -672,11 +672,9 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
const auto count_low = state.per_invocation_masks ? low : ctx.Arg(inst, 2);
const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 3);
const auto count =
Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), count_low),
Unary(state, OpBitCount, TypeU32(state), count_high));
Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), low),
Unary(state, OpBitCount, TypeU32(state), high));
const auto first = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), first,
ConstantU32(state, ScopeSubgroup), ballot});

106
src/graphics/shader/recompiler/frontend/translate/Control.cpp
View File

@ -7,37 +7,26 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) {
switch (inst.op) {
case IR::Opcode::SaveexecB32:
case IR::Opcode::SaveexecB64: {
const auto old_low = ir.GetExecLo();
const auto old_high = ir.GetExecHi();
const auto src_low = ReadRawU32(inst.src[0]);
const auto src_high = inst.op == IR::Opcode::SaveexecB64
? ReadRawU32(OffsetOperand(inst.src[0], 1))
: IR::U32(IR::Value(0u));
const auto apply = [&](IR::U32 old_value, IR::U32 src_value) {
switch (inst.saveexec_mode) {
case IR::SaveexecMode::And: return ir.BitwiseAnd(old_value, src_value);
case IR::SaveexecMode::Andn1:
return ir.BitwiseAnd(old_value, ir.BitwiseNot(src_value));
case IR::SaveexecMode::Orn2:
return ir.BitwiseOr(ir.BitwiseNot(old_value), src_value);
}
return old_value;
};
const auto result_low = apply(old_low, src_low);
const auto result_high =
inst.op == IR::Opcode::SaveexecB64 ? apply(old_high, src_high) : old_high;
if (inst.op == IR::Opcode::SaveexecB64) {
WriteU32Pair(inst.dst, {old_low, old_high});
} else {
WriteOperand(inst.dst, old_low);
const auto old = ir.GetExec();
const auto src = ReadMask(inst.src[0]);
IR::U1 result;
switch (inst.saveexec_mode) {
case IR::SaveexecMode::And: result = ir.LogicalAnd(old, src); break;
case IR::SaveexecMode::Andn1:
result = ir.LogicalAnd(old, ir.LogicalNot(src));
break;
case IR::SaveexecMode::Orn2: result = ir.LogicalOr(ir.LogicalNot(old), src); break;
}
ir.SetExecLo(result_low);
ir.SetExecHi(result_high);
ir.SetExec(ThreadBit(result_low, result_high));
const auto nonzero = inst.op == IR::Opcode::SaveexecB64
? ir.BitwiseOr(result_low, result_high)
: result_low;
ir.SetScc(ir.INotEqual(nonzero, zero));
if (inst.op == IR::Opcode::SaveexecB64) {
WriteMask64(inst.dst, old);
} else {
WriteMask(inst.dst, old);
}
const auto mask = BallotMask(result);
ir.SetExec(result);
ir.SetExecLo(mask[0]);
ir.SetExecHi(mask[1]);
ir.SetScc(result);
return true;
}
case IR::Opcode::ScalarAddCarryU32:
@ -157,24 +146,61 @@ bool Translator::TranslateMove(const IR::Instruction& inst) {
WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32));
return true;
case IR::Opcode::MoveU64: {
const auto is_exec_or_vcc = [](const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc);
};
if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0])) {
WriteMask64(inst.dst, ReadMask(inst.src[0]));
return true;
}
const bool scalar_copy = inst.dst.kind == IR::OperandKind::Register &&
inst.src[0].kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar &&
inst.src[0].reg.file == IR::RegisterFile::Scalar;
IR::U1 source_mask;
IR::U1 source_mask_valid;
if (scalar_copy) {
const auto src = static_cast<IR::ScalarReg>(inst.src[0].reg.index);
source_mask = ir.GetThreadBitScalarReg(src);
source_mask_valid = ir.GetScalarMaskTag(src);
}
WriteU32Pair(inst.dst, ReadU32Pair(inst.src[0]));
if (inst.dst.kind == IR::OperandKind::Register &&
inst.src[0].kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar &&
inst.src[0].reg.file == IR::RegisterFile::Scalar) {
ir.SetThreadBitScalarReg(
static_cast<IR::ScalarReg>(inst.dst.reg.index),
ir.GetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.src[0].reg.index)));
if (scalar_copy) {
ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg.index),
source_mask);
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(inst.dst.reg.index),
source_mask_valid);
}
return true;
}
case IR::Opcode::WqmB64: {
if (!current_per_invocation_masks) {
const auto is_exec_or_vcc = [](const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc);
};
if (!is_exec_or_vcc(inst.dst) && !is_exec_or_vcc(inst.src[0])) {
const auto mask_valid = ReadMaskValid(inst.src[0]);
const auto invocation_result =
IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));
const auto result = IR::U64(
ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src[0], IR::Type::U64)}));
WriteOperand(inst.dst, result);
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
ir.SetThreadBitScalarReg(dst, invocation_result);
ir.SetScalarMaskTag(dst, mask_valid);
const auto raw_nonzero = IR::U1(
ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}));
ir.SetScc(IR::U1(ir.Emit(
IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
} else {
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
}
return true;
}
const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));

75
src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp
View File

@ -3,13 +3,76 @@
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) {
if (current_per_invocation_masks) {
return TranslatePerInvocationU64Mask(inst);
const auto invocation_result = EvaluateU64Mask(inst);
const auto is_exec_or_vcc = [](const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc);
};
if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0]) ||
(inst.src_count > 1u && is_exec_or_vcc(inst.src[1]))) {
WriteMask64(inst.dst, invocation_result);
ir.SetScc(invocation_result);
return true;
}
return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 ||
inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64
? TranslateSimpleInteger(inst)
: TranslateComposedInteger(inst);
const auto lhs = ReadU32Pair(inst.src[0]);
auto mask_valid = ReadMaskValid(inst.src[0]);
if (inst.src_count > 1u) {
mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src[1]));
}
std::array<IR::U32, 2> result;
if (inst.op == IR::Opcode::BitwiseNotU64) {
result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
} else {
const auto rhs = ReadU32Pair(inst.src[1]);
for (uint32_t component = 0; component < 2u; component++) {
switch (inst.op) {
case IR::Opcode::BitwiseAndU64:
result[component] = ir.BitwiseAnd(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseAndNotU64:
result[component] =
ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseOrU64:
result[component] = ir.BitwiseOr(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseOrNotU64:
result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseXorU64:
result[component] = ir.BitwiseXor(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseNandU64:
result[component] =
ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component]));
break;
case IR::Opcode::BitwiseNorU64:
result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component]));
break;
case IR::Opcode::BitwiseXnorU64:
result[component] =
ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component]));
break;
default: return false;
}
}
}
WriteU32Pair(inst.dst, result);
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
ir.SetThreadBitScalarReg(dst, invocation_result);
ir.SetScalarMaskTag(dst, mask_valid);
const auto raw_nonzero =
ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)));
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
} else {
ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
}
return true;
}
bool Translator::TranslateInstruction(const IR::Instruction& inst) {

111
src/graphics/shader/recompiler/frontend/translate/Integer.cpp
View File

@ -126,62 +126,26 @@ bool Translator::TranslatePackedInteger16(const IR::Instruction& inst) {
}
}
bool Translator::TranslatePerInvocationU64Mask(const IR::Instruction& inst) {
if (!current_per_invocation_masks) {
return false;
}
IR::U1 Translator::EvaluateU64Mask(const IR::Instruction& inst) {
const auto lhs = [&] { return ReadMask(inst.src[0]); };
const auto rhs = [&] { return ReadMask(inst.src[1]); };
IR::U1 result;
switch (inst.op) {
case IR::Opcode::BitwiseAndU64: result = ir.LogicalAnd(lhs(), rhs()); break;
case IR::Opcode::BitwiseOrU64: result = ir.LogicalOr(lhs(), rhs()); break;
case IR::Opcode::BitwiseAndU64: return ir.LogicalAnd(lhs(), rhs());
case IR::Opcode::BitwiseOrU64: return ir.LogicalOr(lhs(), rhs());
case IR::Opcode::BitwiseXorU64:
result = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}));
break;
case IR::Opcode::BitwiseAndNotU64:
result = ir.LogicalAnd(lhs(), ir.LogicalNot(rhs()));
break;
case IR::Opcode::BitwiseOrNotU64: result = ir.LogicalOr(lhs(), ir.LogicalNot(rhs())); break;
case IR::Opcode::BitwiseNandU64: result = ir.LogicalNot(ir.LogicalAnd(lhs(), rhs())); break;
case IR::Opcode::BitwiseNorU64: result = ir.LogicalNot(ir.LogicalOr(lhs(), rhs())); break;
return IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}));
case IR::Opcode::BitwiseAndNotU64: return ir.LogicalAnd(lhs(), ir.LogicalNot(rhs()));
case IR::Opcode::BitwiseOrNotU64: return ir.LogicalOr(lhs(), ir.LogicalNot(rhs()));
case IR::Opcode::BitwiseNandU64: return ir.LogicalNot(ir.LogicalAnd(lhs(), rhs()));
case IR::Opcode::BitwiseNorU64: return ir.LogicalNot(ir.LogicalOr(lhs(), rhs()));
case IR::Opcode::BitwiseXnorU64:
result = ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})));
break;
case IR::Opcode::BitwiseNotU64: result = ir.LogicalNot(lhs()); break;
default: return false;
return ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})));
case IR::Opcode::BitwiseNotU64: return ir.LogicalNot(lhs());
default: EXIT("invalid 64-bit mask opcode");
}
WriteMask64(inst.dst, result);
return true;
}
bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
if (inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 ||
inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64) {
const auto lhs = ReadU32Pair(inst.src[0]);
std::array<IR::U32, 2> result;
if (inst.op == IR::Opcode::BitwiseNotU64) {
result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
} else {
const auto rhs = ReadU32Pair(inst.src[1]);
for (uint32_t component = 0; component < 2; component++) {
switch (inst.op) {
case IR::Opcode::BitwiseAndU64:
result[component] = ir.BitwiseAnd(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseOrU64:
result[component] = ir.BitwiseOr(lhs[component], rhs[component]);
break;
default:
result[component] = ir.BitwiseXor(lhs[component], rhs[component]);
break;
}
}
}
WriteU32Pair(inst.dst, result);
return true;
}
IR::ValueOpcode opcode {};
IR::Type type = IR::Type::U32;
switch (inst.op) {
@ -250,37 +214,6 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
}
bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
if (inst.op == IR::Opcode::BitwiseAndNotU64 || inst.op == IR::Opcode::BitwiseOrNotU64 ||
inst.op == IR::Opcode::BitwiseNandU64 || inst.op == IR::Opcode::BitwiseNorU64 ||
inst.op == IR::Opcode::BitwiseXnorU64) {
const auto lhs = ReadU32Pair(inst.src[0]);
const auto rhs = ReadU32Pair(inst.src[1]);
std::array<IR::U32, 2> result;
for (uint32_t component = 0; component < 2; component++) {
switch (inst.op) {
case IR::Opcode::BitwiseAndNotU64:
result[component] =
ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseOrNotU64:
result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseNandU64:
result[component] =
ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component]));
break;
case IR::Opcode::BitwiseNorU64:
result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component]));
break;
default:
result[component] =
ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component]));
break;
}
}
WriteU32Pair(inst.dst, result);
return true;
}
const auto binary_u32 = [&](auto operation) {
const auto lhs = ReadU32(inst.src[0]);
const auto rhs = ReadU32(inst.src[1]);
@ -612,10 +545,28 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
}
case IR::Opcode::SelectU64: {
const auto condition = ReadCondition(inst.src[0]);
const auto lhs = ReadU32Pair(inst.src[1]);
const auto rhs = ReadU32Pair(inst.src[2]);
const auto selected_mask =
IR::U1(ir.Emit(IR::ValueOpcode::SelectU1,
{condition, ReadMask(inst.src[1]), ReadMask(inst.src[2])}));
const auto selected_mask_valid =
IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMaskValid(inst.src[1]),
ReadMaskValid(inst.src[2])}));
if (inst.dst.kind == IR::OperandKind::Register &&
(inst.dst.reg.file == IR::RegisterFile::Exec ||
inst.dst.reg.file == IR::RegisterFile::Vcc)) {
WriteMask64(inst.dst, selected_mask);
return true;
}
const auto lhs = ReadU32Pair(inst.src[1]);
const auto rhs = ReadU32Pair(inst.src[2]);
WriteU32Pair(inst.dst, {ir.Select(condition, lhs[0], rhs[0]),
ir.Select(condition, lhs[1], rhs[1])});
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
ir.SetThreadBitScalarReg(dst, selected_mask);
ir.SetScalarMaskTag(dst, selected_mask_valid);
}
return true;
}
case IR::Opcode::PackLowLowU16:

160
src/graphics/shader/recompiler/frontend/translate/Translate.cpp
View File

@ -91,19 +91,8 @@ IR::U32 Translator::ReadPcRelativeU32(const IR::Operand& operand) {
}
std::array<IR::U32, 2> Translator::BallotMask(IR::U1 value) {
if (current_per_invocation_masks) {
return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))),
IR::U32(IR::Value(0u))};
}
const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {value});
const auto low =
IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(0u)}));
// RDNA2 wave32 masks have one architectural dword; the upper half is unused.
const auto high =
current_wave_size == 32u
? IR::U32(IR::Value(0u))
: IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(1u)}));
return {low, high};
return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))),
IR::U32(IR::Value(0u))};
}
IR::U32 Translator::ReadRawU32(const IR::Operand& operand) {
@ -271,9 +260,17 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) {
}
}
switch (operand.reg.file) {
case IR::RegisterFile::Scalar:
ir.SetScalarReg(static_cast<IR::ScalarReg>(operand.reg.index), value);
case IR::RegisterFile::Scalar: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
ir.SetScalarReg(reg, value);
ir.SetThreadBitScalarReg(reg, ThreadBit(value));
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
IR::U1(IR::Value(false)));
}
break;
}
case IR::RegisterFile::Vector: {
const auto reg = static_cast<IR::VectorReg>(operand.reg.index);
const auto old = ir.GetVectorReg(reg);
@ -296,20 +293,20 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) {
case IR::RegisterFile::Vcc:
if (operand.reg.index == 0) {
ir.SetVccLo(IR::U32(value));
ir.SetVcc(ThreadBit(IR::U32(value), ir.GetVccHi()));
ir.SetVcc(ThreadBit(IR::U32(value)));
} else {
ir.SetVccHi(IR::U32(value));
ir.SetVcc(ThreadBit(ir.GetVccLo(), IR::U32(value)));
ir.SetVcc(ThreadBit(ir.GetVccLo()));
}
break;
case IR::RegisterFile::M0: ir.SetM0(IR::U32(value)); break;
case IR::RegisterFile::Exec:
if (operand.reg.index == 0) {
ir.SetExecLo(IR::U32(value));
ir.SetExec(ThreadBit(IR::U32(value), ir.GetExecHi()));
ir.SetExec(ThreadBit(IR::U32(value)));
} else {
ir.SetExecHi(IR::U32(value));
ir.SetExec(ThreadBit(ir.GetExecLo(), IR::U32(value)));
ir.SetExec(ThreadBit(ir.GetExecLo()));
}
break;
case IR::RegisterFile::Scc: ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u)))); break;
@ -459,10 +456,10 @@ IR::U64 Translator::ReadU64(const IR::Operand& operand) {
IR::F32 Translator::ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed) {
if (operand.float_inline) {
const bool use_zero = packed && (high_lane ? operand.op_sel_hi : operand.op_sel);
auto value = use_zero ? IR::F32(IR::Value::F32(0.0f))
: ir.BitCastF32(IR::U32(IR::Value(operand.imm)));
const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value}));
value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
auto value = use_zero ? IR::F32(IR::Value::F32(0.0f))
: ir.BitCastF32(IR::U32(IR::Value(operand.imm)));
const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value}));
value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
if (operand.absolute) {
value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value}));
}
@ -561,7 +558,7 @@ void Translator::WriteU32Pair(const IR::Operand& operand, const std::array<IR::U
return;
}
if (operand.kind == IR::OperandKind::Register) {
const auto thread_bit = ThreadBit(value[0], value[1]);
const auto thread_bit = ThreadBit(value[0]);
switch (operand.reg.file) {
case IR::RegisterFile::Exec:
ir.SetExec(thread_bit);
@ -573,9 +570,7 @@ void Translator::WriteU32Pair(const IR::Operand& operand, const std::array<IR::U
ir.SetVccLo(value[0]);
ir.SetVccHi(value[1]);
return;
case IR::RegisterFile::Scalar:
ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(operand.reg.index), thread_bit);
break;
case IR::RegisterFile::Scalar: break;
default: break;
}
}
@ -583,21 +578,8 @@ void Translator::WriteU32Pair(const IR::Operand& operand, const std::array<IR::U
WriteRawU32(OffsetOperand(operand, 1), value[1]);
}
IR::U1 Translator::ThreadBit(IR::U32 low, IR::U32 high) {
if (current_per_invocation_masks) {
return ir.INotEqual(low, IR::U32(IR::Value(0u)));
}
const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
auto word = low;
if (current_wave_size == 64u) {
const auto high_lane =
IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, IR::Value(32u)}));
word = ir.Select(high_lane, high, low);
}
const auto bit =
ir.BitwiseAnd(ir.ShiftRightLogical(word, ir.BitwiseAnd(lane, IR::U32(IR::Value(31u)))),
IR::U32(IR::Value(1u)));
return ir.INotEqual(bit, IR::U32(IR::Value(0u)));
IR::U1 Translator::ThreadBit(IR::U32 low) {
return ir.INotEqual(low, IR::U32(IR::Value(0u)));
}
IR::U1 Translator::ReadCondition(const IR::Operand& operand) {
@ -619,11 +601,7 @@ IR::U1 Translator::ReadMask(const IR::Operand& operand) {
switch (operand.reg.file) {
case IR::RegisterFile::Scalar: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto high =
current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs
? ir.GetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u))
: IR::U32(IR::Value(0u));
return ThreadBit(ir.GetScalarReg(reg), high);
return ir.GetThreadBitScalarReg(reg);
}
case IR::RegisterFile::Exec: return ir.GetExec();
case IR::RegisterFile::Vcc: return ir.GetVcc();
@ -632,6 +610,25 @@ IR::U1 Translator::ReadMask(const IR::Operand& operand) {
}
}
IR::U1 Translator::ReadMaskValid(const IR::Operand& operand) {
if (operand.kind == IR::OperandKind::ImmediateU32) {
const bool zero = operand.imm == 0u && !operand.sext_64;
const bool ones = operand.imm == 0xffffffffu && operand.sext_64;
return IR::U1(IR::Value(zero || ones));
}
if (operand.kind != IR::OperandKind::Register) {
return IR::U1(IR::Value(false));
}
switch (operand.reg.file) {
case IR::RegisterFile::Scalar:
return ir.GetScalarMaskTag(static_cast<IR::ScalarReg>(operand.reg.index));
case IR::RegisterFile::Exec:
case IR::RegisterFile::Vcc:
case IR::RegisterFile::Scc: return IR::U1(IR::Value(true));
default: return IR::U1(IR::Value(false));
}
}
void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) {
EXIT_IF(operand.kind != IR::OperandKind::Register);
switch (operand.reg.file) {
@ -639,10 +636,18 @@ void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto mask = BallotMask(value);
ir.SetThreadBitScalarReg(reg, value);
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
IR::U1(IR::Value(false)));
}
ir.SetScalarReg(reg, mask[0]);
// A wave32 VALU mask destination must not overwrite the neighboring SGPR.
if (current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
ir.SetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u), mask[1]);
const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
ir.SetScalarReg(high, mask[1]);
ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
}
return;
}
@ -676,9 +681,17 @@ void Translator::WriteMask64(const IR::Operand& operand, IR::U1 value) {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto mask = BallotMask(value);
ir.SetThreadBitScalarReg(reg, value);
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
IR::U1(IR::Value(false)));
}
ir.SetScalarReg(reg, mask[0]);
if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
ir.SetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u), mask[1]);
const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
ir.SetScalarReg(high, mask[1]);
ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
}
}
@ -733,23 +746,17 @@ bool Translator::AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlock
if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) {
return true;
}
IR::U1 condition;
const auto mask_non_zero = [&](IR::U1 value) {
const auto mask = BallotMask(value);
return ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u)));
};
// EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive
// invocations leave the region without reconstructing a host-subgroup mask.
IR::U1 condition;
switch (source.terminator.condition) {
case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
case CFG::BranchCondition::VccZero:
condition = ir.LogicalNot(mask_non_zero(ir.GetVcc()));
break;
case CFG::BranchCondition::VccNonZero: condition = mask_non_zero(ir.GetVcc()); break;
case CFG::BranchCondition::ExecZero:
condition = ir.LogicalNot(mask_non_zero(ir.GetExec()));
break;
case CFG::BranchCondition::ExecNonZero: condition = mask_non_zero(ir.GetExec()); break;
case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
case CFG::BranchCondition::GotoVariable:
if (source.terminator.goto_variable == UINT32_MAX) {
return Fail(error,
@ -771,11 +778,10 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::string* error) {
result = {};
const uint32_t wave_size = source.wave_size;
const bool per_invocation_masks = source.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
uint32_t vector_limit = 1u;
const auto include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) {
result = {};
const uint32_t wave_size = source.wave_size;
uint32_t vector_limit = 1u;
const auto include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) {
if (operand.kind == IR::OperandKind::Register &&
operand.reg.file == IR::RegisterFile::Vector) {
vector_limit =
@ -852,19 +858,14 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
if (IR::RegIndex(reg) >= IR::NumScalarRegs) {
break;
}
entry_ir.SetScalarReg(reg, entry_ir.GetUserData(reg));
const auto value = entry_ir.GetUserData(reg);
entry_ir.SetScalarReg(reg, value);
entry_ir.SetThreadBitScalarReg(reg, entry_ir.INotEqual(value, IR::U32(IR::Value(0u))));
entry_ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
}
entry_ir.SetExec(IR::U1(IR::Value(true)));
if (source.stage == ShaderType::Compute && !per_invocation_masks) {
const auto active = entry_ir.Emit(IR::ValueOpcode::Ballot, {IR::Value(true)});
entry_ir.SetExecLo(entry_ir.CompositeExtract(active, 0));
entry_ir.SetExecHi(source.wave_size > 32u ? entry_ir.CompositeExtract(active, 1)
: IR::U32(IR::Value(0u)));
} else {
entry_ir.SetExecLo(IR::U32(IR::Value(per_invocation_masks ? 1u : 0xffffffffu)));
entry_ir.SetExecHi(IR::U32(
IR::Value(!per_invocation_masks && source.wave_size > 32u ? 0xffffffffu : 0u)));
}
entry_ir.SetExecLo(IR::U32(IR::Value(1u)));
entry_ir.SetExecHi(IR::U32(IR::Value(0u)));
if (source.stage == ShaderType::Compute) {
const auto* cs = compute_input_info;
const auto thread_ids =
@ -930,8 +931,7 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
}
}
for (size_t index = 0; index < source.blocks.size(); index++) {
Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size,
per_invocation_masks);
Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size);
if (!translator.TranslateBlock(source.blocks[index], error) ||
!translator.AddBranchCondition(source.blocks[index], result.block_info[index + 1u],
error)) {

71
src/graphics/shader/recompiler/frontend/translate/Translator.h
View File

@ -10,9 +10,9 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
class Translator {
public:
Translator(IR::ValueProgram& program, IR::Block* block, uint32_t vector_limit,
uint32_t wave_size, bool per_invocation_masks)
uint32_t wave_size)
: value_program(program), ir(block), current_vector_limit(vector_limit),
current_wave_size(wave_size), current_per_invocation_masks(per_invocation_masks) {}
current_wave_size(wave_size) {}
bool TranslateBlock(const IR::BasicBlock& source, std::string* error);
bool AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info,
@ -28,7 +28,7 @@ private:
IR::U32 ReadScalarCode(uint32_t code);
IR::U32 ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value);
IR::Value ReadOperand(const IR::Operand& operand, IR::Type type);
IR::U1 ThreadBit(IR::U32 low, IR::U32 high);
IR::U1 ThreadBit(IR::U32 low);
void WriteRawU32(const IR::Operand& operand, IR::U32 value);
IR::F32 ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value);
void WriteOperand(const IR::Operand& operand, IR::Value value);
@ -38,11 +38,10 @@ private:
IR::U32 ReadU32(const IR::Operand& operand);
std::array<IR::U32, 2> ReadU32Pair(const IR::Operand& operand);
IR::U64 ReadU64(const IR::Operand& operand);
IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane,
bool packed = false);
IR::F32 ReadF16AsF32(const IR::Operand& operand);
IR::F32 ReadMixF32(const IR::Operand& operand);
IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane);
IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed = false);
IR::F32 ReadF16AsF32(const IR::Operand& operand);
IR::F32 ReadMixF32(const IR::Operand& operand);
IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane);
IR::U32 ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend);
IR::U32 ReadU16AsU32(const IR::Operand& operand, bool sign_extend);
IR::U32 ReadF16LaneBits(const IR::Operand& operand, bool high_lane);
@ -51,6 +50,7 @@ private:
IR::U1 ReadCondition(const IR::Operand& operand);
IR::U32 ConditionBit(const IR::Operand& operand);
IR::U1 ReadMask(const IR::Operand& operand);
IR::U1 ReadMaskValid(const IR::Operand& operand);
void WriteMask(const IR::Operand& operand, IR::U1 value);
void WriteMask64(const IR::Operand& operand, IR::U1 value);
void WriteCompareResult(const IR::Operand& operand, IR::U1 value);
@ -76,8 +76,8 @@ private:
IR::Value GetSamplerResource(const IR::MemoryInfo& memory);
IR::Value MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base);
IR::Value ConstructU32x4(const IR::Operand& base, uint32_t count);
void WriteImageComponents(const IR::Operand& dst, IR::Value value,
const IR::MemoryInfo& memory, uint32_t component_limit);
void WriteImageComponents(const IR::Operand& dst, IR::Value value, const IR::MemoryInfo& memory,
uint32_t component_limit);
IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode);
BufferAddress ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign);
@ -101,35 +101,34 @@ private:
uint32_t upper_result);
IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
bool TranslateInstruction(const IR::Instruction& inst);
bool TranslateStateOperation(const IR::Instruction& inst);
bool TranslateControlOperation(const IR::Instruction& inst);
bool TranslateMove(const IR::Instruction& inst);
bool TranslateLaneOperation(const IR::Instruction& inst);
bool TranslateAttributeOperation(const IR::Instruction& inst);
bool TranslateMemoryOperation(const IR::Instruction& inst);
bool TranslateIntegerCompare(const IR::Instruction& inst);
bool TranslateInteger16Compare(const IR::Instruction& inst);
bool TranslateFloatCompare(const IR::Instruction& inst);
bool TranslateConversion(const IR::Instruction& inst);
bool TranslateInteger16Operation(const IR::Instruction& inst);
bool TranslatePackedInteger16(const IR::Instruction& inst);
bool TranslatePackedFloat16(const IR::Instruction& inst);
bool TranslateFloat16Operation(const IR::Instruction& inst);
bool TranslateFloatOperation(const IR::Instruction& inst);
bool TranslatePerInvocationU64Mask(const IR::Instruction& inst);
bool TranslateU64MaskOperation(const IR::Instruction& inst);
bool TranslateSimpleInteger(const IR::Instruction& inst);
bool TranslateComposedInteger(const IR::Instruction& inst);
bool TranslateExtendedInteger(const IR::Instruction& inst);
bool TranslateInstruction(const IR::Instruction& inst);
bool TranslateStateOperation(const IR::Instruction& inst);
bool TranslateControlOperation(const IR::Instruction& inst);
bool TranslateMove(const IR::Instruction& inst);
bool TranslateLaneOperation(const IR::Instruction& inst);
bool TranslateAttributeOperation(const IR::Instruction& inst);
bool TranslateMemoryOperation(const IR::Instruction& inst);
bool TranslateIntegerCompare(const IR::Instruction& inst);
bool TranslateInteger16Compare(const IR::Instruction& inst);
bool TranslateFloatCompare(const IR::Instruction& inst);
bool TranslateConversion(const IR::Instruction& inst);
bool TranslateInteger16Operation(const IR::Instruction& inst);
bool TranslatePackedInteger16(const IR::Instruction& inst);
bool TranslatePackedFloat16(const IR::Instruction& inst);
bool TranslateFloat16Operation(const IR::Instruction& inst);
bool TranslateFloatOperation(const IR::Instruction& inst);
IR::U1 EvaluateU64Mask(const IR::Instruction& inst);
bool TranslateU64MaskOperation(const IR::Instruction& inst);
bool TranslateSimpleInteger(const IR::Instruction& inst);
bool TranslateComposedInteger(const IR::Instruction& inst);
bool TranslateExtendedInteger(const IR::Instruction& inst);
IR::ValueProgram& value_program;
IR::IREmitter ir;
IR::Opcode current_opcode = IR::Opcode::ControlNop;
uint32_t current_pc = 0;
uint32_t current_vector_limit = 1;
uint32_t current_wave_size = 64;
bool current_per_invocation_masks = false;
IR::Opcode current_opcode = IR::Opcode::ControlNop;
uint32_t current_pc = 0;
uint32_t current_vector_limit = 1;
uint32_t current_wave_size = 64;
};
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail

1
src/graphics/shader/recompiler/ir/Block.h
View File

@ -49,6 +49,7 @@ public:
std::array<Value, NumScalarRegs> ssa_sreg_values {};
std::array<Value, NumScalarRegs> ssa_thread_bit_sreg_values {};
std::array<Value, NumScalarRegs> ssa_sreg_mask_tags {};
std::array<Value, NumVectorRegs> ssa_vreg_values {};
private:

8
src/graphics/shader/recompiler/ir/IREmitter.cpp
View File

@ -36,6 +36,14 @@ void IREmitter::SetThreadBitScalarReg(ScalarReg reg, U1 value) {
Emit(ValueOpcode::SetThreadBitScalarRegister, {Value(reg), value});
}
U1 IREmitter::GetScalarMaskTag(ScalarReg reg) {
return U1(Emit(ValueOpcode::GetScalarMaskTag, {Value(reg)}));
}
void IREmitter::SetScalarMaskTag(ScalarReg reg, U1 value) {
Emit(ValueOpcode::SetScalarMaskTag, {Value(reg), value});
}
U32 IREmitter::GetVectorReg(VectorReg reg) {
return U32(Emit(ValueOpcode::GetVectorRegister, {Value(reg)}));
}

2
src/graphics/shader/recompiler/ir/IREmitter.h
View File

@ -29,6 +29,8 @@ public:
void SetScalarReg(ScalarReg reg, U32 value);
U1 GetThreadBitScalarReg(ScalarReg reg);
void SetThreadBitScalarReg(ScalarReg reg, U1 value);
U1 GetScalarMaskTag(ScalarReg reg);
void SetScalarMaskTag(ScalarReg reg, U1 value);
U32 GetVectorReg(VectorReg reg);
void SetVectorReg(VectorReg reg, U32 value);
U1 GetGotoVariable(uint32_t id);

39
src/graphics/shader/recompiler/ir/ShaderIR.cpp
View File

@ -44,6 +44,23 @@ bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBloc
if (!ScalarResultWritesSccNonZero(decoded.opcode)) {
return true;
}
switch (decoded.opcode) {
case Decoder::Opcode::S_AND_B64:
case Decoder::Opcode::S_ANDN2_B64:
case Decoder::Opcode::S_NOT_B64:
case Decoder::Opcode::S_OR_B64:
case Decoder::Opcode::S_ORN2_B64:
case Decoder::Opcode::S_XOR_B64:
case Decoder::Opcode::S_NAND_B64:
case Decoder::Opcode::S_NOR_B64:
case Decoder::Opcode::S_XNOR_B64:
case Decoder::Opcode::S_WQM_B64:
// These instructions maintain an invocation-local mask shadow and update SCC
// while lowering the operation. A raw dword comparison would disagree after
// complementing a mask temporary.
return true;
default: break;
}
if (block.instructions.empty()) {
SetError(error, "internal IR error: missing scalar result for SCC update");
return false;
@ -206,14 +223,14 @@ uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand) {
MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind) {
MemoryInfo mem;
mem.kind = kind;
mem.offset = decoded.offset;
mem.secondary_offset = decoded.secondary_offset;
mem.dmask = decoded.dmask;
mem.data_dwords = decoded.data_dwords;
mem.data_bits = decoded.data_bits;
mem.component_count = decoded.data_components != 0u ? decoded.data_components
: decoded.data_dwords;
mem.kind = kind;
mem.offset = decoded.offset;
mem.secondary_offset = decoded.secondary_offset;
mem.dmask = decoded.dmask;
mem.data_dwords = decoded.data_dwords;
mem.data_bits = decoded.data_bits;
mem.component_count =
decoded.data_components != 0u ? decoded.data_components : decoded.data_dwords;
mem.data_format = decoded.data_format;
mem.number_format = decoded.number_format;
mem.image_sample_flags = decoded.image_sample_flags;
@ -224,9 +241,9 @@ MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKi
mem.image_nsa_addr[i] = decoded.image_nsa_addr[i];
}
mem.memory_segment = decoded.memory_segment;
mem.address_is_full = kind == ResourceKind::Flat ||
(kind == ResourceKind::Global &&
decoded.src1.kind == Decoder::OperandKind::Vgpr);
mem.address_is_full =
kind == ResourceKind::Flat ||
(kind == ResourceKind::Global && decoded.src1.kind == Decoder::OperandKind::Vgpr);
mem.data_signed = decoded.data_signed;
mem.typed = decoded.typed;
mem.formatted = decoded.formatted;

2
src/graphics/shader/recompiler/ir/ShaderIR.h
View File

@ -456,7 +456,6 @@ struct ShaderInfo {
};
struct SpirvRequirements {
bool requires_exact_subgroup = false;
bool subgroup_ballot = false;
bool subgroup_shuffle = false;
bool subgroup_local_invocation_id = false;
@ -469,7 +468,6 @@ struct SpirvRequirements {
struct Program {
ShaderType stage = ShaderType::Unknown;
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave;
uint64_t shader_hash = 0;
uint32_t wave_size = 64;
uint32_t user_data_base = 0;

4
src/graphics/shader/recompiler/ir/ShaderIRLog.cpp
View File

@ -192,10 +192,8 @@ std::string ExportTargetKindToString(ExportTargetKind kind) {
std::string ProgramToString(const Program& program) {
std::string text;
text += fmt::format("mode={} masks={} fallback={} reason={}\n",
text += fmt::format("mode={} fallback={} reason={}\n",
program.dispatcher_fallback ? "dispatcher" : "structured",
program.lane_mask_mode == ShaderLaneMaskMode::NativeWave ? "native-wave"
: "per-invocation",
CFG::FailureKindToString(program.cfg_failure_kind).c_str(),
program.fallback_reason.c_str());
for (const auto& block: program.blocks) {

2
src/graphics/shader/recompiler/ir/ValueProgram.cpp
View File

@ -17,6 +17,8 @@ bool IsRegisterStatePseudo(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::GetThreadBitScalarRegister:
case ValueOpcode::SetThreadBitScalarRegister:
case ValueOpcode::GetScalarMaskTag:
case ValueOpcode::SetScalarMaskTag:
case ValueOpcode::GetScalarRegister:
case ValueOpcode::SetScalarRegister:
case ValueOpcode::GetVectorRegister:

2
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc
View File

@ -11,6 +11,8 @@ VALUE_OPCODE(GetShaderBase, U64)
VALUE_OPCODE(GetBuiltin, U32, U32, U32)
VALUE_OPCODE(GetThreadBitScalarRegister, U1, ScalarReg)
VALUE_OPCODE(SetThreadBitScalarRegister, Void, ScalarReg, U1)
VALUE_OPCODE(GetScalarMaskTag, U1, ScalarReg)
VALUE_OPCODE(SetScalarMaskTag, Void, ScalarReg, U1)
VALUE_OPCODE(GetScalarRegister, U32, ScalarReg)
VALUE_OPCODE(SetScalarRegister, Void, ScalarReg, U32)
VALUE_OPCODE(GetVectorRegister, U32, VectorReg)

29
src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp
View File

@ -15,6 +15,10 @@ struct ThreadBitScalarReg {
ScalarReg reg {};
auto operator<=>(const ThreadBitScalarReg&) const = default;
};
struct ScalarMaskTag {
ScalarReg reg {};
auto operator<=>(const ScalarMaskTag&) const = default;
};
struct ExecTag {
auto operator<=>(const ExecTag&) const = default;
};
@ -41,8 +45,9 @@ struct GotoVariable {
auto operator<=>(const GotoVariable&) const = default;
};
using Variable = std::variant<ScalarReg, ThreadBitScalarReg, VectorReg, GotoVariable, SccTag,
ExecTag, ExecLoTag, ExecHiTag, VccTag, VccLoTag, VccHiTag, M0Tag>;
using Variable =
std::variant<ScalarReg, ThreadBitScalarReg, ScalarMaskTag, VectorReg, GotoVariable, SccTag,
ExecTag, ExecLoTag, ExecHiTag, VccTag, VccLoTag, VccHiTag, M0Tag>;
using ValueMap = std::unordered_map<Block*, Value>;
struct DefTable {
@ -66,6 +71,14 @@ struct DefTable {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
block->ssa_thread_bit_sreg_values[RegIndex(value.reg)] = definition;
}
const Value& Get(Block* block, ScalarMaskTag value) {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
return block->ssa_sreg_mask_tags[RegIndex(value.reg)];
}
void Set(Block* block, ScalarMaskTag value, Value definition) {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
block->ssa_sreg_mask_tags[RegIndex(value.reg)] = definition;
}
const Value& Get(Block* block, VectorReg reg) {
EXIT_IF(RegIndex(reg) >= NumVectorRegs);
@ -116,6 +129,9 @@ ValueOpcode UndefOpcode(ScalarReg) {
ValueOpcode UndefOpcode(ThreadBitScalarReg) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(ScalarMaskTag) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(VectorReg) {
return ValueOpcode::UndefU32;
}
@ -153,6 +169,9 @@ Value InitialValue(ScalarReg) {
Value InitialValue(ThreadBitScalarReg) {
return Value(false);
}
Value InitialValue(ScalarMaskTag) {
return Value(false);
}
Value InitialValue(VectorReg) {
return Value(0u);
}
@ -321,6 +340,9 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
case ValueOpcode::SetThreadBitScalarRegister:
pass.Write(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block, inst.Arg(1));
break;
case ValueOpcode::SetScalarMaskTag:
pass.Write(ScalarMaskTag {inst.Arg(0).ScalarRegister()}, block, inst.Arg(1));
break;
case ValueOpcode::SetVectorRegister:
pass.Write(inst.Arg(0).VectorRegister(), block, inst.Arg(1));
break;
@ -342,6 +364,9 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
inst.ReplaceUsesWith(
pass.Read(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block));
break;
case ValueOpcode::GetScalarMaskTag:
inst.ReplaceUsesWith(pass.Read(ScalarMaskTag {inst.Arg(0).ScalarRegister()}, block));
break;
case ValueOpcode::GetVectorRegister:
inst.ReplaceUsesWith(pass.Read(inst.Arg(0).VectorRegister(), block));
break;

42
src/graphics/shader/shader.cpp
View File

@ -112,16 +112,14 @@ static std::unique_ptr<std::unordered_map<uint64_t, ShaderMappedData>> g_shader_
static std::mutex g_shader_map_mutex;
struct ShaderStageProgramKey {
ShaderType stage = ShaderType::Unknown;
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave;
uint64_t shader_hash = 0;
ShaderType stage = ShaderType::Unknown;
uint64_t shader_hash = 0;
ShaderId program_id;
Config::ShaderOptimizationType optimization_type = Config::ShaderOptimizationType::None;
bool operator==(const ShaderStageProgramKey& other) const {
return stage == other.stage && lane_mask_mode == other.lane_mask_mode &&
shader_hash == other.shader_hash && program_id == other.program_id &&
optimization_type == other.optimization_type;
return stage == other.stage && shader_hash == other.shader_hash &&
program_id == other.program_id && optimization_type == other.optimization_type;
}
};
@ -133,7 +131,6 @@ struct ShaderStageProgramKeyHash {
};
mix(static_cast<uint32_t>(key.stage));
mix(static_cast<uint32_t>(key.lane_mask_mode));
mix(Common::hash64(key.shader_hash));
mix(key.program_id.hash0);
mix(key.program_id.crc32);
@ -920,11 +917,9 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs,
}
static ShaderStageProgramKey MakeShaderStageProgramKey(ShaderType stage, uint64_t shader_hash,
const ShaderId& program_id,
ShaderLaneMaskMode lane_mask_mode) {
const ShaderId& program_id) {
ShaderStageProgramKey key {};
key.stage = stage;
key.lane_mask_mode = lane_mask_mode;
key.shader_hash = shader_hash;
key.program_id = program_id;
key.optimization_type = Config::GetShaderOptimizationType();
@ -1055,7 +1050,6 @@ static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Prog
static void ShaderAppendNativeSpecialization(std::vector<uint32_t>& ids,
const ShaderRecompiler::IR::Program& program) {
EXIT_IF(!program.binding_layout_complete);
ids.push_back(static_cast<uint32_t>(program.lane_mask_mode));
ids.push_back(program.bindings.descriptor_set);
ids.push_back(program.bindings.push_constant_offset);
ids.push_back(program.bindings.push_constant_size);
@ -1136,8 +1130,7 @@ static std::span<const uint32_t> AddShaderProgramPermutation(const char* stage,
}
bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& info,
std::span<const uint32_t>& spirv) {
ShaderVertexInputInfo& info, std::span<const uint32_t>& spirv) {
spirv = {};
if (!ShaderGetStaticInputInfoVS(regs, sh, info)) {
@ -1145,8 +1138,7 @@ bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegis
}
const auto shader_hash = regs.gs_regs.chksum;
const auto program_id = ShaderGetIdVS(regs, info, false);
const auto key =
MakeShaderStageProgramKey(ShaderType::Vertex, shader_hash, program_id, lane_mask_mode);
const auto key = MakeShaderStageProgramKey(ShaderType::Vertex, shader_hash, program_id);
{
std::scoped_lock lock(g_shader_program_cache_mutex);
@ -1163,7 +1155,7 @@ bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegis
}
std::vector<uint32_t> compiled_spirv;
if (!ShaderCompileSpirvVS(regs, sh, lane_mask_mode, info, compiled_spirv)) {
if (!ShaderCompileSpirvVS(regs, sh, info, compiled_spirv)) {
return false;
}
@ -1175,7 +1167,7 @@ bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegis
}
bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, const ShaderVertexInputInfo& vs_info,
const ShaderVertexInputInfo& vs_info,
std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
ShaderPixelInputInfo& ps_info, std::span<const uint32_t>& spirv) {
spirv = {};
@ -1184,8 +1176,7 @@ bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegist
const auto shader_hash =
regs.ps_regs.chksum != 0 ? regs.ps_regs.chksum : regs.ps_regs.data_addr;
const auto program_id = ShaderGetIdPS(regs, ps_info, false);
const auto key =
MakeShaderStageProgramKey(ShaderType::Pixel, shader_hash, program_id, lane_mask_mode);
const auto key = MakeShaderStageProgramKey(ShaderType::Pixel, shader_hash, program_id);
{
std::scoped_lock lock(g_shader_program_cache_mutex);
@ -1202,7 +1193,7 @@ bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegist
}
std::vector<uint32_t> compiled_spirv;
if (!ShaderCompileSpirvPS(regs, sh, lane_mask_mode, ps_info, compiled_spirv)) {
if (!ShaderCompileSpirvPS(regs, sh, ps_info, compiled_spirv)) {
return false;
}
@ -1220,8 +1211,7 @@ bool ShaderCompileInfoCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegi
ShaderGetStaticInputInfoCS(regs, sh, info);
const auto shader_hash = regs.cs_regs.data_addr;
const auto program_id = ShaderGetIdCS(regs, info, false);
const auto key = MakeShaderStageProgramKey(ShaderType::Compute, shader_hash, program_id,
ShaderLaneMaskMode::NativeWave);
const auto key = MakeShaderStageProgramKey(ShaderType::Compute, shader_hash, program_id);
{
std::scoped_lock lock(g_shader_program_cache_mutex);
@ -1449,8 +1439,7 @@ static void DumpShaderRecompilerOriginal(const char* type, uint64_t shader_hash,
}
bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& input_info,
std::vector<uint32_t>& spirv) {
ShaderVertexInputInfo& input_info, std::vector<uint32_t>& spirv) {
KYTY_PROFILER_FUNCTION(profiler::colors::Amber300);
EXIT_NOT_IMPLEMENTED(regs.es_regs.data_addr == 0 || regs.gs_regs.chksum == 0);
@ -1460,7 +1449,6 @@ bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegi
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Vertex;
options.lane_mask_mode = lane_mask_mode;
options.shader_hash = regs.gs_regs.chksum;
options.shader_base = shader_addr;
options.user_data_base = 8;
@ -1506,8 +1494,7 @@ bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegi
}
bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderPixelInputInfo& input_info,
std::vector<uint32_t>& spirv) {
ShaderPixelInputInfo& input_info, std::vector<uint32_t>& spirv) {
KYTY_PROFILER_FUNCTION(profiler::colors::Blue300);
const uint64_t shader_addr = regs.ps_regs.data_addr;
@ -1516,7 +1503,6 @@ bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegis
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Pixel;
options.lane_mask_mode = lane_mask_mode;
options.shader_hash = shader_hash;
options.shader_base = shader_addr;
options.user_data_count = regs.ps_regs.rsrc2.user_sgpr;

13
src/graphics/shader/shader.h
View File

@ -35,8 +35,6 @@ struct ShaderRegisters;
enum class ShaderType { Unknown, Vertex, Pixel, Fetch, Compute };
enum class ShaderLaneMaskMode { NativeWave, PerInvocation };
namespace ShaderRecompiler::IR {
struct Program;
struct ResourceSnapshot;
@ -261,20 +259,17 @@ ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo& regs, const ShaderComputeInp
bool include_bind_specialization);
// Returned SPIR-V spans are read-only views backed by the shader program cache.
bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& input_info,
std::span<const uint32_t>& spirv);
ShaderVertexInputInfo& input_info, std::span<const uint32_t>& spirv);
bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, const ShaderVertexInputInfo& vs_info,
const ShaderVertexInputInfo& vs_info,
std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
ShaderPixelInputInfo& input_info, std::span<const uint32_t>& spirv);
bool ShaderCompileInfoCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderComputeInputInfo& input_info, std::span<const uint32_t>& spirv);
bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& input_info,
std::vector<uint32_t>& spirv);
ShaderVertexInputInfo& input_info, std::vector<uint32_t>& spirv);
bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderPixelInputInfo& input_info,
std::vector<uint32_t>& spirv);
ShaderPixelInputInfo& input_info, std::vector<uint32_t>& spirv);
bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderComputeInputInfo& input_info, std::vector<uint32_t>& spirv);
bool ShaderAddressValid(uint64_t addr);

565
tests/ShaderRecompilerComputeTests.cpp
View File

@ -574,15 +574,14 @@ constexpr u32 EncodeMimg0(u32 opcode, u32 dmask, u32 nsa_dwords = 0,
return (0x3cu << 26u) | ((opcode >> 7u) & 0x1u) |
((nsa_dwords & 0x3u) << 1u) | ((dim & 0x7u) << 3u) |
((dmask & 0xfu) << 8u) | (glc ? (1u << 13u) : 0u) |
(r128 ? (1u << 15u) : 0u) |
((opcode & 0x7fu) << 18u);
(r128 ? (1u << 15u) : 0u) | ((opcode & 0x7fu) << 18u);
}
constexpr u32 EncodeMimg1(u32 vdata, u32 vaddr, u32 srsrc = 0, u32 ssamp = 0,
bool a16 = false, bool d16 = false) {
return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) |
((vdata & 0xffu) << 8u) | (vaddr & 0xffu) |
(a16 ? (1u << 30u) : 0u) | (d16 ? (1u << 31u) : 0u);
((vdata & 0xffu) << 8u) | (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u) |
(d16 ? (1u << 31u) : 0u);
}
constexpr u32 EncodeVintrp(u32 opcode, u32 dst, u32 attr, u32 chan, u32 src) {
@ -1178,7 +1177,8 @@ CompiledShader CompileCase(const TestCase &test) {
result.resources.user_data[reg - result.program.user_data_base]);
}
packed_user_data.resize(result.program.bindings.ShaderDataDwords());
const auto buffer_count = static_cast<u32>(result.program.info.buffers.size());
const auto buffer_count =
static_cast<u32>(result.program.info.buffers.size());
for (u32 i = 0; i < buffer_count; i++) {
u32 offset = 0;
if (i < test.storage_buffer_offsets.size()) {
@ -1197,8 +1197,8 @@ CompiledShader CompileCase(const TestCase &test) {
Require(test.name, "shader data", offset < 256,
"address-memory offset is not representable");
const auto index = buffer_count + i;
packed_user_data[result.program.bindings.memory_offset_dword + index / 4u] |=
offset << ((index % 4u) * 8u);
packed_user_data[result.program.bindings.memory_offset_dword +
index / 4u] |= offset << ((index % 4u) * 8u);
}
return {std::move(result.spirv), std::move(result.program),
std::move(result.resources.flattened_srt),
@ -3562,8 +3562,7 @@ public:
narrow_target.info.tile_mode = Prospero::TileMode::kRenderTarget;
narrow_target.info.mip_layout[0] = {0, 0x10000, 128, 1};
narrow_target.view_info.format = narrow_target.info.pixel_format;
narrow_target.view_info.usage =
vk::ImageUsageFlagBits::eColorAttachment;
narrow_target.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto narrow_target_id = texture_cache.FindImage(narrow_target);
const auto narrow_target_view =
texture_cache.FindRenderTarget(narrow_target_id, narrow_target);
@ -3868,14 +3867,13 @@ public:
std::memcpy(memory + raw_d16_offset, raw_d16_values.data(),
sizeof(raw_d16_values));
auto raw_d16 = MakeLinearDesc(
base + raw_d16_offset, sizeof(raw_d16_values),
vk::Format::eD16Unorm, Prospero::BufferFormat::k16UNorm,
Prospero::ImageType::kColor2D, {2, 1, 1}, 1, sizeof(uint16_t), 1);
base + raw_d16_offset, sizeof(raw_d16_values), vk::Format::eD16Unorm,
Prospero::BufferFormat::k16UNorm, Prospero::ImageType::kColor2D,
{2, 1, 1}, 1, sizeof(uint16_t), 1);
raw_d16.type = BindingType::DepthTarget;
raw_d16.view_info.format = vk::Format::eD16Unorm;
raw_d16.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
raw_d16.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
raw_d16.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto raw_d16_image = texture_cache.FindImage(raw_d16);
(void)texture_cache.FindDepthTarget(raw_d16_image, raw_d16);
@ -3887,13 +3885,13 @@ public:
raw_d16_uint.view_info.aspect = vk::ImageAspectFlagBits::eColor;
raw_d16_uint.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto raw_d16_uint_image = texture_cache.FindImage(raw_d16_uint);
Require(name, "raw D16 uint texture backing",
raw_d16_uint_image && raw_d16_uint_image != raw_d16_image &&
!TextureCacheTestAccess::Contains(texture_cache,
raw_d16_image) &&
texture_cache.GetImage(raw_d16_uint_image)
.backing.format == vk::Format::eR16Uint,
"D16 uint sampling did not recreate a true integer color backing");
Require(
name, "raw D16 uint texture backing",
raw_d16_uint_image && raw_d16_uint_image != raw_d16_image &&
!TextureCacheTestAccess::Contains(texture_cache, raw_d16_image) &&
texture_cache.GetImage(raw_d16_uint_image).backing.format ==
vk::Format::eR16Uint,
"D16 uint sampling did not recreate a true integer color backing");
Require(name, "raw D16 uint texture view",
texture_cache.FindTexture(raw_d16_uint_image, raw_d16_uint) !=
nullptr,
@ -3916,9 +3914,9 @@ public:
layered_d16_color);
auto &layered_d16_color_native =
texture_cache.GetImage(layered_d16_color_image);
layered_d16_color_native.Transit(
vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {}, scheduler.Current().Handle());
layered_d16_color_native.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {},
scheduler.Current().Handle());
vk::ClearColorValue layered_raw_d16_clear_value{};
layered_raw_d16_clear_value.float32[0] = 1.0f;
const vk::ImageSubresourceRange layered_raw_d16_clear_range{
@ -3931,8 +3929,7 @@ public:
auto layered_raw_d16_depth = layered_d16_color;
layered_raw_d16_depth.type = BindingType::DepthTarget;
layered_raw_d16_depth.info.data.size =
sizeof(uint16_t) * 2 * 5;
layered_raw_d16_depth.info.data.size = sizeof(uint16_t) * 2 * 5;
layered_raw_d16_depth.info.pixel_format = vk::Format::eD16Unorm;
layered_raw_d16_depth.info.guest_format =
Prospero::BufferFormat::k16UNorm;
@ -3940,8 +3937,7 @@ public:
layered_raw_d16_depth.info.mip_layout[0].size =
layered_raw_d16_depth.info.data.size;
layered_raw_d16_depth.view_info.format = vk::Format::eD16Unorm;
layered_raw_d16_depth.view_info.aspect =
vk::ImageAspectFlagBits::eDepth;
layered_raw_d16_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
layered_raw_d16_depth.view_info.layer_count = 5;
layered_raw_d16_depth.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
@ -3949,28 +3945,25 @@ public:
texture_cache.FindImage(layered_raw_d16_depth);
const auto &layered_depth_owner =
texture_cache.GetImage(layered_raw_d16_depth_image);
Require(
name, "layered raw D16 owner layout",
layered_raw_d16_depth_image != layered_d16_color_image &&
!TextureCacheTestAccess::Contains(
texture_cache, layered_d16_color_image) &&
layered_depth_owner.info.resources.layers == 6 &&
layered_depth_owner.info.data.size ==
sizeof(layered_raw_d16_values) &&
layered_depth_owner.info.mip_layout[0].size ==
sizeof(layered_raw_d16_values),
"a partial depth view detached the retained six-layer owner from "
"its full guest range");
Require(name, "layered raw D16 owner layout",
layered_raw_d16_depth_image != layered_d16_color_image &&
!TextureCacheTestAccess::Contains(texture_cache,
layered_d16_color_image) &&
layered_depth_owner.info.resources.layers == 6 &&
layered_depth_owner.info.data.size ==
sizeof(layered_raw_d16_values) &&
layered_depth_owner.info.mip_layout[0].size ==
sizeof(layered_raw_d16_values),
"a partial depth view detached the retained six-layer owner from "
"its full guest range");
(void)texture_cache.FindDepthTarget(layered_raw_d16_depth_image,
layered_raw_d16_depth);
auto reacquired_layered_raw_d16_uint = layered_d16_color;
reacquired_layered_raw_d16_uint.info.pixel_format =
vk::Format::eR16Uint;
reacquired_layered_raw_d16_uint.info.pixel_format = vk::Format::eR16Uint;
reacquired_layered_raw_d16_uint.info.guest_format =
Prospero::BufferFormat::k16UInt;
reacquired_layered_raw_d16_uint.view_info.format =
vk::Format::eR16Uint;
reacquired_layered_raw_d16_uint.view_info.format = vk::Format::eR16Uint;
const auto reacquired_layered_raw_d16_uint_image =
texture_cache.FindImage(reacquired_layered_raw_d16_uint);
const auto reacquired_layered_raw_d16_uint_view =
@ -3986,13 +3979,13 @@ public:
vk::AccessFlagBits2::eTransferRead, {}, scheduler.Current().Handle());
vk::BufferImageCopy layered_raw_d16_copy{};
layered_raw_d16_copy.bufferRowLength = 2;
layered_raw_d16_copy.imageSubresource = {
vk::ImageAspectFlagBits::eColor, 0, 0, 6};
layered_raw_d16_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor,
0, 0, 6};
layered_raw_d16_copy.imageExtent = {2, 1, 1};
scheduler.Current().Handle().copyImageToBuffer(
reacquired_layered_raw_d16_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal,
layered_raw_d16_readback.buffer, 1, &layered_raw_d16_copy);
vk::ImageLayout::eTransferSrcOptimal, layered_raw_d16_readback.buffer,
1, &layered_raw_d16_copy);
HostReadBarrier(layered_raw_d16_readback.buffer,
layered_raw_d16_readback.size,
vk::PipelineStageFlagBits::eTransfer,
@ -4010,8 +4003,8 @@ public:
reacquired_layered_raw_d16_uint_image &&
reacquired_layered_raw_d16_uint_image !=
layered_raw_d16_depth_image &&
!TextureCacheTestAccess::Contains(
texture_cache, layered_raw_d16_depth_image) &&
!TextureCacheTestAccess::Contains(texture_cache,
layered_raw_d16_depth_image) &&
reacquired_layered_raw_d16_native.backing.format ==
vk::Format::eR16Uint &&
reacquired_layered_raw_d16_uint_view != nullptr &&
@ -7717,14 +7710,13 @@ public:
RenderDepthInfo read_only_clear_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, 1, scheduler.Current(), read_only_clear_depth);
Require(
name, "read-only stencil clear suppression",
read_only_clear_depth.image_id == phased_depth.image_id &&
!read_only_clear_depth.stencil_clear_enable &&
!read_only_clear_depth.AttachmentWriteAspects() &&
depth_attachment_layout(read_only_clear_depth) ==
vk::ImageLayout::eDepthStencilReadOnlyOptimal,
"a PS5 target-level stencil write disable retained a host clear");
Require(name, "read-only stencil clear suppression",
read_only_clear_depth.image_id == phased_depth.image_id &&
!read_only_clear_depth.stencil_clear_enable &&
!read_only_clear_depth.AttachmentWriteAspects() &&
depth_attachment_layout(read_only_clear_depth) ==
vk::ImageLayout::eDepthStencilReadOnlyOptimal,
"a PS5 target-level stencil write disable retained a host clear");
registers.SetDepthRenderTarget(phased_depth_target);
registers.SetDepthControl(phased_depth_control);
@ -9972,21 +9964,21 @@ public:
const auto surface_format = TextureGetSurfaceFormatInfo(format);
constexpr u32 tail_xy[7][2] = {{64, 0}, {0, 64}, {32, 0}, {0, 32},
{16, 0}, {8, 16}, {0, 24}};
bool valid =
built && Prospero::NumBytesPerElement(format) == 4 &&
Prospero::BlockCompressedBytesPerBlock(format) == 0 &&
Prospero::IsSampledTextureFormat(format) &&
Prospero::IsUintTextureFormat(format) &&
Prospero::RemapTextureFormat(format) ==
Prospero::BufferFormat::k32UInt &&
surface_format.vk_format == vk::Format::eR32Uint &&
surface_format.conversion_format == format &&
VulkanFormat(format) == vk::Format::eUndefined &&
surface.texture.block.block_width == 128 &&
surface.texture.block.block_height == 128 &&
surface.texture.block.block_depth == 1 &&
surface.first_tail_level == 1 && surface.mips[0].offset == 0x10000 &&
total.size == 0x20000 && total.align == 0x10000;
bool valid = built && Prospero::NumBytesPerElement(format) == 4 &&
Prospero::BlockCompressedBytesPerBlock(format) == 0 &&
Prospero::IsSampledTextureFormat(format) &&
Prospero::IsUintTextureFormat(format) &&
Prospero::RemapTextureFormat(format) ==
Prospero::BufferFormat::k32UInt &&
surface_format.vk_format == vk::Format::eR32Uint &&
surface_format.conversion_format == format &&
VulkanFormat(format) == vk::Format::eUndefined &&
surface.texture.block.block_width == 128 &&
surface.texture.block.block_height == 128 &&
surface.texture.block.block_depth == 1 &&
surface.first_tail_level == 1 &&
surface.mips[0].offset == 0x10000 && total.size == 0x20000 &&
total.align == 0x10000;
for (u32 level = 1; level < levels && valid; level++) {
valid &= surface.mips[level].offset == 0 &&
surface.mips[level].tail_x == tail_xy[level - 1][0] &&
@ -9999,16 +9991,15 @@ public:
const auto storage_only_format = Prospero::BufferFormat::k32SInt;
const auto storage_only_surface =
TextureGetSurfaceFormatInfo(storage_only_format);
Require(
name, "storage-only surface mapping",
!Prospero::IsSampledTextureFormat(storage_only_format) &&
!Prospero::IsUintTextureFormat(storage_only_format) &&
Prospero::RemapTextureFormat(storage_only_format) ==
storage_only_format &&
storage_only_surface.vk_format == vk::Format::eR32Sint &&
storage_only_surface.conversion_format ==
Prospero::BufferFormat::kInvalid,
"storage-only native backing was conflated with sampled support");
Require(name, "storage-only surface mapping",
!Prospero::IsSampledTextureFormat(storage_only_format) &&
!Prospero::IsUintTextureFormat(storage_only_format) &&
Prospero::RemapTextureFormat(storage_only_format) ==
storage_only_format &&
storage_only_surface.vk_format == vk::Format::eR32Sint &&
storage_only_surface.conversion_format ==
Prospero::BufferFormat::kInvalid,
"storage-only native backing was conflated with sampled support");
}
{
@ -11997,18 +11988,26 @@ TestCase ScalarNotB64UpdatesScc() {
code.push_back(EncodeSop1(0x08, 8, 6));
capture_scc(10);
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 12, 106));
code.push_back(EncodeSop1(0x08, 14, 12));
capture_scc(16);
AppendStoreSgpr(&code, 4, 0);
AppendStoreSgpr(&code, 10, 1);
AppendStoreSgprPair(&code, 2, 2);
AppendStoreSgprPair(&code, 8, 4);
AppendStoreSgpr(&code, 16, 6);
AppendStoreSgprPair(&code, 14, 7);
AppendEnd(&code);
return {"ScalarNotB64UpdatesScc",
code,
{},
{0, 1, 0, 0, 1, 0},
{0, 1, 0, 0, 1, 0, 0, 0xfffffffeu, 0xffffffffu},
{O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_NOT_B64, O::S_CSELECT_B32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarQuadmaskB64() {
@ -12109,9 +12108,8 @@ TestCase ScalarSaveExecOps() {
return {"ScalarSaveExecOps",
code,
{},
{0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0xffffffffu,
0xffffffffu, 0xffffffffu, 0x00000003u, 0xffffffffu, 0x00000003u,
0x00000002u, 0xffffffffu, 1},
// EXEC and saved EXEC values are invocation-local Booleans.
{1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0},
{O::S_MOV_B32, O::S_AND_SAVEEXEC_B64, O::S_ORN2_SAVEEXEC_B64,
O::S_ANDN1_SAVEEXEC_B64, O::S_AND_SAVEEXEC_B32,
O::S_ANDN1_SAVEEXEC_B32, O::S_MOV_B64, O::V_MOV_B32,
@ -12138,7 +12136,7 @@ TestCase ScalarOrn2SaveexecUsesSourceOrNotExec() {
return {"ScalarOrn2SaveexecUsesSourceOrNotExec",
code,
{},
{0x0000000cu, 0x80000000u, 0xfffffff3u, 0x7fffffffu, 1},
{1, 0, 1, 0, 1},
{O::S_MOV_B32, O::S_ORN2_SAVEEXEC_B64, O::S_MOV_B64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
@ -12380,15 +12378,13 @@ TestCase Scalar64BitOps() {
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarAndn2B64SccBranch() {
TestCase ScalarAndn2B64SccUsesMaskShadow() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSMovB32(0, InlineU32(3)),
EncodeSMovB32(1, InlineU32(0)),
EncodeSMovB32(2, InlineU32(1)),
EncodeSMovB32(3, InlineU32(0)),
EncodeSop2(0x15, 4, 0, 2),
EncodeVopc(0xc2, InlineU32(0), 0),
EncodeSop1(0x04, 0, 106),
EncodeSop2(0x15, 4, 0, InlineU32(0)),
EncodeVop1(0x01, 0, InlineU32(1)),
EncodeSopp(0x04, 1),
EncodeVop1(0x01, 0, InlineU32(7)),
@ -12396,14 +12392,141 @@ TestCase ScalarAndn2B64SccBranch() {
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"ScalarAndn2B64SccBranch",
return {"ScalarAndn2B64SccUsesMaskShadow",
code,
{},
{7},
{O::S_MOV_B32, O::S_ANDN2_B64, O::V_MOV_B32, O::S_CBRANCH_SCC0,
{O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_ANDN2_B64, O::V_MOV_B32,
O::S_CBRANCH_SCC0, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarMaskHighWriteInvalidatesProvenance() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVopc(0xc2, Vgpr(0), 0), EncodeSop1(0x04, 0, 106),
EncodeSMovB32(1, InlineU32(1)), EncodeSop1(0x08, 2, 0),
EncodeVop1(0x01, 0, InlineU32(1)), EncodeSopp(0x04, 1),
EncodeVop1(0x01, 0, InlineU32(7)),
};
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"ScalarMaskHighWriteInvalidatesProvenance",
code,
{},
{7},
{O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_MOV_B32, O::S_NOT_B64,
O::V_MOV_B32, O::S_CBRANCH_SCC0, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarSelectB64PreservesMaskProvenance() {
using O = ShaderOpcode;
std::vector<u32> code;
const auto set_scc = [&] {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1)));
};
const auto capture_scc = [&](u32 dst) {
code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
};
AppendSMovLiteral(&code, 0, 0xfffffffeu);
AppendSMovLiteral(&code, 1, 0xffffffffu);
code.push_back(EncodeSMovB32(2, InlineU32(0)));
code.push_back(EncodeSMovB32(3, InlineU32(0)));
set_scc();
code.push_back(EncodeSop2(0x0b, 4, 0, 2));
code.push_back(EncodeSop1(0x08, 6, 4));
capture_scc(8);
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 10, 106));
set_scc();
code.push_back(EncodeSop2(0x0b, 12, 10, 2));
code.push_back(EncodeSop1(0x08, 14, 12));
capture_scc(16);
AppendStoreSgpr(&code, 8, 0);
AppendStoreSgpr(&code, 16, 1);
AppendEnd(&code);
return {"ScalarSelectB64PreservesMaskProvenance",
code,
{},
{1, 0},
{O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CSELECT_B64, O::S_NOT_B64,
O::S_CSELECT_B32, O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarWqmB64SelectsSccDomain() {
using O = ShaderOpcode;
std::vector<u32> code;
const auto capture_scc = [&](u32 dst) {
code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
};
code.push_back(EncodeSMovB32(0, InlineU32(0)));
code.push_back(EncodeSMovB32(1, InlineU32(1)));
code.push_back(EncodeSop1(0x0a, 2, 0));
capture_scc(4);
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 6, 106));
code.push_back(EncodeSop1(0x08, 8, 6));
code.push_back(EncodeSop1(0x0a, 10, 8));
capture_scc(12);
AppendStoreSgpr(&code, 4, 0);
AppendStoreSgpr(&code, 12, 1);
AppendEnd(&code);
return {"ScalarWqmB64SelectsSccDomain",
code,
{},
{1, 0},
{O::S_MOV_B32, O::S_WQM_B64, O::S_CSELECT_B32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::S_NOT_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarMaskProvenanceOverlapAndMixedBinary() {
using O = ShaderOpcode;
std::vector<u32> code;
const auto capture_scc = [&](u32 dst) {
code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
};
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 0, 106));
code.push_back(EncodeSop1(0x04, 1, 0));
code.push_back(EncodeSop1(0x08, 3, 1));
capture_scc(5);
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 8, 106));
code.push_back(EncodeSMovB32(10, InlineU32(1)));
code.push_back(EncodeSMovB32(11, InlineU32(0)));
code.push_back(EncodeSop2(0x19, 12, 8, 10));
capture_scc(14);
AppendStoreSgpr(&code, 5, 0);
AppendStoreSgpr(&code, 14, 1);
AppendEnd(&code);
return {"ScalarMaskProvenanceOverlapAndMixedBinary",
code,
{},
{0, 1},
{O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_NOT_B64, O::S_CSELECT_B32,
O::S_MOV_B32, O::S_NAND_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarLiteral() {
using O = ShaderOpcode;
@ -13003,7 +13126,9 @@ TestCase VectorAddcWritesPerLaneCarryOut() {
TestCase test;
test.name = "VectorAddcWritesPerLaneCarryOut";
test.code = code;
test.expected = std::vector<u32>(8, 0x0fu);
// VCC is represented as the current invocation's carry bit, not a shared
// four-lane ballot word.
test.expected = std::vector<u32>(8, 1u);
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_F_U32, O::V_ADDC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
@ -13121,7 +13246,8 @@ TestCase VectorVop3BCarryOutWritesSgprMask() {
TestCase test;
test.name = "VectorVop3BCarryOutWritesSgprMask";
test.code = code;
test.expected = std::vector<u32>(12, 0x0fu);
// Scalar mask destinations keep the current invocation's bit.
test.expected = std::vector<u32>(12, 1u);
test.opcodes = {O::V_MOV_B32, O::V_ADD_I32, O::V_SUB_I32,
O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
@ -13153,7 +13279,7 @@ TestCase VectorVop3BCarryOutUsesEncodedSdst() {
TestCase test;
test.name = "VectorVop3BCarryOutUsesEncodedSdst";
test.code = code;
test.expected = std::vector<u32>(12, 0x0fu);
test.expected = std::vector<u32>(12, 1u);
test.opcodes = {O::V_MOV_B32, O::V_ADD_I32, O::V_SUB_I32,
O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
@ -13192,7 +13318,7 @@ TestCase VectorVop3BSubCoU32UsesRdna2Opcode310() {
TestCase test;
test.name = "VectorVop3BSubCoU32UsesRdna2Opcode310";
test.code = code;
test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 9, 9, 9, 9};
test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 1, 0, 0, 1};
test.opcodes = {O::V_MOV_B32, O::V_CMP_EQ_U32, O::V_CNDMASK_B32,
O::V_SUB_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
@ -13631,9 +13757,12 @@ TestCase VectorPermlane16FetchInactiveZero() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0), EncodeVop2(0x25, 1, InlineU32(10), 1),
EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)),
EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)),
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x25, 1, InlineU32(10), 1),
EncodeVopc(0xc2, InlineU32(0), 0),
EncodeSop1(0x04, 126, 106),
EncodeSMovB32(0, InlineU32(1)),
EncodeSMovB32(1, InlineU32(0)),
};
AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1);
AppendStoreVgpr(&code, 2, 0);
@ -13643,8 +13772,9 @@ TestCase VectorPermlane16FetchInactiveZero() {
test.name = "VectorPermlane16FetchInactiveZero";
test.code = code;
test.expected = {0};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32,
O::V_PERMLANE16_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::S_MOV_B32, O::V_PERMLANE16_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
@ -13657,9 +13787,9 @@ TestCase VectorPermlane16FetchInactiveFi() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, Vgpr(0)), EncodeVop2(0x25, 1, InlineU32(10), 1),
EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)),
EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)),
EncodeVop1(0x01, 1, Vgpr(0)), EncodeVop2(0x25, 1, InlineU32(10), 1),
EncodeVopc(0xc2, InlineU32(0), 0), EncodeSop1(0x04, 126, 106),
EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)),
};
AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1, 0, 1);
AppendStoreVgpr(&code, 2, 0);
@ -13669,8 +13799,9 @@ TestCase VectorPermlane16FetchInactiveFi() {
test.name = "VectorPermlane16FetchInactiveFi";
test.code = code;
test.expected = {11};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32,
O::V_PERMLANE16_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::S_MOV_B32, O::V_PERMLANE16_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
@ -13803,11 +13934,10 @@ TestCase Vop1MoveRelDestination() {
AppendVMovLiteral(&code, 6, 0xbbbbbbbbu);
AppendVMovLiteral(&code, 7, 0xccccccccu);
code.push_back(EncodeSMovB32(124, InlineU32(2)));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeSMovB32(127, InlineU32(0)));
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(EncodeVop1(0x42, 5, Vgpr(1)));
code.push_back(EncodeSMovB32(126, InlineU32(0xf)));
code.push_back(EncodeSMovB32(127, InlineU32(0)));
code.push_back(EncodeSop1(0x04, 126, 193u));
code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
AppendBufferStoreDword(&code, 7, 4);
AppendEnd(&code);
@ -13816,8 +13946,9 @@ TestCase Vop1MoveRelDestination() {
test.name = "Vop1MoveRelDestination";
test.code = code;
test.expected = {0x12345678u, 0xccccccccu, 0xccccccccu, 0xccccccccu};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_MOVRELD_B32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::V_MOVRELD_B32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
@ -14542,7 +14673,6 @@ TestCase VectorVop3CompareGtU64OnGpu() {
{1, 1, 0},
{O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_GT_U64, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.required_spirv = {"OpUGreaterThan"};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
@ -14731,13 +14861,10 @@ TestCase Vop3CndmaskUsesSgprMaskLaneBits() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1),
EncodeVopc(0xc0, Vgpr(0), 0),
EncodeVop1(0x01, 1, 0), EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1), EncodeVopc(0xc2, InlineU32(0), 0),
EncodeSop1(0x04, 4, 106),
};
AppendSMovLiteral(&code, 4, 1);
AppendSMovLiteral(&code, 5, 0);
AppendVop3(&code, 0x101, 2, InlineU32(10), InlineU32(20), 4);
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 1));
AppendBufferStoreDword(&code, 2, 3);
@ -14748,7 +14875,7 @@ TestCase Vop3CndmaskUsesSgprMaskLaneBits() {
test.code = code;
test.expected = {20, 10, 10, 10, 20, 10, 10, 10};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_F_U32, O::S_MOV_B32, O::V_CNDMASK_B32,
O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
@ -14962,7 +15089,7 @@ TestCase SimpleLoop() {
O::S_BRANCH, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BranchVccnzUsesWholeMask() {
TestCase BranchVccnzUsesInvocationMask() {
using O = ShaderOpcode;
std::vector<u32> code = {
@ -14980,9 +15107,9 @@ TestCase BranchVccnzUsesWholeMask() {
AppendEnd(&code);
TestCase test;
test.name = "BranchVccnzUsesWholeMask";
test.name = "BranchVccnzUsesInvocationMask";
test.code = code;
test.expected = std::vector<u32>(8, 42);
test.expected = {42, 11, 11, 11, 42, 11, 11, 11};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_EQ_U32, O::S_CBRANCH_VCCNZ, O::S_BRANCH,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
@ -16647,7 +16774,7 @@ TestCase FlatSubdwordLoadsApplyByteOffset() {
return test;
}
TestCase BranchVccnzUsesCarryProducedWholeMask() {
TestCase BranchVccnzUsesCarryProducedInvocationMask() {
using O = ShaderOpcode;
std::vector<u32> code = {
@ -16665,9 +16792,9 @@ TestCase BranchVccnzUsesCarryProducedWholeMask() {
AppendEnd(&code);
TestCase test;
test.name = "BranchVccnzUsesCarryProducedWholeMask";
test.name = "BranchVccnzUsesCarryProducedInvocationMask";
test.code = code;
test.expected = std::vector<u32>(8, 42);
test.expected = {11, 42, 42, 42, 42, 42, 42, 42};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_F_U32, O::V_ADDC_U32, O::S_CBRANCH_VCCNZ,
O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
@ -18165,7 +18292,8 @@ TestCase ImageD16GatherPacksHalfPairs() {
O::S_ENDPGM};
test.sampled_image_rgba = image;
test.decoded_counts = {{"d16=1", 1}};
test.native_ir_counts = {{"dmask=0x1 data_dwords=2 data_bits=16 component=0/4", 1}};
test.native_ir_counts = {
{"dmask=0x1 data_dwords=2 data_bits=16 component=0/4", 1}};
test.required_spirv = {"OpImageGather", "PackHalf2x16"};
return test;
}
@ -18376,8 +18504,8 @@ TestCase ImageD16StoreUnpacksHalfPairs() {
AppendEnd(&code);
auto expected_image = MakeRgbaImage(4, 4);
SetRgbaPixel(&expected_image, 4, 1, 2, 0x3e800000u, 0x3f000000u,
0x3f400000u, 0x3f800000u);
SetRgbaPixel(&expected_image, 4, 1, 2, 0x3e800000u, 0x3f000000u, 0x3f400000u,
0x3f800000u);
TestCase test;
test.name = "ImageD16StoreUnpacksHalfPairs";
@ -18386,7 +18514,8 @@ TestCase ImageD16StoreUnpacksHalfPairs() {
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.expected_storage_image_rgba = expected_image;
test.decoded_counts = {{"d16=1", 1}};
test.native_ir_counts = {{"dmask=0xf data_dwords=2 data_bits=16 component=0/4", 1}};
test.native_ir_counts = {
{"dmask=0xf data_dwords=2 data_bits=16 component=0/4", 1}};
test.required_spirv = {"UnpackHalf2x16", "OpImageWrite"};
return test;
}
@ -19286,7 +19415,11 @@ std::vector<TestCase> MakeCases() {
AddCase(ScalarBfeI32CapturedRawSignExtends);
AddCase(BitfieldExtractWidthPastEndEdges);
AddCase(Scalar64BitOps);
AddCase(ScalarAndn2B64SccBranch);
AddCase(ScalarAndn2B64SccUsesMaskShadow);
AddCase(ScalarMaskHighWriteInvalidatesProvenance);
AddCase(ScalarSelectB64PreservesMaskProvenance);
AddCase(ScalarWqmB64SelectsSccDomain);
AddCase(ScalarMaskProvenanceOverlapAndMixedBinary);
AddCase(ScalarLiteral);
AddCase(VectorMoves);
AddCase(VectorVop3MoveAppliesFloatSourceModifiers);
@ -19365,8 +19498,8 @@ std::vector<TestCase> MakeCases() {
AddCase(VectorCompareInvertedMaskSelect);
AddCase(BranchSelect);
AddCase(SimpleLoop);
AddCase(BranchVccnzUsesWholeMask);
AddCase(BranchVccnzUsesCarryProducedWholeMask);
AddCase(BranchVccnzUsesInvocationMask);
AddCase(BranchVccnzUsesCarryProducedInvocationMask);
AddCase(ScalarMemoryLoadVariants);
AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset);
AddCase(ScalarLoadAlignsComponentsAndMasksAddress);
@ -21053,8 +21186,8 @@ void CheckBasicStorageTextureDescriptor() {
"PPSA02604 BGRA 2D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), bgra, 0x870000);
const ShaderTextureResource r128{{0x0202e500u, 0xc8200000u, 0x010dc1dfu,
0x91b00facu, 0, 0, 0, 0}};
const ShaderTextureResource r128{
{0x0202e500u, 0xc8200000u, 0x010dc1dfu, 0x91b00facu, 0, 0, 0, 0}};
auto r128_resource = BasicBgraStorageTextureResource();
r128_resource.r128 = true;
Require("BasicStorageTexture", "PPSA01736 R128 descriptor",
@ -22542,7 +22675,8 @@ void CheckPm4DirectShaderRegisterFallback(RenderContext &renderer) {
processor.Process(execution, packet.data(), packet.size()) ==
Pm4ProcessResult::Complete &&
processor.GetShCtx().GetVs().es_regs.data_addr == shader_address,
"direct SET_SH_REG did not route its ES base pair through the register handlers");
"direct SET_SH_REG did not route its ES base pair through the "
"register handlers");
std::printf("[host] %-32s ok\n", "Pm4DirectShaderRegisterFallback");
}
@ -22577,7 +22711,8 @@ void CheckPm4GuardBandRegisterRanges(RenderContext &renderer) {
std::bit_cast<uint32_t>(10.0f),
};
Pm4Execution range_execution;
const auto range_result = processor.Process(range_execution, range.data(), range.size());
const auto range_result =
processor.Process(range_execution, range.data(), range.size());
const auto &range_viewport = processor.GetCtx().GetScreenViewport();
Require("Pm4GuardBandRegisterRanges", "full range",
range_result == Pm4ProcessResult::Complete &&
@ -22644,12 +22779,13 @@ void CheckPm4BlendColorRegisterRanges(RenderContext &renderer) {
std::bit_cast<uint32_t>(14.0f),
};
Pm4Execution full_execution;
const auto full_result = processor.Process(full_execution, full.data(), full.size());
const auto full_result =
processor.Process(full_execution, full.data(), full.size());
const auto &full_color = processor.GetCtx().GetBlendColor();
Require("Pm4BlendColorRegisterRanges", "full range",
full_result == Pm4ProcessResult::Complete && full_color.red == 11.0f &&
full_color.green == 12.0f && full_color.blue == 13.0f &&
full_color.alpha == 14.0f,
full_result == Pm4ProcessResult::Complete &&
full_color.red == 11.0f && full_color.green == 12.0f &&
full_color.blue == 13.0f && full_color.alpha == 14.0f,
"full blend-color range did not preserve native PS5 register order");
std::printf("[host] %-32s ok\n", "Pm4BlendColorRegisterRanges");
}
@ -22727,28 +22863,27 @@ void CheckAgcWaitPackets(RenderContext &renderer) {
nullptr,
0};
const auto address32 = reinterpret_cast<uint64_t>(&label32);
auto *emitted32 = Gen5::AgcDcbWaitRegMem(
reinterpret_cast<Gen5::CommandBuffer *>(&dcb32), 0, 3, 0, 0,
&label32, label32, UINT32_MAX, 400);
Require(
"AgcWaitPackets", "native 32-bit packet",
Gen5::AgcDcbWaitOnAddressGetSize(0) == 56 &&
emitted32 == packet32.data() &&
dcb32.cursor_up == packet32.data() + 14 &&
packet32[0] == 0xc0027904u && packet32[1] == 0x342u &&
packet32[2] ==
(0xc8010000u |
(static_cast<uint32_t>(address32 >> 32u) & 0xffffu)) &&
packet32[3] == static_cast<uint32_t>(address32) &&
packet32[4] == 0xc0053c00u && packet32[5] == 0x13u &&
packet32[6] == (static_cast<uint32_t>(address32) & ~0x3u) &&
packet32[7] ==
(static_cast<uint32_t>(address32 >> 32u) & 0x3ffffu) &&
packet32[8] == label32 && packet32[9] == UINT32_MAX &&
packet32[10] == 25u && packet32[11] == 0xc0017904u &&
packet32[12] == 0x342u && packet32[13] == 0xc8000000u &&
execute(packet32.data(), 14),
"32-bit waitOnAddress stream differs from native AGC");
auto *emitted32 =
Gen5::AgcDcbWaitRegMem(reinterpret_cast<Gen5::CommandBuffer *>(&dcb32), 0,
3, 0, 0, &label32, label32, UINT32_MAX, 400);
Require("AgcWaitPackets", "native 32-bit packet",
Gen5::AgcDcbWaitOnAddressGetSize(0) == 56 &&
emitted32 == packet32.data() &&
dcb32.cursor_up == packet32.data() + 14 &&
packet32[0] == 0xc0027904u && packet32[1] == 0x342u &&
packet32[2] ==
(0xc8010000u |
(static_cast<uint32_t>(address32 >> 32u) & 0xffffu)) &&
packet32[3] == static_cast<uint32_t>(address32) &&
packet32[4] == 0xc0053c00u && packet32[5] == 0x13u &&
packet32[6] == (static_cast<uint32_t>(address32) & ~0x3u) &&
packet32[7] ==
(static_cast<uint32_t>(address32 >> 32u) & 0x3ffffu) &&
packet32[8] == label32 && packet32[9] == UINT32_MAX &&
packet32[10] == 25u && packet32[11] == 0xc0017904u &&
packet32[12] == 0x342u && packet32[13] == 0xc8000000u &&
execute(packet32.data(), 14),
"32-bit waitOnAddress stream differs from native AGC");
alignas(8) uint32_t patched_label32 = 0xaabbccddu;
const auto patched_address32 = reinterpret_cast<uint64_t>(&patched_label32);
@ -22757,7 +22892,8 @@ void CheckAgcWaitPackets(RenderContext &renderer) {
Require(
"AgcWaitPackets", "native 32-bit patch",
Gen5::AgcWaitRegMemPatchAddress(packet32.data(), &patched_label32) == 0 &&
Gen5::AgcWaitRegMemPatchReference(packet32.data(), patched_label32) == 0 &&
Gen5::AgcWaitRegMemPatchReference(packet32.data(), patched_label32) ==
0 &&
packet32[2] ==
(0xc8010000u |
(static_cast<uint32_t>(patched_address32 >> 32u) & 0xffffu)) &&
@ -22780,31 +22916,30 @@ void CheckAgcWaitPackets(RenderContext &renderer) {
nullptr,
0};
const auto address64 = reinterpret_cast<uint64_t>(&label64);
auto *emitted64 = Gen5::AgcDcbWaitRegMem(
reinterpret_cast<Gen5::CommandBuffer *>(&dcb64), 1, 3, 0, 0,
&label64, label64, UINT64_MAX, 400);
Require(
"AgcWaitPackets", "native 64-bit packet",
Gen5::AgcDcbWaitOnAddressGetSize(1) == 64 &&
Gen5::AgcDcbWaitOnAddressGetSize(2) == 0 &&
emitted64 == packet64.data() &&
dcb64.cursor_up == packet64.data() + packet64.size() &&
packet64[0] == 0xc0027904u && packet64[1] == 0x342u &&
packet64[2] ==
(0xc8020000u |
(static_cast<uint32_t>(address64 >> 32u) & 0xffffu)) &&
packet64[3] == static_cast<uint32_t>(address64) &&
packet64[4] == 0xc0079300u && packet64[5] == 0x13u &&
packet64[6] == (static_cast<uint32_t>(address64) & ~0x7u) &&
packet64[7] ==
(static_cast<uint32_t>(address64 >> 32u) & 0x3ffffu) &&
packet64[8] == static_cast<uint32_t>(label64) &&
packet64[9] == static_cast<uint32_t>(label64 >> 32u) &&
packet64[10] == UINT32_MAX && packet64[11] == UINT32_MAX &&
packet64[12] == 25u && packet64[13] == 0xc0017904u &&
packet64[14] == 0x342u && packet64[15] == 0xc8000000u &&
execute(packet64.data(), packet64.size()),
"64-bit waitOnAddress stream differs from native AGC");
auto *emitted64 =
Gen5::AgcDcbWaitRegMem(reinterpret_cast<Gen5::CommandBuffer *>(&dcb64), 1,
3, 0, 0, &label64, label64, UINT64_MAX, 400);
Require("AgcWaitPackets", "native 64-bit packet",
Gen5::AgcDcbWaitOnAddressGetSize(1) == 64 &&
Gen5::AgcDcbWaitOnAddressGetSize(2) == 0 &&
emitted64 == packet64.data() &&
dcb64.cursor_up == packet64.data() + packet64.size() &&
packet64[0] == 0xc0027904u && packet64[1] == 0x342u &&
packet64[2] ==
(0xc8020000u |
(static_cast<uint32_t>(address64 >> 32u) & 0xffffu)) &&
packet64[3] == static_cast<uint32_t>(address64) &&
packet64[4] == 0xc0079300u && packet64[5] == 0x13u &&
packet64[6] == (static_cast<uint32_t>(address64) & ~0x7u) &&
packet64[7] ==
(static_cast<uint32_t>(address64 >> 32u) & 0x3ffffu) &&
packet64[8] == static_cast<uint32_t>(label64) &&
packet64[9] == static_cast<uint32_t>(label64 >> 32u) &&
packet64[10] == UINT32_MAX && packet64[11] == UINT32_MAX &&
packet64[12] == 25u && packet64[13] == 0xc0017904u &&
packet64[14] == 0x342u && packet64[15] == 0xc8000000u &&
execute(packet64.data(), packet64.size()),
"64-bit waitOnAddress stream differs from native AGC");
alignas(8) uint64_t patched_label64 = 0x11223344aabbccddull;
const auto patched_address64 = reinterpret_cast<uint64_t>(&patched_label64);
@ -22826,18 +22961,20 @@ void CheckAgcWaitPackets(RenderContext &renderer) {
0x5a580000u) &&
packet64[8] == 0xaabbccddu && packet64[9] == 0x11223344u &&
execute(packet64.data(), packet64.size()),
"64-bit native wait patch did not preserve the native high reference DWORD");
"64-bit native wait patch did not preserve the native high reference "
"DWORD");
std::array<uint32_t, 16> invalid{};
const uint32_t short_payload = 0x342u;
Require("AgcWaitPackets", "invalid packet",
Gen5::AgcWaitRegMemPatchAddress(invalid.data(), &label32) ==
packet_mismatch &&
Gen5::AgcWaitRegMemPatchReference(invalid.data(), label32) ==
packet_mismatch &&
!Gen5::AgcIsInternalDataPacket(
KYTY_PM4(2, Pm4::IT_SET_UCONFIG_REG, 1), &short_payload),
"native wait patch accepted a packet without the AGC metadata prefix");
Require(
"AgcWaitPackets", "invalid packet",
Gen5::AgcWaitRegMemPatchAddress(invalid.data(), &label32) ==
packet_mismatch &&
Gen5::AgcWaitRegMemPatchReference(invalid.data(), label32) ==
packet_mismatch &&
!Gen5::AgcIsInternalDataPacket(
KYTY_PM4(2, Pm4::IT_SET_UCONFIG_REG, 1), &short_payload),
"native wait patch accepted a packet without the AGC metadata prefix");
std::printf("[host] %-32s ok\n", "AgcWaitPackets");
}
@ -22873,17 +23010,17 @@ void CheckAgcDrawIndirectMultiPacket(RenderContext &renderer) {
0x55667788u, count_address, 0xaabbccddu, modifier);
const uint32_t invalid_payload[]{0x342u, 0xc6000010u};
Require(
"AgcDrawIndirectMulti", "native packet",
emitted == packet.data() && dcb.cursor_up == packet.data() + packet.size() &&
packet == expected && execute(packet.data(), 3u) &&
execute(packet.data() + 13u, 3u) &&
Gen5::AgcWaitRegMemPatchAddress(packet.data(), count_address) ==
packet_mismatch &&
Gen5::AgcWaitRegMemPatchReference(packet.data(), 0x12345678u) ==
packet_mismatch &&
!Gen5::AgcIsInternalDataPacket(0xc0017904u, invalid_payload),
"drawIndirectMulti stream differs from native AGC");
Require("AgcDrawIndirectMulti", "native packet",
emitted == packet.data() &&
dcb.cursor_up == packet.data() + packet.size() &&
packet == expected && execute(packet.data(), 3u) &&
execute(packet.data() + 13u, 3u) &&
Gen5::AgcWaitRegMemPatchAddress(packet.data(), count_address) ==
packet_mismatch &&
Gen5::AgcWaitRegMemPatchReference(packet.data(), 0x12345678u) ==
packet_mismatch &&
!Gen5::AgcIsInternalDataPacket(0xc0017904u, invalid_payload),
"drawIndirectMulti stream differs from native AGC");
std::printf("[host] %-32s ok\n", "AgcDrawIndirectMulti");
}

443
tests/shaderCfgTests.cpp
View File

@ -8,7 +8,6 @@
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "graphics/host_gpu/renderer/image/textureCommon.h"
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/shader/recompiler/ShaderRecompiler.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
@ -988,8 +987,7 @@ void TestSpirvRequirementsAnalysis() {
&analysis_error),
"initial SPIR-V requirements analysis failed");
const auto requirements = *program.spirv_requirements;
Check(requirements.requires_exact_subgroup && requirements.subgroup_ballot &&
requirements.subgroup_shuffle &&
Check(requirements.subgroup_ballot && requirements.subgroup_shuffle &&
requirements.subgroup_local_invocation_id &&
requirements.compute_derivatives &&
requirements.image_gather_extended && requirements.function_lds &&
@ -1044,8 +1042,7 @@ void TestSpirvRequirementsAnalysis() {
&analysis_error),
"ADD_TID SPIR-V requirements analysis failed");
const auto add_tid_requirements = *add_tid.spirv_requirements;
Check(add_tid_requirements.requires_exact_subgroup &&
add_tid_requirements.subgroup_local_invocation_id &&
Check(add_tid_requirements.subgroup_local_invocation_id &&
!add_tid_requirements.subgroup_ballot &&
!add_tid_requirements.subgroup_shuffle,
"buffer ADD_TID requested the wrong subgroup contract");
@ -1071,8 +1068,7 @@ void TestSpirvRequirementsAnalysis() {
&analysis_error),
"empty SPIR-V requirements analysis failed");
const auto empty_requirements = *empty.spirv_requirements;
Check(!empty_requirements.requires_exact_subgroup &&
!empty_requirements.subgroup_ballot &&
Check(!empty_requirements.subgroup_ballot &&
!empty_requirements.subgroup_shuffle &&
!empty_requirements.subgroup_local_invocation_id &&
!empty_requirements.compute_derivatives &&
@ -1082,129 +1078,6 @@ void TestSpirvRequirementsAnalysis() {
"empty IR unexpectedly requested SPIR-V features");
}
void TestNativeSubgroupPolicy() {
const auto SetTypedValues = [](ShaderRecompiler::IR::Program &program,
ShaderRecompiler::IR::ValueOpcode opcode =
ShaderRecompiler::IR::ValueOpcode::Void) {
using namespace ShaderRecompiler::IR;
program.values = std::make_shared<ValueProgram>();
program.values->block_storage.push_back(std::make_unique<Block>());
auto *block = program.values->block_storage.back().get();
program.values->blocks.push_back(block);
if (opcode == ValueOpcode::ReadLane) {
block->AppendNewInst(opcode, {Value(0u), Value(0u)});
} else if (opcode == ValueOpcode::Ballot) {
block->AppendNewInst(opcode, {Value(true)});
} else if (opcode == ValueOpcode::DataAppend) {
program.values->memory_info.push_back({.kind = ResourceKind::Lds});
auto &append = block->AppendNewInst(
opcode, {Value(0u), Value(true), Value(1u), Value(0u)});
append.SetFlags(MemoryFlags{.index = 0});
}
std::string error;
Check(ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program, &error),
"native subgroup requirements analysis failed");
};
GraphicContext context;
context.subgroup_size = 32;
context.min_subgroup_size = 32;
context.max_subgroup_size = 32;
context.subgroup_size_control_enabled = true;
context.required_subgroup_size_stages = vk::ShaderStageFlagBits::eAll;
ShaderRecompiler::IR::Program safe;
safe.wave_size = 32;
safe.lane_mask_mode = ShaderLaneMaskMode::NativeWave;
SetTypedValues(safe);
Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context},
vk::ShaderStageFlagBits::eVertex, safe)
.mode == ShaderSubgroupMode::Natural,
"native wave32 policy changed");
safe.wave_size = 64;
Check(SelectGraphicsLaneMaskMode(safe.wave_size) ==
ShaderLaneMaskMode::PerInvocation &&
ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context},
vk::ShaderStageFlagBits::eVertex, safe)
.mode == ShaderSubgroupMode::Unsupported,
"wave64 graphics mismatch accepted native-wave mask lowering");
safe.lane_mask_mode = ShaderLaneMaskMode::PerInvocation;
Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context},
vk::ShaderStageFlagBits::eVertex, safe)
.mode == ShaderSubgroupMode::PerInvocationGraphics,
"wave64 graphics mismatch did not select per-invocation masks");
ShaderRecompiler::IR::Program cross_lane = safe;
SetTypedValues(cross_lane, ShaderRecompiler::IR::ValueOpcode::ReadLane);
Check(cross_lane.spirv_requirements->requires_exact_subgroup &&
ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context},
vk::ShaderStageFlagBits::eVertex,
cross_lane)
.mode == ShaderSubgroupMode::PerInvocationGraphics,
"graphics mismatch did not select per-invocation masks");
auto cross_lane_compute = cross_lane;
cross_lane_compute.lane_mask_mode = ShaderLaneMaskMode::NativeWave;
Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context},
vk::ShaderStageFlagBits::eCompute,
cross_lane_compute)
.mode == ShaderSubgroupMode::FlattenedMasks,
"split wave64 compute did not use native subgroup operations");
ShaderRecompiler::IR::Program zero_exec = safe;
zero_exec.lane_mask_mode = ShaderLaneMaskMode::NativeWave;
SetTypedValues(zero_exec);
Check(!zero_exec.spirv_requirements->requires_exact_subgroup &&
ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context},
vk::ShaderStageFlagBits::eCompute,
zero_exec)
.mode == ShaderSubgroupMode::FlattenedMasks,
"compile-time uniform-zero EXEC write did not stay on the mask-free "
"path");
ShaderRecompiler::IR::Program selective_exec = safe;
SetTypedValues(selective_exec, ShaderRecompiler::IR::ValueOpcode::Ballot);
Check(selective_exec.spirv_requirements->requires_exact_subgroup,
"selective EXEC write was classified mask-free");
ShaderRecompiler::IR::Program carry = safe;
SetTypedValues(carry, ShaderRecompiler::IR::ValueOpcode::Ballot);
Check(carry.spirv_requirements->requires_exact_subgroup,
"lane-varying VCC carry producer was classified mask-free");
ShaderRecompiler::IR::Program vcc_branch = safe;
SetTypedValues(vcc_branch, ShaderRecompiler::IR::ValueOpcode::Ballot);
Check(vcc_branch.spirv_requirements->requires_exact_subgroup,
"whole-wave VCC branch was classified mask-free");
ShaderRecompiler::IR::Program ds_partial = safe;
ds_partial.lane_mask_mode = ShaderLaneMaskMode::NativeWave;
SetTypedValues(ds_partial, ShaderRecompiler::IR::ValueOpcode::DataAppend);
Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context},
vk::ShaderStageFlagBits::eCompute, ds_partial)
.mode == ShaderSubgroupMode::FlattenedMasks,
"split wave64 DS append did not use native subgroup operations");
context.max_subgroup_size = 64;
const auto controlled = ConfigureShaderSubgroup(
ShaderSubgroupCapabilities{context}, vk::ShaderStageFlagBits::eCompute,
cross_lane_compute);
Check(controlled.mode == ShaderSubgroupMode::Controlled &&
controlled.required_size == 64,
"supported controlled wave64 was not preferred over splitting");
context.subgroup_size = 64;
context.subgroup_size_control_enabled = false;
cross_lane.wave_size = 32;
cross_lane.lane_mask_mode = ShaderLaneMaskMode::PerInvocation;
Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context},
vk::ShaderStageFlagBits::eFragment, cross_lane)
.mode == ShaderSubgroupMode::Unsupported,
"inverse graphics mismatch was accepted as one guest wave");
cross_lane_compute.wave_size = 32;
Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context},
vk::ShaderStageFlagBits::eCompute,
cross_lane_compute)
.mode == ShaderSubgroupMode::FlattenedMasks,
"inverse compute mismatch did not use the native subgroup");
}
std::array<uint32_t, 64>
ImageTestUserData(Prospero::ImageType type = Prospero::ImageType::kColor2D) {
std::array<uint32_t, 64> data{};
@ -5567,8 +5440,7 @@ void TestGraphicsCreateInterpolantMapping() {
"flat/custom interpolant mapping bits were unexpected");
Check(regs[3].value == 0x01580304u,
"f16 interpolant mapping bits were unexpected");
Check(regs[6].offset == Pm4::SPI_PS_INPUT_CNTL_0 + 6u &&
regs[6].value == 6u,
Check(regs[6].offset == Pm4::SPI_PS_INPUT_CNTL_0 + 6u && regs[6].value == 6u,
"interpolant identity tail was not filled");
Check(Gen5::AgcCreateInterpolantMapping2(native_regs, &gs, &ps) == 0,
@ -7502,7 +7374,8 @@ void TestNewShaderRecompilerCfgMultipleLoopLatches() {
"multiple-latch fixture lacks two native backedges");
Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
Check(graph.blocks.size() == original_block_count + 2u,
"multiple native latches did not create one synthetic continue and one empty header");
"multiple native latches did not create one synthetic continue and one "
"empty header");
Check(graph.back_edges.size() == 1u && graph.natural_loops.size() == 1u,
"multiple native latches were not coalesced to one SPIR-V backedge");
const auto &loop = graph.natural_loops.front();
@ -7700,23 +7573,22 @@ void TestNewShaderRecompilerCfgExecSccSharedArm() {
}
}
for (const auto lane_mode :
{ShaderLaneMaskMode::NativeWave, ShaderLaneMaskMode::PerInvocation}) {
auto options = MakeCompileOptions(ShaderType::Compute);
options.lane_mask_mode = lane_mode;
options.dump_ir = true;
ShaderRecompiler::CompileResult result;
Check(ShaderRecompiler::TryRecompile(shader, options, result, &error),
error.c_str());
Check(!result.program.dispatcher_fallback &&
Common::ContainsStr(result.ir_dump, "mode=structured"),
"EXEC/SCC shared-arm epilogue did not stay structured");
Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 3u,
"EXEC/SCC shared-arm SPIR-V has the wrong selection-merge count");
Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u,
"EXEC/SCC shared-arm SPIR-V unexpectedly used dispatcher OpSwitch");
CheckSpirvBinaryValidates(result.spirv);
}
auto options = MakeCompileOptions(ShaderType::Compute);
options.dump_ir = true;
ShaderRecompiler::CompileResult result;
Check(ShaderRecompiler::TryRecompile(shader, options, result, &error),
error.c_str());
Check(!result.program.dispatcher_fallback &&
Common::ContainsStr(result.ir_dump, "mode=structured"),
"EXEC/SCC shared-arm epilogue did not stay structured");
Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 3u,
"EXEC/SCC shared-arm SPIR-V has the wrong selection-merge count");
Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u,
"EXEC/SCC shared-arm SPIR-V unexpectedly used dispatcher OpSwitch");
Check(!Common::ContainsStr(DisassembleSpirvBinary(result.spirv),
"OpGroupNonUniformBallot"),
"per-invocation EXEC/SCC branch reconstructed a native subgroup mask");
CheckSpirvBinaryValidates(result.spirv);
}
void TestNewShaderRecompilerCfgNestedTailEarlyExit() {
@ -7755,13 +7627,16 @@ void TestNewShaderRecompilerCfgNestedTailEarlyExit() {
Check(CfgInstructionCoverage(graph, decoded.instructions.size()) ==
original_coverage,
"nested-tail routing changed semantic instruction coverage");
Check(std::ranges::count_if(graph.blocks, [](const auto &block) {
return block.terminator.condition ==
ShaderRecompiler::CFG::BranchCondition::GotoVariable;
}) == 1u &&
std::ranges::count_if(graph.blocks, [](const auto &block) {
return block.terminator.goto_value >= 0;
}) == 3u,
Check(std::ranges::count_if(
graph.blocks,
[](const auto &block) {
return block.terminator.condition ==
ShaderRecompiler::CFG::BranchCondition::GotoVariable;
}) == 1u &&
std::ranges::count_if(graph.blocks,
[](const auto &block) {
return block.terminator.goto_value >= 0;
}) == 3u,
"nested-tail early exit did not use typed route state");
auto options = MakeCompileOptions(ShaderType::Pixel);
@ -8000,9 +7875,10 @@ void TestNewShaderRecompilerCfgSharedTerminalEarlyExit() {
const auto original_coverage =
CfgInstructionCoverage(graph, decoded.instructions.size());
Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
Check(CfgInstructionCoverage(graph, decoded.instructions.size()) ==
original_coverage,
"shared-terminal structurization changed semantic instruction coverage");
Check(
CfgInstructionCoverage(graph, decoded.instructions.size()) ==
original_coverage,
"shared-terminal structurization changed semantic instruction coverage");
auto options = MakeCompileOptions(ShaderType::Pixel);
options.dump_ir = true;
@ -8171,7 +8047,6 @@ void TestNewShaderRecompilerU64PairLowering() {
ir.Emit(IR::ValueOpcode::ULessThan64, {shifted, base});
ir.Emit(IR::ValueOpcode::SLessThan64, {shifted, base});
ir.Emit(IR::ValueOpcode::BitCount64, {shifted});
ir.Emit(IR::ValueOpcode::WqmU64, {shifted});
for (const uint32_t count : {1u, 31u, 32u, 33u, 63u}) {
ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {base, IR::Value(count)});
ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {base, IR::Value(count)});
@ -8710,10 +8585,9 @@ void TestDemandDrivenSpirvDeclarations() {
void TestTypedEntryStateIsMinimal() {
using namespace ShaderRecompiler;
const auto translate = [](ShaderLaneMaskMode lane_mode, uint32_t wave_size) {
const auto check = [](uint32_t wave_size) {
IR::Program source;
source.stage = ShaderType::Compute;
source.lane_mask_mode = lane_mode;
source.wave_size = wave_size;
source.user_data_count = 0;
source.blocks.emplace_back();
@ -8726,11 +8600,7 @@ void TestTypedEntryStateIsMinimal() {
Check(Frontend::TranslateProgram(source, values, nullptr, nullptr, &compute,
&error),
error.c_str());
return values;
};
const auto check = [&](ShaderLaneMaskMode lane_mode, uint32_t wave_size) {
auto values = translate(lane_mode, wave_size);
uint32_t set_exec = 0;
uint32_t set_exec_lo = 0;
uint32_t set_exec_hi = 0;
@ -8738,13 +8608,11 @@ void TestTypedEntryStateIsMinimal() {
IR::Value exec;
IR::Value exec_lo;
IR::Value exec_hi;
const IR::Inst *ballot = nullptr;
for (const auto *block : values.blocks) {
for (const auto &inst : *block) {
switch (inst.GetOpcode()) {
case IR::ValueOpcode::Ballot:
ballots++;
ballot = &inst;
break;
case IR::ValueOpcode::SetExec:
set_exec++;
@ -8760,6 +8628,7 @@ void TestTypedEntryStateIsMinimal() {
break;
case IR::ValueOpcode::SetScalarRegister:
case IR::ValueOpcode::SetThreadBitScalarRegister:
case IR::ValueOpcode::SetScalarMaskTag:
case IR::ValueOpcode::SetVectorRegister:
case IR::ValueOpcode::SetScc:
case IR::ValueOpcode::SetVcc:
@ -8774,44 +8643,19 @@ void TestTypedEntryStateIsMinimal() {
}
}
Check(set_exec == 1u && set_exec_lo == 1u && set_exec_hi == 1u &&
exec.IsImmediate() && exec.U1(),
"typed entry does not define EXEC exactly once");
if (lane_mode == ShaderLaneMaskMode::PerInvocation) {
Check(ballots == 0u && exec_lo.IsImmediate() && exec_lo.U32() == 1u &&
exec_hi.IsImmediate() && exec_hi.U32() == 0u,
"per-invocation entry EXEC is not the local {1,0} mask");
} else {
Check(ballots == 1u && !exec_lo.IsImmediate(),
"native compute entry EXEC does not use the active subgroup mask");
const auto *lo = exec_lo.ResolveInstruction();
Check(lo != nullptr &&
lo->GetOpcode() == IR::ValueOpcode::CompositeExtractU32x4 &&
lo->Arg(0).ResolveInstruction() == ballot &&
lo->Arg(1).Resolve().U32() == 0u,
"native compute EXEC_LO does not use ballot word zero");
if (wave_size == 32u) {
Check(exec_hi.IsImmediate() && exec_hi.U32() == 0u,
"native wave32 entry defined EXEC_HI");
} else {
const auto *hi = exec_hi.ResolveInstruction();
Check(hi != nullptr &&
hi->GetOpcode() == IR::ValueOpcode::CompositeExtractU32x4 &&
hi->Arg(0).ResolveInstruction() == ballot &&
hi->Arg(1).Resolve().U32() == 1u,
"native wave64 EXEC_HI does not use ballot word one");
}
}
ballots == 0u && exec.IsImmediate() && exec.U1() &&
exec_lo.IsImmediate() && exec_lo.U32() == 1u &&
exec_hi.IsImmediate() && exec_hi.U32() == 0u,
"typed entry is not the local {1,0} invocation mask");
IR::RewriteToSsa(values.blocks);
IR::RemoveIdentities(values.blocks);
IR::EliminateDeadCode(values.blocks);
std::string error;
Check(IR::ValidateValueProgram(values, true, &error), error.c_str());
};
check(ShaderLaneMaskMode::NativeWave, 32u);
check(ShaderLaneMaskMode::NativeWave, 64u);
check(ShaderLaneMaskMode::PerInvocation, 64u);
check(32u);
check(64u);
}
void TestFinalSsaRejectsRegisterStatePseudos() {
@ -9145,36 +8989,9 @@ void TestValuePhiValidation() {
}
}
void TestWqmConstantPropagation() {
void TestWqmMaskSignatureAndU64ShiftConstantPropagation() {
using namespace ShaderRecompiler::IR;
ValueProgram program;
program.block_storage.push_back(std::make_unique<Block>());
program.blocks.push_back(program.block_storage.back().get());
program.block_info.emplace_back();
IREmitter ir(program.blocks.front());
constexpr std::array cases = {
std::pair{uint64_t{0}, uint64_t{0}},
std::pair{uint64_t{1}, uint64_t{0xfull}},
std::pair{uint64_t{1} << 31u, uint64_t{0xf} << 28u},
std::pair{uint64_t{1} << 32u, uint64_t{0xf} << 32u},
std::pair{uint64_t{0x8000001180000001ull},
uint64_t{0xf00000fff000000full}},
};
std::array<Value, cases.size()> results;
for (size_t index = 0; index < cases.size(); index++) {
results[index] = ir.Emit(ValueOpcode::WqmU64, {Value(cases[index].first)});
}
ConstantPropagationPass(program.blocks);
for (size_t index = 0; index < cases.size(); index++) {
const auto value = results[index].Resolve();
Check(value.IsImmediate() && value.GetType() == Type::U64 &&
value.U64() == cases[index].second,
"WqmU64 constant propagation crossed a quad or dword boundary");
}
const auto check_invalid_signature = [](ValueOpcode opcode, Value valid,
Value invalid) {
ValueProgram malformed;
@ -9189,7 +9006,6 @@ void TestWqmConstantPropagation() {
Common::ContainsStr(error, "argument 0 has type"),
"value IR accepted a WQM operand that violates opcode metadata");
};
check_invalid_signature(ValueOpcode::WqmU64, Value(uint64_t{1}), Value(true));
check_invalid_signature(ValueOpcode::WqmMask, Value(true),
Value(uint64_t{1}));
@ -9624,7 +9440,7 @@ void TestNewShaderRecompilerVertexSystemInputsWithoutMirrors() {
"vertex system values were routed through guest VGPR mirrors");
}
void TestNewShaderRecompilerVertexExportUsesLaneExecMask() {
void TestNewShaderRecompilerVertexExportUsesInvocationExecMask() {
const uint32_t shader[] = {
EncodeSop1(0x04, 126, 129), // s_mov_b64 exec, 1
EncodeExp0(0x0c, 0xf),
@ -9640,27 +9456,11 @@ void TestNewShaderRecompilerVertexExportUsesLaneExecMask() {
Check(ShaderRecompiler::TryRecompile(shader, options, result, &error),
error.c_str());
CheckSpirvBinaryValidates(result.spirv);
const auto source = DisassembleSpirvBinary(result.spirv);
Check(Common::ContainsStr(source, "gl_SubgroupInvocationID"),
"vertex export EXEC guard did not test the current lane");
Check(!Common::ContainsStr(source, "OpLoad %uint %gl_SubgroupInvocationID"),
"vertex EXEC guard depends on the native subgroup lane");
Check(Common::ContainsStr(source, "OpBranchConditional"),
"vertex export EXEC guard did not branch on the lane-active mask");
options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation;
ShaderRecompiler::CompileResult local_result;
Check(ShaderRecompiler::TryRecompile(shader, options, local_result, &error),
error.c_str());
Check(local_result.program.lane_mask_mode ==
ShaderLaneMaskMode::PerInvocation,
"per-invocation mask mode was not preserved in the immutable program");
CheckSpirvBinaryValidates(local_result.spirv);
const auto local_source = DisassembleSpirvBinary(local_result.spirv);
Check(!Common::ContainsStr(local_source,
"OpLoad %uint %gl_SubgroupInvocationID"),
"per-invocation EXEC still depends on the native subgroup lane");
Check(Common::ContainsStr(local_source, "OpBranchConditional"),
"per-invocation vertex export lost its EXEC guard");
"vertex export lost its per-invocation EXEC guard");
}
void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
@ -9677,7 +9477,6 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
};
auto options = MakeCompileOptions(ShaderType::Vertex);
options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation;
options.dump_ir = true;
ShaderRecompiler::CompileResult result;
@ -9720,45 +9519,6 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 1u,
"wave32 WQM retained the unused high ballot-word expansion");
const uint32_t native_wqm_shader[] = {
EncodeSop1(0x0a, 2, 0), // s_wqm_b64 s[2:3], s[0:1]
EncodeVop1(0x01, 0, 2), // v_mov_b32 v0, s2
EncodeExp0(0x0c, 0x1), EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01),
};
auto native_options = options;
native_options.lane_mask_mode = ShaderLaneMaskMode::NativeWave;
Check(ShaderRecompiler::TryRecompile(native_wqm_shader, native_options,
result, &error),
error.c_str());
CheckSpirvBinaryValidates(result.spirv);
Check(Common::ContainsStr(result.ir_dump, "WqmU64") &&
!Common::ContainsStr(DisassembleSpirvBinary(result.spirv),
"OpGroupNonUniformBallot"),
"native scalar WQM retained frontend expansion or subgroup machinery");
Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 1u &&
SpirvInstructionOpcodeCount(result.spirv, 194u) == 2u,
"native scalar WQM emitter was folded away or did not use the compact "
"path");
const uint32_t native_special_wqm_shader[] = {
EncodeSop1(0x04, 126, 0), // s_mov_b64 exec, s[0:1]
EncodeSop1(0x0a, 126, 126), // s_wqm_b64 exec, exec
EncodeSop1(0x04, 106, 0), // s_mov_b64 vcc, s[0:1]
EncodeSop1(0x0a, 106, 106), // s_wqm_b64 vcc, vcc
EncodeVop1(0x01, 0, 126), // v_mov_b32 v0, exec_lo
EncodeVop1(0x01, 1, 106), // v_mov_b32 v1, vcc_lo
EncodeExp0(0x0c, 0x3), EncodeExp1(0, 1, 0, 0), EncodeSopp(0x01),
};
Check(ShaderRecompiler::TryRecompile(native_special_wqm_shader,
native_options, result, &error),
error.c_str());
CheckSpirvBinaryValidates(result.spirv);
const auto native_special_source = DisassembleSpirvBinary(result.spirv);
Check(CountSourceOccurrences(result.ir_dump, "WqmU64") == 2u &&
!Common::ContainsStr(native_special_source,
"OpGroupNonUniformBallot"),
"native EXEC/VCC WQM retained destination-dependent ballot lowering");
const uint32_t cross_lane_shader[] = {
EncodeSop2(0x25, 126, 132, 128), // s_bfm_b64 exec, 4, 0
EncodeVop1(0x01, 0, 250),
@ -9789,7 +9549,6 @@ void TestNewShaderRecompilerPerInvocationU64Complement() {
};
auto options = MakeCompileOptions(ShaderType::Vertex);
options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation;
options.dump_ir = true;
ShaderRecompiler::CompileResult result;
@ -9919,7 +9678,8 @@ void TestNewShaderRecompilerExpPixelOutputs() {
Check(ShaderRecompiler::TryRecompile(compressed_ba_shader, options,
unorm16_ba_result, &error),
error.c_str());
const auto unorm16_ba_source = DisassembleSpirvBinary(unorm16_ba_result.spirv);
const auto unorm16_ba_source =
DisassembleSpirvBinary(unorm16_ba_result.spirv);
Check(CountSourceOccurrences(unorm16_ba_source, "UnpackUnorm2x16") == 1u,
"compressed UNORM16 BA-only export did not unpack VSRC1");
CheckSpirvBinaryValidates(unorm16_ba_result.spirv);
@ -10114,14 +9874,14 @@ void TestRenderTargetReverseExportMapping() {
mappings[0] = gr32.export_mapping;
ShaderPixelInputInfo compiled_info{};
std::span<const uint32_t> compiled_spirv;
Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave, vs_info,
mappings, compiled_info, compiled_spirv) &&
Check(ShaderCompileInfoPS(regs, sh, vs_info, mappings, compiled_info,
compiled_spirv) &&
compiled_info.target_export_mapping[0].IsIdentity(),
"inactive reverse MRT mapping was not normalized out of the shader "
"cache key");
sh.target_output_mode[0] = 4;
Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave, vs_info,
mappings, compiled_info, compiled_spirv) &&
Check(ShaderCompileInfoPS(regs, sh, vs_info, mappings, compiled_info,
compiled_spirv) &&
compiled_info.target_export_mapping[0] == gr32.export_mapping,
"active reverse MRT mapping was lost before shader specialization");
}
@ -11098,8 +10858,7 @@ void TestNewShaderRecompilerPixelPipelineEntry() {
HW::ShaderRegisters sh{};
ShaderPixelInputInfo input_info{};
std::vector<uint32_t> spirv;
Check(ShaderCompileSpirvPS(regs, sh, ShaderLaneMaskMode::NativeWave,
input_info, spirv),
Check(ShaderCompileSpirvPS(regs, sh, input_info, spirv),
"new pixel shader recompiler wrapper did not produce SPIR-V");
Check(!spirv.empty(),
"new pixel shader recompiler wrapper returned empty SPIR-V");
@ -11132,8 +10891,7 @@ void TestNewShaderRecompilerPixelPipelineEntry() {
ShaderPixelInputInfo vcc_input{};
std::vector<uint32_t> vcc_spirv;
Check(ShaderCompileSpirvPS(vcc_regs, sh, ShaderLaneMaskMode::NativeWave,
vcc_input, vcc_spirv),
Check(ShaderCompileSpirvPS(vcc_regs, sh, vcc_input, vcc_spirv),
"VCC raw-load pointer was lost");
CheckSpirvBinaryValidates(vcc_spirv);
}
@ -11226,7 +10984,6 @@ void TestComputeLdsAllocationIdentity() {
void TestPixelProgramCacheDescriptorSetIdentity() {
const uint32_t shader_01[] = {0xbf810000u};
const uint32_t shader_10[] = {0xbf810000u};
const uint32_t shader_mask[] = {0xbf810000u};
HW::ShaderRegisters sh{};
auto check_transition = [&](const uint32_t *shader, uint64_t checksum,
@ -11249,8 +11006,8 @@ void TestPixelProgramCacheDescriptorSetIdentity() {
const std::array<Prospero::ColorComponentMapping, 8> identity_mappings{};
ShaderPixelInputInfo ps_info{};
std::span<const uint32_t> spirv;
Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave,
vs_info, identity_mappings, ps_info, spirv),
Check(ShaderCompileInfoPS(regs, sh, vs_info, identity_mappings, ps_info,
spirv),
"pixel program-cache transition failed to compile");
const auto expected_set = has_vs_descriptors ? 1u : 0u;
Check(ps_info.descriptor_set == expected_set &&
@ -11266,28 +11023,6 @@ void TestPixelProgramCacheDescriptorSetIdentity() {
check_transition(shader_01, 0x91a27e6300000001ull, false, true);
check_transition(shader_10, 0x91a27e6300000002ull, true, false);
HW::PixelShaderInfo mask_regs{};
mask_regs.ps_regs.data_addr = reinterpret_cast<uint64_t>(shader_mask);
mask_regs.ps_regs.chksum = 0x91a27e6300000003ull;
ShaderMappedData mapped{};
mapped.code_size_bytes = sizeof(shader_mask);
ShaderMapUserData(mask_regs.ps_regs.data_addr, mapped);
auto compile_mask_mode = [&](ShaderLaneMaskMode mode) {
ShaderVertexInputInfo vs_info{};
vs_info.stage.program = std::make_shared<ShaderRecompiler::IR::Program>();
const std::array<Prospero::ColorComponentMapping, 8> identity_mappings{};
ShaderPixelInputInfo ps_info{};
std::span<const uint32_t> spirv;
Check(ShaderCompileInfoPS(mask_regs, sh, mode, vs_info, identity_mappings,
ps_info, spirv),
"pixel lane-mask cache transition failed to compile");
Check(ps_info.stage.program != nullptr &&
ps_info.stage.program->lane_mask_mode == mode,
"pixel program cache reused a different lane-mask lowering");
};
compile_mask_mode(ShaderLaneMaskMode::NativeWave);
compile_mask_mode(ShaderLaneMaskMode::PerInvocation);
}
void TestNewShaderRecompilerUnsupportedMemoryDecode() {
@ -11379,23 +11114,21 @@ void TestNewShaderRecompilerFlatAddressProvenanceBoundaries() {
}
void TestNewShaderRecompilerSpirvSizeBaselines() {
const auto compile =
[](const char *name, std::span<const uint32_t> shader,
const SpirvMetrics &budget, ShaderType stage = ShaderType::Compute,
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave) {
auto options = MakeCompileOptions(stage);
options.dump_ir = true;
options.lane_mask_mode = lane_mask_mode;
const auto compile = [](const char *name, std::span<const uint32_t> shader,
const SpirvMetrics &budget,
ShaderType stage = ShaderType::Compute) {
auto options = MakeCompileOptions(stage);
options.dump_ir = true;
ShaderRecompiler::CompileResult result;
std::string error;
const bool compiled =
ShaderRecompiler::TryRecompile(shader, options, result, &error);
Check(compiled, error.c_str());
CheckSpirvBinaryValidates(result.spirv);
CheckSpirvBudget(name, result.spirv, budget);
return result;
};
ShaderRecompiler::CompileResult result;
std::string error;
const bool compiled =
ShaderRecompiler::TryRecompile(shader, options, result, &error);
Check(compiled, error.c_str());
CheckSpirvBinaryValidates(result.spirv);
CheckSpirvBudget(name, result.spirv, budget);
return result;
};
const uint32_t empty[] = {EncodeSopp(0x01)};
const auto empty_result =
@ -11595,19 +11328,18 @@ void TestNewShaderRecompilerSpirvSizeBaselines() {
EncodeExp1(0, 0, 0, 0), // POS0.x
EncodeSopp(0x01),
};
const auto wqm_result =
compile("wqm", wqm,
{.words = 407,
.instructions = 98,
.variables = 4,
.loads = 3,
.stores = 1,
.labels = 6,
.selection_merges = 1,
.branches = 4,
.conditional_branches = 1,
.ballots = 1},
ShaderType::Vertex, ShaderLaneMaskMode::PerInvocation);
const auto wqm_result = compile("wqm", wqm,
{.words = 407,
.instructions = 98,
.variables = 4,
.loads = 3,
.stores = 1,
.labels = 6,
.selection_merges = 1,
.branches = 4,
.conditional_branches = 1,
.ballots = 1},
ShaderType::Vertex);
Check(Common::ContainsStr(wqm_result.ir_dump, "WqmMask"),
"WQM size fixture no longer reaches per-invocation WqmMask IR");
@ -11653,7 +11385,6 @@ int main() {
TestSpirvRequirementsAnalysis();
TestNewShaderRecompilerSpirvSizeBaselines();
TestDemandDrivenSpirvDeclarations();
TestNativeSubgroupPolicy();
TestNewShaderRecompilerSMovB32();
TestNewShaderRecompilerNativeWideScalarMemoryIr();
TestNewShaderRecompilerNativeWideBufferIr();
@ -11720,13 +11451,13 @@ int main() {
TestNewShaderRecompilerPrunesUnreachableSetpcMetadata();
TestNewShaderRecompilerSetpcDwordJumpTable();
TestTypedEntryStateIsMinimal();
TestWqmConstantPropagation();
TestWqmMaskSignatureAndU64ShiftConstantPropagation();
TestFinalSsaRejectsRegisterStatePseudos();
TestValuePhiValidation();
TestNativeWideValueValidation();
TestNewShaderRecompilerZeroInitialRegisterState();
TestNewShaderRecompilerVertexSystemInputsWithoutMirrors();
TestNewShaderRecompilerVertexExportUsesLaneExecMask();
TestNewShaderRecompilerVertexExportUsesInvocationExecMask();
TestNewShaderRecompilerPerInvocationMasksWithoutMirrors();
TestNewShaderRecompilerPerInvocationU64Complement();
TestNewShaderRecompilerExpPixelOutputs();