From 49ea1680f8ee3ae331b0f29b9c418e3718d375d9 Mon Sep 17 00:00:00 2001 From: nmzik Date: Fri, 21 Aug 2026 08:02:21 +0200 Subject: [PATCH] graphics: use per-invocation shader masks --- .../renderer/pipeline/shaderSubgroup.cpp | 58 -- .../renderer/pipeline/shaderSubgroup.h | 40 -- .../host_gpu/renderer/pipeline/shaders.cpp | 70 +-- .../host_gpu/renderer/renderCompute.cpp | 1 - src/graphics/host_gpu/renderer/renderDraw.cpp | 7 +- .../presentation/window/vulkanWindow.cpp | 25 +- .../shader/recompiler/ShaderRecompiler.cpp | 1 - .../shader/recompiler/ShaderRecompiler.h | 1 - .../recompiler/backend/spirv/SpirvEmitter.cpp | 23 +- .../backend/spirv/spirvEmitterFlow.cpp | 23 +- .../backend/spirv/spirvEmitterInternal.h | 107 ++-- .../backend/spirv/spirvEmitterMemory.cpp | 6 +- .../recompiler/frontend/translate/Control.cpp | 106 ++-- .../frontend/translate/Dispatch.cpp | 75 ++- .../recompiler/frontend/translate/Integer.cpp | 111 +--- .../frontend/translate/Translate.cpp | 160 ++--- .../frontend/translate/Translator.h | 71 ++- src/graphics/shader/recompiler/ir/Block.h | 1 + .../shader/recompiler/ir/IREmitter.cpp | 8 + src/graphics/shader/recompiler/ir/IREmitter.h | 2 + .../shader/recompiler/ir/ShaderIR.cpp | 39 +- src/graphics/shader/recompiler/ir/ShaderIR.h | 2 - .../shader/recompiler/ir/ShaderIRLog.cpp | 4 +- .../shader/recompiler/ir/ValueProgram.cpp | 2 + .../recompiler/ir/opcodes/ValueOpcodes.inc | 2 + .../recompiler/ir/passes/SsaRewrite.cpp | 29 +- src/graphics/shader/shader.cpp | 42 +- src/graphics/shader/shader.h | 13 +- tests/ShaderRecompilerComputeTests.cpp | 565 +++++++++++------- tests/shaderCfgTests.cpp | 443 +++----------- 30 files changed, 897 insertions(+), 1140 deletions(-) delete mode 100644 src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp delete mode 100644 src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.h diff --git a/src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp b/src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp deleted file mode 100644 index b35e291d..00000000 --- a/src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h" - -#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" - -namespace Libs::Graphics { - -ShaderSubgroupCapabilities::ShaderSubgroupCapabilities(const GraphicContext& graphics) - : subgroup_size(graphics.subgroup_size), min_subgroup_size(graphics.min_subgroup_size), - max_subgroup_size(graphics.max_subgroup_size), - required_subgroup_size_stages(graphics.required_subgroup_size_stages), - subgroup_size_control_enabled(graphics.subgroup_size_control_enabled) {} - -ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size) { - // A 64-wide guest wave can't assume the host GPU's lane layout matches - // GCN's. AMD RDNA also reports a 64-wide subgroup but uses a different - // lane/pixel layout, which corrupted rendering under "native wave" mode. - // Per-invocation emulation works regardless of the host's lane layout. - if (guest_wave_size == 64u) { - return ShaderLaneMaskMode::PerInvocation; - } - return ShaderLaneMaskMode::NativeWave; -} - -ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities, - vk::ShaderStageFlagBits stage, - const ShaderRecompiler::IR::Program& program) { - const auto guest_wave_size = program.wave_size; - if ((guest_wave_size != 32u && guest_wave_size != 64u) || - !program.spirv_requirements.has_value()) { - return {}; - } - if (stage != vk::ShaderStageFlagBits::eCompute) { - const auto expected = SelectGraphicsLaneMaskMode(guest_wave_size); - if (program.lane_mask_mode != expected || (capabilities.subgroup_size != guest_wave_size && - expected != ShaderLaneMaskMode::PerInvocation)) { - return {}; - } - return {expected == ShaderLaneMaskMode::NativeWave - ? ShaderSubgroupMode::Natural - : ShaderSubgroupMode::PerInvocationGraphics, - 0}; - } - if (program.lane_mask_mode != ShaderLaneMaskMode::NativeWave) { - return {}; - } - if (capabilities.subgroup_size == guest_wave_size) { - return {ShaderSubgroupMode::Natural, 0}; - } - if (capabilities.subgroup_size_control_enabled && - (capabilities.required_subgroup_size_stages & stage) && - guest_wave_size >= capabilities.min_subgroup_size && - guest_wave_size <= capabilities.max_subgroup_size) { - return {ShaderSubgroupMode::Controlled, guest_wave_size}; - } - return {ShaderSubgroupMode::FlattenedMasks, 0}; -} - -} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.h b/src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.h deleted file mode 100644 index 96b382f6..00000000 --- a/src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_ -#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_ - -#include "graphics/host_gpu/graphicContext.h" -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -namespace Libs::Graphics { - -enum class ShaderSubgroupMode { - Natural, - Controlled, - PerInvocationGraphics, - FlattenedMasks, - Unsupported -}; - -struct ShaderSubgroupConfiguration { - ShaderSubgroupMode mode = ShaderSubgroupMode::Unsupported; - uint32_t required_size = 0; -}; - -struct ShaderSubgroupCapabilities { - explicit ShaderSubgroupCapabilities(const GraphicContext& graphics); - - uint32_t subgroup_size = 0; - uint32_t min_subgroup_size = 0; - uint32_t max_subgroup_size = 0; - vk::ShaderStageFlags required_subgroup_size_stages; - bool subgroup_size_control_enabled = false; -}; - -ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size); - -ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities, - vk::ShaderStageFlagBits stage, - const ShaderRecompiler::IR::Program& program); - -} // namespace Libs::Graphics - -#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_ diff --git a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp index 7f05fa02..89b8c10a 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp @@ -8,7 +8,6 @@ #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" -#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/renderTarget.h" @@ -415,48 +414,6 @@ static void CreateLayout(DescriptorCache& descriptor_cache, } } -static void ConfigureSubgroupSize(const GraphicContext& graphics, vk::ShaderStageFlagBits vk_stage, - const ShaderRecompiler::IR::Program& program, - vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo& required, - vk::PipelineShaderStageCreateInfo& stage) { - const auto config = - ConfigureShaderSubgroup(ShaderSubgroupCapabilities {graphics}, vk_stage, program); - switch (config.mode) { - case ShaderSubgroupMode::Natural: return; - case ShaderSubgroupMode::PerInvocationGraphics: { - static std::atomic log_count {0}; - if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { - LOGF("Vulkan running %u-lane graphics shader stage 0x%08x with per-invocation " - "EXEC/VCC on the host-native subgroup\n", - program.wave_size, static_cast(vk_stage)); - } - return; - } - case ShaderSubgroupMode::FlattenedMasks: { - static std::atomic log_count {0}; - if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { - LOGF("Vulkan running %u-lane shader stage 0x%08x with flattened per-invocation " - "EXEC/VCC\n", - program.wave_size, static_cast(vk_stage)); - } - return; - } - case ShaderSubgroupMode::Controlled: break; - case ShaderSubgroupMode::Unsupported: - default: - EXIT("Vulkan cannot run %u-lane shader stage 0x%08x: default=%u min=%u max=%u " - "controlled_stages=0x%08x\n", - program.wave_size, static_cast(vk_stage), graphics.subgroup_size, - graphics.min_subgroup_size, graphics.max_subgroup_size, - static_cast( - graphics.required_subgroup_size_stages)); - } - - required.sType = vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo; - required.requiredSubgroupSize = config.required_size; - stage.pNext = &required; -} - // NOLINTNEXTLINE(readability-function-cognitive-complexity) void CreatePipelineInternal( GraphicContext& graphics, DescriptorCache& descriptor_cache, @@ -528,8 +485,7 @@ void CreatePipelineInternal( rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr)); EXIT_NOT_IMPLEMENTED(ps_active && frag_shader_module == nullptr); - vk::PipelineShaderStageCreateInfo vert_shader_stage_info {}; - vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo vert_subgroup_size {}; + vk::PipelineShaderStageCreateInfo vert_shader_stage_info {}; vert_shader_stage_info.sType = vk::StructureType::ePipelineShaderStageCreateInfo; vert_shader_stage_info.pNext = nullptr; @@ -538,12 +494,8 @@ void CreatePipelineInternal( vert_shader_stage_info.module = vert_shader_module; vert_shader_stage_info.pName = "main"; vert_shader_stage_info.pSpecializationInfo = nullptr; - EXIT_IF(!vs_input_info.stage); - ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eVertex, *vs_input_info.stage.program, - vert_subgroup_size, vert_shader_stage_info); - vk::PipelineShaderStageCreateInfo frag_shader_stage_info {}; - vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo frag_subgroup_size {}; + vk::PipelineShaderStageCreateInfo frag_shader_stage_info {}; frag_shader_stage_info.sType = vk::StructureType::ePipelineShaderStageCreateInfo; frag_shader_stage_info.pNext = nullptr; frag_shader_stage_info.flags = {}; @@ -551,12 +503,6 @@ void CreatePipelineInternal( frag_shader_stage_info.module = frag_shader_module; frag_shader_stage_info.pName = "main"; frag_shader_stage_info.pSpecializationInfo = nullptr; - if (ps_active) { - EXIT_IF(!ps_input_info->stage); - ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eFragment, - *ps_input_info->stage.program, frag_subgroup_size, - frag_shader_stage_info); - } vk::PipelineShaderStageCreateInfo tess_control_shader_stage_info {}; tess_control_shader_stage_info.sType = vk::StructureType::ePipelineShaderStageCreateInfo; @@ -1074,8 +1020,15 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto comp_shader_stage_info.pName = "main"; comp_shader_stage_info.pSpecializationInfo = nullptr; EXIT_IF(!input_info.stage); - ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eCompute, *input_info.stage.program, - comp_subgroup_size, comp_shader_stage_info); + const auto wave_size = input_info.stage.program->wave_size; + if (graphics.subgroup_size_control_enabled && + (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) && + wave_size >= graphics.min_subgroup_size && wave_size <= graphics.max_subgroup_size) { + comp_subgroup_size.sType = + vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo; + comp_subgroup_size.requiredSubgroupSize = wave_size; + comp_shader_stage_info.pNext = &comp_subgroup_size; + } vk::DescriptorSetLayout set_layouts[1] = {}; uint32_t set_layouts_num = 0; @@ -1083,7 +1036,6 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto vk::PushConstantRange push_constant_info[1]; uint32_t push_constant_info_num = 0; - EXIT_IF(!input_info.stage); CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info, push_constant_info_num, *input_info.stage.program, vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 37731d84..c863c45b 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -15,7 +15,6 @@ #include "graphics/host_gpu/renderer/pipeline/descriptors.h" #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h" -#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/vulkanCommon.h" diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index a55bee6f..6ef035ae 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -18,7 +18,6 @@ #include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h" -#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/vulkanCommon.h" @@ -1061,12 +1060,10 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw for (uint32_t i = 0; i < state.color_count; i++) { target_export_mapping[state.color_info[i].target_slot] = state.color_info[i].export_mapping; } - const auto lane_mask_mode = SelectGraphicsLaneMaskMode(64u); - if (log_phases) { LogDrawPhase(draw.name, "ShaderCompileInfoVS"); } - if (!ShaderCompileInfoVS(vertex_shader_info, shader_regs, lane_mask_mode, state.vs_input_info, + if (!ShaderCompileInfoVS(vertex_shader_info, shader_regs, state.vs_input_info, state.vs_shader)) { EXIT("ShaderCompileInfoVS failed for draw %s\n", draw.name); } @@ -1077,7 +1074,7 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw if (log_phases) { LogDrawPhase(draw.name, "ShaderCompileInfoPS"); } - if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, lane_mask_mode, state.vs_input_info, + if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, state.vs_input_info, target_export_mapping, state.ps_input_info, state.ps_shader)) { EXIT("ShaderCompileInfoPS failed for draw %s\n", draw.name); } diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index 923cded1..b81c27b1 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -514,10 +514,6 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V features12.pNext = &depth_clip_control; features12.samplerMirrorClampToEdge = VK_TRUE; - vk::PhysicalDeviceSubgroupSizeControlFeatures subgroup_size_control {}; - subgroup_size_control.sType = vk::StructureType::ePhysicalDeviceSubgroupSizeControlFeatures; - subgroup_size_control.pNext = &features12; - vk::PhysicalDeviceVulkan13Features supported_features13 {}; supported_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features; @@ -578,28 +574,23 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V device_features.shaderInt64 = supported_features2.features.shaderInt64; } - if (graphics.subgroup_size_control_enabled && - supported_features13.subgroupSizeControl == VK_TRUE) { - subgroup_size_control.subgroupSizeControl = VK_TRUE; - } - - const auto* base_feature_chain = (subgroup_size_control.subgroupSizeControl == VK_TRUE - ? static_cast(&subgroup_size_control) - : static_cast(&features12)); - vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {}; robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT; - robustness2.pNext = const_cast(base_feature_chain); + robustness2.pNext = &features12; if (robustness2_ext_enabled) { robustness2.robustBufferAccess2 = supported_robustness2.robustBufferAccess2; robustness2.robustImageAccess2 = supported_robustness2.robustImageAccess2; robustness2.nullDescriptor = supported_robustness2.nullDescriptor; } - auto features13 = required_features13; - features13.pNext = - robustness2_ext_enabled ? &robustness2 : const_cast(base_feature_chain); + const bool subgroup_size_control_enabled = + graphics.subgroup_size_control_enabled && supported_features13.subgroupSizeControl == VK_TRUE; + + auto features13 = required_features13; + features13.pNext = robustness2_ext_enabled ? static_cast(&robustness2) + : static_cast(&features12); features13.robustImageAccess = supported_features13.robustImageAccess; + features13.subgroupSizeControl = subgroup_size_control_enabled ? VK_TRUE : VK_FALSE; LOGF("Vulkan robustness: robustImageAccess=%s robustImageAccess2=%s\n", features13.robustImageAccess == VK_TRUE ? "true" : "false", diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index aaaaade9..f5b1a438 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -683,7 +683,6 @@ bool TryRecompile(std::span code, const CompileOptions& options, if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, output, error)) { return false; } - output.lane_mask_mode = options.lane_mask_mode; output.shader_hash = options.shader_hash; output.user_data_base = options.user_data_base; output.user_data_count = options.user_data_count; diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h index 84169566..d66322cc 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.h +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -14,7 +14,6 @@ namespace Libs::Graphics::ShaderRecompiler { struct CompileOptions { ShaderType stage = ShaderType::Compute; - ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave; uint32_t wave_size = 64; uint32_t user_data_base = 0; uint32_t user_data_count = 64; diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index af8162f8..262c2387 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -198,10 +198,7 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { program.spirv_requirements.emplace(requirements); return true; } - const auto MarkExactSubgroup = [&] { - requirements.requires_exact_subgroup = true; - requirements.subgroup_ballot = true; - }; + const auto MarkBallot = [&] { requirements.subgroup_ballot = true; }; for (const auto* block: program.values->blocks) { for (const auto& inst: *block) { if (IR::AddressOpcodeInfoOf(inst.GetOpcode()).access != IR::AddressAccess::None) { @@ -230,7 +227,6 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { if (program.stage != ShaderType::Compute) { return Fail(error, "buffer ADD_TID is only valid for compute shaders"); } - requirements.requires_exact_subgroup = true; requirements.subgroup_local_invocation_id = true; } } @@ -250,17 +246,17 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { } if (shared_access == IR::SharedAccess::Append || shared_access == IR::SharedAccess::Consume) { - MarkExactSubgroup(); + MarkBallot(); requirements.subgroup_shuffle = true; requirements.subgroup_local_invocation_id = true; } } switch (inst.GetOpcode()) { - case IR::ValueOpcode::Ballot: MarkExactSubgroup(); break; + case IR::ValueOpcode::Ballot: MarkBallot(); break; case IR::ValueOpcode::DppMoveU32: case IR::ValueOpcode::ReadFirstLane: case IR::ValueOpcode::ReadLane: { - MarkExactSubgroup(); + MarkBallot(); requirements.subgroup_shuffle = true; if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) { requirements.subgroup_local_invocation_id = true; @@ -270,18 +266,18 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { case IR::ValueOpcode::DppUpdateU32: case IR::ValueOpcode::WqmMask: case IR::ValueOpcode::WriteLane: { - MarkExactSubgroup(); + MarkBallot(); requirements.subgroup_local_invocation_id = true; break; } case IR::ValueOpcode::Permlane16U32: { - MarkExactSubgroup(); + MarkBallot(); requirements.subgroup_shuffle = true; requirements.subgroup_local_invocation_id = true; break; } case IR::ValueOpcode::SwizzleU32: { - MarkExactSubgroup(); + MarkBallot(); requirements.subgroup_shuffle = true; requirements.subgroup_local_invocation_id = true; break; @@ -346,9 +342,8 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc return false; } EmitterState state(program, resources, input_info); - state.stage = program.stage; - state.wave_size = program.wave_size; - state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation; + state.stage = program.stage; + state.wave_size = program.wave_size; state.inputs.reserve(program.info.inputs.size()); state.outputs.reserve(program.info.outputs.size()); state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size()); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 12bb7296..cdc2b256 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -58,13 +58,6 @@ uint32_t EmitWqmU64(EmitterState& state, uint32_t value) { return result; } -uint32_t BoolAsU32(ValueEmitContext& ctx, uint32_t value) { - const auto result = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), result, value, - ConstantU32(ctx.state, 1), ConstantU32(ctx.state, 0)}); - return result; -} - uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) { auto& state = ctx.state; if (kind == IR::StageInputKind::LocalInvocationIndex) { @@ -222,8 +215,8 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo bool uint_output) { auto& state = ctx.state; if (exp.compr && !uint_output) { - const auto unpack = MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16 - : GlslUnpackHalf2x16; + const auto unpack = + MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16 : GlslUnpackHalf2x16; uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)}; for (uint32_t pair = 0; pair < 2u; pair++) { @@ -441,16 +434,8 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Define(inst, EmitSubgroupLocalInvocationId(state)); return true; case IR::ValueOpcode::Ballot: - if (state.per_invocation_masks) { - const auto result = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), result, - BoolAsU32(ctx, ctx.Arg(inst, 0)), ConstantU32(state, 0), - ConstantU32(state, 0), ConstantU32(state, 0)}); - ctx.Define(inst, result); - } else { - ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4, - {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)}); - } + ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4, + {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)}); return true; case IR::ValueOpcode::ReadFirstLane: { const auto ballot = state.builder.AllocateId(); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index d6f53eee..5db3bc8d 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -251,30 +251,30 @@ enum : uint32_t { }; enum : uint32_t { - GlslRoundEven = 2, - GlslTrunc = 3, - GlslFAbs = 4, - GlslFloor = 8, - GlslCeil = 9, - GlslFract = 10, - GlslSin = 13, - GlslCos = 14, - GlslExp2 = 29, - GlslLog2 = 30, - GlslSqrt = 31, - GlslInverseSqrt = 32, - GlslFMin = 37, - GlslFMax = 40, - GlslFClamp = 43, - GlslLdexp = 53, - GlslFma = 50, - GlslPackSnorm2x16 = 56, - GlslPackUnorm2x16 = 57, - GlslPackHalf2x16 = 58, + GlslRoundEven = 2, + GlslTrunc = 3, + GlslFAbs = 4, + GlslFloor = 8, + GlslCeil = 9, + GlslFract = 10, + GlslSin = 13, + GlslCos = 14, + GlslExp2 = 29, + GlslLog2 = 30, + GlslSqrt = 31, + GlslInverseSqrt = 32, + GlslFMin = 37, + GlslFMax = 40, + GlslFClamp = 43, + GlslLdexp = 53, + GlslFma = 50, + GlslPackSnorm2x16 = 56, + GlslPackUnorm2x16 = 57, + GlslPackHalf2x16 = 58, GlslUnpackUnorm2x16 = 61, - GlslUnpackHalf2x16 = 62, - GlslFindILsb = 73, - GlslFindUMsb = 75, + GlslUnpackHalf2x16 = 62, + GlslFindILsb = 73, + GlslFindUMsb = 75, }; struct InputBinding { @@ -325,41 +325,40 @@ struct EmitterState { : program(program_), resources(resources_), input_info(input_info_), requirements(*program_.spirv_requirements) {} - Builder builder; - const IR::Program& program; - const IR::ResourceSnapshot& resources; - ShaderStageInputInfo input_info; - const IR::SpirvRequirements& requirements; - ShaderType stage = ShaderType::Unknown; - uint32_t wave_size = 64; - bool per_invocation_masks = false; - uint32_t storage_buffer_variable = 0; + Builder builder; + const IR::Program& program; + const IR::ResourceSnapshot& resources; + ShaderStageInputInfo input_info; + const IR::SpirvRequirements& requirements; + ShaderType stage = ShaderType::Unknown; + uint32_t wave_size = 64; + uint32_t storage_buffer_variable = 0; std::array - memory_byte_offsets {}; - uint32_t address_memory_variable = 0; - uint32_t gds_variable = 0; - uint32_t gds_length = 0; - uint32_t push_constant_variable = 0; - uint32_t vsharp_storage_variable = 0; - uint32_t flattened_srt_variable = 0; - uint32_t lds_variable = 0; - uint32_t scratch_variable = 0; + memory_byte_offsets {}; + uint32_t address_memory_variable = 0; + uint32_t gds_variable = 0; + uint32_t gds_length = 0; + uint32_t push_constant_variable = 0; + uint32_t vsharp_storage_variable = 0; + uint32_t flattened_srt_variable = 0; + uint32_t lds_variable = 0; + uint32_t scratch_variable = 0; std::array sampled_image_variables {}; std::array(StorageImageClass::Count) * StorageImageViewKindCount> - storage_image_variables {}; - uint32_t sampler_variable = 0; - uint32_t main_func = 0; - uint32_t entry_label = 0; - uint32_t current_label = 0; - uint32_t pixel_valid_mask_variable = 0; - uint32_t subgroup_local_invocation_id_variable = 0; - uint32_t per_vertex_variable = 0; - uint32_t depth_variable = 0; - uint32_t sample_mask_variable = 0; - std::vector inputs; - std::vector outputs; - std::vector interface_variables; + storage_image_variables {}; + uint32_t sampler_variable = 0; + uint32_t main_func = 0; + uint32_t entry_label = 0; + uint32_t current_label = 0; + uint32_t pixel_valid_mask_variable = 0; + uint32_t subgroup_local_invocation_id_variable = 0; + uint32_t per_vertex_variable = 0; + uint32_t depth_variable = 0; + uint32_t sample_mask_variable = 0; + std::vector inputs; + std::vector outputs; + std::vector interface_variables; }; uint32_t TypeVoid(EmitterState& state); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index ec5f347d..22216af9 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -672,11 +672,9 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) const auto high = state.builder.AllocateId(); state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0}); state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1}); - const auto count_low = state.per_invocation_masks ? low : ctx.Arg(inst, 2); - const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 3); const auto count = - Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), count_low), - Unary(state, OpBitCount, TypeU32(state), count_high)); + Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), low), + Unary(state, OpBitCount, TypeU32(state), high)); const auto first = state.builder.AllocateId(); state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), first, ConstantU32(state, ScopeSubgroup), ballot}); diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index 11c04533..dbfd29b2 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -7,37 +7,26 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) { switch (inst.op) { case IR::Opcode::SaveexecB32: case IR::Opcode::SaveexecB64: { - const auto old_low = ir.GetExecLo(); - const auto old_high = ir.GetExecHi(); - const auto src_low = ReadRawU32(inst.src[0]); - const auto src_high = inst.op == IR::Opcode::SaveexecB64 - ? ReadRawU32(OffsetOperand(inst.src[0], 1)) - : IR::U32(IR::Value(0u)); - const auto apply = [&](IR::U32 old_value, IR::U32 src_value) { - switch (inst.saveexec_mode) { - case IR::SaveexecMode::And: return ir.BitwiseAnd(old_value, src_value); - case IR::SaveexecMode::Andn1: - return ir.BitwiseAnd(old_value, ir.BitwiseNot(src_value)); - case IR::SaveexecMode::Orn2: - return ir.BitwiseOr(ir.BitwiseNot(old_value), src_value); - } - return old_value; - }; - const auto result_low = apply(old_low, src_low); - const auto result_high = - inst.op == IR::Opcode::SaveexecB64 ? apply(old_high, src_high) : old_high; - if (inst.op == IR::Opcode::SaveexecB64) { - WriteU32Pair(inst.dst, {old_low, old_high}); - } else { - WriteOperand(inst.dst, old_low); + const auto old = ir.GetExec(); + const auto src = ReadMask(inst.src[0]); + IR::U1 result; + switch (inst.saveexec_mode) { + case IR::SaveexecMode::And: result = ir.LogicalAnd(old, src); break; + case IR::SaveexecMode::Andn1: + result = ir.LogicalAnd(old, ir.LogicalNot(src)); + break; + case IR::SaveexecMode::Orn2: result = ir.LogicalOr(ir.LogicalNot(old), src); break; } - ir.SetExecLo(result_low); - ir.SetExecHi(result_high); - ir.SetExec(ThreadBit(result_low, result_high)); - const auto nonzero = inst.op == IR::Opcode::SaveexecB64 - ? ir.BitwiseOr(result_low, result_high) - : result_low; - ir.SetScc(ir.INotEqual(nonzero, zero)); + if (inst.op == IR::Opcode::SaveexecB64) { + WriteMask64(inst.dst, old); + } else { + WriteMask(inst.dst, old); + } + const auto mask = BallotMask(result); + ir.SetExec(result); + ir.SetExecLo(mask[0]); + ir.SetExecHi(mask[1]); + ir.SetScc(result); return true; } case IR::Opcode::ScalarAddCarryU32: @@ -157,24 +146,61 @@ bool Translator::TranslateMove(const IR::Instruction& inst) { WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32)); return true; case IR::Opcode::MoveU64: { + const auto is_exec_or_vcc = [](const IR::Operand& operand) { + return operand.kind == IR::OperandKind::Register && + (operand.reg.file == IR::RegisterFile::Exec || + operand.reg.file == IR::RegisterFile::Vcc); + }; + if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0])) { + WriteMask64(inst.dst, ReadMask(inst.src[0])); + return true; + } + const bool scalar_copy = inst.dst.kind == IR::OperandKind::Register && + inst.src[0].kind == IR::OperandKind::Register && + inst.dst.reg.file == IR::RegisterFile::Scalar && + inst.src[0].reg.file == IR::RegisterFile::Scalar; + IR::U1 source_mask; + IR::U1 source_mask_valid; + if (scalar_copy) { + const auto src = static_cast(inst.src[0].reg.index); + source_mask = ir.GetThreadBitScalarReg(src); + source_mask_valid = ir.GetScalarMaskTag(src); + } WriteU32Pair(inst.dst, ReadU32Pair(inst.src[0])); - if (inst.dst.kind == IR::OperandKind::Register && - inst.src[0].kind == IR::OperandKind::Register && - inst.dst.reg.file == IR::RegisterFile::Scalar && - inst.src[0].reg.file == IR::RegisterFile::Scalar) { - ir.SetThreadBitScalarReg( - static_cast(inst.dst.reg.index), - ir.GetThreadBitScalarReg(static_cast(inst.src[0].reg.index))); + if (scalar_copy) { + ir.SetThreadBitScalarReg(static_cast(inst.dst.reg.index), + source_mask); + ir.SetScalarMaskTag(static_cast(inst.dst.reg.index), + source_mask_valid); } return true; } case IR::Opcode::WqmB64: { - if (!current_per_invocation_masks) { + const auto is_exec_or_vcc = [](const IR::Operand& operand) { + return operand.kind == IR::OperandKind::Register && + (operand.reg.file == IR::RegisterFile::Exec || + operand.reg.file == IR::RegisterFile::Vcc); + }; + if (!is_exec_or_vcc(inst.dst) && !is_exec_or_vcc(inst.src[0])) { + const auto mask_valid = ReadMaskValid(inst.src[0]); + const auto invocation_result = + IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])})); const auto result = IR::U64( ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src[0], IR::Type::U64)})); WriteOperand(inst.dst, result); - ir.SetScc(IR::U1( - ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); + if (inst.dst.kind == IR::OperandKind::Register && + inst.dst.reg.file == IR::RegisterFile::Scalar) { + const auto dst = static_cast(inst.dst.reg.index); + ir.SetThreadBitScalarReg(dst, invocation_result); + ir.SetScalarMaskTag(dst, mask_valid); + const auto raw_nonzero = IR::U1( + ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})); + ir.SetScc(IR::U1(ir.Emit( + IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero}))); + } else { + ir.SetScc(IR::U1( + ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); + } return true; } const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])})); diff --git a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp index 6caf7544..fbee46ae 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp @@ -3,13 +3,76 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) { - if (current_per_invocation_masks) { - return TranslatePerInvocationU64Mask(inst); + const auto invocation_result = EvaluateU64Mask(inst); + const auto is_exec_or_vcc = [](const IR::Operand& operand) { + return operand.kind == IR::OperandKind::Register && + (operand.reg.file == IR::RegisterFile::Exec || + operand.reg.file == IR::RegisterFile::Vcc); + }; + if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0]) || + (inst.src_count > 1u && is_exec_or_vcc(inst.src[1]))) { + WriteMask64(inst.dst, invocation_result); + ir.SetScc(invocation_result); + return true; } - return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 || - inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64 - ? TranslateSimpleInteger(inst) - : TranslateComposedInteger(inst); + + const auto lhs = ReadU32Pair(inst.src[0]); + auto mask_valid = ReadMaskValid(inst.src[0]); + if (inst.src_count > 1u) { + mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src[1])); + } + std::array result; + if (inst.op == IR::Opcode::BitwiseNotU64) { + result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])}; + } else { + const auto rhs = ReadU32Pair(inst.src[1]); + for (uint32_t component = 0; component < 2u; component++) { + switch (inst.op) { + case IR::Opcode::BitwiseAndU64: + result[component] = ir.BitwiseAnd(lhs[component], rhs[component]); + break; + case IR::Opcode::BitwiseAndNotU64: + result[component] = + ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component])); + break; + case IR::Opcode::BitwiseOrU64: + result[component] = ir.BitwiseOr(lhs[component], rhs[component]); + break; + case IR::Opcode::BitwiseOrNotU64: + result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component])); + break; + case IR::Opcode::BitwiseXorU64: + result[component] = ir.BitwiseXor(lhs[component], rhs[component]); + break; + case IR::Opcode::BitwiseNandU64: + result[component] = + ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component])); + break; + case IR::Opcode::BitwiseNorU64: + result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component])); + break; + case IR::Opcode::BitwiseXnorU64: + result[component] = + ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component])); + break; + default: return false; + } + } + } + WriteU32Pair(inst.dst, result); + if (inst.dst.kind == IR::OperandKind::Register && + inst.dst.reg.file == IR::RegisterFile::Scalar) { + const auto dst = static_cast(inst.dst.reg.index); + ir.SetThreadBitScalarReg(dst, invocation_result); + ir.SetScalarMaskTag(dst, mask_valid); + const auto raw_nonzero = + ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))); + ir.SetScc(IR::U1( + ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero}))); + } else { + ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)))); + } + return true; } bool Translator::TranslateInstruction(const IR::Instruction& inst) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp index f66591fc..bf69c4c7 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp @@ -126,62 +126,26 @@ bool Translator::TranslatePackedInteger16(const IR::Instruction& inst) { } } -bool Translator::TranslatePerInvocationU64Mask(const IR::Instruction& inst) { - if (!current_per_invocation_masks) { - return false; - } - +IR::U1 Translator::EvaluateU64Mask(const IR::Instruction& inst) { const auto lhs = [&] { return ReadMask(inst.src[0]); }; const auto rhs = [&] { return ReadMask(inst.src[1]); }; - IR::U1 result; switch (inst.op) { - case IR::Opcode::BitwiseAndU64: result = ir.LogicalAnd(lhs(), rhs()); break; - case IR::Opcode::BitwiseOrU64: result = ir.LogicalOr(lhs(), rhs()); break; + case IR::Opcode::BitwiseAndU64: return ir.LogicalAnd(lhs(), rhs()); + case IR::Opcode::BitwiseOrU64: return ir.LogicalOr(lhs(), rhs()); case IR::Opcode::BitwiseXorU64: - result = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})); - break; - case IR::Opcode::BitwiseAndNotU64: - result = ir.LogicalAnd(lhs(), ir.LogicalNot(rhs())); - break; - case IR::Opcode::BitwiseOrNotU64: result = ir.LogicalOr(lhs(), ir.LogicalNot(rhs())); break; - case IR::Opcode::BitwiseNandU64: result = ir.LogicalNot(ir.LogicalAnd(lhs(), rhs())); break; - case IR::Opcode::BitwiseNorU64: result = ir.LogicalNot(ir.LogicalOr(lhs(), rhs())); break; + return IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})); + case IR::Opcode::BitwiseAndNotU64: return ir.LogicalAnd(lhs(), ir.LogicalNot(rhs())); + case IR::Opcode::BitwiseOrNotU64: return ir.LogicalOr(lhs(), ir.LogicalNot(rhs())); + case IR::Opcode::BitwiseNandU64: return ir.LogicalNot(ir.LogicalAnd(lhs(), rhs())); + case IR::Opcode::BitwiseNorU64: return ir.LogicalNot(ir.LogicalOr(lhs(), rhs())); case IR::Opcode::BitwiseXnorU64: - result = ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}))); - break; - case IR::Opcode::BitwiseNotU64: result = ir.LogicalNot(lhs()); break; - default: return false; + return ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}))); + case IR::Opcode::BitwiseNotU64: return ir.LogicalNot(lhs()); + default: EXIT("invalid 64-bit mask opcode"); } - WriteMask64(inst.dst, result); - return true; } bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) { - if (inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 || - inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64) { - const auto lhs = ReadU32Pair(inst.src[0]); - std::array result; - if (inst.op == IR::Opcode::BitwiseNotU64) { - result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])}; - } else { - const auto rhs = ReadU32Pair(inst.src[1]); - for (uint32_t component = 0; component < 2; component++) { - switch (inst.op) { - case IR::Opcode::BitwiseAndU64: - result[component] = ir.BitwiseAnd(lhs[component], rhs[component]); - break; - case IR::Opcode::BitwiseOrU64: - result[component] = ir.BitwiseOr(lhs[component], rhs[component]); - break; - default: - result[component] = ir.BitwiseXor(lhs[component], rhs[component]); - break; - } - } - } - WriteU32Pair(inst.dst, result); - return true; - } IR::ValueOpcode opcode {}; IR::Type type = IR::Type::U32; switch (inst.op) { @@ -250,37 +214,6 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) { } bool Translator::TranslateComposedInteger(const IR::Instruction& inst) { - if (inst.op == IR::Opcode::BitwiseAndNotU64 || inst.op == IR::Opcode::BitwiseOrNotU64 || - inst.op == IR::Opcode::BitwiseNandU64 || inst.op == IR::Opcode::BitwiseNorU64 || - inst.op == IR::Opcode::BitwiseXnorU64) { - const auto lhs = ReadU32Pair(inst.src[0]); - const auto rhs = ReadU32Pair(inst.src[1]); - std::array result; - for (uint32_t component = 0; component < 2; component++) { - switch (inst.op) { - case IR::Opcode::BitwiseAndNotU64: - result[component] = - ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component])); - break; - case IR::Opcode::BitwiseOrNotU64: - result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component])); - break; - case IR::Opcode::BitwiseNandU64: - result[component] = - ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component])); - break; - case IR::Opcode::BitwiseNorU64: - result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component])); - break; - default: - result[component] = - ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component])); - break; - } - } - WriteU32Pair(inst.dst, result); - return true; - } const auto binary_u32 = [&](auto operation) { const auto lhs = ReadU32(inst.src[0]); const auto rhs = ReadU32(inst.src[1]); @@ -612,10 +545,28 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { } case IR::Opcode::SelectU64: { const auto condition = ReadCondition(inst.src[0]); - const auto lhs = ReadU32Pair(inst.src[1]); - const auto rhs = ReadU32Pair(inst.src[2]); + const auto selected_mask = + IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, + {condition, ReadMask(inst.src[1]), ReadMask(inst.src[2])})); + const auto selected_mask_valid = + IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMaskValid(inst.src[1]), + ReadMaskValid(inst.src[2])})); + if (inst.dst.kind == IR::OperandKind::Register && + (inst.dst.reg.file == IR::RegisterFile::Exec || + inst.dst.reg.file == IR::RegisterFile::Vcc)) { + WriteMask64(inst.dst, selected_mask); + return true; + } + const auto lhs = ReadU32Pair(inst.src[1]); + const auto rhs = ReadU32Pair(inst.src[2]); WriteU32Pair(inst.dst, {ir.Select(condition, lhs[0], rhs[0]), ir.Select(condition, lhs[1], rhs[1])}); + if (inst.dst.kind == IR::OperandKind::Register && + inst.dst.reg.file == IR::RegisterFile::Scalar) { + const auto dst = static_cast(inst.dst.reg.index); + ir.SetThreadBitScalarReg(dst, selected_mask); + ir.SetScalarMaskTag(dst, selected_mask_valid); + } return true; } case IR::Opcode::PackLowLowU16: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index e1fdaf2c..1581a87d 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -91,19 +91,8 @@ IR::U32 Translator::ReadPcRelativeU32(const IR::Operand& operand) { } std::array Translator::BallotMask(IR::U1 value) { - if (current_per_invocation_masks) { - return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))), - IR::U32(IR::Value(0u))}; - } - const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {value}); - const auto low = - IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(0u)})); - // RDNA2 wave32 masks have one architectural dword; the upper half is unused. - const auto high = - current_wave_size == 32u - ? IR::U32(IR::Value(0u)) - : IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(1u)})); - return {low, high}; + return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))), + IR::U32(IR::Value(0u))}; } IR::U32 Translator::ReadRawU32(const IR::Operand& operand) { @@ -271,9 +260,17 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) { } } switch (operand.reg.file) { - case IR::RegisterFile::Scalar: - ir.SetScalarReg(static_cast(operand.reg.index), value); + case IR::RegisterFile::Scalar: { + const auto reg = static_cast(operand.reg.index); + ir.SetScalarReg(reg, value); + ir.SetThreadBitScalarReg(reg, ThreadBit(value)); + ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false))); + if (IR::RegIndex(reg) > 0u) { + ir.SetScalarMaskTag(static_cast(IR::RegIndex(reg) - 1u), + IR::U1(IR::Value(false))); + } break; + } case IR::RegisterFile::Vector: { const auto reg = static_cast(operand.reg.index); const auto old = ir.GetVectorReg(reg); @@ -296,20 +293,20 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) { case IR::RegisterFile::Vcc: if (operand.reg.index == 0) { ir.SetVccLo(IR::U32(value)); - ir.SetVcc(ThreadBit(IR::U32(value), ir.GetVccHi())); + ir.SetVcc(ThreadBit(IR::U32(value))); } else { ir.SetVccHi(IR::U32(value)); - ir.SetVcc(ThreadBit(ir.GetVccLo(), IR::U32(value))); + ir.SetVcc(ThreadBit(ir.GetVccLo())); } break; case IR::RegisterFile::M0: ir.SetM0(IR::U32(value)); break; case IR::RegisterFile::Exec: if (operand.reg.index == 0) { ir.SetExecLo(IR::U32(value)); - ir.SetExec(ThreadBit(IR::U32(value), ir.GetExecHi())); + ir.SetExec(ThreadBit(IR::U32(value))); } else { ir.SetExecHi(IR::U32(value)); - ir.SetExec(ThreadBit(ir.GetExecLo(), IR::U32(value))); + ir.SetExec(ThreadBit(ir.GetExecLo())); } break; case IR::RegisterFile::Scc: ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u)))); break; @@ -459,10 +456,10 @@ IR::U64 Translator::ReadU64(const IR::Operand& operand) { IR::F32 Translator::ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed) { if (operand.float_inline) { const bool use_zero = packed && (high_lane ? operand.op_sel_hi : operand.op_sel); - auto value = use_zero ? IR::F32(IR::Value::F32(0.0f)) - : ir.BitCastF32(IR::U32(IR::Value(operand.imm))); - const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value})); - value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half})); + auto value = use_zero ? IR::F32(IR::Value::F32(0.0f)) + : ir.BitCastF32(IR::U32(IR::Value(operand.imm))); + const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value})); + value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half})); if (operand.absolute) { value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value})); } @@ -561,7 +558,7 @@ void Translator::WriteU32Pair(const IR::Operand& operand, const std::array(operand.reg.index), thread_bit); - break; + case IR::RegisterFile::Scalar: break; default: break; } } @@ -583,21 +578,8 @@ void Translator::WriteU32Pair(const IR::Operand& operand, const std::array(operand.reg.index); - const auto high = - current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs - ? ir.GetScalarReg(static_cast(IR::RegIndex(reg) + 1u)) - : IR::U32(IR::Value(0u)); - return ThreadBit(ir.GetScalarReg(reg), high); + return ir.GetThreadBitScalarReg(reg); } case IR::RegisterFile::Exec: return ir.GetExec(); case IR::RegisterFile::Vcc: return ir.GetVcc(); @@ -632,6 +610,25 @@ IR::U1 Translator::ReadMask(const IR::Operand& operand) { } } +IR::U1 Translator::ReadMaskValid(const IR::Operand& operand) { + if (operand.kind == IR::OperandKind::ImmediateU32) { + const bool zero = operand.imm == 0u && !operand.sext_64; + const bool ones = operand.imm == 0xffffffffu && operand.sext_64; + return IR::U1(IR::Value(zero || ones)); + } + if (operand.kind != IR::OperandKind::Register) { + return IR::U1(IR::Value(false)); + } + switch (operand.reg.file) { + case IR::RegisterFile::Scalar: + return ir.GetScalarMaskTag(static_cast(operand.reg.index)); + case IR::RegisterFile::Exec: + case IR::RegisterFile::Vcc: + case IR::RegisterFile::Scc: return IR::U1(IR::Value(true)); + default: return IR::U1(IR::Value(false)); + } +} + void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) { EXIT_IF(operand.kind != IR::OperandKind::Register); switch (operand.reg.file) { @@ -639,10 +636,18 @@ void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) { const auto reg = static_cast(operand.reg.index); const auto mask = BallotMask(value); ir.SetThreadBitScalarReg(reg, value); + ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true))); + if (IR::RegIndex(reg) > 0u) { + ir.SetScalarMaskTag(static_cast(IR::RegIndex(reg) - 1u), + IR::U1(IR::Value(false))); + } ir.SetScalarReg(reg, mask[0]); // A wave32 VALU mask destination must not overwrite the neighboring SGPR. if (current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs) { - ir.SetScalarReg(static_cast(IR::RegIndex(reg) + 1u), mask[1]); + const auto high = static_cast(IR::RegIndex(reg) + 1u); + ir.SetScalarReg(high, mask[1]); + ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false))); + ir.SetScalarMaskTag(high, IR::U1(IR::Value(false))); } return; } @@ -676,9 +681,17 @@ void Translator::WriteMask64(const IR::Operand& operand, IR::U1 value) { const auto reg = static_cast(operand.reg.index); const auto mask = BallotMask(value); ir.SetThreadBitScalarReg(reg, value); + ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true))); + if (IR::RegIndex(reg) > 0u) { + ir.SetScalarMaskTag(static_cast(IR::RegIndex(reg) - 1u), + IR::U1(IR::Value(false))); + } ir.SetScalarReg(reg, mask[0]); if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) { - ir.SetScalarReg(static_cast(IR::RegIndex(reg) + 1u), mask[1]); + const auto high = static_cast(IR::RegIndex(reg) + 1u); + ir.SetScalarReg(high, mask[1]); + ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false))); + ir.SetScalarMaskTag(high, IR::U1(IR::Value(false))); } } @@ -733,23 +746,17 @@ bool Translator::AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlock if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) { return true; } - IR::U1 condition; - const auto mask_non_zero = [&](IR::U1 value) { - const auto mask = BallotMask(value); - return ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u))); - }; + // EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive + // invocations leave the region without reconstructing a host-subgroup mask. + IR::U1 condition; switch (source.terminator.condition) { case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break; case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break; case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break; - case CFG::BranchCondition::VccZero: - condition = ir.LogicalNot(mask_non_zero(ir.GetVcc())); - break; - case CFG::BranchCondition::VccNonZero: condition = mask_non_zero(ir.GetVcc()); break; - case CFG::BranchCondition::ExecZero: - condition = ir.LogicalNot(mask_non_zero(ir.GetExec())); - break; - case CFG::BranchCondition::ExecNonZero: condition = mask_non_zero(ir.GetExec()); break; + case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break; + case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break; + case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break; + case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break; case CFG::BranchCondition::GotoVariable: if (source.terminator.goto_variable == UINT32_MAX) { return Fail(error, @@ -771,11 +778,10 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, const ShaderVertexInputInfo* vertex_input_info, const ShaderPixelInputInfo* pixel_input_info, const ShaderComputeInputInfo* compute_input_info, std::string* error) { - result = {}; - const uint32_t wave_size = source.wave_size; - const bool per_invocation_masks = source.lane_mask_mode == ShaderLaneMaskMode::PerInvocation; - uint32_t vector_limit = 1u; - const auto include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) { + result = {}; + const uint32_t wave_size = source.wave_size; + uint32_t vector_limit = 1u; + const auto include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) { if (operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Vector) { vector_limit = @@ -852,19 +858,14 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, if (IR::RegIndex(reg) >= IR::NumScalarRegs) { break; } - entry_ir.SetScalarReg(reg, entry_ir.GetUserData(reg)); + const auto value = entry_ir.GetUserData(reg); + entry_ir.SetScalarReg(reg, value); + entry_ir.SetThreadBitScalarReg(reg, entry_ir.INotEqual(value, IR::U32(IR::Value(0u)))); + entry_ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false))); } entry_ir.SetExec(IR::U1(IR::Value(true))); - if (source.stage == ShaderType::Compute && !per_invocation_masks) { - const auto active = entry_ir.Emit(IR::ValueOpcode::Ballot, {IR::Value(true)}); - entry_ir.SetExecLo(entry_ir.CompositeExtract(active, 0)); - entry_ir.SetExecHi(source.wave_size > 32u ? entry_ir.CompositeExtract(active, 1) - : IR::U32(IR::Value(0u))); - } else { - entry_ir.SetExecLo(IR::U32(IR::Value(per_invocation_masks ? 1u : 0xffffffffu))); - entry_ir.SetExecHi(IR::U32( - IR::Value(!per_invocation_masks && source.wave_size > 32u ? 0xffffffffu : 0u))); - } + entry_ir.SetExecLo(IR::U32(IR::Value(1u))); + entry_ir.SetExecHi(IR::U32(IR::Value(0u))); if (source.stage == ShaderType::Compute) { const auto* cs = compute_input_info; const auto thread_ids = @@ -930,8 +931,7 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, } } for (size_t index = 0; index < source.blocks.size(); index++) { - Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size, - per_invocation_masks); + Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size); if (!translator.TranslateBlock(source.blocks[index], error) || !translator.AddBranchCondition(source.blocks[index], result.block_info[index + 1u], error)) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 7b88919f..76d630bb 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -10,9 +10,9 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { class Translator { public: Translator(IR::ValueProgram& program, IR::Block* block, uint32_t vector_limit, - uint32_t wave_size, bool per_invocation_masks) + uint32_t wave_size) : value_program(program), ir(block), current_vector_limit(vector_limit), - current_wave_size(wave_size), current_per_invocation_masks(per_invocation_masks) {} + current_wave_size(wave_size) {} bool TranslateBlock(const IR::BasicBlock& source, std::string* error); bool AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info, @@ -28,7 +28,7 @@ private: IR::U32 ReadScalarCode(uint32_t code); IR::U32 ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value); IR::Value ReadOperand(const IR::Operand& operand, IR::Type type); - IR::U1 ThreadBit(IR::U32 low, IR::U32 high); + IR::U1 ThreadBit(IR::U32 low); void WriteRawU32(const IR::Operand& operand, IR::U32 value); IR::F32 ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value); void WriteOperand(const IR::Operand& operand, IR::Value value); @@ -38,11 +38,10 @@ private: IR::U32 ReadU32(const IR::Operand& operand); std::array ReadU32Pair(const IR::Operand& operand); IR::U64 ReadU64(const IR::Operand& operand); - IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, - bool packed = false); - IR::F32 ReadF16AsF32(const IR::Operand& operand); - IR::F32 ReadMixF32(const IR::Operand& operand); - IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane); + IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed = false); + IR::F32 ReadF16AsF32(const IR::Operand& operand); + IR::F32 ReadMixF32(const IR::Operand& operand); + IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane); IR::U32 ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend); IR::U32 ReadU16AsU32(const IR::Operand& operand, bool sign_extend); IR::U32 ReadF16LaneBits(const IR::Operand& operand, bool high_lane); @@ -51,6 +50,7 @@ private: IR::U1 ReadCondition(const IR::Operand& operand); IR::U32 ConditionBit(const IR::Operand& operand); IR::U1 ReadMask(const IR::Operand& operand); + IR::U1 ReadMaskValid(const IR::Operand& operand); void WriteMask(const IR::Operand& operand, IR::U1 value); void WriteMask64(const IR::Operand& operand, IR::U1 value); void WriteCompareResult(const IR::Operand& operand, IR::U1 value); @@ -76,8 +76,8 @@ private: IR::Value GetSamplerResource(const IR::MemoryInfo& memory); IR::Value MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base); IR::Value ConstructU32x4(const IR::Operand& base, uint32_t count); - void WriteImageComponents(const IR::Operand& dst, IR::Value value, - const IR::MemoryInfo& memory, uint32_t component_limit); + void WriteImageComponents(const IR::Operand& dst, IR::Value value, const IR::MemoryInfo& memory, + uint32_t component_limit); IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode); BufferAddress ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset); IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign); @@ -101,35 +101,34 @@ private: uint32_t upper_result); IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high); - bool TranslateInstruction(const IR::Instruction& inst); - bool TranslateStateOperation(const IR::Instruction& inst); - bool TranslateControlOperation(const IR::Instruction& inst); - bool TranslateMove(const IR::Instruction& inst); - bool TranslateLaneOperation(const IR::Instruction& inst); - bool TranslateAttributeOperation(const IR::Instruction& inst); - bool TranslateMemoryOperation(const IR::Instruction& inst); - bool TranslateIntegerCompare(const IR::Instruction& inst); - bool TranslateInteger16Compare(const IR::Instruction& inst); - bool TranslateFloatCompare(const IR::Instruction& inst); - bool TranslateConversion(const IR::Instruction& inst); - bool TranslateInteger16Operation(const IR::Instruction& inst); - bool TranslatePackedInteger16(const IR::Instruction& inst); - bool TranslatePackedFloat16(const IR::Instruction& inst); - bool TranslateFloat16Operation(const IR::Instruction& inst); - bool TranslateFloatOperation(const IR::Instruction& inst); - bool TranslatePerInvocationU64Mask(const IR::Instruction& inst); - bool TranslateU64MaskOperation(const IR::Instruction& inst); - bool TranslateSimpleInteger(const IR::Instruction& inst); - bool TranslateComposedInteger(const IR::Instruction& inst); - bool TranslateExtendedInteger(const IR::Instruction& inst); + bool TranslateInstruction(const IR::Instruction& inst); + bool TranslateStateOperation(const IR::Instruction& inst); + bool TranslateControlOperation(const IR::Instruction& inst); + bool TranslateMove(const IR::Instruction& inst); + bool TranslateLaneOperation(const IR::Instruction& inst); + bool TranslateAttributeOperation(const IR::Instruction& inst); + bool TranslateMemoryOperation(const IR::Instruction& inst); + bool TranslateIntegerCompare(const IR::Instruction& inst); + bool TranslateInteger16Compare(const IR::Instruction& inst); + bool TranslateFloatCompare(const IR::Instruction& inst); + bool TranslateConversion(const IR::Instruction& inst); + bool TranslateInteger16Operation(const IR::Instruction& inst); + bool TranslatePackedInteger16(const IR::Instruction& inst); + bool TranslatePackedFloat16(const IR::Instruction& inst); + bool TranslateFloat16Operation(const IR::Instruction& inst); + bool TranslateFloatOperation(const IR::Instruction& inst); + IR::U1 EvaluateU64Mask(const IR::Instruction& inst); + bool TranslateU64MaskOperation(const IR::Instruction& inst); + bool TranslateSimpleInteger(const IR::Instruction& inst); + bool TranslateComposedInteger(const IR::Instruction& inst); + bool TranslateExtendedInteger(const IR::Instruction& inst); IR::ValueProgram& value_program; IR::IREmitter ir; - IR::Opcode current_opcode = IR::Opcode::ControlNop; - uint32_t current_pc = 0; - uint32_t current_vector_limit = 1; - uint32_t current_wave_size = 64; - bool current_per_invocation_masks = false; + IR::Opcode current_opcode = IR::Opcode::ControlNop; + uint32_t current_pc = 0; + uint32_t current_vector_limit = 1; + uint32_t current_wave_size = 64; }; } // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail diff --git a/src/graphics/shader/recompiler/ir/Block.h b/src/graphics/shader/recompiler/ir/Block.h index 33edbd33..fe778cd6 100644 --- a/src/graphics/shader/recompiler/ir/Block.h +++ b/src/graphics/shader/recompiler/ir/Block.h @@ -49,6 +49,7 @@ public: std::array ssa_sreg_values {}; std::array ssa_thread_bit_sreg_values {}; + std::array ssa_sreg_mask_tags {}; std::array ssa_vreg_values {}; private: diff --git a/src/graphics/shader/recompiler/ir/IREmitter.cpp b/src/graphics/shader/recompiler/ir/IREmitter.cpp index 8c714f91..50db3c27 100644 --- a/src/graphics/shader/recompiler/ir/IREmitter.cpp +++ b/src/graphics/shader/recompiler/ir/IREmitter.cpp @@ -36,6 +36,14 @@ void IREmitter::SetThreadBitScalarReg(ScalarReg reg, U1 value) { Emit(ValueOpcode::SetThreadBitScalarRegister, {Value(reg), value}); } +U1 IREmitter::GetScalarMaskTag(ScalarReg reg) { + return U1(Emit(ValueOpcode::GetScalarMaskTag, {Value(reg)})); +} + +void IREmitter::SetScalarMaskTag(ScalarReg reg, U1 value) { + Emit(ValueOpcode::SetScalarMaskTag, {Value(reg), value}); +} + U32 IREmitter::GetVectorReg(VectorReg reg) { return U32(Emit(ValueOpcode::GetVectorRegister, {Value(reg)})); } diff --git a/src/graphics/shader/recompiler/ir/IREmitter.h b/src/graphics/shader/recompiler/ir/IREmitter.h index beef3cab..e429ac04 100644 --- a/src/graphics/shader/recompiler/ir/IREmitter.h +++ b/src/graphics/shader/recompiler/ir/IREmitter.h @@ -29,6 +29,8 @@ public: void SetScalarReg(ScalarReg reg, U32 value); U1 GetThreadBitScalarReg(ScalarReg reg); void SetThreadBitScalarReg(ScalarReg reg, U1 value); + U1 GetScalarMaskTag(ScalarReg reg); + void SetScalarMaskTag(ScalarReg reg, U1 value); U32 GetVectorReg(VectorReg reg); void SetVectorReg(VectorReg reg, U32 value); U1 GetGotoVariable(uint32_t id); diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.cpp b/src/graphics/shader/recompiler/ir/ShaderIR.cpp index 50dcfdba..b744e906 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIR.cpp @@ -44,6 +44,23 @@ bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBloc if (!ScalarResultWritesSccNonZero(decoded.opcode)) { return true; } + switch (decoded.opcode) { + case Decoder::Opcode::S_AND_B64: + case Decoder::Opcode::S_ANDN2_B64: + case Decoder::Opcode::S_NOT_B64: + case Decoder::Opcode::S_OR_B64: + case Decoder::Opcode::S_ORN2_B64: + case Decoder::Opcode::S_XOR_B64: + case Decoder::Opcode::S_NAND_B64: + case Decoder::Opcode::S_NOR_B64: + case Decoder::Opcode::S_XNOR_B64: + case Decoder::Opcode::S_WQM_B64: + // These instructions maintain an invocation-local mask shadow and update SCC + // while lowering the operation. A raw dword comparison would disagree after + // complementing a mask temporary. + return true; + default: break; + } if (block.instructions.empty()) { SetError(error, "internal IR error: missing scalar result for SCC update"); return false; @@ -206,14 +223,14 @@ uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand) { MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind) { MemoryInfo mem; - mem.kind = kind; - mem.offset = decoded.offset; - mem.secondary_offset = decoded.secondary_offset; - mem.dmask = decoded.dmask; - mem.data_dwords = decoded.data_dwords; - mem.data_bits = decoded.data_bits; - mem.component_count = decoded.data_components != 0u ? decoded.data_components - : decoded.data_dwords; + mem.kind = kind; + mem.offset = decoded.offset; + mem.secondary_offset = decoded.secondary_offset; + mem.dmask = decoded.dmask; + mem.data_dwords = decoded.data_dwords; + mem.data_bits = decoded.data_bits; + mem.component_count = + decoded.data_components != 0u ? decoded.data_components : decoded.data_dwords; mem.data_format = decoded.data_format; mem.number_format = decoded.number_format; mem.image_sample_flags = decoded.image_sample_flags; @@ -224,9 +241,9 @@ MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKi mem.image_nsa_addr[i] = decoded.image_nsa_addr[i]; } mem.memory_segment = decoded.memory_segment; - mem.address_is_full = kind == ResourceKind::Flat || - (kind == ResourceKind::Global && - decoded.src1.kind == Decoder::OperandKind::Vgpr); + mem.address_is_full = + kind == ResourceKind::Flat || + (kind == ResourceKind::Global && decoded.src1.kind == Decoder::OperandKind::Vgpr); mem.data_signed = decoded.data_signed; mem.typed = decoded.typed; mem.formatted = decoded.formatted; diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index e7730b1a..17688c83 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -456,7 +456,6 @@ struct ShaderInfo { }; struct SpirvRequirements { - bool requires_exact_subgroup = false; bool subgroup_ballot = false; bool subgroup_shuffle = false; bool subgroup_local_invocation_id = false; @@ -469,7 +468,6 @@ struct SpirvRequirements { struct Program { ShaderType stage = ShaderType::Unknown; - ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave; uint64_t shader_hash = 0; uint32_t wave_size = 64; uint32_t user_data_base = 0; diff --git a/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp b/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp index 8dfb2b74..6e4afe05 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp @@ -192,10 +192,8 @@ std::string ExportTargetKindToString(ExportTargetKind kind) { std::string ProgramToString(const Program& program) { std::string text; - text += fmt::format("mode={} masks={} fallback={} reason={}\n", + text += fmt::format("mode={} fallback={} reason={}\n", program.dispatcher_fallback ? "dispatcher" : "structured", - program.lane_mask_mode == ShaderLaneMaskMode::NativeWave ? "native-wave" - : "per-invocation", CFG::FailureKindToString(program.cfg_failure_kind).c_str(), program.fallback_reason.c_str()); for (const auto& block: program.blocks) { diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.cpp b/src/graphics/shader/recompiler/ir/ValueProgram.cpp index 000b1e4c..0c0ea261 100644 --- a/src/graphics/shader/recompiler/ir/ValueProgram.cpp +++ b/src/graphics/shader/recompiler/ir/ValueProgram.cpp @@ -17,6 +17,8 @@ bool IsRegisterStatePseudo(ValueOpcode opcode) { switch (opcode) { case ValueOpcode::GetThreadBitScalarRegister: case ValueOpcode::SetThreadBitScalarRegister: + case ValueOpcode::GetScalarMaskTag: + case ValueOpcode::SetScalarMaskTag: case ValueOpcode::GetScalarRegister: case ValueOpcode::SetScalarRegister: case ValueOpcode::GetVectorRegister: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 19626802..e11b6c13 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -11,6 +11,8 @@ VALUE_OPCODE(GetShaderBase, U64) VALUE_OPCODE(GetBuiltin, U32, U32, U32) VALUE_OPCODE(GetThreadBitScalarRegister, U1, ScalarReg) VALUE_OPCODE(SetThreadBitScalarRegister, Void, ScalarReg, U1) +VALUE_OPCODE(GetScalarMaskTag, U1, ScalarReg) +VALUE_OPCODE(SetScalarMaskTag, Void, ScalarReg, U1) VALUE_OPCODE(GetScalarRegister, U32, ScalarReg) VALUE_OPCODE(SetScalarRegister, Void, ScalarReg, U32) VALUE_OPCODE(GetVectorRegister, U32, VectorReg) diff --git a/src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp b/src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp index 96c1e521..ce89b900 100644 --- a/src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp @@ -15,6 +15,10 @@ struct ThreadBitScalarReg { ScalarReg reg {}; auto operator<=>(const ThreadBitScalarReg&) const = default; }; +struct ScalarMaskTag { + ScalarReg reg {}; + auto operator<=>(const ScalarMaskTag&) const = default; +}; struct ExecTag { auto operator<=>(const ExecTag&) const = default; }; @@ -41,8 +45,9 @@ struct GotoVariable { auto operator<=>(const GotoVariable&) const = default; }; -using Variable = std::variant; +using Variable = + std::variant; using ValueMap = std::unordered_map; struct DefTable { @@ -66,6 +71,14 @@ struct DefTable { EXIT_IF(RegIndex(value.reg) >= NumScalarRegs); block->ssa_thread_bit_sreg_values[RegIndex(value.reg)] = definition; } + const Value& Get(Block* block, ScalarMaskTag value) { + EXIT_IF(RegIndex(value.reg) >= NumScalarRegs); + return block->ssa_sreg_mask_tags[RegIndex(value.reg)]; + } + void Set(Block* block, ScalarMaskTag value, Value definition) { + EXIT_IF(RegIndex(value.reg) >= NumScalarRegs); + block->ssa_sreg_mask_tags[RegIndex(value.reg)] = definition; + } const Value& Get(Block* block, VectorReg reg) { EXIT_IF(RegIndex(reg) >= NumVectorRegs); @@ -116,6 +129,9 @@ ValueOpcode UndefOpcode(ScalarReg) { ValueOpcode UndefOpcode(ThreadBitScalarReg) { return ValueOpcode::UndefU1; } +ValueOpcode UndefOpcode(ScalarMaskTag) { + return ValueOpcode::UndefU1; +} ValueOpcode UndefOpcode(VectorReg) { return ValueOpcode::UndefU32; } @@ -153,6 +169,9 @@ Value InitialValue(ScalarReg) { Value InitialValue(ThreadBitScalarReg) { return Value(false); } +Value InitialValue(ScalarMaskTag) { + return Value(false); +} Value InitialValue(VectorReg) { return Value(0u); } @@ -321,6 +340,9 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) { case ValueOpcode::SetThreadBitScalarRegister: pass.Write(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block, inst.Arg(1)); break; + case ValueOpcode::SetScalarMaskTag: + pass.Write(ScalarMaskTag {inst.Arg(0).ScalarRegister()}, block, inst.Arg(1)); + break; case ValueOpcode::SetVectorRegister: pass.Write(inst.Arg(0).VectorRegister(), block, inst.Arg(1)); break; @@ -342,6 +364,9 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) { inst.ReplaceUsesWith( pass.Read(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block)); break; + case ValueOpcode::GetScalarMaskTag: + inst.ReplaceUsesWith(pass.Read(ScalarMaskTag {inst.Arg(0).ScalarRegister()}, block)); + break; case ValueOpcode::GetVectorRegister: inst.ReplaceUsesWith(pass.Read(inst.Arg(0).VectorRegister(), block)); break; diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index ff74e59c..01dc6fa0 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -112,16 +112,14 @@ static std::unique_ptr> g_shader_ static std::mutex g_shader_map_mutex; struct ShaderStageProgramKey { - ShaderType stage = ShaderType::Unknown; - ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave; - uint64_t shader_hash = 0; + ShaderType stage = ShaderType::Unknown; + uint64_t shader_hash = 0; ShaderId program_id; Config::ShaderOptimizationType optimization_type = Config::ShaderOptimizationType::None; bool operator==(const ShaderStageProgramKey& other) const { - return stage == other.stage && lane_mask_mode == other.lane_mask_mode && - shader_hash == other.shader_hash && program_id == other.program_id && - optimization_type == other.optimization_type; + return stage == other.stage && shader_hash == other.shader_hash && + program_id == other.program_id && optimization_type == other.optimization_type; } }; @@ -133,7 +131,6 @@ struct ShaderStageProgramKeyHash { }; mix(static_cast(key.stage)); - mix(static_cast(key.lane_mask_mode)); mix(Common::hash64(key.shader_hash)); mix(key.program_id.hash0); mix(key.program_id.crc32); @@ -920,11 +917,9 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs, } static ShaderStageProgramKey MakeShaderStageProgramKey(ShaderType stage, uint64_t shader_hash, - const ShaderId& program_id, - ShaderLaneMaskMode lane_mask_mode) { + const ShaderId& program_id) { ShaderStageProgramKey key {}; key.stage = stage; - key.lane_mask_mode = lane_mask_mode; key.shader_hash = shader_hash; key.program_id = program_id; key.optimization_type = Config::GetShaderOptimizationType(); @@ -1055,7 +1050,6 @@ static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Prog static void ShaderAppendNativeSpecialization(std::vector& ids, const ShaderRecompiler::IR::Program& program) { EXIT_IF(!program.binding_layout_complete); - ids.push_back(static_cast(program.lane_mask_mode)); ids.push_back(program.bindings.descriptor_set); ids.push_back(program.bindings.push_constant_offset); ids.push_back(program.bindings.push_constant_size); @@ -1136,8 +1130,7 @@ static std::span AddShaderProgramPermutation(const char* stage, } bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& info, - std::span& spirv) { + ShaderVertexInputInfo& info, std::span& spirv) { spirv = {}; if (!ShaderGetStaticInputInfoVS(regs, sh, info)) { @@ -1145,8 +1138,7 @@ bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegis } const auto shader_hash = regs.gs_regs.chksum; const auto program_id = ShaderGetIdVS(regs, info, false); - const auto key = - MakeShaderStageProgramKey(ShaderType::Vertex, shader_hash, program_id, lane_mask_mode); + const auto key = MakeShaderStageProgramKey(ShaderType::Vertex, shader_hash, program_id); { std::scoped_lock lock(g_shader_program_cache_mutex); @@ -1163,7 +1155,7 @@ bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegis } std::vector compiled_spirv; - if (!ShaderCompileSpirvVS(regs, sh, lane_mask_mode, info, compiled_spirv)) { + if (!ShaderCompileSpirvVS(regs, sh, info, compiled_spirv)) { return false; } @@ -1175,7 +1167,7 @@ bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegis } bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderLaneMaskMode lane_mask_mode, const ShaderVertexInputInfo& vs_info, + const ShaderVertexInputInfo& vs_info, std::span target_export_mapping, ShaderPixelInputInfo& ps_info, std::span& spirv) { spirv = {}; @@ -1184,8 +1176,7 @@ bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegist const auto shader_hash = regs.ps_regs.chksum != 0 ? regs.ps_regs.chksum : regs.ps_regs.data_addr; const auto program_id = ShaderGetIdPS(regs, ps_info, false); - const auto key = - MakeShaderStageProgramKey(ShaderType::Pixel, shader_hash, program_id, lane_mask_mode); + const auto key = MakeShaderStageProgramKey(ShaderType::Pixel, shader_hash, program_id); { std::scoped_lock lock(g_shader_program_cache_mutex); @@ -1202,7 +1193,7 @@ bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegist } std::vector compiled_spirv; - if (!ShaderCompileSpirvPS(regs, sh, lane_mask_mode, ps_info, compiled_spirv)) { + if (!ShaderCompileSpirvPS(regs, sh, ps_info, compiled_spirv)) { return false; } @@ -1220,8 +1211,7 @@ bool ShaderCompileInfoCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegi ShaderGetStaticInputInfoCS(regs, sh, info); const auto shader_hash = regs.cs_regs.data_addr; const auto program_id = ShaderGetIdCS(regs, info, false); - const auto key = MakeShaderStageProgramKey(ShaderType::Compute, shader_hash, program_id, - ShaderLaneMaskMode::NativeWave); + const auto key = MakeShaderStageProgramKey(ShaderType::Compute, shader_hash, program_id); { std::scoped_lock lock(g_shader_program_cache_mutex); @@ -1449,8 +1439,7 @@ static void DumpShaderRecompilerOriginal(const char* type, uint64_t shader_hash, } bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& input_info, - std::vector& spirv) { + ShaderVertexInputInfo& input_info, std::vector& spirv) { KYTY_PROFILER_FUNCTION(profiler::colors::Amber300); EXIT_NOT_IMPLEMENTED(regs.es_regs.data_addr == 0 || regs.gs_regs.chksum == 0); @@ -1460,7 +1449,6 @@ bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegi ShaderRecompiler::CompileOptions options; options.stage = ShaderType::Vertex; - options.lane_mask_mode = lane_mask_mode; options.shader_hash = regs.gs_regs.chksum; options.shader_base = shader_addr; options.user_data_base = 8; @@ -1506,8 +1494,7 @@ bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegi } bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderLaneMaskMode lane_mask_mode, ShaderPixelInputInfo& input_info, - std::vector& spirv) { + ShaderPixelInputInfo& input_info, std::vector& spirv) { KYTY_PROFILER_FUNCTION(profiler::colors::Blue300); const uint64_t shader_addr = regs.ps_regs.data_addr; @@ -1516,7 +1503,6 @@ bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegis ShaderRecompiler::CompileOptions options; options.stage = ShaderType::Pixel; - options.lane_mask_mode = lane_mask_mode; options.shader_hash = shader_hash; options.shader_base = shader_addr; options.user_data_count = regs.ps_regs.rsrc2.user_sgpr; diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index bfc244f0..7b950616 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -35,8 +35,6 @@ struct ShaderRegisters; enum class ShaderType { Unknown, Vertex, Pixel, Fetch, Compute }; -enum class ShaderLaneMaskMode { NativeWave, PerInvocation }; - namespace ShaderRecompiler::IR { struct Program; struct ResourceSnapshot; @@ -261,20 +259,17 @@ ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo& regs, const ShaderComputeInp bool include_bind_specialization); // Returned SPIR-V spans are read-only views backed by the shader program cache. bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& input_info, - std::span& spirv); + ShaderVertexInputInfo& input_info, std::span& spirv); bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderLaneMaskMode lane_mask_mode, const ShaderVertexInputInfo& vs_info, + const ShaderVertexInputInfo& vs_info, std::span target_export_mapping, ShaderPixelInputInfo& input_info, std::span& spirv); bool ShaderCompileInfoCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, ShaderComputeInputInfo& input_info, std::span& spirv); bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& input_info, - std::vector& spirv); + ShaderVertexInputInfo& input_info, std::vector& spirv); bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderLaneMaskMode lane_mask_mode, ShaderPixelInputInfo& input_info, - std::vector& spirv); + ShaderPixelInputInfo& input_info, std::vector& spirv); bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, ShaderComputeInputInfo& input_info, std::vector& spirv); bool ShaderAddressValid(uint64_t addr); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index f05dea01..e4658679 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -574,15 +574,14 @@ constexpr u32 EncodeMimg0(u32 opcode, u32 dmask, u32 nsa_dwords = 0, return (0x3cu << 26u) | ((opcode >> 7u) & 0x1u) | ((nsa_dwords & 0x3u) << 1u) | ((dim & 0x7u) << 3u) | ((dmask & 0xfu) << 8u) | (glc ? (1u << 13u) : 0u) | - (r128 ? (1u << 15u) : 0u) | - ((opcode & 0x7fu) << 18u); + (r128 ? (1u << 15u) : 0u) | ((opcode & 0x7fu) << 18u); } constexpr u32 EncodeMimg1(u32 vdata, u32 vaddr, u32 srsrc = 0, u32 ssamp = 0, bool a16 = false, bool d16 = false) { return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) | - ((vdata & 0xffu) << 8u) | (vaddr & 0xffu) | - (a16 ? (1u << 30u) : 0u) | (d16 ? (1u << 31u) : 0u); + ((vdata & 0xffu) << 8u) | (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u) | + (d16 ? (1u << 31u) : 0u); } constexpr u32 EncodeVintrp(u32 opcode, u32 dst, u32 attr, u32 chan, u32 src) { @@ -1178,7 +1177,8 @@ CompiledShader CompileCase(const TestCase &test) { result.resources.user_data[reg - result.program.user_data_base]); } packed_user_data.resize(result.program.bindings.ShaderDataDwords()); - const auto buffer_count = static_cast(result.program.info.buffers.size()); + const auto buffer_count = + static_cast(result.program.info.buffers.size()); for (u32 i = 0; i < buffer_count; i++) { u32 offset = 0; if (i < test.storage_buffer_offsets.size()) { @@ -1197,8 +1197,8 @@ CompiledShader CompileCase(const TestCase &test) { Require(test.name, "shader data", offset < 256, "address-memory offset is not representable"); const auto index = buffer_count + i; - packed_user_data[result.program.bindings.memory_offset_dword + index / 4u] |= - offset << ((index % 4u) * 8u); + packed_user_data[result.program.bindings.memory_offset_dword + + index / 4u] |= offset << ((index % 4u) * 8u); } return {std::move(result.spirv), std::move(result.program), std::move(result.resources.flattened_srt), @@ -3562,8 +3562,7 @@ public: narrow_target.info.tile_mode = Prospero::TileMode::kRenderTarget; narrow_target.info.mip_layout[0] = {0, 0x10000, 128, 1}; narrow_target.view_info.format = narrow_target.info.pixel_format; - narrow_target.view_info.usage = - vk::ImageUsageFlagBits::eColorAttachment; + narrow_target.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; const auto narrow_target_id = texture_cache.FindImage(narrow_target); const auto narrow_target_view = texture_cache.FindRenderTarget(narrow_target_id, narrow_target); @@ -3868,14 +3867,13 @@ public: std::memcpy(memory + raw_d16_offset, raw_d16_values.data(), sizeof(raw_d16_values)); auto raw_d16 = MakeLinearDesc( - base + raw_d16_offset, sizeof(raw_d16_values), - vk::Format::eD16Unorm, Prospero::BufferFormat::k16UNorm, - Prospero::ImageType::kColor2D, {2, 1, 1}, 1, sizeof(uint16_t), 1); + base + raw_d16_offset, sizeof(raw_d16_values), vk::Format::eD16Unorm, + Prospero::BufferFormat::k16UNorm, Prospero::ImageType::kColor2D, + {2, 1, 1}, 1, sizeof(uint16_t), 1); raw_d16.type = BindingType::DepthTarget; raw_d16.view_info.format = vk::Format::eD16Unorm; raw_d16.view_info.aspect = vk::ImageAspectFlagBits::eDepth; - raw_d16.view_info.usage = - vk::ImageUsageFlagBits::eDepthStencilAttachment; + raw_d16.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; const auto raw_d16_image = texture_cache.FindImage(raw_d16); (void)texture_cache.FindDepthTarget(raw_d16_image, raw_d16); @@ -3887,13 +3885,13 @@ public: raw_d16_uint.view_info.aspect = vk::ImageAspectFlagBits::eColor; raw_d16_uint.view_info.usage = vk::ImageUsageFlagBits::eSampled; const auto raw_d16_uint_image = texture_cache.FindImage(raw_d16_uint); - Require(name, "raw D16 uint texture backing", - raw_d16_uint_image && raw_d16_uint_image != raw_d16_image && - !TextureCacheTestAccess::Contains(texture_cache, - raw_d16_image) && - texture_cache.GetImage(raw_d16_uint_image) - .backing.format == vk::Format::eR16Uint, - "D16 uint sampling did not recreate a true integer color backing"); + Require( + name, "raw D16 uint texture backing", + raw_d16_uint_image && raw_d16_uint_image != raw_d16_image && + !TextureCacheTestAccess::Contains(texture_cache, raw_d16_image) && + texture_cache.GetImage(raw_d16_uint_image).backing.format == + vk::Format::eR16Uint, + "D16 uint sampling did not recreate a true integer color backing"); Require(name, "raw D16 uint texture view", texture_cache.FindTexture(raw_d16_uint_image, raw_d16_uint) != nullptr, @@ -3916,9 +3914,9 @@ public: layered_d16_color); auto &layered_d16_color_native = texture_cache.GetImage(layered_d16_color_image); - layered_d16_color_native.Transit( - vk::ImageLayout::eTransferDstOptimal, - vk::AccessFlagBits2::eTransferWrite, {}, scheduler.Current().Handle()); + layered_d16_color_native.Transit(vk::ImageLayout::eTransferDstOptimal, + vk::AccessFlagBits2::eTransferWrite, {}, + scheduler.Current().Handle()); vk::ClearColorValue layered_raw_d16_clear_value{}; layered_raw_d16_clear_value.float32[0] = 1.0f; const vk::ImageSubresourceRange layered_raw_d16_clear_range{ @@ -3931,8 +3929,7 @@ public: auto layered_raw_d16_depth = layered_d16_color; layered_raw_d16_depth.type = BindingType::DepthTarget; - layered_raw_d16_depth.info.data.size = - sizeof(uint16_t) * 2 * 5; + layered_raw_d16_depth.info.data.size = sizeof(uint16_t) * 2 * 5; layered_raw_d16_depth.info.pixel_format = vk::Format::eD16Unorm; layered_raw_d16_depth.info.guest_format = Prospero::BufferFormat::k16UNorm; @@ -3940,8 +3937,7 @@ public: layered_raw_d16_depth.info.mip_layout[0].size = layered_raw_d16_depth.info.data.size; layered_raw_d16_depth.view_info.format = vk::Format::eD16Unorm; - layered_raw_d16_depth.view_info.aspect = - vk::ImageAspectFlagBits::eDepth; + layered_raw_d16_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth; layered_raw_d16_depth.view_info.layer_count = 5; layered_raw_d16_depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; @@ -3949,28 +3945,25 @@ public: texture_cache.FindImage(layered_raw_d16_depth); const auto &layered_depth_owner = texture_cache.GetImage(layered_raw_d16_depth_image); - Require( - name, "layered raw D16 owner layout", - layered_raw_d16_depth_image != layered_d16_color_image && - !TextureCacheTestAccess::Contains( - texture_cache, layered_d16_color_image) && - layered_depth_owner.info.resources.layers == 6 && - layered_depth_owner.info.data.size == - sizeof(layered_raw_d16_values) && - layered_depth_owner.info.mip_layout[0].size == - sizeof(layered_raw_d16_values), - "a partial depth view detached the retained six-layer owner from " - "its full guest range"); + Require(name, "layered raw D16 owner layout", + layered_raw_d16_depth_image != layered_d16_color_image && + !TextureCacheTestAccess::Contains(texture_cache, + layered_d16_color_image) && + layered_depth_owner.info.resources.layers == 6 && + layered_depth_owner.info.data.size == + sizeof(layered_raw_d16_values) && + layered_depth_owner.info.mip_layout[0].size == + sizeof(layered_raw_d16_values), + "a partial depth view detached the retained six-layer owner from " + "its full guest range"); (void)texture_cache.FindDepthTarget(layered_raw_d16_depth_image, layered_raw_d16_depth); auto reacquired_layered_raw_d16_uint = layered_d16_color; - reacquired_layered_raw_d16_uint.info.pixel_format = - vk::Format::eR16Uint; + reacquired_layered_raw_d16_uint.info.pixel_format = vk::Format::eR16Uint; reacquired_layered_raw_d16_uint.info.guest_format = Prospero::BufferFormat::k16UInt; - reacquired_layered_raw_d16_uint.view_info.format = - vk::Format::eR16Uint; + reacquired_layered_raw_d16_uint.view_info.format = vk::Format::eR16Uint; const auto reacquired_layered_raw_d16_uint_image = texture_cache.FindImage(reacquired_layered_raw_d16_uint); const auto reacquired_layered_raw_d16_uint_view = @@ -3986,13 +3979,13 @@ public: vk::AccessFlagBits2::eTransferRead, {}, scheduler.Current().Handle()); vk::BufferImageCopy layered_raw_d16_copy{}; layered_raw_d16_copy.bufferRowLength = 2; - layered_raw_d16_copy.imageSubresource = { - vk::ImageAspectFlagBits::eColor, 0, 0, 6}; + layered_raw_d16_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, + 0, 0, 6}; layered_raw_d16_copy.imageExtent = {2, 1, 1}; scheduler.Current().Handle().copyImageToBuffer( reacquired_layered_raw_d16_native.backing.image, - vk::ImageLayout::eTransferSrcOptimal, - layered_raw_d16_readback.buffer, 1, &layered_raw_d16_copy); + vk::ImageLayout::eTransferSrcOptimal, layered_raw_d16_readback.buffer, + 1, &layered_raw_d16_copy); HostReadBarrier(layered_raw_d16_readback.buffer, layered_raw_d16_readback.size, vk::PipelineStageFlagBits::eTransfer, @@ -4010,8 +4003,8 @@ public: reacquired_layered_raw_d16_uint_image && reacquired_layered_raw_d16_uint_image != layered_raw_d16_depth_image && - !TextureCacheTestAccess::Contains( - texture_cache, layered_raw_d16_depth_image) && + !TextureCacheTestAccess::Contains(texture_cache, + layered_raw_d16_depth_image) && reacquired_layered_raw_d16_native.backing.format == vk::Format::eR16Uint && reacquired_layered_raw_d16_uint_view != nullptr && @@ -7717,14 +7710,13 @@ public: RenderDepthInfo read_only_clear_depth{}; RenderExecutorTestAccess::ResolveRenderDepthTarget( executor, 1, scheduler.Current(), read_only_clear_depth); - Require( - name, "read-only stencil clear suppression", - read_only_clear_depth.image_id == phased_depth.image_id && - !read_only_clear_depth.stencil_clear_enable && - !read_only_clear_depth.AttachmentWriteAspects() && - depth_attachment_layout(read_only_clear_depth) == - vk::ImageLayout::eDepthStencilReadOnlyOptimal, - "a PS5 target-level stencil write disable retained a host clear"); + Require(name, "read-only stencil clear suppression", + read_only_clear_depth.image_id == phased_depth.image_id && + !read_only_clear_depth.stencil_clear_enable && + !read_only_clear_depth.AttachmentWriteAspects() && + depth_attachment_layout(read_only_clear_depth) == + vk::ImageLayout::eDepthStencilReadOnlyOptimal, + "a PS5 target-level stencil write disable retained a host clear"); registers.SetDepthRenderTarget(phased_depth_target); registers.SetDepthControl(phased_depth_control); @@ -9972,21 +9964,21 @@ public: const auto surface_format = TextureGetSurfaceFormatInfo(format); constexpr u32 tail_xy[7][2] = {{64, 0}, {0, 64}, {32, 0}, {0, 32}, {16, 0}, {8, 16}, {0, 24}}; - bool valid = - built && Prospero::NumBytesPerElement(format) == 4 && - Prospero::BlockCompressedBytesPerBlock(format) == 0 && - Prospero::IsSampledTextureFormat(format) && - Prospero::IsUintTextureFormat(format) && - Prospero::RemapTextureFormat(format) == - Prospero::BufferFormat::k32UInt && - surface_format.vk_format == vk::Format::eR32Uint && - surface_format.conversion_format == format && - VulkanFormat(format) == vk::Format::eUndefined && - surface.texture.block.block_width == 128 && - surface.texture.block.block_height == 128 && - surface.texture.block.block_depth == 1 && - surface.first_tail_level == 1 && surface.mips[0].offset == 0x10000 && - total.size == 0x20000 && total.align == 0x10000; + bool valid = built && Prospero::NumBytesPerElement(format) == 4 && + Prospero::BlockCompressedBytesPerBlock(format) == 0 && + Prospero::IsSampledTextureFormat(format) && + Prospero::IsUintTextureFormat(format) && + Prospero::RemapTextureFormat(format) == + Prospero::BufferFormat::k32UInt && + surface_format.vk_format == vk::Format::eR32Uint && + surface_format.conversion_format == format && + VulkanFormat(format) == vk::Format::eUndefined && + surface.texture.block.block_width == 128 && + surface.texture.block.block_height == 128 && + surface.texture.block.block_depth == 1 && + surface.first_tail_level == 1 && + surface.mips[0].offset == 0x10000 && total.size == 0x20000 && + total.align == 0x10000; for (u32 level = 1; level < levels && valid; level++) { valid &= surface.mips[level].offset == 0 && surface.mips[level].tail_x == tail_xy[level - 1][0] && @@ -9999,16 +9991,15 @@ public: const auto storage_only_format = Prospero::BufferFormat::k32SInt; const auto storage_only_surface = TextureGetSurfaceFormatInfo(storage_only_format); - Require( - name, "storage-only surface mapping", - !Prospero::IsSampledTextureFormat(storage_only_format) && - !Prospero::IsUintTextureFormat(storage_only_format) && - Prospero::RemapTextureFormat(storage_only_format) == - storage_only_format && - storage_only_surface.vk_format == vk::Format::eR32Sint && - storage_only_surface.conversion_format == - Prospero::BufferFormat::kInvalid, - "storage-only native backing was conflated with sampled support"); + Require(name, "storage-only surface mapping", + !Prospero::IsSampledTextureFormat(storage_only_format) && + !Prospero::IsUintTextureFormat(storage_only_format) && + Prospero::RemapTextureFormat(storage_only_format) == + storage_only_format && + storage_only_surface.vk_format == vk::Format::eR32Sint && + storage_only_surface.conversion_format == + Prospero::BufferFormat::kInvalid, + "storage-only native backing was conflated with sampled support"); } { @@ -11997,18 +11988,26 @@ TestCase ScalarNotB64UpdatesScc() { code.push_back(EncodeSop1(0x08, 8, 6)); capture_scc(10); + code.push_back(EncodeVopc(0xc2, Vgpr(0), 0)); + code.push_back(EncodeSop1(0x04, 12, 106)); + code.push_back(EncodeSop1(0x08, 14, 12)); + capture_scc(16); + AppendStoreSgpr(&code, 4, 0); AppendStoreSgpr(&code, 10, 1); AppendStoreSgprPair(&code, 2, 2); AppendStoreSgprPair(&code, 8, 4); + AppendStoreSgpr(&code, 16, 6); + AppendStoreSgprPair(&code, 14, 7); AppendEnd(&code); return {"ScalarNotB64UpdatesScc", code, {}, - {0, 1, 0, 0, 1, 0}, + {0, 1, 0, 0, 1, 0, 0, 0xfffffffeu, 0xffffffffu}, {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_NOT_B64, O::S_CSELECT_B32, - O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase ScalarQuadmaskB64() { @@ -12109,9 +12108,8 @@ TestCase ScalarSaveExecOps() { return {"ScalarSaveExecOps", code, {}, - {0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0xffffffffu, - 0xffffffffu, 0xffffffffu, 0x00000003u, 0xffffffffu, 0x00000003u, - 0x00000002u, 0xffffffffu, 1}, + // EXEC and saved EXEC values are invocation-local Booleans. + {1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0}, {O::S_MOV_B32, O::S_AND_SAVEEXEC_B64, O::S_ORN2_SAVEEXEC_B64, O::S_ANDN1_SAVEEXEC_B64, O::S_AND_SAVEEXEC_B32, O::S_ANDN1_SAVEEXEC_B32, O::S_MOV_B64, O::V_MOV_B32, @@ -12138,7 +12136,7 @@ TestCase ScalarOrn2SaveexecUsesSourceOrNotExec() { return {"ScalarOrn2SaveexecUsesSourceOrNotExec", code, {}, - {0x0000000cu, 0x80000000u, 0xfffffff3u, 0x7fffffffu, 1}, + {1, 0, 1, 0, 1}, {O::S_MOV_B32, O::S_ORN2_SAVEEXEC_B64, O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } @@ -12380,15 +12378,13 @@ TestCase Scalar64BitOps() { O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } -TestCase ScalarAndn2B64SccBranch() { +TestCase ScalarAndn2B64SccUsesMaskShadow() { using O = ShaderOpcode; std::vector code = { - EncodeSMovB32(0, InlineU32(3)), - EncodeSMovB32(1, InlineU32(0)), - EncodeSMovB32(2, InlineU32(1)), - EncodeSMovB32(3, InlineU32(0)), - EncodeSop2(0x15, 4, 0, 2), + EncodeVopc(0xc2, InlineU32(0), 0), + EncodeSop1(0x04, 0, 106), + EncodeSop2(0x15, 4, 0, InlineU32(0)), EncodeVop1(0x01, 0, InlineU32(1)), EncodeSopp(0x04, 1), EncodeVop1(0x01, 0, InlineU32(7)), @@ -12396,14 +12392,141 @@ TestCase ScalarAndn2B64SccBranch() { AppendStoreVgpr(&code, 0, 0); AppendEnd(&code); - return {"ScalarAndn2B64SccBranch", + return {"ScalarAndn2B64SccUsesMaskShadow", code, {}, {7}, - {O::S_MOV_B32, O::S_ANDN2_B64, O::V_MOV_B32, O::S_CBRANCH_SCC0, + {O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_ANDN2_B64, O::V_MOV_B32, + O::S_CBRANCH_SCC0, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; +} + +TestCase ScalarMaskHighWriteInvalidatesProvenance() { + using O = ShaderOpcode; + + std::vector code = { + EncodeVopc(0xc2, Vgpr(0), 0), EncodeSop1(0x04, 0, 106), + EncodeSMovB32(1, InlineU32(1)), EncodeSop1(0x08, 2, 0), + EncodeVop1(0x01, 0, InlineU32(1)), EncodeSopp(0x04, 1), + EncodeVop1(0x01, 0, InlineU32(7)), + }; + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + return {"ScalarMaskHighWriteInvalidatesProvenance", + code, + {}, + {7}, + {O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_MOV_B32, O::S_NOT_B64, + O::V_MOV_B32, O::S_CBRANCH_SCC0, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; +} + +TestCase ScalarSelectB64PreservesMaskProvenance() { + using O = ShaderOpcode; + + std::vector code; + const auto set_scc = [&] { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1))); + }; + const auto capture_scc = [&](u32 dst) { + code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0))); + }; + + AppendSMovLiteral(&code, 0, 0xfffffffeu); + AppendSMovLiteral(&code, 1, 0xffffffffu); + code.push_back(EncodeSMovB32(2, InlineU32(0))); + code.push_back(EncodeSMovB32(3, InlineU32(0))); + set_scc(); + code.push_back(EncodeSop2(0x0b, 4, 0, 2)); + code.push_back(EncodeSop1(0x08, 6, 4)); + capture_scc(8); + + code.push_back(EncodeVopc(0xc2, Vgpr(0), 0)); + code.push_back(EncodeSop1(0x04, 10, 106)); + set_scc(); + code.push_back(EncodeSop2(0x0b, 12, 10, 2)); + code.push_back(EncodeSop1(0x08, 14, 12)); + capture_scc(16); + + AppendStoreSgpr(&code, 8, 0); + AppendStoreSgpr(&code, 16, 1); + AppendEnd(&code); + + return {"ScalarSelectB64PreservesMaskProvenance", + code, + {}, + {1, 0}, + {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CSELECT_B64, O::S_NOT_B64, + O::S_CSELECT_B32, O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase ScalarWqmB64SelectsSccDomain() { + using O = ShaderOpcode; + + std::vector code; + const auto capture_scc = [&](u32 dst) { + code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0))); + }; + + code.push_back(EncodeSMovB32(0, InlineU32(0))); + code.push_back(EncodeSMovB32(1, InlineU32(1))); + code.push_back(EncodeSop1(0x0a, 2, 0)); + capture_scc(4); + + code.push_back(EncodeVopc(0xc2, Vgpr(0), 0)); + code.push_back(EncodeSop1(0x04, 6, 106)); + code.push_back(EncodeSop1(0x08, 8, 6)); + code.push_back(EncodeSop1(0x0a, 10, 8)); + capture_scc(12); + + AppendStoreSgpr(&code, 4, 0); + AppendStoreSgpr(&code, 12, 1); + AppendEnd(&code); + + return {"ScalarWqmB64SelectsSccDomain", + code, + {}, + {1, 0}, + {O::S_MOV_B32, O::S_WQM_B64, O::S_CSELECT_B32, O::V_CMP_EQ_U32, + O::S_MOV_B64, O::S_NOT_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; +} + +TestCase ScalarMaskProvenanceOverlapAndMixedBinary() { + using O = ShaderOpcode; + + std::vector code; + const auto capture_scc = [&](u32 dst) { + code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0))); + }; + + code.push_back(EncodeVopc(0xc2, Vgpr(0), 0)); + code.push_back(EncodeSop1(0x04, 0, 106)); + code.push_back(EncodeSop1(0x04, 1, 0)); + code.push_back(EncodeSop1(0x08, 3, 1)); + capture_scc(5); + + code.push_back(EncodeVopc(0xc2, Vgpr(0), 0)); + code.push_back(EncodeSop1(0x04, 8, 106)); + code.push_back(EncodeSMovB32(10, InlineU32(1))); + code.push_back(EncodeSMovB32(11, InlineU32(0))); + code.push_back(EncodeSop2(0x19, 12, 8, 10)); + capture_scc(14); + + AppendStoreSgpr(&code, 5, 0); + AppendStoreSgpr(&code, 14, 1); + AppendEnd(&code); + + return {"ScalarMaskProvenanceOverlapAndMixedBinary", + code, + {}, + {0, 1}, + {O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_NOT_B64, O::S_CSELECT_B32, + O::S_MOV_B32, O::S_NAND_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; +} + TestCase ScalarLiteral() { using O = ShaderOpcode; @@ -13003,7 +13126,9 @@ TestCase VectorAddcWritesPerLaneCarryOut() { TestCase test; test.name = "VectorAddcWritesPerLaneCarryOut"; test.code = code; - test.expected = std::vector(8, 0x0fu); + // VCC is represented as the current invocation's carry bit, not a shared + // four-lane ballot word. + test.expected = std::vector(8, 1u); test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::V_CMP_F_U32, O::V_ADDC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; @@ -13121,7 +13246,8 @@ TestCase VectorVop3BCarryOutWritesSgprMask() { TestCase test; test.name = "VectorVop3BCarryOutWritesSgprMask"; test.code = code; - test.expected = std::vector(12, 0x0fu); + // Scalar mask destinations keep the current invocation's bit. + test.expected = std::vector(12, 1u); test.opcodes = {O::V_MOV_B32, O::V_ADD_I32, O::V_SUB_I32, O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; @@ -13153,7 +13279,7 @@ TestCase VectorVop3BCarryOutUsesEncodedSdst() { TestCase test; test.name = "VectorVop3BCarryOutUsesEncodedSdst"; test.code = code; - test.expected = std::vector(12, 0x0fu); + test.expected = std::vector(12, 1u); test.opcodes = {O::V_MOV_B32, O::V_ADD_I32, O::V_SUB_I32, O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; @@ -13192,7 +13318,7 @@ TestCase VectorVop3BSubCoU32UsesRdna2Opcode310() { TestCase test; test.name = "VectorVop3BSubCoU32UsesRdna2Opcode310"; test.code = code; - test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 9, 9, 9, 9}; + test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 1, 0, 0, 1}; test.opcodes = {O::V_MOV_B32, O::V_CMP_EQ_U32, O::V_CNDMASK_B32, O::V_SUB_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; @@ -13631,9 +13757,12 @@ TestCase VectorPermlane16FetchInactiveZero() { using O = ShaderOpcode; std::vector code = { - EncodeVop1(0x01, 1, 0), EncodeVop2(0x25, 1, InlineU32(10), 1), - EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)), - EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)), + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x25, 1, InlineU32(10), 1), + EncodeVopc(0xc2, InlineU32(0), 0), + EncodeSop1(0x04, 126, 106), + EncodeSMovB32(0, InlineU32(1)), + EncodeSMovB32(1, InlineU32(0)), }; AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1); AppendStoreVgpr(&code, 2, 0); @@ -13643,8 +13772,9 @@ TestCase VectorPermlane16FetchInactiveZero() { test.name = "VectorPermlane16FetchInactiveZero"; test.code = code; test.expected = {0}; - test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, - O::V_PERMLANE16_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32, + O::S_MOV_B64, O::S_MOV_B32, O::V_PERMLANE16_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -13657,9 +13787,9 @@ TestCase VectorPermlane16FetchInactiveFi() { using O = ShaderOpcode; std::vector code = { - EncodeVop1(0x01, 1, Vgpr(0)), EncodeVop2(0x25, 1, InlineU32(10), 1), - EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)), - EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)), + EncodeVop1(0x01, 1, Vgpr(0)), EncodeVop2(0x25, 1, InlineU32(10), 1), + EncodeVopc(0xc2, InlineU32(0), 0), EncodeSop1(0x04, 126, 106), + EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)), }; AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1, 0, 1); AppendStoreVgpr(&code, 2, 0); @@ -13669,8 +13799,9 @@ TestCase VectorPermlane16FetchInactiveFi() { test.name = "VectorPermlane16FetchInactiveFi"; test.code = code; test.expected = {11}; - test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, - O::V_PERMLANE16_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32, + O::S_MOV_B64, O::S_MOV_B32, O::V_PERMLANE16_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -13803,11 +13934,10 @@ TestCase Vop1MoveRelDestination() { AppendVMovLiteral(&code, 6, 0xbbbbbbbbu); AppendVMovLiteral(&code, 7, 0xccccccccu); code.push_back(EncodeSMovB32(124, InlineU32(2))); - code.push_back(EncodeSMovB32(126, InlineU32(1))); - code.push_back(EncodeSMovB32(127, InlineU32(0))); + code.push_back(EncodeVopc(0xc2, InlineU32(0), 0)); + code.push_back(EncodeSop1(0x04, 126, 106)); code.push_back(EncodeVop1(0x42, 5, Vgpr(1))); - code.push_back(EncodeSMovB32(126, InlineU32(0xf))); - code.push_back(EncodeSMovB32(127, InlineU32(0))); + code.push_back(EncodeSop1(0x04, 126, 193u)); code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0)); AppendBufferStoreDword(&code, 7, 4); AppendEnd(&code); @@ -13816,8 +13946,9 @@ TestCase Vop1MoveRelDestination() { test.name = "Vop1MoveRelDestination"; test.code = code; test.expected = {0x12345678u, 0xccccccccu, 0xccccccccu, 0xccccccccu}; - test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_MOVRELD_B32, - O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_EQ_U32, + O::S_MOV_B64, O::V_MOVRELD_B32, O::V_LSHLREV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -14542,7 +14673,6 @@ TestCase VectorVop3CompareGtU64OnGpu() { {1, 1, 0}, {O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_GT_U64, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; - test.required_spirv = {"OpUGreaterThan"}; test.compute_info.threads_num[0] = 64; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -14731,13 +14861,10 @@ TestCase Vop3CndmaskUsesSgprMaskLaneBits() { using O = ShaderOpcode; std::vector code = { - EncodeVop1(0x01, 1, 0), - EncodeVop2(0x1a, 1, InlineU32(2), 1), - EncodeVop2(0x25, 1, Vgpr(0), 1), - EncodeVopc(0xc0, Vgpr(0), 0), + EncodeVop1(0x01, 1, 0), EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), EncodeVopc(0xc2, InlineU32(0), 0), + EncodeSop1(0x04, 4, 106), }; - AppendSMovLiteral(&code, 4, 1); - AppendSMovLiteral(&code, 5, 0); AppendVop3(&code, 0x101, 2, InlineU32(10), InlineU32(20), 4); code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 1)); AppendBufferStoreDword(&code, 2, 3); @@ -14748,7 +14875,7 @@ TestCase Vop3CndmaskUsesSgprMaskLaneBits() { test.code = code; test.expected = {20, 10, 10, 10, 20, 10, 10, 10}; test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, - O::V_CMP_F_U32, O::S_MOV_B32, O::V_CNDMASK_B32, + O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; @@ -14962,7 +15089,7 @@ TestCase SimpleLoop() { O::S_BRANCH, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } -TestCase BranchVccnzUsesWholeMask() { +TestCase BranchVccnzUsesInvocationMask() { using O = ShaderOpcode; std::vector code = { @@ -14980,9 +15107,9 @@ TestCase BranchVccnzUsesWholeMask() { AppendEnd(&code); TestCase test; - test.name = "BranchVccnzUsesWholeMask"; + test.name = "BranchVccnzUsesInvocationMask"; test.code = code; - test.expected = std::vector(8, 42); + test.expected = {42, 11, 11, 11, 42, 11, 11, 11}; test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32, O::S_CBRANCH_VCCNZ, O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; @@ -16647,7 +16774,7 @@ TestCase FlatSubdwordLoadsApplyByteOffset() { return test; } -TestCase BranchVccnzUsesCarryProducedWholeMask() { +TestCase BranchVccnzUsesCarryProducedInvocationMask() { using O = ShaderOpcode; std::vector code = { @@ -16665,9 +16792,9 @@ TestCase BranchVccnzUsesCarryProducedWholeMask() { AppendEnd(&code); TestCase test; - test.name = "BranchVccnzUsesCarryProducedWholeMask"; + test.name = "BranchVccnzUsesCarryProducedInvocationMask"; test.code = code; - test.expected = std::vector(8, 42); + test.expected = {11, 42, 42, 42, 42, 42, 42, 42}; test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::V_CMP_F_U32, O::V_ADDC_U32, O::S_CBRANCH_VCCNZ, O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; @@ -18165,7 +18292,8 @@ TestCase ImageD16GatherPacksHalfPairs() { O::S_ENDPGM}; test.sampled_image_rgba = image; test.decoded_counts = {{"d16=1", 1}}; - test.native_ir_counts = {{"dmask=0x1 data_dwords=2 data_bits=16 component=0/4", 1}}; + test.native_ir_counts = { + {"dmask=0x1 data_dwords=2 data_bits=16 component=0/4", 1}}; test.required_spirv = {"OpImageGather", "PackHalf2x16"}; return test; } @@ -18376,8 +18504,8 @@ TestCase ImageD16StoreUnpacksHalfPairs() { AppendEnd(&code); auto expected_image = MakeRgbaImage(4, 4); - SetRgbaPixel(&expected_image, 4, 1, 2, 0x3e800000u, 0x3f000000u, - 0x3f400000u, 0x3f800000u); + SetRgbaPixel(&expected_image, 4, 1, 2, 0x3e800000u, 0x3f000000u, 0x3f400000u, + 0x3f800000u); TestCase test; test.name = "ImageD16StoreUnpacksHalfPairs"; @@ -18386,7 +18514,8 @@ TestCase ImageD16StoreUnpacksHalfPairs() { test.storage_image_rgba = MakeRgbaImage(4, 4); test.expected_storage_image_rgba = expected_image; test.decoded_counts = {{"d16=1", 1}}; - test.native_ir_counts = {{"dmask=0xf data_dwords=2 data_bits=16 component=0/4", 1}}; + test.native_ir_counts = { + {"dmask=0xf data_dwords=2 data_bits=16 component=0/4", 1}}; test.required_spirv = {"UnpackHalf2x16", "OpImageWrite"}; return test; } @@ -19286,7 +19415,11 @@ std::vector MakeCases() { AddCase(ScalarBfeI32CapturedRawSignExtends); AddCase(BitfieldExtractWidthPastEndEdges); AddCase(Scalar64BitOps); - AddCase(ScalarAndn2B64SccBranch); + AddCase(ScalarAndn2B64SccUsesMaskShadow); + AddCase(ScalarMaskHighWriteInvalidatesProvenance); + AddCase(ScalarSelectB64PreservesMaskProvenance); + AddCase(ScalarWqmB64SelectsSccDomain); + AddCase(ScalarMaskProvenanceOverlapAndMixedBinary); AddCase(ScalarLiteral); AddCase(VectorMoves); AddCase(VectorVop3MoveAppliesFloatSourceModifiers); @@ -19365,8 +19498,8 @@ std::vector MakeCases() { AddCase(VectorCompareInvertedMaskSelect); AddCase(BranchSelect); AddCase(SimpleLoop); - AddCase(BranchVccnzUsesWholeMask); - AddCase(BranchVccnzUsesCarryProducedWholeMask); + AddCase(BranchVccnzUsesInvocationMask); + AddCase(BranchVccnzUsesCarryProducedInvocationMask); AddCase(ScalarMemoryLoadVariants); AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset); AddCase(ScalarLoadAlignsComponentsAndMasksAddress); @@ -21053,8 +21186,8 @@ void CheckBasicStorageTextureDescriptor() { "PPSA02604 BGRA 2D storage descriptor fixture is malformed"); ValidateStorageTexture(BasicBgraStorageTextureResource(), bgra, 0x870000); - const ShaderTextureResource r128{{0x0202e500u, 0xc8200000u, 0x010dc1dfu, - 0x91b00facu, 0, 0, 0, 0}}; + const ShaderTextureResource r128{ + {0x0202e500u, 0xc8200000u, 0x010dc1dfu, 0x91b00facu, 0, 0, 0, 0}}; auto r128_resource = BasicBgraStorageTextureResource(); r128_resource.r128 = true; Require("BasicStorageTexture", "PPSA01736 R128 descriptor", @@ -22542,7 +22675,8 @@ void CheckPm4DirectShaderRegisterFallback(RenderContext &renderer) { processor.Process(execution, packet.data(), packet.size()) == Pm4ProcessResult::Complete && processor.GetShCtx().GetVs().es_regs.data_addr == shader_address, - "direct SET_SH_REG did not route its ES base pair through the register handlers"); + "direct SET_SH_REG did not route its ES base pair through the " + "register handlers"); std::printf("[host] %-32s ok\n", "Pm4DirectShaderRegisterFallback"); } @@ -22577,7 +22711,8 @@ void CheckPm4GuardBandRegisterRanges(RenderContext &renderer) { std::bit_cast(10.0f), }; Pm4Execution range_execution; - const auto range_result = processor.Process(range_execution, range.data(), range.size()); + const auto range_result = + processor.Process(range_execution, range.data(), range.size()); const auto &range_viewport = processor.GetCtx().GetScreenViewport(); Require("Pm4GuardBandRegisterRanges", "full range", range_result == Pm4ProcessResult::Complete && @@ -22644,12 +22779,13 @@ void CheckPm4BlendColorRegisterRanges(RenderContext &renderer) { std::bit_cast(14.0f), }; Pm4Execution full_execution; - const auto full_result = processor.Process(full_execution, full.data(), full.size()); + const auto full_result = + processor.Process(full_execution, full.data(), full.size()); const auto &full_color = processor.GetCtx().GetBlendColor(); Require("Pm4BlendColorRegisterRanges", "full range", - full_result == Pm4ProcessResult::Complete && full_color.red == 11.0f && - full_color.green == 12.0f && full_color.blue == 13.0f && - full_color.alpha == 14.0f, + full_result == Pm4ProcessResult::Complete && + full_color.red == 11.0f && full_color.green == 12.0f && + full_color.blue == 13.0f && full_color.alpha == 14.0f, "full blend-color range did not preserve native PS5 register order"); std::printf("[host] %-32s ok\n", "Pm4BlendColorRegisterRanges"); } @@ -22727,28 +22863,27 @@ void CheckAgcWaitPackets(RenderContext &renderer) { nullptr, 0}; const auto address32 = reinterpret_cast(&label32); - auto *emitted32 = Gen5::AgcDcbWaitRegMem( - reinterpret_cast(&dcb32), 0, 3, 0, 0, - &label32, label32, UINT32_MAX, 400); - Require( - "AgcWaitPackets", "native 32-bit packet", - Gen5::AgcDcbWaitOnAddressGetSize(0) == 56 && - emitted32 == packet32.data() && - dcb32.cursor_up == packet32.data() + 14 && - packet32[0] == 0xc0027904u && packet32[1] == 0x342u && - packet32[2] == - (0xc8010000u | - (static_cast(address32 >> 32u) & 0xffffu)) && - packet32[3] == static_cast(address32) && - packet32[4] == 0xc0053c00u && packet32[5] == 0x13u && - packet32[6] == (static_cast(address32) & ~0x3u) && - packet32[7] == - (static_cast(address32 >> 32u) & 0x3ffffu) && - packet32[8] == label32 && packet32[9] == UINT32_MAX && - packet32[10] == 25u && packet32[11] == 0xc0017904u && - packet32[12] == 0x342u && packet32[13] == 0xc8000000u && - execute(packet32.data(), 14), - "32-bit waitOnAddress stream differs from native AGC"); + auto *emitted32 = + Gen5::AgcDcbWaitRegMem(reinterpret_cast(&dcb32), 0, + 3, 0, 0, &label32, label32, UINT32_MAX, 400); + Require("AgcWaitPackets", "native 32-bit packet", + Gen5::AgcDcbWaitOnAddressGetSize(0) == 56 && + emitted32 == packet32.data() && + dcb32.cursor_up == packet32.data() + 14 && + packet32[0] == 0xc0027904u && packet32[1] == 0x342u && + packet32[2] == + (0xc8010000u | + (static_cast(address32 >> 32u) & 0xffffu)) && + packet32[3] == static_cast(address32) && + packet32[4] == 0xc0053c00u && packet32[5] == 0x13u && + packet32[6] == (static_cast(address32) & ~0x3u) && + packet32[7] == + (static_cast(address32 >> 32u) & 0x3ffffu) && + packet32[8] == label32 && packet32[9] == UINT32_MAX && + packet32[10] == 25u && packet32[11] == 0xc0017904u && + packet32[12] == 0x342u && packet32[13] == 0xc8000000u && + execute(packet32.data(), 14), + "32-bit waitOnAddress stream differs from native AGC"); alignas(8) uint32_t patched_label32 = 0xaabbccddu; const auto patched_address32 = reinterpret_cast(&patched_label32); @@ -22757,7 +22892,8 @@ void CheckAgcWaitPackets(RenderContext &renderer) { Require( "AgcWaitPackets", "native 32-bit patch", Gen5::AgcWaitRegMemPatchAddress(packet32.data(), &patched_label32) == 0 && - Gen5::AgcWaitRegMemPatchReference(packet32.data(), patched_label32) == 0 && + Gen5::AgcWaitRegMemPatchReference(packet32.data(), patched_label32) == + 0 && packet32[2] == (0xc8010000u | (static_cast(patched_address32 >> 32u) & 0xffffu)) && @@ -22780,31 +22916,30 @@ void CheckAgcWaitPackets(RenderContext &renderer) { nullptr, 0}; const auto address64 = reinterpret_cast(&label64); - auto *emitted64 = Gen5::AgcDcbWaitRegMem( - reinterpret_cast(&dcb64), 1, 3, 0, 0, - &label64, label64, UINT64_MAX, 400); - Require( - "AgcWaitPackets", "native 64-bit packet", - Gen5::AgcDcbWaitOnAddressGetSize(1) == 64 && - Gen5::AgcDcbWaitOnAddressGetSize(2) == 0 && - emitted64 == packet64.data() && - dcb64.cursor_up == packet64.data() + packet64.size() && - packet64[0] == 0xc0027904u && packet64[1] == 0x342u && - packet64[2] == - (0xc8020000u | - (static_cast(address64 >> 32u) & 0xffffu)) && - packet64[3] == static_cast(address64) && - packet64[4] == 0xc0079300u && packet64[5] == 0x13u && - packet64[6] == (static_cast(address64) & ~0x7u) && - packet64[7] == - (static_cast(address64 >> 32u) & 0x3ffffu) && - packet64[8] == static_cast(label64) && - packet64[9] == static_cast(label64 >> 32u) && - packet64[10] == UINT32_MAX && packet64[11] == UINT32_MAX && - packet64[12] == 25u && packet64[13] == 0xc0017904u && - packet64[14] == 0x342u && packet64[15] == 0xc8000000u && - execute(packet64.data(), packet64.size()), - "64-bit waitOnAddress stream differs from native AGC"); + auto *emitted64 = + Gen5::AgcDcbWaitRegMem(reinterpret_cast(&dcb64), 1, + 3, 0, 0, &label64, label64, UINT64_MAX, 400); + Require("AgcWaitPackets", "native 64-bit packet", + Gen5::AgcDcbWaitOnAddressGetSize(1) == 64 && + Gen5::AgcDcbWaitOnAddressGetSize(2) == 0 && + emitted64 == packet64.data() && + dcb64.cursor_up == packet64.data() + packet64.size() && + packet64[0] == 0xc0027904u && packet64[1] == 0x342u && + packet64[2] == + (0xc8020000u | + (static_cast(address64 >> 32u) & 0xffffu)) && + packet64[3] == static_cast(address64) && + packet64[4] == 0xc0079300u && packet64[5] == 0x13u && + packet64[6] == (static_cast(address64) & ~0x7u) && + packet64[7] == + (static_cast(address64 >> 32u) & 0x3ffffu) && + packet64[8] == static_cast(label64) && + packet64[9] == static_cast(label64 >> 32u) && + packet64[10] == UINT32_MAX && packet64[11] == UINT32_MAX && + packet64[12] == 25u && packet64[13] == 0xc0017904u && + packet64[14] == 0x342u && packet64[15] == 0xc8000000u && + execute(packet64.data(), packet64.size()), + "64-bit waitOnAddress stream differs from native AGC"); alignas(8) uint64_t patched_label64 = 0x11223344aabbccddull; const auto patched_address64 = reinterpret_cast(&patched_label64); @@ -22826,18 +22961,20 @@ void CheckAgcWaitPackets(RenderContext &renderer) { 0x5a580000u) && packet64[8] == 0xaabbccddu && packet64[9] == 0x11223344u && execute(packet64.data(), packet64.size()), - "64-bit native wait patch did not preserve the native high reference DWORD"); + "64-bit native wait patch did not preserve the native high reference " + "DWORD"); std::array invalid{}; const uint32_t short_payload = 0x342u; - Require("AgcWaitPackets", "invalid packet", - Gen5::AgcWaitRegMemPatchAddress(invalid.data(), &label32) == - packet_mismatch && - Gen5::AgcWaitRegMemPatchReference(invalid.data(), label32) == - packet_mismatch && - !Gen5::AgcIsInternalDataPacket( - KYTY_PM4(2, Pm4::IT_SET_UCONFIG_REG, 1), &short_payload), - "native wait patch accepted a packet without the AGC metadata prefix"); + Require( + "AgcWaitPackets", "invalid packet", + Gen5::AgcWaitRegMemPatchAddress(invalid.data(), &label32) == + packet_mismatch && + Gen5::AgcWaitRegMemPatchReference(invalid.data(), label32) == + packet_mismatch && + !Gen5::AgcIsInternalDataPacket( + KYTY_PM4(2, Pm4::IT_SET_UCONFIG_REG, 1), &short_payload), + "native wait patch accepted a packet without the AGC metadata prefix"); std::printf("[host] %-32s ok\n", "AgcWaitPackets"); } @@ -22873,17 +23010,17 @@ void CheckAgcDrawIndirectMultiPacket(RenderContext &renderer) { 0x55667788u, count_address, 0xaabbccddu, modifier); const uint32_t invalid_payload[]{0x342u, 0xc6000010u}; - Require( - "AgcDrawIndirectMulti", "native packet", - emitted == packet.data() && dcb.cursor_up == packet.data() + packet.size() && - packet == expected && execute(packet.data(), 3u) && - execute(packet.data() + 13u, 3u) && - Gen5::AgcWaitRegMemPatchAddress(packet.data(), count_address) == - packet_mismatch && - Gen5::AgcWaitRegMemPatchReference(packet.data(), 0x12345678u) == - packet_mismatch && - !Gen5::AgcIsInternalDataPacket(0xc0017904u, invalid_payload), - "drawIndirectMulti stream differs from native AGC"); + Require("AgcDrawIndirectMulti", "native packet", + emitted == packet.data() && + dcb.cursor_up == packet.data() + packet.size() && + packet == expected && execute(packet.data(), 3u) && + execute(packet.data() + 13u, 3u) && + Gen5::AgcWaitRegMemPatchAddress(packet.data(), count_address) == + packet_mismatch && + Gen5::AgcWaitRegMemPatchReference(packet.data(), 0x12345678u) == + packet_mismatch && + !Gen5::AgcIsInternalDataPacket(0xc0017904u, invalid_payload), + "drawIndirectMulti stream differs from native AGC"); std::printf("[host] %-32s ok\n", "AgcDrawIndirectMulti"); } diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index c51eddee..0cf89156 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -8,7 +8,6 @@ #include "graphics/host_gpu/renderer/image/imageView.h" #include "graphics/host_gpu/renderer/image/textureCommon.h" #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h" -#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" @@ -988,8 +987,7 @@ void TestSpirvRequirementsAnalysis() { &analysis_error), "initial SPIR-V requirements analysis failed"); const auto requirements = *program.spirv_requirements; - Check(requirements.requires_exact_subgroup && requirements.subgroup_ballot && - requirements.subgroup_shuffle && + Check(requirements.subgroup_ballot && requirements.subgroup_shuffle && requirements.subgroup_local_invocation_id && requirements.compute_derivatives && requirements.image_gather_extended && requirements.function_lds && @@ -1044,8 +1042,7 @@ void TestSpirvRequirementsAnalysis() { &analysis_error), "ADD_TID SPIR-V requirements analysis failed"); const auto add_tid_requirements = *add_tid.spirv_requirements; - Check(add_tid_requirements.requires_exact_subgroup && - add_tid_requirements.subgroup_local_invocation_id && + Check(add_tid_requirements.subgroup_local_invocation_id && !add_tid_requirements.subgroup_ballot && !add_tid_requirements.subgroup_shuffle, "buffer ADD_TID requested the wrong subgroup contract"); @@ -1071,8 +1068,7 @@ void TestSpirvRequirementsAnalysis() { &analysis_error), "empty SPIR-V requirements analysis failed"); const auto empty_requirements = *empty.spirv_requirements; - Check(!empty_requirements.requires_exact_subgroup && - !empty_requirements.subgroup_ballot && + Check(!empty_requirements.subgroup_ballot && !empty_requirements.subgroup_shuffle && !empty_requirements.subgroup_local_invocation_id && !empty_requirements.compute_derivatives && @@ -1082,129 +1078,6 @@ void TestSpirvRequirementsAnalysis() { "empty IR unexpectedly requested SPIR-V features"); } -void TestNativeSubgroupPolicy() { - const auto SetTypedValues = [](ShaderRecompiler::IR::Program &program, - ShaderRecompiler::IR::ValueOpcode opcode = - ShaderRecompiler::IR::ValueOpcode::Void) { - using namespace ShaderRecompiler::IR; - program.values = std::make_shared(); - program.values->block_storage.push_back(std::make_unique()); - auto *block = program.values->block_storage.back().get(); - program.values->blocks.push_back(block); - if (opcode == ValueOpcode::ReadLane) { - block->AppendNewInst(opcode, {Value(0u), Value(0u)}); - } else if (opcode == ValueOpcode::Ballot) { - block->AppendNewInst(opcode, {Value(true)}); - } else if (opcode == ValueOpcode::DataAppend) { - program.values->memory_info.push_back({.kind = ResourceKind::Lds}); - auto &append = block->AppendNewInst( - opcode, {Value(0u), Value(true), Value(1u), Value(0u)}); - append.SetFlags(MemoryFlags{.index = 0}); - } - std::string error; - Check(ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program, &error), - "native subgroup requirements analysis failed"); - }; - - GraphicContext context; - context.subgroup_size = 32; - context.min_subgroup_size = 32; - context.max_subgroup_size = 32; - context.subgroup_size_control_enabled = true; - context.required_subgroup_size_stages = vk::ShaderStageFlagBits::eAll; - ShaderRecompiler::IR::Program safe; - safe.wave_size = 32; - safe.lane_mask_mode = ShaderLaneMaskMode::NativeWave; - SetTypedValues(safe); - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, - vk::ShaderStageFlagBits::eVertex, safe) - .mode == ShaderSubgroupMode::Natural, - "native wave32 policy changed"); - safe.wave_size = 64; - Check(SelectGraphicsLaneMaskMode(safe.wave_size) == - ShaderLaneMaskMode::PerInvocation && - ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, - vk::ShaderStageFlagBits::eVertex, safe) - .mode == ShaderSubgroupMode::Unsupported, - "wave64 graphics mismatch accepted native-wave mask lowering"); - safe.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, - vk::ShaderStageFlagBits::eVertex, safe) - .mode == ShaderSubgroupMode::PerInvocationGraphics, - "wave64 graphics mismatch did not select per-invocation masks"); - - ShaderRecompiler::IR::Program cross_lane = safe; - SetTypedValues(cross_lane, ShaderRecompiler::IR::ValueOpcode::ReadLane); - Check(cross_lane.spirv_requirements->requires_exact_subgroup && - ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, - vk::ShaderStageFlagBits::eVertex, - cross_lane) - .mode == ShaderSubgroupMode::PerInvocationGraphics, - "graphics mismatch did not select per-invocation masks"); - auto cross_lane_compute = cross_lane; - cross_lane_compute.lane_mask_mode = ShaderLaneMaskMode::NativeWave; - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, - vk::ShaderStageFlagBits::eCompute, - cross_lane_compute) - .mode == ShaderSubgroupMode::FlattenedMasks, - "split wave64 compute did not use native subgroup operations"); - - ShaderRecompiler::IR::Program zero_exec = safe; - zero_exec.lane_mask_mode = ShaderLaneMaskMode::NativeWave; - SetTypedValues(zero_exec); - Check(!zero_exec.spirv_requirements->requires_exact_subgroup && - ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, - vk::ShaderStageFlagBits::eCompute, - zero_exec) - .mode == ShaderSubgroupMode::FlattenedMasks, - "compile-time uniform-zero EXEC write did not stay on the mask-free " - "path"); - - ShaderRecompiler::IR::Program selective_exec = safe; - SetTypedValues(selective_exec, ShaderRecompiler::IR::ValueOpcode::Ballot); - Check(selective_exec.spirv_requirements->requires_exact_subgroup, - "selective EXEC write was classified mask-free"); - - ShaderRecompiler::IR::Program carry = safe; - SetTypedValues(carry, ShaderRecompiler::IR::ValueOpcode::Ballot); - Check(carry.spirv_requirements->requires_exact_subgroup, - "lane-varying VCC carry producer was classified mask-free"); - - ShaderRecompiler::IR::Program vcc_branch = safe; - SetTypedValues(vcc_branch, ShaderRecompiler::IR::ValueOpcode::Ballot); - Check(vcc_branch.spirv_requirements->requires_exact_subgroup, - "whole-wave VCC branch was classified mask-free"); - - ShaderRecompiler::IR::Program ds_partial = safe; - ds_partial.lane_mask_mode = ShaderLaneMaskMode::NativeWave; - SetTypedValues(ds_partial, ShaderRecompiler::IR::ValueOpcode::DataAppend); - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, - vk::ShaderStageFlagBits::eCompute, ds_partial) - .mode == ShaderSubgroupMode::FlattenedMasks, - "split wave64 DS append did not use native subgroup operations"); - context.max_subgroup_size = 64; - const auto controlled = ConfigureShaderSubgroup( - ShaderSubgroupCapabilities{context}, vk::ShaderStageFlagBits::eCompute, - cross_lane_compute); - Check(controlled.mode == ShaderSubgroupMode::Controlled && - controlled.required_size == 64, - "supported controlled wave64 was not preferred over splitting"); - context.subgroup_size = 64; - context.subgroup_size_control_enabled = false; - cross_lane.wave_size = 32; - cross_lane.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, - vk::ShaderStageFlagBits::eFragment, cross_lane) - .mode == ShaderSubgroupMode::Unsupported, - "inverse graphics mismatch was accepted as one guest wave"); - cross_lane_compute.wave_size = 32; - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, - vk::ShaderStageFlagBits::eCompute, - cross_lane_compute) - .mode == ShaderSubgroupMode::FlattenedMasks, - "inverse compute mismatch did not use the native subgroup"); -} - std::array ImageTestUserData(Prospero::ImageType type = Prospero::ImageType::kColor2D) { std::array data{}; @@ -5567,8 +5440,7 @@ void TestGraphicsCreateInterpolantMapping() { "flat/custom interpolant mapping bits were unexpected"); Check(regs[3].value == 0x01580304u, "f16 interpolant mapping bits were unexpected"); - Check(regs[6].offset == Pm4::SPI_PS_INPUT_CNTL_0 + 6u && - regs[6].value == 6u, + Check(regs[6].offset == Pm4::SPI_PS_INPUT_CNTL_0 + 6u && regs[6].value == 6u, "interpolant identity tail was not filled"); Check(Gen5::AgcCreateInterpolantMapping2(native_regs, &gs, &ps) == 0, @@ -7502,7 +7374,8 @@ void TestNewShaderRecompilerCfgMultipleLoopLatches() { "multiple-latch fixture lacks two native backedges"); Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); Check(graph.blocks.size() == original_block_count + 2u, - "multiple native latches did not create one synthetic continue and one empty header"); + "multiple native latches did not create one synthetic continue and one " + "empty header"); Check(graph.back_edges.size() == 1u && graph.natural_loops.size() == 1u, "multiple native latches were not coalesced to one SPIR-V backedge"); const auto &loop = graph.natural_loops.front(); @@ -7700,23 +7573,22 @@ void TestNewShaderRecompilerCfgExecSccSharedArm() { } } - for (const auto lane_mode : - {ShaderLaneMaskMode::NativeWave, ShaderLaneMaskMode::PerInvocation}) { - auto options = MakeCompileOptions(ShaderType::Compute); - options.lane_mask_mode = lane_mode; - options.dump_ir = true; - ShaderRecompiler::CompileResult result; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), - error.c_str()); - Check(!result.program.dispatcher_fallback && - Common::ContainsStr(result.ir_dump, "mode=structured"), - "EXEC/SCC shared-arm epilogue did not stay structured"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 3u, - "EXEC/SCC shared-arm SPIR-V has the wrong selection-merge count"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, - "EXEC/SCC shared-arm SPIR-V unexpectedly used dispatcher OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); - } + auto options = MakeCompileOptions(ShaderType::Compute); + options.dump_ir = true; + ShaderRecompiler::CompileResult result; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(!result.program.dispatcher_fallback && + Common::ContainsStr(result.ir_dump, "mode=structured"), + "EXEC/SCC shared-arm epilogue did not stay structured"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 3u, + "EXEC/SCC shared-arm SPIR-V has the wrong selection-merge count"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, + "EXEC/SCC shared-arm SPIR-V unexpectedly used dispatcher OpSwitch"); + Check(!Common::ContainsStr(DisassembleSpirvBinary(result.spirv), + "OpGroupNonUniformBallot"), + "per-invocation EXEC/SCC branch reconstructed a native subgroup mask"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgNestedTailEarlyExit() { @@ -7755,13 +7627,16 @@ void TestNewShaderRecompilerCfgNestedTailEarlyExit() { Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "nested-tail routing changed semantic instruction coverage"); - Check(std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - }) == 1u && - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.goto_value >= 0; - }) == 3u, + Check(std::ranges::count_if( + graph.blocks, + [](const auto &block) { + return block.terminator.condition == + ShaderRecompiler::CFG::BranchCondition::GotoVariable; + }) == 1u && + std::ranges::count_if(graph.blocks, + [](const auto &block) { + return block.terminator.goto_value >= 0; + }) == 3u, "nested-tail early exit did not use typed route state"); auto options = MakeCompileOptions(ShaderType::Pixel); @@ -8000,9 +7875,10 @@ void TestNewShaderRecompilerCfgSharedTerminalEarlyExit() { const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); - Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == - original_coverage, - "shared-terminal structurization changed semantic instruction coverage"); + Check( + CfgInstructionCoverage(graph, decoded.instructions.size()) == + original_coverage, + "shared-terminal structurization changed semantic instruction coverage"); auto options = MakeCompileOptions(ShaderType::Pixel); options.dump_ir = true; @@ -8171,7 +8047,6 @@ void TestNewShaderRecompilerU64PairLowering() { ir.Emit(IR::ValueOpcode::ULessThan64, {shifted, base}); ir.Emit(IR::ValueOpcode::SLessThan64, {shifted, base}); ir.Emit(IR::ValueOpcode::BitCount64, {shifted}); - ir.Emit(IR::ValueOpcode::WqmU64, {shifted}); for (const uint32_t count : {1u, 31u, 32u, 33u, 63u}) { ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {base, IR::Value(count)}); ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {base, IR::Value(count)}); @@ -8710,10 +8585,9 @@ void TestDemandDrivenSpirvDeclarations() { void TestTypedEntryStateIsMinimal() { using namespace ShaderRecompiler; - const auto translate = [](ShaderLaneMaskMode lane_mode, uint32_t wave_size) { + const auto check = [](uint32_t wave_size) { IR::Program source; source.stage = ShaderType::Compute; - source.lane_mask_mode = lane_mode; source.wave_size = wave_size; source.user_data_count = 0; source.blocks.emplace_back(); @@ -8726,11 +8600,7 @@ void TestTypedEntryStateIsMinimal() { Check(Frontend::TranslateProgram(source, values, nullptr, nullptr, &compute, &error), error.c_str()); - return values; - }; - const auto check = [&](ShaderLaneMaskMode lane_mode, uint32_t wave_size) { - auto values = translate(lane_mode, wave_size); uint32_t set_exec = 0; uint32_t set_exec_lo = 0; uint32_t set_exec_hi = 0; @@ -8738,13 +8608,11 @@ void TestTypedEntryStateIsMinimal() { IR::Value exec; IR::Value exec_lo; IR::Value exec_hi; - const IR::Inst *ballot = nullptr; for (const auto *block : values.blocks) { for (const auto &inst : *block) { switch (inst.GetOpcode()) { case IR::ValueOpcode::Ballot: ballots++; - ballot = &inst; break; case IR::ValueOpcode::SetExec: set_exec++; @@ -8760,6 +8628,7 @@ void TestTypedEntryStateIsMinimal() { break; case IR::ValueOpcode::SetScalarRegister: case IR::ValueOpcode::SetThreadBitScalarRegister: + case IR::ValueOpcode::SetScalarMaskTag: case IR::ValueOpcode::SetVectorRegister: case IR::ValueOpcode::SetScc: case IR::ValueOpcode::SetVcc: @@ -8774,44 +8643,19 @@ void TestTypedEntryStateIsMinimal() { } } Check(set_exec == 1u && set_exec_lo == 1u && set_exec_hi == 1u && - exec.IsImmediate() && exec.U1(), - "typed entry does not define EXEC exactly once"); - if (lane_mode == ShaderLaneMaskMode::PerInvocation) { - Check(ballots == 0u && exec_lo.IsImmediate() && exec_lo.U32() == 1u && - exec_hi.IsImmediate() && exec_hi.U32() == 0u, - "per-invocation entry EXEC is not the local {1,0} mask"); - } else { - Check(ballots == 1u && !exec_lo.IsImmediate(), - "native compute entry EXEC does not use the active subgroup mask"); - const auto *lo = exec_lo.ResolveInstruction(); - Check(lo != nullptr && - lo->GetOpcode() == IR::ValueOpcode::CompositeExtractU32x4 && - lo->Arg(0).ResolveInstruction() == ballot && - lo->Arg(1).Resolve().U32() == 0u, - "native compute EXEC_LO does not use ballot word zero"); - if (wave_size == 32u) { - Check(exec_hi.IsImmediate() && exec_hi.U32() == 0u, - "native wave32 entry defined EXEC_HI"); - } else { - const auto *hi = exec_hi.ResolveInstruction(); - Check(hi != nullptr && - hi->GetOpcode() == IR::ValueOpcode::CompositeExtractU32x4 && - hi->Arg(0).ResolveInstruction() == ballot && - hi->Arg(1).Resolve().U32() == 1u, - "native wave64 EXEC_HI does not use ballot word one"); - } - } + ballots == 0u && exec.IsImmediate() && exec.U1() && + exec_lo.IsImmediate() && exec_lo.U32() == 1u && + exec_hi.IsImmediate() && exec_hi.U32() == 0u, + "typed entry is not the local {1,0} invocation mask"); IR::RewriteToSsa(values.blocks); IR::RemoveIdentities(values.blocks); IR::EliminateDeadCode(values.blocks); - std::string error; Check(IR::ValidateValueProgram(values, true, &error), error.c_str()); }; - check(ShaderLaneMaskMode::NativeWave, 32u); - check(ShaderLaneMaskMode::NativeWave, 64u); - check(ShaderLaneMaskMode::PerInvocation, 64u); + check(32u); + check(64u); } void TestFinalSsaRejectsRegisterStatePseudos() { @@ -9145,36 +8989,9 @@ void TestValuePhiValidation() { } } -void TestWqmConstantPropagation() { +void TestWqmMaskSignatureAndU64ShiftConstantPropagation() { using namespace ShaderRecompiler::IR; - ValueProgram program; - program.block_storage.push_back(std::make_unique()); - program.blocks.push_back(program.block_storage.back().get()); - program.block_info.emplace_back(); - IREmitter ir(program.blocks.front()); - - constexpr std::array cases = { - std::pair{uint64_t{0}, uint64_t{0}}, - std::pair{uint64_t{1}, uint64_t{0xfull}}, - std::pair{uint64_t{1} << 31u, uint64_t{0xf} << 28u}, - std::pair{uint64_t{1} << 32u, uint64_t{0xf} << 32u}, - std::pair{uint64_t{0x8000001180000001ull}, - uint64_t{0xf00000fff000000full}}, - }; - std::array results; - for (size_t index = 0; index < cases.size(); index++) { - results[index] = ir.Emit(ValueOpcode::WqmU64, {Value(cases[index].first)}); - } - - ConstantPropagationPass(program.blocks); - for (size_t index = 0; index < cases.size(); index++) { - const auto value = results[index].Resolve(); - Check(value.IsImmediate() && value.GetType() == Type::U64 && - value.U64() == cases[index].second, - "WqmU64 constant propagation crossed a quad or dword boundary"); - } - const auto check_invalid_signature = [](ValueOpcode opcode, Value valid, Value invalid) { ValueProgram malformed; @@ -9189,7 +9006,6 @@ void TestWqmConstantPropagation() { Common::ContainsStr(error, "argument 0 has type"), "value IR accepted a WQM operand that violates opcode metadata"); }; - check_invalid_signature(ValueOpcode::WqmU64, Value(uint64_t{1}), Value(true)); check_invalid_signature(ValueOpcode::WqmMask, Value(true), Value(uint64_t{1})); @@ -9624,7 +9440,7 @@ void TestNewShaderRecompilerVertexSystemInputsWithoutMirrors() { "vertex system values were routed through guest VGPR mirrors"); } -void TestNewShaderRecompilerVertexExportUsesLaneExecMask() { +void TestNewShaderRecompilerVertexExportUsesInvocationExecMask() { const uint32_t shader[] = { EncodeSop1(0x04, 126, 129), // s_mov_b64 exec, 1 EncodeExp0(0x0c, 0xf), @@ -9640,27 +9456,11 @@ void TestNewShaderRecompilerVertexExportUsesLaneExecMask() { Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(Common::ContainsStr(source, "gl_SubgroupInvocationID"), - "vertex export EXEC guard did not test the current lane"); + Check(!Common::ContainsStr(source, "OpLoad %uint %gl_SubgroupInvocationID"), + "vertex EXEC guard depends on the native subgroup lane"); Check(Common::ContainsStr(source, "OpBranchConditional"), - "vertex export EXEC guard did not branch on the lane-active mask"); - - options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; - ShaderRecompiler::CompileResult local_result; - Check(ShaderRecompiler::TryRecompile(shader, options, local_result, &error), - error.c_str()); - Check(local_result.program.lane_mask_mode == - ShaderLaneMaskMode::PerInvocation, - "per-invocation mask mode was not preserved in the immutable program"); - CheckSpirvBinaryValidates(local_result.spirv); - const auto local_source = DisassembleSpirvBinary(local_result.spirv); - Check(!Common::ContainsStr(local_source, - "OpLoad %uint %gl_SubgroupInvocationID"), - "per-invocation EXEC still depends on the native subgroup lane"); - Check(Common::ContainsStr(local_source, "OpBranchConditional"), - "per-invocation vertex export lost its EXEC guard"); + "vertex export lost its per-invocation EXEC guard"); } void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { @@ -9677,7 +9477,6 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { }; auto options = MakeCompileOptions(ShaderType::Vertex); - options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; options.dump_ir = true; ShaderRecompiler::CompileResult result; @@ -9720,45 +9519,6 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 1u, "wave32 WQM retained the unused high ballot-word expansion"); - const uint32_t native_wqm_shader[] = { - EncodeSop1(0x0a, 2, 0), // s_wqm_b64 s[2:3], s[0:1] - EncodeVop1(0x01, 0, 2), // v_mov_b32 v0, s2 - EncodeExp0(0x0c, 0x1), EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01), - }; - auto native_options = options; - native_options.lane_mask_mode = ShaderLaneMaskMode::NativeWave; - Check(ShaderRecompiler::TryRecompile(native_wqm_shader, native_options, - result, &error), - error.c_str()); - CheckSpirvBinaryValidates(result.spirv); - Check(Common::ContainsStr(result.ir_dump, "WqmU64") && - !Common::ContainsStr(DisassembleSpirvBinary(result.spirv), - "OpGroupNonUniformBallot"), - "native scalar WQM retained frontend expansion or subgroup machinery"); - Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 1u && - SpirvInstructionOpcodeCount(result.spirv, 194u) == 2u, - "native scalar WQM emitter was folded away or did not use the compact " - "path"); - - const uint32_t native_special_wqm_shader[] = { - EncodeSop1(0x04, 126, 0), // s_mov_b64 exec, s[0:1] - EncodeSop1(0x0a, 126, 126), // s_wqm_b64 exec, exec - EncodeSop1(0x04, 106, 0), // s_mov_b64 vcc, s[0:1] - EncodeSop1(0x0a, 106, 106), // s_wqm_b64 vcc, vcc - EncodeVop1(0x01, 0, 126), // v_mov_b32 v0, exec_lo - EncodeVop1(0x01, 1, 106), // v_mov_b32 v1, vcc_lo - EncodeExp0(0x0c, 0x3), EncodeExp1(0, 1, 0, 0), EncodeSopp(0x01), - }; - Check(ShaderRecompiler::TryRecompile(native_special_wqm_shader, - native_options, result, &error), - error.c_str()); - CheckSpirvBinaryValidates(result.spirv); - const auto native_special_source = DisassembleSpirvBinary(result.spirv); - Check(CountSourceOccurrences(result.ir_dump, "WqmU64") == 2u && - !Common::ContainsStr(native_special_source, - "OpGroupNonUniformBallot"), - "native EXEC/VCC WQM retained destination-dependent ballot lowering"); - const uint32_t cross_lane_shader[] = { EncodeSop2(0x25, 126, 132, 128), // s_bfm_b64 exec, 4, 0 EncodeVop1(0x01, 0, 250), @@ -9789,7 +9549,6 @@ void TestNewShaderRecompilerPerInvocationU64Complement() { }; auto options = MakeCompileOptions(ShaderType::Vertex); - options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; options.dump_ir = true; ShaderRecompiler::CompileResult result; @@ -9919,7 +9678,8 @@ void TestNewShaderRecompilerExpPixelOutputs() { Check(ShaderRecompiler::TryRecompile(compressed_ba_shader, options, unorm16_ba_result, &error), error.c_str()); - const auto unorm16_ba_source = DisassembleSpirvBinary(unorm16_ba_result.spirv); + const auto unorm16_ba_source = + DisassembleSpirvBinary(unorm16_ba_result.spirv); Check(CountSourceOccurrences(unorm16_ba_source, "UnpackUnorm2x16") == 1u, "compressed UNORM16 BA-only export did not unpack VSRC1"); CheckSpirvBinaryValidates(unorm16_ba_result.spirv); @@ -10114,14 +9874,14 @@ void TestRenderTargetReverseExportMapping() { mappings[0] = gr32.export_mapping; ShaderPixelInputInfo compiled_info{}; std::span compiled_spirv; - Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave, vs_info, - mappings, compiled_info, compiled_spirv) && + Check(ShaderCompileInfoPS(regs, sh, vs_info, mappings, compiled_info, + compiled_spirv) && compiled_info.target_export_mapping[0].IsIdentity(), "inactive reverse MRT mapping was not normalized out of the shader " "cache key"); sh.target_output_mode[0] = 4; - Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave, vs_info, - mappings, compiled_info, compiled_spirv) && + Check(ShaderCompileInfoPS(regs, sh, vs_info, mappings, compiled_info, + compiled_spirv) && compiled_info.target_export_mapping[0] == gr32.export_mapping, "active reverse MRT mapping was lost before shader specialization"); } @@ -11098,8 +10858,7 @@ void TestNewShaderRecompilerPixelPipelineEntry() { HW::ShaderRegisters sh{}; ShaderPixelInputInfo input_info{}; std::vector spirv; - Check(ShaderCompileSpirvPS(regs, sh, ShaderLaneMaskMode::NativeWave, - input_info, spirv), + Check(ShaderCompileSpirvPS(regs, sh, input_info, spirv), "new pixel shader recompiler wrapper did not produce SPIR-V"); Check(!spirv.empty(), "new pixel shader recompiler wrapper returned empty SPIR-V"); @@ -11132,8 +10891,7 @@ void TestNewShaderRecompilerPixelPipelineEntry() { ShaderPixelInputInfo vcc_input{}; std::vector vcc_spirv; - Check(ShaderCompileSpirvPS(vcc_regs, sh, ShaderLaneMaskMode::NativeWave, - vcc_input, vcc_spirv), + Check(ShaderCompileSpirvPS(vcc_regs, sh, vcc_input, vcc_spirv), "VCC raw-load pointer was lost"); CheckSpirvBinaryValidates(vcc_spirv); } @@ -11226,7 +10984,6 @@ void TestComputeLdsAllocationIdentity() { void TestPixelProgramCacheDescriptorSetIdentity() { const uint32_t shader_01[] = {0xbf810000u}; const uint32_t shader_10[] = {0xbf810000u}; - const uint32_t shader_mask[] = {0xbf810000u}; HW::ShaderRegisters sh{}; auto check_transition = [&](const uint32_t *shader, uint64_t checksum, @@ -11249,8 +11006,8 @@ void TestPixelProgramCacheDescriptorSetIdentity() { const std::array identity_mappings{}; ShaderPixelInputInfo ps_info{}; std::span spirv; - Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave, - vs_info, identity_mappings, ps_info, spirv), + Check(ShaderCompileInfoPS(regs, sh, vs_info, identity_mappings, ps_info, + spirv), "pixel program-cache transition failed to compile"); const auto expected_set = has_vs_descriptors ? 1u : 0u; Check(ps_info.descriptor_set == expected_set && @@ -11266,28 +11023,6 @@ void TestPixelProgramCacheDescriptorSetIdentity() { check_transition(shader_01, 0x91a27e6300000001ull, false, true); check_transition(shader_10, 0x91a27e6300000002ull, true, false); - - HW::PixelShaderInfo mask_regs{}; - mask_regs.ps_regs.data_addr = reinterpret_cast(shader_mask); - mask_regs.ps_regs.chksum = 0x91a27e6300000003ull; - ShaderMappedData mapped{}; - mapped.code_size_bytes = sizeof(shader_mask); - ShaderMapUserData(mask_regs.ps_regs.data_addr, mapped); - auto compile_mask_mode = [&](ShaderLaneMaskMode mode) { - ShaderVertexInputInfo vs_info{}; - vs_info.stage.program = std::make_shared(); - const std::array identity_mappings{}; - ShaderPixelInputInfo ps_info{}; - std::span spirv; - Check(ShaderCompileInfoPS(mask_regs, sh, mode, vs_info, identity_mappings, - ps_info, spirv), - "pixel lane-mask cache transition failed to compile"); - Check(ps_info.stage.program != nullptr && - ps_info.stage.program->lane_mask_mode == mode, - "pixel program cache reused a different lane-mask lowering"); - }; - compile_mask_mode(ShaderLaneMaskMode::NativeWave); - compile_mask_mode(ShaderLaneMaskMode::PerInvocation); } void TestNewShaderRecompilerUnsupportedMemoryDecode() { @@ -11379,23 +11114,21 @@ void TestNewShaderRecompilerFlatAddressProvenanceBoundaries() { } void TestNewShaderRecompilerSpirvSizeBaselines() { - const auto compile = - [](const char *name, std::span shader, - const SpirvMetrics &budget, ShaderType stage = ShaderType::Compute, - ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave) { - auto options = MakeCompileOptions(stage); - options.dump_ir = true; - options.lane_mask_mode = lane_mask_mode; + const auto compile = [](const char *name, std::span shader, + const SpirvMetrics &budget, + ShaderType stage = ShaderType::Compute) { + auto options = MakeCompileOptions(stage); + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - const bool compiled = - ShaderRecompiler::TryRecompile(shader, options, result, &error); - Check(compiled, error.c_str()); - CheckSpirvBinaryValidates(result.spirv); - CheckSpirvBudget(name, result.spirv, budget); - return result; - }; + ShaderRecompiler::CompileResult result; + std::string error; + const bool compiled = + ShaderRecompiler::TryRecompile(shader, options, result, &error); + Check(compiled, error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + CheckSpirvBudget(name, result.spirv, budget); + return result; + }; const uint32_t empty[] = {EncodeSopp(0x01)}; const auto empty_result = @@ -11595,19 +11328,18 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { EncodeExp1(0, 0, 0, 0), // POS0.x EncodeSopp(0x01), }; - const auto wqm_result = - compile("wqm", wqm, - {.words = 407, - .instructions = 98, - .variables = 4, - .loads = 3, - .stores = 1, - .labels = 6, - .selection_merges = 1, - .branches = 4, - .conditional_branches = 1, - .ballots = 1}, - ShaderType::Vertex, ShaderLaneMaskMode::PerInvocation); + const auto wqm_result = compile("wqm", wqm, + {.words = 407, + .instructions = 98, + .variables = 4, + .loads = 3, + .stores = 1, + .labels = 6, + .selection_merges = 1, + .branches = 4, + .conditional_branches = 1, + .ballots = 1}, + ShaderType::Vertex); Check(Common::ContainsStr(wqm_result.ir_dump, "WqmMask"), "WQM size fixture no longer reaches per-invocation WqmMask IR"); @@ -11653,7 +11385,6 @@ int main() { TestSpirvRequirementsAnalysis(); TestNewShaderRecompilerSpirvSizeBaselines(); TestDemandDrivenSpirvDeclarations(); - TestNativeSubgroupPolicy(); TestNewShaderRecompilerSMovB32(); TestNewShaderRecompilerNativeWideScalarMemoryIr(); TestNewShaderRecompilerNativeWideBufferIr(); @@ -11720,13 +11451,13 @@ int main() { TestNewShaderRecompilerPrunesUnreachableSetpcMetadata(); TestNewShaderRecompilerSetpcDwordJumpTable(); TestTypedEntryStateIsMinimal(); - TestWqmConstantPropagation(); + TestWqmMaskSignatureAndU64ShiftConstantPropagation(); TestFinalSsaRejectsRegisterStatePseudos(); TestValuePhiValidation(); TestNativeWideValueValidation(); TestNewShaderRecompilerZeroInitialRegisterState(); TestNewShaderRecompilerVertexSystemInputsWithoutMirrors(); - TestNewShaderRecompilerVertexExportUsesLaneExecMask(); + TestNewShaderRecompilerVertexExportUsesInvocationExecMask(); TestNewShaderRecompilerPerInvocationMasksWithoutMirrors(); TestNewShaderRecompilerPerInvocationU64Complement(); TestNewShaderRecompilerExpPixelOutputs();