From 337f4320165f21759687333331ea3db9e8b52720 Mon Sep 17 00:00:00 2001 From: nmzik Date: Thu, 1 Oct 2026 07:55:28 +0400 Subject: [PATCH] shader: execute PS5 BVH intersections and local FLAT stacks Co-authored-by: Ali Almohaya Co-authored-by: Mac Co-authored-by: Senaxx Co-authored-by: Jean-Philippe --- .../renderer/pipeline/pipelineCache.cpp | 9 - .../host_gpu/renderer/renderCompute.cpp | 8 - .../shader/recompiler/ShaderRecompiler.cpp | 16 - .../shader/recompiler/ShaderRecompiler.h | 1 - .../recompiler/backend/spirv/SpirvEmitter.cpp | 20 +- .../backend/spirv/spirvEmitterBvh.cpp | 348 ++++++++++++++++++ .../backend/spirv/spirvEmitterInstructions.h | 1 + .../backend/spirv/spirvEmitterInternal.h | 6 + .../backend/spirv/spirvEmitterMemory.cpp | 114 ++++-- .../backend/spirv/spirvEmitterProgram.cpp | 1 + .../recompiler/frontend/decode/ImageOps.cpp | 7 + .../frontend/decode/ShaderDecoder.cpp | 9 +- .../frontend/decode/ShaderDecoder.h | 8 +- .../recompiler/frontend/translate/Memory.cpp | 9 + .../frontend/translate/Translate.cpp | 8 + src/graphics/shader/recompiler/ir/ShaderIR.h | 4 +- .../recompiler/ir/opcodes/ValueOpcodes.inc | 1 + .../recompiler/ir/passes/ResourceTracking.cpp | 41 +++ tests/ShaderRayTracingGpuTests.inc | 229 ++++++++++++ tests/ShaderRayTracingTests.inc | 127 +++---- tests/ShaderRecompilerComputeTests.cpp | 22 ++ tests/shaderCfgTests.cpp | 26 +- 22 files changed, 859 insertions(+), 156 deletions(-) create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp create mode 100644 tests/ShaderRayTracingGpuTests.inc diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index 6be59b83..9760753a 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -220,7 +220,6 @@ struct PipelineCache::ProgramCache { ShaderRecompiler::IR::ResourceSnapshot resources; ShaderRecompiler::IR::ResourceSpecialization specialization; std::vector permutations; - bool skip_dispatch = false; }; struct ProgramKeyHash { @@ -288,9 +287,6 @@ struct PipelineCache::ProgramCache { lookup_key.code_size = static_cast(params.code.size()); BuildStageStaticKey(input_info, lookup_key.static_state); auto entry = programs.find(lookup_key); - if (entry != programs.end() && entry->second.skip_dispatch) { - return {}; - } const ShaderRecompiler::IR::SrtRuntime runtime { .user_data = user_data, .shader_base = params.Base(), @@ -358,11 +354,6 @@ struct PipelineCache::ProgramCache { } DumpShaderOriginal(stage_name, options.shader_hash, params.code); auto translated = ShaderRecompiler::TranslateProgram(params.code, options); - if (translated.skip_dispatch) { - entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan {}).first; - entry->second.skip_dispatch = true; - return {}; - } if (entry == programs.end()) { entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first; diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 1a0c6c6b..715ef9af 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -256,10 +256,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, input_info.dispatch_thread_dimensions = use_thread_dimensions; const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info); - if (!compute_program) { - // Temporary until RT is implemented. - return; - } if (use_thread_dimensions) { input_info.dispatch_threads_num[0] = thread_group_x; input_info.dispatch_threads_num[1] = thread_group_y; @@ -417,10 +413,6 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, ShaderComputeInputInfo input_info {}; const auto compute_program = m_context.GetPipelineCache().GetComputeProgram( cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info); - if (!compute_program) { - // Temporary until RT is implemented. - return; - } buffer.EndRendering(); auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program); auto& bindings = m_compute_bindings; diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index b930791f..8a0d6cc2 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include @@ -523,20 +522,6 @@ TranslateResult TranslateProgram(std::span code, const CompileOp GetDumpLabel(options), StageName(options.stage), options.shader_hash, static_cast(decoded.instructions.size()), phase_ms()); - // Temporary workaround for games that compile ray-tracing shaders before - // the player can select a mode without ray tracing. - if (options.stage == ShaderType::Compute && decoded.has_bvh) { - static std::atomic_flag warned = ATOMIC_FLAG_INIT; - if (!warned.test_and_set(std::memory_order_relaxed)) { - const auto& bvh = decoded.instructions.back(); - Log::WriteToConsoleAndLog(fmt::format( - "Warning: ray tracing is not implemented; skipping compute dispatches containing " - "BVH intersection instructions (shader=0x{:016x}, pc=0x{:08x}, opcode=0x{:02x}).\n", - options.shader_hash, bvh.pc, bvh.opcode_id)); - } - return {.skip_dispatch = true}; - } - std::string decoded_dump; if (options.dump_ir) { decoded_dump = Decoder::ProgramToString(decoded); @@ -645,7 +630,6 @@ TranslateResult TranslateProgram(std::span code, const CompileOp CompileResult CompileProgram(TranslateResult translated, const CompileOptions& options, const IR::ResourceSpecialization& specialization, uint32_t push_data_start_dword) { - EXIT_IF(translated.skip_dispatch); const auto emit_begin = std::chrono::steady_clock::now(); auto& ir = translated.program; IR::ApplyResourceSpecialization(ir, specialization); diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h index 8e8912e7..29b3a40e 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.h +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -28,7 +28,6 @@ struct TranslateResult { IR::Program program; std::string decoded_dump; std::string cfg_dump; - bool skip_dispatch = false; }; struct CompileResult { diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index ee6d5644..5910f349 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -122,6 +122,16 @@ void ValidateNativeProgram(const IR::Program& program) { program.memory_info[index].kind == IR::ResourceKind::IndirectBuffer; }); }; + const auto local_flat_handle = [&](const IR::Inst& handle) { + return !handle.Uses().empty() && + std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) { + if (IR::AddressOpcodeInfoOf(use.user->GetOpcode()).access == IR::AddressAccess::None) + return false; + const auto index = use.user->Flags().index; + return index < program.memory_info.size() && + program.memory_info[index].kind == IR::ResourceKind::FlatLocal; + }); + }; for (const auto* block: program.blocks) { for (const auto& inst: *block) { const auto dense = inst.Flags(); @@ -138,7 +148,7 @@ void ValidateNativeProgram(const IR::Program& program) { if (planning_only_handle(inst)) { break; } - if (inst.NumArgs() != 2 || !program.info.uses_dma) { + if (inst.NumArgs() != 2 || (!program.info.uses_dma && !local_flat_handle(inst))) { Fail(program, "typed address handle has invalid DMA metadata"); } break; @@ -188,11 +198,16 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program if (memory_index >= program.memory_info.size()) { Fail(program, "address operation has invalid memory metadata"); } - if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) { + const auto kind = program.memory_info[memory_index].kind; + if (kind == IR::ResourceKind::Scratch || kind == IR::ResourceKind::FlatLocal) { if (program.scratch_dwords == 0) { Fail(program, "scratch operation has no per-thread storage"); } requirements.function_scratch = true; + if (kind == IR::ResourceKind::FlatLocal && program.stage != ShaderType::Compute && + program.stage != ShaderType::Mesh) { + requirements.function_lds = true; + } } else if (address_access == IR::AddressAccess::Write) { Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking"); } @@ -247,6 +262,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program } } switch (inst.GetOpcode()) { + case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break; case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break; case IR::ValueOpcode::DppMoveU32: case IR::ValueOpcode::ReadFirstLane: diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp new file mode 100644 index 00000000..1389054f --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp @@ -0,0 +1,348 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { +namespace { + +using Vec3 = std::array; + +uint32_t Extract(EmitterState& s, uint32_t type, uint32_t vector, uint32_t index) { + const auto result = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpCompositeExtract, type, result, vector, index); + return result; +} + +uint32_t Vector(EmitterState& s, uint32_t type, std::span components) { + const auto result = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpCompositeConstruct, type, result, components); + return result; +} + +uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) { + const auto offset = Binary(s, spv::OpShiftLeftLogical, TypeU32(s), word, ConstantU32(s, 2)); + const auto address = Binary(s, spv::OpIAdd, TypeScalarU64(s), block, + Unary(s, spv::OpUConvert, TypeScalarU64(s), offset)); + const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address); + const auto result = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpLoad, TypeU32(s), result, pointer, spv::MemoryAccessAlignedMask, 4u); + return result; +} + +uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary, + Vec3 origin, Vec3 direction) { + const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s); + const auto zero = ConstantF32Value(s, 0.0f); + const auto op = [&](spv::Op code, uint32_t x, uint32_t y) { + const auto value = Binary(s, code, f, x, y); + s.builder.AddAnnotation(spv::OpDecorate, value, spv::DecorationNoContraction); + return value; + }; + const auto cmp = [&](spv::Op code, uint32_t x, uint32_t y) { return Binary(s, code, b, x, y); }; + const auto either = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); }; + const auto both = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); }; + const auto abs = [&](uint32_t x) { return EmitGlsl(s, x); }; + const auto is_kind = [&](uint32_t n) { return cmp(spv::OpIEqual, kind, ConstantU32(s, n)); }; + // The four triangles are (0,1,2), (1,3,2), (2,3,4), (2,4,0). + const std::array vertices { + EmitGlsl(s, kind, ConstantU32(s, 2)), + Select(s, u, is_kind(0), ConstantU32(s, 1), + Select(s, u, is_kind(3), ConstantU32(s, 4), ConstantU32(s, 3))), + Select(s, u, is_kind(2), ConstantU32(s, 4), + Select(s, u, is_kind(3), ConstantU32(s, 0), ConstantU32(s, 2))) }; + std::array positions; + for (uint32_t i = 0; i < 3; ++i) { + const auto first = Binary(s, spv::OpIMul, u, vertices[i], ConstantU32(s, 3)); + for (uint32_t axis = 0; axis < 3; ++axis) + positions[i][axis] = Unary(s, spv::OpBitcast, f, + NodeWord(s, block, Binary(s, spv::OpIAdd, u, first, ConstantU32(s, axis)))); + } + const auto flag = NodeWord(s, block, ConstantU32(s, 15)); + const auto y_largest = cmp(spv::OpFOrdLessThan, abs(direction[0]), abs(direction[1])); + const auto z_largest = cmp(spv::OpFOrdLessThan, + Select(s, f, y_largest, abs(direction[1]), abs(direction[0])), abs(direction[2])); + const auto rotate = [&](Vec3 value) { + Vec3 result; + for (uint32_t i = 0; i < 3; ++i) + result[i] = Select(s, f, z_largest, value[i], + Select(s, f, y_largest, value[(i + 2) % 3], value[(i + 1) % 3])); + return result; + }; + origin = rotate(origin); + direction = rotate(direction); + std::array projected; + for (uint32_t i = 0; i < 3; ++i) { + const auto position = rotate(positions[i]); + Vec3 relative; + for (uint32_t axis = 0; axis < 3; ++axis) + relative[axis] = op(spv::OpFSub, position[axis], origin[axis]); + projected[i] = {op(spv::OpFSub, op(spv::OpFMul, relative[0], direction[2]), + op(spv::OpFMul, direction[0], relative[2])), + op(spv::OpFSub, op(spv::OpFMul, relative[1], direction[2]), + op(spv::OpFMul, direction[1], relative[2])), relative[2]}; + } + Vec3 edge, weighted; + for (uint32_t i = 0; i < 3; ++i) { + const auto& first = projected[(i + 1) % 3]; + const auto& second = projected[(i + 2) % 3]; + edge[i] = op(spv::OpFSub, op(spv::OpFMul, second[0], first[1]), + op(spv::OpFMul, second[1], first[0])); + weighted[i] = op(spv::OpFMul, edge[i], direction[2]); + } + auto numerator = op(spv::OpFAdd, op(spv::OpFAdd, + op(spv::OpFMul, edge[0], projected[0][2]), op(spv::OpFMul, edge[1], projected[1][2])), + op(spv::OpFMul, edge[2], projected[2][2])); + auto denominator = op(spv::OpFAdd, op(spv::OpFAdd, weighted[0], weighted[1]), weighted[2]); + uint32_t negative = ConstantBool(s, false), positive = negative; + for (auto value: edge) { + negative = either(negative, cmp(spv::OpFOrdLessThan, value, zero)); + positive = either(positive, cmp(spv::OpFOrdGreaterThan, value, zero)); + } + const auto winding = cmp(spv::OpFOrdGreaterThan, denominator, zero); + auto missed = either(both(negative, positive), cmp(spv::OpFOrdEqual, denominator, zero)); + missed = either(missed, Unary(s, spv::OpIsNan, b, numerator)); + missed = either(missed, cmp(spv::OpFOrdLessThan, + Select(s, f, winding, numerator, Unary(s, spv::OpFNegate, f, numerator)), zero)); + for (uint32_t i = 0; i < 3; ++i) { + const auto first = projected[(i + 1) % 3][1], second = projected[(i + 2) % 3][1]; + const auto first_zero = cmp(spv::OpFOrdEqual, first, zero); + const auto horizontal = both(first_zero, cmp(spv::OpFOrdEqual, second, zero)); + const auto below = either(cmp(spv::OpFOrdLessThan, first, zero), + both(first_zero, cmp(spv::OpFOrdGreaterThan, second, zero))); + const auto right = Binary(s, spv::OpLogicalNotEqual, b, below, winding); + const auto excluded = Select(s, b, horizontal, + cmp(spv::OpFOrdGreaterThan, projected[i][1], zero), right); + missed = either(missed, both(cmp(spv::OpFOrdEqual, edge[i], zero), excluded)); + } + const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag, + Binary(s, spv::OpIMul, u, kind, ConstantU32(s, 8))); + const auto coordinate = [&](uint32_t shift) { + const auto index = Binary(s, spv::OpBitwiseAnd, u, + Binary(s, spv::OpShiftRightLogical, u, remap, ConstantU32(s, shift)), ConstantU32(s, 3)); + return Unary(s, spv::OpBitcast, u, Select(s, f, + cmp(spv::OpIEqual, index, ConstantU32(s, 1)), weighted[1], + Select(s, f, cmp(spv::OpIEqual, index, ConstantU32(s, 2)), weighted[2], weighted[0]))); + }; + const std::array result { + Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32(s, 0x7f800000), numerator)), + Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32Value(s, 1.0f), denominator)), + Select(s, u, bary, coordinate(0), Binary(s, spv::OpIAdd, u, flag, kind)), + Select(s, u, bary, coordinate(2), Select(s, u, missed, ConstantU32(s, 0), ConstantU32(s, 1))) }; + return Vector(s, TypeU32Vector(s, 4), result); +} + +std::array Box(EmitterState& s, uint32_t first, uint32_t second, + uint32_t half, uint32_t child) { + const auto f = TypeF32(s); + const auto half_label = s.builder.AllocateId(), full_label = s.builder.AllocateId(); + const auto merge = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + s.builder.AddFunction(spv::OpBranchConditional, half, half_label, full_label); + EmitLabel(s, half_label); + std::array packed, full, result; + for (uint32_t pair = 0; pair < 3; ++pair) { + const auto word = NodeWord(s, first, ConstantU32(s, 4 + child * 3 + pair)); + const auto unpacked = EmitGlsl(s, word); + packed[pair * 2] = Extract(s, f, unpacked, 0); + packed[pair * 2 + 1] = Extract(s, f, unpacked, 1); + } + s.builder.AddFunction(spv::OpBranch, merge); + EmitLabel(s, full_label); + for (uint32_t component = 0; component < 6; ++component) { + const auto word = 4 + child * 6 + component; + full[component] = Unary(s, spv::OpBitcast, f, + NodeWord(s, word < 16 ? first : second, ConstantU32(s, word % 16))); + } + s.builder.AddFunction(spv::OpBranch, merge); + EmitLabel(s, merge); + for (uint32_t component = 0; component < 6; ++component) { + result[component] = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpPhi, f, result[component], packed[component], half_label, + full[component], full_label); + } + return result; +} + +uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half, + uint32_t grow, uint32_t sort, uint32_t extent, const Vec3& origin, const Vec3& inverse) { + const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s); + const auto zero = ConstantF32Value(s, 0.0f); + std::array depths, indices; + Vec3 forward; + for (uint32_t axis = 0; axis < 3; ++axis) + forward[axis] = Binary(s, spv::OpFOrdGreaterThanEqual, b, inverse[axis], zero); + for (uint32_t child = 0; child < 4; ++child) { + const auto box = Box(s, first, second, half, child); + Vec3 near, far; + for (uint32_t axis = 0; axis < 3; ++axis) { + const auto lo = Binary(s, spv::OpFMul, f, + Binary(s, spv::OpFSub, f, box[axis], origin[axis]), inverse[axis]); + const auto hi = Binary(s, spv::OpFMul, f, + Binary(s, spv::OpFSub, f, box[axis + 3], origin[axis]), inverse[axis]); + near[axis] = Select(s, f, forward[axis], lo, hi); + far[axis] = Select(s, f, forward[axis], hi, lo); + } + const auto entry = EmitGlsl(s, + EmitGlsl(s, near[0], near[1]), near[2]); + const auto exit = EmitGlsl(s, + EmitGlsl(s, far[0], far[1]), far[2]); + // Valid hits have nonnegative exit distance. Saturate ULP growth at +infinity. + const auto bits = Unary(s, spv::OpBitcast, u, Select(s, f, + Binary(s, spv::OpFOrdEqual, b, exit, zero), zero, exit)); + const auto grown_bits = EmitGlsl(s, + Binary(s, spv::OpIAdd, u, bits, grow), ConstantU32(s, 0x7f800000)); + const auto grown = Unary(s, spv::OpBitcast, f, grown_bits); + auto hit = Binary(s, spv::OpLogicalAnd, b, + Binary(s, spv::OpFOrdGreaterThanEqual, b, exit, zero), + Binary(s, spv::OpFOrdLessThan, b, entry, extent)); + hit = Binary(s, spv::OpLogicalAnd, b, hit, + Binary(s, spv::OpFOrdLessThanEqual, b, entry, grown)); + depths[child] = Unary(s, spv::OpBitcast, u, Select(s, f, hit, + Select(s, f, Binary(s, spv::OpFOrdGreaterThan, b, entry, zero), entry, zero), + ConstantF32(s, 0x7f800000))); + indices[child] = Select(s, u, hit, NodeWord(s, first, ConstantU32(s, child)), ConstantU32(s, ~0u)); + } + // A select network keeps the original child order when sorting is disabled. + constexpr std::array, 5> network {{{0,1}, {2,3}, {0,2}, {1,3}, {1,2}}}; + for (size_t step = 0; step < network.size(); ++step) { + const auto [a, c] = network[step]; + const auto swap = Binary(s, spv::OpLogicalAnd, b, sort, + Binary(s, spv::OpUGreaterThan, b, depths[a], depths[c])); + const auto depth = depths[a], index = indices[a]; + const auto later = [&](uint32_t position) { + return std::ranges::any_of(std::span(network).subspan(step + 1), [=](auto pair) { + return pair.first == position || pair.second == position; + }); + }; + if (later(a)) depths[a] = Select(s, u, swap, depths[c], depth); + if (later(c)) depths[c] = Select(s, u, swap, depth, depths[c]); + indices[a] = Select(s, u, swap, indices[c], index); + indices[c] = Select(s, u, swap, index, indices[c]); + } + return Vector(s, TypeU32Vector(s, 4), indices); +} + +} // namespace + +void DefineBvhIntersect(EmitterState& s) { + if (!s.requirements.bvh) return; + const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s); + const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3); + const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, u, f, vec3, vec3, vec3); + s.bvh_intersect_function = s.builder.AllocateId(); + s.builder.AddName(s.bvh_intersect_function, "bvh_intersect"); + s.builder.AddFunction(spv::OpFunction, vec4, s.bvh_intersect_function, + spv::FunctionControlMaskNone, signature); + std::array args; + const std::array types {vec4, u, f, vec3, vec3, vec3}; + for (uint32_t i = 0; i < args.size(); ++i) { + args[i] = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpFunctionParameter, types[i], args[i]); + } + EmitLabel(s, s.builder.AllocateId()); + const auto descriptor = args[0], node = args[1], extent = args[2]; + Vec3 origin, direction, inverse; + for (uint32_t i = 0; i < 3; ++i) { + origin[i] = Extract(s, f, args[3], i); + direction[i] = Extract(s, f, args[4], i); + inverse[i] = Extract(s, f, args[5], i); + } + std::array words; + for (uint32_t i = 0; i < 4; ++i) words[i] = Extract(s, u, descriptor, i); + const auto and_bits = [&](uint32_t value, uint32_t mask) { + return Binary(s, spv::OpBitwiseAnd, u, value, ConstantU32(s, mask)); + }; + const auto shr = [&](uint32_t value, uint32_t shift) { + return Binary(s, spv::OpShiftRightLogical, u, value, ConstantU32(s, shift)); + }; + const auto kind = and_bits(node, 7); + const auto triangle = Binary(s, spv::OpULessThan, b, kind, ConstantU32(s, 4)); + const auto full = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 5)); + const auto half = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 4)); + const auto bary = Binary(s, spv::OpINotEqual, b, and_bits(words[3], 0x01000000), ConstantU32(s, 0)); + const auto sort = Binary(s, spv::OpINotEqual, b, and_bits(words[1], 0x80000000), ConstantU32(s, 0)); + const auto grow = and_bits(shr(words[1], 23), 0xff); + const auto base = Binary(s, spv::OpShiftLeftLogical, wide, + DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8)); + const auto index = Unary(s, spv::OpUConvert, wide, shr(node, 3)); + const auto last_index = Binary(s, spv::OpIAdd, wide, index, + Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0))); + const auto address = Binary(s, spv::OpIAdd, wide, base, + Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantDeviceAddress(s, 6))); + const auto last_address = Binary(s, spv::OpIAdd, wide, base, + Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantDeviceAddress(s, 6))); + auto valid = Binary(s, spv::OpIEqual, b, shr(words[3], 28), ConstantU32(s, 8)); + const auto require = [&](uint32_t condition) { + valid = Binary(s, spv::OpLogicalAnd, b, valid, condition); + }; + require(Binary(s, spv::OpULessThanEqual, b, kind, ConstantU32(s, 5))); + require(Binary(s, spv::OpULessThanEqual, b, last_index, DeviceAddressFromWords(s, words[2], and_bits(words[3], 0x3ff)))); + require(Binary(s, spv::OpULessThan, b, last_address, ConstantDeviceAddress(s, uint64_t{1} << 48))); + require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, extent))); + for (uint32_t i = 0; i < 3; ++i) { + for (const auto value: {origin[i], direction[i], inverse[i]}) + require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, value))); + require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsInf, b, origin[i]))); + } + const std::array triangle_invalid {ConstantU32(s, 0x7f800000), ConstantU32(s, 0x3f800000), + ConstantU32(s, ~0u), ConstantU32(s, 0)}; + const std::array box_invalid {ConstantU32(s, ~0u), ConstantU32(s, ~0u), + ConstantU32(s, ~0u), ConstantU32(s, ~0u)}; + std::array invalid_words; + for (uint32_t i = 0; i < 4; ++i) + invalid_words[i] = Select(s, u, triangle, triangle_invalid[i], box_invalid[i]); + const auto invalid = Vector(s, vec4, invalid_words); + const auto result = EmitValueOrDefaultIfCondition(s, valid, vec4, invalid, [&] { + const auto first = GetBdaPointer(s, address); + const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantDeviceAddress(s, 0), [&] { + return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantDeviceAddress(s, 64))); + }); + const auto present = Binary(s, spv::OpLogicalAnd, b, + Binary(s, spv::OpINotEqual, b, first, ConstantDeviceAddress(s, 0)), + Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, full), + Binary(s, spv::OpINotEqual, b, second, ConstantDeviceAddress(s, 0)))); + return EmitValueOrDefaultIfCondition(s, present, vec4, invalid, [&] { + const auto tri_label = s.builder.AllocateId(), box_label = s.builder.AllocateId(); + const auto merge = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + s.builder.AddFunction(spv::OpBranchConditional, triangle, tri_label, box_label); + EmitLabel(s, tri_label); + const auto tri_value = Triangle(s, first, kind, bary, origin, direction); + const auto tri_exit = s.current_label; + s.builder.AddFunction(spv::OpBranch, merge); + EmitLabel(s, box_label); + const auto box_value = Boxes(s, first, second, half, grow, sort, extent, origin, inverse); + const auto box_exit = s.current_label; + s.builder.AddFunction(spv::OpBranch, merge); + EmitLabel(s, merge); + const auto value = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpPhi, vec4, value, tri_value, tri_exit, box_value, box_exit); + return value; + }); + }); + s.builder.AddFunction(spv::OpReturnValue, result); + s.builder.AddFunction(spv::OpFunctionEnd); +} + +uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& s = ctx.state; + const auto* ray = ctx.ImageAddress(inst.Arg(1)); + if (ray == nullptr || ray->NumArgs() < 11 || s.bvh_intersect_function == 0) + ctx.Fail(inst, "BVH intersection requires eleven address components and its shared function"); + const auto vector = [&](uint32_t first) { + Vec3 values; + for (uint32_t i = 0; i < 3; ++i) + values[i] = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, first + i)); + return Vector(s, TypeF32Vector(s, 3), values); + }; + const auto descriptor = ctx.Arg(inst, 0), node = ctx.Arg(*ray, 0); + const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, 1)); + const auto origin = vector(2), direction = vector(5), inverse = vector(8); + return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), TypeU32Vector(s, 4), + ConstantU32CompositeZero(s, 4), [&] { + const auto result = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpFunctionCall, TypeU32Vector(s, 4), result, + s.bvh_intersect_function, descriptor, node, extent, origin, direction, inverse); + return result; + }); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index bc74f0eb..f18314d2 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -294,6 +294,7 @@ inline constexpr auto EmitDataAppend = EmitAppendConsume; inline constexpr auto EmitDataConsume = EmitAppendConsume; uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst); void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst); inline constexpr auto EmitImageQueryDimensions = EmitImage; inline constexpr auto EmitImageQueryLod = EmitImage; inline constexpr auto EmitImageRead = EmitImage; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 17d68a6a..7c5fb594 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -58,6 +58,7 @@ constexpr std::array ImageDimensions {{ const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension); struct SpirvRequirements { + bool bvh = false; bool subgroup_ballot = false; bool subgroup_shuffle = false; bool subgroup_local_invocation_id = false; @@ -96,6 +97,7 @@ struct EmitterState { uint32_t bda_pagetable_variable = 0; uint32_t fault_buffer_variable = 0; uint32_t bda_pointer_function = 0; + uint32_t bvh_intersect_function = 0; uint32_t gds_variable = 0; uint32_t gds_length = 0; uint32_t push_constant_variable = 0; @@ -487,6 +489,10 @@ uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits); void EmitProgram(EmitterState& state); void DefineGetBdaPointer(EmitterState& state); +void DefineBvhIntersect(EmitterState& state); +uint32_t GetBdaPointer(EmitterState& state, uint32_t address); +uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value); +uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high); // These templates accept local lambdas from several emitter translation units. template diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 741d9428..91520d40 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -1,6 +1,7 @@ #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h" #include "graphics/host_gpu/renderer/cache/bufferCache.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include @@ -105,31 +106,23 @@ uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t ad return result; } -uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low, - uint32_t high) { - auto& state = ctx.state; - auto immediate = static_cast(mem.offset); +uint32_t AddAddressOffset(EmitterState& state, const IR::MemoryInfo& mem, uint32_t low, + uint32_t& high) { + const auto immediate = static_cast(mem.offset); + if (immediate == 0) return low; const auto immediate_low = ConstantU32(state, static_cast(immediate)); const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u); - low = AddU64Low(state, low, high, immediate_low, immediate_high, high); + return AddU64Low(state, low, high, immediate_low, immediate_high, high); +} + +uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low, + uint32_t high) { + auto& state = ctx.state; + low = AddAddressOffset(state, mem, low, high); const auto valid = Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, 0)); return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX)); } -uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) { - return state.builder.Constant(spv::OpConstant, TypeScalarU64(state), - static_cast(value), - static_cast(value >> 32u)); -} - -uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) { - const auto low64 = Unary(state, spv::OpUConvert, TypeScalarU64(state), low); - const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state), - Unary(state, spv::OpUConvert, TypeScalarU64(state), high), - ConstantDeviceAddress(state, 32)); - return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64); -} - uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { auto& state = ctx.state; auto low = ctx.Arg(inst, 1); @@ -186,17 +179,9 @@ void RecordBdaFault(EmitterState& state, uint32_t page) { Binary(state, spv::OpBitwiseOr, TypeU32(state), value, bit)); } -uint32_t GetBdaPointer(ValueEmitContext& ctx, uint32_t address) { - auto& state = ctx.state; - const auto result = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result, - state.bda_pointer_function, address); - return result; -} - uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) { auto& state = ctx.state; - const auto bda = GetBdaPointer(ctx, address); + const auto bda = GetBdaPointer(state, address); const auto present = Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantDeviceAddress(state, 0)); return EmitValueOrZeroIfCondition(state, present, [&]() { @@ -511,6 +496,50 @@ void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resour resource.memory_access); } +template +void ForEachLocalFlatAccess(ValueEmitContext& ctx, const IR::Inst& inst, Fn&& emit) { + auto& state = ctx.state; + auto mem = ctx.Memory(inst); + auto high = ctx.Arg(inst, 2); + const auto low = AddAddressOffset(state, mem, ctx.Arg(inst, 1), high); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), low, ConstantU32(state, 2)); + const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), ConstantU32(state, 0), + Binary(state, spv::OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, 3))); + for (const auto kind: {IR::ResourceKind::Scratch, IR::ResourceKind::Lds}) { + mem.kind = kind; + const auto resource = PrepareMemoryResourceAccess(state, mem); + const auto aperture = kind == IR::ResourceKind::Scratch ? Decoder::PrivateApertureHigh + : Decoder::SharedApertureHigh; + const auto selected = Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, aperture)); + const auto valid = AndCondition(state, aligned, + AndCondition(state, selected, EmitMemoryElementInBounds(state, resource, index))); + emit(resource, index, valid); + } +} + +uint32_t LoadLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) { + return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + uint32_t result = 0; + ForEachLocalFlatAccess(ctx, inst, [&](const auto& resource, uint32_t index, uint32_t valid) { + const auto value = EmitValueOrZeroIfCondition(ctx.state, valid, [&]() { + return LoadWordInBounds(ctx, resource, index); + }); + result = result == 0 ? value : Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state), result, value); + }); + return result; + }); +} + +void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) { + EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + ForEachLocalFlatAccess(ctx, inst, [&](const auto& resource, uint32_t index, uint32_t valid) { + EmitIfCondition(ctx.state, valid, [&]() { + StoreWordInBounds(ctx, resource, index, ctx.Arg(inst, inst.NumArgs() - 2)); + }); + }); + }); +} + void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, uint32_t data) { const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem)); @@ -961,6 +990,27 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo } // namespace +uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) { + return state.builder.Constant(spv::OpConstant, TypeScalarU64(state), + static_cast(value), + static_cast(value >> 32u)); +} + +uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) { + const auto low64 = Unary(state, spv::OpUConvert, TypeScalarU64(state), low); + const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state), + Unary(state, spv::OpUConvert, TypeScalarU64(state), high), + ConstantDeviceAddress(state, 32)); + return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64); +} + +uint32_t GetBdaPointer(EmitterState& state, uint32_t address) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result, + state.bda_pointer_function, address); + return result; +} + void DefineGetBdaPointer(EmitterState& state) { if (!state.program.info.uses_dma) { return; @@ -1243,7 +1293,9 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) { const auto shared_components = IR::SharedComponentCount(op); const auto address_info = IR::AddressOpcodeInfoOf(op); uint32_t value; - if (buffer_components > 1u) + if (mem.kind == IR::ResourceKind::FlatLocal) + value = LoadLocalFlat(ctx, inst); + else if (buffer_components > 1u) value = LoadWideBuffer(ctx, inst, buffer_components); else if (shared_components > 1u) value = LoadWideShared(ctx, inst, shared_components); @@ -1270,7 +1322,9 @@ void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst) { const auto buffer_components = IR::BufferComponentCount(op); const auto shared_components = IR::SharedComponentCount(op); const auto type = inst.Arg(inst.NumArgs() - 2).GetType(); - if (buffer_components > 1u) + if (mem.kind == IR::ResourceKind::FlatLocal) + StoreLocalFlat(ctx, inst); + else if (buffer_components > 1u) StoreWideBuffer(ctx, inst, buffer_components); else if (shared_components > 1u) StoreWideShared(ctx, inst, shared_components); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 6f3b0632..25286e3d 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -679,6 +679,7 @@ void EmitProgram(EmitterState& state) { } } DefineGetBdaPointer(state); + DefineBvhIntersect(state); for (const auto* block: program.blocks) { if (std::ranges::any_of(*block, [](const IR::Inst& inst) { return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 || diff --git a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp index b42248fc..972f979d 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp @@ -225,6 +225,7 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim case 0x09u: return Opcode::IMAGE_STORE_MIP; case 0x0eu: return Opcode::IMAGE_GET_RESINFO; case 0x60u: return Opcode::IMAGE_GET_LOD; + case 0xe6u: return Opcode::IMAGE_BVH_INTERSECT_RAY; default: return Opcode::UNSUPPORTED; } } @@ -253,6 +254,7 @@ uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension, } switch (opcode) { + case 0xe6u: return 11u; case 0x0eu: return 1u; case 0x01u: case 0x09u: return ImageCoordComponents(dimension) + 1u; @@ -360,6 +362,11 @@ void DecodeMimg(uint32_t pc, std::span code, uint32_t word_index if (d16 && !supports_d16) { SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data"); } + if (opcode == 0xe6u && + (a16 || !r128 || inst.dmask != 0xfu || (nsa_dwords != 0u && nsa_dwords != 3u))) { + SetUnsupported(inst, Family::MIMG, opcode, + "BVH intersection requires eleven full-float ray DWORDs and R128/dmask:0xf"); + } DecodeVectorGpr(vdata, inst.dst); DecodeVectorGpr(vaddr, inst.src0); diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index dae75e56..a65bba2a 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -254,6 +254,8 @@ void DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand) { case 125u: operand.kind = OperandKind::Null; return; case 126u: operand.kind = OperandKind::ExecLo; return; case 127u: operand.kind = OperandKind::ExecHi; return; + case 235u: operand.kind = OperandKind::SharedBase; return; + case 237u: operand.kind = OperandKind::PrivateBase; return; case 239u: operand.kind = OperandKind::PopsExitingWaveId; return; case 248u: operand.kind = OperandKind::FloatInlineConstant; @@ -404,7 +406,6 @@ void DecodeProgram(std::span code, Program& program) { program.instructions.clear(); program.instructions.reserve(code.size()); program.code = code; - program.has_bvh = false; std::vector branch_targets; for (uint32_t word_index = 0; word_index < code.size();) { @@ -413,10 +414,6 @@ void DecodeProgram(std::span code, Program& program) { const auto& inst = program.instructions.back(); word_index += inst.word_count; - if (inst.family == Family::MIMG && (inst.opcode_id == 0xe6u || inst.opcode_id == 0xe7u)) { - program.has_bvh = true; - return; - } if (IsDirectBranch(inst.opcode)) { const auto target_index = inst.branch_target / sizeof(uint32_t); @@ -455,6 +452,8 @@ std::string OperandToString(const Operand& operand) { case OperandKind::Scc: text = "scc"; break; case OperandKind::M0: text = "m0"; break; case OperandKind::PopsExitingWaveId: text = "pops_exiting_wave_id"; break; + case OperandKind::SharedBase: text = "shared_base"; break; + case OperandKind::PrivateBase: text = "private_base"; break; case OperandKind::Null: text = "null"; break; default: text = "unknown"; break; } diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 3da20d0f..02c185fc 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -10,6 +10,10 @@ namespace Libs::Graphics::ShaderRecompiler::Decoder { +// PS5 flat stack addresses use these SH_MEM_BASES aperture tags in VA[63:32]. +constexpr uint32_t PrivateApertureHigh = 0x70000000u; +constexpr uint32_t SharedApertureHigh = 0x80000000u; + enum class Family { Unknown, SOP1, @@ -569,6 +573,7 @@ enum class Opcode { DS_WRITE_B128, DS_WRITE_ADDTID_B32, DS_READ_ADDTID_B32, + IMAGE_BVH_INTERSECT_RAY, IMAGE_GET_RESINFO, IMAGE_GET_LOD, IMAGE_LOAD, @@ -636,6 +641,8 @@ enum class OperandKind { Scc, M0, PopsExitingWaveId, + SharedBase, + PrivateBase, Null, Vgpr, }; @@ -747,7 +754,6 @@ struct Instruction { struct Program { std::span code; std::vector instructions; - bool has_bvh = false; }; // Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs. diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 28fb49dd..a81da2eb 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -883,6 +883,15 @@ void Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) { void Translator::EmitMemory(const Decoder::Instruction& inst) { switch (inst.opcode) { + case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: { + const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect, + {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()}); + for (uint32_t component = 0; component < 4; ++component) { + WriteOperand(OffsetOperand(inst.dst, component), + ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(component)})); + } + return; + } case Decoder::Opcode::S_LOAD_DWORD: case Decoder::Opcode::S_LOAD_DWORDX2: case Decoder::Opcode::S_LOAD_DWORDX4: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 3479d5a7..25478144 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -135,6 +135,8 @@ IR::U32 Translator::ReadRawU32(const Decoder::Operand& operand) { case Decoder::OperandKind::IntegerInlineConstant: case Decoder::OperandKind::FloatInlineConstant: return IR::U32(IR::Value(operand.value)); case Decoder::OperandKind::Null: + case Decoder::OperandKind::SharedBase: + case Decoder::OperandKind::PrivateBase: case Decoder::OperandKind::PopsExitingWaveId: return IR::U32(IR::Value(0u)); case Decoder::OperandKind::Sgpr: return ir.GetScalarReg(static_cast(operand.reg)); @@ -474,6 +476,12 @@ IR::U32 Translator::ReadU32(const Decoder::Operand& operand) { } std::array Translator::ReadU32Pair(const Decoder::Operand& operand) { + if (operand.kind == Decoder::OperandKind::PrivateBase || + operand.kind == Decoder::OperandKind::SharedBase) { + return {IR::U32(IR::Value(0u)), IR::U32(IR::Value( + operand.kind == Decoder::OperandKind::PrivateBase ? Decoder::PrivateApertureHigh + : Decoder::SharedApertureHigh))}; + } if (operand.kind == Decoder::OperandKind::ExecLo) { return {ir.GetExecLo(), ir.GetExecHi()}; } diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 7380672f..e95830b5 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -30,6 +30,7 @@ enum class ResourceKind { Buffer, IndirectBuffer, Flat, + FlatLocal, Global, Scratch, Lds, @@ -40,7 +41,8 @@ enum class ResourceKind { [[nodiscard]] constexpr bool IsAddressResourceKind(ResourceKind kind) { return kind == ResourceKind::ScalarAddress || kind == ResourceKind::Flat || - kind == ResourceKind::Global || kind == ResourceKind::Scratch; + kind == ResourceKind::FlatLocal || kind == ResourceKind::Global || + kind == ResourceKind::Scratch; } struct MemoryInfo { diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 273079ca..6774ad8a 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -284,6 +284,7 @@ VALUE_OPCODE(DataConsume, U32, U32, U1) VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1) VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress) +VALUE_OPCODE(BvhIntersect, U32x4, U32x4, ImageAddress, U1) VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress) VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1) VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1) diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index ce795034..b76f19de 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -44,6 +44,34 @@ uint32_t PossibleU32Bits(Value value) { } } +bool IsNoncanonicalFlatAddress(Value high, Value active) { + high = high.Resolve(); + active = active.Resolve(); + // A predicated VGPR write supplies its new value on exactly the lanes which + // execute this access. Do not discard a selection under a different mask. + while (const auto* select = high.TryInstruction()) { + if (select->GetOpcode() != ValueOpcode::SelectU32 || select->Arg(0).Resolve() != active) break; + high = select->Arg(1).Resolve(); + } + uint32_t lower = 0, upper = UINT32_MAX; + if (high.IsImmediate() && high.GetType() == Type::U32) { + lower = upper = high.U32(); + } else if (const auto* inst = high.TryInstruction(); + inst != nullptr && inst->GetOpcode() == ValueOpcode::UMedTri32) { + // The median lies between any two constant inputs, regardless of the third. + uint32_t count = 0; + for (size_t index = 0; index < inst->NumArgs(); ++index) { + const auto value = inst->Arg(index).Resolve(); + if (!value.IsImmediate() || value.GetType() != Type::U32) continue; + if (count++ == 0) lower = upper = value.U32(); + else { lower = std::min(lower, value.U32()); upper = std::max(upper, value.U32()); } + } + if (count < 2) return false; + } + // Leave one high DWORD on each side for the signed instruction offset/carry. + return lower > 0x00008000u && upper < 0xffff7fffu; +} + Value CanonicalizeSampleAdjustDword3(Value value) { for (;;) { value = value.Resolve(); @@ -1649,6 +1677,10 @@ private: void Collect(Inst& inst) { const auto op = inst.GetOpcode(); + if (op == ValueOpcode::BvhIntersect) { + m_info.uses_dma = true; + return; + } const auto buffer = BufferAccessOf(op); const auto address_info = AddressOpcodeInfoOf(op); const auto image_info = ImageOpcodeInfoOf(op); @@ -1695,6 +1727,15 @@ private: if (!IsAddressResourceKind(memory.kind)) { Fail(flags.pc, "address operation has invalid resource kind"); } + if (memory.kind == ResourceKind::Flat && memory.address_is_full && + IsNoncanonicalFlatAddress(inst.Arg(2), inst.Arg(inst.NumArgs() - 1))) { + ValidateAddressHandle(inst.Arg(0), flags.pc); + if (memory.data_bits != 32u || m_program.scratch_dwords == 0) { + Fail(flags.pc, "local FLAT access requires DWORD data and per-thread scratch storage"); + } + m_program.memory_info[flags.index].kind = ResourceKind::FlatLocal; + return; + } if (memory.kind == ResourceKind::Scratch) { handle = inst.Arg(0).Resolve().TryInstruction(); if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetScratchResource || diff --git a/tests/ShaderRayTracingGpuTests.inc b/tests/ShaderRayTracingGpuTests.inc new file mode 100644 index 00000000..bae25ca3 --- /dev/null +++ b/tests/ShaderRayTracingGpuTests.inc @@ -0,0 +1,229 @@ +TestCase FlatStackApertures(u32 wave_size) { + TestCase test; + test.name = wave_size == 32 ? "FlatStackApertures32" : "FlatStackApertures64"; + auto &code = test.code; + code = {EncodeSop1(0x04, 8, 237), // private base + EncodeSop1(0x04, 10, 235), // shared base + EncodeVop2(0x1a, 20, InlineU32(2), 0), + EncodeVop2(0x1b, 22, InlineU32(1), 0), + EncodeVop2(0x1a, 22, InlineU32(31), 22)}; + AppendVop3(&code, 0x159, 21, 9, 11, Vgpr(22)); // Clamp the stack address to the local apertures. + AppendVMovU32(&code, 30, 100); + code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30)); + code.insert(code.end(), {EncodeFlat0(0x1c, 1), EncodeFlat1(0, 0x7d, 30, 20)}); + AppendVMovU32(&code, 30, 200); + code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30)); + code.insert(code.end(), {EncodeDs0(0x0d), EncodeDs1(0, 30, 20)}); + AppendVMovU32(&code, 30, 300); + code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30)); + code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20)}); + // FLAT writes must alias both explicit memory instructions, per invocation. + code.insert(code.end(), {EncodeFlat0(0x0c, 1), EncodeFlat1(24, 0x7d, 0, 20), + EncodeDs0(0x36), EncodeDs1(25, 0, 20), + EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)}); + AppendStoreVgprAtLaneDwordOffset(&code, 24, 0, 0); + AppendStoreVgprAtLaneDwordOffset(&code, 25, 0, wave_size); + AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 2 * wave_size); + // Inactive lanes must preserve both stores and destination VGPRs. + AppendSMovLiteral(&code, 126, 0x55555555); + if (wave_size == 64) AppendSMovLiteral(&code, 127, 0x55555555); + code.push_back(EncodeVop2(0x25, 22, InlineU32(1), 22)); + AppendVop3(&code, 0x159, 21, 9, 11, Vgpr(22)); + AppendVMovU32(&code, 30, 400); + code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30)); + code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20)}); + code.insert(code.end(), {EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)}); + code.push_back(EncodeSop1(0x04, 126, 193)); // -1 sign-extends across EXEC + AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 3 * wave_size); + // Noncanonical addresses outside either aperture, aperture overflow, and + // offsets outside allocated local storage must not wrap into valid storage. + AppendVMovLiteral(&code, 21, 0x78000000); + code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20), + EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)}); + AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 4 * wave_size); + code.push_back(EncodeVop1(0x01, 21, 11)); + AppendVMovLiteral(&code, 20, 0xfffffffc); + code.insert(code.end(), {EncodeFlat0(0x1c, 0, 4), EncodeFlat1(0, 0x7d, 30, 20), + EncodeFlat0(0x0c, 0, 4), EncodeFlat1(26, 0x7d, 0, 20)}); + AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 5 * wave_size); + AppendVMovU32(&code, 20, wave_size * 4); + code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20), + EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)}); + AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 6 * wave_size); + AppendVMovU32(&code, 20, 1); + code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20), + EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)}); + AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 9 * wave_size); + code.push_back(EncodeVop2(0x1a, 20, InlineU32(2), 0)); + code.insert(code.end(), {EncodeFlat0(0x0c, 1), EncodeFlat1(24, 0x7d, 0, 20), + EncodeDs0(0x36), EncodeDs1(25, 0, 20)}); + AppendStoreVgprAtLaneDwordOffset(&code, 24, 0, 7 * wave_size); + AppendStoreVgprAtLaneDwordOffset(&code, 25, 0, 8 * wave_size); + AppendEnd(&code); + test.expected.resize(10 * wave_size); + for (u32 lane = 0; lane < wave_size; ++lane) { + const bool shared = lane & 1; + test.expected[lane] = (shared ? 100 : 300) + lane; + test.expected[wave_size + lane] = (shared ? 300 : 200) + lane; + test.expected[2 * wave_size + lane] = 300 + lane; + test.expected[3 * wave_size + lane] = (shared ? 300 : 400) + lane; + test.expected[7 * wave_size + lane] = (shared ? 100 : 400) + lane; + test.expected[8 * wave_size + lane] = (shared ? 300 : 200) + lane; + } + test.compute_info.wave_size = wave_size; + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.scratch_size_dwords = wave_size; + test.compute_info.lds_size_dwords = wave_size; + test.has_compute_info = true; + test.forbidden_spirv = {"get_bda_pointer"}; + return test; +} + +TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0) { + TestCase test; + test.name = barycentrics ? (sorted ? "BvhBarycentricSorted" : "BvhBarycentricUnsorted") + : (sorted ? "BvhTriangleIdSorted" : "BvhTriangleIdUnsorted"); + if (grow != 0) test.name = "BvhBoxExitGrowth"; + constexpr u32 NodeStart = 1024, OutputStart = 512, LaneCount = 32; + constexpr uint64_t GuestBase = 0x10000; + const auto bits = [](float value) { return std::bit_cast(value); }; + constexpr u32 Infinity = 0x7f800000, Invalid = UINT32_MAX, Flag = 0x04040404; + test.initial.resize(NodeStart + 258 * 16); + auto &data = test.initial; + // Pack five float3 vertices and the triangle flag into a 64-byte node. + const std::array vertices{-1, -1, 2, 1, -1, 2, 0, 1, 2, 2, 1, 2, 0, 3, 2}; + for (u32 i = 0; i < vertices.size(); ++i) data[NodeStart + i] = bits(vertices[i]); + data[NodeStart + 15] = Flag; + constexpr std::array children{0x88, 0x90, 0x98, 0xa0}; + constexpr std::array z{5, 1, 3, 7}; + constexpr std::array half_z{0x4500, 0x3c00, 0x4200, 0x4700}; + constexpr std::array half_far{0x4600, 0x4000, 0x4400, 0x4800}; + for (u32 child = 0; child < 4; ++child) { + data[NodeStart + 16 + child] = data[NodeStart + 32 + child] = children[child]; + const auto fp16 = NodeStart + 20 + child * 3; + data[fp16] = 0xbc00bc00; // min.xy = -1,-1 + data[fp16 + 1] = 0x3c000000 | half_z[child]; + data[fp16 + 2] = (half_far[child] << 16) | 0x3c00; + const auto fp32 = NodeStart + 36 + child * 6; + const std::array bounds{-1, -1, z[child], 1, 1, z[child] + 1}; + for (u32 i = 0; i < 6; ++i) data[fp32 + i] = bits(bounds[i]); + } + // A grazing ray enters one ULP after exiting on another axis. + for (u32 child = 0; child < 4; ++child) { + data[NodeStart + 64 + child] = children[child]; + const std::array bounds = child == 0 + ? std::array{0, 0, std::nextafter(1.0f, 2.0f), 1, 2, 2} + : child == 1 ? std::array{-1, 1, -1, -0.0f, 2, 2} + : std::array{1, 1, 1, -1, -1, -1}; + for (u32 i = 0; i < 6; ++i) data[NodeStart + 68 + child * 6 + i] = bits(bounds[i]); + } + // An FP32 node straddles two guest pages with a 64-byte gap in physical backing. + std::copy_n(data.begin() + NodeStart + 32, 16, data.begin() + NodeStart + 255 * 16); + std::copy_n(data.begin() + NodeStart + 48, 16, data.begin() + NodeStart + 257 * 16); + const std::array, 4> origins{{{0, 0}, {1, 0}, {.5f, 1.5f}, {-.25f, 1}}}; + std::array, LaneCount> expected{}; + for (u32 lane = 0; lane < LaneCount; ++lane) { + const auto kind = lane % 8; + const auto input = lane * 16; + data[input] = kind < 4 ? kind : kind == 4 ? 12 : kind == 5 ? 21 : kind == 6 ? 6 : 0x1000; + data[input + 1] = bits(6); + data[input + 2] = bits(kind < 4 ? origins[kind][0] : 0); + data[input + 3] = bits(kind < 4 ? origins[kind][1] : 0); + data[input + 7] = bits(1); + data[input + 8] = data[input + 9] = Infinity; + data[input + 10] = bits(1); + if (kind < 4) { + const float denominator = kind == 3 ? -2 : -4; + const float i = kind == 0 || kind == 3 ? -1 : -2; + const float j = kind == 3 ? -.5f : -1; + expected[lane] = {bits(2 * denominator), bits(denominator), + barycentrics ? bits(i) : Flag + kind, + barycentrics ? bits(j) : 1}; + } else if (kind < 6) { + expected[lane] = sorted ? std::array{children[1], children[2], children[0], Invalid} + : std::array{children[0], children[1], children[2], Invalid}; + } else { + expected[lane] = kind == 6 ? std::array{Invalid, Invalid, Invalid, Invalid} + : std::array{Infinity, bits(1), Invalid, 0}; + } + if (lane >= 8 && lane < 12) { + data[input + 2] = 0x7fc00000; // invalid ray origin + expected[lane] = {Infinity, bits(1), Invalid, 0}; + } else if (lane == 12 || lane == 13) { + data[input + 1] = bits(1); // nearest face equals the exclusive extent + expected[lane] = {Invalid, Invalid, Invalid, Invalid}; + } else if (lane >= 16 && lane < 20) { + const u32 axis = lane & 1; + data[input + 2 + axis] = bits(origins[kind][axis] - 4); + data[input + 5 + axis] = bits(2); + data[input + 8 + axis] = bits(.5f); + // Dominant X/Y projection scales all intersection numerators and the + // denominator by four; the distance and barycentrics remain unchanged. + for (u32 i = 0; i < (barycentrics ? 4u : 2u); ++i) + expected[lane][i] = bits(std::bit_cast(expected[lane][i]) * 4); + } else if (lane == 22) { + data[input] = 37; // node 4, FP32 + data[input + 1] = bits(10); + for (u32 i = 5; i <= 10; ++i) data[input + i] = bits(1); + expected[lane] = {grow ? children[0] : Invalid, Invalid, Invalid, Invalid}; + } else if (lane >= 24 && lane < 28) { + data[input + 4] = bits(4); + data[input + 7] = data[input + 10] = bits(-1); + for (u32 i = 0; i < (barycentrics ? 4u : 2u); ++i) + expected[lane][i] = bits(-std::bit_cast(expected[lane][i])); + } else if (lane == 28) { + data[input] = 0; + data[input + 3] = bits(-1); // horizontal triangle edge: top-left rule rejects + expected[lane] = {Infinity, bits(1), barycentrics ? bits(-2) : Flag, + barycentrics ? bits(-2) : 0}; + } else if (lane == 29) { + data[input] = (255 << 3) | 5; + } + } + test.expected = test.initial; + for (u32 lane = 0; lane < LaneCount; ++lane) { + std::copy(expected[lane].begin(), expected[lane].end(), + test.expected.begin() + OutputStart + lane * 4); + auto bounded = expected[lane]; + if (lane % 8 == 5) bounded = {Invalid, Invalid, Invalid, Invalid}; + if (lane >= 20) bounded = {0x12345678, 0x12345679, 0x1234567a, 0x1234567b}; + std::copy(bounded.begin(), bounded.end(), + test.expected.begin() + OutputStart + LaneCount * 4 + lane * 4); + } + auto &code = test.code; + // The resource is a GPU value, not a materialized host image descriptor. + AppendSMovLiteral(&code, 8, GuestBase >> 8); + AppendSMovLiteral(&code, 9, (sorted ? 0x80000000 : 0) | (grow << 23)); + AppendSMovLiteral(&code, 10, 256); + AppendSMovLiteral(&code, 11, barycentrics ? 0x81000000 : 0x80000000); + code.push_back(EncodeVop2(0x1a, 1, InlineU32(6), 0)); + for (u32 offset : {0u, 4u, 8u}) { + code.insert(code.end(), {EncodeMubuf0(offset == 8 ? 0x0f : 0x0e, offset * 4), + EncodeMubuf1(4 + offset, 0, 1)}); + } + code.insert(code.end(), {EncodeMimg0(0xe6, 15, 0, false, 0, true), + EncodeMimg1(20, 4, 2)}); + code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0)); + for (u32 i = 0; i < 4; ++i) AppendStoreVgprAtLaneDwordOffset(&code, 20 + i, 2, OutputStart + i); + // NSA addressing, the FP32 second-block bound, and inactive destination lanes. + AppendSMovLiteral(&code, 10, 2); + for (u32 i = 0; i < 4; ++i) AppendVMovLiteral(&code, 24 + i, 0x12345678 + i); + AppendSMovLiteral(&code, 126, 0x000fffff); + code.insert(code.end(), {EncodeMimg0(0xe6, 15, 3, false, 0, true), + EncodeMimg1(24, 4, 2), 0x08070605, 0x0c0b0a09, 0x00000e0d}); + code.push_back(EncodeSop1(0x04, 126, 193)); + for (u32 i = 0; i < 4; ++i) + AppendStoreVgprAtLaneDwordOffset(&code, 24 + i, 2, OutputStart + LaneCount * 4 + i); + AppendEnd(&code); + test.bda_mappings = {{GuestBase, NodeStart * sizeof(u32)}, + {GuestBase + 0x4000, (NodeStart + 257 * 16) * sizeof(u32)}}; + test.compute_info.wave_size = LaneCount; + test.compute_info.threads_num[0] = LaneCount; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + test.required_spirv = {"get_bda_pointer"}; + return test; +} diff --git a/tests/ShaderRayTracingTests.inc b/tests/ShaderRayTracingTests.inc index 21c51ee4..dd79ee70 100644 --- a/tests/ShaderRayTracingTests.inc +++ b/tests/ShaderRayTracingTests.inc @@ -1,87 +1,50 @@ -void TestRayTracingDispatchDetection() { +void TestRayTracingInstructions() { namespace D = ShaderRecompiler::Decoder; - const auto check_decoded = [](std::span code, bool expected, - const char *message) { - D::Program decoded; - D::DecodeProgram(code, decoded); - Check(decoded.has_bvh == expected, message); - if (expected) { - const auto &last = decoded.instructions.back(); - Check(last.family == D::Family::MIMG && - (last.opcode_id == 0xe6u || last.opcode_id == 0xe7u), - "decoder continued past the first BVH instruction"); - const auto result = ShaderRecompiler::TranslateProgram( - code, MakeCompileOptions(ShaderType::Compute)); - Check(result.skip_dispatch && result.program.blocks.empty() && result.cfg_dump.empty(), - "BVH compute shader reached IR translation"); - } - }; - const auto check = [&](std::initializer_list code, bool expected, - const char *message) { - check_decoded(std::span{code.begin(), code.size()}, expected, message); - }; - - check({0xf1989f01u, 0x00081014u, EncodeSopp(0x01)}, true, - "captured contiguous BVH instruction did not skip dispatch"); - check({0xf1989f07u, 0x00071001u, 0x3d3e3c15u, 0x38095851u, 0x00003a39u, - EncodeSopp(0x01)}, true, - "captured NSA BVH instruction did not skip dispatch"); - check({0xf19c9f01u, 0x00081014u, EncodeSopp(0x01)}, true, - "RDNA2 BVH64 instruction did not skip dispatch"); - check({0xbe8e210eu, 0xf1989f01u, 0x00081014u, EncodeSopp(0x01)}, true, - "unsupported SWAPPC before BVH prevented dispatch detection"); - check({0xf1989f01u, 0x00081014u, 0xc0000000u, EncodeSopp(0x01)}, true, - "unknown instruction after BVH prevented dispatch skipping"); - - for (uint32_t opcode : {0xe6u, 0xe7u}) { - for (uint32_t nsa = 0; nsa < 4; ++nsa) { - for (uint32_t slc = 0; slc < 2; ++slc) { - std::array code{ - EncodeMimg0(opcode, 0xf, true) | (nsa << 1u) | (slc << 25u), - EncodeMimg1(0, 0, 0, 0)}; - code[2 + nsa] = EncodeSopp(0x01); - check_decoded(std::span(code).first(3 + nsa), true, - "BVH cache flags or NSA length prevented dispatch detection"); - } - } + const std::array code{ + 0xf1989f01u, 0x00081014u, + EncodeMubuf0(0x1e, 0, false), EncodeMubuf1(16, 0, 0), + 0xf1989f07u, 0x00071001u, 0x3d3e3c15u, 0x38095851u, 0x00003a39u, + EncodeMubuf0(0x1e, 16, false), EncodeMubuf1(16, 0, 0), + EncodeSopp(0x01)}; + D::Program decoded; + D::DecodeProgram(code, decoded); + Check(decoded.instructions.size() == 5 && + decoded.instructions.back().opcode == D::Opcode::S_ENDPGM, + "decoder stopped before instructions following BVH intersections"); + for (const auto index : {0u, 2u}) { + const auto &inst = decoded.instructions[index]; + Check(inst.opcode == D::Opcode::IMAGE_BVH_INTERSECT_RAY && + inst.image_address_components == 11 && inst.data_dwords == 4 && inst.image_r128, + "BVH32 instruction lost its ray or descriptor width"); } - for (uint32_t opcode : {0x66u, 0x67u, 0xe4u, 0xe5u, 0xe8u, 0xe9u}) { - check({EncodeMimg0(opcode, 0xf), EncodeMimg1(0, 0, 0, 0), EncodeSopp(0x01)}, - false, "non-BVH image opcode was incorrectly skipped"); + Check(decoded.instructions[2].image_nsa_addr[9] == 58, + "BVH NSA ray components were not decoded"); + const auto result = RecompileForTest(code, MakeCompileOptions(ShaderType::Compute)); + Check(CountSourceOccurrences(result.ir_dump, "BvhIntersect") == 2 && + result.program.info.uses_dma && result.program.info.images.empty() && + result.program.info.samplers.empty(), + "BVH intersections did not use GPU descriptors and raw guest memory"); + CheckSpirvBinaryValidates(result.spirv); + + const std::array literal{EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)}; + D::DecodeProgram(literal, decoded); + Check(decoded.instructions.size() == 2 && decoded.instructions[0].opcode == D::Opcode::S_MOV_B32, + "BVH-shaped scalar literal was decoded as an instruction"); +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS + for (const auto words : {std::array{0xf19c9f01u, 0x00081014u}, // BVH64 + std::array{0xf1989f01u, 0x40081014u}, // A16 + std::array{0xf1981f01u, 0x00081014u}, // not R128 + std::array{0xf1989f01u, 0x80081014u}}) { // D16 + const std::array unsupported{words[0], words[1], EncodeSopp(0x01)}; + ExpectFatal([&]() { (void)RecompileForTest(unsupported, MakeCompileOptions(ShaderType::Compute)); }, + "unsupported BVH encoding was silently accepted"); } - - check({EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)}, false, - "scalar literal was mistaken for a BVH instruction"); - check({EncodeMimg0(0x20, 0xf) | 6u, EncodeMimg1(0, 0, 0, 0), - 0xf1989f01u, 0xf19c9f01u, 0u, EncodeSopp(0x01)}, false, - "NSA address words were mistaken for BVH instructions"); - check({EncodeMimg0(0x20, 0xf), 0xf1989f01u, EncodeSopp(0x01)}, false, - "MIMG operand word was mistaken for a BVH instruction"); - check({EncodeSopp(0x01), 0xf1989f01u, 0x00081014u}, false, - "data after the shader end was mistaken for a BVH instruction"); - check({EncodeMimg0(0x20, 0xf), EncodeMimg1(0, 0, 0, 0), EncodeSopp(0x01)}, - false, "ordinary image shader was incorrectly skipped"); - - // An end instruction on one branch does not end another branch's code. - check({EncodeSopp(0x04, 1), EncodeSopp(0x01), 0xf1989f01u, 0x00081014u, - EncodeSopp(0x01)}, true, - "BVH at a branch target after an early end was not detected"); - - const std::array bvh_code{0xf1989f01u, 0x00081014u, 0xf19c9f01u, 0x00081014u, - EncodeSopp(0x01)}; - const std::array ordinary_code{EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)}; - D::Program reused; - D::DecodeProgram(bvh_code, reused); - Check(reused.has_bvh && reused.instructions.size() == 1, - "decoder did not stop at the first BVH instruction"); - D::DecodeProgram(ordinary_code, reused); - Check(!reused.has_bvh, "decoder retained stale BVH metadata"); - for (const auto stage : {ShaderType::Compute, ShaderType::Vertex, ShaderType::Pixel}) { - const auto options = MakeCompileOptions(stage); - Check(!ShaderRecompiler::TranslateProgram(ordinary_code, options).skip_dispatch, - "BVH-shaped literal marked an ordinary shader for skipping"); - const auto compiled = RecompileForTest(ordinary_code, options); - Check(!compiled.spirv.empty(), "ordinary shader did not compile"); - CheckSpirvBinaryValidates(compiled.spirv); + for (const auto nsa : {1u, 2u}) { + std::array truncated{0xf1989f01u | (nsa << 1), 0x00081014u}; + truncated[2 + nsa] = EncodeSopp(0x01); + ExpectFatal([&]() { + (void)RecompileForTest(std::span(truncated).first(3 + nsa), MakeCompileOptions(ShaderType::Compute)); + }, "BVH NSA instruction accepted fewer than eleven ray addresses"); } +#endif } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index cb2dbae3..945fed41 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -30715,6 +30715,8 @@ TestCase DispatcherIrreducibleControlFlow() { return test; } +#include "ShaderRayTracingGpuTests.inc" + std::vector MakeCases() { std::vector cases; cases.reserve(128); @@ -31006,6 +31008,14 @@ std::vector MakeCases() { AddCase(GlobalSignedImmediateRebasesBeforeSaddr); AddCase(FlatSegmentIgnoresSaddrAndMasksOffsetMsb); AddCase(ScratchIsPrivatePerInvocation); + cases.push_back(FlatStackApertures(32)); + cases.push_back(FlatStackApertures(64)); + cases.push_back(BvhIntersections(true, true, 1)); + for (bool barycentrics : {false, true}) { + for (bool sorted : {false, true}) { + cases.push_back(BvhIntersections(barycentrics, sorted)); + } + } AddCase(DsReadWriteVariants); AddCase(DsWriteB16D16HiCapturedUsesHighHalf); AddCase(DsWriteB8D16HiWritesByteTwo); @@ -35903,6 +35913,18 @@ int main(int argc, char **argv) { return 0; } #endif + if (argc == 2 && std::strcmp(argv[1], "--ray-tracing-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, FlatStackApertures(32)); + RunCase(&vulkan, FlatStackApertures(64)); + RunCase(&vulkan, BvhIntersections(true, true, 1)); + for (bool barycentrics : {false, true}) { + for (bool sorted : {false, true}) { + RunCase(&vulkan, BvhIntersections(barycentrics, sorted)); + } + } + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--indirect-buffer-only") == 0) { VulkanHarness vulkan; RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors()); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index e8b92335..09cb2201 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -13658,6 +13658,30 @@ void TestNewShaderRecompilerFlatAddressDomainsUseDma() { Check((result.ir_dump.find("GetScratchResource") != std::string::npos) && result.program.scratch_dwords == 1, "SCRATCH incorrectly entered guest address tracking"); +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS + // A local aperture proof must not admit a median which can still produce a + // canonical global address. Such stores need the existing ownership proof. + const uint32_t mixed_domain_store[] = { + EncodeSop1(0x04, 8, 237), // s_mov_b64 s[8:9], private_base + EncodeVop3Word0(0x159, 1), + EncodeVop3Word1(128, 9, 256 + 2), // median(0, private high, dynamic v2) + EncodeFlat0(0x1c, 0, 0), EncodeFlat1(0, 0x7d, 3, 0), + EncodeSopp(0x01), + }; + ExpectFatal([&]() { (void)RecompileForTest(mixed_domain_store, options); }, + "mixed local/global FLAT store bypassed GPU ownership tracking"); + const uint32_t changed_mask_store[] = { + EncodeSop1(0x04, 8, 237), EncodeSop1(0x04, 10, 235), + EncodeVop1(0x01, 1, 128), // canonical old address high + EncodeSop1(0x03, 126, 255), 0x55555555u, + EncodeVop3Word0(0x159, 1), EncodeVop3Word1(9, 11, 256 + 2), + EncodeSop1(0x04, 126, 193), // restore all lanes after the median + EncodeFlat0(0x1c, 0, 0), EncodeFlat1(0, 0x7d, 3, 0), + EncodeSopp(0x01), + }; + ExpectFatal([&]() { (void)RecompileForTest(changed_mask_store, options); }, + "FLAT proof discarded a differently predicated address value"); +#endif } void TestCompilerStageInputOwnership() { @@ -14043,7 +14067,7 @@ int main() { using namespace Libs::Graphics; EnsureConfigInitialized(); - TestRayTracingDispatchDetection(); + TestRayTracingInstructions(); TestResourceDescriptorClassification(); TestShaderBufferResourceSize(); TestNativeShaderResourceDependencies();