shader: execute PS5 BVH intersections and local FLAT stacks
Co-authored-by: Ali Almohaya <AliAlmohaya@gmail.com> Co-authored-by: Mac <me@itsmemac.com> Co-authored-by: Senaxx <michelbeld@gmail.com> Co-authored-by: Jean-Philippe <github@meunier.jp>
This commit is contained in:
nmzik
parent
581dad7dd3
commit
337f432016
|
|
@ -220,7 +220,6 @@ struct PipelineCache::ProgramCache {
|
|||
ShaderRecompiler::IR::ResourceSnapshot resources;
|
||||
ShaderRecompiler::IR::ResourceSpecialization specialization;
|
||||
std::vector<Permutation> permutations;
|
||||
bool skip_dispatch = false;
|
||||
};
|
||||
|
||||
struct ProgramKeyHash {
|
||||
|
|
@ -288,9 +287,6 @@ struct PipelineCache::ProgramCache {
|
|||
lookup_key.code_size = static_cast<uint32_t>(params.code.size());
|
||||
BuildStageStaticKey(input_info, lookup_key.static_state);
|
||||
auto entry = programs.find(lookup_key);
|
||||
if (entry != programs.end() && entry->second.skip_dispatch) {
|
||||
return {};
|
||||
}
|
||||
const ShaderRecompiler::IR::SrtRuntime runtime {
|
||||
.user_data = user_data,
|
||||
.shader_base = params.Base(),
|
||||
|
|
@ -358,11 +354,6 @@ struct PipelineCache::ProgramCache {
|
|||
}
|
||||
DumpShaderOriginal(stage_name, options.shader_hash, params.code);
|
||||
auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
|
||||
if (translated.skip_dispatch) {
|
||||
entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan {}).first;
|
||||
entry->second.skip_dispatch = true;
|
||||
return {};
|
||||
}
|
||||
if (entry == programs.end()) {
|
||||
entry = programs.try_emplace(lookup_key,
|
||||
ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first;
|
||||
|
|
|
|||
|
|
@ -256,10 +256,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
|
|||
input_info.dispatch_thread_dimensions = use_thread_dimensions;
|
||||
const auto compute_program =
|
||||
m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
|
||||
if (!compute_program) {
|
||||
// Temporary until RT is implemented.
|
||||
return;
|
||||
}
|
||||
if (use_thread_dimensions) {
|
||||
input_info.dispatch_threads_num[0] = thread_group_x;
|
||||
input_info.dispatch_threads_num[1] = thread_group_y;
|
||||
|
|
@ -417,10 +413,6 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
|
|||
ShaderComputeInputInfo input_info {};
|
||||
const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
|
||||
cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
|
||||
if (!compute_program) {
|
||||
// Temporary until RT is implemented.
|
||||
return;
|
||||
}
|
||||
buffer.EndRendering();
|
||||
auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
|
||||
auto& bindings = m_compute_bindings;
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@
|
|||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <fmt/format.h>
|
||||
#include <map>
|
||||
|
|
@ -523,20 +522,6 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
|
|||
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
|
||||
static_cast<uint64_t>(decoded.instructions.size()), phase_ms());
|
||||
|
||||
// Temporary workaround for games that compile ray-tracing shaders before
|
||||
// the player can select a mode without ray tracing.
|
||||
if (options.stage == ShaderType::Compute && decoded.has_bvh) {
|
||||
static std::atomic_flag warned = ATOMIC_FLAG_INIT;
|
||||
if (!warned.test_and_set(std::memory_order_relaxed)) {
|
||||
const auto& bvh = decoded.instructions.back();
|
||||
Log::WriteToConsoleAndLog(fmt::format(
|
||||
"Warning: ray tracing is not implemented; skipping compute dispatches containing "
|
||||
"BVH intersection instructions (shader=0x{:016x}, pc=0x{:08x}, opcode=0x{:02x}).\n",
|
||||
options.shader_hash, bvh.pc, bvh.opcode_id));
|
||||
}
|
||||
return {.skip_dispatch = true};
|
||||
}
|
||||
|
||||
std::string decoded_dump;
|
||||
if (options.dump_ir) {
|
||||
decoded_dump = Decoder::ProgramToString(decoded);
|
||||
|
|
@ -645,7 +630,6 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
|
|||
CompileResult CompileProgram(TranslateResult translated, const CompileOptions& options,
|
||||
const IR::ResourceSpecialization& specialization,
|
||||
uint32_t push_data_start_dword) {
|
||||
EXIT_IF(translated.skip_dispatch);
|
||||
const auto emit_begin = std::chrono::steady_clock::now();
|
||||
auto& ir = translated.program;
|
||||
IR::ApplyResourceSpecialization(ir, specialization);
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ struct TranslateResult {
|
|||
IR::Program program;
|
||||
std::string decoded_dump;
|
||||
std::string cfg_dump;
|
||||
bool skip_dispatch = false;
|
||||
};
|
||||
|
||||
struct CompileResult {
|
||||
|
|
|
|||
|
|
@ -122,6 +122,16 @@ void ValidateNativeProgram(const IR::Program& program) {
|
|||
program.memory_info[index].kind == IR::ResourceKind::IndirectBuffer;
|
||||
});
|
||||
};
|
||||
const auto local_flat_handle = [&](const IR::Inst& handle) {
|
||||
return !handle.Uses().empty() &&
|
||||
std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
|
||||
if (IR::AddressOpcodeInfoOf(use.user->GetOpcode()).access == IR::AddressAccess::None)
|
||||
return false;
|
||||
const auto index = use.user->Flags<IR::MemoryFlags>().index;
|
||||
return index < program.memory_info.size() &&
|
||||
program.memory_info[index].kind == IR::ResourceKind::FlatLocal;
|
||||
});
|
||||
};
|
||||
for (const auto* block: program.blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
const auto dense = inst.Flags<uint32_t>();
|
||||
|
|
@ -138,7 +148,7 @@ void ValidateNativeProgram(const IR::Program& program) {
|
|||
if (planning_only_handle(inst)) {
|
||||
break;
|
||||
}
|
||||
if (inst.NumArgs() != 2 || !program.info.uses_dma) {
|
||||
if (inst.NumArgs() != 2 || (!program.info.uses_dma && !local_flat_handle(inst))) {
|
||||
Fail(program, "typed address handle has invalid DMA metadata");
|
||||
}
|
||||
break;
|
||||
|
|
@ -188,11 +198,16 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
|
|||
if (memory_index >= program.memory_info.size()) {
|
||||
Fail(program, "address operation has invalid memory metadata");
|
||||
}
|
||||
if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) {
|
||||
const auto kind = program.memory_info[memory_index].kind;
|
||||
if (kind == IR::ResourceKind::Scratch || kind == IR::ResourceKind::FlatLocal) {
|
||||
if (program.scratch_dwords == 0) {
|
||||
Fail(program, "scratch operation has no per-thread storage");
|
||||
}
|
||||
requirements.function_scratch = true;
|
||||
if (kind == IR::ResourceKind::FlatLocal && program.stage != ShaderType::Compute &&
|
||||
program.stage != ShaderType::Mesh) {
|
||||
requirements.function_lds = true;
|
||||
}
|
||||
} else if (address_access == IR::AddressAccess::Write) {
|
||||
Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
|
||||
}
|
||||
|
|
@ -247,6 +262,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
|
|||
}
|
||||
}
|
||||
switch (inst.GetOpcode()) {
|
||||
case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break;
|
||||
case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
|
||||
case IR::ValueOpcode::DppMoveU32:
|
||||
case IR::ValueOpcode::ReadFirstLane:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,348 @@
|
|||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
using Vec3 = std::array<uint32_t, 3>;
|
||||
|
||||
uint32_t Extract(EmitterState& s, uint32_t type, uint32_t vector, uint32_t index) {
|
||||
const auto result = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpCompositeExtract, type, result, vector, index);
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t Vector(EmitterState& s, uint32_t type, std::span<const uint32_t> components) {
|
||||
const auto result = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpCompositeConstruct, type, result, components);
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) {
|
||||
const auto offset = Binary(s, spv::OpShiftLeftLogical, TypeU32(s), word, ConstantU32(s, 2));
|
||||
const auto address = Binary(s, spv::OpIAdd, TypeScalarU64(s), block,
|
||||
Unary(s, spv::OpUConvert, TypeScalarU64(s), offset));
|
||||
const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address);
|
||||
const auto result = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpLoad, TypeU32(s), result, pointer, spv::MemoryAccessAlignedMask, 4u);
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
|
||||
Vec3 origin, Vec3 direction) {
|
||||
const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
|
||||
const auto zero = ConstantF32Value(s, 0.0f);
|
||||
const auto op = [&](spv::Op code, uint32_t x, uint32_t y) {
|
||||
const auto value = Binary(s, code, f, x, y);
|
||||
s.builder.AddAnnotation(spv::OpDecorate, value, spv::DecorationNoContraction);
|
||||
return value;
|
||||
};
|
||||
const auto cmp = [&](spv::Op code, uint32_t x, uint32_t y) { return Binary(s, code, b, x, y); };
|
||||
const auto either = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); };
|
||||
const auto both = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); };
|
||||
const auto abs = [&](uint32_t x) { return EmitGlsl<GLSLstd450FAbs, IR::Type::F32>(s, x); };
|
||||
const auto is_kind = [&](uint32_t n) { return cmp(spv::OpIEqual, kind, ConstantU32(s, n)); };
|
||||
// The four triangles are (0,1,2), (1,3,2), (2,3,4), (2,4,0).
|
||||
const std::array vertices {
|
||||
EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s, kind, ConstantU32(s, 2)),
|
||||
Select(s, u, is_kind(0), ConstantU32(s, 1),
|
||||
Select(s, u, is_kind(3), ConstantU32(s, 4), ConstantU32(s, 3))),
|
||||
Select(s, u, is_kind(2), ConstantU32(s, 4),
|
||||
Select(s, u, is_kind(3), ConstantU32(s, 0), ConstantU32(s, 2))) };
|
||||
std::array<Vec3, 3> positions;
|
||||
for (uint32_t i = 0; i < 3; ++i) {
|
||||
const auto first = Binary(s, spv::OpIMul, u, vertices[i], ConstantU32(s, 3));
|
||||
for (uint32_t axis = 0; axis < 3; ++axis)
|
||||
positions[i][axis] = Unary(s, spv::OpBitcast, f,
|
||||
NodeWord(s, block, Binary(s, spv::OpIAdd, u, first, ConstantU32(s, axis))));
|
||||
}
|
||||
const auto flag = NodeWord(s, block, ConstantU32(s, 15));
|
||||
const auto y_largest = cmp(spv::OpFOrdLessThan, abs(direction[0]), abs(direction[1]));
|
||||
const auto z_largest = cmp(spv::OpFOrdLessThan,
|
||||
Select(s, f, y_largest, abs(direction[1]), abs(direction[0])), abs(direction[2]));
|
||||
const auto rotate = [&](Vec3 value) {
|
||||
Vec3 result;
|
||||
for (uint32_t i = 0; i < 3; ++i)
|
||||
result[i] = Select(s, f, z_largest, value[i],
|
||||
Select(s, f, y_largest, value[(i + 2) % 3], value[(i + 1) % 3]));
|
||||
return result;
|
||||
};
|
||||
origin = rotate(origin);
|
||||
direction = rotate(direction);
|
||||
std::array<Vec3, 3> projected;
|
||||
for (uint32_t i = 0; i < 3; ++i) {
|
||||
const auto position = rotate(positions[i]);
|
||||
Vec3 relative;
|
||||
for (uint32_t axis = 0; axis < 3; ++axis)
|
||||
relative[axis] = op(spv::OpFSub, position[axis], origin[axis]);
|
||||
projected[i] = {op(spv::OpFSub, op(spv::OpFMul, relative[0], direction[2]),
|
||||
op(spv::OpFMul, direction[0], relative[2])),
|
||||
op(spv::OpFSub, op(spv::OpFMul, relative[1], direction[2]),
|
||||
op(spv::OpFMul, direction[1], relative[2])), relative[2]};
|
||||
}
|
||||
Vec3 edge, weighted;
|
||||
for (uint32_t i = 0; i < 3; ++i) {
|
||||
const auto& first = projected[(i + 1) % 3];
|
||||
const auto& second = projected[(i + 2) % 3];
|
||||
edge[i] = op(spv::OpFSub, op(spv::OpFMul, second[0], first[1]),
|
||||
op(spv::OpFMul, second[1], first[0]));
|
||||
weighted[i] = op(spv::OpFMul, edge[i], direction[2]);
|
||||
}
|
||||
auto numerator = op(spv::OpFAdd, op(spv::OpFAdd,
|
||||
op(spv::OpFMul, edge[0], projected[0][2]), op(spv::OpFMul, edge[1], projected[1][2])),
|
||||
op(spv::OpFMul, edge[2], projected[2][2]));
|
||||
auto denominator = op(spv::OpFAdd, op(spv::OpFAdd, weighted[0], weighted[1]), weighted[2]);
|
||||
uint32_t negative = ConstantBool(s, false), positive = negative;
|
||||
for (auto value: edge) {
|
||||
negative = either(negative, cmp(spv::OpFOrdLessThan, value, zero));
|
||||
positive = either(positive, cmp(spv::OpFOrdGreaterThan, value, zero));
|
||||
}
|
||||
const auto winding = cmp(spv::OpFOrdGreaterThan, denominator, zero);
|
||||
auto missed = either(both(negative, positive), cmp(spv::OpFOrdEqual, denominator, zero));
|
||||
missed = either(missed, Unary(s, spv::OpIsNan, b, numerator));
|
||||
missed = either(missed, cmp(spv::OpFOrdLessThan,
|
||||
Select(s, f, winding, numerator, Unary(s, spv::OpFNegate, f, numerator)), zero));
|
||||
for (uint32_t i = 0; i < 3; ++i) {
|
||||
const auto first = projected[(i + 1) % 3][1], second = projected[(i + 2) % 3][1];
|
||||
const auto first_zero = cmp(spv::OpFOrdEqual, first, zero);
|
||||
const auto horizontal = both(first_zero, cmp(spv::OpFOrdEqual, second, zero));
|
||||
const auto below = either(cmp(spv::OpFOrdLessThan, first, zero),
|
||||
both(first_zero, cmp(spv::OpFOrdGreaterThan, second, zero)));
|
||||
const auto right = Binary(s, spv::OpLogicalNotEqual, b, below, winding);
|
||||
const auto excluded = Select(s, b, horizontal,
|
||||
cmp(spv::OpFOrdGreaterThan, projected[i][1], zero), right);
|
||||
missed = either(missed, both(cmp(spv::OpFOrdEqual, edge[i], zero), excluded));
|
||||
}
|
||||
const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag,
|
||||
Binary(s, spv::OpIMul, u, kind, ConstantU32(s, 8)));
|
||||
const auto coordinate = [&](uint32_t shift) {
|
||||
const auto index = Binary(s, spv::OpBitwiseAnd, u,
|
||||
Binary(s, spv::OpShiftRightLogical, u, remap, ConstantU32(s, shift)), ConstantU32(s, 3));
|
||||
return Unary(s, spv::OpBitcast, u, Select(s, f,
|
||||
cmp(spv::OpIEqual, index, ConstantU32(s, 1)), weighted[1],
|
||||
Select(s, f, cmp(spv::OpIEqual, index, ConstantU32(s, 2)), weighted[2], weighted[0])));
|
||||
};
|
||||
const std::array result {
|
||||
Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32(s, 0x7f800000), numerator)),
|
||||
Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32Value(s, 1.0f), denominator)),
|
||||
Select(s, u, bary, coordinate(0), Binary(s, spv::OpIAdd, u, flag, kind)),
|
||||
Select(s, u, bary, coordinate(2), Select(s, u, missed, ConstantU32(s, 0), ConstantU32(s, 1))) };
|
||||
return Vector(s, TypeU32Vector(s, 4), result);
|
||||
}
|
||||
|
||||
std::array<uint32_t, 6> Box(EmitterState& s, uint32_t first, uint32_t second,
|
||||
uint32_t half, uint32_t child) {
|
||||
const auto f = TypeF32(s);
|
||||
const auto half_label = s.builder.AllocateId(), full_label = s.builder.AllocateId();
|
||||
const auto merge = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
|
||||
s.builder.AddFunction(spv::OpBranchConditional, half, half_label, full_label);
|
||||
EmitLabel(s, half_label);
|
||||
std::array<uint32_t, 6> packed, full, result;
|
||||
for (uint32_t pair = 0; pair < 3; ++pair) {
|
||||
const auto word = NodeWord(s, first, ConstantU32(s, 4 + child * 3 + pair));
|
||||
const auto unpacked = EmitGlsl<GLSLstd450UnpackHalf2x16, IR::Type::F32x2>(s, word);
|
||||
packed[pair * 2] = Extract(s, f, unpacked, 0);
|
||||
packed[pair * 2 + 1] = Extract(s, f, unpacked, 1);
|
||||
}
|
||||
s.builder.AddFunction(spv::OpBranch, merge);
|
||||
EmitLabel(s, full_label);
|
||||
for (uint32_t component = 0; component < 6; ++component) {
|
||||
const auto word = 4 + child * 6 + component;
|
||||
full[component] = Unary(s, spv::OpBitcast, f,
|
||||
NodeWord(s, word < 16 ? first : second, ConstantU32(s, word % 16)));
|
||||
}
|
||||
s.builder.AddFunction(spv::OpBranch, merge);
|
||||
EmitLabel(s, merge);
|
||||
for (uint32_t component = 0; component < 6; ++component) {
|
||||
result[component] = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpPhi, f, result[component], packed[component], half_label,
|
||||
full[component], full_label);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half,
|
||||
uint32_t grow, uint32_t sort, uint32_t extent, const Vec3& origin, const Vec3& inverse) {
|
||||
const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
|
||||
const auto zero = ConstantF32Value(s, 0.0f);
|
||||
std::array<uint32_t, 4> depths, indices;
|
||||
Vec3 forward;
|
||||
for (uint32_t axis = 0; axis < 3; ++axis)
|
||||
forward[axis] = Binary(s, spv::OpFOrdGreaterThanEqual, b, inverse[axis], zero);
|
||||
for (uint32_t child = 0; child < 4; ++child) {
|
||||
const auto box = Box(s, first, second, half, child);
|
||||
Vec3 near, far;
|
||||
for (uint32_t axis = 0; axis < 3; ++axis) {
|
||||
const auto lo = Binary(s, spv::OpFMul, f,
|
||||
Binary(s, spv::OpFSub, f, box[axis], origin[axis]), inverse[axis]);
|
||||
const auto hi = Binary(s, spv::OpFMul, f,
|
||||
Binary(s, spv::OpFSub, f, box[axis + 3], origin[axis]), inverse[axis]);
|
||||
near[axis] = Select(s, f, forward[axis], lo, hi);
|
||||
far[axis] = Select(s, f, forward[axis], hi, lo);
|
||||
}
|
||||
const auto entry = EmitGlsl<GLSLstd450FMax, IR::Type::F32>(s,
|
||||
EmitGlsl<GLSLstd450FMax, IR::Type::F32>(s, near[0], near[1]), near[2]);
|
||||
const auto exit = EmitGlsl<GLSLstd450FMin, IR::Type::F32>(s,
|
||||
EmitGlsl<GLSLstd450FMin, IR::Type::F32>(s, far[0], far[1]), far[2]);
|
||||
// Valid hits have nonnegative exit distance. Saturate ULP growth at +infinity.
|
||||
const auto bits = Unary(s, spv::OpBitcast, u, Select(s, f,
|
||||
Binary(s, spv::OpFOrdEqual, b, exit, zero), zero, exit));
|
||||
const auto grown_bits = EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
|
||||
Binary(s, spv::OpIAdd, u, bits, grow), ConstantU32(s, 0x7f800000));
|
||||
const auto grown = Unary(s, spv::OpBitcast, f, grown_bits);
|
||||
auto hit = Binary(s, spv::OpLogicalAnd, b,
|
||||
Binary(s, spv::OpFOrdGreaterThanEqual, b, exit, zero),
|
||||
Binary(s, spv::OpFOrdLessThan, b, entry, extent));
|
||||
hit = Binary(s, spv::OpLogicalAnd, b, hit,
|
||||
Binary(s, spv::OpFOrdLessThanEqual, b, entry, grown));
|
||||
depths[child] = Unary(s, spv::OpBitcast, u, Select(s, f, hit,
|
||||
Select(s, f, Binary(s, spv::OpFOrdGreaterThan, b, entry, zero), entry, zero),
|
||||
ConstantF32(s, 0x7f800000)));
|
||||
indices[child] = Select(s, u, hit, NodeWord(s, first, ConstantU32(s, child)), ConstantU32(s, ~0u));
|
||||
}
|
||||
// A select network keeps the original child order when sorting is disabled.
|
||||
constexpr std::array<std::pair<uint32_t, uint32_t>, 5> network {{{0,1}, {2,3}, {0,2}, {1,3}, {1,2}}};
|
||||
for (size_t step = 0; step < network.size(); ++step) {
|
||||
const auto [a, c] = network[step];
|
||||
const auto swap = Binary(s, spv::OpLogicalAnd, b, sort,
|
||||
Binary(s, spv::OpUGreaterThan, b, depths[a], depths[c]));
|
||||
const auto depth = depths[a], index = indices[a];
|
||||
const auto later = [&](uint32_t position) {
|
||||
return std::ranges::any_of(std::span(network).subspan(step + 1), [=](auto pair) {
|
||||
return pair.first == position || pair.second == position;
|
||||
});
|
||||
};
|
||||
if (later(a)) depths[a] = Select(s, u, swap, depths[c], depth);
|
||||
if (later(c)) depths[c] = Select(s, u, swap, depth, depths[c]);
|
||||
indices[a] = Select(s, u, swap, indices[c], index);
|
||||
indices[c] = Select(s, u, swap, index, indices[c]);
|
||||
}
|
||||
return Vector(s, TypeU32Vector(s, 4), indices);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void DefineBvhIntersect(EmitterState& s) {
|
||||
if (!s.requirements.bvh) return;
|
||||
const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s);
|
||||
const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3);
|
||||
const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, u, f, vec3, vec3, vec3);
|
||||
s.bvh_intersect_function = s.builder.AllocateId();
|
||||
s.builder.AddName(s.bvh_intersect_function, "bvh_intersect");
|
||||
s.builder.AddFunction(spv::OpFunction, vec4, s.bvh_intersect_function,
|
||||
spv::FunctionControlMaskNone, signature);
|
||||
std::array<uint32_t, 6> args;
|
||||
const std::array types {vec4, u, f, vec3, vec3, vec3};
|
||||
for (uint32_t i = 0; i < args.size(); ++i) {
|
||||
args[i] = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpFunctionParameter, types[i], args[i]);
|
||||
}
|
||||
EmitLabel(s, s.builder.AllocateId());
|
||||
const auto descriptor = args[0], node = args[1], extent = args[2];
|
||||
Vec3 origin, direction, inverse;
|
||||
for (uint32_t i = 0; i < 3; ++i) {
|
||||
origin[i] = Extract(s, f, args[3], i);
|
||||
direction[i] = Extract(s, f, args[4], i);
|
||||
inverse[i] = Extract(s, f, args[5], i);
|
||||
}
|
||||
std::array<uint32_t, 4> words;
|
||||
for (uint32_t i = 0; i < 4; ++i) words[i] = Extract(s, u, descriptor, i);
|
||||
const auto and_bits = [&](uint32_t value, uint32_t mask) {
|
||||
return Binary(s, spv::OpBitwiseAnd, u, value, ConstantU32(s, mask));
|
||||
};
|
||||
const auto shr = [&](uint32_t value, uint32_t shift) {
|
||||
return Binary(s, spv::OpShiftRightLogical, u, value, ConstantU32(s, shift));
|
||||
};
|
||||
const auto kind = and_bits(node, 7);
|
||||
const auto triangle = Binary(s, spv::OpULessThan, b, kind, ConstantU32(s, 4));
|
||||
const auto full = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 5));
|
||||
const auto half = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 4));
|
||||
const auto bary = Binary(s, spv::OpINotEqual, b, and_bits(words[3], 0x01000000), ConstantU32(s, 0));
|
||||
const auto sort = Binary(s, spv::OpINotEqual, b, and_bits(words[1], 0x80000000), ConstantU32(s, 0));
|
||||
const auto grow = and_bits(shr(words[1], 23), 0xff);
|
||||
const auto base = Binary(s, spv::OpShiftLeftLogical, wide,
|
||||
DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8));
|
||||
const auto index = Unary(s, spv::OpUConvert, wide, shr(node, 3));
|
||||
const auto last_index = Binary(s, spv::OpIAdd, wide, index,
|
||||
Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0)));
|
||||
const auto address = Binary(s, spv::OpIAdd, wide, base,
|
||||
Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantDeviceAddress(s, 6)));
|
||||
const auto last_address = Binary(s, spv::OpIAdd, wide, base,
|
||||
Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantDeviceAddress(s, 6)));
|
||||
auto valid = Binary(s, spv::OpIEqual, b, shr(words[3], 28), ConstantU32(s, 8));
|
||||
const auto require = [&](uint32_t condition) {
|
||||
valid = Binary(s, spv::OpLogicalAnd, b, valid, condition);
|
||||
};
|
||||
require(Binary(s, spv::OpULessThanEqual, b, kind, ConstantU32(s, 5)));
|
||||
require(Binary(s, spv::OpULessThanEqual, b, last_index, DeviceAddressFromWords(s, words[2], and_bits(words[3], 0x3ff))));
|
||||
require(Binary(s, spv::OpULessThan, b, last_address, ConstantDeviceAddress(s, uint64_t{1} << 48)));
|
||||
require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, extent)));
|
||||
for (uint32_t i = 0; i < 3; ++i) {
|
||||
for (const auto value: {origin[i], direction[i], inverse[i]})
|
||||
require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, value)));
|
||||
require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsInf, b, origin[i])));
|
||||
}
|
||||
const std::array triangle_invalid {ConstantU32(s, 0x7f800000), ConstantU32(s, 0x3f800000),
|
||||
ConstantU32(s, ~0u), ConstantU32(s, 0)};
|
||||
const std::array box_invalid {ConstantU32(s, ~0u), ConstantU32(s, ~0u),
|
||||
ConstantU32(s, ~0u), ConstantU32(s, ~0u)};
|
||||
std::array<uint32_t, 4> invalid_words;
|
||||
for (uint32_t i = 0; i < 4; ++i)
|
||||
invalid_words[i] = Select(s, u, triangle, triangle_invalid[i], box_invalid[i]);
|
||||
const auto invalid = Vector(s, vec4, invalid_words);
|
||||
const auto result = EmitValueOrDefaultIfCondition(s, valid, vec4, invalid, [&] {
|
||||
const auto first = GetBdaPointer(s, address);
|
||||
const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantDeviceAddress(s, 0), [&] {
|
||||
return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantDeviceAddress(s, 64)));
|
||||
});
|
||||
const auto present = Binary(s, spv::OpLogicalAnd, b,
|
||||
Binary(s, spv::OpINotEqual, b, first, ConstantDeviceAddress(s, 0)),
|
||||
Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, full),
|
||||
Binary(s, spv::OpINotEqual, b, second, ConstantDeviceAddress(s, 0))));
|
||||
return EmitValueOrDefaultIfCondition(s, present, vec4, invalid, [&] {
|
||||
const auto tri_label = s.builder.AllocateId(), box_label = s.builder.AllocateId();
|
||||
const auto merge = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
|
||||
s.builder.AddFunction(spv::OpBranchConditional, triangle, tri_label, box_label);
|
||||
EmitLabel(s, tri_label);
|
||||
const auto tri_value = Triangle(s, first, kind, bary, origin, direction);
|
||||
const auto tri_exit = s.current_label;
|
||||
s.builder.AddFunction(spv::OpBranch, merge);
|
||||
EmitLabel(s, box_label);
|
||||
const auto box_value = Boxes(s, first, second, half, grow, sort, extent, origin, inverse);
|
||||
const auto box_exit = s.current_label;
|
||||
s.builder.AddFunction(spv::OpBranch, merge);
|
||||
EmitLabel(s, merge);
|
||||
const auto value = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpPhi, vec4, value, tri_value, tri_exit, box_value, box_exit);
|
||||
return value;
|
||||
});
|
||||
});
|
||||
s.builder.AddFunction(spv::OpReturnValue, result);
|
||||
s.builder.AddFunction(spv::OpFunctionEnd);
|
||||
}
|
||||
|
||||
uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
auto& s = ctx.state;
|
||||
const auto* ray = ctx.ImageAddress(inst.Arg(1));
|
||||
if (ray == nullptr || ray->NumArgs() < 11 || s.bvh_intersect_function == 0)
|
||||
ctx.Fail(inst, "BVH intersection requires eleven address components and its shared function");
|
||||
const auto vector = [&](uint32_t first) {
|
||||
Vec3 values;
|
||||
for (uint32_t i = 0; i < 3; ++i)
|
||||
values[i] = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, first + i));
|
||||
return Vector(s, TypeF32Vector(s, 3), values);
|
||||
};
|
||||
const auto descriptor = ctx.Arg(inst, 0), node = ctx.Arg(*ray, 0);
|
||||
const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, 1));
|
||||
const auto origin = vector(2), direction = vector(5), inverse = vector(8);
|
||||
return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), TypeU32Vector(s, 4),
|
||||
ConstantU32CompositeZero(s, 4), [&] {
|
||||
const auto result = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpFunctionCall, TypeU32Vector(s, 4), result,
|
||||
s.bvh_intersect_function, descriptor, node, extent, origin, direction, inverse);
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
|
|
@ -294,6 +294,7 @@ inline constexpr auto EmitDataAppend = EmitAppendConsume;
|
|||
inline constexpr auto EmitDataConsume = EmitAppendConsume;
|
||||
uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
inline constexpr auto EmitImageQueryDimensions = EmitImage;
|
||||
inline constexpr auto EmitImageQueryLod = EmitImage;
|
||||
inline constexpr auto EmitImageRead = EmitImage;
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ constexpr std::array<ImageDimensionInfo, 7> ImageDimensions {{
|
|||
const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
|
||||
|
||||
struct SpirvRequirements {
|
||||
bool bvh = false;
|
||||
bool subgroup_ballot = false;
|
||||
bool subgroup_shuffle = false;
|
||||
bool subgroup_local_invocation_id = false;
|
||||
|
|
@ -96,6 +97,7 @@ struct EmitterState {
|
|||
uint32_t bda_pagetable_variable = 0;
|
||||
uint32_t fault_buffer_variable = 0;
|
||||
uint32_t bda_pointer_function = 0;
|
||||
uint32_t bvh_intersect_function = 0;
|
||||
uint32_t gds_variable = 0;
|
||||
uint32_t gds_length = 0;
|
||||
uint32_t push_constant_variable = 0;
|
||||
|
|
@ -487,6 +489,10 @@ uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
|
|||
void EmitProgram(EmitterState& state);
|
||||
|
||||
void DefineGetBdaPointer(EmitterState& state);
|
||||
void DefineBvhIntersect(EmitterState& state);
|
||||
uint32_t GetBdaPointer(EmitterState& state, uint32_t address);
|
||||
uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value);
|
||||
uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high);
|
||||
|
||||
// These templates accept local lambdas from several emitter translation units.
|
||||
template <typename Fn>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
|
||||
|
||||
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
|
||||
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
|
|
@ -105,31 +106,23 @@ uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t ad
|
|||
return result;
|
||||
}
|
||||
|
||||
uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low,
|
||||
uint32_t high) {
|
||||
auto& state = ctx.state;
|
||||
auto immediate = static_cast<int32_t>(mem.offset);
|
||||
uint32_t AddAddressOffset(EmitterState& state, const IR::MemoryInfo& mem, uint32_t low,
|
||||
uint32_t& high) {
|
||||
const auto immediate = static_cast<int32_t>(mem.offset);
|
||||
if (immediate == 0) return low;
|
||||
const auto immediate_low = ConstantU32(state, static_cast<uint32_t>(immediate));
|
||||
const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u);
|
||||
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
|
||||
return AddU64Low(state, low, high, immediate_low, immediate_high, high);
|
||||
}
|
||||
|
||||
uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low,
|
||||
uint32_t high) {
|
||||
auto& state = ctx.state;
|
||||
low = AddAddressOffset(state, mem, low, high);
|
||||
const auto valid = Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, 0));
|
||||
return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX));
|
||||
}
|
||||
|
||||
uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
|
||||
return state.builder.Constant(spv::OpConstant, TypeScalarU64(state),
|
||||
static_cast<uint32_t>(value),
|
||||
static_cast<uint32_t>(value >> 32u));
|
||||
}
|
||||
|
||||
uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
|
||||
const auto low64 = Unary(state, spv::OpUConvert, TypeScalarU64(state), low);
|
||||
const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state),
|
||||
Unary(state, spv::OpUConvert, TypeScalarU64(state), high),
|
||||
ConstantDeviceAddress(state, 32));
|
||||
return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
|
||||
}
|
||||
|
||||
uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
|
||||
auto& state = ctx.state;
|
||||
auto low = ctx.Arg(inst, 1);
|
||||
|
|
@ -186,17 +179,9 @@ void RecordBdaFault(EmitterState& state, uint32_t page) {
|
|||
Binary(state, spv::OpBitwiseOr, TypeU32(state), value, bit));
|
||||
}
|
||||
|
||||
uint32_t GetBdaPointer(ValueEmitContext& ctx, uint32_t address) {
|
||||
auto& state = ctx.state;
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result,
|
||||
state.bda_pointer_function, address);
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
|
||||
auto& state = ctx.state;
|
||||
const auto bda = GetBdaPointer(ctx, address);
|
||||
const auto bda = GetBdaPointer(state, address);
|
||||
const auto present =
|
||||
Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantDeviceAddress(state, 0));
|
||||
return EmitValueOrZeroIfCondition(state, present, [&]() {
|
||||
|
|
@ -511,6 +496,50 @@ void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resour
|
|||
resource.memory_access);
|
||||
}
|
||||
|
||||
template <typename Fn>
|
||||
void ForEachLocalFlatAccess(ValueEmitContext& ctx, const IR::Inst& inst, Fn&& emit) {
|
||||
auto& state = ctx.state;
|
||||
auto mem = ctx.Memory(inst);
|
||||
auto high = ctx.Arg(inst, 2);
|
||||
const auto low = AddAddressOffset(state, mem, ctx.Arg(inst, 1), high);
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), low, ConstantU32(state, 2));
|
||||
const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), ConstantU32(state, 0),
|
||||
Binary(state, spv::OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, 3)));
|
||||
for (const auto kind: {IR::ResourceKind::Scratch, IR::ResourceKind::Lds}) {
|
||||
mem.kind = kind;
|
||||
const auto resource = PrepareMemoryResourceAccess(state, mem);
|
||||
const auto aperture = kind == IR::ResourceKind::Scratch ? Decoder::PrivateApertureHigh
|
||||
: Decoder::SharedApertureHigh;
|
||||
const auto selected = Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, aperture));
|
||||
const auto valid = AndCondition(state, aligned,
|
||||
AndCondition(state, selected, EmitMemoryElementInBounds(state, resource, index)));
|
||||
emit(resource, index, valid);
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t LoadLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
uint32_t result = 0;
|
||||
ForEachLocalFlatAccess(ctx, inst, [&](const auto& resource, uint32_t index, uint32_t valid) {
|
||||
const auto value = EmitValueOrZeroIfCondition(ctx.state, valid, [&]() {
|
||||
return LoadWordInBounds(ctx, resource, index);
|
||||
});
|
||||
result = result == 0 ? value : Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state), result, value);
|
||||
});
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
ForEachLocalFlatAccess(ctx, inst, [&](const auto& resource, uint32_t index, uint32_t valid) {
|
||||
EmitIfCondition(ctx.state, valid, [&]() {
|
||||
StoreWordInBounds(ctx, resource, index, ctx.Arg(inst, inst.NumArgs() - 2));
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource, uint32_t data) {
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
|
||||
|
|
@ -961,6 +990,27 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
|
|||
|
||||
} // namespace
|
||||
|
||||
uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
|
||||
return state.builder.Constant(spv::OpConstant, TypeScalarU64(state),
|
||||
static_cast<uint32_t>(value),
|
||||
static_cast<uint32_t>(value >> 32u));
|
||||
}
|
||||
|
||||
uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
|
||||
const auto low64 = Unary(state, spv::OpUConvert, TypeScalarU64(state), low);
|
||||
const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state),
|
||||
Unary(state, spv::OpUConvert, TypeScalarU64(state), high),
|
||||
ConstantDeviceAddress(state, 32));
|
||||
return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
|
||||
}
|
||||
|
||||
uint32_t GetBdaPointer(EmitterState& state, uint32_t address) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result,
|
||||
state.bda_pointer_function, address);
|
||||
return result;
|
||||
}
|
||||
|
||||
void DefineGetBdaPointer(EmitterState& state) {
|
||||
if (!state.program.info.uses_dma) {
|
||||
return;
|
||||
|
|
@ -1243,7 +1293,9 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
|
|||
const auto shared_components = IR::SharedComponentCount(op);
|
||||
const auto address_info = IR::AddressOpcodeInfoOf(op);
|
||||
uint32_t value;
|
||||
if (buffer_components > 1u)
|
||||
if (mem.kind == IR::ResourceKind::FlatLocal)
|
||||
value = LoadLocalFlat(ctx, inst);
|
||||
else if (buffer_components > 1u)
|
||||
value = LoadWideBuffer(ctx, inst, buffer_components);
|
||||
else if (shared_components > 1u)
|
||||
value = LoadWideShared(ctx, inst, shared_components);
|
||||
|
|
@ -1270,7 +1322,9 @@ void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
|
|||
const auto buffer_components = IR::BufferComponentCount(op);
|
||||
const auto shared_components = IR::SharedComponentCount(op);
|
||||
const auto type = inst.Arg(inst.NumArgs() - 2).GetType();
|
||||
if (buffer_components > 1u)
|
||||
if (mem.kind == IR::ResourceKind::FlatLocal)
|
||||
StoreLocalFlat(ctx, inst);
|
||||
else if (buffer_components > 1u)
|
||||
StoreWideBuffer(ctx, inst, buffer_components);
|
||||
else if (shared_components > 1u)
|
||||
StoreWideShared(ctx, inst, shared_components);
|
||||
|
|
|
|||
|
|
@ -679,6 +679,7 @@ void EmitProgram(EmitterState& state) {
|
|||
}
|
||||
}
|
||||
DefineGetBdaPointer(state);
|
||||
DefineBvhIntersect(state);
|
||||
for (const auto* block: program.blocks) {
|
||||
if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
|
||||
return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
|
||||
|
|
|
|||
|
|
@ -225,6 +225,7 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim
|
|||
case 0x09u: return Opcode::IMAGE_STORE_MIP;
|
||||
case 0x0eu: return Opcode::IMAGE_GET_RESINFO;
|
||||
case 0x60u: return Opcode::IMAGE_GET_LOD;
|
||||
case 0xe6u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
|
||||
default: return Opcode::UNSUPPORTED;
|
||||
}
|
||||
}
|
||||
|
|
@ -253,6 +254,7 @@ uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
|
|||
}
|
||||
|
||||
switch (opcode) {
|
||||
case 0xe6u: return 11u;
|
||||
case 0x0eu: return 1u;
|
||||
case 0x01u:
|
||||
case 0x09u: return ImageCoordComponents(dimension) + 1u;
|
||||
|
|
@ -360,6 +362,11 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
|
|||
if (d16 && !supports_d16) {
|
||||
SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
|
||||
}
|
||||
if (opcode == 0xe6u &&
|
||||
(a16 || !r128 || inst.dmask != 0xfu || (nsa_dwords != 0u && nsa_dwords != 3u))) {
|
||||
SetUnsupported(inst, Family::MIMG, opcode,
|
||||
"BVH intersection requires eleven full-float ray DWORDs and R128/dmask:0xf");
|
||||
}
|
||||
|
||||
DecodeVectorGpr(vdata, inst.dst);
|
||||
DecodeVectorGpr(vaddr, inst.src0);
|
||||
|
|
|
|||
|
|
@ -254,6 +254,8 @@ void DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand) {
|
|||
case 125u: operand.kind = OperandKind::Null; return;
|
||||
case 126u: operand.kind = OperandKind::ExecLo; return;
|
||||
case 127u: operand.kind = OperandKind::ExecHi; return;
|
||||
case 235u: operand.kind = OperandKind::SharedBase; return;
|
||||
case 237u: operand.kind = OperandKind::PrivateBase; return;
|
||||
case 239u: operand.kind = OperandKind::PopsExitingWaveId; return;
|
||||
case 248u:
|
||||
operand.kind = OperandKind::FloatInlineConstant;
|
||||
|
|
@ -404,7 +406,6 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
|
|||
program.instructions.clear();
|
||||
program.instructions.reserve(code.size());
|
||||
program.code = code;
|
||||
program.has_bvh = false;
|
||||
|
||||
std::vector<bool> branch_targets;
|
||||
for (uint32_t word_index = 0; word_index < code.size();) {
|
||||
|
|
@ -413,10 +414,6 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
|
|||
|
||||
const auto& inst = program.instructions.back();
|
||||
word_index += inst.word_count;
|
||||
if (inst.family == Family::MIMG && (inst.opcode_id == 0xe6u || inst.opcode_id == 0xe7u)) {
|
||||
program.has_bvh = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (IsDirectBranch(inst.opcode)) {
|
||||
const auto target_index = inst.branch_target / sizeof(uint32_t);
|
||||
|
|
@ -455,6 +452,8 @@ std::string OperandToString(const Operand& operand) {
|
|||
case OperandKind::Scc: text = "scc"; break;
|
||||
case OperandKind::M0: text = "m0"; break;
|
||||
case OperandKind::PopsExitingWaveId: text = "pops_exiting_wave_id"; break;
|
||||
case OperandKind::SharedBase: text = "shared_base"; break;
|
||||
case OperandKind::PrivateBase: text = "private_base"; break;
|
||||
case OperandKind::Null: text = "null"; break;
|
||||
default: text = "unknown"; break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@
|
|||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Decoder {
|
||||
|
||||
// PS5 flat stack addresses use these SH_MEM_BASES aperture tags in VA[63:32].
|
||||
constexpr uint32_t PrivateApertureHigh = 0x70000000u;
|
||||
constexpr uint32_t SharedApertureHigh = 0x80000000u;
|
||||
|
||||
enum class Family {
|
||||
Unknown,
|
||||
SOP1,
|
||||
|
|
@ -569,6 +573,7 @@ enum class Opcode {
|
|||
DS_WRITE_B128,
|
||||
DS_WRITE_ADDTID_B32,
|
||||
DS_READ_ADDTID_B32,
|
||||
IMAGE_BVH_INTERSECT_RAY,
|
||||
IMAGE_GET_RESINFO,
|
||||
IMAGE_GET_LOD,
|
||||
IMAGE_LOAD,
|
||||
|
|
@ -636,6 +641,8 @@ enum class OperandKind {
|
|||
Scc,
|
||||
M0,
|
||||
PopsExitingWaveId,
|
||||
SharedBase,
|
||||
PrivateBase,
|
||||
Null,
|
||||
Vgpr,
|
||||
};
|
||||
|
|
@ -747,7 +754,6 @@ struct Instruction {
|
|||
struct Program {
|
||||
std::span<const uint32_t> code;
|
||||
std::vector<Instruction> instructions;
|
||||
bool has_bvh = false;
|
||||
};
|
||||
|
||||
// Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs.
|
||||
|
|
|
|||
|
|
@ -883,6 +883,15 @@ void Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
|
|||
|
||||
void Translator::EmitMemory(const Decoder::Instruction& inst) {
|
||||
switch (inst.opcode) {
|
||||
case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: {
|
||||
const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect,
|
||||
{ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()});
|
||||
for (uint32_t component = 0; component < 4; ++component) {
|
||||
WriteOperand(OffsetOperand(inst.dst, component),
|
||||
ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(component)}));
|
||||
}
|
||||
return;
|
||||
}
|
||||
case Decoder::Opcode::S_LOAD_DWORD:
|
||||
case Decoder::Opcode::S_LOAD_DWORDX2:
|
||||
case Decoder::Opcode::S_LOAD_DWORDX4:
|
||||
|
|
|
|||
|
|
@ -135,6 +135,8 @@ IR::U32 Translator::ReadRawU32(const Decoder::Operand& operand) {
|
|||
case Decoder::OperandKind::IntegerInlineConstant:
|
||||
case Decoder::OperandKind::FloatInlineConstant: return IR::U32(IR::Value(operand.value));
|
||||
case Decoder::OperandKind::Null:
|
||||
case Decoder::OperandKind::SharedBase:
|
||||
case Decoder::OperandKind::PrivateBase:
|
||||
case Decoder::OperandKind::PopsExitingWaveId: return IR::U32(IR::Value(0u));
|
||||
case Decoder::OperandKind::Sgpr:
|
||||
return ir.GetScalarReg(static_cast<IR::ScalarReg>(operand.reg));
|
||||
|
|
@ -474,6 +476,12 @@ IR::U32 Translator::ReadU32(const Decoder::Operand& operand) {
|
|||
}
|
||||
|
||||
std::array<IR::U32, 2> Translator::ReadU32Pair(const Decoder::Operand& operand) {
|
||||
if (operand.kind == Decoder::OperandKind::PrivateBase ||
|
||||
operand.kind == Decoder::OperandKind::SharedBase) {
|
||||
return {IR::U32(IR::Value(0u)), IR::U32(IR::Value(
|
||||
operand.kind == Decoder::OperandKind::PrivateBase ? Decoder::PrivateApertureHigh
|
||||
: Decoder::SharedApertureHigh))};
|
||||
}
|
||||
if (operand.kind == Decoder::OperandKind::ExecLo) {
|
||||
return {ir.GetExecLo(), ir.GetExecHi()};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ enum class ResourceKind {
|
|||
Buffer,
|
||||
IndirectBuffer,
|
||||
Flat,
|
||||
FlatLocal,
|
||||
Global,
|
||||
Scratch,
|
||||
Lds,
|
||||
|
|
@ -40,7 +41,8 @@ enum class ResourceKind {
|
|||
|
||||
[[nodiscard]] constexpr bool IsAddressResourceKind(ResourceKind kind) {
|
||||
return kind == ResourceKind::ScalarAddress || kind == ResourceKind::Flat ||
|
||||
kind == ResourceKind::Global || kind == ResourceKind::Scratch;
|
||||
kind == ResourceKind::FlatLocal || kind == ResourceKind::Global ||
|
||||
kind == ResourceKind::Scratch;
|
||||
}
|
||||
|
||||
struct MemoryInfo {
|
||||
|
|
|
|||
|
|
@ -284,6 +284,7 @@ VALUE_OPCODE(DataConsume, U32, U32, U1)
|
|||
VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
|
||||
|
||||
VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)
|
||||
VALUE_OPCODE(BvhIntersect, U32x4, U32x4, ImageAddress, U1)
|
||||
VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress)
|
||||
VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1)
|
||||
VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1)
|
||||
|
|
|
|||
|
|
@ -44,6 +44,34 @@ uint32_t PossibleU32Bits(Value value) {
|
|||
}
|
||||
}
|
||||
|
||||
bool IsNoncanonicalFlatAddress(Value high, Value active) {
|
||||
high = high.Resolve();
|
||||
active = active.Resolve();
|
||||
// A predicated VGPR write supplies its new value on exactly the lanes which
|
||||
// execute this access. Do not discard a selection under a different mask.
|
||||
while (const auto* select = high.TryInstruction()) {
|
||||
if (select->GetOpcode() != ValueOpcode::SelectU32 || select->Arg(0).Resolve() != active) break;
|
||||
high = select->Arg(1).Resolve();
|
||||
}
|
||||
uint32_t lower = 0, upper = UINT32_MAX;
|
||||
if (high.IsImmediate() && high.GetType() == Type::U32) {
|
||||
lower = upper = high.U32();
|
||||
} else if (const auto* inst = high.TryInstruction();
|
||||
inst != nullptr && inst->GetOpcode() == ValueOpcode::UMedTri32) {
|
||||
// The median lies between any two constant inputs, regardless of the third.
|
||||
uint32_t count = 0;
|
||||
for (size_t index = 0; index < inst->NumArgs(); ++index) {
|
||||
const auto value = inst->Arg(index).Resolve();
|
||||
if (!value.IsImmediate() || value.GetType() != Type::U32) continue;
|
||||
if (count++ == 0) lower = upper = value.U32();
|
||||
else { lower = std::min(lower, value.U32()); upper = std::max(upper, value.U32()); }
|
||||
}
|
||||
if (count < 2) return false;
|
||||
}
|
||||
// Leave one high DWORD on each side for the signed instruction offset/carry.
|
||||
return lower > 0x00008000u && upper < 0xffff7fffu;
|
||||
}
|
||||
|
||||
Value CanonicalizeSampleAdjustDword3(Value value) {
|
||||
for (;;) {
|
||||
value = value.Resolve();
|
||||
|
|
@ -1649,6 +1677,10 @@ private:
|
|||
|
||||
void Collect(Inst& inst) {
|
||||
const auto op = inst.GetOpcode();
|
||||
if (op == ValueOpcode::BvhIntersect) {
|
||||
m_info.uses_dma = true;
|
||||
return;
|
||||
}
|
||||
const auto buffer = BufferAccessOf(op);
|
||||
const auto address_info = AddressOpcodeInfoOf(op);
|
||||
const auto image_info = ImageOpcodeInfoOf(op);
|
||||
|
|
@ -1695,6 +1727,15 @@ private:
|
|||
if (!IsAddressResourceKind(memory.kind)) {
|
||||
Fail(flags.pc, "address operation has invalid resource kind");
|
||||
}
|
||||
if (memory.kind == ResourceKind::Flat && memory.address_is_full &&
|
||||
IsNoncanonicalFlatAddress(inst.Arg(2), inst.Arg(inst.NumArgs() - 1))) {
|
||||
ValidateAddressHandle(inst.Arg(0), flags.pc);
|
||||
if (memory.data_bits != 32u || m_program.scratch_dwords == 0) {
|
||||
Fail(flags.pc, "local FLAT access requires DWORD data and per-thread scratch storage");
|
||||
}
|
||||
m_program.memory_info[flags.index].kind = ResourceKind::FlatLocal;
|
||||
return;
|
||||
}
|
||||
if (memory.kind == ResourceKind::Scratch) {
|
||||
handle = inst.Arg(0).Resolve().TryInstruction();
|
||||
if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetScratchResource ||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,229 @@
|
|||
TestCase FlatStackApertures(u32 wave_size) {
|
||||
TestCase test;
|
||||
test.name = wave_size == 32 ? "FlatStackApertures32" : "FlatStackApertures64";
|
||||
auto &code = test.code;
|
||||
code = {EncodeSop1(0x04, 8, 237), // private base
|
||||
EncodeSop1(0x04, 10, 235), // shared base
|
||||
EncodeVop2(0x1a, 20, InlineU32(2), 0),
|
||||
EncodeVop2(0x1b, 22, InlineU32(1), 0),
|
||||
EncodeVop2(0x1a, 22, InlineU32(31), 22)};
|
||||
AppendVop3(&code, 0x159, 21, 9, 11, Vgpr(22)); // Clamp the stack address to the local apertures.
|
||||
AppendVMovU32(&code, 30, 100);
|
||||
code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
|
||||
code.insert(code.end(), {EncodeFlat0(0x1c, 1), EncodeFlat1(0, 0x7d, 30, 20)});
|
||||
AppendVMovU32(&code, 30, 200);
|
||||
code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
|
||||
code.insert(code.end(), {EncodeDs0(0x0d), EncodeDs1(0, 30, 20)});
|
||||
AppendVMovU32(&code, 30, 300);
|
||||
code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
|
||||
code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20)});
|
||||
// FLAT writes must alias both explicit memory instructions, per invocation.
|
||||
code.insert(code.end(), {EncodeFlat0(0x0c, 1), EncodeFlat1(24, 0x7d, 0, 20),
|
||||
EncodeDs0(0x36), EncodeDs1(25, 0, 20),
|
||||
EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 24, 0, 0);
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 25, 0, wave_size);
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 2 * wave_size);
|
||||
// Inactive lanes must preserve both stores and destination VGPRs.
|
||||
AppendSMovLiteral(&code, 126, 0x55555555);
|
||||
if (wave_size == 64) AppendSMovLiteral(&code, 127, 0x55555555);
|
||||
code.push_back(EncodeVop2(0x25, 22, InlineU32(1), 22));
|
||||
AppendVop3(&code, 0x159, 21, 9, 11, Vgpr(22));
|
||||
AppendVMovU32(&code, 30, 400);
|
||||
code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
|
||||
code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20)});
|
||||
code.insert(code.end(), {EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
|
||||
code.push_back(EncodeSop1(0x04, 126, 193)); // -1 sign-extends across EXEC
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 3 * wave_size);
|
||||
// Noncanonical addresses outside either aperture, aperture overflow, and
|
||||
// offsets outside allocated local storage must not wrap into valid storage.
|
||||
AppendVMovLiteral(&code, 21, 0x78000000);
|
||||
code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20),
|
||||
EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 4 * wave_size);
|
||||
code.push_back(EncodeVop1(0x01, 21, 11));
|
||||
AppendVMovLiteral(&code, 20, 0xfffffffc);
|
||||
code.insert(code.end(), {EncodeFlat0(0x1c, 0, 4), EncodeFlat1(0, 0x7d, 30, 20),
|
||||
EncodeFlat0(0x0c, 0, 4), EncodeFlat1(26, 0x7d, 0, 20)});
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 5 * wave_size);
|
||||
AppendVMovU32(&code, 20, wave_size * 4);
|
||||
code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20),
|
||||
EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 6 * wave_size);
|
||||
AppendVMovU32(&code, 20, 1);
|
||||
code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20),
|
||||
EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 9 * wave_size);
|
||||
code.push_back(EncodeVop2(0x1a, 20, InlineU32(2), 0));
|
||||
code.insert(code.end(), {EncodeFlat0(0x0c, 1), EncodeFlat1(24, 0x7d, 0, 20),
|
||||
EncodeDs0(0x36), EncodeDs1(25, 0, 20)});
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 24, 0, 7 * wave_size);
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 25, 0, 8 * wave_size);
|
||||
AppendEnd(&code);
|
||||
test.expected.resize(10 * wave_size);
|
||||
for (u32 lane = 0; lane < wave_size; ++lane) {
|
||||
const bool shared = lane & 1;
|
||||
test.expected[lane] = (shared ? 100 : 300) + lane;
|
||||
test.expected[wave_size + lane] = (shared ? 300 : 200) + lane;
|
||||
test.expected[2 * wave_size + lane] = 300 + lane;
|
||||
test.expected[3 * wave_size + lane] = (shared ? 300 : 400) + lane;
|
||||
test.expected[7 * wave_size + lane] = (shared ? 100 : 400) + lane;
|
||||
test.expected[8 * wave_size + lane] = (shared ? 300 : 200) + lane;
|
||||
}
|
||||
test.compute_info.wave_size = wave_size;
|
||||
test.compute_info.threads_num[0] = wave_size;
|
||||
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
|
||||
test.compute_info.thread_ids_num = 1;
|
||||
test.compute_info.scratch_size_dwords = wave_size;
|
||||
test.compute_info.lds_size_dwords = wave_size;
|
||||
test.has_compute_info = true;
|
||||
test.forbidden_spirv = {"get_bda_pointer"};
|
||||
return test;
|
||||
}
|
||||
|
||||
TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0) {
|
||||
TestCase test;
|
||||
test.name = barycentrics ? (sorted ? "BvhBarycentricSorted" : "BvhBarycentricUnsorted")
|
||||
: (sorted ? "BvhTriangleIdSorted" : "BvhTriangleIdUnsorted");
|
||||
if (grow != 0) test.name = "BvhBoxExitGrowth";
|
||||
constexpr u32 NodeStart = 1024, OutputStart = 512, LaneCount = 32;
|
||||
constexpr uint64_t GuestBase = 0x10000;
|
||||
const auto bits = [](float value) { return std::bit_cast<u32>(value); };
|
||||
constexpr u32 Infinity = 0x7f800000, Invalid = UINT32_MAX, Flag = 0x04040404;
|
||||
test.initial.resize(NodeStart + 258 * 16);
|
||||
auto &data = test.initial;
|
||||
// Pack five float3 vertices and the triangle flag into a 64-byte node.
|
||||
const std::array<float, 15> vertices{-1, -1, 2, 1, -1, 2, 0, 1, 2, 2, 1, 2, 0, 3, 2};
|
||||
for (u32 i = 0; i < vertices.size(); ++i) data[NodeStart + i] = bits(vertices[i]);
|
||||
data[NodeStart + 15] = Flag;
|
||||
constexpr std::array<u32, 4> children{0x88, 0x90, 0x98, 0xa0};
|
||||
constexpr std::array<float, 4> z{5, 1, 3, 7};
|
||||
constexpr std::array<u32, 4> half_z{0x4500, 0x3c00, 0x4200, 0x4700};
|
||||
constexpr std::array<u32, 4> half_far{0x4600, 0x4000, 0x4400, 0x4800};
|
||||
for (u32 child = 0; child < 4; ++child) {
|
||||
data[NodeStart + 16 + child] = data[NodeStart + 32 + child] = children[child];
|
||||
const auto fp16 = NodeStart + 20 + child * 3;
|
||||
data[fp16] = 0xbc00bc00; // min.xy = -1,-1
|
||||
data[fp16 + 1] = 0x3c000000 | half_z[child];
|
||||
data[fp16 + 2] = (half_far[child] << 16) | 0x3c00;
|
||||
const auto fp32 = NodeStart + 36 + child * 6;
|
||||
const std::array<float, 6> bounds{-1, -1, z[child], 1, 1, z[child] + 1};
|
||||
for (u32 i = 0; i < 6; ++i) data[fp32 + i] = bits(bounds[i]);
|
||||
}
|
||||
// A grazing ray enters one ULP after exiting on another axis.
|
||||
for (u32 child = 0; child < 4; ++child) {
|
||||
data[NodeStart + 64 + child] = children[child];
|
||||
const std::array<float, 6> bounds = child == 0
|
||||
? std::array<float, 6>{0, 0, std::nextafter(1.0f, 2.0f), 1, 2, 2}
|
||||
: child == 1 ? std::array<float, 6>{-1, 1, -1, -0.0f, 2, 2}
|
||||
: std::array<float, 6>{1, 1, 1, -1, -1, -1};
|
||||
for (u32 i = 0; i < 6; ++i) data[NodeStart + 68 + child * 6 + i] = bits(bounds[i]);
|
||||
}
|
||||
// An FP32 node straddles two guest pages with a 64-byte gap in physical backing.
|
||||
std::copy_n(data.begin() + NodeStart + 32, 16, data.begin() + NodeStart + 255 * 16);
|
||||
std::copy_n(data.begin() + NodeStart + 48, 16, data.begin() + NodeStart + 257 * 16);
|
||||
const std::array<std::array<float, 2>, 4> origins{{{0, 0}, {1, 0}, {.5f, 1.5f}, {-.25f, 1}}};
|
||||
std::array<std::array<u32, 4>, LaneCount> expected{};
|
||||
for (u32 lane = 0; lane < LaneCount; ++lane) {
|
||||
const auto kind = lane % 8;
|
||||
const auto input = lane * 16;
|
||||
data[input] = kind < 4 ? kind : kind == 4 ? 12 : kind == 5 ? 21 : kind == 6 ? 6 : 0x1000;
|
||||
data[input + 1] = bits(6);
|
||||
data[input + 2] = bits(kind < 4 ? origins[kind][0] : 0);
|
||||
data[input + 3] = bits(kind < 4 ? origins[kind][1] : 0);
|
||||
data[input + 7] = bits(1);
|
||||
data[input + 8] = data[input + 9] = Infinity;
|
||||
data[input + 10] = bits(1);
|
||||
if (kind < 4) {
|
||||
const float denominator = kind == 3 ? -2 : -4;
|
||||
const float i = kind == 0 || kind == 3 ? -1 : -2;
|
||||
const float j = kind == 3 ? -.5f : -1;
|
||||
expected[lane] = {bits(2 * denominator), bits(denominator),
|
||||
barycentrics ? bits(i) : Flag + kind,
|
||||
barycentrics ? bits(j) : 1};
|
||||
} else if (kind < 6) {
|
||||
expected[lane] = sorted ? std::array<u32, 4>{children[1], children[2], children[0], Invalid}
|
||||
: std::array<u32, 4>{children[0], children[1], children[2], Invalid};
|
||||
} else {
|
||||
expected[lane] = kind == 6 ? std::array<u32, 4>{Invalid, Invalid, Invalid, Invalid}
|
||||
: std::array<u32, 4>{Infinity, bits(1), Invalid, 0};
|
||||
}
|
||||
if (lane >= 8 && lane < 12) {
|
||||
data[input + 2] = 0x7fc00000; // invalid ray origin
|
||||
expected[lane] = {Infinity, bits(1), Invalid, 0};
|
||||
} else if (lane == 12 || lane == 13) {
|
||||
data[input + 1] = bits(1); // nearest face equals the exclusive extent
|
||||
expected[lane] = {Invalid, Invalid, Invalid, Invalid};
|
||||
} else if (lane >= 16 && lane < 20) {
|
||||
const u32 axis = lane & 1;
|
||||
data[input + 2 + axis] = bits(origins[kind][axis] - 4);
|
||||
data[input + 5 + axis] = bits(2);
|
||||
data[input + 8 + axis] = bits(.5f);
|
||||
// Dominant X/Y projection scales all intersection numerators and the
|
||||
// denominator by four; the distance and barycentrics remain unchanged.
|
||||
for (u32 i = 0; i < (barycentrics ? 4u : 2u); ++i)
|
||||
expected[lane][i] = bits(std::bit_cast<float>(expected[lane][i]) * 4);
|
||||
} else if (lane == 22) {
|
||||
data[input] = 37; // node 4, FP32
|
||||
data[input + 1] = bits(10);
|
||||
for (u32 i = 5; i <= 10; ++i) data[input + i] = bits(1);
|
||||
expected[lane] = {grow ? children[0] : Invalid, Invalid, Invalid, Invalid};
|
||||
} else if (lane >= 24 && lane < 28) {
|
||||
data[input + 4] = bits(4);
|
||||
data[input + 7] = data[input + 10] = bits(-1);
|
||||
for (u32 i = 0; i < (barycentrics ? 4u : 2u); ++i)
|
||||
expected[lane][i] = bits(-std::bit_cast<float>(expected[lane][i]));
|
||||
} else if (lane == 28) {
|
||||
data[input] = 0;
|
||||
data[input + 3] = bits(-1); // horizontal triangle edge: top-left rule rejects
|
||||
expected[lane] = {Infinity, bits(1), barycentrics ? bits(-2) : Flag,
|
||||
barycentrics ? bits(-2) : 0};
|
||||
} else if (lane == 29) {
|
||||
data[input] = (255 << 3) | 5;
|
||||
}
|
||||
}
|
||||
test.expected = test.initial;
|
||||
for (u32 lane = 0; lane < LaneCount; ++lane) {
|
||||
std::copy(expected[lane].begin(), expected[lane].end(),
|
||||
test.expected.begin() + OutputStart + lane * 4);
|
||||
auto bounded = expected[lane];
|
||||
if (lane % 8 == 5) bounded = {Invalid, Invalid, Invalid, Invalid};
|
||||
if (lane >= 20) bounded = {0x12345678, 0x12345679, 0x1234567a, 0x1234567b};
|
||||
std::copy(bounded.begin(), bounded.end(),
|
||||
test.expected.begin() + OutputStart + LaneCount * 4 + lane * 4);
|
||||
}
|
||||
auto &code = test.code;
|
||||
// The resource is a GPU value, not a materialized host image descriptor.
|
||||
AppendSMovLiteral(&code, 8, GuestBase >> 8);
|
||||
AppendSMovLiteral(&code, 9, (sorted ? 0x80000000 : 0) | (grow << 23));
|
||||
AppendSMovLiteral(&code, 10, 256);
|
||||
AppendSMovLiteral(&code, 11, barycentrics ? 0x81000000 : 0x80000000);
|
||||
code.push_back(EncodeVop2(0x1a, 1, InlineU32(6), 0));
|
||||
for (u32 offset : {0u, 4u, 8u}) {
|
||||
code.insert(code.end(), {EncodeMubuf0(offset == 8 ? 0x0f : 0x0e, offset * 4),
|
||||
EncodeMubuf1(4 + offset, 0, 1)});
|
||||
}
|
||||
code.insert(code.end(), {EncodeMimg0(0xe6, 15, 0, false, 0, true),
|
||||
EncodeMimg1(20, 4, 2)});
|
||||
code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0));
|
||||
for (u32 i = 0; i < 4; ++i) AppendStoreVgprAtLaneDwordOffset(&code, 20 + i, 2, OutputStart + i);
|
||||
// NSA addressing, the FP32 second-block bound, and inactive destination lanes.
|
||||
AppendSMovLiteral(&code, 10, 2);
|
||||
for (u32 i = 0; i < 4; ++i) AppendVMovLiteral(&code, 24 + i, 0x12345678 + i);
|
||||
AppendSMovLiteral(&code, 126, 0x000fffff);
|
||||
code.insert(code.end(), {EncodeMimg0(0xe6, 15, 3, false, 0, true),
|
||||
EncodeMimg1(24, 4, 2), 0x08070605, 0x0c0b0a09, 0x00000e0d});
|
||||
code.push_back(EncodeSop1(0x04, 126, 193));
|
||||
for (u32 i = 0; i < 4; ++i)
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 24 + i, 2, OutputStart + LaneCount * 4 + i);
|
||||
AppendEnd(&code);
|
||||
test.bda_mappings = {{GuestBase, NodeStart * sizeof(u32)},
|
||||
{GuestBase + 0x4000, (NodeStart + 257 * 16) * sizeof(u32)}};
|
||||
test.compute_info.wave_size = LaneCount;
|
||||
test.compute_info.threads_num[0] = LaneCount;
|
||||
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
|
||||
test.compute_info.thread_ids_num = 1;
|
||||
test.has_compute_info = true;
|
||||
test.required_spirv = {"get_bda_pointer"};
|
||||
return test;
|
||||
}
|
||||
|
|
@ -1,87 +1,50 @@
|
|||
void TestRayTracingDispatchDetection() {
|
||||
void TestRayTracingInstructions() {
|
||||
namespace D = ShaderRecompiler::Decoder;
|
||||
const auto check_decoded = [](std::span<const uint32_t> code, bool expected,
|
||||
const char *message) {
|
||||
D::Program decoded;
|
||||
D::DecodeProgram(code, decoded);
|
||||
Check(decoded.has_bvh == expected, message);
|
||||
if (expected) {
|
||||
const auto &last = decoded.instructions.back();
|
||||
Check(last.family == D::Family::MIMG &&
|
||||
(last.opcode_id == 0xe6u || last.opcode_id == 0xe7u),
|
||||
"decoder continued past the first BVH instruction");
|
||||
const auto result = ShaderRecompiler::TranslateProgram(
|
||||
code, MakeCompileOptions(ShaderType::Compute));
|
||||
Check(result.skip_dispatch && result.program.blocks.empty() && result.cfg_dump.empty(),
|
||||
"BVH compute shader reached IR translation");
|
||||
}
|
||||
};
|
||||
const auto check = [&](std::initializer_list<uint32_t> code, bool expected,
|
||||
const char *message) {
|
||||
check_decoded(std::span<const uint32_t>{code.begin(), code.size()}, expected, message);
|
||||
};
|
||||
|
||||
check({0xf1989f01u, 0x00081014u, EncodeSopp(0x01)}, true,
|
||||
"captured contiguous BVH instruction did not skip dispatch");
|
||||
check({0xf1989f07u, 0x00071001u, 0x3d3e3c15u, 0x38095851u, 0x00003a39u,
|
||||
EncodeSopp(0x01)}, true,
|
||||
"captured NSA BVH instruction did not skip dispatch");
|
||||
check({0xf19c9f01u, 0x00081014u, EncodeSopp(0x01)}, true,
|
||||
"RDNA2 BVH64 instruction did not skip dispatch");
|
||||
check({0xbe8e210eu, 0xf1989f01u, 0x00081014u, EncodeSopp(0x01)}, true,
|
||||
"unsupported SWAPPC before BVH prevented dispatch detection");
|
||||
check({0xf1989f01u, 0x00081014u, 0xc0000000u, EncodeSopp(0x01)}, true,
|
||||
"unknown instruction after BVH prevented dispatch skipping");
|
||||
|
||||
for (uint32_t opcode : {0xe6u, 0xe7u}) {
|
||||
for (uint32_t nsa = 0; nsa < 4; ++nsa) {
|
||||
for (uint32_t slc = 0; slc < 2; ++slc) {
|
||||
std::array<uint32_t, 6> code{
|
||||
EncodeMimg0(opcode, 0xf, true) | (nsa << 1u) | (slc << 25u),
|
||||
EncodeMimg1(0, 0, 0, 0)};
|
||||
code[2 + nsa] = EncodeSopp(0x01);
|
||||
check_decoded(std::span(code).first(3 + nsa), true,
|
||||
"BVH cache flags or NSA length prevented dispatch detection");
|
||||
}
|
||||
}
|
||||
const std::array code{
|
||||
0xf1989f01u, 0x00081014u,
|
||||
EncodeMubuf0(0x1e, 0, false), EncodeMubuf1(16, 0, 0),
|
||||
0xf1989f07u, 0x00071001u, 0x3d3e3c15u, 0x38095851u, 0x00003a39u,
|
||||
EncodeMubuf0(0x1e, 16, false), EncodeMubuf1(16, 0, 0),
|
||||
EncodeSopp(0x01)};
|
||||
D::Program decoded;
|
||||
D::DecodeProgram(code, decoded);
|
||||
Check(decoded.instructions.size() == 5 &&
|
||||
decoded.instructions.back().opcode == D::Opcode::S_ENDPGM,
|
||||
"decoder stopped before instructions following BVH intersections");
|
||||
for (const auto index : {0u, 2u}) {
|
||||
const auto &inst = decoded.instructions[index];
|
||||
Check(inst.opcode == D::Opcode::IMAGE_BVH_INTERSECT_RAY &&
|
||||
inst.image_address_components == 11 && inst.data_dwords == 4 && inst.image_r128,
|
||||
"BVH32 instruction lost its ray or descriptor width");
|
||||
}
|
||||
for (uint32_t opcode : {0x66u, 0x67u, 0xe4u, 0xe5u, 0xe8u, 0xe9u}) {
|
||||
check({EncodeMimg0(opcode, 0xf), EncodeMimg1(0, 0, 0, 0), EncodeSopp(0x01)},
|
||||
false, "non-BVH image opcode was incorrectly skipped");
|
||||
Check(decoded.instructions[2].image_nsa_addr[9] == 58,
|
||||
"BVH NSA ray components were not decoded");
|
||||
const auto result = RecompileForTest(code, MakeCompileOptions(ShaderType::Compute));
|
||||
Check(CountSourceOccurrences(result.ir_dump, "BvhIntersect") == 2 &&
|
||||
result.program.info.uses_dma && result.program.info.images.empty() &&
|
||||
result.program.info.samplers.empty(),
|
||||
"BVH intersections did not use GPU descriptors and raw guest memory");
|
||||
CheckSpirvBinaryValidates(result.spirv);
|
||||
|
||||
const std::array literal{EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)};
|
||||
D::DecodeProgram(literal, decoded);
|
||||
Check(decoded.instructions.size() == 2 && decoded.instructions[0].opcode == D::Opcode::S_MOV_B32,
|
||||
"BVH-shaped scalar literal was decoded as an instruction");
|
||||
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
|
||||
for (const auto words : {std::array{0xf19c9f01u, 0x00081014u}, // BVH64
|
||||
std::array{0xf1989f01u, 0x40081014u}, // A16
|
||||
std::array{0xf1981f01u, 0x00081014u}, // not R128
|
||||
std::array{0xf1989f01u, 0x80081014u}}) { // D16
|
||||
const std::array unsupported{words[0], words[1], EncodeSopp(0x01)};
|
||||
ExpectFatal([&]() { (void)RecompileForTest(unsupported, MakeCompileOptions(ShaderType::Compute)); },
|
||||
"unsupported BVH encoding was silently accepted");
|
||||
}
|
||||
|
||||
check({EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)}, false,
|
||||
"scalar literal was mistaken for a BVH instruction");
|
||||
check({EncodeMimg0(0x20, 0xf) | 6u, EncodeMimg1(0, 0, 0, 0),
|
||||
0xf1989f01u, 0xf19c9f01u, 0u, EncodeSopp(0x01)}, false,
|
||||
"NSA address words were mistaken for BVH instructions");
|
||||
check({EncodeMimg0(0x20, 0xf), 0xf1989f01u, EncodeSopp(0x01)}, false,
|
||||
"MIMG operand word was mistaken for a BVH instruction");
|
||||
check({EncodeSopp(0x01), 0xf1989f01u, 0x00081014u}, false,
|
||||
"data after the shader end was mistaken for a BVH instruction");
|
||||
check({EncodeMimg0(0x20, 0xf), EncodeMimg1(0, 0, 0, 0), EncodeSopp(0x01)},
|
||||
false, "ordinary image shader was incorrectly skipped");
|
||||
|
||||
// An end instruction on one branch does not end another branch's code.
|
||||
check({EncodeSopp(0x04, 1), EncodeSopp(0x01), 0xf1989f01u, 0x00081014u,
|
||||
EncodeSopp(0x01)}, true,
|
||||
"BVH at a branch target after an early end was not detected");
|
||||
|
||||
const std::array bvh_code{0xf1989f01u, 0x00081014u, 0xf19c9f01u, 0x00081014u,
|
||||
EncodeSopp(0x01)};
|
||||
const std::array ordinary_code{EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)};
|
||||
D::Program reused;
|
||||
D::DecodeProgram(bvh_code, reused);
|
||||
Check(reused.has_bvh && reused.instructions.size() == 1,
|
||||
"decoder did not stop at the first BVH instruction");
|
||||
D::DecodeProgram(ordinary_code, reused);
|
||||
Check(!reused.has_bvh, "decoder retained stale BVH metadata");
|
||||
for (const auto stage : {ShaderType::Compute, ShaderType::Vertex, ShaderType::Pixel}) {
|
||||
const auto options = MakeCompileOptions(stage);
|
||||
Check(!ShaderRecompiler::TranslateProgram(ordinary_code, options).skip_dispatch,
|
||||
"BVH-shaped literal marked an ordinary shader for skipping");
|
||||
const auto compiled = RecompileForTest(ordinary_code, options);
|
||||
Check(!compiled.spirv.empty(), "ordinary shader did not compile");
|
||||
CheckSpirvBinaryValidates(compiled.spirv);
|
||||
for (const auto nsa : {1u, 2u}) {
|
||||
std::array<uint32_t, 5> truncated{0xf1989f01u | (nsa << 1), 0x00081014u};
|
||||
truncated[2 + nsa] = EncodeSopp(0x01);
|
||||
ExpectFatal([&]() {
|
||||
(void)RecompileForTest(std::span(truncated).first(3 + nsa), MakeCompileOptions(ShaderType::Compute));
|
||||
}, "BVH NSA instruction accepted fewer than eleven ray addresses");
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30715,6 +30715,8 @@ TestCase DispatcherIrreducibleControlFlow() {
|
|||
return test;
|
||||
}
|
||||
|
||||
#include "ShaderRayTracingGpuTests.inc"
|
||||
|
||||
std::vector<TestCase> MakeCases() {
|
||||
std::vector<TestCase> cases;
|
||||
cases.reserve(128);
|
||||
|
|
@ -31006,6 +31008,14 @@ std::vector<TestCase> MakeCases() {
|
|||
AddCase(GlobalSignedImmediateRebasesBeforeSaddr);
|
||||
AddCase(FlatSegmentIgnoresSaddrAndMasksOffsetMsb);
|
||||
AddCase(ScratchIsPrivatePerInvocation);
|
||||
cases.push_back(FlatStackApertures(32));
|
||||
cases.push_back(FlatStackApertures(64));
|
||||
cases.push_back(BvhIntersections(true, true, 1));
|
||||
for (bool barycentrics : {false, true}) {
|
||||
for (bool sorted : {false, true}) {
|
||||
cases.push_back(BvhIntersections(barycentrics, sorted));
|
||||
}
|
||||
}
|
||||
AddCase(DsReadWriteVariants);
|
||||
AddCase(DsWriteB16D16HiCapturedUsesHighHalf);
|
||||
AddCase(DsWriteB8D16HiWritesByteTwo);
|
||||
|
|
@ -35903,6 +35913,18 @@ int main(int argc, char **argv) {
|
|||
return 0;
|
||||
}
|
||||
#endif
|
||||
if (argc == 2 && std::strcmp(argv[1], "--ray-tracing-only") == 0) {
|
||||
VulkanHarness vulkan;
|
||||
RunCase(&vulkan, FlatStackApertures(32));
|
||||
RunCase(&vulkan, FlatStackApertures(64));
|
||||
RunCase(&vulkan, BvhIntersections(true, true, 1));
|
||||
for (bool barycentrics : {false, true}) {
|
||||
for (bool sorted : {false, true}) {
|
||||
RunCase(&vulkan, BvhIntersections(barycentrics, sorted));
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
if (argc == 2 && std::strcmp(argv[1], "--indirect-buffer-only") == 0) {
|
||||
VulkanHarness vulkan;
|
||||
RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors());
|
||||
|
|
|
|||
|
|
@ -13658,6 +13658,30 @@ void TestNewShaderRecompilerFlatAddressDomainsUseDma() {
|
|||
Check((result.ir_dump.find("GetScratchResource") != std::string::npos) &&
|
||||
result.program.scratch_dwords == 1,
|
||||
"SCRATCH incorrectly entered guest address tracking");
|
||||
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
|
||||
// A local aperture proof must not admit a median which can still produce a
|
||||
// canonical global address. Such stores need the existing ownership proof.
|
||||
const uint32_t mixed_domain_store[] = {
|
||||
EncodeSop1(0x04, 8, 237), // s_mov_b64 s[8:9], private_base
|
||||
EncodeVop3Word0(0x159, 1),
|
||||
EncodeVop3Word1(128, 9, 256 + 2), // median(0, private high, dynamic v2)
|
||||
EncodeFlat0(0x1c, 0, 0), EncodeFlat1(0, 0x7d, 3, 0),
|
||||
EncodeSopp(0x01),
|
||||
};
|
||||
ExpectFatal([&]() { (void)RecompileForTest(mixed_domain_store, options); },
|
||||
"mixed local/global FLAT store bypassed GPU ownership tracking");
|
||||
const uint32_t changed_mask_store[] = {
|
||||
EncodeSop1(0x04, 8, 237), EncodeSop1(0x04, 10, 235),
|
||||
EncodeVop1(0x01, 1, 128), // canonical old address high
|
||||
EncodeSop1(0x03, 126, 255), 0x55555555u,
|
||||
EncodeVop3Word0(0x159, 1), EncodeVop3Word1(9, 11, 256 + 2),
|
||||
EncodeSop1(0x04, 126, 193), // restore all lanes after the median
|
||||
EncodeFlat0(0x1c, 0, 0), EncodeFlat1(0, 0x7d, 3, 0),
|
||||
EncodeSopp(0x01),
|
||||
};
|
||||
ExpectFatal([&]() { (void)RecompileForTest(changed_mask_store, options); },
|
||||
"FLAT proof discarded a differently predicated address value");
|
||||
#endif
|
||||
}
|
||||
|
||||
void TestCompilerStageInputOwnership() {
|
||||
|
|
@ -14043,7 +14067,7 @@ int main() {
|
|||
using namespace Libs::Graphics;
|
||||
|
||||
EnsureConfigInitialized();
|
||||
TestRayTracingDispatchDetection();
|
||||
TestRayTracingInstructions();
|
||||
TestResourceDescriptorClassification();
|
||||
TestShaderBufferResourceSize();
|
||||
TestNativeShaderResourceDependencies();
|
||||
|
|
|
|||
Loading…
Reference in New Issue