shader: link scalar leaf calls only for SWAPPC programs
Co-authored-by: Behzad <behzadkhan1736@gmail.com>
This commit is contained in:
nmzik
parent
165cd884ca
commit
1818aae562
|
|
@ -217,17 +217,20 @@ struct PipelineCache::ProgramCache {
|
|||
};
|
||||
|
||||
struct Permutation {
|
||||
std::vector<uint32_t> function_code;
|
||||
ShaderRecompiler::IR::ResourceSpecialization specialization;
|
||||
ShaderRecompiler::IR::CompiledShaderInfo program;
|
||||
ShaderProgram handle;
|
||||
};
|
||||
|
||||
struct SourceEntry {
|
||||
explicit SourceEntry(ShaderRecompiler::IR::ResourcePlan plan)
|
||||
: resource_plan(std::move(plan)) {
|
||||
SourceEntry(ShaderRecompiler::IR::ResourcePlan plan,
|
||||
std::unique_ptr<ShaderRecompiler::ShaderSource> call_source)
|
||||
: call_source(std::move(call_source)), resource_plan(std::move(plan)) {
|
||||
permutations.reserve(8);
|
||||
}
|
||||
|
||||
std::unique_ptr<ShaderRecompiler::ShaderSource> call_source;
|
||||
ShaderRecompiler::IR::ResourcePlan resource_plan;
|
||||
ShaderRecompiler::IR::ResourceSnapshot resources;
|
||||
ShaderRecompiler::IR::ResourceSpecialization specialization;
|
||||
|
|
@ -256,7 +259,8 @@ struct PipelineCache::ProgramCache {
|
|||
const ShaderRecompiler::CompileOptions& options,
|
||||
ShaderRecompiler::TranslateResult translated,
|
||||
ShaderRecompiler::IR::ResourceSpecialization specialization,
|
||||
uint32_t push_data_start_dword) {
|
||||
uint32_t push_data_start_dword,
|
||||
std::span<const uint32_t> function_code) {
|
||||
auto result = ShaderRecompiler::CompileProgram(std::move(translated), options,
|
||||
specialization, push_data_start_dword);
|
||||
if (!ValidateShaderSpirv(options.dump_label, options.shader_hash, result.spirv)) {
|
||||
|
|
@ -273,6 +277,7 @@ struct PipelineCache::ProgramCache {
|
|||
static_cast<uint64_t>(result.spirv.size()), options.wave_size);
|
||||
}
|
||||
return {
|
||||
.function_code = {function_code.begin(), function_code.end()},
|
||||
.specialization = std::move(specialization),
|
||||
.program = std::move(result.program).TakeCompiledInfo(),
|
||||
.handle = {.id = ++next_shader_id, .module = module},
|
||||
|
|
@ -307,21 +312,33 @@ struct PipelineCache::ProgramCache {
|
|||
if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) {
|
||||
runtime.workgroup_counts = input_info.workgroup_counts;
|
||||
}
|
||||
if (entry != programs.end()) {
|
||||
bool source_changed = false;
|
||||
const auto function_code = [](const std::unique_ptr<ShaderRecompiler::ShaderSource>& source) {
|
||||
return source ? std::span<const uint32_t>(source->linked->code).subspan(source->code.size())
|
||||
: std::span<const uint32_t>{};
|
||||
};
|
||||
if (entry != programs.end() && entry->second.call_source) {
|
||||
auto& cached = entry->second;
|
||||
const auto revision = cached.call_source->revision;
|
||||
ShaderRecompiler::RefreshShaderSource(*cached.call_source, runtime);
|
||||
source_changed = revision != cached.call_source->revision;
|
||||
cached.resource_plan.source_reads = cached.call_source->reads;
|
||||
}
|
||||
const auto matches = [&](const Permutation& candidate) {
|
||||
const auto& layout = candidate.program.bindings;
|
||||
return layout.push_data_start_dword == ShaderRecompiler::IR::PushData::StartFor(
|
||||
push_data_cursor, layout.ShaderDataDwords()) &&
|
||||
candidate.specialization == entry->second.specialization &&
|
||||
(!entry->second.call_source || std::ranges::equal(
|
||||
candidate.function_code, function_code(entry->second.call_source)));
|
||||
};
|
||||
if (entry != programs.end() && !source_changed) {
|
||||
EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(
|
||||
entry->second.resource_plan, runtime, entry->second.resources,
|
||||
entry->second.specialization));
|
||||
if (const auto permutation = std::ranges::find_if(
|
||||
entry->second.permutations, [&](const Permutation& candidate) {
|
||||
const auto& layout = candidate.program.bindings;
|
||||
return layout.push_data_start_dword ==
|
||||
ShaderRecompiler::IR::PushData::StartFor(
|
||||
push_data_cursor, layout.ShaderDataDwords()) &&
|
||||
candidate.specialization == entry->second.specialization;
|
||||
});
|
||||
if (const auto permutation = std::ranges::find_if(entry->second.permutations, matches);
|
||||
permutation != entry->second.permutations.end()) {
|
||||
input_info.stage = {.program = &permutation->program,
|
||||
.resources = &entry->second.resources};
|
||||
input_info.stage = {.program = &permutation->program, .resources = &entry->second.resources};
|
||||
permutation->program.bindings.AdvancePushData(push_data_cursor);
|
||||
return permutation->handle;
|
||||
}
|
||||
|
|
@ -368,16 +385,41 @@ struct PipelineCache::ProgramCache {
|
|||
options.wave_size = input_info.wave_size;
|
||||
}
|
||||
DumpShaderOriginal(stage_name, options.shader_hash, params.code);
|
||||
auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
|
||||
ShaderRecompiler::ShaderSource source;
|
||||
if (entry == programs.end()) source = ShaderRecompiler::PrepareShaderSource(params.code, options);
|
||||
auto* current = entry == programs.end() ? &source : entry->second.call_source.get();
|
||||
ShaderRecompiler::TranslateResult translated;
|
||||
if (current != nullptr) {
|
||||
const auto& decoded = entry == programs.end()
|
||||
? ShaderRecompiler::RefreshShaderSource(*current, runtime)
|
||||
: current->linked->decoded;
|
||||
translated = ShaderRecompiler::TranslateProgram(decoded, options);
|
||||
translated.program.source_reads = current->reads;
|
||||
} else {
|
||||
translated = ShaderRecompiler::TranslateProgram(params.code, options);
|
||||
}
|
||||
if (entry == programs.end()) {
|
||||
auto retained = source.call ? std::make_unique<ShaderRecompiler::ShaderSource>(std::move(source)) : nullptr;
|
||||
entry = programs.try_emplace(lookup_key,
|
||||
ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first;
|
||||
ShaderRecompiler::IR::ExtractResourcePlan(translated.program), std::move(retained)).first;
|
||||
EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(
|
||||
entry->second.resource_plan, runtime, entry->second.resources,
|
||||
entry->second.specialization));
|
||||
} else if (source_changed) {
|
||||
entry->second.resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
|
||||
EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(
|
||||
entry->second.resource_plan, runtime, entry->second.resources,
|
||||
entry->second.specialization));
|
||||
if (const auto permutation = std::ranges::find_if(entry->second.permutations, matches);
|
||||
permutation != entry->second.permutations.end()) {
|
||||
input_info.stage = {.program = &permutation->program, .resources = &entry->second.resources};
|
||||
permutation->program.bindings.AdvancePushData(push_data_cursor);
|
||||
return permutation->handle;
|
||||
}
|
||||
}
|
||||
entry->second.permutations.push_back(CompilePermutation(
|
||||
stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor));
|
||||
stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor,
|
||||
function_code(entry->second.call_source)));
|
||||
const auto& permutation = entry->second.permutations.back();
|
||||
input_info.stage = {.program = &permutation.program, .resources = &entry->second.resources};
|
||||
permutation.program.bindings.AdvancePushData(push_data_cursor);
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
|
||||
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
|
||||
#include "graphics/shader/recompiler/frontend/translate/Translate.h"
|
||||
#include "graphics/shader/recompiler/frontend/translate/Translator.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
|
||||
|
|
@ -20,6 +21,7 @@
|
|||
#include <array>
|
||||
#include <chrono>
|
||||
#include <fmt/format.h>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <span>
|
||||
#include <utility>
|
||||
|
|
@ -478,11 +480,229 @@ Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<c
|
|||
return result;
|
||||
}
|
||||
|
||||
void PrepareCallTarget(ShaderSource& source, const CompileOptions& options) {
|
||||
using namespace IR;
|
||||
for (uint32_t i = 0; i < source.decoded.instructions.size(); ++i) {
|
||||
if (source.decoded.instructions[i].opcode != Decoder::Opcode::S_SWAPPC_B64) continue;
|
||||
EXIT_NOT_IMPLEMENTED(source.call.has_value() || options.stage != ShaderType::Compute);
|
||||
source.call = ShaderSource::Call {.instruction = i};
|
||||
}
|
||||
if (!source.call) return;
|
||||
EXIT_IF(options.input_info.compute == nullptr);
|
||||
const auto& call = source.decoded.instructions[source.call->instruction];
|
||||
EXIT_NOT_IMPLEMENTED(call.src0.kind != Decoder::OperandKind::Sgpr ||
|
||||
call.dst.kind != Decoder::OperandKind::Sgpr ||
|
||||
call.src0.reg != call.dst.reg || call.dst.reg >= 105u);
|
||||
const auto graph = CFG::BuildGraph(source.decoded);
|
||||
Program query;
|
||||
query.stage = options.stage;
|
||||
query.user_data_base = options.user_data_base;
|
||||
query.user_data_count = static_cast<uint32_t>(options.user_data.size());
|
||||
query.wave_size = options.wave_size;
|
||||
auto& block = *query.block_storage.emplace_back(std::make_unique<Block>());
|
||||
query.blocks.push_back(&block);
|
||||
struct Producer { bool ready = false; std::map<uint32_t, Value> values; };
|
||||
std::map<uint32_t, Producer> producers;
|
||||
std::function<Value(uint32_t, uint32_t)> Register;
|
||||
std::function<Value(uint32_t, Value)> Resolve = [&](uint32_t before, Value value) -> Value {
|
||||
value = value.Resolve();
|
||||
auto* inst = value.TryInstruction();
|
||||
if (inst == nullptr) return value;
|
||||
uint32_t code = UINT32_MAX;
|
||||
switch (inst->GetOpcode()) {
|
||||
case ValueOpcode::GetScalarRegister: code = static_cast<uint32_t>(inst->Arg(0).ScalarRegister()); break;
|
||||
case ValueOpcode::GetVccLo: code = 106u; break;
|
||||
case ValueOpcode::GetVccHi: code = 107u; break;
|
||||
default: break;
|
||||
}
|
||||
if (code != UINT32_MAX) {
|
||||
const auto result = Register(before, code);
|
||||
inst->ReplaceUsesWith(result);
|
||||
return result;
|
||||
}
|
||||
for (size_t i = 0; i < inst->NumArgs(); ++i) inst->SetArg(i, Resolve(before, inst->Arg(i)));
|
||||
return value;
|
||||
};
|
||||
Register = [&](uint32_t before, uint32_t code) -> Value {
|
||||
const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code);
|
||||
if (index == UINT32_MAX) {
|
||||
EXIT_IF(code < query.user_data_base || code - query.user_data_base >= query.user_data_count);
|
||||
return IREmitter(&block).GetUserData(static_cast<ScalarReg>(code));
|
||||
}
|
||||
auto [entry, inserted] = producers.try_emplace(index);
|
||||
auto& producer = entry->second;
|
||||
if (inserted) {
|
||||
Block translated;
|
||||
Frontend::Translator translator(query, &translated, 1u,
|
||||
(options.input_info.compute->float_mode & 0x10u) == 0u);
|
||||
translator.TranslateInstruction(source.decoded.instructions[index]);
|
||||
for (const auto& inst: translated) {
|
||||
uint32_t destination = UINT32_MAX;
|
||||
Value value;
|
||||
switch (inst.GetOpcode()) {
|
||||
case ValueOpcode::SetScalarRegister:
|
||||
destination = static_cast<uint32_t>(inst.Arg(0).ScalarRegister()); value = inst.Arg(1); break;
|
||||
case ValueOpcode::SetVccLo: destination = 106u; value = inst.Arg(0); break;
|
||||
case ValueOpcode::SetVccHi: destination = 107u; value = inst.Arg(0); break;
|
||||
default: break;
|
||||
}
|
||||
if (destination != UINT32_MAX) producer.values.insert_or_assign(destination, value);
|
||||
}
|
||||
for (auto& [destination, value]: producer.values) value = Resolve(index, value);
|
||||
for (auto& inst: translated) inst.SetParent(&block);
|
||||
block.Instructions().splice(block.end(), translated.Instructions());
|
||||
producer.ready = true;
|
||||
}
|
||||
EXIT_IF(!producer.ready || !producer.values.contains(code));
|
||||
return producer.values.at(code).Resolve();
|
||||
};
|
||||
for (uint32_t i = 0; i < 2u; ++i) {
|
||||
const auto value = Register(source.call->instruction, call.src0.reg + i);
|
||||
EXIT_IF(!ValidateRuntimeValue(query, value, RuntimeValueType::Integer));
|
||||
auto& root = query.value_storage.emplace_back(ValueOpcode::ReferenceU32);
|
||||
root.SetArg(0, value);
|
||||
source.call->target[i] = Value(&root);
|
||||
}
|
||||
RewriteToSsa(query.blocks);
|
||||
ConstantPropagationPass(query.blocks, query.wave_size);
|
||||
RemoveIdentities(query.blocks);
|
||||
EliminateDeadCode(query.blocks);
|
||||
for (auto& value: source.call->target) value = value.Instruction()->Arg(0).Resolve();
|
||||
query.value_storage.clear();
|
||||
for (auto& inst: block) {
|
||||
inst.SetParent(nullptr);
|
||||
if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 || inst.GetOpcode() == ValueOpcode::ReadConstBuffer)
|
||||
inst.SetFlags(SrtReadFlags {.index = inst.Flags<MemoryFlags>().index});
|
||||
}
|
||||
query.value_storage.splice(query.value_storage.end(), block.Instructions());
|
||||
source.call_targets = std::move(static_cast<ResourcePlan&>(query));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
ShaderSource PrepareShaderSource(std::span<const uint32_t> code, const CompileOptions& options) {
|
||||
EXIT_IF(code.empty());
|
||||
ShaderSource source;
|
||||
if (!options.back_code.empty()) {
|
||||
source.decoded = DecodeFusedProgram(code, options.back_code, source.code);
|
||||
} else if (options.stage == ShaderType::Local) {
|
||||
source.decoded = Decoder::DecodeFrontProgram(code);
|
||||
auto& handoff = source.decoded.instructions.back();
|
||||
handoff.opcode = Decoder::Opcode::S_ENDPGM;
|
||||
handoff.src_count = 0;
|
||||
} else {
|
||||
Decoder::DecodeProgram(code, source.decoded);
|
||||
}
|
||||
PrepareCallTarget(source, options);
|
||||
if (source.call) {
|
||||
source.code.assign(code.begin(), code.end());
|
||||
source.decoded.code = source.code;
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtRuntime& runtime) {
|
||||
auto& reads = source.reads;
|
||||
reads.clear();
|
||||
if (!source.call) return source.decoded;
|
||||
IR::SrtReadCapture capture(runtime, reads);
|
||||
const auto clean = IR::CleanRuntime(capture.ObservedRuntime());
|
||||
IR::SrtWalker walker(source.call_targets, clean);
|
||||
uint32_t low = 0, high = 0;
|
||||
const auto& call = source.decoded.instructions[source.call->instruction];
|
||||
if (!walker.Evaluate(source.call->target[0], low) || !walker.Evaluate(source.call->target[1], high))
|
||||
EXIT("shader call at pc 0x%08x has an unavailable scalar target", call.pc);
|
||||
// SWAPPC reads the old aliased pair and ignores the target's low two bits.
|
||||
const auto address = ((uint64_t {high} << 32u) | low) & ~uint64_t {3};
|
||||
EXIT_IF(address == 0u);
|
||||
const auto query_reads = reads.size();
|
||||
if (source.linked) {
|
||||
auto& linked = *source.linked;
|
||||
const auto function = std::span(linked.code).subspan(source.code.size());
|
||||
if (clean.read_memory(clean.userdata, address, linked.observed_function) &&
|
||||
std::ranges::equal(function, linked.observed_function)) {
|
||||
return linked.decoded;
|
||||
}
|
||||
reads.resize(query_reads);
|
||||
}
|
||||
ShaderSource::Linked linked {.code = source.code,
|
||||
.decoded = source.decoded};
|
||||
const auto& last = source.decoded.instructions.back();
|
||||
EXIT_NOT_IMPLEMENTED(last.opcode != Decoder::Opcode::S_ENDPGM);
|
||||
// Keep the caller's data/footer bytes at their native offsets. The leaf's virtual CFG
|
||||
// PCs follow decoded instructions; its physical code is independently read and owned.
|
||||
const auto function_pc = last.pc + last.word_count * sizeof(uint32_t);
|
||||
for (uint32_t index = 0;;) {
|
||||
// The observed queue helper is a scalar leaf. Scalar encodings need at most one literal.
|
||||
EXIT_IF(index >= 64u || address > UINT64_MAX - (uint64_t {index} + 2u) * sizeof(uint32_t));
|
||||
std::array<uint32_t, 2> words {};
|
||||
if (!clean.read_memory(clean.userdata, address + uint64_t {index} * sizeof(uint32_t),
|
||||
std::span(words).first(1)))
|
||||
EXIT("shader call at pc 0x%08x cannot read leaf code", call.pc);
|
||||
const auto family = Decoder::GetInstructionFamily(words[0]);
|
||||
EXIT_NOT_IMPLEMENTED(family != Decoder::Family::SOP1 && family != Decoder::Family::SOP2 &&
|
||||
family != Decoder::Family::SOPC && family != Decoder::Family::SOPK &&
|
||||
family != Decoder::Family::SOPP);
|
||||
Decoder::Instruction inst;
|
||||
Decoder::DecodeInstruction(words, 0u, inst);
|
||||
if (inst.word_count == 2u) {
|
||||
if (!clean.read_memory(clean.userdata, address + uint64_t {index + 1u} * sizeof(uint32_t),
|
||||
std::span(words).last(1)))
|
||||
EXIT("shader call at pc 0x%08x cannot read leaf literal", call.pc);
|
||||
inst = {};
|
||||
Decoder::DecodeInstruction(words, 0u, inst);
|
||||
}
|
||||
inst.pc = index * sizeof(uint32_t);
|
||||
if (Decoder::IsDirectBranch(inst.opcode)) inst.branch_target += inst.pc;
|
||||
linked.code.insert(linked.code.end(), inst.raw, inst.raw + inst.word_count);
|
||||
index += inst.word_count;
|
||||
EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_SWAPPC_B64 ||
|
||||
inst.opcode == Decoder::Opcode::S_GETPC_B64 ||
|
||||
inst.opcode == Decoder::Opcode::S_ENDPGM);
|
||||
EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg) || CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u));
|
||||
const bool returns = inst.opcode == Decoder::Opcode::S_SETPC_B64;
|
||||
if (returns) {
|
||||
EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr || inst.src0.reg != call.dst.reg);
|
||||
inst.opcode = Decoder::Opcode::S_BRANCH;
|
||||
inst.branch_target = call.pc + call.word_count * sizeof(uint32_t);
|
||||
} else if (Decoder::IsDirectBranch(inst.opcode)) {
|
||||
EXIT_IF(inst.branch_target > UINT32_MAX - function_pc);
|
||||
inst.branch_target += function_pc;
|
||||
}
|
||||
inst.pc += function_pc;
|
||||
linked.decoded.instructions.push_back(std::move(inst));
|
||||
if (returns) break;
|
||||
}
|
||||
const auto function_end = linked.decoded.instructions.back().pc +
|
||||
linked.decoded.instructions.back().word_count * sizeof(uint32_t);
|
||||
for (size_t i = source.decoded.instructions.size(); i + 1u < linked.decoded.instructions.size(); ++i) {
|
||||
const auto& inst = linked.decoded.instructions[i];
|
||||
if (Decoder::IsDirectBranch(inst.opcode))
|
||||
EXIT_IF(inst.branch_target < function_pc || inst.branch_target >= function_end);
|
||||
}
|
||||
reads.resize(query_reads);
|
||||
reads.emplace_back(address, (linked.code.size() - source.code.size()) * sizeof(uint32_t));
|
||||
linked.decoded.instructions[source.call->instruction].branch_target = function_pc;
|
||||
linked.decoded.code = linked.code;
|
||||
linked.observed_function.resize(linked.code.size() - source.code.size());
|
||||
source.linked = std::move(linked);
|
||||
++source.revision;
|
||||
return source.linked->decoded;
|
||||
}
|
||||
|
||||
TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOptions& options) {
|
||||
if (code.empty()) {
|
||||
EXIT("shader recompiler input is empty\n");
|
||||
auto source = PrepareShaderSource(code, options);
|
||||
return TranslateProgram(source.decoded, options);
|
||||
}
|
||||
|
||||
TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileOptions& options) {
|
||||
EXIT_IF(decoded.instructions.empty());
|
||||
bool has_call = false;
|
||||
for (const auto& inst: decoded.instructions) {
|
||||
if (inst.opcode != Decoder::Opcode::S_SWAPPC_B64) continue;
|
||||
has_call = true;
|
||||
if (inst.branch_target == UINT32_MAX)
|
||||
EXIT("shader call at pc 0x%08x must be linked before translation", inst.pc);
|
||||
}
|
||||
if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex &&
|
||||
options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh &&
|
||||
|
|
@ -493,34 +713,11 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
|
|||
}
|
||||
|
||||
const auto compile_begin = std::chrono::steady_clock::now();
|
||||
const auto phase_ms = [&compile_begin]() {
|
||||
const auto phase_ms = [&compile_begin]() {
|
||||
return static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - compile_begin)
|
||||
.count());
|
||||
std::chrono::steady_clock::now() - compile_begin).count());
|
||||
};
|
||||
|
||||
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " code_words=%" PRIu64 " decode\n",
|
||||
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
|
||||
static_cast<uint64_t>(code.size()));
|
||||
|
||||
Decoder::Program decoded;
|
||||
std::vector<uint32_t> joined_code;
|
||||
if (!options.back_code.empty()) {
|
||||
decoded = DecodeFusedProgram(code, options.back_code, joined_code);
|
||||
} else if (options.stage == ShaderType::Local) {
|
||||
decoded = Decoder::DecodeFrontProgram(code);
|
||||
// The separately compiled hull half runs in the next Vulkan stage.
|
||||
auto& handoff = decoded.instructions.back();
|
||||
handoff.opcode = Decoder::Opcode::S_ENDPGM;
|
||||
handoff.src_count = 0;
|
||||
} else {
|
||||
Decoder::DecodeProgram(code, decoded);
|
||||
}
|
||||
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " decode instructions=%" PRIu64
|
||||
" elapsed_ms=%" PRIu64 "\n",
|
||||
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
|
||||
static_cast<uint64_t>(decoded.instructions.size()), phase_ms());
|
||||
|
||||
std::string decoded_dump;
|
||||
if (options.dump_ir) {
|
||||
decoded_dump = Decoder::ProgramToString(decoded);
|
||||
|
|
@ -606,6 +803,12 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
|
|||
IR::RemoveIdentities(ir.blocks);
|
||||
IR::EliminateDeadCode(ir.blocks);
|
||||
}
|
||||
if (has_call) {
|
||||
// Leaf returns are resolved CFG edges; native PC values must not escape into GPU data.
|
||||
for (const auto* block: ir.blocks)
|
||||
for (const auto& inst: *block)
|
||||
EXIT_NOT_IMPLEMENTED(inst.GetOpcode() == IR::ValueOpcode::GetShaderBase);
|
||||
}
|
||||
LowerTessellationMemory(ir, options);
|
||||
std::string cfg_dump;
|
||||
if (options.dump_ir) {
|
||||
|
|
|
|||
|
|
@ -4,8 +4,11 @@
|
|||
#include "common/common.h"
|
||||
#include "common/stringUtils.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
|
||||
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
|
||||
#include "graphics/shader/shader.h"
|
||||
|
||||
#include <array>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
|
|
@ -37,6 +40,32 @@ struct CompileResult {
|
|||
IR::Program program;
|
||||
};
|
||||
|
||||
// Decoded source and typed scalar call queries are retained once per cache source.
|
||||
struct ShaderSource {
|
||||
struct Call {
|
||||
uint32_t instruction;
|
||||
std::array<IR::Value, 2> target;
|
||||
};
|
||||
struct Linked {
|
||||
std::vector<uint32_t> code;
|
||||
Decoder::Program decoded;
|
||||
std::vector<uint32_t> observed_function;
|
||||
};
|
||||
std::vector<uint32_t> code;
|
||||
Decoder::Program decoded;
|
||||
IR::ResourcePlan call_targets;
|
||||
std::optional<Call> call;
|
||||
std::optional<Linked> linked;
|
||||
uint64_t revision = 0;
|
||||
std::vector<std::pair<uint64_t, uint64_t>> reads;
|
||||
};
|
||||
|
||||
[[nodiscard]] ShaderSource PrepareShaderSource(std::span<const uint32_t> code,
|
||||
const CompileOptions& options);
|
||||
const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtRuntime& runtime);
|
||||
[[nodiscard]] TranslateResult TranslateProgram(const Decoder::Program& decoded,
|
||||
const CompileOptions& options);
|
||||
|
||||
[[nodiscard]] TranslateResult TranslateProgram(std::span<const uint32_t> code,
|
||||
const CompileOptions& options);
|
||||
[[nodiscard]] CompileResult CompileProgram(TranslateResult translated,
|
||||
|
|
|
|||
|
|
@ -1468,7 +1468,8 @@ Graph BuildGraph(const Decoder::Program& program) {
|
|||
for (uint32_t i = 0; i < program.instructions.size(); i++) {
|
||||
const auto& inst = program.instructions[i];
|
||||
const auto next_pc = InstructionEndPc(inst);
|
||||
if (Decoder::IsDirectBranch(inst.opcode)) {
|
||||
if (Decoder::IsDirectBranch(inst.opcode) ||
|
||||
(inst.opcode == Opcode::S_SWAPPC_B64 && inst.branch_target != UINT32_MAX)) {
|
||||
if (!IsValidTarget(inst.branch_target, program.instructions, end_pc)) {
|
||||
ExitBuildFailure(graph, FailureKind::InvalidBranchTarget, UINT32_MAX,
|
||||
fmt::format("branch at pc 0x{:08x} targets invalid pc 0x{:08x}",
|
||||
|
|
@ -1501,7 +1502,7 @@ Graph BuildGraph(const Decoder::Program& program) {
|
|||
if (next_pc <= end_pc) {
|
||||
labels.push_back(next_pc);
|
||||
}
|
||||
} else if (inst.opcode == Opcode::S_ENDPGM) {
|
||||
} else if (inst.opcode == Opcode::S_ENDPGM || inst.opcode == Opcode::S_SWAPPC_B64) {
|
||||
labels.push_back(next_pc);
|
||||
}
|
||||
}
|
||||
|
|
@ -1550,6 +1551,9 @@ Graph BuildGraph(const Decoder::Program& program) {
|
|||
const auto next_pc = InstructionEndPc(last);
|
||||
if (last.opcode == Opcode::S_ENDPGM) {
|
||||
block.terminator.kind = TerminatorKind::Return;
|
||||
} else if (last.opcode == Opcode::S_SWAPPC_B64 && last.branch_target != UINT32_MAX) {
|
||||
block.terminator.kind = TerminatorKind::Branch;
|
||||
block.terminator.true_block = block_at_pc(last.branch_target);
|
||||
} else if (last.opcode == Opcode::S_SETPC_B64) {
|
||||
const auto& target_info = setpc_targets.at(last.pc);
|
||||
if (target_info.indirect) {
|
||||
|
|
@ -1676,6 +1680,36 @@ Graph Structurize(const Graph& graph) {
|
|||
return GotoStructurizer(graph).Run();
|
||||
}
|
||||
|
||||
bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code) {
|
||||
uint32_t destination = UINT32_MAX;
|
||||
// Scalar ALU pair widths are not all decoded. The typed producer must confirm this word.
|
||||
return InstructionWritesScalarCode(inst, code) ||
|
||||
(ScalarOperandCode(inst.dst, destination) && code == destination + 1u);
|
||||
}
|
||||
|
||||
uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
|
||||
uint32_t before, uint32_t code) {
|
||||
EXIT_IF(before >= program.instructions.size());
|
||||
const auto found = std::ranges::find_if(graph.blocks, [before](const auto& block) {
|
||||
return block.inst_begin <= before && before < block.inst_end;
|
||||
});
|
||||
EXIT_IF(found == graph.blocks.end());
|
||||
const auto* block = &*found;
|
||||
for (size_t depth = 0; depth < graph.blocks.size(); ++depth) {
|
||||
for (uint32_t i = before; i > block->inst_begin;) {
|
||||
const auto& inst = program.instructions[--i];
|
||||
EXIT_IF(inst.opcode == Opcode::S_SWAPPC_B64);
|
||||
if (MayWriteScalarRegister(inst, code)) return i;
|
||||
}
|
||||
if (block->id == graph.entry_block) return UINT32_MAX;
|
||||
EXIT_IF(block->predecessors.size() != 1u);
|
||||
block = graph.FindBlock(block->predecessors[0]);
|
||||
EXIT_IF(block == nullptr);
|
||||
before = block->inst_end;
|
||||
}
|
||||
EXIT("scalar shader call source has cyclic reaching definitions");
|
||||
}
|
||||
|
||||
std::string BranchConditionToString(BranchCondition condition) {
|
||||
switch (condition) {
|
||||
case BranchCondition::Always: return "always";
|
||||
|
|
|
|||
|
|
@ -113,6 +113,9 @@ struct Graph {
|
|||
};
|
||||
|
||||
Graph BuildGraph(const Decoder::Program& program);
|
||||
bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code);
|
||||
uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
|
||||
uint32_t before, uint32_t code);
|
||||
// Returns structured control flow or failure diagnostics without changing the native graph.
|
||||
// On failure, failure_block is an original block ID or UINT32_MAX.
|
||||
Graph Structurize(const Graph& graph);
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
|
|||
{0x1eu, Opcode::S_BITSET1_B64},
|
||||
{0x1fu, Opcode::S_GETPC_B64},
|
||||
{0x20u, Opcode::S_SETPC_B64},
|
||||
{0x21u, Opcode::S_SWAPPC_B64},
|
||||
{0x24u, Opcode::S_AND_SAVEEXEC_B64},
|
||||
{0x28u, Opcode::S_ORN2_SAVEEXEC_B64},
|
||||
{0x2du, Opcode::S_QUADMASK_B64},
|
||||
|
|
@ -155,6 +156,10 @@ void DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
|
|||
}
|
||||
|
||||
switch (inst.opcode) {
|
||||
case Opcode::S_SWAPPC_B64:
|
||||
inst.branch_target = UINT32_MAX;
|
||||
inst.data_dwords = 2;
|
||||
break;
|
||||
case Opcode::S_GETPC_B64:
|
||||
inst.src_count = 0;
|
||||
DecodeScalarDestination(sdst, pc, inst.dst);
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ enum class Opcode {
|
|||
S_QUADMASK_B64,
|
||||
S_GETPC_B64,
|
||||
S_SETPC_B64,
|
||||
S_SWAPPC_B64,
|
||||
S_SUBVECTOR_LOOP_BEGIN,
|
||||
S_SUBVECTOR_LOOP_END,
|
||||
S_AND_SAVEEXEC_B32,
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
|
|||
case O::S_MOV_B64: S_MOV_B64(inst); return;
|
||||
case O::S_WQM_B32: S_WQM(inst, false); return;
|
||||
case O::S_WQM_B64: S_WQM(inst, true); return;
|
||||
case O::S_GETPC_B64: S_GETPC_B64(inst); return;
|
||||
case O::S_GETPC_B64:
|
||||
case O::S_SWAPPC_B64: S_GETPC_B64(inst); return;
|
||||
case O::S_SETPC_B64: return;
|
||||
case O::S_SUBVECTOR_LOOP_BEGIN: S_SUBVECTOR_LOOP(inst, true); return;
|
||||
case O::S_SUBVECTOR_LOOP_END: S_SUBVECTOR_LOOP(inst, false); return;
|
||||
|
|
|
|||
|
|
@ -590,6 +590,7 @@ struct ResourcePlan {
|
|||
std::vector<ResourceBlock> control_flow;
|
||||
std::vector<SrtRead> srt_reads;
|
||||
bool requires_specialization_memory = false;
|
||||
std::vector<std::pair<uint64_t, uint64_t>> source_reads;
|
||||
bool capture_specialization_reads = false;
|
||||
bool srt_plan_complete = false;
|
||||
bool resource_tracking_complete = false;
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@
|
|||
#include <array>
|
||||
#include <bit>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fmt/format.h>
|
||||
#include <functional>
|
||||
#include <numeric>
|
||||
|
|
@ -141,31 +140,6 @@ bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResou
|
|||
return true;
|
||||
}
|
||||
|
||||
struct ReadCapture {
|
||||
SrtRuntime source;
|
||||
std::vector<std::pair<uint64_t, uint64_t>>& ranges;
|
||||
};
|
||||
|
||||
bool CaptureStrictRead(void* userdata, uint64_t address, std::span<uint32_t> values) {
|
||||
auto& capture = *static_cast<ReadCapture*>(userdata);
|
||||
if (!capture.source.read_specialization_memory(capture.source.userdata, address, values)) {
|
||||
return false;
|
||||
}
|
||||
capture.ranges.emplace_back(address, values.size_bytes());
|
||||
return true;
|
||||
}
|
||||
|
||||
bool CaptureOrdinaryRead(void* userdata, uint64_t address, std::span<uint32_t> values) {
|
||||
auto& capture = *static_cast<ReadCapture*>(userdata);
|
||||
if (capture.source.read_memory != nullptr) {
|
||||
if (!capture.source.read_memory(capture.source.userdata, address, values)) return false;
|
||||
} else {
|
||||
std::memcpy(values.data(), reinterpret_cast<const void*>(address), values.size_bytes());
|
||||
}
|
||||
capture.ranges.emplace_back(address, values.size_bytes());
|
||||
return true;
|
||||
}
|
||||
|
||||
const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) {
|
||||
if (source >= program.descriptor_sources.size()) {
|
||||
return nullptr;
|
||||
|
|
@ -953,6 +927,7 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
|
|||
plan.user_data_count = program.user_data_count;
|
||||
plan.info = program.info;
|
||||
plan.memory_info = program.memory_info;
|
||||
plan.source_reads = program.source_reads;
|
||||
plan.srt_plan_complete = program.srt_plan_complete;
|
||||
plan.resource_tracking_complete = program.resource_tracking_complete;
|
||||
|
||||
|
|
@ -1071,7 +1046,8 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
|
|||
for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) {
|
||||
plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
|
||||
}
|
||||
plan.capture_specialization_reads |= capture_indirect_reads || capture_control_reads;
|
||||
plan.capture_specialization_reads |=
|
||||
capture_indirect_reads || capture_control_reads || !plan.source_reads.empty();
|
||||
if (plan.capture_specialization_reads) {
|
||||
// Every former flat alias now targets its retained raw read. Mark those reads
|
||||
// on the normalized graph, so discarded Phi/EXEC dependencies stay discarded.
|
||||
|
|
@ -1106,27 +1082,23 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
|
|||
if (image.written) MarkCleanReads(planned_reads, Source(plan, image.source));
|
||||
}
|
||||
}
|
||||
if (capture_indirect_reads) plan.resource_tracking_complete &= !program.has_address_writes;
|
||||
if (capture_indirect_reads || !plan.source_reads.empty())
|
||||
plan.resource_tracking_complete &= !program.has_address_writes;
|
||||
return plan;
|
||||
}
|
||||
|
||||
bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime,
|
||||
ResourceSnapshot& snapshot, ResourceSpecialization& specialization) {
|
||||
if (!program.resource_tracking_complete ||
|
||||
(program.requires_specialization_memory && runtime.read_specialization_memory == nullptr)) {
|
||||
((program.requires_specialization_memory || !program.source_reads.empty()) &&
|
||||
runtime.read_specialization_memory == nullptr)) {
|
||||
return false;
|
||||
}
|
||||
const bool capture_reads = program.capture_specialization_reads;
|
||||
const bool capture_reads = program.capture_specialization_reads || !program.source_reads.empty();
|
||||
auto& reads = snapshot.specialization_reads;
|
||||
reads.clear();
|
||||
ReadCapture capture {runtime, reads};
|
||||
SrtRuntime observed = runtime;
|
||||
if (capture_reads) {
|
||||
observed.userdata = &capture;
|
||||
observed.read_specialization_memory = runtime.read_specialization_memory != nullptr
|
||||
? CaptureStrictRead : nullptr;
|
||||
observed.read_memory = CaptureOrdinaryRead;
|
||||
}
|
||||
reads = program.source_reads;
|
||||
SrtReadCapture capture {runtime, reads};
|
||||
const auto observed = capture_reads ? capture.ObservedRuntime() : runtime;
|
||||
SrtWalker clean(program, CleanRuntime(observed));
|
||||
SrtWalker walker(program, observed,
|
||||
capture_reads || program.requires_specialization_memory ? &clean : nullptr);
|
||||
|
|
|
|||
|
|
@ -11,6 +11,35 @@
|
|||
|
||||
namespace Libs::Graphics::ShaderRecompiler::IR {
|
||||
|
||||
SrtRuntime SrtReadCapture::ObservedRuntime() {
|
||||
auto runtime = m_source;
|
||||
runtime.userdata = this;
|
||||
runtime.read_specialization_memory = m_source.read_specialization_memory != nullptr
|
||||
? ReadStrict : nullptr;
|
||||
runtime.read_memory = ReadOrdinary;
|
||||
return runtime;
|
||||
}
|
||||
|
||||
bool SrtReadCapture::ReadStrict(void* userdata, uint64_t address, std::span<uint32_t> values) {
|
||||
auto& capture = *static_cast<SrtReadCapture*>(userdata);
|
||||
if (!capture.m_source.read_specialization_memory(capture.m_source.userdata, address, values)) {
|
||||
return false;
|
||||
}
|
||||
capture.m_ranges.emplace_back(address, values.size_bytes());
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SrtReadCapture::ReadOrdinary(void* userdata, uint64_t address, std::span<uint32_t> values) {
|
||||
auto& capture = *static_cast<SrtReadCapture*>(userdata);
|
||||
if (capture.m_source.read_memory != nullptr) {
|
||||
if (!capture.m_source.read_memory(capture.m_source.userdata, address, values)) return false;
|
||||
} else {
|
||||
std::memcpy(values.data(), reinterpret_cast<const void*>(address), values.size_bytes());
|
||||
}
|
||||
capture.m_ranges.emplace_back(address, values.size_bytes());
|
||||
return true;
|
||||
}
|
||||
|
||||
SrtRuntime CleanRuntime(SrtRuntime runtime) {
|
||||
runtime.read_memory = runtime.read_specialization_memory != nullptr
|
||||
? runtime.read_specialization_memory
|
||||
|
|
|
|||
|
|
@ -20,6 +20,20 @@ struct SrtRuntime {
|
|||
std::span<const uint32_t> workgroup_counts;
|
||||
};
|
||||
|
||||
class SrtReadCapture {
|
||||
public:
|
||||
SrtReadCapture(SrtRuntime source, std::vector<std::pair<uint64_t, uint64_t>>& ranges):
|
||||
m_source(source), m_ranges(ranges) {}
|
||||
SrtRuntime ObservedRuntime();
|
||||
|
||||
private:
|
||||
static bool ReadStrict(void* userdata, uint64_t address, std::span<uint32_t> values);
|
||||
static bool ReadOrdinary(void* userdata, uint64_t address, std::span<uint32_t> values);
|
||||
|
||||
SrtRuntime m_source;
|
||||
std::vector<std::pair<uint64_t, uint64_t>>& m_ranges;
|
||||
};
|
||||
|
||||
// Retained reads no longer need the guest instruction PC. A clean read evaluates
|
||||
// its address and bounds through the strict reader as well as the final DWORD.
|
||||
struct SrtReadFlags {
|
||||
|
|
|
|||
|
|
@ -9950,6 +9950,105 @@ public:
|
|||
return result;
|
||||
}
|
||||
|
||||
void CheckScalarLeafCall() {
|
||||
constexpr const char* name = "ScalarLeafCall";
|
||||
constexpr uintptr_t base = 0x0000000204800000ull;
|
||||
constexpr uint64_t size = 0x10000;
|
||||
constexpr auto table = base + 0x100;
|
||||
constexpr auto first = base + 0x1000;
|
||||
constexpr auto second = base + 0x2000;
|
||||
constexpr auto output = base + 0x4000;
|
||||
std::vector<u32> code{
|
||||
EncodeSMovB32(20, InlineU32(0)),
|
||||
0xbeea03ffu, static_cast<u32>(base), 0xbeeb03ffu, static_cast<u32>(base >> 32u),
|
||||
0xf4080135u, 0xfa000000u, 0xbf8cc07fu, 0xf4240382u, 0xfa000020u,
|
||||
0xbf8cc07fu, 0xbf130e80u, 0xbf840004u, 0xf4240402u, 0xfa000028u,
|
||||
0xbf8cc07fu, 0xbe8e210eu, EncodeVop1(0x01, 1, 20),
|
||||
EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(1, 0, 0)};
|
||||
AppendEnd(&code);
|
||||
ShaderMapUserData(reinterpret_cast<uint64_t>(code.data()),
|
||||
{.type = Prospero::ShaderBinaryType::kCs,
|
||||
.code_size_bytes = static_cast<u32>(code.size() * sizeof(u32))});
|
||||
EnsureRuntimeContext();
|
||||
int64_t physical = -1;
|
||||
Require(name, "allocate", LibKernel::Memory::KernelAllocateDirectMemory(
|
||||
0, LibKernel::Memory::KernelGetDirectMemorySize(), size, size, 0, &physical) == 0,
|
||||
"call fixture allocation failed");
|
||||
void* mapped = reinterpret_cast<void*>(base);
|
||||
Require(name, "map", LibKernel::Memory::KernelMapDirectMemory(
|
||||
&mapped, size, 0x3, 0x10, physical, size) == 0 && mapped == reinterpret_cast<void*>(base),
|
||||
"call fixture mapping failed");
|
||||
std::memset(mapped, 0, size);
|
||||
const ShaderBufferResource table_desc{{static_cast<u32>(table),
|
||||
static_cast<u32>(table >> 32u) | (16u << 16u), 8, 0x5204u}};
|
||||
LibKernel::Memory::WriteBacking(base, table_desc.fields, sizeof(table_desc.fields));
|
||||
const std::array leaf{EncodeSMovB32(20, 255), 41u, EncodeSop1(0x21, 125, 14)};
|
||||
for (const auto address : {first, second})
|
||||
LibKernel::Memory::WriteBacking(address, leaf.data(), sizeof(leaf));
|
||||
|
||||
RenderContext context(m_runtime_context);
|
||||
context.InitializeGpu(nullptr);
|
||||
LibKernel::Memory::InstallGpuResources(&context);
|
||||
context.GetGpu().SendCommandSync([&] {
|
||||
CommandProcessor processor(context, 0);
|
||||
processor.BufferInit();
|
||||
context.MapMemory(base, size);
|
||||
auto& cache = context.GetBufferCache();
|
||||
auto& shaders = processor.GetShCtx();
|
||||
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(code.data()),
|
||||
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
|
||||
.wave_size = 64, .user_sgpr = 4});
|
||||
ShaderBufferResource out{};
|
||||
out.UpdateAddress48(output);
|
||||
out.fields[2] = 4;
|
||||
out.fields[3] = DstSel(4, 5, 6, 7) |
|
||||
(static_cast<u32>(Prospero::BufferFormat::k32UInt) << 12u);
|
||||
for (u32 i = 0; i < 4; ++i)
|
||||
shaders.SetCsUserSgpr(i, out.fields[i], HW::UserSgprType::Unknown);
|
||||
uint64_t original_id = 0;
|
||||
struct Case { uint64_t address; u32 value; bool update; };
|
||||
for (const auto test : {Case{first, 41, false}, Case{first, 41, false},
|
||||
Case{second, 41, false}, Case{second, 73, true},
|
||||
Case{first, 41, false}}) {
|
||||
Require(name, "refresh function pointer", context.InvalidateMemory(table + 0x20, 8),
|
||||
"function table escaped mapped memory");
|
||||
const uint64_t target = test.address | 3u;
|
||||
LibKernel::Memory::WriteBacking(table + 0x20, &target, sizeof(target));
|
||||
if (test.update) {
|
||||
auto [buffer, offset] = cache.ObtainBuffer(test.address + 4, 4, true);
|
||||
buffer->Fill(offset, 4, test.value);
|
||||
Require(name, "GPU-written function code", cache.HasGpuDirtyBytes(test.address + 4, 4),
|
||||
"code mutation did not remain GPU-owned before refresh");
|
||||
}
|
||||
ShaderComputeInputInfo input{};
|
||||
const auto program = context.GetPipelineCache().GetComputeProgram(
|
||||
shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
|
||||
if (original_id == 0) original_id = program.id;
|
||||
Require(name, "exact function identity",
|
||||
(program.id == original_id) == (test.value == 41),
|
||||
"helper bytes were stale or identical relocated/restored code recompiled");
|
||||
Require(name, "current code dependency",
|
||||
std::ranges::find(input.stage.resources->specialization_reads,
|
||||
std::pair<uint64_t, uint64_t>{test.address, sizeof(leaf)}) !=
|
||||
input.stage.resources->specialization_reads.end(),
|
||||
"function relocation retained an old code dependency");
|
||||
processor.DispatchDirect(1, 1, 1, 0x41u);
|
||||
cache.ReadMemory(output, sizeof(u32));
|
||||
u32 actual = 0;
|
||||
Require(name, "scalar leaf execution and return",
|
||||
LibKernel::Memory::TryReadBacking(output, &actual, sizeof(actual)) && actual == test.value,
|
||||
"caller did not observe the leaf's scalar result after return");
|
||||
}
|
||||
context.UnmapMemory(base, size);
|
||||
context.GetCommandScheduler().Finish();
|
||||
});
|
||||
LibKernel::Memory::InstallGpuResources(nullptr);
|
||||
context.ShutdownGpu();
|
||||
Require(name, "release", LibKernel::Memory::KernelCheckedReleaseDirectMemory(physical, size) == 0,
|
||||
"call fixture release failed");
|
||||
std::printf("[gpu] %-32s ok\n", name);
|
||||
}
|
||||
|
||||
void CheckNativeIndirectDispatch() {
|
||||
constexpr const char *name = "NativeIndirectDispatch";
|
||||
constexpr uintptr_t base = 0x0000000204600000ull;
|
||||
|
|
@ -42796,6 +42895,7 @@ int main(int argc, char **argv) {
|
|||
if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) {
|
||||
VulkanHarness vulkan;
|
||||
vulkan.CheckGpuCommandLane();
|
||||
vulkan.CheckScalarLeafCall();
|
||||
CheckAgcSystemTable(vulkan.RuntimeRenderer());
|
||||
CheckPm4IndirectControlFlow(vulkan.RuntimeRenderer());
|
||||
CheckPm4WaitResume(vulkan.RuntimeRenderer());
|
||||
|
|
@ -43290,6 +43390,7 @@ int main(int argc, char **argv) {
|
|||
}
|
||||
vulkan.CheckGpuSuspendPoint();
|
||||
vulkan.CheckGpuCommandLane();
|
||||
vulkan.CheckScalarLeafCall();
|
||||
if (skipped_device_checks) {
|
||||
std::printf(
|
||||
"ShaderRecompilerComputeTests: device rasterization checks skipped, this "
|
||||
|
|
|
|||
|
|
@ -10308,6 +10308,147 @@ void TestNewShaderRecompilerBranchConditionForms() {
|
|||
}
|
||||
}
|
||||
|
||||
void TestMemoryFedScalarLeafCall() {
|
||||
using namespace ShaderRecompiler;
|
||||
constexpr uint64_t base = 0xfe0040000ull;
|
||||
const uint32_t shader[] = {
|
||||
0xbeea03ffu, 0xe0040000u, 0xbeeb03ffu, 0x0000000fu,
|
||||
0xf4080135u, 0xfa000000u, 0xbf8cc07fu, 0xf4240382u, 0xfa000020u,
|
||||
0xbf8cc07fu, 0xbf130e80u, 0xbf840004u, 0xf4240402u, 0xfa000028u,
|
||||
0xbf8cc07fu, 0xbe8e210eu, 0xbf810000u};
|
||||
struct Memory {
|
||||
std::array<uint32_t, 128> words{};
|
||||
uint64_t end = base + 0x10c;
|
||||
bool clean = true;
|
||||
} memory;
|
||||
std::copy_n(std::array{0xe0040060u, 0x0010000fu, 8u, 0x5204u}.begin(),
|
||||
4, memory.words.begin());
|
||||
memory.words[32] = static_cast<uint32_t>(base + 0x103); // SWAPPC aligns the old pair.
|
||||
memory.words[33] = static_cast<uint32_t>(base >> 32u);
|
||||
memory.words[64] = EncodeSMovB32(3, 255);
|
||||
memory.words[65] = 0x12345678;
|
||||
memory.words[66] = EncodeSop1(0x21, 125, 14);
|
||||
const auto read = [](void *data, uint64_t address, std::span<uint32_t> out) {
|
||||
const auto& m = *static_cast<Memory*>(data);
|
||||
if (!m.clean || address < base || address > m.end || out.size_bytes() > m.end - address)
|
||||
return false;
|
||||
std::memcpy(out.data(), reinterpret_cast<const uint8_t*>(m.words.data()) + address - base,
|
||||
out.size_bytes());
|
||||
return true;
|
||||
};
|
||||
IR::SrtRuntime runtime{.read_memory = read, .userdata = &memory, .read_specialization_memory = read};
|
||||
const auto options = MakeCompileOptions(ShaderType::Compute);
|
||||
auto source = PrepareShaderSource(shader, options);
|
||||
Check(source.call && source.decoded.instructions[source.call->instruction].branch_target == UINT32_MAX,
|
||||
"memory-fed SWAPPC did not retain an unresolved target");
|
||||
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
|
||||
ExpectFatal([&] { (void)TranslateProgram(source.decoded, options); },
|
||||
"unresolved SWAPPC silently fell through");
|
||||
#endif
|
||||
const auto& linked = RefreshShaderSource(source, runtime);
|
||||
Check(source.revision == 1 && linked.instructions.back().branch_target == 0x40 &&
|
||||
linked.instructions[source.call->instruction].branch_target == 0x44 &&
|
||||
source.reads.back() == std::pair<uint64_t, uint64_t>{base + 0x100, 12},
|
||||
"aliased call target, exact leaf range, or continuation was lost");
|
||||
const auto* storage = linked.instructions.data();
|
||||
const auto* scratch = source.linked->observed_function.data();
|
||||
(void)RefreshShaderSource(source, runtime);
|
||||
Check(source.revision == 1 && source.linked->decoded.instructions.data() == storage &&
|
||||
source.linked->observed_function.data() == scratch,
|
||||
"unchanged leaf rebuilt decoded instructions or refresh storage");
|
||||
auto translated = TranslateProgram(linked, options);
|
||||
translated.program.source_reads = source.reads;
|
||||
const auto plan = IR::ExtractResourcePlan(translated.program);
|
||||
Check(plan.source_reads == source.reads && plan.capture_specialization_reads,
|
||||
"linked code dependencies were omitted from resource alias validation");
|
||||
|
||||
std::vector<uint32_t> writer(std::begin(shader), std::end(shader) - 1);
|
||||
writer.insert(writer.end(), {EncodeVop1(0x01, 1, 3), EncodeMubuf0(0x1c, 0, false),
|
||||
EncodeMubuf1(1, 6, 0), EncodeSopp(0x01)});
|
||||
std::array<uint32_t, 64> user_data{};
|
||||
user_data[25] = static_cast<uint32_t>(base >> 32u);
|
||||
user_data[26] = 4;
|
||||
user_data[27] = 3u << 28u;
|
||||
auto write_options = options;
|
||||
write_options.user_data = user_data;
|
||||
auto write_runtime = runtime;
|
||||
write_runtime.user_data = user_data;
|
||||
auto write_source = PrepareShaderSource(writer, write_options);
|
||||
auto write_program = TranslateProgram(RefreshShaderSource(write_source, write_runtime), write_options);
|
||||
write_program.program.source_reads = write_source.reads;
|
||||
auto write_plan = IR::ExtractResourcePlan(write_program.program);
|
||||
IR::ResourceSnapshot snapshot;
|
||||
IR::ResourceSpecialization specialization;
|
||||
const auto written = std::ranges::find_if(write_plan.info.buffers,
|
||||
[](const auto& buffer) { return buffer.written; });
|
||||
Check(written != write_plan.info.buffers.end(), "caller store lost its writable binding");
|
||||
const auto output_index = static_cast<size_t>(written - write_plan.info.buffers.begin());
|
||||
for (const auto offset : {0x80u, 0x100u, 0x1f0u}) {
|
||||
user_data[24] = static_cast<uint32_t>(base + offset);
|
||||
Check(IR::MaterializeResources(write_plan, write_runtime, snapshot, specialization) &&
|
||||
snapshot.buffers[output_index].dwords[0] == user_data[24] &&
|
||||
snapshot.buffers[output_index].dwords[2] == 4 &&
|
||||
std::ranges::find(snapshot.specialization_reads,
|
||||
std::pair<uint64_t, uint64_t>{base + 0x80, 4}) != snapshot.specialization_reads.end() &&
|
||||
std::ranges::find(snapshot.specialization_reads,
|
||||
std::pair<uint64_t, uint64_t>{base + 0x100, 12}) != snapshot.specialization_reads.end(),
|
||||
"call dependencies or writable ranges were lost before binding validation");
|
||||
}
|
||||
|
||||
// Relocating identical code updates dependencies without rebuilding the source.
|
||||
std::copy_n(memory.words.begin() + 64, 3, memory.words.begin() + 96);
|
||||
memory.words[32] = static_cast<uint32_t>(base + 0x180);
|
||||
memory.end = base + 0x18c;
|
||||
(void)RefreshShaderSource(source, runtime);
|
||||
Check(source.revision == 1 && source.reads.back().first == base + 0x180,
|
||||
"identical relocated leaf recompiled or retained its old memory dependency");
|
||||
memory.words[97] = 0x87654321;
|
||||
(void)RefreshShaderSource(source, runtime);
|
||||
Check(source.revision == 2 && source.linked->code.back() == EncodeSop1(0x21, 125, 14),
|
||||
"changed leaf bytes retained stale source");
|
||||
// A newly selected shorter function may end before the old bulk-read length.
|
||||
memory.words[96] = EncodeSMovB32(3, 129);
|
||||
memory.words[97] = EncodeSop1(0x21, 125, 14);
|
||||
memory.end = base + 0x188;
|
||||
(void)RefreshShaderSource(source, runtime);
|
||||
Check(source.revision == 3 && source.reads.back().second == 8,
|
||||
"shorter exact-boundary leaf was not decoded after a failed old-span read");
|
||||
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
|
||||
memory.clean = false;
|
||||
ExpectFatal([&] { (void)RefreshShaderSource(source, runtime); },
|
||||
"unreadable call dependencies reused stale linked code");
|
||||
memory.clean = true;
|
||||
auto ordinary = runtime;
|
||||
ordinary.read_specialization_memory = nullptr;
|
||||
ExpectFatal([&] { (void)RefreshShaderSource(source, ordinary); },
|
||||
"call target query accepted an ordinary non-strict memory reader");
|
||||
std::vector<uint32_t> escaping(std::begin(shader), std::end(shader) - 1);
|
||||
escaping.insert(escaping.end(), {EncodeVop1(0x01, 1, 14),
|
||||
EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(1, 0, 0), EncodeSopp(0x01)});
|
||||
ExpectFatal([&] {
|
||||
auto escaping_source = PrepareShaderSource(escaping, options);
|
||||
(void)TranslateProgram(RefreshShaderSource(escaping_source, runtime), options);
|
||||
}, "saved native return PC escaped into GPU data");
|
||||
memory.words[96] = EncodeSMovB32(14, 129);
|
||||
ExpectFatal([&] { (void)RefreshShaderSource(source, runtime); },
|
||||
"leaf clobbering its return pair was linked");
|
||||
// AND_B64 writes the high half despite the decoder's generic scalar width.
|
||||
const uint32_t high_writer[] = {
|
||||
EncodeSMovB32(15, 129), EncodeSop2(0x0f, 14, 128, 128),
|
||||
EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)};
|
||||
auto pair_source = PrepareShaderSource(high_writer, options);
|
||||
IR::SrtWalker pair_walker(pair_source.call_targets, runtime);
|
||||
uint32_t high = 1;
|
||||
Check(pair_walker.Evaluate(pair_source.call->target[1], high) && high == 0,
|
||||
"scalar pair producer used a stale preceding high-half definition");
|
||||
const uint32_t joined[] = {
|
||||
EncodeSopp(0x04, 2), EncodeSMovB32(14, 129), EncodeSopp(0x02, 1),
|
||||
EncodeSMovB32(14, 130), EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)};
|
||||
ExpectFatal([&] { (void)PrepareShaderSource(joined, options); },
|
||||
"ambiguous reaching definitions were accepted for a call target");
|
||||
#endif
|
||||
}
|
||||
|
||||
void TestNewShaderRecompilerSetpcBranch() {
|
||||
const uint32_t shader[] = {
|
||||
EncodeSop1(0x1f, 4, 0), // s_getpc_b64 s[4:5]
|
||||
|
|
@ -15445,6 +15586,7 @@ int main() {
|
|||
TestNewShaderRecompilerPixelImageSampleLodSelection();
|
||||
TestNewShaderRecompilerBranchConditionForms();
|
||||
TestNewShaderRecompilerSetpcBranch();
|
||||
TestMemoryFedScalarLeafCall();
|
||||
TestFusedShaderHandoffPreservesRegisters();
|
||||
TestMeshExportStorage();
|
||||
TestMergedShaderUserDataSnapshot();
|
||||
|
|
|
|||
Loading…
Reference in New Issue