diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index 8a680c63..b1989e08 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -217,17 +217,20 @@ struct PipelineCache::ProgramCache { }; struct Permutation { + std::vector function_code; ShaderRecompiler::IR::ResourceSpecialization specialization; ShaderRecompiler::IR::CompiledShaderInfo program; ShaderProgram handle; }; struct SourceEntry { - explicit SourceEntry(ShaderRecompiler::IR::ResourcePlan plan) - : resource_plan(std::move(plan)) { + SourceEntry(ShaderRecompiler::IR::ResourcePlan plan, + std::unique_ptr call_source) + : call_source(std::move(call_source)), resource_plan(std::move(plan)) { permutations.reserve(8); } + std::unique_ptr call_source; ShaderRecompiler::IR::ResourcePlan resource_plan; ShaderRecompiler::IR::ResourceSnapshot resources; ShaderRecompiler::IR::ResourceSpecialization specialization; @@ -256,7 +259,8 @@ struct PipelineCache::ProgramCache { const ShaderRecompiler::CompileOptions& options, ShaderRecompiler::TranslateResult translated, ShaderRecompiler::IR::ResourceSpecialization specialization, - uint32_t push_data_start_dword) { + uint32_t push_data_start_dword, + std::span function_code) { auto result = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization, push_data_start_dword); if (!ValidateShaderSpirv(options.dump_label, options.shader_hash, result.spirv)) { @@ -273,6 +277,7 @@ struct PipelineCache::ProgramCache { static_cast(result.spirv.size()), options.wave_size); } return { + .function_code = {function_code.begin(), function_code.end()}, .specialization = std::move(specialization), .program = std::move(result.program).TakeCompiledInfo(), .handle = {.id = ++next_shader_id, .module = module}, @@ -307,21 +312,33 @@ struct PipelineCache::ProgramCache { if constexpr (std::is_same_v) { runtime.workgroup_counts = input_info.workgroup_counts; } - if (entry != programs.end()) { + bool source_changed = false; + const auto function_code = [](const std::unique_ptr& source) { + return source ? std::span(source->linked->code).subspan(source->code.size()) + : std::span{}; + }; + if (entry != programs.end() && entry->second.call_source) { + auto& cached = entry->second; + const auto revision = cached.call_source->revision; + ShaderRecompiler::RefreshShaderSource(*cached.call_source, runtime); + source_changed = revision != cached.call_source->revision; + cached.resource_plan.source_reads = cached.call_source->reads; + } + const auto matches = [&](const Permutation& candidate) { + const auto& layout = candidate.program.bindings; + return layout.push_data_start_dword == ShaderRecompiler::IR::PushData::StartFor( + push_data_cursor, layout.ShaderDataDwords()) && + candidate.specialization == entry->second.specialization && + (!entry->second.call_source || std::ranges::equal( + candidate.function_code, function_code(entry->second.call_source))); + }; + if (entry != programs.end() && !source_changed) { EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( entry->second.resource_plan, runtime, entry->second.resources, entry->second.specialization)); - if (const auto permutation = std::ranges::find_if( - entry->second.permutations, [&](const Permutation& candidate) { - const auto& layout = candidate.program.bindings; - return layout.push_data_start_dword == - ShaderRecompiler::IR::PushData::StartFor( - push_data_cursor, layout.ShaderDataDwords()) && - candidate.specialization == entry->second.specialization; - }); + if (const auto permutation = std::ranges::find_if(entry->second.permutations, matches); permutation != entry->second.permutations.end()) { - input_info.stage = {.program = &permutation->program, - .resources = &entry->second.resources}; + input_info.stage = {.program = &permutation->program, .resources = &entry->second.resources}; permutation->program.bindings.AdvancePushData(push_data_cursor); return permutation->handle; } @@ -368,16 +385,41 @@ struct PipelineCache::ProgramCache { options.wave_size = input_info.wave_size; } DumpShaderOriginal(stage_name, options.shader_hash, params.code); - auto translated = ShaderRecompiler::TranslateProgram(params.code, options); + ShaderRecompiler::ShaderSource source; + if (entry == programs.end()) source = ShaderRecompiler::PrepareShaderSource(params.code, options); + auto* current = entry == programs.end() ? &source : entry->second.call_source.get(); + ShaderRecompiler::TranslateResult translated; + if (current != nullptr) { + const auto& decoded = entry == programs.end() + ? ShaderRecompiler::RefreshShaderSource(*current, runtime) + : current->linked->decoded; + translated = ShaderRecompiler::TranslateProgram(decoded, options); + translated.program.source_reads = current->reads; + } else { + translated = ShaderRecompiler::TranslateProgram(params.code, options); + } if (entry == programs.end()) { + auto retained = source.call ? std::make_unique(std::move(source)) : nullptr; entry = programs.try_emplace(lookup_key, - ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first; + ShaderRecompiler::IR::ExtractResourcePlan(translated.program), std::move(retained)).first; EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( entry->second.resource_plan, runtime, entry->second.resources, entry->second.specialization)); + } else if (source_changed) { + entry->second.resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( + entry->second.resource_plan, runtime, entry->second.resources, + entry->second.specialization)); + if (const auto permutation = std::ranges::find_if(entry->second.permutations, matches); + permutation != entry->second.permutations.end()) { + input_info.stage = {.program = &permutation->program, .resources = &entry->second.resources}; + permutation->program.bindings.AdvancePushData(push_data_cursor); + return permutation->handle; + } } entry->second.permutations.push_back(CompilePermutation( - stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor)); + stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor, + function_code(entry->second.call_source))); const auto& permutation = entry->second.permutations.back(); input_info.stage = {.program = &permutation.program, .resources = &entry->second.resources}; permutation.program.bindings.AdvancePushData(push_data_cursor); diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index b600adbd..080d0624 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -7,6 +7,7 @@ #include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h" #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include "graphics/shader/recompiler/frontend/translate/Translate.h" +#include "graphics/shader/recompiler/frontend/translate/Translator.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" @@ -20,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -478,11 +480,229 @@ Decoder::Program DecodeFusedProgram(std::span front, std::spaninstruction]; + EXIT_NOT_IMPLEMENTED(call.src0.kind != Decoder::OperandKind::Sgpr || + call.dst.kind != Decoder::OperandKind::Sgpr || + call.src0.reg != call.dst.reg || call.dst.reg >= 105u); + const auto graph = CFG::BuildGraph(source.decoded); + Program query; + query.stage = options.stage; + query.user_data_base = options.user_data_base; + query.user_data_count = static_cast(options.user_data.size()); + query.wave_size = options.wave_size; + auto& block = *query.block_storage.emplace_back(std::make_unique()); + query.blocks.push_back(&block); + struct Producer { bool ready = false; std::map values; }; + std::map producers; + std::function Register; + std::function Resolve = [&](uint32_t before, Value value) -> Value { + value = value.Resolve(); + auto* inst = value.TryInstruction(); + if (inst == nullptr) return value; + uint32_t code = UINT32_MAX; + switch (inst->GetOpcode()) { + case ValueOpcode::GetScalarRegister: code = static_cast(inst->Arg(0).ScalarRegister()); break; + case ValueOpcode::GetVccLo: code = 106u; break; + case ValueOpcode::GetVccHi: code = 107u; break; + default: break; + } + if (code != UINT32_MAX) { + const auto result = Register(before, code); + inst->ReplaceUsesWith(result); + return result; + } + for (size_t i = 0; i < inst->NumArgs(); ++i) inst->SetArg(i, Resolve(before, inst->Arg(i))); + return value; + }; + Register = [&](uint32_t before, uint32_t code) -> Value { + const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code); + if (index == UINT32_MAX) { + EXIT_IF(code < query.user_data_base || code - query.user_data_base >= query.user_data_count); + return IREmitter(&block).GetUserData(static_cast(code)); + } + auto [entry, inserted] = producers.try_emplace(index); + auto& producer = entry->second; + if (inserted) { + Block translated; + Frontend::Translator translator(query, &translated, 1u, + (options.input_info.compute->float_mode & 0x10u) == 0u); + translator.TranslateInstruction(source.decoded.instructions[index]); + for (const auto& inst: translated) { + uint32_t destination = UINT32_MAX; + Value value; + switch (inst.GetOpcode()) { + case ValueOpcode::SetScalarRegister: + destination = static_cast(inst.Arg(0).ScalarRegister()); value = inst.Arg(1); break; + case ValueOpcode::SetVccLo: destination = 106u; value = inst.Arg(0); break; + case ValueOpcode::SetVccHi: destination = 107u; value = inst.Arg(0); break; + default: break; + } + if (destination != UINT32_MAX) producer.values.insert_or_assign(destination, value); + } + for (auto& [destination, value]: producer.values) value = Resolve(index, value); + for (auto& inst: translated) inst.SetParent(&block); + block.Instructions().splice(block.end(), translated.Instructions()); + producer.ready = true; + } + EXIT_IF(!producer.ready || !producer.values.contains(code)); + return producer.values.at(code).Resolve(); + }; + for (uint32_t i = 0; i < 2u; ++i) { + const auto value = Register(source.call->instruction, call.src0.reg + i); + EXIT_IF(!ValidateRuntimeValue(query, value, RuntimeValueType::Integer)); + auto& root = query.value_storage.emplace_back(ValueOpcode::ReferenceU32); + root.SetArg(0, value); + source.call->target[i] = Value(&root); + } + RewriteToSsa(query.blocks); + ConstantPropagationPass(query.blocks, query.wave_size); + RemoveIdentities(query.blocks); + EliminateDeadCode(query.blocks); + for (auto& value: source.call->target) value = value.Instruction()->Arg(0).Resolve(); + query.value_storage.clear(); + for (auto& inst: block) { + inst.SetParent(nullptr); + if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 || inst.GetOpcode() == ValueOpcode::ReadConstBuffer) + inst.SetFlags(SrtReadFlags {.index = inst.Flags().index}); + } + query.value_storage.splice(query.value_storage.end(), block.Instructions()); + source.call_targets = std::move(static_cast(query)); +} + } // namespace +ShaderSource PrepareShaderSource(std::span code, const CompileOptions& options) { + EXIT_IF(code.empty()); + ShaderSource source; + if (!options.back_code.empty()) { + source.decoded = DecodeFusedProgram(code, options.back_code, source.code); + } else if (options.stage == ShaderType::Local) { + source.decoded = Decoder::DecodeFrontProgram(code); + auto& handoff = source.decoded.instructions.back(); + handoff.opcode = Decoder::Opcode::S_ENDPGM; + handoff.src_count = 0; + } else { + Decoder::DecodeProgram(code, source.decoded); + } + PrepareCallTarget(source, options); + if (source.call) { + source.code.assign(code.begin(), code.end()); + source.decoded.code = source.code; + } + return source; +} + +const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtRuntime& runtime) { + auto& reads = source.reads; + reads.clear(); + if (!source.call) return source.decoded; + IR::SrtReadCapture capture(runtime, reads); + const auto clean = IR::CleanRuntime(capture.ObservedRuntime()); + IR::SrtWalker walker(source.call_targets, clean); + uint32_t low = 0, high = 0; + const auto& call = source.decoded.instructions[source.call->instruction]; + if (!walker.Evaluate(source.call->target[0], low) || !walker.Evaluate(source.call->target[1], high)) + EXIT("shader call at pc 0x%08x has an unavailable scalar target", call.pc); + // SWAPPC reads the old aliased pair and ignores the target's low two bits. + const auto address = ((uint64_t {high} << 32u) | low) & ~uint64_t {3}; + EXIT_IF(address == 0u); + const auto query_reads = reads.size(); + if (source.linked) { + auto& linked = *source.linked; + const auto function = std::span(linked.code).subspan(source.code.size()); + if (clean.read_memory(clean.userdata, address, linked.observed_function) && + std::ranges::equal(function, linked.observed_function)) { + return linked.decoded; + } + reads.resize(query_reads); + } + ShaderSource::Linked linked {.code = source.code, + .decoded = source.decoded}; + const auto& last = source.decoded.instructions.back(); + EXIT_NOT_IMPLEMENTED(last.opcode != Decoder::Opcode::S_ENDPGM); + // Keep the caller's data/footer bytes at their native offsets. The leaf's virtual CFG + // PCs follow decoded instructions; its physical code is independently read and owned. + const auto function_pc = last.pc + last.word_count * sizeof(uint32_t); + for (uint32_t index = 0;;) { + // The observed queue helper is a scalar leaf. Scalar encodings need at most one literal. + EXIT_IF(index >= 64u || address > UINT64_MAX - (uint64_t {index} + 2u) * sizeof(uint32_t)); + std::array words {}; + if (!clean.read_memory(clean.userdata, address + uint64_t {index} * sizeof(uint32_t), + std::span(words).first(1))) + EXIT("shader call at pc 0x%08x cannot read leaf code", call.pc); + const auto family = Decoder::GetInstructionFamily(words[0]); + EXIT_NOT_IMPLEMENTED(family != Decoder::Family::SOP1 && family != Decoder::Family::SOP2 && + family != Decoder::Family::SOPC && family != Decoder::Family::SOPK && + family != Decoder::Family::SOPP); + Decoder::Instruction inst; + Decoder::DecodeInstruction(words, 0u, inst); + if (inst.word_count == 2u) { + if (!clean.read_memory(clean.userdata, address + uint64_t {index + 1u} * sizeof(uint32_t), + std::span(words).last(1))) + EXIT("shader call at pc 0x%08x cannot read leaf literal", call.pc); + inst = {}; + Decoder::DecodeInstruction(words, 0u, inst); + } + inst.pc = index * sizeof(uint32_t); + if (Decoder::IsDirectBranch(inst.opcode)) inst.branch_target += inst.pc; + linked.code.insert(linked.code.end(), inst.raw, inst.raw + inst.word_count); + index += inst.word_count; + EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_SWAPPC_B64 || + inst.opcode == Decoder::Opcode::S_GETPC_B64 || + inst.opcode == Decoder::Opcode::S_ENDPGM); + EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg) || CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u)); + const bool returns = inst.opcode == Decoder::Opcode::S_SETPC_B64; + if (returns) { + EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr || inst.src0.reg != call.dst.reg); + inst.opcode = Decoder::Opcode::S_BRANCH; + inst.branch_target = call.pc + call.word_count * sizeof(uint32_t); + } else if (Decoder::IsDirectBranch(inst.opcode)) { + EXIT_IF(inst.branch_target > UINT32_MAX - function_pc); + inst.branch_target += function_pc; + } + inst.pc += function_pc; + linked.decoded.instructions.push_back(std::move(inst)); + if (returns) break; + } + const auto function_end = linked.decoded.instructions.back().pc + + linked.decoded.instructions.back().word_count * sizeof(uint32_t); + for (size_t i = source.decoded.instructions.size(); i + 1u < linked.decoded.instructions.size(); ++i) { + const auto& inst = linked.decoded.instructions[i]; + if (Decoder::IsDirectBranch(inst.opcode)) + EXIT_IF(inst.branch_target < function_pc || inst.branch_target >= function_end); + } + reads.resize(query_reads); + reads.emplace_back(address, (linked.code.size() - source.code.size()) * sizeof(uint32_t)); + linked.decoded.instructions[source.call->instruction].branch_target = function_pc; + linked.decoded.code = linked.code; + linked.observed_function.resize(linked.code.size() - source.code.size()); + source.linked = std::move(linked); + ++source.revision; + return source.linked->decoded; +} + TranslateResult TranslateProgram(std::span code, const CompileOptions& options) { - if (code.empty()) { - EXIT("shader recompiler input is empty\n"); + auto source = PrepareShaderSource(code, options); + return TranslateProgram(source.decoded, options); +} + +TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileOptions& options) { + EXIT_IF(decoded.instructions.empty()); + bool has_call = false; + for (const auto& inst: decoded.instructions) { + if (inst.opcode != Decoder::Opcode::S_SWAPPC_B64) continue; + has_call = true; + if (inst.branch_target == UINT32_MAX) + EXIT("shader call at pc 0x%08x must be linked before translation", inst.pc); } if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex && options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh && @@ -493,34 +713,11 @@ TranslateResult TranslateProgram(std::span code, const CompileOp } const auto compile_begin = std::chrono::steady_clock::now(); - const auto phase_ms = [&compile_begin]() { + const auto phase_ms = [&compile_begin]() { return static_cast(std::chrono::duration_cast( - std::chrono::steady_clock::now() - compile_begin) - .count()); + std::chrono::steady_clock::now() - compile_begin).count()); }; - LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " code_words=%" PRIu64 " decode\n", - GetDumpLabel(options), StageName(options.stage), options.shader_hash, - static_cast(code.size())); - - Decoder::Program decoded; - std::vector joined_code; - if (!options.back_code.empty()) { - decoded = DecodeFusedProgram(code, options.back_code, joined_code); - } else if (options.stage == ShaderType::Local) { - decoded = Decoder::DecodeFrontProgram(code); - // The separately compiled hull half runs in the next Vulkan stage. - auto& handoff = decoded.instructions.back(); - handoff.opcode = Decoder::Opcode::S_ENDPGM; - handoff.src_count = 0; - } else { - Decoder::DecodeProgram(code, decoded); - } - LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " decode instructions=%" PRIu64 - " elapsed_ms=%" PRIu64 "\n", - GetDumpLabel(options), StageName(options.stage), options.shader_hash, - static_cast(decoded.instructions.size()), phase_ms()); - std::string decoded_dump; if (options.dump_ir) { decoded_dump = Decoder::ProgramToString(decoded); @@ -606,6 +803,12 @@ TranslateResult TranslateProgram(std::span code, const CompileOp IR::RemoveIdentities(ir.blocks); IR::EliminateDeadCode(ir.blocks); } + if (has_call) { + // Leaf returns are resolved CFG edges; native PC values must not escape into GPU data. + for (const auto* block: ir.blocks) + for (const auto& inst: *block) + EXIT_NOT_IMPLEMENTED(inst.GetOpcode() == IR::ValueOpcode::GetShaderBase); + } LowerTessellationMemory(ir, options); std::string cfg_dump; if (options.dump_ir) { diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h index 29b3a40e..91b538bd 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.h +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -4,8 +4,11 @@ #include "common/common.h" #include "common/stringUtils.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include "graphics/shader/shader.h" +#include +#include #include #include @@ -37,6 +40,32 @@ struct CompileResult { IR::Program program; }; +// Decoded source and typed scalar call queries are retained once per cache source. +struct ShaderSource { + struct Call { + uint32_t instruction; + std::array target; + }; + struct Linked { + std::vector code; + Decoder::Program decoded; + std::vector observed_function; + }; + std::vector code; + Decoder::Program decoded; + IR::ResourcePlan call_targets; + std::optional call; + std::optional linked; + uint64_t revision = 0; + std::vector> reads; +}; + +[[nodiscard]] ShaderSource PrepareShaderSource(std::span code, + const CompileOptions& options); +const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtRuntime& runtime); +[[nodiscard]] TranslateResult TranslateProgram(const Decoder::Program& decoded, + const CompileOptions& options); + [[nodiscard]] TranslateResult TranslateProgram(std::span code, const CompileOptions& options); [[nodiscard]] CompileResult CompileProgram(TranslateResult translated, diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp index 860fd229..7eb9cafb 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp @@ -1468,7 +1468,8 @@ Graph BuildGraph(const Decoder::Program& program) { for (uint32_t i = 0; i < program.instructions.size(); i++) { const auto& inst = program.instructions[i]; const auto next_pc = InstructionEndPc(inst); - if (Decoder::IsDirectBranch(inst.opcode)) { + if (Decoder::IsDirectBranch(inst.opcode) || + (inst.opcode == Opcode::S_SWAPPC_B64 && inst.branch_target != UINT32_MAX)) { if (!IsValidTarget(inst.branch_target, program.instructions, end_pc)) { ExitBuildFailure(graph, FailureKind::InvalidBranchTarget, UINT32_MAX, fmt::format("branch at pc 0x{:08x} targets invalid pc 0x{:08x}", @@ -1501,7 +1502,7 @@ Graph BuildGraph(const Decoder::Program& program) { if (next_pc <= end_pc) { labels.push_back(next_pc); } - } else if (inst.opcode == Opcode::S_ENDPGM) { + } else if (inst.opcode == Opcode::S_ENDPGM || inst.opcode == Opcode::S_SWAPPC_B64) { labels.push_back(next_pc); } } @@ -1550,6 +1551,9 @@ Graph BuildGraph(const Decoder::Program& program) { const auto next_pc = InstructionEndPc(last); if (last.opcode == Opcode::S_ENDPGM) { block.terminator.kind = TerminatorKind::Return; + } else if (last.opcode == Opcode::S_SWAPPC_B64 && last.branch_target != UINT32_MAX) { + block.terminator.kind = TerminatorKind::Branch; + block.terminator.true_block = block_at_pc(last.branch_target); } else if (last.opcode == Opcode::S_SETPC_B64) { const auto& target_info = setpc_targets.at(last.pc); if (target_info.indirect) { @@ -1676,6 +1680,36 @@ Graph Structurize(const Graph& graph) { return GotoStructurizer(graph).Run(); } +bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code) { + uint32_t destination = UINT32_MAX; + // Scalar ALU pair widths are not all decoded. The typed producer must confirm this word. + return InstructionWritesScalarCode(inst, code) || + (ScalarOperandCode(inst.dst, destination) && code == destination + 1u); +} + +uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph, + uint32_t before, uint32_t code) { + EXIT_IF(before >= program.instructions.size()); + const auto found = std::ranges::find_if(graph.blocks, [before](const auto& block) { + return block.inst_begin <= before && before < block.inst_end; + }); + EXIT_IF(found == graph.blocks.end()); + const auto* block = &*found; + for (size_t depth = 0; depth < graph.blocks.size(); ++depth) { + for (uint32_t i = before; i > block->inst_begin;) { + const auto& inst = program.instructions[--i]; + EXIT_IF(inst.opcode == Opcode::S_SWAPPC_B64); + if (MayWriteScalarRegister(inst, code)) return i; + } + if (block->id == graph.entry_block) return UINT32_MAX; + EXIT_IF(block->predecessors.size() != 1u); + block = graph.FindBlock(block->predecessors[0]); + EXIT_IF(block == nullptr); + before = block->inst_end; + } + EXIT("scalar shader call source has cyclic reaching definitions"); +} + std::string BranchConditionToString(BranchCondition condition) { switch (condition) { case BranchCondition::Always: return "always"; diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h index a051bf2f..8caad316 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h @@ -113,6 +113,9 @@ struct Graph { }; Graph BuildGraph(const Decoder::Program& program); +bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code); +uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph, + uint32_t before, uint32_t code); // Returns structured control flow or failure diagnostics without changing the native graph. // On failure, failure_block is an original block ID or UINT32_MAX. Graph Structurize(const Graph& graph); diff --git a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp index 63428193..12fc9411 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp @@ -61,6 +61,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = { {0x1eu, Opcode::S_BITSET1_B64}, {0x1fu, Opcode::S_GETPC_B64}, {0x20u, Opcode::S_SETPC_B64}, + {0x21u, Opcode::S_SWAPPC_B64}, {0x24u, Opcode::S_AND_SAVEEXEC_B64}, {0x28u, Opcode::S_ORN2_SAVEEXEC_B64}, {0x2du, Opcode::S_QUADMASK_B64}, @@ -155,6 +156,10 @@ void DecodeSop1(uint32_t pc, std::span code, uint32_t word_index } switch (inst.opcode) { + case Opcode::S_SWAPPC_B64: + inst.branch_target = UINT32_MAX; + inst.data_dwords = 2; + break; case Opcode::S_GETPC_B64: inst.src_count = 0; DecodeScalarDestination(sdst, pc, inst.dst); diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 5ceaabeb..5520a35e 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -60,6 +60,7 @@ enum class Opcode { S_QUADMASK_B64, S_GETPC_B64, S_SETPC_B64, + S_SWAPPC_B64, S_SUBVECTOR_LOOP_BEGIN, S_SUBVECTOR_LOOP_END, S_AND_SAVEEXEC_B32, diff --git a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp index 3653807f..d9cfaafc 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp @@ -10,7 +10,8 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) { case O::S_MOV_B64: S_MOV_B64(inst); return; case O::S_WQM_B32: S_WQM(inst, false); return; case O::S_WQM_B64: S_WQM(inst, true); return; - case O::S_GETPC_B64: S_GETPC_B64(inst); return; + case O::S_GETPC_B64: + case O::S_SWAPPC_B64: S_GETPC_B64(inst); return; case O::S_SETPC_B64: return; case O::S_SUBVECTOR_LOOP_BEGIN: S_SUBVECTOR_LOOP(inst, true); return; case O::S_SUBVECTOR_LOOP_END: S_SUBVECTOR_LOOP(inst, false); return; diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 0e473f60..ea4cf7bf 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -590,6 +590,7 @@ struct ResourcePlan { std::vector control_flow; std::vector srt_reads; bool requires_specialization_memory = false; + std::vector> source_reads; bool capture_specialization_reads = false; bool srt_plan_complete = false; bool resource_tracking_complete = false; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index a56c339c..49a6993d 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -141,31 +140,6 @@ bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResou return true; } -struct ReadCapture { - SrtRuntime source; - std::vector>& ranges; -}; - -bool CaptureStrictRead(void* userdata, uint64_t address, std::span values) { - auto& capture = *static_cast(userdata); - if (!capture.source.read_specialization_memory(capture.source.userdata, address, values)) { - return false; - } - capture.ranges.emplace_back(address, values.size_bytes()); - return true; -} - -bool CaptureOrdinaryRead(void* userdata, uint64_t address, std::span values) { - auto& capture = *static_cast(userdata); - if (capture.source.read_memory != nullptr) { - if (!capture.source.read_memory(capture.source.userdata, address, values)) return false; - } else { - std::memcpy(values.data(), reinterpret_cast(address), values.size_bytes()); - } - capture.ranges.emplace_back(address, values.size_bytes()); - return true; -} - const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) { if (source >= program.descriptor_sources.size()) { return nullptr; @@ -953,6 +927,7 @@ ResourcePlan ExtractResourcePlan(const Program& program) { plan.user_data_count = program.user_data_count; plan.info = program.info; plan.memory_info = program.memory_info; + plan.source_reads = program.source_reads; plan.srt_plan_complete = program.srt_plan_complete; plan.resource_tracking_complete = program.resource_tracking_complete; @@ -1071,7 +1046,8 @@ ResourcePlan ExtractResourcePlan(const Program& program) { for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) { plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]); } - plan.capture_specialization_reads |= capture_indirect_reads || capture_control_reads; + plan.capture_specialization_reads |= + capture_indirect_reads || capture_control_reads || !plan.source_reads.empty(); if (plan.capture_specialization_reads) { // Every former flat alias now targets its retained raw read. Mark those reads // on the normalized graph, so discarded Phi/EXEC dependencies stay discarded. @@ -1106,27 +1082,23 @@ ResourcePlan ExtractResourcePlan(const Program& program) { if (image.written) MarkCleanReads(planned_reads, Source(plan, image.source)); } } - if (capture_indirect_reads) plan.resource_tracking_complete &= !program.has_address_writes; + if (capture_indirect_reads || !plan.source_reads.empty()) + plan.resource_tracking_complete &= !program.has_address_writes; return plan; } bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot, ResourceSpecialization& specialization) { if (!program.resource_tracking_complete || - (program.requires_specialization_memory && runtime.read_specialization_memory == nullptr)) { + ((program.requires_specialization_memory || !program.source_reads.empty()) && + runtime.read_specialization_memory == nullptr)) { return false; } - const bool capture_reads = program.capture_specialization_reads; + const bool capture_reads = program.capture_specialization_reads || !program.source_reads.empty(); auto& reads = snapshot.specialization_reads; - reads.clear(); - ReadCapture capture {runtime, reads}; - SrtRuntime observed = runtime; - if (capture_reads) { - observed.userdata = &capture; - observed.read_specialization_memory = runtime.read_specialization_memory != nullptr - ? CaptureStrictRead : nullptr; - observed.read_memory = CaptureOrdinaryRead; - } + reads = program.source_reads; + SrtReadCapture capture {runtime, reads}; + const auto observed = capture_reads ? capture.ObservedRuntime() : runtime; SrtWalker clean(program, CleanRuntime(observed)); SrtWalker walker(program, observed, capture_reads || program.requires_specialization_memory ? &clean : nullptr); diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 492ab303..ab9d4a32 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -11,6 +11,35 @@ namespace Libs::Graphics::ShaderRecompiler::IR { +SrtRuntime SrtReadCapture::ObservedRuntime() { + auto runtime = m_source; + runtime.userdata = this; + runtime.read_specialization_memory = m_source.read_specialization_memory != nullptr + ? ReadStrict : nullptr; + runtime.read_memory = ReadOrdinary; + return runtime; +} + +bool SrtReadCapture::ReadStrict(void* userdata, uint64_t address, std::span values) { + auto& capture = *static_cast(userdata); + if (!capture.m_source.read_specialization_memory(capture.m_source.userdata, address, values)) { + return false; + } + capture.m_ranges.emplace_back(address, values.size_bytes()); + return true; +} + +bool SrtReadCapture::ReadOrdinary(void* userdata, uint64_t address, std::span values) { + auto& capture = *static_cast(userdata); + if (capture.m_source.read_memory != nullptr) { + if (!capture.m_source.read_memory(capture.m_source.userdata, address, values)) return false; + } else { + std::memcpy(values.data(), reinterpret_cast(address), values.size_bytes()); + } + capture.m_ranges.emplace_back(address, values.size_bytes()); + return true; +} + SrtRuntime CleanRuntime(SrtRuntime runtime) { runtime.read_memory = runtime.read_specialization_memory != nullptr ? runtime.read_specialization_memory diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index 2628a4fa..6e353e4b 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -20,6 +20,20 @@ struct SrtRuntime { std::span workgroup_counts; }; +class SrtReadCapture { +public: + SrtReadCapture(SrtRuntime source, std::vector>& ranges): + m_source(source), m_ranges(ranges) {} + SrtRuntime ObservedRuntime(); + +private: + static bool ReadStrict(void* userdata, uint64_t address, std::span values); + static bool ReadOrdinary(void* userdata, uint64_t address, std::span values); + + SrtRuntime m_source; + std::vector>& m_ranges; +}; + // Retained reads no longer need the guest instruction PC. A clean read evaluates // its address and bounds through the strict reader as well as the final DWORD. struct SrtReadFlags { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 58c607eb..74ab26f2 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -9950,6 +9950,105 @@ public: return result; } + void CheckScalarLeafCall() { + constexpr const char* name = "ScalarLeafCall"; + constexpr uintptr_t base = 0x0000000204800000ull; + constexpr uint64_t size = 0x10000; + constexpr auto table = base + 0x100; + constexpr auto first = base + 0x1000; + constexpr auto second = base + 0x2000; + constexpr auto output = base + 0x4000; + std::vector code{ + EncodeSMovB32(20, InlineU32(0)), + 0xbeea03ffu, static_cast(base), 0xbeeb03ffu, static_cast(base >> 32u), + 0xf4080135u, 0xfa000000u, 0xbf8cc07fu, 0xf4240382u, 0xfa000020u, + 0xbf8cc07fu, 0xbf130e80u, 0xbf840004u, 0xf4240402u, 0xfa000028u, + 0xbf8cc07fu, 0xbe8e210eu, EncodeVop1(0x01, 1, 20), + EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(1, 0, 0)}; + AppendEnd(&code); + ShaderMapUserData(reinterpret_cast(code.data()), + {.type = Prospero::ShaderBinaryType::kCs, + .code_size_bytes = static_cast(code.size() * sizeof(u32))}); + EnsureRuntimeContext(); + int64_t physical = -1; + Require(name, "allocate", LibKernel::Memory::KernelAllocateDirectMemory( + 0, LibKernel::Memory::KernelGetDirectMemorySize(), size, size, 0, &physical) == 0, + "call fixture allocation failed"); + void* mapped = reinterpret_cast(base); + Require(name, "map", LibKernel::Memory::KernelMapDirectMemory( + &mapped, size, 0x3, 0x10, physical, size) == 0 && mapped == reinterpret_cast(base), + "call fixture mapping failed"); + std::memset(mapped, 0, size); + const ShaderBufferResource table_desc{{static_cast(table), + static_cast(table >> 32u) | (16u << 16u), 8, 0x5204u}}; + LibKernel::Memory::WriteBacking(base, table_desc.fields, sizeof(table_desc.fields)); + const std::array leaf{EncodeSMovB32(20, 255), 41u, EncodeSop1(0x21, 125, 14)}; + for (const auto address : {first, second}) + LibKernel::Memory::WriteBacking(address, leaf.data(), sizeof(leaf)); + + RenderContext context(m_runtime_context); + context.InitializeGpu(nullptr); + LibKernel::Memory::InstallGpuResources(&context); + context.GetGpu().SendCommandSync([&] { + CommandProcessor processor(context, 0); + processor.BufferInit(); + context.MapMemory(base, size); + auto& cache = context.GetBufferCache(); + auto& shaders = processor.GetShCtx(); + shaders.SetCsShader({.data_addr = reinterpret_cast(code.data()), + .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1, + .wave_size = 64, .user_sgpr = 4}); + ShaderBufferResource out{}; + out.UpdateAddress48(output); + out.fields[2] = 4; + out.fields[3] = DstSel(4, 5, 6, 7) | + (static_cast(Prospero::BufferFormat::k32UInt) << 12u); + for (u32 i = 0; i < 4; ++i) + shaders.SetCsUserSgpr(i, out.fields[i], HW::UserSgprType::Unknown); + uint64_t original_id = 0; + struct Case { uint64_t address; u32 value; bool update; }; + for (const auto test : {Case{first, 41, false}, Case{first, 41, false}, + Case{second, 41, false}, Case{second, 73, true}, + Case{first, 41, false}}) { + Require(name, "refresh function pointer", context.InvalidateMemory(table + 0x20, 8), + "function table escaped mapped memory"); + const uint64_t target = test.address | 3u; + LibKernel::Memory::WriteBacking(table + 0x20, &target, sizeof(target)); + if (test.update) { + auto [buffer, offset] = cache.ObtainBuffer(test.address + 4, 4, true); + buffer->Fill(offset, 4, test.value); + Require(name, "GPU-written function code", cache.HasGpuDirtyBytes(test.address + 4, 4), + "code mutation did not remain GPU-owned before refresh"); + } + ShaderComputeInputInfo input{}; + const auto program = context.GetPipelineCache().GetComputeProgram( + shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input); + if (original_id == 0) original_id = program.id; + Require(name, "exact function identity", + (program.id == original_id) == (test.value == 41), + "helper bytes were stale or identical relocated/restored code recompiled"); + Require(name, "current code dependency", + std::ranges::find(input.stage.resources->specialization_reads, + std::pair{test.address, sizeof(leaf)}) != + input.stage.resources->specialization_reads.end(), + "function relocation retained an old code dependency"); + processor.DispatchDirect(1, 1, 1, 0x41u); + cache.ReadMemory(output, sizeof(u32)); + u32 actual = 0; + Require(name, "scalar leaf execution and return", + LibKernel::Memory::TryReadBacking(output, &actual, sizeof(actual)) && actual == test.value, + "caller did not observe the leaf's scalar result after return"); + } + context.UnmapMemory(base, size); + context.GetCommandScheduler().Finish(); + }); + LibKernel::Memory::InstallGpuResources(nullptr); + context.ShutdownGpu(); + Require(name, "release", LibKernel::Memory::KernelCheckedReleaseDirectMemory(physical, size) == 0, + "call fixture release failed"); + std::printf("[gpu] %-32s ok\n", name); + } + void CheckNativeIndirectDispatch() { constexpr const char *name = "NativeIndirectDispatch"; constexpr uintptr_t base = 0x0000000204600000ull; @@ -42796,6 +42895,7 @@ int main(int argc, char **argv) { if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) { VulkanHarness vulkan; vulkan.CheckGpuCommandLane(); + vulkan.CheckScalarLeafCall(); CheckAgcSystemTable(vulkan.RuntimeRenderer()); CheckPm4IndirectControlFlow(vulkan.RuntimeRenderer()); CheckPm4WaitResume(vulkan.RuntimeRenderer()); @@ -43290,6 +43390,7 @@ int main(int argc, char **argv) { } vulkan.CheckGpuSuspendPoint(); vulkan.CheckGpuCommandLane(); + vulkan.CheckScalarLeafCall(); if (skipped_device_checks) { std::printf( "ShaderRecompilerComputeTests: device rasterization checks skipped, this " diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index b4ee2771..cb1a8743 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -10308,6 +10308,147 @@ void TestNewShaderRecompilerBranchConditionForms() { } } +void TestMemoryFedScalarLeafCall() { + using namespace ShaderRecompiler; + constexpr uint64_t base = 0xfe0040000ull; + const uint32_t shader[] = { + 0xbeea03ffu, 0xe0040000u, 0xbeeb03ffu, 0x0000000fu, + 0xf4080135u, 0xfa000000u, 0xbf8cc07fu, 0xf4240382u, 0xfa000020u, + 0xbf8cc07fu, 0xbf130e80u, 0xbf840004u, 0xf4240402u, 0xfa000028u, + 0xbf8cc07fu, 0xbe8e210eu, 0xbf810000u}; + struct Memory { + std::array words{}; + uint64_t end = base + 0x10c; + bool clean = true; + } memory; + std::copy_n(std::array{0xe0040060u, 0x0010000fu, 8u, 0x5204u}.begin(), + 4, memory.words.begin()); + memory.words[32] = static_cast(base + 0x103); // SWAPPC aligns the old pair. + memory.words[33] = static_cast(base >> 32u); + memory.words[64] = EncodeSMovB32(3, 255); + memory.words[65] = 0x12345678; + memory.words[66] = EncodeSop1(0x21, 125, 14); + const auto read = [](void *data, uint64_t address, std::span out) { + const auto& m = *static_cast(data); + if (!m.clean || address < base || address > m.end || out.size_bytes() > m.end - address) + return false; + std::memcpy(out.data(), reinterpret_cast(m.words.data()) + address - base, + out.size_bytes()); + return true; + }; + IR::SrtRuntime runtime{.read_memory = read, .userdata = &memory, .read_specialization_memory = read}; + const auto options = MakeCompileOptions(ShaderType::Compute); + auto source = PrepareShaderSource(shader, options); + Check(source.call && source.decoded.instructions[source.call->instruction].branch_target == UINT32_MAX, + "memory-fed SWAPPC did not retain an unresolved target"); +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS + ExpectFatal([&] { (void)TranslateProgram(source.decoded, options); }, + "unresolved SWAPPC silently fell through"); +#endif + const auto& linked = RefreshShaderSource(source, runtime); + Check(source.revision == 1 && linked.instructions.back().branch_target == 0x40 && + linked.instructions[source.call->instruction].branch_target == 0x44 && + source.reads.back() == std::pair{base + 0x100, 12}, + "aliased call target, exact leaf range, or continuation was lost"); + const auto* storage = linked.instructions.data(); + const auto* scratch = source.linked->observed_function.data(); + (void)RefreshShaderSource(source, runtime); + Check(source.revision == 1 && source.linked->decoded.instructions.data() == storage && + source.linked->observed_function.data() == scratch, + "unchanged leaf rebuilt decoded instructions or refresh storage"); + auto translated = TranslateProgram(linked, options); + translated.program.source_reads = source.reads; + const auto plan = IR::ExtractResourcePlan(translated.program); + Check(plan.source_reads == source.reads && plan.capture_specialization_reads, + "linked code dependencies were omitted from resource alias validation"); + + std::vector writer(std::begin(shader), std::end(shader) - 1); + writer.insert(writer.end(), {EncodeVop1(0x01, 1, 3), EncodeMubuf0(0x1c, 0, false), + EncodeMubuf1(1, 6, 0), EncodeSopp(0x01)}); + std::array user_data{}; + user_data[25] = static_cast(base >> 32u); + user_data[26] = 4; + user_data[27] = 3u << 28u; + auto write_options = options; + write_options.user_data = user_data; + auto write_runtime = runtime; + write_runtime.user_data = user_data; + auto write_source = PrepareShaderSource(writer, write_options); + auto write_program = TranslateProgram(RefreshShaderSource(write_source, write_runtime), write_options); + write_program.program.source_reads = write_source.reads; + auto write_plan = IR::ExtractResourcePlan(write_program.program); + IR::ResourceSnapshot snapshot; + IR::ResourceSpecialization specialization; + const auto written = std::ranges::find_if(write_plan.info.buffers, + [](const auto& buffer) { return buffer.written; }); + Check(written != write_plan.info.buffers.end(), "caller store lost its writable binding"); + const auto output_index = static_cast(written - write_plan.info.buffers.begin()); + for (const auto offset : {0x80u, 0x100u, 0x1f0u}) { + user_data[24] = static_cast(base + offset); + Check(IR::MaterializeResources(write_plan, write_runtime, snapshot, specialization) && + snapshot.buffers[output_index].dwords[0] == user_data[24] && + snapshot.buffers[output_index].dwords[2] == 4 && + std::ranges::find(snapshot.specialization_reads, + std::pair{base + 0x80, 4}) != snapshot.specialization_reads.end() && + std::ranges::find(snapshot.specialization_reads, + std::pair{base + 0x100, 12}) != snapshot.specialization_reads.end(), + "call dependencies or writable ranges were lost before binding validation"); + } + + // Relocating identical code updates dependencies without rebuilding the source. + std::copy_n(memory.words.begin() + 64, 3, memory.words.begin() + 96); + memory.words[32] = static_cast(base + 0x180); + memory.end = base + 0x18c; + (void)RefreshShaderSource(source, runtime); + Check(source.revision == 1 && source.reads.back().first == base + 0x180, + "identical relocated leaf recompiled or retained its old memory dependency"); + memory.words[97] = 0x87654321; + (void)RefreshShaderSource(source, runtime); + Check(source.revision == 2 && source.linked->code.back() == EncodeSop1(0x21, 125, 14), + "changed leaf bytes retained stale source"); + // A newly selected shorter function may end before the old bulk-read length. + memory.words[96] = EncodeSMovB32(3, 129); + memory.words[97] = EncodeSop1(0x21, 125, 14); + memory.end = base + 0x188; + (void)RefreshShaderSource(source, runtime); + Check(source.revision == 3 && source.reads.back().second == 8, + "shorter exact-boundary leaf was not decoded after a failed old-span read"); +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS + memory.clean = false; + ExpectFatal([&] { (void)RefreshShaderSource(source, runtime); }, + "unreadable call dependencies reused stale linked code"); + memory.clean = true; + auto ordinary = runtime; + ordinary.read_specialization_memory = nullptr; + ExpectFatal([&] { (void)RefreshShaderSource(source, ordinary); }, + "call target query accepted an ordinary non-strict memory reader"); + std::vector escaping(std::begin(shader), std::end(shader) - 1); + escaping.insert(escaping.end(), {EncodeVop1(0x01, 1, 14), + EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(1, 0, 0), EncodeSopp(0x01)}); + ExpectFatal([&] { + auto escaping_source = PrepareShaderSource(escaping, options); + (void)TranslateProgram(RefreshShaderSource(escaping_source, runtime), options); + }, "saved native return PC escaped into GPU data"); + memory.words[96] = EncodeSMovB32(14, 129); + ExpectFatal([&] { (void)RefreshShaderSource(source, runtime); }, + "leaf clobbering its return pair was linked"); + // AND_B64 writes the high half despite the decoder's generic scalar width. + const uint32_t high_writer[] = { + EncodeSMovB32(15, 129), EncodeSop2(0x0f, 14, 128, 128), + EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)}; + auto pair_source = PrepareShaderSource(high_writer, options); + IR::SrtWalker pair_walker(pair_source.call_targets, runtime); + uint32_t high = 1; + Check(pair_walker.Evaluate(pair_source.call->target[1], high) && high == 0, + "scalar pair producer used a stale preceding high-half definition"); + const uint32_t joined[] = { + EncodeSopp(0x04, 2), EncodeSMovB32(14, 129), EncodeSopp(0x02, 1), + EncodeSMovB32(14, 130), EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)}; + ExpectFatal([&] { (void)PrepareShaderSource(joined, options); }, + "ambiguous reaching definitions were accepted for a call target"); +#endif +} + void TestNewShaderRecompilerSetpcBranch() { const uint32_t shader[] = { EncodeSop1(0x1f, 4, 0), // s_getpc_b64 s[4:5] @@ -15445,6 +15586,7 @@ int main() { TestNewShaderRecompilerPixelImageSampleLodSelection(); TestNewShaderRecompilerBranchConditionForms(); TestNewShaderRecompilerSetpcBranch(); + TestMemoryFedScalarLeafCall(); TestFusedShaderHandoffPreservesRegisters(); TestMeshExportStorage(); TestMergedShaderUserDataSnapshot();