From 56f8a62428042a682a7c32bc4e7eb0ea26d004fd Mon Sep 17 00:00:00 2001 From: Celegans12 <53881228+Celegans12@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:16:09 +0200 Subject: [PATCH] feat(agc): implement deferred faults for reads through null SRT pointers Capturing a shader threw "null or misaligned address" when the flat SRT walk read through a pointer loaded from memory that is null or unmapped, even when the program skips that s_load. Such a read now yields zero and a record with its pc and address, as does a slot whose address needs it. The registered artifact checks one flag word per guarded slot at each ReadConst; the capture appends the flags and the records to the flattened SRT, and a set flag raises an Unmapped BDA fault through the fault buffer and stops the invocation as a BDA read does. Nothing per capture enters a key: poisoned and clean captures share one variant and specialization. Descriptors, branch conditions, uniform fills, table bases and user-data pointers that need such a value still throw, and poisoned results stay out of the dispatch and draw caches. The shader disk cache format becomes 21. --- .../include/Driver/Dispatch/DispatchCache.hpp | 2 + .../Execution/include/ShaderMemory.hpp | 1 + .../src/Driver/Dispatch/Dispatch.cpp | 2 +- .../src/Driver/Dispatch/DispatchInsert.cpp | 4 + .../Execution/src/Driver/Draw/DrawCapture.cpp | 2 +- .../Execution/src/ShaderMemory.cpp | 23 ++ .../Graphics/src/ShaderAddressResources.cpp | 4 +- .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 3 + .../libSceAgcDriver/tests/ShaderMemory.cpp | 200 +++++++++++++- .../tests/execution/GuardedNullPointer.cpp | 244 ++++++++++++++++++ .../IrMetadata/ControlFlowInfo.hpp | 8 + .../IrMetadata/ResourcePlan.hpp | 3 + .../IrMetadata/ShaderInfo.hpp | 1 + .../Optimization/RequestMemoryView.hpp | 1 + .../include/Optimization/SrtWalker.hpp | 4 +- .../SrtWalker/SrtDescriptorEvaluation.hpp | 2 +- .../Optimization/SrtWalker/SrtEvaluator.hpp | 7 + .../SrtWalker/SrtFlatSlotClasses.hpp | 1 + .../SrtWalker/SrtInstructionPredicates.hpp | 1 + .../Optimization/src/BindingAllocator.cpp | 2 + .../Optimization/src/RequestMemoryView.cpp | 24 ++ .../Optimization/src/ResourceMaterializer.cpp | 20 +- .../src/SrtWalker/SrtDescriptorEvaluation.cpp | 18 +- .../src/SrtWalker/SrtEvaluator.cpp | 5 + .../src/SrtWalker/SrtFlatSlotClasses.cpp | 44 ++++ .../SrtWalker/SrtInstructionPredicates.cpp | 21 ++ .../Optimization/src/SrtWalker/SrtWalker.cpp | 4 +- core/shader/recompiler/Recompiler.cpp | 3 +- core/shader/recompiler/Recompiler.hpp | 1 + core/shader/recompiler/ShaderDiskCache.cpp | 10 +- core/shader/recompiler/ShaderDiskCache.hpp | 2 +- .../include/SpirvBackend/SpirvBda.hpp | 1 + .../SpirvBackend/src/SpirvBdaFault.cpp | 27 +- .../SpirvBackend/src/SpirvBdaLookup.cpp | 2 +- .../SpirvBackend/src/SpirvBdaRead.cpp | 1 + .../SpirvBackend/src/SpirvFlowEmitter.cpp | 3 +- .../SpirvMemory/SpirvMemoryInstructions.cpp | 35 +++ docs/dev/TechnicalDebt.md | 3 + 38 files changed, 715 insertions(+), 24 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/GuardedNullPointer.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp index 8bbb76d59..1591221fd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp @@ -82,6 +82,8 @@ struct DataMask { enum class EntryOutcome { Equal, EqualData, Differing, Inaccessible, QueuedLabel, FlushingImage, PublishMoved, PendingMoved, ForgetMoved }; +bool CacheableResult(const ShaderRecompiler::RecompileResult& compiled); + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp index 457fd8360..75f597a2d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp @@ -85,6 +85,7 @@ private: }; static bool read(void* context, std::uint64_t address, std::uint32_t* value); + static bool accessible(void* context, std::uint64_t address, std::uint64_t bytes); Page& page(std::uint64_t base); // Regions given at construction (the registered shader's code and header), referenced as given: diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index b0909f721..975e68edb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -231,7 +231,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, } recompileMs += phaseTiming.Elapsed(); phaseTiming.Phase(PhaseRecompile); - insertDispatch(address, key, noDispatchCache, profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming); + insertDispatch(address, key, noDispatchCache || !CacheableResult(*compiledResult), profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming); } if (verifyDataHits() && dataHit) verifyDataHit(snapshot, codeOffset, request, memory, address, *keepVariant, liveWords, *compiledResult); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp index 459a69cf9..8478e84a6 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp @@ -6,6 +6,10 @@ namespace AgcDriver::DriverDetail { +bool CacheableResult(const ShaderRecompiler::RecompileResult& compiled) { + return compiled.poisonedSrtReads == 0u; +} + void Driver::insertDispatch(std::uint64_t address, std::uint64_t key, bool noDispatchCache, bool profile, const std::shared_ptr& registeredShader, std::uint64_t forgetAtCapture, std::span memory, const std::shared_ptr& shaderMemory, const std::vector& captured, const std::shared_ptr& capture, const std::shared_ptr& compiledResult, const std::shared_ptr& missedEntry, bool missedDiffering, std::shared_ptr& attachVariant, DispatchPhaseTiming& phaseTiming) { if (!noDispatchCache) { auto fresh = std::make_shared(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp index 97f85726f..ad07dcdc6 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp @@ -88,7 +88,7 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP std::uint64_t unstable = 0, mismatches = 0; for (std::size_t i = 0; i < programs.size(); ++i) { const auto& stageCapture = stageCaptures[i]; - if (stageCapture.compiled == nullptr) continue; + if (stageCapture.compiled == nullptr || !CacheableResult(*stageCapture.compiled)) continue; auto variant = std::make_shared(); variant->compiled = stageCapture.compiled; variant->shader = programs[i].snapshot; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp index 8c07b7b74..7344a3418 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp @@ -201,6 +201,28 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val return true; } +bool ShaderMemory::accessible(void* context, std::uint64_t address, std::uint64_t bytes) { + auto& self = *static_cast(context); + if (bytes == 0 || address > std::numeric_limits::max() - bytes) return false; + const auto end = address + bytes; + while (address < end) { + const auto next = self.initial.upper_bound(address); + if (next != self.initial.begin()) { + const auto previous = std::prev(next); + if (address - previous->first < previous->second.size()) { + address = previous->first + previous->second.size(); + continue; + } + } + const auto base = address & ~static_cast(PageBytes - 1); + const auto pageEnd = std::min(base + PageBytes, end); + const auto& page = self.page(base); + if (!page.wordwise && !page.valid.all() && !GuestMemory::Accessible(reinterpret_cast(address), static_cast(pageEnd - address))) return false; + address = pageEnd; + } + return true; +} + ShaderMemory::KnownValueCounts ShaderMemory::KnownValues() { auto& totals = CaptureTotals(); return {totals.wordsKnown.load(std::memory_order_relaxed), totals.wordsKnownVerified.load(std::memory_order_relaxed), totals.wordsKnownMismatches.load(std::memory_order_relaxed)}; @@ -233,6 +255,7 @@ std::shared_ptr ShaderMemory::capture(c runtime.userContext = this; runtime.readMemory = &read; runtime.readSpecializationMemory = &read; + runtime.accessible = &accessible; auto capture = invocation != nullptr ? invocation->Capture(runtime) : handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime); if (profile) { totals.captureNanoseconds += NanosecondsSince(started); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp index 895fa359d..04601fb8e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp @@ -15,8 +15,8 @@ void ShaderResources::prepareAddressBindings(std::span sha if (binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable) ++tables; else ++faults; } - const bool rectListFault = shader.stage == ShaderRecompiler::ShaderStage::TessellationControl && tables == 0 && faults == 1; - Require((tables == faults || rectListFault) && tables <= 1 && faults <= 1, "invalid BDA table and fault descriptors"); + const bool faultOnly = tables == 0 && faults == 1 && shader.program->bdaAbiVersion == ShaderRecompiler::BdaAbi::Version; + Require((tables == faults || faultOnly) && tables <= 1 && faults <= 1, "invalid BDA table and fault descriptors"); Require(shader.program->bdaAbiVersion == (faults == 0 ? 0u : ShaderRecompiler::BdaAbi::Version), "incompatible BDA ABI version"); // Rect-list validation needs a fault buffer, but never accesses guest addresses. usesBda = usesBda || tables != 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 3facf9ad5..e5db59f1b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -79,6 +79,7 @@ void requireSameResult(const RecompileResult& left, const RecompileResult& right require(sameBindings(left.bindings, right.bindings), prefix + "bindings differ"); require(left.pushConstants == right.pushConstants, prefix + "push constants differ"); require(left.specialization == right.specialization, prefix + "specialization constants differ"); + require(left.poisonedSrtReads == right.poisonedSrtReads, prefix + "poisoned SRT read counts differ"); require(left.vertexAttributes.size() == right.vertexAttributes.size(), prefix + "vertex attribute count differs"); for (std::size_t i = 0; i < left.vertexAttributes.size(); ++i) { const auto& a = left.vertexAttributes[i]; @@ -149,6 +150,7 @@ RecompileResult sampleResult() { result.instanceOffsetConflict = true; result.parameterExports = {0, 3, 7}; result.fragmentParameters = {{0, 1, true, false, true}, {2, 3, false, true}}; + result.poisonedSrtReads = 3; result.variantId = 99; return result; } @@ -220,6 +222,7 @@ CompiledVariant sampleVariant() { info.info.vertexOffsetSgpr = 6; info.info.hasBitwiseXor = true; info.info.usesDma = true; + info.info.usesFaultBuffer = true; info.bindings.pushDataStartDword = 2; info.bindings.memoryOffsetDword = 1; info.bindings.memoryOffsetCount = 5; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 0d4c3bd60..3525d7900 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -29,6 +29,17 @@ #include #include #include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#endif namespace { @@ -1761,6 +1772,182 @@ void verifyFunctionLdsBound() { require(unsized == FunctionLdsDwordLimit, "function LDS: an access without a known width must keep the full array"); } +struct NestedRequest { + std::vector code; + std::array userData{}; + ShaderRecompiler::RecompileRequest request{}; + + NestedRequest(std::vector words, const void* srt) : code(std::move(words)) { + using namespace ShaderRecompiler; + const auto address = static_cast(reinterpret_cast(srt)); + userData = {static_cast(address), static_cast(address >> 32u)}; + request.shader = {ShaderStage::Compute, 0x60000u + static_cast(code.size()) * 4u, code, 0, {}}; + request.context.waveSize = 64; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target = BufferTarget(); + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32; + request.target.fragmentShaderBarycentricEnabled = false; + request.layout.pushConstantSizeBytes = 128; + } +}; + +bool RegionsCover(const std::vector& regions, const void* pointer, std::size_t bytes) { + auto address = static_cast(reinterpret_cast(pointer)); + const auto end = address + bytes; + while (address < end) { + const auto region = std::find_if(regions.begin(), regions.end(), [&](const ShaderRecompiler::MemoryRegion& candidate) { return address >= candidate.guestAddress && address < candidate.guestAddress + candidate.bytes.size(); }); + if (region == regions.end()) return false; + address = region->guestAddress + region->bytes.size(); + } + return true; +} + +bool RegionsTouch(const std::vector& regions, std::uint64_t address, std::size_t bytes) { + return std::any_of(regions.begin(), regions.end(), [&](const ShaderRecompiler::MemoryRegion& region) { return region.guestAddress < address + bytes && address < region.guestAddress + region.bytes.size(); }); +} + +bool GuardRecords(const ShaderRecompiler::ResourceCapture& capture) { + const auto& plan = *capture.plan; + const auto& flat = capture.snapshot.flattenedSrt; + const auto& poison = capture.snapshot.srtPoison; + const auto flags = plan.srtReads.size(); + const auto guards = plan.guardedSrtSlots.size(); + if (flat.size() != flags + guards + 3u * poison.size()) return false; + for (std::size_t guard = 0; guard < guards; ++guard) { + const auto record = std::find_if(poison.begin(), poison.end(), [&](const ShaderRecompiler::SrtReadPoison& entry) { return entry.slot == plan.guardedSrtSlots[guard]; }); + const auto expected = record == poison.end() ? 0u : static_cast(record - poison.begin()) + 1u; + if (flat[flags + guard] != expected) return false; + } + for (std::size_t record = 0; record < poison.size(); ++record) { + const auto* words = flat.data() + flags + guards + 3u * record; + if (words[0] != poison[record].pc || words[1] != static_cast(poison[record].address) || words[2] != static_cast(poison[record].address >> 32u) || flat.at(poison[record].slot) != 0u) return false; + } + return true; +} + +void verifyGuardedNullPointers() { + using namespace ShaderRecompiler; + constexpr std::size_t Reserved = 0x10000; + void* reserved = nullptr; +#ifdef _WIN32 + reserved = VirtualAlloc(nullptr, Reserved, MEM_RESERVE, PAGE_NOACCESS); +#else + reserved = mmap(nullptr, Reserved, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) reserved = nullptr; +#endif + require(reserved != nullptr, "guarded pointer: cannot reserve an inaccessible range"); + const auto unmapped = static_cast(reinterpret_cast(reserved)); + alignas(256) static std::array output{}; + alignas(256) static std::array root{}; + static std::uint32_t payload = 0x3f800000u; + static std::uint64_t indirect = 0; + const auto outputAddress = static_cast(reinterpret_cast(output.data())); + root = {static_cast(outputAddress), static_cast((outputAddress >> 32u) & 0xffffu), static_cast(sizeof(output)), 0x30027facu, 1u}; + const auto point = [&](std::uint64_t address) { + root[6] = static_cast(address); + root[7] = static_cast(address >> 32u); + }; + const auto payloadAddress = static_cast(reinterpret_cast(&payload)); + const std::vector branchCode{0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xbf068008u, 0xbf850003u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u}; + const std::vector chainCode{0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xbf068008u, 0xbf850006u, 0xf4040385u, 0xfa000000u, 0xbf8cc07fu, 0xf4000307u, 0xfa000000u, 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, + 0xe0701000u, 0x80010102u, 0xbf810000u}; + const std::vector descriptorCode{0xf4040280u, 0xfa000018u, 0xbf8cc07fu, 0xf4080105u, 0xfa000000u, 0xbf8cc07fu, 0x34040082u, 0xe0701000u, + 0x80010002u, 0xbf810000u}; + const auto slotPc = [](const IrResourcePlan& plan, std::uint32_t slot) { return plan.srtReads.at(slot).value->Flags().pc; }; + const auto faultBinding = [](const RecompileResult& result) { + return std::any_of(result.bindings.begin(), result.bindings.end(), [](const DescriptorBinding& binding) { return binding.role == DescriptorRole::FaultBuffer; }); + }; + const auto samePipeline = [](const RecompileResult& left, const RecompileResult& right) { + return left.variantId == right.variantId && left.specializationId == right.specializationId && left.PipelineVariantId() == right.PipelineVariantId() && left.spirv == right.spirv; + }; + + NestedRequest branch(branchCode, root.data()); + point(payloadAddress); + AgcDriver::ShaderMemory validMemory({}); + const auto valid = validMemory.Capture(branch.request); + require(valid->snapshot.srtPoison.empty() && valid->plan->guardedSrtSlots.size() == 1u && slotPc(*valid->plan, valid->plan->guardedSrtSlots[0]) == 0x2cu && GuardRecords(*valid), "guarded pointer: a mapped nested pointer was poisoned, or its read has no guard"); + require(valid->snapshot.flattenedSrt.at(valid->plan->guardedSrtSlots[0]) == payload, "guarded pointer: a mapped nested read does not hold its word"); + branch.request.context.memory = validMemory.Regions(); + const auto validResult = Recompile(branch.request, *valid); + require(validResult->poisonedSrtReads == 0u && faultBinding(*validResult) && validResult->bdaAbiVersion == BdaAbi::Version, "guarded pointer: a guarded program has no fault buffer"); + + for (const auto pointer : {std::uint64_t{0}, unmapped}) { + for (const auto condition : {1u, 0u}) { + point(pointer); + root[4] = condition; + branch.request.context.memory = {}; + AgcDriver::ShaderMemory memory({}); + const auto capture = memory.Capture(branch.request); + const auto& poison = capture->snapshot.srtPoison; + require(poison.size() == 1u && poison[0].pc == 0x2cu && poison[0].address == pointer && slotPc(*capture->plan, poison[0].slot) == 0x2cu, "guarded pointer: the nested read was not poisoned at its pc and address"); + require(GuardRecords(*capture), "guarded pointer: the poison record is not where the guard reads it"); + const auto regions = memory.Regions(); + require(!RegionsTouch(regions, pointer, 4u) && RegionsCover(regions, root.data() + 6, 8u), "guarded pointer: the captured regions are wrong"); + branch.request.context.memory = regions; + const auto result = Recompile(branch.request, *capture); + require(result->poisonedSrtReads == 1u && faultBinding(*result) && result->bdaAbiVersion == BdaAbi::Version, "guarded pointer: the poisoned capture does not report through the fault buffer"); + require(samePipeline(*result, *validResult), "guarded pointer: poison changed the variant, the specialization or the module"); + const auto replay = Recompile(branch.request); + require(replay.poisonedSrtReads == 1u && samePipeline(replay, *result), "guarded pointer: a replay of the captured regions did not reproduce the poison"); + verifyResult(*result, replay); +#if ANYPS5_ENABLE_SPIRV_TOOLS + static_cast(ValidateAndOptimizeSpirv(result->spirv, branch.request.target.vulkanVersion, branch.request.target.spirvVersion)); +#endif + } + } + root[4] = 1u; + + NestedRequest chain(chainCode, root.data()); + point(unmapped); + AgcDriver::ShaderMemory chainMemory({}); + const auto chained = chainMemory.Capture(chain.request); + const auto& chainPoison = chained->snapshot.srtPoison; + require(chainPoison.size() == 3u && GuardRecords(*chained), "guarded pointer: a pointer read through an unmapped pointer did not poison its two words and the read through it"); + for (const auto& poison : chainPoison) { + const auto own = slotPc(*chained->plan, poison.slot); + const bool pointerWord = own == 0x2cu && (poison.address == unmapped || poison.address == unmapped + 4u); + const bool throughPointer = own == 0x38u && poison.address == unmapped; + require(poison.pc == 0x2cu && (pointerWord || throughPointer), "guarded pointer: a derived poison does not name the inaccessible read"); + } + chain.request.context.memory = chainMemory.Regions(); + const auto chainResult = Recompile(chain.request, *chained); + indirect = 0; + point(static_cast(reinterpret_cast(&indirect))); + AgcDriver::ShaderMemory nullChainMemory({}); + const auto nullChain = nullChainMemory.Capture(chain.request); + require(nullChain->snapshot.srtPoison.size() == 1u && nullChain->snapshot.srtPoison[0].pc == 0x38u && nullChain->snapshot.srtPoison[0].address == 0u && GuardRecords(*nullChain), "guarded pointer: a null pointer read from memory did not poison the read through it"); + chain.request.context.memory = nullChainMemory.Regions(); + const auto nullChainResult = Recompile(chain.request, *nullChain); + indirect = payloadAddress; + AgcDriver::ShaderMemory mappedChainMemory({}); + const auto mappedChain = mappedChainMemory.Capture(chain.request); + require(mappedChain->snapshot.srtPoison.empty() && mappedChain->plan->guardedSrtSlots.size() == 3u && GuardRecords(*mappedChain), "guarded pointer: a mapped pointer chain was poisoned, or its reads have no guards"); + chain.request.context.memory = mappedChainMemory.Regions(); + const auto mappedChainResult = Recompile(chain.request, *mappedChain); + require(chainResult->poisonedSrtReads == 3u && nullChainResult->poisonedSrtReads == 1u && mappedChainResult->poisonedSrtReads == 0u, "guarded pointer: a chain result does not count its poisoned reads"); + require(samePipeline(*chainResult, *mappedChainResult) && samePipeline(*nullChainResult, *mappedChainResult), "guarded pointer: a chain's poison changed the variant, the specialization or the module"); + + NestedRequest descriptor(descriptorCode, root.data()); + point(0u); + AgcDriver::ShaderMemory descriptorMemory({}); + expectFailure([&] { static_cast(descriptorMemory.Capture(descriptor.request)); }, "null or misaligned address", "guarded pointer: a V# loaded through a null nested pointer was accepted"); + NestedRequest rootless(branchCode, nullptr); + AgcDriver::ShaderMemory rootlessMemory({}); + expectFailure([&] { static_cast(rootlessMemory.Capture(rootless.request)); }, "null or misaligned address", "guarded pointer: a null user-data pointer was accepted"); + point(0u); +#ifdef _WIN32 + VirtualFree(reserved, 0, MEM_RELEASE); +#else + munmap(reserved, Reserved); +#endif +} + int main(int argc, char** argv) { try { using namespace ShaderRecompiler; @@ -1793,6 +1980,7 @@ int main(int argc, char** argv) { verifyTwoLaneUniformValues(); verifyBdaReadFallbackFunctions(); verifyFunctionLdsBound(); + verifyGuardedNullPointers(); #if ANYPS5_ENABLE_SPIRV_TOOLS const std::vector minimalSpirv{ 0x07230203u, 0x00010000u, 0u, 5u, 0u, @@ -1979,8 +2167,18 @@ int main(int argc, char** argv) { expectFailure([&] { static_cast(PrepareResourceProgram(request)); }, "shader user data exceeds the scalar register bank", "user data overran scalar register bank"); request.context.userDataBaseRegister = 8; request.context.memory = {}; + AgcDriver::ShaderMemory guarded({}); + const auto poisoned = guarded.Capture(request); + require(poisoned->snapshot.srtPoison.size() == 1u && poisoned->snapshot.srtPoison[0].pc == 8u && poisoned->snapshot.srtPoison[0].address == 0u && GuardRecords(*poisoned), "null nested pointer was not poisoned at its read"); + const auto guardedRegions = guarded.Regions(); + request.context.memory = guardedRegions; + const auto poisonedVertex = Recompile(request, *poisoned); + require(poisonedVertex->poisonedSrtReads == 1u && poisonedVertex->bdaAbiVersion == BdaAbi::Version && poisonedVertex->PipelineVariantId() == first.PipelineVariantId() && std::any_of(poisonedVertex->bindings.begin(), poisonedVertex->bindings.end(), [](const DescriptorBinding& binding) { return binding.role == DescriptorRole::FaultBuffer; }), "a vertex shader reading through a null nested pointer has no fault buffer or another variant"); + request.context.memory = {}; + const std::array nullUserData{}; + request.context.userData = nullUserData; AgcDriver::ShaderMemory invalid({}); - expectFailure([&] { invalid.Capture(request); }, "null or misaligned address", "null nested pointer was accepted"); + expectFailure([&] { invalid.Capture(request); }, "null or misaligned address", "null user-data pointer was accepted"); std::cout << "Shader memory capture, strict validation and deterministic replay passed\n"; return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GuardedNullPointer.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GuardedNullPointer.cpp new file mode 100644 index 000000000..2f71f42a9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GuardedNullPointer.cpp @@ -0,0 +1,244 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#else +#include +#include +#endif + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Lanes = 64; +constexpr std::uint32_t Default = 0x11111111u; +constexpr std::uint32_t Payload = 0x3f800000u; + +alignas(256) std::array Output{}; +alignas(256) std::array Root{}; +alignas(256) std::array Data{Payload, 0u, 0u, 0u}; +alignas(256) std::array Indirect{}; + +alignas(256) constexpr std::array BranchCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xbf068008u, 0xbf850003u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, + 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +alignas(256) constexpr std::array ExecCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0x7da80008u, 0xbf880004u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, + 0x7e02020cu, 0xbefe04c1u, 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +alignas(256) constexpr std::array EmptyExecCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0x7da80008u, 0xbf800000u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, + 0x7e02020cu, 0xbefe04c1u, 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +alignas(256) constexpr std::array ChainCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xf4040385u, 0xfa000000u, 0xbf8cc07fu, 0xbf068008u, 0xbf850003u, 0xf4000307u, 0xfa000000u, + 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +class StderrCapture { +public: + StderrCapture() { + std::fflush(stderr); + file = std::tmpfile(); + Require(file != nullptr, "cannot open a temporary file for stderr"); +#ifdef _WIN32 + saved = _dup(_fileno(stderr)); + _dup2(_fileno(file), _fileno(stderr)); +#else + saved = dup(fileno(stderr)); + dup2(fileno(file), fileno(stderr)); +#endif + } + std::string Finish() { + std::fflush(stderr); +#ifdef _WIN32 + _dup2(saved, _fileno(stderr)); + _close(saved); +#else + dup2(saved, fileno(stderr)); + close(saved); +#endif + std::rewind(file); + std::string text; + char buffer[4096]; + for (std::size_t read = 0; (read = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, read); + std::fclose(file); + std::fwrite(text.data(), 1, text.size(), stderr); + return text; + } + +private: + std::FILE* file = nullptr; + int saved = -1; +}; + +struct Outcome { + std::vector words; + std::string log; + std::uint64_t variantId = 0; + std::uint64_t pipelineVariantId = 0; + std::uint32_t poisoned = 0; + bool cacheable = false; +}; + +template +Outcome Run(AgcDriver::VulkanDevice& device, const std::array& code, std::uint64_t pointer, std::uint32_t guard) { + const auto output = static_cast(reinterpret_cast(Output.data())); + Root = {}; + Root[0] = static_cast(output); + Root[1] = static_cast((output >> 32u) & 0xffffu); + Root[2] = static_cast(sizeof(Output)); + Root[3] = 0x30027facu; + Root[4] = guard; + Root[6] = static_cast(pointer); + Root[7] = static_cast(pointer >> 32u); + std::fill(Output.begin(), Output.end(), 0xdeadbeefu); + const auto root = static_cast(reinterpret_cast(Root.data())); + const std::array userData{static_cast(root), static_cast(root >> 32u)}; + const std::span span(code); + const ShaderRecompiler::ShaderComputeStageInfo compute{{Lanes, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(span.data()), span, 0, {}}, + {64, 0, userData, compute, std::nullopt, std::nullopt, {}}, + device.Target(), + {0, 0, 0, 128} + }; + AgcDriver::ShaderMemory memory({}); + const auto capture = memory.Capture(request); + const auto regions = memory.Regions(); + request.context.memory = regions; + const auto result = ShaderRecompiler::Recompile(request, *capture); + Outcome outcome; + outcome.variantId = result->variantId; + outcome.pipelineVariantId = result->PipelineVariantId(); + outcome.poisoned = result->poisonedSrtReads; + outcome.cacheable = AgcDriver::DriverDetail::CacheableResult(*result); + StderrCapture log; + try { + device.Dispatch(*result, 1, 1, 1, {}, reinterpret_cast(span.data())); + device.WaitIdle(); + } catch (...) { + static_cast(log.Finish()); + throw; + } + outcome.log = log.Finish(); + outcome.words.assign(Output.begin(), Output.end()); + return outcome; +} + +bool Faulted(const Outcome& outcome) { + return outcome.log.find("BDA access failed") != std::string::npos; +} + +std::string Fault(std::uint64_t address, std::uint32_t pc) { + char text[96]; + std::snprintf(text, sizeof(text), "address=0x%llx instruction=0x%x bytes=4", static_cast(address), pc); + return text; +} + +void RequireWords(const Outcome& outcome, std::uint32_t lanes, std::uint32_t inside, const std::string& what) { + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + const auto expected = lane < lanes ? inside : Default; + Require(outcome.words[lane] == expected, what + ": lane " + std::to_string(lane) + " wrote " + std::to_string(outcome.words[lane]) + ", expected " + std::to_string(expected)); + } +} + +void RunTests(AgcDriver::VulkanDevice& device) { + const auto valid = static_cast(reinterpret_cast(Data.data())); + void* reserved = nullptr; +#ifdef _WIN32 + reserved = VirtualAlloc(nullptr, 65536, MEM_RESERVE, PAGE_NOACCESS); +#else + reserved = mmap(nullptr, 65536, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) reserved = nullptr; +#endif + Require(reserved != nullptr, "cannot reserve an inaccessible range"); + const auto unmapped = static_cast(reinterpret_cast(reserved)); + + const auto taken = Run(device, BranchCode, valid, 1u); + Require(!Faulted(taken) && taken.poisoned == 0u && taken.cacheable, "guarded pointer: a mapped pointer faulted, was poisoned or left the caches:\n" + taken.log); + RequireWords(taken, Lanes, Payload, "guarded pointer: a mapped pointer under a taken branch"); + const auto skippedNull = Run(device, BranchCode, 0u, 0u); + Require(!Faulted(skippedNull) && skippedNull.poisoned == 1u && !skippedNull.cacheable, "guarded pointer: a null pointer behind a branch that skips its read faulted or stayed cacheable:\n" + skippedNull.log); + RequireWords(skippedNull, 0u, 0u, "guarded pointer: a skipped read through a null pointer"); + const auto takenNull = Run(device, BranchCode, 0u, 1u); + Require(takenNull.pipelineVariantId == skippedNull.pipelineVariantId && takenNull.pipelineVariantId == taken.pipelineVariantId && takenNull.variantId == taken.variantId, "guarded pointer: the poison or the branch condition changed the pipeline variant"); + Require(Faulted(takenNull) && takenNull.log.find(Fault(0u, 0x2cu)) != std::string::npos && takenNull.log.find("reason=1") != std::string::npos, "guarded pointer: a read through a null pointer that runs did not fault at its pc and address:\n" + takenNull.log); + Require(std::all_of(takenNull.words.begin(), takenNull.words.end(), [](std::uint32_t word) { return word == 0u || word == 0xdeadbeefu; }), "guarded pointer: a faulting read did not return zero"); + const auto takenUnmapped = Run(device, BranchCode, unmapped, 1u); + Require(Faulted(takenUnmapped) && takenUnmapped.log.find(Fault(unmapped, 0x2cu)) != std::string::npos, "guarded pointer: a read through an unmapped pointer that runs did not fault at its address:\n" + takenUnmapped.log); + Require(takenUnmapped.pipelineVariantId == taken.pipelineVariantId, "guarded pointer: an unmapped pointer changed the pipeline variant"); + + const auto someLanes = Run(device, ExecCode, valid, 5u); + Require(!Faulted(someLanes), "guarded pointer: a mapped pointer under EXEC faulted:\n" + someLanes.log); + RequireWords(someLanes, 5u, Payload, "guarded pointer: a mapped pointer read under EXEC"); + const auto noLanes = Run(device, ExecCode, 0u, 0u); + Require(!Faulted(noLanes) && noLanes.poisoned == 1u, "guarded pointer: a null pointer read skipped by s_cbranch_execz faulted:\n" + noLanes.log); + RequireWords(noLanes, 0u, 0u, "guarded pointer: a read skipped by s_cbranch_execz"); + const auto oneLane = Run(device, ExecCode, 0u, 1u); + Require(Faulted(oneLane) && oneLane.log.find(Fault(0u, 0x34u)) != std::string::npos, "guarded pointer: a null pointer read with one EXEC lane did not fault:\n" + oneLane.log); + const auto emptyMapped = Run(device, EmptyExecCode, valid, 0u); + Require(!Faulted(emptyMapped), "guarded pointer: a mapped pointer read with an empty EXEC faulted:\n" + emptyMapped.log); + RequireWords(emptyMapped, 0u, 0u, "guarded pointer: a mapped pointer read with an empty EXEC"); + const auto emptyNull = Run(device, EmptyExecCode, 0u, 0u); + Require(Faulted(emptyNull) && emptyNull.log.find(Fault(0u, 0x34u)) != std::string::npos, "guarded pointer: a null pointer read that runs with an empty EXEC did not fault:\n" + emptyNull.log); + Require(noLanes.pipelineVariantId == someLanes.pipelineVariantId && oneLane.pipelineVariantId == someLanes.pipelineVariantId && emptyNull.pipelineVariantId == emptyMapped.pipelineVariantId, "guarded pointer: the poison changed an EXEC program's pipeline variant"); + + const auto chainSkipped = Run(device, ChainCode, 0u, 0u); + Require(chainSkipped.poisoned == 3u && Faulted(chainSkipped) && chainSkipped.log.find(Fault(0u, 0x24u)) != std::string::npos, "guarded pointer: a pointer read through a null pointer, used only by a skipped read, did not fault at its own pc:\n" + chainSkipped.log); + const auto chainTaken = Run(device, ChainCode, 0u, 1u); + Require(Faulted(chainTaken) && chainTaken.log.find(Fault(0u, 0x24u)) != std::string::npos, "guarded pointer: a read through a pointer read through a null pointer did not fault at the first read:\n" + chainTaken.log); + Indirect[0] = valid; + const auto chainMapped = Run(device, ChainCode, static_cast(reinterpret_cast(Indirect.data())), 1u); + Require(!Faulted(chainMapped) && chainMapped.poisoned == 0u && chainMapped.cacheable && chainMapped.pipelineVariantId == chainTaken.pipelineVariantId && chainSkipped.pipelineVariantId == chainTaken.pipelineVariantId, "guarded pointer: a chain's poison changed its pipeline variant, or a mapped chain faulted:\n" + chainMapped.log); + RequireWords(chainMapped, Lanes, Payload, "guarded pointer: a read through a mapped pointer chain"); + +#ifdef _WIN32 + VirtualFree(reserved, 0, MEM_RELEASE); +#else + munmap(reserved, 65536); +#endif +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + RunTests(*device); + std::puts("guarded null pointer tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index 5447bef4e..eed5b9551 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -50,6 +50,14 @@ struct SrtRead { bool operator==(const SrtRead& other) const = default; }; +struct SrtReadPoison { + std::uint32_t slot = 0; + std::uint32_t pc = 0; + std::uint64_t address = 0; + + bool operator==(const SrtReadPoison& other) const = default; +}; + struct ResourceBlock { IrValue* condition = nullptr; std::vector successors; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp index a59dbab67..953dacc16 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp @@ -41,6 +41,7 @@ struct ResourceSnapshot { std::vector flattenedSrt; std::vector userData; UniformFill uniformFill; + std::vector srtPoison; }; struct UniformFillPlan { @@ -63,6 +64,8 @@ struct IrResourcePlan { std::vector controlFlow; std::vector materializationSources; std::vector srtReads; + std::vector guardedSrtSlots; + std::uint32_t srtGuardOffset = 0; std::vector cleanFlatSlots; // One byte per srtReads slot, 1 when the CPU walk never consumes the slot's value (see // Detail::ComputePureFlatSlots): a driver may reuse a capture whose words differ only there. diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp index 1a3f4f790..ca81c0fec 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp @@ -33,6 +33,7 @@ struct ShaderInfo { bool hasBitwiseXor = false; bool usesDma = false; bool bdaWrites = false; + bool usesFaultBuffer = false; bool dispatchThreadLimit = false; bool operator==(const ShaderInfo& other) const = default; diff --git a/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp b/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp index 5abfd7503..321474d50 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp @@ -16,6 +16,7 @@ public: private: static bool ReadGuestMemory(void* userContext, std::uint64_t address, std::uint32_t* value); + static bool Covered(void* userContext, std::uint64_t address, std::uint64_t bytes); std::vector regions; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp index f5f5a4070..16cdedac5 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp @@ -11,6 +11,7 @@ namespace ShaderRecompiler { using SrtMemoryReader = bool (*)(void* userData, std::uint64_t address, std::uint32_t* value); +using SrtMemoryProbe = bool (*)(void* userData, std::uint64_t address, std::uint64_t bytes); // The guest addresses a walk dereferenced (Detail::Evaluator::EvaluateRawRead): the leaf read of // each pure flat slot (IrResourcePlan::pureFlatSlots) as (flat offset, address), set by the @@ -30,6 +31,7 @@ struct SrtRuntime { void* userContext = nullptr; SrtMemoryReader readSpecializationMemory = nullptr; SrtReadTrace* readTrace = nullptr; + SrtMemoryProbe accessible = nullptr; }; enum class RuntimeValueType { @@ -44,7 +46,7 @@ public: void EvaluateUniformValues(const IrResourcePlan& program, std::span values, const SrtRuntime& runtime, std::span results) const; void EvaluateDescriptorSource(const IrResourcePlan& program, std::uint32_t source, const SrtRuntime& runtime, DescriptorValue& result) const; void EvaluateDescriptorSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results) const; - void EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources) const; + void EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison = nullptr) const; void Walk(const IrResourcePlan& program, const SrtRuntime& runtime, std::vector& flat) const; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp index c1b008e9b..eb9c568ad 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp @@ -15,7 +15,7 @@ namespace ShaderRecompiler::Detail { // Why the last EvaluateRuntimeSourcesImpl call on this thread returned false. const std::string& RuntimeSourceFailureReason(); -bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources); +bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison = nullptr); } diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp index 7877c5928..269bfceb4 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp @@ -66,12 +66,18 @@ private: std::size_t _count = 0; }; +struct InaccessibleRead { + const IrValue* read = nullptr; + std::uint64_t address = 0; +}; + class Evaluator { public: Evaluator(const IrResourcePlan& program, const SrtRuntime& runtime, std::span cleanFlatSlots = {}, Evaluator* cleanEvaluator = nullptr, IrValue* activeMask = nullptr) : _program(program), _runtime(runtime), _cleanFlatSlots(cleanFlatSlots), _cleanEvaluator(cleanEvaluator), _activeMask(activeMask != nullptr ? activeMask->Resolve() : nullptr) {} bool Evaluate(IrValue* value, std::uint32_t& result); bool EvaluateWide(IrValue* raw, std::uint64_t& result); + void ReportInaccessibleReads(InaccessibleRead* sink) { _inaccessible = sink; } private: static float Float32(std::uint64_t bits); @@ -88,6 +94,7 @@ private: std::span _cleanFlatSlots; Evaluator* _cleanEvaluator = nullptr; IrValue* _activeMask = nullptr; + InaccessibleRead* _inaccessible = nullptr; EvaluatedValues _cache; std::vector _visiting; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp index f1f806f08..85bd0e7ab 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp @@ -16,6 +16,7 @@ namespace ShaderRecompiler::Detail { // fill) has no pure slot. Reachability over-approximates what the evaluators consume, so an // unknown opcode only makes more slots impure. std::vector ComputePureFlatSlots(const IrResourcePlan& plan); +std::vector ComputeGuardedFlatSlots(const IrResourcePlan& plan); } diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp index e4ab1ce06..00c598e3b 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp @@ -6,6 +6,7 @@ namespace ShaderRecompiler::Detail { bool IsRawRead(const IrResourcePlan& program, const IrValue& inst); +bool LoadedFromMemory(const IrValue& handle); bool IsDescriptorHandle(IrOpcode opcode); bool IsRuntimeSelect(IrOpcode opcode); bool IsRuntimeUniformOp(IrOpcode opcode); diff --git a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp index 7d18b8a79..518ac228e 100644 --- a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp +++ b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp @@ -161,6 +161,8 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin } if (info.usesDma) { addBinding(next, DescriptorBindingKind::BdaPagetable); + } + if (info.usesDma || info.usesFaultBuffer) { addBinding(next, DescriptorBindingKind::FaultBuffer); } diff --git a/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp b/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp index 249778f06..58981be3a 100644 --- a/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp +++ b/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp @@ -64,6 +64,29 @@ bool RequestMemoryView::ReadGuestMemory(void* userContext, std::uint64_t address return true; } +bool RequestMemoryView::Covered(void* userContext, std::uint64_t address, std::uint64_t bytes) { + const auto* self = static_cast(userContext); + if (bytes > std::numeric_limits::max() - address) { + return false; + } + const std::uint64_t end = address + bytes; + while (address < end) { + const auto it = std::upper_bound(self->regions.begin(), self->regions.end(), address, [](std::uint64_t addressValue, const MemoryRegion& region) { + return addressValue < region.guestAddress; + }); + if (it == self->regions.begin()) { + return false; + } + const MemoryRegion& region = *std::prev(it); + const std::uint64_t regionEnd = region.guestAddress + region.bytes.size(); + if (address >= regionEnd) { + return false; + } + address = regionEnd; + } + return true; +} + SrtRuntime RequestMemoryView::MakeRuntime(std::span userData, std::uint64_t shaderBase) { SrtRuntime runtime; runtime.userData = userData; @@ -71,6 +94,7 @@ SrtRuntime RequestMemoryView::MakeRuntime(std::span userDat runtime.userContext = this; runtime.readMemory = &ReadGuestMemory; runtime.readSpecializationMemory = &ReadGuestMemory; + runtime.accessible = &Covered; return runtime; } diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 1623afd77..fe2061160 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -444,7 +444,7 @@ void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime, std::vector values; std::vector activeSources; - walker.EvaluateRuntimeSources(plan, plan.materializationSources, runtime, values, snapshot.flattenedSrt, plan.cleanFlatSlots, activeSources); + walker.EvaluateRuntimeSources(plan, plan.materializationSources, runtime, values, snapshot.flattenedSrt, plan.cleanFlatSlots, activeSources, &snapshot.srtPoison); std::size_t cursor = 0; if (values.size() < plan.info.buffers.size()) { @@ -593,6 +593,19 @@ void materializeTables(const IrResourcePlan& plan, ResourceSnapshot& snapshot, c } } +void materializeSrtGuards(const IrResourcePlan& plan, ResourceSnapshot& snapshot) { + const auto& guarded = plan.guardedSrtSlots; + const auto flags = snapshot.flattenedSrt.size(); + snapshot.flattenedSrt.resize(flags + guarded.size(), 0u); + for (std::uint32_t record = 0u; record < snapshot.srtPoison.size(); ++record) { + const auto& poison = snapshot.srtPoison[record]; + const auto found = std::lower_bound(guarded.begin(), guarded.end(), poison.slot); + if (found == guarded.end() || *found != poison.slot) throw std::runtime_error("SRT read at flat offset " + std::to_string(poison.slot) + " reads inaccessible memory but has no guard"); + snapshot.flattenedSrt[flags + static_cast(found - guarded.begin())] = record + 1u; + snapshot.flattenedSrt.insert(snapshot.flattenedSrt.end(), {poison.pc, static_cast(poison.address), static_cast(poison.address >> 32u)}); + } +} + } std::vector ResourceMaterializer::RuntimeImageModes(const ImageResource& image) { @@ -780,6 +793,9 @@ void ResourceMaterializer::ApplyStaticInterface(IrProgram& program, bool nativeS sampler.depthCompare = sampler.depthCompare || images[pair.image].depthCompare; } resources.info.images = std::move(images); + resources.guardedSrtSlots = Detail::ComputeGuardedFlatSlots(resources); + resources.srtGuardOffset = mappingOffset; + if (!resources.guardedSrtSlots.empty()) resources.info.usesFaultBuffer = true; PrepareImageModes(resources.info); } @@ -893,6 +909,7 @@ IrResourcePlan ResourceMaterializer::ExtractPlan(const IrProgram& program) const } for (const auto& sampler : plan.info.samplers) addSource(sampler.source); plan.pureFlatSlots = Detail::ComputePureFlatSlots(plan); + plan.guardedSrtSlots = Detail::ComputeGuardedFlatSlots(plan); return plan; } @@ -915,6 +932,7 @@ void ResourceMaterializer::Materialize(const IrResourcePlan& program, const SrtR } const auto started = MaterializeProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; materializeTables(plan, nextSnapshot, tables); + materializeSrtGuards(plan, nextSnapshot); if (MaterializeProfiled()) specializationNanoseconds.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed); snapshot = std::move(nextSnapshot); reportBindless(); diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp index 7ea4c4a78..2ef278ff6 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp @@ -44,7 +44,7 @@ const DescriptorSource* Source(const IrResourcePlan& program, std::uint32_t sour } -bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources) { +bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison) { failureReason().clear(); static const bool debug = std::getenv("APS5_SRT_DEBUG") != nullptr; if (debug) { @@ -117,8 +117,14 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span flattened; + std::vector poisoned; if (evaluateFlat) { flattened.resize(program.srtReads.size()); + InaccessibleRead inaccessible; + if (poison != nullptr) { + evaluator.ReportInaccessibleReads(&inaccessible); + cleanEvaluator.ReportInaccessibleReads(&inaccessible); + } for (const auto& read : program.srtReads) { const bool clean = read.flatOffset < cleanFlatSlots.size() && cleanFlatSlots[read.flatOffset] != 0u; auto& selected = clean ? cleanEvaluator : evaluator; @@ -131,8 +137,15 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::spanleaf = read.value->Resolve(); trace->leafSlot = read.flatOffset; } + inaccessible = {}; const bool evaluated = read.flatOffset < flattened.size() && selected.Evaluate(read.value, flattened[read.flatOffset]); if (pure) trace->leaf = nullptr; + if (!evaluated && inaccessible.read != nullptr) { + if (debug) std::fprintf(stderr, "[srt] flat offset %u reads inaccessible 0x%llx at pc 0x%x: poisoned\n", read.flatOffset, static_cast(inaccessible.address), inaccessible.read->Flags().pc); + flattened[read.flatOffset] = 0u; + poisoned.push_back({read.flatOffset, inaccessible.read->Flags().pc, inaccessible.address}); + continue; + } if (!evaluated) { return Fail(std::string(clean ? "clean " : "") + "SRT read at flat offset " + std::to_string(read.flatOffset) + ": " + DescribeValue(read.value, 4)); } @@ -143,6 +156,9 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::spanleaf) trace->leaves.emplace_back(trace->leafSlot, address); else trace->otherReads.push_back(address); @@ -187,6 +191,7 @@ bool Evaluator::EvaluateInst(IrValue& inst, std::uint64_t& result) { case IrOpcode::Phi: return EvaluatePhi(inst, result); case IrOpcode::ReadFirstLane: { Evaluator active(_program, _runtime, _cleanFlatSlots, _cleanEvaluator, inst.Argument(1)); + active.ReportInaccessibleReads(_inaccessible); return active.EvaluateWide(inst.Argument(0), result); } case IrOpcode::BitCastU32F32: diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp index d0b787b7f..cca8ad626 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp @@ -1,4 +1,5 @@ #include "Optimization/SrtWalker/SrtFlatSlotClasses.hpp" +#include "Optimization/SrtWalker/SrtInstructionPredicates.hpp" #include #include @@ -61,4 +62,47 @@ std::vector ComputePureFlatSlots(const IrResourcePlan& plan) { return pure; } +std::vector ComputeGuardedFlatSlots(const IrResourcePlan& plan) { + std::unordered_map> users; + std::unordered_set seen; + std::vector pending; + std::vector poisonable; + for (const auto& read : plan.srtReads) { + if (read.value != nullptr) pending.push_back(read.value->Resolve()); + } + while (!pending.empty()) { + const IrValue* value = pending.back(); + pending.pop_back(); + if (!seen.insert(value).second) continue; + const IrValue* handle = value->ArgumentCount() != 0u && value->Argument(0) != nullptr ? value->Argument(0)->Resolve() : nullptr; + if (IsRawRead(plan, *value) && handle != nullptr && LoadedFromMemory(*handle)) poisonable.push_back(value); + const auto follow = [&](const IrValue* argument) { + if (argument == nullptr) return; + argument = argument->Resolve(); + users[argument].push_back(value); + pending.push_back(argument); + }; + for (std::size_t i = 0; i < value->ArgumentCount(); ++i) follow(value->Argument(i)); + if (value->Opcode() == IrOpcode::ReadConst && value->ArgumentCount() >= 2 && value->Argument(1) != nullptr) { + const IrValue* slot = value->Argument(1)->Resolve(); + if (slot->HasImmediate() && slot->Type() == IrType::U32 && slot->ImmediateU32() < plan.srtReads.size()) follow(plan.srtReads[slot->ImmediateU32()].value); + } + } + std::unordered_set reaching(poisonable.begin(), poisonable.end()); + while (!poisonable.empty()) { + const IrValue* value = poisonable.back(); + poisonable.pop_back(); + const auto found = users.find(value); + if (found == users.end()) continue; + for (const IrValue* user : found->second) { + if (reaching.insert(user).second) poisonable.push_back(user); + } + } + std::vector guarded; + for (std::uint32_t slot = 0; slot < plan.srtReads.size(); ++slot) { + if (plan.srtReads[slot].value != nullptr && reaching.contains(plan.srtReads[slot].value->Resolve())) guarded.push_back(slot); + } + return guarded; +} + } diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp index 46735d2cc..274524be7 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp @@ -1,5 +1,8 @@ #include "Optimization/SrtWalker/SrtInstructionPredicates.hpp" +#include +#include + namespace ShaderRecompiler::Detail { bool IsRawRead(const IrResourcePlan& program, const IrValue& inst) { @@ -15,6 +18,24 @@ bool IsRawRead(const IrResourcePlan& program, const IrValue& inst) { return (op == IrOpcode::LoadAddressU32 && kind == ResourceKind::ScalarAddress) || (op == IrOpcode::ReadConstBuffer && kind == ResourceKind::ScalarBuffer); } +bool LoadedFromMemory(const IrValue& handle) { + std::vector pending; + std::vector visited; + for (std::size_t index = 0; index < handle.ArgumentCount(); ++index) pending.push_back(handle.Argument(index)); + while (!pending.empty()) { + const IrValue* value = pending.back(); + pending.pop_back(); + if (value == nullptr) continue; + value = value->Resolve(); + if (value->HasImmediate() || std::find(visited.begin(), visited.end(), value) != visited.end()) continue; + visited.push_back(value); + const auto opcode = value->Opcode(); + if (opcode == IrOpcode::ReadConst || opcode == IrOpcode::LoadAddressU32 || opcode == IrOpcode::ReadConstBuffer) return true; + for (std::size_t index = 0; index < value->ArgumentCount(); ++index) pending.push_back(value->Argument(index)); + } + return false; +} + bool IsDescriptorHandle(IrOpcode opcode) { switch (opcode) { case IrOpcode::GetBufferResource: diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp index 2000b1c90..4e8a85d89 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp @@ -49,8 +49,8 @@ void SrtWalker::EvaluateDescriptorSources(const IrResourcePlan& program, std::sp } } -void SrtWalker::EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources) const { - if (!Detail::EvaluateRuntimeSourcesImpl(program, sources, runtime, results, flat, true, cleanFlatSlots, activeSources)) { +void SrtWalker::EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison) const { + if (!Detail::EvaluateRuntimeSourcesImpl(program, sources, runtime, results, flat, true, cleanFlatSlots, activeSources, poison)) { throw std::runtime_error("SrtWalker::EvaluateRuntimeSources failed to evaluate runtime sources: " + Detail::RuntimeSourceFailureReason()); } } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 358b31a24..92017bf68 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -344,7 +344,7 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra result.variantId = nextVariantId(); result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions); - result.bdaAbiVersion = program.Info().usesDma ? request.target.bdaAbiVersion : 0u; + result.bdaAbiVersion = program.Info().usesDma || program.Info().usesFaultBuffer ? request.target.bdaAbiVersion : 0u; result.memoryOffsetDword = bindings.layout.memoryOffsetDword; result.shaderDataDwords = bindings.layout.ShaderDataDwords(); result.imageMetadataDword = bindings.layout.ImageMetadataDword(); @@ -680,6 +680,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil DescriptorBindingBuilder{}.Populate(bindings, variant.bindings, *bindingPlan, variant.info.userDataBase, snapshot, partialThreads(request)); result.bindings = std::move(bindings.bindings); result.pushConstants = std::move(bindings.pushConstants); + result.poisonedSrtReads = static_cast(snapshot.srtPoison.size()); return result; } diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 3ca2409ea..603d7b937 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -462,6 +462,7 @@ struct ShaderInvocation { std::vector bindings; std::vector pushConstants; std::vector vertexAttributes; + std::uint32_t poisonedSrtReads = 0; }; struct RecompileResult : CompiledShaderArtifact, ShaderInvocation { diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 10a60c556..40f10ab42 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -65,14 +65,14 @@ namespace ShaderRecompiler::ShaderDiskCache { #if defined(__linux__) && defined(__x86_64__) && defined(__GLIBCXX__) static_assert(sizeof(CompiledShaderArtifact) == 184, "CompiledShaderArtifact changed: update the artifact encoder"); -static_assert(sizeof(ShaderInvocation) == 104, "ShaderInvocation changed: update the invocation encoder"); +static_assert(sizeof(ShaderInvocation) == 112, "ShaderInvocation changed: update the invocation encoder"); static_assert(sizeof(RecompileResult) == 296, "RecompileResult changed: update EncodeResult and DecodeResult"); static_assert(sizeof(DescriptorBinding) == 448, "DescriptorBinding changed: update the binding encoder"); static_assert(sizeof(VertexAttribute) == 32, "VertexAttribute changed: update the attribute encoder"); static_assert(sizeof(VertexInput) == 16, "VertexInput changed: update the vertex input encoder"); static_assert(sizeof(FragmentParameter) == 12, "FragmentParameter changed: update the parameter encoder"); -static_assert(sizeof(CompiledShaderInfo) == 336, "CompiledShaderInfo changed: update the info encoder"); -static_assert(sizeof(ShaderInfo) == 224, "ShaderInfo changed: update the info encoder"); +static_assert(sizeof(CompiledShaderInfo) == 344, "CompiledShaderInfo changed: update the info encoder"); +static_assert(sizeof(ShaderInfo) == 232, "ShaderInfo changed: update the info encoder"); static_assert(sizeof(BufferResource) == 32, "BufferResource changed: update the info encoder"); static_assert(sizeof(ImageResource) == 112, "ImageResource changed: update the info encoder"); static_assert(sizeof(SamplerResource) == 16, "SamplerResource changed: update the info encoder"); @@ -374,6 +374,7 @@ void encodeInvocation(Writer& writer, const ShaderInvocation& invocation) { out.Value(attribute.fetchIndex); out.Value(attribute.formatComponents); }); + writer.Value(invocation.poisonedSrtReads); } void decodeInvocation(Reader& reader, ShaderInvocation& invocation) { @@ -394,6 +395,7 @@ void decodeInvocation(Reader& reader, ShaderInvocation& invocation) { in.Value(attribute.fetchIndex); in.Value(attribute.formatComponents); }); + reader.Value(invocation.poisonedSrtReads); } void encodeLayout(Writer& writer, const IrBindingLayout& layout) { @@ -518,6 +520,7 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { writer.Value(info.hasBitwiseXor); writer.Value(info.usesDma); writer.Value(info.bdaWrites); + writer.Value(info.usesFaultBuffer); writer.Value(info.dispatchThreadLimit); encodeLayout(writer, compiled.bindings); } @@ -620,6 +623,7 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { reader.Value(info.hasBitwiseXor); reader.Value(info.usesDma); reader.Value(info.bdaWrites); + reader.Value(info.usesFaultBuffer); reader.Value(info.dispatchThreadLimit); decodeLayout(reader, compiled.bindings); } diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index d8aa5338d..a9262a584 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -12,7 +12,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 20; +inline constexpr std::uint32_t FormatVersion = 21; enum class LoadStatus { Loaded, diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp index 4b68725c7..501ef58fc 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp @@ -16,6 +16,7 @@ std::uint32_t BdaLoadAddress(SpirvEmitterState& state, std::uint32_t index); void DefineBdaFaultFunction(SpirvEmitterState& state); void DefineBdaDwordReadFunctions(SpirvEmitterState& state); void RecordBdaFault(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); +void RecordBdaFaultWords(SpirvEmitterState& state, std::uint32_t addressLow, std::uint32_t addressHigh, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ReturnBdaFailureIf(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& target); // Whether a faulting BDA access may end its invocation. Programs with workgroup barriers must keep diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp index 411f7fc0a..ca8c1a267 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp @@ -29,16 +29,20 @@ std::uint32_t BdaLoadAddress(SpirvEmitterState& state, std::uint32_t index) { void DefineBdaFaultFunction(SpirvEmitterState& state) { const auto u32 = TypeU32(state); - const auto u64 = TypeScalarU64(state); + const bool words = !state.program.Info().usesDma; + const auto addressType = words ? u32 : TypeScalarU64(state); state.bdaFaultFunction = state.module.AllocateId(); state.module.AddName(state.bdaFaultFunction, "record_bda_fault"); - state.module.AddFunction(spv::OpFunction, TypeVoid(state), state.bdaFaultFunction, spv::FunctionControlDontInlineMask, state.module.Type(spv::OpTypeFunction, TypeVoid(state), u32, u64, u32, u32)); + const auto type = words ? state.module.Type(spv::OpTypeFunction, TypeVoid(state), u32, addressType, u32, u32, u32) : state.module.Type(spv::OpTypeFunction, TypeVoid(state), u32, addressType, u32, u32); + state.module.AddFunction(spv::OpFunction, TypeVoid(state), state.bdaFaultFunction, spv::FunctionControlDontInlineMask, type); const auto reason = state.module.AllocateId(); const auto address = state.module.AllocateId(); + const auto addressHigh = words ? state.module.AllocateId() : 0u; const auto bytes = state.module.AllocateId(); const auto instruction = state.module.AllocateId(); state.module.AddFunction(spv::OpFunctionParameter, u32, reason); - state.module.AddFunction(spv::OpFunctionParameter, u64, address); + state.module.AddFunction(spv::OpFunctionParameter, addressType, address); + if (words) state.module.AddFunction(spv::OpFunctionParameter, u32, addressHigh); state.module.AddFunction(spv::OpFunctionParameter, u32, bytes); state.module.AddFunction(spv::OpFunctionParameter, u32, instruction); EmitLabel(state, state.module.AllocateId()); @@ -53,8 +57,8 @@ void DefineBdaFaultFunction(SpirvEmitterState& state) { state.module.AddFunction(spv::OpStore, BdaWord(state, state.faultBufferVariable, ConstantU32(state, index)), value); }; store(1, reason); - store(2, Unary(state, spv::OpUConvert, TypeU32(state), address)); - store(3, Unary(state, spv::OpUConvert, TypeU32(state), Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, 32u)))); + store(2, words ? address : Unary(state, spv::OpUConvert, TypeU32(state), address)); + store(3, words ? addressHigh : Unary(state, spv::OpUConvert, TypeU32(state), Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, 32u)))); store(4, bytes); store(5, ConstantU32(state, static_cast(state.program.Resources().stage))); store(6, instruction); @@ -67,10 +71,21 @@ void DefineBdaFaultFunction(SpirvEmitterState& state) { } void RecordBdaFault(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason) { - if (state.bdaFaultFunction == 0) throw std::runtime_error("BDA fault function is missing"); + if (state.bdaFaultFunction == 0 || !state.program.Info().usesDma) throw std::runtime_error("BDA fault function is missing"); state.module.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.module.AllocateId(), state.bdaFaultFunction, ConstantU32(state, static_cast(reason)), address, bytes, instruction); } +void RecordBdaFaultWords(SpirvEmitterState& state, std::uint32_t addressLow, std::uint32_t addressHigh, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason) { + if (state.program.Info().usesDma) { + const auto u64 = TypeScalarU64(state); + const auto high = Binary(state, spv::OpShiftLeftLogical, u64, Unary(state, spv::OpUConvert, u64, addressHigh), BdaConstant(state, 32u)); + RecordBdaFault(state, Binary(state, spv::OpBitwiseOr, u64, Unary(state, spv::OpUConvert, u64, addressLow), high), bytes, instruction, reason); + return; + } + if (state.bdaFaultFunction == 0) throw std::runtime_error("BDA fault function is missing"); + state.module.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.module.AllocateId(), state.bdaFaultFunction, ConstantU32(state, static_cast(reason)), addressLow, addressHigh, bytes, instruction); +} + void ReturnBdaFailureIf(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason) { const auto failed = state.module.AllocateId(); const auto next = state.module.AllocateId(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp index e6d905db3..77701040f 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp @@ -157,8 +157,8 @@ static std::uint32_t DefineBdaNoteWrite(SpirvEmitterState& state) { } void DefineGetBdaPointer(SpirvEmitterState& state) { + if (state.faultBufferVariable != 0) DefineBdaFaultFunction(state); if (!state.program.Info().usesDma) return; - DefineBdaFaultFunction(state); state.bdaPointerFunction = DefineBdaLookup(state, "get_bda_pointer", true); if (!BdaByteReadsForced()) state.bdaProbeFunction = DefineBdaLookup(state, "probe_bda_pointer", false); DefineBdaDwordReadFunctions(state); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index 60df7b542..5b37506a5 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -167,6 +167,7 @@ std::uint32_t AddBdaImmediate(SpirvValueEmitContext& ctx, const IrValue& inst, s } void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& target) { + if (program.Info().usesFaultBuffer && target.bdaAbiVersion != BdaAbi::Version) throw std::runtime_error("the fault buffer needs the BDA fault ABI, which the target lacks"); if (!program.Info().usesDma) return; if (target.bdaAbiVersion != BdaAbi::Version) throw std::runtime_error("unsupported BDA ABI version"); for (const auto capability : {spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index c08132cb3..8dc88285f 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -85,7 +85,8 @@ void EmitReturnTerminator(SpirvValueEmitContext& ctx) { const auto pc = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, TypeU32(state), pc, state.loopGuardPc); EmitIfCondition(state, Binary(state, spv::OpINotEqual, TypeBool(state), pc, ConstantU32(state, 0u)), [&] { - RecordBdaFault(state, BdaConstant(state, state.program.Resources().shaderHash), ConstantU32(state, state.loopGuardLimit), EmitBinaryU32(state, spv::OpISub, pc, ConstantU32(state, 1u)), BdaAbi::FaultReason::LoopLimit); + const auto hash = state.program.Resources().shaderHash; + RecordBdaFaultWords(state, ConstantU32(state, static_cast(hash)), ConstantU32(state, static_cast(hash >> 32u)), ConstantU32(state, state.loopGuardLimit), EmitBinaryU32(state, spv::OpISub, pc, ConstantU32(state, 1u)), BdaAbi::FaultReason::LoopLimit); }); } if (OrderedPixelShader(state)) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index 4ef487bfc..a0ab53817 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -1020,6 +1020,41 @@ std::uint32_t EmitReadConst(SpirvValueEmitContext& ctx, const IrValue& inst) { if (state.flattenedSrtVariable == 0) { ctx.Fail(inst, "requires the flattened SRT descriptor"); } + const auto& resources = state.program.Resources(); + const auto& guarded = resources.guardedSrtSlots; + if (!guarded.empty()) { + const IrValue* slot = inst.Argument(1)->Resolve(); + if (!slot->HasImmediate()) { + ctx.Fail(inst, "reads the flattened SRT at a runtime slot while some slots are guarded"); + } + const auto found = std::lower_bound(guarded.begin(), guarded.end(), slot->ImmediateU32()); + if (found != guarded.end() && *found == slot->ImmediateU32()) { + if (state.faultBufferVariable == 0) { + ctx.Fail(inst, "reads a guarded SRT slot without a fault buffer"); + } + const auto u32 = TypeU32(state); + const auto load = [&](std::uint32_t index) { + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.flattenedSrtVariable, ConstantU32(state, 0u), index); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, u32, value, pointer); + return value; + }; + const auto guard = static_cast(found - guarded.begin()); + const auto flag = load(ConstantU32(state, resources.srtGuardOffset + guard)); + const auto poisoned = Binary(state, spv::OpINotEqual, TypeBool(state), flag, ConstantU32(state, 0u)); + EmitIfCondition(state, poisoned, [&] { + const auto records = resources.srtGuardOffset + static_cast(guarded.size()); + const auto record = Binary(state, spv::OpIAdd, u32, ConstantU32(state, records), Binary(state, spv::OpIMul, u32, Binary(state, spv::OpISub, u32, flag, ConstantU32(state, 1u)), ConstantU32(state, 3u))); + const auto word = [&](std::uint32_t offset) { return load(Binary(state, spv::OpIAdd, u32, record, ConstantU32(state, offset))); }; + const auto pc = word(0u); + const auto low = word(1u); + const auto high = word(2u); + RecordBdaFaultWords(state, low, high, ConstantU32(state, 4u), pc, BdaAbi::FaultReason::Unmapped); + }); + StopBdaInvocationIf(state, poisoned); + } + } const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.flattenedSrtVariable, ConstantU32(state, 0u), ctx.Arg(inst, 1)); const auto value = state.module.AllocateId(); diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 67b1dc406..29b3078fe 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -271,6 +271,9 @@ Throughout the project, every function at every stage either **does exactly what - [Samples and gathers](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8_SRGB and 8_8_SRGB textures throw on a device that cannot sample R8_SRGB or R8G8_SRGB. - [Bindless image table](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) entries of 8_SRGB and 8_8_SRGB textures read zeros, like converted formats, on a device that cannot sample R8_SRGB or R8G8_SRGB. - The shader recompiler [ignores `s_setreg_b32`](../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp): writes to MODE (float rounding and denormal controls) and the other hardware registers have no effect, and `s_getreg_b32` reads of the MODE round fields report round to nearest even even after `s_setreg_b32` wrote another mode. The initial round mode from the shader's program registers is not read either. +- A [flat SRT read](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) through a null or unmapped pointer loaded from memory raises a [BDA fault](../../core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp) only if the program executes it, then stops or reads zero like a BDA read. +- Repeated `s_load`s of one [flat SRT](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) dword through a null or unmapped pointer report one of their pcs, and an `s_load` whose inaccessible dwords are all unused, or only address repeated loads, reports no fault of its own. +- A value read through a null or unmapped pointer still throws at record time if a [descriptor, branch condition, uniform fill or table base](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) needs it, or if the pointer comes from user data, even if the console skips the read. - On a host subgroup narrower than 32 lanes, [`threadBit`](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) reads an EXEC written as a value (`s_mov_b32`/`s_mov_b64`) at bit (subgroup invocation id) of EXEC_LO, so lanes at or above the subgroup size read a lower lane's bit. - [EXEC read as a value](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) (`s_mov_b64 sN, exec`, the `s_*_saveexec` copy, ALU sources) holds only lanes with a host invocation; the console's also holds the wave's other lanes, e.g. 32-63 of a wave64 on 32 host lanes. - [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) and fetch-inactive `v_permlane16_b32`/`v_permlanex16_b32` read undefined data from a lane that the host subgroup does not hold or whose host invocation is inactive; the console reads the lane's VGPR.