diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 069a14e85..77678af84 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -559,6 +559,64 @@ void verifyDescriptorPhis() { auto dynamic = makeRequest(dynamicCode); expectFailure([&] { static_cast(GetResourcePlan(dynamic)); }, "GetSamplerResource dword 0 is not a valid runtime value", "descriptor Phi: an edge without an SRT slot was accepted"); + + const std::vector loopCode{0xf4080500u, 0xfa000020u, 0xf4080600u, 0xfa000040u, 0xf40c0200u, 0xfa000000u, 0xbe910380u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf068011u, 0xbf850003u, 0xf40c0200u, 0xfa000000u, 0xbf820002u, 0xf40c0200u, 0xfa000060u, 0xbf8cc07fu, 0x80118111u, 0xbf0a8211u, 0xbf85fff0u, 0xbf810000u}; + const auto loopCapture = compile(loopCode, 2u, 1u); + const auto& loopImages = loopCapture->snapshot.images; + const auto holdsLoopImage = [&](const std::array& words) { + return std::ranges::any_of(loopImages, [&](const DescriptorValue& value) { + return std::equal(words.begin(), words.end(), value.dwords.begin()); + }); + }; + require(loopImages.size() == 2u && holdsLoopImage(first) && holdsLoopImage(second), "descriptor Phi: the loop's chained T# Phis were not split into the two SRT T#s"); + + const auto entryWrites = [&](std::initializer_list words) { + auto code = loopCode; + code.insert(code.begin() + 8, words); + return code; + }; + const std::array, const char*>, 3> invalidEntries{{ + {entryWrites({0xbe8e1f00u}), "descriptor Phi: a T# holding the program counter was accepted"}, + {entryWrites({0xbe8e037eu}), "descriptor Phi: a T# holding EXEC was accepted"}, + {entryWrites({0x7d840080u, 0xbe8e036au}), "descriptor Phi: a T# holding a compare mask was accepted"}, + }}; + for (const auto& [code, message] : invalidEntries) { + auto request = makeRequest(code); + expectFailure([&] { static_cast(GetResourcePlan(request)); }, "GetImageResource dword 0 is not a valid runtime value", message); + } + auto unwritten = loopCode; + unwritten[4] = 0xf4080200u; + auto unwrittenRequest = makeRequest(unwritten); + require(GetResourcePlan(unwrittenRequest)->info.images.size() == 3u, "descriptor Phi: a T# whose unwritten registers read as 0 was not planned as its own image"); + + const std::array prologue{0xf4080500u, 0xfa000020u, 0xf4080600u, 0xfa000040u, 0xf40c0200u, 0xfa000000u, 0xbf8cc07fu}; + const std::array epilogue{0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u}; + const auto planImages = [&](const std::vector& body) { + std::vector code(prologue.begin(), prologue.end()); + code.insert(code.end(), body.begin(), body.end()); + code.insert(code.end(), epilogue.begin(), epilogue.end()); + auto request = makeRequest(code); + return GetResourcePlan(request)->info.images.size(); + }; + const auto distinctLoads = [](std::uint32_t loads) { + std::vector body; + for (std::uint32_t load = 0; load < loads; load++) { + body.insert(body.end(), {0xbf068014u, 0xbf850002u, 0xf40c0200u, 0xfa000060u + load * 0x20u}); + } + return body; + }; + require(planImages(distinctLoads(63u)) == 64u, "descriptor Phi: 64 distinct T#s were not given one image each"); + expectFailure([&] { static_cast(planImages(distinctLoads(64u))); }, "GetImageResource dword 0 is not a valid runtime value", "descriptor Phi: more than 64 distinct T#s were accepted"); + + const auto splitReloads = [](std::uint32_t joins) { + std::vector body; + for (std::uint32_t join = 0; join < joins; join++) { + body.insert(body.end(), {0xbf068014u, 0xbf850003u, 0xf40003c0u, join % 2u == 0u ? 0xfa000088u : 0xfa00008cu, 0xbf820006u, 0xf4080200u, 0xfa000060u, 0xf4040300u, 0xfa000070u, 0xf4000380u, 0xfa000078u}); + } + return body; + }; + require(planImages(splitReloads(8u)) == 4u, "descriptor Phi: separate reloads of T# dword 7 and dwords 0-6 were not split into their 4 T#s"); + expectFailure([&] { static_cast(planImages(splitReloads(260u))); }, "GetImageResource dword 0 is not a valid runtime value", "descriptor Phi: a web over more than 512 descriptor tuples was accepted"); } void verifyProgramCounterRelativeData() { diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index ddeeb2cb0..85bda01e6 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -11,8 +11,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -599,7 +601,7 @@ private: void SplitDescriptorPhi(IrValue& inst) { const auto info = ImageOpcodeInfoOf(inst.Opcode()); const std::uint32_t handleCount = info.needsSampler ? 2u : 1u; - if (inst.ArgumentCount() < handleCount) { + if (inst.ArgumentCount() < handleCount || inst.Parent() == nullptr) { return; } std::array handles {}; @@ -626,21 +628,27 @@ private: if (phis.empty()) { return; } + if (!SplitDescriptorEdges(inst, handles, split, handleCount, phis)) { + SplitDescriptorWeb(inst, handles, split, handleCount); + } + } + + bool SplitDescriptorEdges(IrValue& inst, const std::array& handles, const std::array& split, std::uint32_t handleCount, const std::vector& phis) { IrBlock* block = phis.front()->Parent(); - if (block == nullptr || inst.Parent() == nullptr) { - return; + if (block == nullptr) { + return false; } const auto& predecessors = block->Predecessors(); if (predecessors.size() < 2u) { - return; + return false; } for (const IrValue* phi : phis) { if (phi->Parent() != block || phi->PhiBlockCount() != predecessors.size()) { - return; + return false; } for (const IrBlock* predecessor : predecessors) { if (PhiIncoming(*phi, predecessor) == nullptr) { - return; + return false; } } } @@ -666,24 +674,40 @@ private: for (std::uint32_t slot = 0; slot < handleCount; slot++) { for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) { if (SubstituteEdge(handles[slot]->Argument(dword), arm, inst, false, phiSearchDepth) == nullptr) { - return; + return false; } } } } + std::vector conditions; + for (std::uint32_t arm = 0; arm + 1u < arms.size(); arm++) { + std::vector edges(edgeArm.size()); + for (std::size_t edge = 0; edge < edgeArm.size(); edge++) { + edges[edge] = edgeArm[edge] == arm; + } + conditions.push_back(&EdgeSelector(*block, edges)); + } + ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) { + return SubstituteEdge(handles[slot]->Argument(dword), arms[arm], inst, true, phiSearchDepth); + }); + return true; + } + + template + void ReplaceWithArmCopies(IrValue& inst, const std::array& handles, const std::array& split, std::uint32_t handleCount, std::span conditions, Dword&& dword) { std::vector copies; const auto flags = inst.Flags(); const auto memory = m_program.Resources().memoryInfo.at(flags.index); - for (std::uint32_t arm = 0; arm < arms.size(); arm++) { + for (std::uint32_t arm = 0; arm <= conditions.size(); arm++) { std::vector arguments(inst.Arguments().begin(), inst.Arguments().end()); for (std::uint32_t slot = 0; slot < handleCount; slot++) { if (!split[slot]) { continue; } std::vector dwords; - for (std::size_t dword = 0; dword < handles[slot]->ArgumentCount(); dword++) { - dwords.push_back(SubstituteEdge(handles[slot]->Argument(dword), arms[arm], inst, true, phiSearchDepth)); + for (std::size_t index = 0; index < handles[slot]->ArgumentCount(); index++) { + dwords.push_back(dword(arm, slot, index)); } arguments[slot] = EmitCopy(*handles[slot], dwords, inst); } @@ -707,11 +731,7 @@ private: }; IrValue* selected = extract(copies.back()); for (std::uint32_t arm = static_cast(copies.size()) - 1u; arm-- > 0u;) { - std::vector edges(edgeArm.size()); - for (std::size_t edge = 0; edge < edgeArm.size(); edge++) { - edges[edge] = edgeArm[edge] == arm; - } - const std::array arguments {&EdgeSelector(*block, edges), extract(copies[arm]), selected}; + const std::array arguments {conditions[arm], extract(copies[arm]), selected}; selected = EmitBefore(inst, IrOpcode::SelectU32, IrType::U32, arguments); } components[component] = selected; @@ -723,6 +743,140 @@ private: inst.Parent()->RemoveInstruction(&inst); } + static bool HoldsProgramCounter(IrValue* value, std::uint32_t depth) { + value = value->Resolve(); + if (value->Opcode() == IrOpcode::GetShaderBase) { + return true; + } + if (value->HasImmediate() || depth == 0u || !Detail::IsRuntimeUniformOp(value->Opcode())) { + return false; + } + for (std::size_t index = 0; index < value->ArgumentCount(); index++) { + if (HoldsProgramCounter(value->Argument(index), depth - 1u)) { + return true; + } + } + return false; + } + + struct DescriptorWeb { + static constexpr std::size_t maxStates = 512u; + static constexpr std::size_t maxArms = 64u; + std::vector> arms; + std::map, IrValue*> selectors; + std::vector> phis; + std::unordered_map resolved; + }; + + IrValue* WebSelector(std::vector tuple, DescriptorWeb& web, IrValue& position) { + for (IrValue*& value : tuple) { + value = value->Resolve(); + if (!value->IsPhi()) { + continue; + } + const auto [entry, inserted] = web.resolved.try_emplace(value, value); + if (inserted) { + IrValue* invariant = ResolveInvariantPhi(m_program.Resources(), value); + entry->second = invariant == nullptr ? value : invariant->Resolve(); + } + value = entry->second; + } + if (const auto known = web.selectors.find(tuple); known != web.selectors.end()) { + return known->second; + } + const auto phi = std::ranges::find_if(tuple, [](const IrValue* value) { return value->IsPhi(); }); + if (phi == tuple.end()) { + for (IrValue* value : tuple) { + if (HoldsProgramCounter(value, phiSearchDepth) || Rematerialize(value, position, false, phiSearchDepth) == nullptr) { + return nullptr; + } + } + const auto same = std::ranges::find_if(web.arms, [&](const std::vector& arm) { + return std::ranges::equal(tuple, arm, [&](IrValue* left, IrValue* right) { return EquivalentValue(m_program.Resources(), left, right); }); + }); + if (same != web.arms.end()) { + return &m_builder.Constant(static_cast(same - web.arms.begin())); + } + if (web.arms.size() == DescriptorWeb::maxArms) { + return nullptr; + } + web.arms.push_back(std::move(tuple)); + return &m_builder.Constant(static_cast(web.arms.size() - 1u)); + } + IrBlock* block = (*phi)->Parent(); + if (block == nullptr || web.selectors.size() == DescriptorWeb::maxStates) { + return nullptr; + } + const auto& predecessors = block->Predecessors(); + if (predecessors.size() < 2u) { + return nullptr; + } + for (const IrValue* value : tuple) { + if (value->IsPhi() && (value->Parent() != block || value->PhiBlockCount() != predecessors.size())) { + return nullptr; + } + } + IrValue& selector = m_program.CreateValue(IrOpcode::Phi, IrType::U32); + web.selectors.emplace(tuple, &selector); + web.phis.emplace_back(block, &selector); + for (IrBlock* predecessor : predecessors) { + std::vector incoming; + for (IrValue* value : tuple) { + IrValue* edge = value->IsPhi() ? PhiIncoming(*value, predecessor) : value; + if (edge == nullptr) { + return nullptr; + } + incoming.push_back(edge); + } + IrValue* edgeSelector = WebSelector(std::move(incoming), web, position); + if (edgeSelector == nullptr) { + return nullptr; + } + selector.AddPhiOperand(predecessor, edgeSelector); + } + return &selector; + } + + void SplitDescriptorWeb(IrValue& inst, const std::array& handles, const std::array& split, std::uint32_t handleCount) { + std::vector tuple; + std::array offsets {}; + for (std::uint32_t slot = 0; slot < handleCount; slot++) { + offsets[slot] = tuple.size(); + for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) { + tuple.push_back(handles[slot]->Argument(dword)); + } + } + DescriptorWeb web; + IrValue* selector = WebSelector(tuple, web, inst); + const bool select = selector != nullptr && web.arms.size() > 1u; + if (select) { + for (const auto& [block, phi] : web.phis) { + block->InsertInstructionBefore(nullptr, phi); + } + } else { + for (const auto& [block, phi] : web.phis) { + for (std::size_t index = 0; index < phi->ArgumentCount(); index++) { + phi->ReplaceArgument(index, nullptr); + } + } + for (const auto& [block, phi] : web.phis) { + phi->Invalidate(); + } + } + if (selector == nullptr || web.arms.empty()) { + return; + } + + std::vector conditions; + for (std::uint32_t arm = 0; arm + 1u < web.arms.size(); arm++) { + const std::array compare {selector, &m_builder.Constant(arm)}; + conditions.push_back(EmitBefore(inst, IrOpcode::IEqual32, IrType::Bool, compare)); + } + ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) { + return Rematerialize(web.arms[arm][offsets[slot] + dword], inst, true, phiSearchDepth); + }); + } + void SplitDescriptorPhis() { std::vector candidates; for (auto& block : m_program.Blocks()) {