Merge pull request #1274 from CnRJay/fix/recompiler-descriptor-phi-web
fix(recompiler): split image reads whose descriptor is a web of SRT Phis
This commit is contained in:
Adrià
GitHub
commit
dfa8b4338f
|
|
@ -559,6 +559,64 @@ void verifyDescriptorPhis() {
|
|||
|
||||
auto dynamic = makeRequest(dynamicCode);
|
||||
expectFailure([&] { static_cast<void>(GetResourcePlan(dynamic)); }, "GetSamplerResource dword 0 is not a valid runtime value", "descriptor Phi: an edge without an SRT slot was accepted");
|
||||
|
||||
const std::vector<std::uint32_t> loopCode{0xf4080500u, 0xfa000020u, 0xf4080600u, 0xfa000040u, 0xf40c0200u, 0xfa000000u, 0xbe910380u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf068011u, 0xbf850003u, 0xf40c0200u, 0xfa000000u, 0xbf820002u, 0xf40c0200u, 0xfa000060u, 0xbf8cc07fu, 0x80118111u, 0xbf0a8211u, 0xbf85fff0u, 0xbf810000u};
|
||||
const auto loopCapture = compile(loopCode, 2u, 1u);
|
||||
const auto& loopImages = loopCapture->snapshot.images;
|
||||
const auto holdsLoopImage = [&](const std::array<std::uint32_t, 8>& words) {
|
||||
return std::ranges::any_of(loopImages, [&](const DescriptorValue& value) {
|
||||
return std::equal(words.begin(), words.end(), value.dwords.begin());
|
||||
});
|
||||
};
|
||||
require(loopImages.size() == 2u && holdsLoopImage(first) && holdsLoopImage(second), "descriptor Phi: the loop's chained T# Phis were not split into the two SRT T#s");
|
||||
|
||||
const auto entryWrites = [&](std::initializer_list<std::uint32_t> words) {
|
||||
auto code = loopCode;
|
||||
code.insert(code.begin() + 8, words);
|
||||
return code;
|
||||
};
|
||||
const std::array<std::pair<std::vector<std::uint32_t>, const char*>, 3> invalidEntries{{
|
||||
{entryWrites({0xbe8e1f00u}), "descriptor Phi: a T# holding the program counter was accepted"},
|
||||
{entryWrites({0xbe8e037eu}), "descriptor Phi: a T# holding EXEC was accepted"},
|
||||
{entryWrites({0x7d840080u, 0xbe8e036au}), "descriptor Phi: a T# holding a compare mask was accepted"},
|
||||
}};
|
||||
for (const auto& [code, message] : invalidEntries) {
|
||||
auto request = makeRequest(code);
|
||||
expectFailure([&] { static_cast<void>(GetResourcePlan(request)); }, "GetImageResource dword 0 is not a valid runtime value", message);
|
||||
}
|
||||
auto unwritten = loopCode;
|
||||
unwritten[4] = 0xf4080200u;
|
||||
auto unwrittenRequest = makeRequest(unwritten);
|
||||
require(GetResourcePlan(unwrittenRequest)->info.images.size() == 3u, "descriptor Phi: a T# whose unwritten registers read as 0 was not planned as its own image");
|
||||
|
||||
const std::array<std::uint32_t, 7> prologue{0xf4080500u, 0xfa000020u, 0xf4080600u, 0xfa000040u, 0xf40c0200u, 0xfa000000u, 0xbf8cc07fu};
|
||||
const std::array<std::uint32_t, 7> epilogue{0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u};
|
||||
const auto planImages = [&](const std::vector<std::uint32_t>& body) {
|
||||
std::vector<std::uint32_t> code(prologue.begin(), prologue.end());
|
||||
code.insert(code.end(), body.begin(), body.end());
|
||||
code.insert(code.end(), epilogue.begin(), epilogue.end());
|
||||
auto request = makeRequest(code);
|
||||
return GetResourcePlan(request)->info.images.size();
|
||||
};
|
||||
const auto distinctLoads = [](std::uint32_t loads) {
|
||||
std::vector<std::uint32_t> body;
|
||||
for (std::uint32_t load = 0; load < loads; load++) {
|
||||
body.insert(body.end(), {0xbf068014u, 0xbf850002u, 0xf40c0200u, 0xfa000060u + load * 0x20u});
|
||||
}
|
||||
return body;
|
||||
};
|
||||
require(planImages(distinctLoads(63u)) == 64u, "descriptor Phi: 64 distinct T#s were not given one image each");
|
||||
expectFailure([&] { static_cast<void>(planImages(distinctLoads(64u))); }, "GetImageResource dword 0 is not a valid runtime value", "descriptor Phi: more than 64 distinct T#s were accepted");
|
||||
|
||||
const auto splitReloads = [](std::uint32_t joins) {
|
||||
std::vector<std::uint32_t> body;
|
||||
for (std::uint32_t join = 0; join < joins; join++) {
|
||||
body.insert(body.end(), {0xbf068014u, 0xbf850003u, 0xf40003c0u, join % 2u == 0u ? 0xfa000088u : 0xfa00008cu, 0xbf820006u, 0xf4080200u, 0xfa000060u, 0xf4040300u, 0xfa000070u, 0xf4000380u, 0xfa000078u});
|
||||
}
|
||||
return body;
|
||||
};
|
||||
require(planImages(splitReloads(8u)) == 4u, "descriptor Phi: separate reloads of T# dword 7 and dwords 0-6 were not split into their 4 T#s");
|
||||
expectFailure([&] { static_cast<void>(planImages(splitReloads(260u))); }, "GetImageResource dword 0 is not a valid runtime value", "descriptor Phi: a web over more than 512 descriptor tuples was accepted");
|
||||
}
|
||||
|
||||
void verifyProgramCounterRelativeData() {
|
||||
|
|
|
|||
|
|
@ -11,8 +11,10 @@
|
|||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
|
|
@ -599,7 +601,7 @@ private:
|
|||
void SplitDescriptorPhi(IrValue& inst) {
|
||||
const auto info = ImageOpcodeInfoOf(inst.Opcode());
|
||||
const std::uint32_t handleCount = info.needsSampler ? 2u : 1u;
|
||||
if (inst.ArgumentCount() < handleCount) {
|
||||
if (inst.ArgumentCount() < handleCount || inst.Parent() == nullptr) {
|
||||
return;
|
||||
}
|
||||
std::array<IrValue*, 2> handles {};
|
||||
|
|
@ -626,21 +628,27 @@ private:
|
|||
if (phis.empty()) {
|
||||
return;
|
||||
}
|
||||
if (!SplitDescriptorEdges(inst, handles, split, handleCount, phis)) {
|
||||
SplitDescriptorWeb(inst, handles, split, handleCount);
|
||||
}
|
||||
}
|
||||
|
||||
bool SplitDescriptorEdges(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount, const std::vector<IrValue*>& phis) {
|
||||
IrBlock* block = phis.front()->Parent();
|
||||
if (block == nullptr || inst.Parent() == nullptr) {
|
||||
return;
|
||||
if (block == nullptr) {
|
||||
return false;
|
||||
}
|
||||
const auto& predecessors = block->Predecessors();
|
||||
if (predecessors.size() < 2u) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
for (const IrValue* phi : phis) {
|
||||
if (phi->Parent() != block || phi->PhiBlockCount() != predecessors.size()) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
for (const IrBlock* predecessor : predecessors) {
|
||||
if (PhiIncoming(*phi, predecessor) == nullptr) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -666,24 +674,40 @@ private:
|
|||
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
||||
for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) {
|
||||
if (SubstituteEdge(handles[slot]->Argument(dword), arm, inst, false, phiSearchDepth) == nullptr) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<IrValue*> conditions;
|
||||
for (std::uint32_t arm = 0; arm + 1u < arms.size(); arm++) {
|
||||
std::vector<bool> edges(edgeArm.size());
|
||||
for (std::size_t edge = 0; edge < edgeArm.size(); edge++) {
|
||||
edges[edge] = edgeArm[edge] == arm;
|
||||
}
|
||||
conditions.push_back(&EdgeSelector(*block, edges));
|
||||
}
|
||||
ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) {
|
||||
return SubstituteEdge(handles[slot]->Argument(dword), arms[arm], inst, true, phiSearchDepth);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename Dword>
|
||||
void ReplaceWithArmCopies(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount, std::span<IrValue* const> conditions, Dword&& dword) {
|
||||
std::vector<IrValue*> copies;
|
||||
const auto flags = inst.Flags<MemoryFlags>();
|
||||
const auto memory = m_program.Resources().memoryInfo.at(flags.index);
|
||||
for (std::uint32_t arm = 0; arm < arms.size(); arm++) {
|
||||
for (std::uint32_t arm = 0; arm <= conditions.size(); arm++) {
|
||||
std::vector<IrValue*> arguments(inst.Arguments().begin(), inst.Arguments().end());
|
||||
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
||||
if (!split[slot]) {
|
||||
continue;
|
||||
}
|
||||
std::vector<IrValue*> dwords;
|
||||
for (std::size_t dword = 0; dword < handles[slot]->ArgumentCount(); dword++) {
|
||||
dwords.push_back(SubstituteEdge(handles[slot]->Argument(dword), arms[arm], inst, true, phiSearchDepth));
|
||||
for (std::size_t index = 0; index < handles[slot]->ArgumentCount(); index++) {
|
||||
dwords.push_back(dword(arm, slot, index));
|
||||
}
|
||||
arguments[slot] = EmitCopy(*handles[slot], dwords, inst);
|
||||
}
|
||||
|
|
@ -707,11 +731,7 @@ private:
|
|||
};
|
||||
IrValue* selected = extract(copies.back());
|
||||
for (std::uint32_t arm = static_cast<std::uint32_t>(copies.size()) - 1u; arm-- > 0u;) {
|
||||
std::vector<bool> edges(edgeArm.size());
|
||||
for (std::size_t edge = 0; edge < edgeArm.size(); edge++) {
|
||||
edges[edge] = edgeArm[edge] == arm;
|
||||
}
|
||||
const std::array<IrValue*, 3> arguments {&EdgeSelector(*block, edges), extract(copies[arm]), selected};
|
||||
const std::array<IrValue*, 3> arguments {conditions[arm], extract(copies[arm]), selected};
|
||||
selected = EmitBefore(inst, IrOpcode::SelectU32, IrType::U32, arguments);
|
||||
}
|
||||
components[component] = selected;
|
||||
|
|
@ -723,6 +743,140 @@ private:
|
|||
inst.Parent()->RemoveInstruction(&inst);
|
||||
}
|
||||
|
||||
static bool HoldsProgramCounter(IrValue* value, std::uint32_t depth) {
|
||||
value = value->Resolve();
|
||||
if (value->Opcode() == IrOpcode::GetShaderBase) {
|
||||
return true;
|
||||
}
|
||||
if (value->HasImmediate() || depth == 0u || !Detail::IsRuntimeUniformOp(value->Opcode())) {
|
||||
return false;
|
||||
}
|
||||
for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
|
||||
if (HoldsProgramCounter(value->Argument(index), depth - 1u)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
struct DescriptorWeb {
|
||||
static constexpr std::size_t maxStates = 512u;
|
||||
static constexpr std::size_t maxArms = 64u;
|
||||
std::vector<std::vector<IrValue*>> arms;
|
||||
std::map<std::vector<IrValue*>, IrValue*> selectors;
|
||||
std::vector<std::pair<IrBlock*, IrValue*>> phis;
|
||||
std::unordered_map<IrValue*, IrValue*> resolved;
|
||||
};
|
||||
|
||||
IrValue* WebSelector(std::vector<IrValue*> tuple, DescriptorWeb& web, IrValue& position) {
|
||||
for (IrValue*& value : tuple) {
|
||||
value = value->Resolve();
|
||||
if (!value->IsPhi()) {
|
||||
continue;
|
||||
}
|
||||
const auto [entry, inserted] = web.resolved.try_emplace(value, value);
|
||||
if (inserted) {
|
||||
IrValue* invariant = ResolveInvariantPhi(m_program.Resources(), value);
|
||||
entry->second = invariant == nullptr ? value : invariant->Resolve();
|
||||
}
|
||||
value = entry->second;
|
||||
}
|
||||
if (const auto known = web.selectors.find(tuple); known != web.selectors.end()) {
|
||||
return known->second;
|
||||
}
|
||||
const auto phi = std::ranges::find_if(tuple, [](const IrValue* value) { return value->IsPhi(); });
|
||||
if (phi == tuple.end()) {
|
||||
for (IrValue* value : tuple) {
|
||||
if (HoldsProgramCounter(value, phiSearchDepth) || Rematerialize(value, position, false, phiSearchDepth) == nullptr) {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
const auto same = std::ranges::find_if(web.arms, [&](const std::vector<IrValue*>& arm) {
|
||||
return std::ranges::equal(tuple, arm, [&](IrValue* left, IrValue* right) { return EquivalentValue(m_program.Resources(), left, right); });
|
||||
});
|
||||
if (same != web.arms.end()) {
|
||||
return &m_builder.Constant(static_cast<std::uint32_t>(same - web.arms.begin()));
|
||||
}
|
||||
if (web.arms.size() == DescriptorWeb::maxArms) {
|
||||
return nullptr;
|
||||
}
|
||||
web.arms.push_back(std::move(tuple));
|
||||
return &m_builder.Constant(static_cast<std::uint32_t>(web.arms.size() - 1u));
|
||||
}
|
||||
IrBlock* block = (*phi)->Parent();
|
||||
if (block == nullptr || web.selectors.size() == DescriptorWeb::maxStates) {
|
||||
return nullptr;
|
||||
}
|
||||
const auto& predecessors = block->Predecessors();
|
||||
if (predecessors.size() < 2u) {
|
||||
return nullptr;
|
||||
}
|
||||
for (const IrValue* value : tuple) {
|
||||
if (value->IsPhi() && (value->Parent() != block || value->PhiBlockCount() != predecessors.size())) {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
IrValue& selector = m_program.CreateValue(IrOpcode::Phi, IrType::U32);
|
||||
web.selectors.emplace(tuple, &selector);
|
||||
web.phis.emplace_back(block, &selector);
|
||||
for (IrBlock* predecessor : predecessors) {
|
||||
std::vector<IrValue*> incoming;
|
||||
for (IrValue* value : tuple) {
|
||||
IrValue* edge = value->IsPhi() ? PhiIncoming(*value, predecessor) : value;
|
||||
if (edge == nullptr) {
|
||||
return nullptr;
|
||||
}
|
||||
incoming.push_back(edge);
|
||||
}
|
||||
IrValue* edgeSelector = WebSelector(std::move(incoming), web, position);
|
||||
if (edgeSelector == nullptr) {
|
||||
return nullptr;
|
||||
}
|
||||
selector.AddPhiOperand(predecessor, edgeSelector);
|
||||
}
|
||||
return &selector;
|
||||
}
|
||||
|
||||
void SplitDescriptorWeb(IrValue& inst, const std::array<IrValue*, 2>& handles, const std::array<bool, 2>& split, std::uint32_t handleCount) {
|
||||
std::vector<IrValue*> tuple;
|
||||
std::array<std::size_t, 2> offsets {};
|
||||
for (std::uint32_t slot = 0; slot < handleCount; slot++) {
|
||||
offsets[slot] = tuple.size();
|
||||
for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) {
|
||||
tuple.push_back(handles[slot]->Argument(dword));
|
||||
}
|
||||
}
|
||||
DescriptorWeb web;
|
||||
IrValue* selector = WebSelector(tuple, web, inst);
|
||||
const bool select = selector != nullptr && web.arms.size() > 1u;
|
||||
if (select) {
|
||||
for (const auto& [block, phi] : web.phis) {
|
||||
block->InsertInstructionBefore(nullptr, phi);
|
||||
}
|
||||
} else {
|
||||
for (const auto& [block, phi] : web.phis) {
|
||||
for (std::size_t index = 0; index < phi->ArgumentCount(); index++) {
|
||||
phi->ReplaceArgument(index, nullptr);
|
||||
}
|
||||
}
|
||||
for (const auto& [block, phi] : web.phis) {
|
||||
phi->Invalidate();
|
||||
}
|
||||
}
|
||||
if (selector == nullptr || web.arms.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<IrValue*> conditions;
|
||||
for (std::uint32_t arm = 0; arm + 1u < web.arms.size(); arm++) {
|
||||
const std::array<IrValue*, 2> compare {selector, &m_builder.Constant(arm)};
|
||||
conditions.push_back(EmitBefore(inst, IrOpcode::IEqual32, IrType::Bool, compare));
|
||||
}
|
||||
ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) {
|
||||
return Rematerialize(web.arms[arm][offsets[slot] + dword], inst, true, phiSearchDepth);
|
||||
});
|
||||
}
|
||||
|
||||
void SplitDescriptorPhis() {
|
||||
std::vector<IrValue*> candidates;
|
||||
for (auto& block : m_program.Blocks()) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue