611 lines
24 KiB
C++
611 lines
24 KiB
C++
#include "ControlFlow/GraphBuilder.hpp"
|
|
#include "ControlFlow/RequestSerializer.hpp"
|
|
#include "ControlFlow/Structurizer.hpp"
|
|
#include "RdnaDecoder/RdnaInstructionDecoder.hpp"
|
|
#include "Recompiler.hpp"
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <cstdio>
|
|
#include <exception>
|
|
#include <fstream>
|
|
#include <map>
|
|
#include <span>
|
|
#include <sstream>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <tuple>
|
|
#include <vector>
|
|
|
|
namespace {
|
|
|
|
using namespace ShaderRecompiler;
|
|
|
|
enum class Split {
|
|
Clone,
|
|
Route
|
|
};
|
|
|
|
struct Program {
|
|
std::string_view name;
|
|
std::string_view source;
|
|
std::vector<std::uint32_t> code;
|
|
Split split;
|
|
std::size_t clonedLimit = 0;
|
|
std::vector<std::uint32_t> reference = {};
|
|
};
|
|
|
|
using RouteState = std::map<std::uint32_t, bool>;
|
|
|
|
void require(bool condition, const std::string& message) {
|
|
if (!condition) throw std::runtime_error(message);
|
|
}
|
|
|
|
std::vector<std::uint32_t> constructBlocks(const ControlFlowGraph& graph, std::uint32_t header, std::uint32_t exclude) {
|
|
std::vector<std::uint32_t> blocks;
|
|
for (const auto& block : graph.blocks) {
|
|
if (graph.Dominates(header, block.id) && (exclude == InvalidControlFlowId || !graph.Dominates(exclude, block.id))) {
|
|
blocks.push_back(block.id);
|
|
}
|
|
}
|
|
return blocks;
|
|
}
|
|
|
|
void verifyStructured(const std::string& prefix, const ControlFlowGraph& graph) {
|
|
std::map<std::uint32_t, std::uint32_t> mergeOwners;
|
|
std::vector<std::pair<std::uint32_t, std::uint32_t>> loopExits;
|
|
for (const auto& block : graph.blocks) {
|
|
const auto& terminator = block.terminator;
|
|
if (terminator.mergeBlock == InvalidControlFlowId) continue;
|
|
require(mergeOwners.emplace(terminator.mergeBlock, block.id).second, prefix + "block " + std::to_string(terminator.mergeBlock) + " merges two constructs");
|
|
if (terminator.loopHeader) loopExits.emplace_back(terminator.mergeBlock, terminator.continueBlock);
|
|
}
|
|
for (const auto& block : graph.blocks) {
|
|
const auto& terminator = block.terminator;
|
|
if (terminator.loopHeader || terminator.mergeBlock == InvalidControlFlowId) continue;
|
|
for (const auto member : constructBlocks(graph, block.id, terminator.mergeBlock)) {
|
|
for (const auto successor : graph.FindBlock(member).successors) {
|
|
if (successor == terminator.mergeBlock || (graph.Dominates(block.id, successor) && !graph.Dominates(terminator.mergeBlock, successor))) continue;
|
|
const bool loopExit = std::any_of(loopExits.begin(), loopExits.end(), [&](const auto& exits) { return successor == exits.first || successor == exits.second; });
|
|
require(loopExit, prefix + "block " + std::to_string(member) + " of the selection at block " + std::to_string(block.id) + " branches to block " + std::to_string(successor) + " outside the construct");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
std::uint32_t followEmptyBlocks(const std::string& prefix, const ControlFlowGraph& graph, std::uint32_t blockId, RouteState& routes) {
|
|
for (std::size_t steps = 0; steps <= graph.blocks.size(); ++steps) {
|
|
const auto& block = graph.FindBlock(blockId);
|
|
const auto& terminator = block.terminator;
|
|
const bool empty = block.instructionBegin == block.instructionEnd;
|
|
require(terminator.gotoValue < 0 || empty, prefix + "block " + std::to_string(block.id) + " sets a route variable after guest instructions");
|
|
if (terminator.gotoValue >= 0) routes[terminator.gotoVariable] = terminator.gotoValue != 0;
|
|
if (!empty) return blockId;
|
|
if (terminator.kind == TerminatorKind::Branch) {
|
|
blockId = terminator.trueBlock;
|
|
} else if (terminator.kind == TerminatorKind::ConditionalBranch && terminator.condition == BranchCondition::GotoVariable) {
|
|
const auto route = routes.find(terminator.gotoVariable);
|
|
require(route != routes.end(), prefix + "block " + std::to_string(block.id) + " reads route variable " + std::to_string(terminator.gotoVariable) + " before any path sets it");
|
|
blockId = route->second ? terminator.trueBlock : terminator.falseBlock;
|
|
} else {
|
|
return blockId;
|
|
}
|
|
}
|
|
throw std::runtime_error(prefix + "a cycle of empty blocks");
|
|
}
|
|
|
|
void verifySameExecutions(const std::string& prefix, const ControlFlowGraph& original, const ControlFlowGraph& structured) {
|
|
std::vector<std::tuple<std::uint32_t, std::uint32_t, RouteState>> pending{{structured.blocks.front().id, original.entryBlock, {}}};
|
|
std::vector<std::tuple<std::uint32_t, std::uint32_t, RouteState>> visited;
|
|
RouteState none;
|
|
while (!pending.empty()) {
|
|
auto [structuredId, originalId, routes] = pending.back();
|
|
pending.pop_back();
|
|
const auto copyId = followEmptyBlocks(prefix, structured, structuredId, routes);
|
|
const auto sourceId = followEmptyBlocks(prefix, original, originalId, none);
|
|
auto state = std::make_tuple(copyId, sourceId, routes);
|
|
if (std::find(visited.begin(), visited.end(), state) != visited.end()) continue;
|
|
visited.push_back(state);
|
|
const auto& copy = structured.FindBlock(copyId);
|
|
const auto& source = original.FindBlock(sourceId);
|
|
require(copy.instructionBegin == source.instructionBegin && copy.instructionEnd == source.instructionEnd, prefix + "block " + std::to_string(copy.id) + " runs other instructions than block " + std::to_string(source.id));
|
|
require(copy.terminator.kind == source.terminator.kind && copy.terminator.condition == source.terminator.condition, prefix + "block " + std::to_string(copy.id) + " branches differently from block " + std::to_string(source.id));
|
|
if (source.terminator.kind == TerminatorKind::Branch || source.terminator.kind == TerminatorKind::ConditionalBranch) pending.emplace_back(copy.terminator.trueBlock, source.terminator.trueBlock, routes);
|
|
if (source.terminator.kind == TerminatorKind::ConditionalBranch) pending.emplace_back(copy.terminator.falseBlock, source.terminator.falseBlock, routes);
|
|
}
|
|
}
|
|
|
|
std::size_t clonedInstructions(const ControlFlowGraph& graph) {
|
|
std::map<std::pair<std::uint32_t, std::uint32_t>, std::size_t> copies;
|
|
std::size_t cloned = 0;
|
|
for (const auto& block : graph.blocks) {
|
|
if (block.instructionBegin != block.instructionEnd && copies[{block.instructionBegin, block.instructionEnd}]++ != 0) cloned += block.instructionEnd - block.instructionBegin;
|
|
}
|
|
return cloned;
|
|
}
|
|
|
|
std::size_t routeVariables(const ControlFlowGraph& graph) {
|
|
std::vector<std::uint32_t> variables;
|
|
for (const auto& block : graph.blocks) {
|
|
if (block.terminator.condition == BranchCondition::GotoVariable && std::find(variables.begin(), variables.end(), block.terminator.gotoVariable) == variables.end()) variables.push_back(block.terminator.gotoVariable);
|
|
}
|
|
return variables.size();
|
|
}
|
|
|
|
struct Structured {
|
|
std::size_t addedBlocks;
|
|
std::size_t clonedInstructions;
|
|
std::size_t routeVariables;
|
|
};
|
|
|
|
Structured verifyGraph(const std::string& name, std::span<const std::uint32_t> code) {
|
|
const std::string prefix = name + ": ";
|
|
const auto decoded = RdnaInstructionDecoder{}.Decode(code);
|
|
const auto original = GraphBuilder{}.Build(decoded);
|
|
auto graph = original;
|
|
Structurizer{}.Structurize(graph);
|
|
verifyStructured(prefix, graph);
|
|
verifySameExecutions(prefix, original, graph);
|
|
return {graph.blocks.size() - original.blocks.size(), clonedInstructions(graph), routeVariables(graph)};
|
|
}
|
|
|
|
std::vector<std::uint32_t> Store(std::vector<std::uint32_t> code) {
|
|
code.insert(code.end(), {0xe0700000u, 0x80000100u, 0xbf810000u});
|
|
return code;
|
|
}
|
|
|
|
void verifyRequest(const char* path) {
|
|
std::ifstream file(path, std::ios::binary);
|
|
std::ostringstream text;
|
|
text << file.rdbuf();
|
|
const auto request = RequestSerializer{}.Deserialize(text.str());
|
|
const auto result = verifyGraph(path, request.request.shader.code);
|
|
std::printf("%s: structured with %zu added blocks, %zu cloned instructions, %zu route variables\n", path, result.addedBlocks, result.clonedInstructions, result.routeVariables);
|
|
}
|
|
|
|
std::size_t recompile(const std::string& name, std::span<const std::uint32_t> code, std::uint32_t subgroupSize) {
|
|
const std::array<std::uint32_t, 4> userData{0x10000000u, 0x00100000u, 0x40u, 0x00027facu};
|
|
const std::array<std::uint32_t, 2> capabilities{1u, 61u};
|
|
const std::array<std::string_view, 1> extensions{"SPV_KHR_storage_buffer_storage_class"};
|
|
RecompileRequest request{};
|
|
request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}};
|
|
request.context.waveSize = 64;
|
|
request.context.userDataBaseRegister = 0;
|
|
request.context.userData = userData;
|
|
request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
|
|
request.target.vulkanVersion = 0x00403000u;
|
|
request.target.spirvVersion = 0x00010600u;
|
|
request.target.subgroupSize = subgroupSize;
|
|
request.target.bdaAbiVersion = 1;
|
|
request.target.supportedCapabilities = capabilities;
|
|
request.target.supportedExtensions = extensions;
|
|
request.target.maxWorkgroupSize = {1024u, 1024u, 64u};
|
|
request.target.maxWorkgroupInvocations = 1024;
|
|
request.target.maxWorkgroupSharedMemoryBytes = 49152;
|
|
request.layout = {0, 0, 0, 128};
|
|
request.useCache = false;
|
|
const auto result = Recompile(request);
|
|
require(!result.spirv.empty(), name + ": no SPIR-V for a " + std::to_string(subgroupSize) + "-lane subgroup");
|
|
return result.spirv.size();
|
|
}
|
|
|
|
void verifyProgram(const Program& program) {
|
|
const std::string name(program.name);
|
|
const auto result = verifyGraph(name, program.code);
|
|
if (program.split == Split::Clone) {
|
|
require(result.clonedInstructions != 0 && result.routeVariables == 0, name + ": expected a clone, got " + std::to_string(result.clonedInstructions) + " cloned instructions and " + std::to_string(result.routeVariables) + " route variables");
|
|
} else {
|
|
require(result.clonedInstructions <= program.clonedLimit && result.routeVariables != 0, name + ": expected routing with at most " + std::to_string(program.clonedLimit) + " cloned instructions, got " + std::to_string(result.clonedInstructions) + " cloned instructions and " + std::to_string(result.routeVariables) + " route variables");
|
|
}
|
|
const auto words = recompile(name, program.code, 64u);
|
|
static_cast<void>(recompile(name, program.code, 32u));
|
|
if (!program.reference.empty()) {
|
|
const auto reference = recompile(name, program.reference, 64u);
|
|
require(words * 10u <= reference * 11u, name + ": " + std::to_string(words) + " SPIR-V words, more than 10% over the " + std::to_string(reference) + " of the program without the entering branch");
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
int main(int argc, char** argv) {
|
|
if (argc > 1) {
|
|
int failures = 0;
|
|
for (int i = 1; i < argc; ++i) {
|
|
try {
|
|
verifyRequest(argv[i]);
|
|
} catch (const std::exception& error) {
|
|
std::fprintf(stderr, "%s: %s\n", argv[i], error.what());
|
|
++failures;
|
|
}
|
|
}
|
|
return failures == 0 ? 0 : 1;
|
|
}
|
|
const std::vector<Program> programs{
|
|
{"shared tail", R"(
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_cbranch_vccz outer_else
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz tail
|
|
v_add_nc_u32 v1, 1, v0
|
|
s_branch done
|
|
outer_else:
|
|
v_add_nc_u32 v1, 2, v0
|
|
tail:
|
|
v_add_nc_u32 v1, 3, v1
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0x7d880090u, 0xbf860004u, 0x7d880088u, 0xbf870003u, 0x4a020081u, 0xbf820002u, 0x4a020082u, 0x4a020283u}), Split::Clone},
|
|
{"shared tail with a selection", R"(
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_cbranch_vccz outer_else
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz tail
|
|
v_add_nc_u32 v1, 1, v0
|
|
s_branch done
|
|
outer_else:
|
|
v_add_nc_u32 v1, 2, v0
|
|
tail:
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_cbranch_vccz tail_join
|
|
v_add_nc_u32 v1, 5, v1
|
|
tail_join:
|
|
v_add_nc_u32 v1, 3, v1
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0x7d880090u, 0xbf860004u, 0x7d880088u, 0xbf870003u, 0x4a020081u, 0xbf820005u, 0x4a020082u, 0x7d880084u, 0xbf860001u, 0x4a020285u, 0x4a020283u}), Split::Clone},
|
|
{"shared tail in a loop", R"(
|
|
s_mov_b32 s8, 0
|
|
v_mov_b32 v1, 0
|
|
loop:
|
|
v_cmp_gt_u32 vcc, s8, v0
|
|
s_cbranch_vccz outer_else
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz tail
|
|
v_add_nc_u32 v1, 1, v1
|
|
s_branch latch
|
|
outer_else:
|
|
v_add_nc_u32 v1, 2, v1
|
|
tail:
|
|
v_add_nc_u32 v1, 3, v1
|
|
latch:
|
|
s_add_u32 s8, s8, 16
|
|
s_cmp_lt_u32 s8, 64
|
|
s_cbranch_scc1 loop
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0xbe880380u, 0x7e020280u, 0x7d880008u, 0xbf860004u, 0x7d880088u, 0xbf870003u, 0x4a020281u, 0xbf820002u, 0x4a020282u, 0x4a020283u, 0x80089008u, 0xbf0ac008u, 0xbf85fff5u}), Split::Clone},
|
|
{"shared early exit", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_and_saveexec_b64 s[8:9], vcc
|
|
s_cbranch_execz join
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_cbranch_vccz early_exit
|
|
v_add_nc_u32 v1, 1, v0
|
|
join:
|
|
s_mov_b64 exec, s[8:9]
|
|
v_cmp_gt_u32 vcc, 32, v0
|
|
s_cbranch_vccz early_exit
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
early_exit:
|
|
s_mov_b64 exec, 0
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf880003u, 0x7d880084u, 0xbf860007u, 0x4a020081u, 0xbefe0408u, 0x7d8800a0u, 0xbf860003u, 0xe0700000u, 0x80000100u, 0xbf810000u, 0xbefe0480u, 0xbf810000u}, Split::Clone},
|
|
{"short-circuit condition", R"(
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_cbranch_vccz body
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz done
|
|
body:
|
|
v_add_nc_u32 v1, 1, v0
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_cbranch_vccz done
|
|
v_add_nc_u32 v1, 2, v1
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
Store({0x7d880090u, 0xbf860002u, 0x7d880088u, 0xbf870004u, 0x4a020081u, 0x7d880084u, 0xbf860001u, 0x4a020282u}), Split::Clone},
|
|
{"entered region with an expensive body", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_and_saveexec_b64 s[8:9], vcc
|
|
s_cbranch_execz body
|
|
v_add_nc_u32 v1, 1, v0
|
|
v_cmp_gt_u32 vcc, 8, v1
|
|
s_cbranch_vccnz done
|
|
body:
|
|
s_mov_b32 s10, 0
|
|
loop:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
s_add_u32 s10, s10, 1
|
|
s_cmp_lt_u32 s10, 4
|
|
s_cbranch_scc1 loop
|
|
s_mov_b64 exec, s[8:9]
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
done:
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf880003u, 0x4a020081u, 0x7d880288u, 0xbf870015u, 0xbe8a0380u, 0xe0300000u, 0x80000200u,
|
|
0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u,
|
|
0x7e160501u, 0x4a02020bu, 0x800a810au, 0xbf0a840au, 0xbf85ffefu, 0xbefe0408u, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Route, 0,
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf800000u, 0x4a020081u, 0x7d880288u, 0xbf870015u, 0xbe8a0380u, 0xe0300000u, 0x80000200u,
|
|
0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u,
|
|
0x7e160501u, 0x4a02020bu, 0x800a810au, 0xbf0a840au, 0xbf85ffefu, 0xbefe0408u, 0xe0700000u, 0x80000100u, 0xbf810000u}},
|
|
{"return tail past an entered region", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_and_saveexec_b64 s[8:9], vcc
|
|
s_cbranch_execz body
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz done
|
|
v_add_nc_u32 v1, 1, v0
|
|
s_mov_b64 exec, s[8:9]
|
|
body:
|
|
s_mov_b32 s10, 0
|
|
loop:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
s_add_u32 s10, s10, 1
|
|
s_cmp_lt_u32 s10, 4
|
|
s_cbranch_scc1 loop
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
done:
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf880004u, 0x7d880088u, 0xbf870016u, 0x4a020081u, 0xbefe0408u, 0xbe8a0380u, 0xe0300000u,
|
|
0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u,
|
|
0x3a020303u, 0x7e160501u, 0x4a02020bu, 0x800a810au, 0xbf0a840au, 0xbf85ffefu, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Clone},
|
|
{"entered region in a loop", R"(
|
|
s_mov_b32 s12, 0
|
|
v_mov_b32 v1, 0
|
|
outer:
|
|
v_cmp_gt_u32 vcc, s12, v0
|
|
s_cbranch_vccz body
|
|
v_cmp_gt_u32 vcc, 8, v0
|
|
s_cbranch_vccnz join
|
|
v_add_nc_u32 v1, 1, v1
|
|
body:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
join:
|
|
v_add_nc_u32 v1, 2, v1
|
|
s_add_u32 s12, s12, 16
|
|
s_cmp_lt_u32 s12, 64
|
|
s_cbranch_scc0 exit_loop
|
|
s_branch outer
|
|
exit_loop:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
{0xbe8c0380u, 0x7e020280u, 0x7d88000cu, 0xbf860003u, 0x7d880088u, 0xbf87000fu, 0x4a020281u, 0xe0300000u, 0x80000200u, 0x10060302u,
|
|
0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u,
|
|
0x4a02020bu, 0x4a020282u, 0x800c900cu, 0xbf0ac00cu, 0xbf840001u, 0xbf82ffe8u, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Route},
|
|
{"nested entered regions", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_and_saveexec_b64 s[8:9], vcc
|
|
s_cbranch_execz outer_body
|
|
v_add_nc_u32 v1, 1, v0
|
|
v_cmp_gt_u32 vcc, 8, v1
|
|
s_cbranch_vccnz done
|
|
outer_body:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
s_cmp_eq_u32 s11, 64
|
|
s_cbranch_scc0 inner_body
|
|
v_add_nc_u32 v1, 2, v1
|
|
v_cmp_gt_u32 vcc, 4, v1
|
|
s_cbranch_vccz done
|
|
inner_body:
|
|
s_mov_b32 s10, 0
|
|
inner_loop:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
s_add_u32 s10, s10, 1
|
|
s_cmp_lt_u32 s10, 4
|
|
s_cbranch_scc1 inner_loop
|
|
s_mov_b64 exec, s[8:9]
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
done:
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf880003u, 0x4a020081u, 0x7d880288u, 0xbf87001au, 0xe0300000u, 0x80000200u, 0x10060302u,
|
|
0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xbf06c00bu, 0xbf840003u, 0x4a020282u, 0x7d880284u, 0xbf86000eu, 0xbe8a0380u,
|
|
0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0x800a810au, 0xbf0a840au, 0xbf85fff6u,
|
|
0xbefe0408u, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Route},
|
|
{"shared body after an overwritten condition", R"(
|
|
v_mov_b32 v1, 0
|
|
s_cmp_eq_u32 s2, 64
|
|
s_cbranch_scc0 shared
|
|
v_add_nc_u32 v1, 1, v0
|
|
s_cmp_eq_u32 s3, 0
|
|
s_cbranch_scc1 done
|
|
shared:
|
|
s_cbranch_scc1 store
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
store:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
done:
|
|
s_endpgm)",
|
|
{0x7e020280u, 0xbf06c002u, 0xbf840003u, 0x4a020081u, 0xbf068003u, 0xbf850011u, 0xbf85000eu, 0xe0300000u, 0x80000200u, 0x10060302u,
|
|
0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u,
|
|
0x4a02020bu, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Route},
|
|
{"entered region beside early returns", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_cbranch_vccz body
|
|
v_add_nc_u32 v1, 1, v0
|
|
v_cmp_gt_u32 vcc, 8, v1
|
|
s_cbranch_vccnz tail
|
|
v_readfirstlane_b32 s11, v1
|
|
s_cmp_eq_u32 s11, 12
|
|
s_cbranch_scc0 done
|
|
s_cmp_eq_u32 s11, 38
|
|
s_cbranch_scc1 done
|
|
body:
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
tail:
|
|
v_add_nc_u32 v1, 1, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 2, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 3, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 4, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 5, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 6, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 7, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 8, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 9, v1
|
|
v_xor_b32 v1, v3, v1
|
|
v_add_nc_u32 v1, 10, v1
|
|
v_xor_b32 v1, v3, v1
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbf860008u, 0x4a020081u, 0x7d880288u, 0xbf870013u, 0x7e160501u, 0xbf068c0bu, 0xbf840024u, 0xbf06a60bu,
|
|
0xbf850022u, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u,
|
|
0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0x4a020281u, 0x3a020303u, 0x4a020282u, 0x3a020303u, 0x4a020283u,
|
|
0x3a020303u, 0x4a020284u, 0x3a020303u, 0x4a020285u, 0x3a020303u, 0x4a020286u, 0x3a020303u, 0x4a020287u, 0x3a020303u, 0x4a020288u,
|
|
0x3a020303u, 0x4a020289u, 0x3a020303u, 0x4a02028au, 0x3a020303u, 0xe0700000u, 0x80000100u, 0xbf810000u},
|
|
Split::Route, 4},
|
|
{"selection arm that only returns", R"(
|
|
s_cbranch_execz inner
|
|
s_cbranch_scc1 done
|
|
inner:
|
|
v_cmp_gt_u32 vcc, 3, v1
|
|
s_cbranch_vccnz second
|
|
v_mul_f32 v3, v2, v1
|
|
s_cbranch_vccz join
|
|
second:
|
|
s_cbranch_vccnz done
|
|
join:
|
|
v_add_nc_u32 v1, 2, v1
|
|
done:
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm)",
|
|
{0xbf880001u, 0xbf850006u, 0x7d880283u, 0xbf870002u, 0x10060302u, 0xbf860001u, 0xbf870001u, 0x4a020282u, 0xe0700000u, 0x80000100u,
|
|
0xbf810000u},
|
|
Split::Route, 9},
|
|
{"shared early exit beside an expensive region", R"(
|
|
v_mov_b32 v1, 0
|
|
v_cmp_gt_u32 vcc, 16, v0
|
|
s_and_saveexec_b64 s[8:9], vcc
|
|
s_cbranch_execz join
|
|
v_cmp_gt_u32 vcc, 4, v0
|
|
s_cbranch_vccz early_exit
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
join:
|
|
s_mov_b64 exec, s[8:9]
|
|
v_cmp_gt_u32 vcc, 32, v0
|
|
s_cbranch_vccz early_exit
|
|
buffer_load_dword v2, off, s[0:3], 0
|
|
v_mul_f32 v3, v2, v1
|
|
v_add_f32 v3, v3, v2
|
|
v_xor_b32 v1, v3, v1
|
|
v_readfirstlane_b32 s11, v1
|
|
v_add_nc_u32 v1, s11, v1
|
|
buffer_store_dword v1, off, s[0:3], 0
|
|
s_endpgm
|
|
early_exit:
|
|
s_mov_b64 exec, 0
|
|
s_endpgm)",
|
|
{0x7e020280u, 0x7d880090u, 0xbe88246au, 0xbf88000eu, 0x7d880084u, 0xbf860019u, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u,
|
|
0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0xbefe0408u, 0x7d8800a0u,
|
|
0xbf86000au, 0xe0300000u, 0x80000200u, 0x10060302u, 0x06060503u, 0x3a020303u, 0x7e160501u, 0x4a02020bu, 0xe0700000u, 0x80000100u,
|
|
0xbf810000u, 0xbefe0480u, 0xbf810000u},
|
|
Split::Clone},
|
|
};
|
|
int failures = 0;
|
|
for (const auto& program : programs) {
|
|
try {
|
|
verifyProgram(program);
|
|
} catch (const std::exception& error) {
|
|
std::fprintf(stderr, "%s\n%.*s\n", error.what(), static_cast<int>(program.source.size()), program.source.data());
|
|
++failures;
|
|
}
|
|
}
|
|
if (failures != 0) return 1;
|
|
std::puts("Control flow structurization tests passed");
|
|
return 0;
|
|
}
|