Merge pull request #327 from slick-daddy/opt-2
perf(shader): intern SPIR-V type and constant ids without a per-call allocation
This commit is contained in:
Adrià
GitHub
commit
e79c971f99
|
|
@ -273,6 +273,11 @@ target_link_libraries(agc_structurizer_tests PRIVATE shader_recompiler_agc_drive
|
|||
set_target_properties(agc_structurizer_tests PROPERTIES CXX_EXTENSIONS OFF)
|
||||
configure_windows_unwind(agc_structurizer_tests)
|
||||
|
||||
add_test_executable(agc_spirv_module_tests EXCLUDE_FROM_ALL tests/SpirvModule.cpp)
|
||||
target_link_libraries(agc_spirv_module_tests PRIVATE shader_recompiler_agc_driver libc)
|
||||
set_target_properties(agc_spirv_module_tests PROPERTIES CXX_EXTENSIONS OFF)
|
||||
configure_windows_unwind(agc_spirv_module_tests)
|
||||
|
||||
add_test_executable(agc_dpp_write_tests EXCLUDE_FROM_ALL tests/DppWrite.cpp)
|
||||
target_link_libraries(agc_dpp_write_tests PRIVATE shader_recompiler_agc_driver libc)
|
||||
set_target_properties(agc_dpp_write_tests PROPERTIES CXX_EXTENSIONS OFF)
|
||||
|
|
@ -516,7 +521,7 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
|
|||
endforeach()
|
||||
|
||||
if(BUILD_TESTING)
|
||||
foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests ${agcExecutionTests})
|
||||
foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests ${agcExecutionTests})
|
||||
string(REGEX REPLACE "_tests$" "" agcTestName "${agcTest}")
|
||||
set_target_properties(${agcTest} PROPERTIES EXCLUDE_FROM_ALL OFF)
|
||||
add_test(NAME ${agcTestName} COMMAND ${agcTest})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,203 @@
|
|||
#include "SpirvBackend/SpirvModule.hpp"
|
||||
#include <cstdio>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <spirv/unified1/spirv.hpp>
|
||||
#include <vector>
|
||||
|
||||
using namespace ShaderRecompiler;
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
void check(bool condition, const char* what) {
|
||||
if (!condition) {
|
||||
std::fprintf(stderr, "%s\n", what);
|
||||
++failures;
|
||||
}
|
||||
}
|
||||
|
||||
std::map<std::uint32_t, std::size_t> opcodeCounts(const std::vector<std::uint32_t>& words) {
|
||||
std::map<std::uint32_t, std::size_t> counts;
|
||||
std::size_t offset = 5;
|
||||
while (offset < words.size()) {
|
||||
const auto header = words[offset];
|
||||
const auto wordCount = header >> spv::WordCountShift;
|
||||
if (wordCount == 0 || offset + wordCount > words.size()) {
|
||||
return counts;
|
||||
}
|
||||
++counts[header & 0xffffu];
|
||||
offset += wordCount;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
std::size_t declared(const SpirvModule& module, std::uint32_t opcode) {
|
||||
const auto counts = opcodeCounts(module.Finalize());
|
||||
const auto found = counts.find(opcode);
|
||||
return found == counts.end() ? 0 : found->second;
|
||||
}
|
||||
|
||||
void testRepeatedTypesShareOneId() {
|
||||
SpirvModule module;
|
||||
const auto first = module.Type(spv::OpTypeInt, 32u, 0u);
|
||||
const auto second = module.Type(spv::OpTypeInt, 32u, 0u);
|
||||
check(first == second, "a repeated OpTypeInt 32 0 returned a different id");
|
||||
check(declared(module, spv::OpTypeInt) == 1, "a repeated OpTypeInt 32 0 was declared more than once");
|
||||
check(module.Type(spv::OpTypeVoid) == module.Type(spv::OpTypeVoid), "a repeated OpTypeVoid returned a different id");
|
||||
check(declared(module, spv::OpTypeVoid) == 1, "OpTypeVoid was not declared exactly once");
|
||||
check(module.Type(spv::OpTypeBool) == module.Type(spv::OpTypeBool), "a repeated OpTypeBool returned a different id");
|
||||
check(module.Type(spv::OpTypeFloat, 32u) == module.Type(spv::OpTypeFloat, 32u), "a repeated OpTypeFloat 32 returned a different id");
|
||||
check(declared(module, spv::OpTypeFloat) == 1, "OpTypeFloat was not declared exactly once");
|
||||
}
|
||||
|
||||
void testOperandShapesDoNotCollide() {
|
||||
SpirvModule module;
|
||||
const auto oneWord = module.Type(spv::OpTypeInt, 32u);
|
||||
const auto twoWords = module.Type(spv::OpTypeInt, 32u, 0u);
|
||||
check(oneWord != twoWords, "OpTypeInt with one operand collided with OpTypeInt with two");
|
||||
check(declared(module, spv::OpTypeInt) == 2, "the two OpTypeInt shapes were not both declared");
|
||||
const auto voidType = module.Type(spv::OpTypeVoid);
|
||||
const auto boolType = module.Type(spv::OpTypeBool);
|
||||
check(voidType != boolType, "OpTypeVoid collided with OpTypeBool");
|
||||
check(module.Type(spv::OpTypeInt, 32u, 0u) != module.Type(spv::OpTypeInt, 32u, 1u), "unsigned and signed OpTypeInt 32 collided");
|
||||
check(module.Type(spv::OpTypeInt, 32u, 0u) != module.Type(spv::OpTypeInt, 64u, 0u), "OpTypeInt 32 collided with OpTypeInt 64");
|
||||
check(module.Type(spv::OpTypeFloat, 32u) != module.Type(spv::OpTypeFloat, 64u), "OpTypeFloat 32 collided with OpTypeFloat 64");
|
||||
}
|
||||
|
||||
void testOperandKindsAgree() {
|
||||
SpirvModule module;
|
||||
const auto fromUnsigned = module.Type(spv::OpTypeInt, 32u, 0u);
|
||||
const auto fromSigned = module.Type(spv::OpTypeInt, static_cast<std::int32_t>(32), static_cast<std::int32_t>(0));
|
||||
check(fromUnsigned == fromSigned, "OpTypeInt disagreed between unsigned and signed operands");
|
||||
const auto typeU32 = module.Type(spv::OpTypeInt, 32u, 0u);
|
||||
const auto fromEnum = module.Type(spv::OpTypePointer, spv::StorageClassFunction, typeU32);
|
||||
const auto fromWord = module.Type(spv::OpTypePointer, static_cast<std::uint32_t>(spv::StorageClassFunction), typeU32);
|
||||
check(fromEnum == fromWord, "OpTypePointer disagreed between enum and integer operands");
|
||||
check(fromEnum == module.Type(spv::OpTypePointer, spv::StorageClassFunction, typeU32), "a repeated enum-operand OpTypePointer returned a different id");
|
||||
check(declared(module, spv::OpTypePointer) == 1, "OpTypePointer was not declared exactly once");
|
||||
}
|
||||
|
||||
void testDecoratedTypesStayDistinct() {
|
||||
SpirvModule module;
|
||||
const auto typeU32 = module.Type(spv::OpTypeInt, 32u, 0u);
|
||||
const auto plain = module.Type(spv::OpTypeRuntimeArray, typeU32);
|
||||
const auto decorated = module.DecoratedType(spv::OpTypeRuntimeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, 4u}}}, typeU32);
|
||||
check(plain != decorated, "a plain OpTypeRuntimeArray collided with the decorated one");
|
||||
check(decorated == module.DecoratedType(spv::OpTypeRuntimeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, 4u}}}, typeU32), "a repeated decorated type returned a different id");
|
||||
check(declared(module, spv::OpTypeRuntimeArray) == 2, "the plain and decorated OpTypeRuntimeArray were not both declared");
|
||||
}
|
||||
|
||||
void testConstantsShareOneId() {
|
||||
SpirvModule module;
|
||||
const auto typeU32 = module.Type(spv::OpTypeInt, 32u, 0u);
|
||||
const auto typeBool = module.Type(spv::OpTypeBool);
|
||||
check(module.Constant(spv::OpConstant, typeU32, 0u) == module.Constant(spv::OpConstant, typeU32, 0u), "a repeated OpConstant 0 returned a different id");
|
||||
check(module.Constant(spv::OpConstant, typeU32, 0u) != module.Constant(spv::OpConstant, typeU32, 1u), "OpConstant 0 collided with OpConstant 1");
|
||||
check(module.Constant(spv::OpConstant, typeU32, 1000u) != module.Constant(spv::OpConstant, typeU32, 0u), "OpConstant 1000 collided with OpConstant 0");
|
||||
check(module.Constant(spv::OpConstantTrue, typeBool) == module.Constant(spv::OpConstantTrue, typeBool), "a repeated OpConstantTrue returned a different id");
|
||||
check(module.Constant(spv::OpConstantTrue, typeBool) != module.Constant(spv::OpConstantFalse, typeBool), "OpConstantTrue collided with OpConstantFalse");
|
||||
check(declared(module, spv::OpConstant) == 3, "OpConstant was not declared once per distinct value");
|
||||
check(declared(module, spv::OpConstantTrue) == 1, "OpConstantTrue was not declared exactly once");
|
||||
const auto signedId = module.Constant(spv::OpConstant, module.Type(spv::OpTypeInt, 32u, 1u), static_cast<std::uint32_t>(-1));
|
||||
check(signedId != 0, "a signed OpConstant produced a zero id");
|
||||
}
|
||||
|
||||
void testOperandFreeConstantsShareOneId() {
|
||||
SpirvModule module;
|
||||
const auto typeBool = module.Type(spv::OpTypeBool);
|
||||
const auto typeVoid = module.Type(spv::OpTypeVoid);
|
||||
check(module.Constant(spv::OpConstantTrue, typeBool) == module.Constant(spv::OpConstantTrue, typeBool), "a repeated OpConstantTrue returned a different id");
|
||||
check(module.Constant(spv::OpConstantTrue, typeBool) != module.Constant(spv::OpConstantFalse, typeBool), "OpConstantTrue collided with OpConstantFalse");
|
||||
check(module.Constant(spv::OpConstantTrue, typeVoid) != module.Constant(spv::OpConstantTrue, typeBool), "OpConstantTrue of different types collided");
|
||||
check(declared(module, spv::OpConstantTrue) == 2, "OpConstantTrue was not declared once per type");
|
||||
check(declared(module, spv::OpConstantFalse) == 1, "OpConstantFalse was not declared exactly once");
|
||||
check(module.Type(spv::OpTypeVoid) == typeVoid, "a repeated operand-free type returned a different id");
|
||||
}
|
||||
|
||||
void testIdsSurviveInterleavedAllocation() {
|
||||
SpirvModule module;
|
||||
const auto typeU32 = module.Type(spv::OpTypeInt, 32u, 0u);
|
||||
const auto constant = module.Constant(spv::OpConstant, typeU32, 4u);
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
check(module.AllocateId() != 0, "AllocateId returned zero");
|
||||
check(module.Type(spv::OpTypeInt, 32u, 0u) == typeU32, "a memoized type changed id after AllocateId");
|
||||
check(module.Constant(spv::OpConstant, typeU32, 4u) == constant, "a memoized constant changed id after AllocateId");
|
||||
}
|
||||
}
|
||||
|
||||
void testModulesAreIndependent() {
|
||||
SpirvModule first;
|
||||
SpirvModule second;
|
||||
const auto firstId = first.Type(spv::OpTypeInt, 32u, 0u);
|
||||
const auto secondId = second.Type(spv::OpTypeInt, 32u, 0u);
|
||||
check(firstId == 1, "the first id of a fresh module was not 1");
|
||||
check(secondId == 1, "a second module did not start its ids from 1");
|
||||
first.Type(spv::OpTypeVoid);
|
||||
check(second.Type(spv::OpTypeInt, 32u, 0u) == secondId, "a second module's ids moved when the first module was used");
|
||||
}
|
||||
|
||||
void testIdsStayUniqueBeyondCacheCapacity() {
|
||||
SpirvModule module;
|
||||
const auto typeU32 = module.Type(spv::OpTypeInt, 32u, 0u);
|
||||
std::vector<std::uint32_t> ids;
|
||||
ids.reserve(4096);
|
||||
for (std::uint32_t value = 0; value < 4096; ++value) {
|
||||
ids.push_back(module.Constant(spv::OpConstant, typeU32, value));
|
||||
}
|
||||
const std::set<std::uint32_t> distinct(ids.begin(), ids.end());
|
||||
check(distinct.size() == ids.size(), "distinct constants shared an id once the cache was overrun");
|
||||
for (std::uint32_t value = 0; value < 4096; ++value) {
|
||||
if (module.Constant(spv::OpConstant, typeU32, value) != ids[value]) {
|
||||
std::fprintf(stderr, "constant %u changed id after the cache was overrun\n", value);
|
||||
++failures;
|
||||
break;
|
||||
}
|
||||
}
|
||||
std::uint32_t largest = 0;
|
||||
for (const auto id : distinct) {
|
||||
largest = largest < id ? id : largest;
|
||||
}
|
||||
const auto first = ids.front();
|
||||
check(largest - first + 1 == distinct.size(), "the constant ids were not contiguous, so some were never declared");
|
||||
check(first > typeU32, "a constant was numbered before the type it uses");
|
||||
}
|
||||
|
||||
void testEmittedWordsAreStable() {
|
||||
SpirvModule module;
|
||||
const auto typeU32 = module.Type(spv::OpTypeInt, 32u, 0u);
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
module.Type(spv::OpTypeVector, typeU32, 4u);
|
||||
module.Type(spv::OpTypePointer, spv::StorageClassFunction, typeU32);
|
||||
module.Constant(spv::OpConstant, typeU32, static_cast<std::uint32_t>(i % 8));
|
||||
module.AddFunction(spv::OpNop);
|
||||
}
|
||||
const auto words = module.Finalize();
|
||||
check(!words.empty(), "Finalize produced no words");
|
||||
const auto counts = opcodeCounts(words);
|
||||
check(counts.at(spv::OpTypeVector) == 1, "OpTypeVector was declared more than once");
|
||||
check(counts.at(spv::OpTypePointer) == 1, "OpTypePointer was declared more than once");
|
||||
check(counts.at(spv::OpConstant) == 8, "OpConstant was not declared once per distinct value");
|
||||
check(counts.at(spv::OpNop) == 32, "the function body lost instructions");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main() {
|
||||
testRepeatedTypesShareOneId();
|
||||
testOperandShapesDoNotCollide();
|
||||
testOperandKindsAgree();
|
||||
testDecoratedTypesStayDistinct();
|
||||
testConstantsShareOneId();
|
||||
testOperandFreeConstantsShareOneId();
|
||||
testIdsSurviveInterleavedAllocation();
|
||||
testModulesAreIndependent();
|
||||
testIdsStayUniqueBeyondCacheCapacity();
|
||||
testEmittedWordsAreStable();
|
||||
if (failures != 0) {
|
||||
std::fprintf(stderr, "%d check(s) failed\n", failures);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
#ifndef CORE_SHADER_RECOMPILIER_SPIRVBACKEND_INCLUDE_SPIRVBACKEND_SPIRVMODULE_HPP
|
||||
#define CORE_SHADER_RECOMPILIER_SPIRVBACKEND_INCLUDE_SPIRVBACKEND_SPIRVMODULE_HPP
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <initializer_list>
|
||||
|
|
@ -25,6 +27,15 @@ struct SpirvDeferredPhi {
|
|||
};
|
||||
|
||||
class SpirvModule {
|
||||
private:
|
||||
template<typename TOperand>
|
||||
static constexpr bool wordSized = (std::is_integral_v<TOperand> || std::is_enum_v<TOperand>) && sizeof(TOperand) <= sizeof(std::uint32_t);
|
||||
|
||||
template<typename TOperand>
|
||||
static std::uint32_t narrowWord(TOperand operand) {
|
||||
return static_cast<std::uint32_t>(operand);
|
||||
}
|
||||
|
||||
public:
|
||||
explicit SpirvModule(std::uint32_t version = 0x00010300u);
|
||||
[[nodiscard]] std::uint32_t AllocateId();
|
||||
|
|
@ -51,6 +62,28 @@ public:
|
|||
return declareType(opcode, makeTypeKey(opcode, operands...));
|
||||
}
|
||||
|
||||
std::uint32_t Type(std::uint32_t opcode) {
|
||||
const std::uint32_t key[2] = {opcode, 0u};
|
||||
return interned(key, 2u, [this, opcode]() { return declareType(opcode, makeTypeKey(opcode)); });
|
||||
}
|
||||
|
||||
template<typename TWord>
|
||||
requires wordSized<TWord>
|
||||
std::uint32_t Type(std::uint32_t opcode, TWord width) {
|
||||
const auto narrowed = narrowWord(width);
|
||||
const std::uint32_t key[3] = {opcode, 1u, narrowed};
|
||||
return interned(key, 3u, [this, opcode, width]() { return declareType(opcode, makeTypeKey(opcode, width)); });
|
||||
}
|
||||
|
||||
template<typename TWord, typename TSign>
|
||||
requires (wordSized<TWord> && wordSized<TSign>)
|
||||
std::uint32_t Type(std::uint32_t opcode, TWord width, TSign signness) {
|
||||
const auto narrowedWidth = narrowWord(width);
|
||||
const auto narrowedSignness = narrowWord(signness);
|
||||
const std::uint32_t key[4] = {opcode, 2u, narrowedWidth, narrowedSignness};
|
||||
return interned(key, 4u, [this, opcode, width, signness]() { return declareType(opcode, makeTypeKey(opcode, width, signness)); });
|
||||
}
|
||||
|
||||
template<typename... TOperands>
|
||||
std::uint32_t DecoratedType(std::uint32_t opcode, std::initializer_list<SpirvTypeAnnotation> annotations, const TOperands&... operands) {
|
||||
return declareDecoratedType(opcode, makeTypeKey(opcode, operands...), annotations);
|
||||
|
|
@ -64,6 +97,29 @@ public:
|
|||
return declareConstant(opcode, std::move(key));
|
||||
}
|
||||
|
||||
std::uint32_t Constant(std::uint32_t opcode, std::uint32_t type) {
|
||||
const std::uint32_t key[2] = {opcode, type};
|
||||
return interned(key, 2u, [this, opcode, type]() {
|
||||
std::vector<std::uint32_t> keyWords;
|
||||
keyWords.reserve(2u);
|
||||
appendOperands(keyWords, opcode, type);
|
||||
return declareConstant(opcode, std::move(keyWords));
|
||||
});
|
||||
}
|
||||
|
||||
template<typename TValue>
|
||||
requires wordSized<TValue>
|
||||
std::uint32_t Constant(std::uint32_t opcode, std::uint32_t type, TValue value) {
|
||||
const auto narrowed = narrowWord(value);
|
||||
const std::uint32_t key[3] = {opcode, type, narrowed};
|
||||
return interned(key, 3u, [this, opcode, type, value]() {
|
||||
std::vector<std::uint32_t> keyWords;
|
||||
keyWords.reserve(3u);
|
||||
appendOperands(keyWords, opcode, type, value);
|
||||
return declareConstant(opcode, std::move(keyWords));
|
||||
});
|
||||
}
|
||||
|
||||
template<typename... TOperands>
|
||||
void AddExecutionMode(std::uint32_t entryPoint, std::uint32_t mode, const TOperands&... operands) {
|
||||
appendInstruction(executionModes, spv::OpExecutionMode, entryPoint, mode, operands...);
|
||||
|
|
@ -80,6 +136,46 @@ public:
|
|||
}
|
||||
|
||||
private:
|
||||
static constexpr std::size_t DeclarationCacheSize = 512;
|
||||
static constexpr std::size_t MaxCachedDeclarationWords = 4;
|
||||
|
||||
struct DeclarationSlot {
|
||||
std::array<std::uint32_t, MaxCachedDeclarationWords> words = {};
|
||||
std::uint32_t length = 0;
|
||||
std::uint32_t id = 0;
|
||||
};
|
||||
|
||||
static std::size_t declarationCacheSlot(const std::uint32_t* words, std::uint32_t length) {
|
||||
std::uint64_t hash = 14695981039346656037ull;
|
||||
for (std::uint32_t index = 0; index < length; index++) {
|
||||
hash ^= words[index];
|
||||
hash *= 1099511628211ull;
|
||||
}
|
||||
return static_cast<std::size_t>(hash) & (DeclarationCacheSize - 1u);
|
||||
}
|
||||
|
||||
template<typename TDeclare>
|
||||
std::uint32_t interned(const std::uint32_t* words, std::uint32_t length, TDeclare&& declare) {
|
||||
const auto start = declarationCacheSlot(words, length);
|
||||
std::size_t target = start;
|
||||
for (std::size_t probe = 0; probe < DeclarationCacheSize; ++probe) {
|
||||
target = (start + probe) & (DeclarationCacheSize - 1u);
|
||||
auto& slot = declarationCache[target];
|
||||
if (slot.id == 0) {
|
||||
break;
|
||||
}
|
||||
if (slot.length == length && std::equal(words, words + length, slot.words.begin())) {
|
||||
return slot.id;
|
||||
}
|
||||
}
|
||||
const auto id = declare();
|
||||
auto& slot = declarationCache[target];
|
||||
std::copy(words, words + length, slot.words.begin());
|
||||
slot.length = length;
|
||||
slot.id = id;
|
||||
return id;
|
||||
}
|
||||
|
||||
static void appendOperand(std::vector<std::uint32_t>& words, std::uint32_t value) {
|
||||
words.push_back(value);
|
||||
}
|
||||
|
|
@ -147,6 +243,7 @@ private:
|
|||
std::set<std::string> requiredExtensions;
|
||||
std::map<std::string, std::uint32_t> importIds;
|
||||
std::map<std::vector<std::uint32_t>, std::uint32_t> declarationIds;
|
||||
std::array<DeclarationSlot, DeclarationCacheSize> declarationCache = {};
|
||||
std::size_t unpatchedPhiIncomings = 0;
|
||||
std::vector<std::uint32_t> capabilities;
|
||||
std::vector<std::uint32_t> extensions;
|
||||
|
|
|
|||
Loading…
Reference in New Issue