428 lines
20 KiB
C++
428 lines
20 KiB
C++
#include "Recompiler.hpp"
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <cstdlib>
|
|
#include <cstdio>
|
|
#include "CacheKey.hpp"
|
|
#include <mutex>
|
|
#include <shared_mutex>
|
|
#include <unordered_map>
|
|
#include "ControlFlow/include/ControlFlow/GraphBuilder.hpp"
|
|
#include "ControlFlow/include/ControlFlow/Structurizer.hpp"
|
|
#include "RdnaDecoder/include/RdnaDecoder/RdnaInstructionDecoder.hpp"
|
|
#include "IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp"
|
|
#include "Optimization/include/Optimization/BindingAllocator.hpp"
|
|
#include "Optimization/include/Optimization/ConstantFolder.hpp"
|
|
#include "Optimization/include/Optimization/DeadCodeEliminator.hpp"
|
|
#include "Optimization/include/Optimization/DescriptorBindingBuilder.hpp"
|
|
#include "Optimization/include/Optimization/ReadLaneEliminator.hpp"
|
|
#include "Optimization/include/Optimization/RequestMemoryView.hpp"
|
|
#include "Optimization/include/Optimization/ResourceMaterializer.hpp"
|
|
#include "Optimization/ResourceProgram.hpp"
|
|
#include "Optimization/include/Optimization/ResourceTracker.hpp"
|
|
#include "Optimization/include/Optimization/ShaderInfoCollector.hpp"
|
|
#include "Optimization/include/Optimization/SrtWalker.hpp"
|
|
#include "Optimization/include/Optimization/SsaBuilder.hpp"
|
|
#include "SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp"
|
|
#if ANYPS5_ENABLE_SPIRV_TOOLS
|
|
#include "SpirvBackend/SpirvOptimizer.hpp"
|
|
#endif
|
|
#include "SpirvBackend/SpirvMemory/SpirvInputOutput.hpp"
|
|
#include "Translation/include/Translation/InstructionTranslator.hpp"
|
|
#include "Translation/include/Translation/ShaderInputInfoBuilder.hpp"
|
|
#include <exception>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <ControlFlow/RequestSerializer.hpp>
|
|
|
|
namespace ShaderRecompiler {
|
|
|
|
namespace {
|
|
|
|
ShaderStageKind toShaderStageKind(ShaderStage stage) {
|
|
switch (stage) {
|
|
case ShaderStage::Compute:
|
|
return ShaderStageKind::Compute;
|
|
case ShaderStage::Vertex:
|
|
return ShaderStageKind::Vertex;
|
|
case ShaderStage::TessellationControl:
|
|
return ShaderStageKind::TessellationControl;
|
|
case ShaderStage::TessellationEvaluation:
|
|
return ShaderStageKind::TessellationEvaluation;
|
|
case ShaderStage::Fragment:
|
|
return ShaderStageKind::Pixel;
|
|
case ShaderStage::Local:
|
|
return ShaderStageKind::Local;
|
|
case ShaderStage::Mesh:
|
|
return ShaderStageKind::Mesh;
|
|
case ShaderStage::Geometry:
|
|
break;
|
|
}
|
|
throw std::runtime_error("ShaderRecompiler::Recompile: unsupported shader stage");
|
|
}
|
|
|
|
}
|
|
|
|
namespace {
|
|
|
|
// The host subgroup width wave64 programs are laid out for. Debug aid: APS5_SINGLE_LANE=<hex code
|
|
// addresses, comma separated, or "all"> keeps the listed programs at one guest lane per invocation.
|
|
std::uint32_t HostSubgroupSize(const RecompileRequest& request) {
|
|
static const std::string list = [] { const char* text = std::getenv("APS5_SINGLE_LANE"); return text ? std::string(text) : std::string(); }();
|
|
if (!list.empty()) {
|
|
if (list == "all") return 64u;
|
|
char address[32];
|
|
std::snprintf(address, sizeof(address), "%llx", static_cast<unsigned long long>(request.shader.codeAddress));
|
|
if (list.find(address) != std::string::npos) return 64u;
|
|
}
|
|
return request.target.subgroupSize;
|
|
}
|
|
|
|
}
|
|
|
|
IrProgram PrepareResourceProgram(const RecompileRequest& request) {
|
|
const auto stageKind = toShaderStageKind(request.shader.stage);
|
|
const auto inputInfo = BuildShaderStageInputInfo(stageKind, request.context, HostSubgroupSize(request));
|
|
|
|
constexpr RdnaInstructionDecoder decoder;
|
|
const auto decoded = decoder.Decode(request.shader.code);
|
|
|
|
constexpr GraphBuilder graphBuilder;
|
|
auto cfg = graphBuilder.Build(decoded);
|
|
|
|
constexpr Structurizer structurizer;
|
|
structurizer.Structurize(cfg);
|
|
|
|
TranslateOptions translateOptions {};
|
|
translateOptions.stage = stageKind;
|
|
translateOptions.waveSize = request.context.waveSize;
|
|
translateOptions.userDataBaseRegister = request.context.userDataBaseRegister;
|
|
translateOptions.userDataCount = static_cast<std::uint32_t>(request.context.userData.size());
|
|
translateOptions.embeddedFetch = nullptr;
|
|
translateOptions.fragmentShaderBarycentricEnabled = request.target.fragmentShaderBarycentricEnabled;
|
|
translateOptions.inputInfo = inputInfo;
|
|
|
|
constexpr InstructionTranslator translator;
|
|
|
|
EmbeddedFetchPlan embeddedFetch;
|
|
if ((stageKind == ShaderStageKind::Vertex || stageKind == ShaderStageKind::Local) && inputInfo.vertex != nullptr && inputInfo.vertex->fetchEmbedded) {
|
|
constexpr EmbeddedVertexFetchAnalyzer embeddedFetchAnalyzer;
|
|
embeddedFetch = embeddedFetchAnalyzer.Analyze(decoded, inputInfo.vertex->fetchAttribReg, inputInfo.vertex->fetchBufferReg, request.context.userDataBaseRegister, static_cast<std::uint32_t>(request.context.userData.size()), request.context.waveSize);
|
|
}
|
|
translateOptions.embeddedFetch = embeddedFetch.loads.empty() ? nullptr : &embeddedFetch;
|
|
|
|
auto program = translator.Translate(decoded, cfg, translateOptions);
|
|
// Debug aid: APS5_DUMP_IR=<hex code address> (or "all") prints the program after each front-end pass.
|
|
const auto dumpIr = [&](const char* pass) {
|
|
static const std::string list = [] { const char* text = std::getenv("APS5_DUMP_IR"); return text ? std::string(text) : std::string(); }();
|
|
if (list.empty()) return;
|
|
char address[32];
|
|
std::snprintf(address, sizeof(address), "%llx", static_cast<unsigned long long>(request.shader.codeAddress));
|
|
if (list != "all" && list.find(address) == std::string::npos) return;
|
|
std::fprintf(stderr, "==== IR 0x%s after %s\n%s\n", address, pass, ProgramToString(program).c_str());
|
|
};
|
|
dumpIr("translate");
|
|
|
|
constexpr SsaBuilder ssaBuilder;
|
|
ssaBuilder.Rewrite(program);
|
|
dumpIr("ssa");
|
|
|
|
constexpr ConstantFolder constantFolder;
|
|
constexpr DeadCodeEliminator deadCodeEliminator;
|
|
|
|
constantFolder.Fold(program);
|
|
ResolveControlFlowIdentities(program);
|
|
deadCodeEliminator.RemoveIdentities(program);
|
|
deadCodeEliminator.Eliminate(program);
|
|
dumpIr("fold");
|
|
|
|
constexpr ReadLaneEliminator readLaneEliminator;
|
|
const auto readLaneStats = readLaneEliminator.Eliminate(program, translateOptions.waveSize);
|
|
if (readLaneStats.rewrittenReads != 0u) {
|
|
constantFolder.Fold(program);
|
|
ResolveControlFlowIdentities(program);
|
|
deadCodeEliminator.RemoveIdentities(program);
|
|
deadCodeEliminator.Eliminate(program);
|
|
}
|
|
|
|
constexpr SrtWalker srtWalker;
|
|
srtWalker.BuildPlan(program);
|
|
deadCodeEliminator.Eliminate(program);
|
|
dumpIr("srt");
|
|
|
|
constexpr ResourceTracker resourceTracker;
|
|
resourceTracker.Track(program);
|
|
deadCodeEliminator.Eliminate(program);
|
|
dumpIr("resources");
|
|
|
|
return program;
|
|
}
|
|
|
|
struct CompiledVariant {
|
|
ResourceSpecialization specialization;
|
|
BindingLayout layout;
|
|
CompiledShaderInfo info;
|
|
BindingAllocationResult bindings;
|
|
RecompileResult result;
|
|
};
|
|
|
|
struct SourceEntry {
|
|
std::mutex mutex;
|
|
// The code the entry was built for: the key carries only a hash of it, so a candidate entry is
|
|
// accepted only when its code matches word for word. Owned here because the request's span
|
|
// points into a registration the driver may replace while the entry lives on.
|
|
std::vector<std::uint32_t> code;
|
|
std::shared_ptr<const IrResourcePlan> plan;
|
|
std::vector<std::shared_ptr<const CompiledVariant>> variants;
|
|
};
|
|
|
|
namespace {
|
|
|
|
struct ResourceProgram {
|
|
explicit ResourceProgram(const RecompileRequest& request) : program(PrepareResourceProgram(request)), plan(ResourceMaterializer{}.ExtractPlan(program)) {}
|
|
|
|
IrProgram program;
|
|
IrResourcePlan plan;
|
|
};
|
|
|
|
std::shared_ptr<const IrResourcePlan> makeResourcePlan(const RecompileRequest& request) {
|
|
const auto resource = std::make_shared<ResourceProgram>(request);
|
|
return std::shared_ptr<const IrResourcePlan>(resource, &resource->plan);
|
|
}
|
|
|
|
struct SourceKeyHash {
|
|
std::size_t operator()(const std::vector<std::uint64_t>& key) const {
|
|
std::size_t hash = 0;
|
|
for (const auto value : key) {
|
|
hash ^= static_cast<std::size_t>(value) + static_cast<std::size_t>(0x9e3779b97f4a7c15ull) + (hash << 6u) + (hash >> 2u);
|
|
if constexpr (sizeof(std::size_t) < sizeof(value)) hash ^= static_cast<std::size_t>(value >> 32u);
|
|
}
|
|
return hash;
|
|
}
|
|
};
|
|
|
|
std::shared_ptr<SourceEntry> getSource(const RecompileRequest& request) {
|
|
static std::shared_mutex mutex;
|
|
// Entries whose code hashes alike share a bucket; the code comparison picks the right one.
|
|
static std::unordered_map<std::vector<std::uint64_t>, std::vector<std::shared_ptr<SourceEntry>>, SourceKeyHash> sources;
|
|
thread_local std::vector<std::uint64_t> key;
|
|
RecompileCacheKey::Build(request, key);
|
|
const auto find = [&]() -> std::shared_ptr<SourceEntry> {
|
|
const auto found = sources.find(key);
|
|
if (found == sources.end()) return nullptr;
|
|
for (const auto& entry : found->second) {
|
|
if (std::equal(entry->code.begin(), entry->code.end(), request.shader.code.begin(), request.shader.code.end())) return entry;
|
|
}
|
|
return nullptr;
|
|
};
|
|
std::shared_ptr<SourceEntry> source;
|
|
{
|
|
std::shared_lock lock(mutex);
|
|
source = find();
|
|
}
|
|
if (source == nullptr) {
|
|
std::unique_lock lock(mutex);
|
|
source = find();
|
|
if (source == nullptr) {
|
|
source = std::make_shared<SourceEntry>();
|
|
source->code.assign(request.shader.code.begin(), request.shader.code.end());
|
|
auto& bucket = sources[key];
|
|
if (!bucket.empty()) {
|
|
// A second entry under one key is a code hash collision (or the unhashed key with
|
|
// identical code, which cannot happen); each is reported under the profile switch.
|
|
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
|
|
static std::uint64_t collisions = 0;
|
|
++collisions;
|
|
if (profile) std::fprintf(stderr, "[recompile] source key collision %llu: %zu entries share a key (%zu code words)\n", static_cast<unsigned long long>(collisions), bucket.size() + 1, request.shader.code.size());
|
|
}
|
|
bucket.push_back(source);
|
|
}
|
|
}
|
|
{
|
|
std::lock_guard lock(source->mutex);
|
|
if (source->plan == nullptr) {
|
|
source->plan = makeResourcePlan(request);
|
|
}
|
|
}
|
|
return source;
|
|
}
|
|
|
|
CompiledVariant compileVariant(const RecompileRequest& request, IrProgram program, const ResourceSnapshot& resourceSnapshot, const ResourceSpecialization& resourceSpecialization) {
|
|
const auto inputInfo = BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request));
|
|
constexpr DeadCodeEliminator deadCodeEliminator;
|
|
constexpr ResourceMaterializer resourceMaterializer;
|
|
resourceMaterializer.Apply(program, resourceSpecialization);
|
|
|
|
deadCodeEliminator.RemoveIdentities(program);
|
|
deadCodeEliminator.Eliminate(program);
|
|
|
|
constexpr ShaderInfoCollector shaderInfoCollector;
|
|
shaderInfoCollector.Collect(program, inputInfo);
|
|
|
|
constexpr BindingAllocator bindingAllocator;
|
|
auto bindings = bindingAllocator.Allocate(program, request.layout);
|
|
|
|
constexpr DescriptorBindingBuilder descriptorBindingBuilder;
|
|
descriptorBindingBuilder.Populate(bindings, program, resourceSnapshot);
|
|
|
|
SpirvTargetOptions targetOptions {};
|
|
targetOptions.vulkanVersion = request.target.vulkanVersion;
|
|
targetOptions.spirvVersion = request.target.spirvVersion;
|
|
targetOptions.subgroupSize = request.target.subgroupSize;
|
|
targetOptions.bdaAbiVersion = request.target.bdaAbiVersion;
|
|
targetOptions.supportedCapabilities = request.target.supportedCapabilities;
|
|
targetOptions.supportedExtensions = request.target.supportedExtensions;
|
|
|
|
constexpr SpirvEmitter spirvEmitter;
|
|
RecompileResult result;
|
|
static std::atomic<std::uint64_t> variants{0};
|
|
result.variantId = variants.fetch_add(1, std::memory_order_relaxed) + 1;
|
|
result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions);
|
|
|
|
#if ANYPS5_ENABLE_SPIRV_TOOLS
|
|
result.spirv = ValidateAndOptimizeSpirv(result.spirv, request.target.vulkanVersion, request.target.spirvVersion);
|
|
#endif
|
|
|
|
result.bdaAbiVersion = program.Info().usesDma ? request.target.bdaAbiVersion : 0u;
|
|
result.vertexOffsetSgpr = program.Info().vertexOffsetSgpr;
|
|
result.instanceOffsetSgpr = program.Info().instanceOffsetSgpr;
|
|
for (const auto& output : program.Info().outputs) {
|
|
if (output.kind == StageOutputKind::Parameter) result.parameterExports.push_back(output.location);
|
|
}
|
|
if (request.shader.stage == ShaderStage::Fragment) result.fragmentParameters = DescribeFragmentParameters(program, inputInfo);
|
|
if (request.shader.stage == ShaderStage::Vertex || request.shader.stage == ShaderStage::Local) {
|
|
if (inputInfo.vertex == nullptr) throw std::runtime_error("vertex input metadata is missing");
|
|
for (const auto& input : program.Info().inputs) {
|
|
if (input.kind != StageInputKind::Parameter) continue;
|
|
if (input.location >= static_cast<std::uint32_t>(inputInfo.vertex->resourcesNum)) throw std::runtime_error("vertex attribute location exceeds resource count");
|
|
result.vertexAttributes.push_back({input.location, input.componentCount, {inputInfo.vertex->resources[input.location].fields}, inputInfo.vertex->resourcesDst[input.location].fetchIndex});
|
|
}
|
|
}
|
|
|
|
result.bindings.clear();
|
|
result.pushConstants.clear();
|
|
for (auto& attribute : result.vertexAttributes) attribute.resource = {};
|
|
bindings.bindings.clear();
|
|
bindings.pushConstants.clear();
|
|
return {resourceSpecialization, request.layout, std::move(program).TakeCompiledInfo(), std::move(bindings), std::move(result)};
|
|
}
|
|
|
|
RecompileResult materializeResult(const CompiledVariant& variant, const RecompileRequest& request, const ResourceSnapshot& snapshot) {
|
|
auto result = variant.result;
|
|
BindingAllocationResult bindings;
|
|
bindings.layout = variant.bindings.layout;
|
|
bindings.pushConstantOffsetBytes = variant.bindings.pushConstantOffsetBytes;
|
|
bindings.pushConstantSizeBytes = variant.bindings.pushConstantSizeBytes;
|
|
DescriptorBindingBuilder{}.Populate(bindings, variant.info.info, variant.info.stage, variant.info.userDataBase, snapshot);
|
|
result.bindings = std::move(bindings.bindings);
|
|
result.pushConstants = std::move(bindings.pushConstants);
|
|
for (auto& attribute : result.vertexAttributes) {
|
|
if (!request.context.vertex || attribute.location >= request.context.vertex->resourcesNum) throw std::runtime_error("Shader cache: invalid vertex attribute metadata");
|
|
attribute.resource = request.context.vertex->resources[attribute.location];
|
|
}
|
|
return result;
|
|
}
|
|
|
|
bool sameLayout(const BindingLayout& left, const BindingLayout& right) {
|
|
return left.descriptorSet == right.descriptorSet && left.firstBinding == right.firstBinding && left.pushConstantOffsetBytes == right.pushConstantOffsetBytes && left.pushConstantSizeBytes == right.pushConstantSizeBytes;
|
|
}
|
|
|
|
// The cached variant of `source` for the specialization, compiled on first use.
|
|
RecompileResult materializeVariant(SourceEntry& source, const RecompileRequest& request, const ResourceSnapshot& snapshot, const ResourceSpecialization& specialization) {
|
|
std::shared_ptr<const CompiledVariant> variant;
|
|
bool cacheHit = false;
|
|
{
|
|
std::lock_guard lock(source.mutex);
|
|
for (const auto& candidate : source.variants) {
|
|
if (sameLayout(candidate->layout, request.layout) && candidate->specialization == specialization) {
|
|
variant = candidate;
|
|
cacheHit = true;
|
|
break;
|
|
}
|
|
}
|
|
if (variant == nullptr) {
|
|
auto program = PrepareResourceProgram(request);
|
|
variant = std::make_shared<CompiledVariant>(compileVariant(request, std::move(program), snapshot, specialization));
|
|
source.variants.push_back(variant);
|
|
}
|
|
}
|
|
auto result = materializeResult(*variant, request, snapshot);
|
|
result.cacheHit = cacheHit;
|
|
return result;
|
|
}
|
|
|
|
RecompileResult RecompileImpl(const RecompileRequest& request) {
|
|
static_cast<void>(BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request)));
|
|
RequestMemoryView memory(request.context.memory);
|
|
const auto runtime = memory.MakeRuntime(request.context.userData, request.shader.codeAddress);
|
|
ResourceSnapshot snapshot;
|
|
ResourceSpecialization specialization;
|
|
constexpr ResourceMaterializer materializer;
|
|
if (!request.useCache) {
|
|
auto program = PrepareResourceProgram(request);
|
|
const auto plan = materializer.ExtractPlan(program);
|
|
materializer.Materialize(plan, runtime, snapshot, specialization);
|
|
const auto variant = compileVariant(request, std::move(program), snapshot, specialization);
|
|
return materializeResult(variant, request, snapshot);
|
|
}
|
|
const auto source = getSource(request);
|
|
materializer.Materialize(*source->plan, runtime, snapshot, specialization);
|
|
return materializeVariant(*source, request, snapshot, specialization);
|
|
}
|
|
|
|
// The capture already resolved the source entry (stage input validation included) and materialized
|
|
// the request over exactly the words the driver captured, so neither is repeated here.
|
|
RecompileResult RecompileImpl(const RecompileRequest& request, const ResourceCapture& capture) {
|
|
if (!request.useCache || capture.source == nullptr) {
|
|
auto program = PrepareResourceProgram(request);
|
|
const auto variant = compileVariant(request, std::move(program), capture.snapshot, capture.specialization);
|
|
return materializeResult(variant, request, capture.snapshot);
|
|
}
|
|
return materializeVariant(*capture.source, request, capture.snapshot, capture.specialization);
|
|
}
|
|
|
|
template <typename Impl>
|
|
RecompileResult recompileReporting(const RecompileRequest& request, Impl&& impl) {
|
|
try {
|
|
return impl();
|
|
} catch (const std::exception& e) {
|
|
constexpr auto requestSerializer = RequestSerializer{};
|
|
const auto inputInfo = "\nRecompileRequest:\n" + requestSerializer.Serialize(request);
|
|
throw std::runtime_error(std::string("ShaderRecompiler::Recompile: ") + e.what() + inputInfo);
|
|
} catch (...) {
|
|
throw std::runtime_error("ShaderRecompiler::Recompile: unknown exception");
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
std::shared_ptr<const IrResourcePlan> GetResourcePlan(const RecompileRequest& request) {
|
|
static_cast<void>(BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request)));
|
|
if (request.useCache) return getSource(request)->plan;
|
|
return makeResourcePlan(request);
|
|
}
|
|
|
|
std::shared_ptr<const ResourceCapture> CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime) {
|
|
// Validates the stage inputs once per request, as GetResourcePlan and Recompile(request) do.
|
|
static_cast<void>(BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request)));
|
|
auto capture = std::make_shared<ResourceCapture>();
|
|
if (request.useCache) {
|
|
capture->source = getSource(request);
|
|
capture->plan = capture->source->plan;
|
|
} else {
|
|
capture->plan = makeResourcePlan(request);
|
|
}
|
|
ResourceMaterializer{}.Materialize(*capture->plan, runtime, capture->snapshot, capture->specialization);
|
|
return capture;
|
|
}
|
|
|
|
RecompileResult Recompile(const RecompileRequest& request) {
|
|
return recompileReporting(request, [&] { return RecompileImpl(request); });
|
|
}
|
|
|
|
RecompileResult Recompile(const RecompileRequest& request, const ResourceCapture& capture) {
|
|
return recompileReporting(request, [&] { return RecompileImpl(request, capture); });
|
|
}
|
|
|
|
}
|