From dc62c432b436bfe26a05ff8579bd4d07d55b9dfd Mon Sep 17 00:00:00 2001 From: boykopovar Date: Tue, 6 Oct 2026 06:55:14 +0300 Subject: [PATCH] perf(shader): eliminate redundant prepared invocation work --- .../include/Driver/Shaders/ShaderRegistry.hpp | 1 + .../Execution/include/ShaderMemory.hpp | 3 + .../Driver/Dispatch/DataHitVerification.cpp | 6 +- .../src/Driver/Dispatch/Dispatch.cpp | 17 +++--- .../Execution/src/Driver/Draw/DrawCapture.cpp | 6 +- .../src/Driver/Shaders/ShaderRegistry.cpp | 14 +++++ .../Execution/src/ShaderMemory.cpp | 10 +++- .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 2 + .../tests/execution/PreparedShaders.cpp | 23 ++++++++ core/shader/recompiler/CMakeLists.txt | 2 + core/shader/recompiler/CacheKey.hpp | 33 +++++++---- .../IrMetadata/ShaderInfo.hpp | 1 + .../Optimization/ResourceMaterializer.hpp | 3 +- .../include/Optimization/ResourceProgram.hpp | 13 +++++ .../src/DescriptorBindingBuilder.cpp | 17 +++--- .../Optimization/src/ResourceMaterializer.cpp | 16 ++++-- core/shader/recompiler/Recompiler.cpp | 55 ++++++++++++++++--- core/shader/recompiler/ShaderDiskCache.cpp | 9 ++- core/shader/recompiler/ShaderDiskCache.hpp | 2 +- 19 files changed, 183 insertions(+), 50 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp index e7512d63a..b4499e5c3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp @@ -46,6 +46,7 @@ std::uint64_t NullPixelProgramAddress(); ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapshot, std::uint64_t vertexId, std::uint64_t fragmentId); std::shared_ptr SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, std::uint64_t deviceSerial, const ShaderRecompiler::RecompileRequest& request); +ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, std::uint64_t deviceSerial, const ShaderRecompiler::RecompileRequest& request); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp index 2c41a0684..457fd8360 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp @@ -14,6 +14,7 @@ namespace ShaderRecompiler { struct SourceHandle; +class PreparedShaderInvocation; } namespace AgcDriver { @@ -61,6 +62,7 @@ public: // (the driver's memoized ShaderRecompiler::ResolveSource result) the capture skips the source // resolution. std::shared_ptr Capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle = nullptr); + std::shared_ptr Capture(const ShaderRecompiler::PreparedShaderInvocation& invocation); [[nodiscard]] std::vector Regions() const; // The page regions read since the previous call (or construction), a word read again // included, the initial regions excluded: one stage's own reads on the draw path's shared @@ -70,6 +72,7 @@ public: static void CountHandleMemo(bool hit); private: + std::shared_ptr capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle, const ShaderRecompiler::PreparedShaderInvocation* invocation); static constexpr std::size_t PageBytes = 4096; static constexpr std::size_t PageWords = PageBytes / sizeof(std::uint32_t); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp index 661367444..a7f958737 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp @@ -13,11 +13,11 @@ void Driver::verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffse std::abort(); }; auto verifyMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); - const auto handle = SourceHandleFor(snapshot, codeOffset, deviceSerial, request); - const auto capture = verifyMemory->Capture(request, handle.get()); + const auto invocation = InvocationFor(snapshot, codeOffset, deviceSerial, request); + const auto capture = verifyMemory->Capture(invocation); const auto regions = verifyMemory->Regions(); request.context.memory = regions; - const auto result = ShaderRecompiler::MaterializeShader(request, *capture, *handle); + const auto result = invocation.Materialize(*capture); std::size_t position = 0, run = 0; for (const auto& region : regions) { const bool registered = std::any_of(memory.begin(), memory.end(), [&](const auto& known) { return region.guestAddress >= known.guestAddress && region.guestAddress < known.guestAddress + known.bytes.size(); }); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index 7da91d8e0..6e0cc8de5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -90,7 +90,12 @@ void Driver::dispatch(QueueState& queue, std::span packet, if (probeThis) std::fprintf(stderr, "[gpu] probing dispatch %llu of 0x%llx\n", static_cast(probeDispatch.second), static_cast(address)); } - const auto preparedHandle = SourceHandleFor(snapshot, codeOffset, localDevice->Serial(), request); + struct ProbeScope { + bool active; + explicit ProbeScope(bool active) : active(active) { if (active) ShaderRecompiler::SetDebugProbeActive(true); } + ~ProbeScope() { if (active) ShaderRecompiler::SetDebugProbeActive(false); } + } probeScope{probeThis}; + const auto invocation = InvocationFor(snapshot, codeOffset, localDevice->Serial(), request); if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) { pendingDispatchPhases().outcome = DispatchOutcome::FillHle; return; @@ -185,17 +190,11 @@ void Driver::dispatch(QueueState& queue, std::span packet, static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; try { - struct ProbeScope { - bool active; - explicit ProbeScope(bool active) : active(active) { if (active) ShaderRecompiler::SetDebugProbeActive(true); } - ~ProbeScope() { if (active) ShaderRecompiler::SetDebugProbeActive(false); } - } probeScope{probeThis}; const auto waitedBefore = traceCapSync() ? Graphics::Recorder::ThreadWaitedMs() : 0.0; forgetAtCapture = GuestMemory::ForgetSerial(); - const auto& handle = preparedHandle; capture = [&] { const SampledReadScope sampling(evidenceReads); - return shaderMemory->Capture(request, handle.get()); + return shaderMemory->Capture(invocation); }(); captured = shaderMemory->Regions(); request.context.memory = captured; @@ -206,7 +205,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, const auto started = std::chrono::steady_clock::now(); - compiledResult = ShaderRecompiler::MaterializeShader(request, *capture, *handle); + compiledResult = invocation.Materialize(*capture); if (compiledResult->cacheHit) ++cacheHits; const auto elapsed = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); static double totalMs = 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp index 414b74a11..41655756b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp @@ -22,13 +22,13 @@ ShaderRecompiler::RecompileResult Driver::materializeDrawStage(std::size_t i, st ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, graphics.stages.mesh, graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}} }; const auto waitedBefore = traceCapSync() || profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; - const auto handle = SourceHandleFor(*program.snapshot, program.codeOffset, localDevice->Serial(), request); + const auto invocation = InvocationFor(*program.snapshot, program.codeOffset, localDevice->Serial(), request); auto& stageCapture = stageCaptures[i]; stageCapture.forgetSerial = GuestMemory::ForgetSerial(); stageCapture.pushOffset = pushOffset; const auto capture = [&] { const SampledReadScope sampling(evidenceReads); - return shaderMemory.Capture(request, handle.get()); + return shaderMemory.Capture(invocation); }(); stageCapture.regions = shaderMemory.TakeRecentRegions(); @@ -71,7 +71,7 @@ ShaderRecompiler::RecompileResult Driver::materializeDrawStage(std::size_t i, st phaseTiming.Phase(DrawRowCapture); - stageCapture.compiled = ShaderRecompiler::MaterializeShader(request, *capture, *handle); + stageCapture.compiled = invocation.Materialize(*capture); ShaderRecompiler::RecompileResult result = *stageCapture.compiled; phaseTiming.Phase(DrawRowRecompile); return result; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 877fd9a0e..7c2df89ac 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -105,6 +105,20 @@ ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapsh } throw std::runtime_error("AGC driver: prepared rectangle artifacts are missing"); } + +ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, std::uint64_t deviceSerial, const ShaderRecompiler::RecompileRequest& request) { + require(codeOffset < snapshot.code.size(), "prepared shader code offset is outside the snapshot"); + const auto code = std::span(snapshot.code).subspan(codeOffset); + require(request.shader.code.data() == code.data() && request.shader.code.size() == code.size(), "prepared invocation does not refer to registered code"); + auto invocationRequest = request; + std::lock_guard lock(snapshot.prepared->mutex); + for (const auto& entry : snapshot.prepared->entries) { + if (entry.codeOffset != codeOffset || entry.deviceSerial != deviceSerial) continue; + invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle); + if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle)) return std::move(*invocation); + } + throw std::runtime_error("AGC driver: prepared shader artifact is missing for the requested static ABI"); +} namespace { template diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp index f624288af..8c07b7b74 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp @@ -212,6 +212,14 @@ void ShaderMemory::CountHandleMemo(bool hit) { } std::shared_ptr ShaderMemory::Capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle) { + return capture(request, handle, nullptr); +} + +std::shared_ptr ShaderMemory::Capture(const ShaderRecompiler::PreparedShaderInvocation& invocation) { + return capture(invocation.Request(), nullptr, &invocation); +} + +std::shared_ptr ShaderMemory::capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle, const ShaderRecompiler::PreparedShaderInvocation* invocation) { // The capture's word and page reads (through `read`) are attributed to it ([hooksync], [guestmem]). const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Capture); const bool profile = CaptureProfiled(); @@ -225,7 +233,7 @@ std::shared_ptr ShaderMemory::Capture(c runtime.userContext = this; runtime.readMemory = &read; runtime.readSpecializationMemory = &read; - auto capture = handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime); + auto capture = invocation != nullptr ? invocation->Capture(runtime) : handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime); if (profile) { totals.captureNanoseconds += NanosecondsSince(started); totals.resolveNanoseconds += capture->sourceNanoseconds; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 70fcacc0a..4f1ca64eb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -1,5 +1,6 @@ #include "ShaderDiskCache.hpp" #include "CacheKey.hpp" +#include "Optimization/ResourceMaterializer.hpp" #include "ShaderCacheDirectory.hpp" #include #include @@ -181,6 +182,7 @@ CompiledVariant sampleVariant() { image.indirectSearchIterations = 3; image.indirectResources = {1, 2, 3}; info.info.images = {image}; + ResourceMaterializer::PrepareImageModes(info.info); info.info.samplers = {{7, 0x10, 3, true, false, SamplerUseExplicitLod | SamplerUseGather}}; info.info.sampledPairs = {{0, 0, 0x10}}; StageInput input{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp index 015cee1df..b0cc27820 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp @@ -49,6 +49,27 @@ void Run(AgcDriver::VulkanDevice& device) { device.Dispatch(*result, 1, 1, 1); } device.WaitIdle(); + auto registeredRequest = request; + registeredRequest.shader.code = snapshot.code; + registeredRequest.useCache = true; + const auto invocation = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, device.Serial(), registeredRequest); + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 0, device.Serial(), request)); }, "does not refer to registered code"); + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 1, device.Serial(), registeredRequest)); }, "outside the snapshot"); + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 0, device.Serial() + 1, registeredRequest)); }, "artifact is missing"); + registeredRequest.context.compute->numThreads[0] = 2; + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 0, device.Serial(), registeredRequest)); }, "artifact is missing"); + ShaderRecompiler::SrtRuntime preparedRuntime{}; + preparedRuntime.userData = users; + const auto preparedCapture = invocation.Capture(preparedRuntime); + const auto firstResult = invocation.Materialize(*preparedCapture); + const auto repeatedCapture = invocation.Capture(preparedRuntime); + Require(invocation.Materialize(*repeatedCapture) == firstResult, "prepared invocation rebuilt an unchanged materialized result"); + users[0] += 1; + const auto changedCapture = invocation.Capture(preparedRuntime); + const auto changedResult = invocation.Materialize(*changedCapture); + Require(changedResult != firstResult && changedResult->spirv.data() == firstResult->spirv.data(), "prepared invocation did not distinguish changed runtime data"); + users[0] -= 1; + Require(invocation.Materialize(*preparedCapture) == firstResult, "prepared invocation evicted the previous runtime data"); ShaderRecompiler::SrtRuntime runtime{}; runtime.userData = users; const auto capture = ShaderRecompiler::CaptureResources(request, runtime, *handle); @@ -57,6 +78,8 @@ void Run(AgcDriver::VulkanDevice& device) { ExpectFailure([&] { static_cast(ShaderRecompiler::CaptureResources(request, runtime, *handle)); }, "does not match the static ABI"); ExpectFailure([&] { static_cast(ShaderRecompiler::MaterializeShader(request, *capture, *handle)); }, "does not match the static ABI"); const auto otherHandle = ShaderRecompiler::PrepareShader(request); + const auto otherCapture = ShaderRecompiler::CaptureResources(request, runtime, *otherHandle); + ExpectFailure([&] { static_cast(invocation.Materialize(*otherCapture)); }, "another prepared shader"); ExpectFailure([&] { static_cast(ShaderRecompiler::MaterializeShader(request, *capture, *otherHandle)); }, "another prepared shader"); request.context.compute->numThreads[0] = 1; request.layout.pushConstantSizeBytes = 124; diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 969e07ca3..19746ac8a 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -217,6 +217,8 @@ function(add_shader_recompiler target runtimeTarget) RecompileCacheKey::ContextHash RecompileResult PrepareShader + GetPreparedCode + PreparedShaderInvocation::TryCreate MatchesPreparedShader GetPreparedArtifact MaterializeShader diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index 1d2b784e8..8a1dee37e 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -26,17 +26,14 @@ public: } else { append(key, request.shader.code); } - append(key, request.context.waveSize); - append(key, request.context.userDataBaseRegister); - append(key, request.context.userData.size()); - append(key, request.context.compute); - append(key, request.context.pixel); - append(key, request.context.vertex); - appendMesh(key, request); - append(key, request.target); - append(key, DebugProbeActive()); - append(key, RayTracingStrict()); - append(key, RayTracingMiss()); + appendInterface(key, request); + } + + static void BuildInterface(const RecompileRequest& request, std::vector& key) { + key.clear(); + append(key, RuntimeAbi::Version); + append(key, request.shader.stage); + appendInterface(key, request); } // A hash over every field Build appends except the code, the target and the probe flag: the @@ -77,6 +74,20 @@ public: } private: + static void appendInterface(std::vector& key, const RecompileRequest& request) { + append(key, request.context.waveSize); + append(key, request.context.userDataBaseRegister); + append(key, request.context.userData.size()); + append(key, request.context.compute); + append(key, request.context.pixel); + append(key, request.context.vertex); + appendMesh(key, request); + append(key, request.target); + append(key, DebugProbeActive()); + append(key, RayTracingStrict()); + append(key, RayTracingMiss()); + } + static void appendMesh(std::vector& key, const RecompileRequest& request) { if (request.shader.stage != ShaderStage::Mesh) return; const auto* mesh = request.graphics && request.graphics->mesh ? &*request.graphics->mesh : nullptr; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp index aa6ee6b96..1a3f4f790 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp @@ -18,6 +18,7 @@ struct ShaderInfo { std::vector buffers; std::vector images; + std::vector> runtimeImageModes; std::vector samplers; std::vector sampledPairs; std::vector inputs; diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp index 0cd825884..fedb59594 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp @@ -17,7 +17,8 @@ public: static std::uint32_t EmulatedCompareState(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint32_t index); void ApplyStaticInterface(IrProgram& program) const; static std::vector RuntimeImageModes(const ImageResource& image); - static std::uint32_t RuntimeImageMode(const ImageResource& image, const DescriptorValue& descriptor); + static std::uint32_t RuntimeImageMode(const ImageResource& image, const DescriptorValue& descriptor, std::span modes); + static void PrepareImageModes(ShaderInfo& info); [[nodiscard]] IrResourcePlan ExtractPlan(const IrProgram& program) const; void Materialize(const IrResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot) const; static std::uint64_t SpecializationNanoseconds(); diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp index e6a2fb226..cbcbe9d49 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp @@ -40,6 +40,19 @@ struct SourceHandle { std::shared_ptr artifact; std::vector staticKey; }; +class PreparedShaderInvocation { +public: + static std::optional TryCreate(const RecompileRequest& request, const std::shared_ptr& handle); + const RecompileRequest& Request() const { return request; } + std::shared_ptr Capture(const SrtRuntime& runtime) const; + std::shared_ptr Materialize(const ResourceCapture& capture) const; + +private: + PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle); + RecompileRequest request; + std::shared_ptr handle; +}; +[[nodiscard]] std::span GetPreparedCode(const SourceHandle& handle); [[nodiscard]] std::shared_ptr PrepareShader(const RecompileRequest& request); [[nodiscard]] bool MatchesPreparedShader(const RecompileRequest& request, const SourceHandle& handle); [[nodiscard]] const CompiledShaderArtifact& GetPreparedArtifact(const SourceHandle& handle); diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index 5b4d2f933..4e496be35 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -227,7 +227,7 @@ std::uint32_t ImageSamplerMask(const ShaderInfo& info, const std::vector ShaderDataDwordsFor(const IrBindingLayout& layout, const ShaderInfo& info, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads) { +std::vector ShaderDataDwordsFor(const IrBindingLayout& layout, const ShaderInfo& info, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads, std::span imageModes) { RuntimeAbi::ShaderData data{}; data.version = RuntimeAbi::Version; if (snapshot.images.size() > data.images.size() || snapshot.samplers.size() > data.samplers.size()) fail("runtime resource metadata capacity exceeded"); @@ -257,9 +257,9 @@ std::vector ShaderDataDwordsFor(const IrBindingLayout& layout, co const auto& descriptor = image ? snapshot.images.at(resource) : snapshot.samplers.at(resource); std::uint32_t modeIndex = 0u; if (image) { - const auto& base = info.images.at(resource); - modeIndex = ResourceMaterializer::RuntimeImageMode(base, descriptor); - const auto mode = ResourceMaterializer::RuntimeImageModes(base).at(modeIndex); + if (resource >= imageModes.size()) fail("runtime image index exceeds the static capacity"); + modeIndex = imageModes[resource]; + const auto& mode = info.runtimeImageModes.at(resource).at(modeIndex); if (DescriptorBindingForImage(mode) != binding.kind) continue; } if (metadata.elementCount == 0u) { @@ -295,7 +295,10 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, const ShaderInfo& info, IrShaderStage stage, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads) const { const IrBindingLayout& layout = allocation.layout; - const std::vector shaderData = ShaderDataDwordsFor(layout, info, userDataBase, snapshot, partialThreads); + if (info.images.size() > ShaderInfo::MaxImages) fail("runtime image count exceeds the static capacity"); + std::array imageModes{}; + for (std::size_t index = 0; index < info.images.size(); ++index) imageModes[index] = ResourceMaterializer::RuntimeImageMode(info.images[index], snapshot.images.at(index), info.runtimeImageModes.at(index)); + const std::vector shaderData = ShaderDataDwordsFor(layout, info, userDataBase, snapshot, partialThreads, imageModes); const UnnormalizedProof unnormalized = ProveUnnormalized(info, snapshot); const std::vector samplerElements = SamplerElements(layout, info); @@ -327,7 +330,7 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con } break; case DescriptorRole::GuestImages: { - const auto modes = ResourceMaterializer::RuntimeImageModes(info.images.at(logical.resources.front())); + const auto& modes = info.runtimeImageModes.at(logical.resources.front()); const auto shape = std::ranges::find_if(modes, [&](const ImageResource& mode) { return DescriptorBindingForImage(mode) == logical.kind; }); if (shape == modes.end()) fail("runtime image heap has no static image type"); physical.imageShape = ImageShapeForResource(*shape); @@ -336,7 +339,7 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con for (const std::uint32_t resource : logical.resources) { const auto& image = info.images.at(resource); const auto& descriptor = snapshot.images.at(resource); - const auto mode = ResourceMaterializer::RuntimeImageModes(image).at(ResourceMaterializer::RuntimeImageMode(image, descriptor)); + const auto& mode = info.runtimeImageModes.at(resource).at(imageModes[resource]); mip = resource == previous ? mip + 1u : 0u; previous = resource; const auto firstMip = (descriptor.dwords[3] >> 12u) & 0xfu; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 67bafb0e5..f3e10cb63 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -375,7 +375,7 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con if (!validImageDescriptor(candidate, image.r128)) throw std::runtime_error("bindless image table contains an invalid descriptor"); const auto decoded = decodeImageDescriptor(candidate, image, plan.srgbDecodeFormats); if (decoded.fmask) throw std::runtime_error("bindless FMASK images are unsupported"); - static_cast(ResourceMaterializer::RuntimeImageMode(image, candidate)); + static_cast(ResourceMaterializer::RuntimeImageMode(image, candidate, plan.info.runtimeImageModes.at(imageIndex))); valid[i] = 1u; } resolution.mapping.clear(); @@ -545,7 +545,7 @@ void materializeTables(const IrResourcePlan& plan, ResourceSnapshot& snapshot, c if (snapshot.flattenedSrt.size() != plan.srtReads.size()) throw std::runtime_error("runtime SRT size differs from the static interface"); for (std::uint32_t i = 0u; i < plan.info.images.size(); ++i) { const auto& image = plan.info.images[i]; - static_cast(ResourceMaterializer::RuntimeImageMode(image, snapshot.images.at(i))); + static_cast(ResourceMaterializer::RuntimeImageMode(image, snapshot.images.at(i), plan.info.runtimeImageModes.at(i))); const auto& table = tables.at(i); if (!plan.descriptorSources.at(image.source).indirectImage.has_value()) { if (!table.slots.empty()) throw std::runtime_error("direct image has runtime table slots"); @@ -622,9 +622,9 @@ std::vector ResourceMaterializer::RuntimeImageModes(const ImageRe return modes; } -std::uint32_t ResourceMaterializer::RuntimeImageMode(const ImageResource& image, const DescriptorValue& descriptor) { +std::uint32_t ResourceMaterializer::RuntimeImageMode(const ImageResource& image, const DescriptorValue& descriptor, std::span modes) { if (descriptor.dwordCount != 8u) throw std::runtime_error("runtime image descriptor must contain eight dwords"); - const auto modes = RuntimeImageModes(image); + if (modes.empty()) throw std::runtime_error("prepared runtime image modes are missing"); if (nullImageDescriptor(descriptor)) return 0u; const auto decoded = decodeImageDescriptor(descriptor, image); if (image.packed && decoded.packedFormat != IrBufferFormat::Invalid) { @@ -688,6 +688,13 @@ void ResourceMaterializer::ApplyStaticInterface(IrProgram& program) const { sampler.depthCompare = sampler.depthCompare || images[pair.image].depthCompare; } resources.info.images = std::move(images); + PrepareImageModes(resources.info); +} + +void ResourceMaterializer::PrepareImageModes(ShaderInfo& info) { + info.runtimeImageModes.clear(); + info.runtimeImageModes.reserve(info.images.size()); + for (const auto& image : info.images) info.runtimeImageModes.push_back(RuntimeImageModes(image)); } namespace { @@ -772,6 +779,7 @@ IrResourcePlan ResourceMaterializer::ExtractPlan(const IrProgram& program) const plan.srtPlanComplete = source.srtPlanComplete; plan.resourceTrackingComplete = source.resourceTrackingComplete; plan.info = source.info; + PrepareImageModes(plan.info); plan.uniformFill = source.uniformFill; ownPlanValues(plan); const auto addSource = [&plan](std::uint32_t index) { diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 28e2fecfe..8b50c7b7d 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -537,16 +537,13 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh // The memo'd result of `source`'s variant for the snapshot (design13 R5): a hit returns the shared // object, a miss materializes outside the source mutex and inserts (a concurrent miss's object is // as good). `memoHit` reports the hit. -std::shared_ptr materializeMemoized(SourceEntry& source, const RecompileRequest& request, const ResourceSnapshot& snapshot, bool* memoHit) { +std::shared_ptr materializePreparedMemoized(SourceEntry& source, const std::shared_ptr& variant, const RecompileRequest& request, const ResourceSnapshot& snapshot, bool cacheHit, bool* memoHit) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - std::shared_ptr variant; - bool cacheHit = false; const auto hash = snapshotHash(request, snapshot); std::uint64_t index = 0; auto& counters = resultMemoCounters(); { std::lock_guard lock(source.mutex); - variant = findOrCompileVariant(source, request, cacheHit); index = (variant->artifact.variantId * 0x9e3779b97f4a7c15ull) ^ hash; const auto found = source.memoIndex.find(index); if (found != source.memoIndex.end() && found->second->variantId == variant->artifact.variantId && found->second->hash == hash) { @@ -590,6 +587,16 @@ std::shared_ptr materializeMemoized(SourceEntry& source, return shared; } +std::shared_ptr materializeMemoized(SourceEntry& source, const RecompileRequest& request, const ResourceSnapshot& snapshot, bool* memoHit) { + std::shared_ptr variant; + bool cacheHit = false; + { + std::lock_guard lock(source.mutex); + variant = findOrCompileVariant(source, request, cacheHit); + } + return materializePreparedMemoized(source, variant, request, snapshot, cacheHit, memoHit); +} + // The capture already resolved the source entry (stage input validation included) and materialized // the request over exactly the words the driver captured, so neither is repeated here. std::shared_ptr RecompileImpl(const RecompileRequest& request, const ResourceCapture& capture, bool* memoHit) { @@ -671,7 +678,7 @@ std::shared_ptr PrepareShader(const RecompileRequest& reques static_cast(RequestInputInfo(request)); auto handle = std::make_shared(); handle->source = getSource(request); - RecompileCacheKey::Build(request, handle->staticKey); + RecompileCacheKey::BuildInterface(request, handle->staticKey); handle->staticKey.push_back(HostSubgroupSize(request)); bool cacheHit = false; std::lock_guard lock(handle->source->mutex); @@ -682,10 +689,41 @@ std::shared_ptr PrepareShader(const RecompileRequest& reques bool MatchesPreparedShader(const RecompileRequest& request, const SourceHandle& handle) { if (handle.source == nullptr || handle.artifact == nullptr || !sameLayout(handle.artifact->layout, request.layout)) return false; - std::vector key; - RecompileCacheKey::Build(request, key); + struct PreparedKeyStorage {}; + auto& key = HostThreadLocal, PreparedKeyStorage>(); + RecompileCacheKey::BuildInterface(request, key); key.push_back(HostSubgroupSize(request)); - return key == handle.staticKey && std::ranges::equal(handle.source->code, request.shader.code); + if (key != handle.staticKey || handle.source->code.size() != request.shader.code.size()) return false; + return handle.source->code.data() == request.shader.code.data() || std::ranges::equal(handle.source->code, request.shader.code); +} + +std::span GetPreparedCode(const SourceHandle& handle) { + if (handle.source == nullptr || handle.artifact == nullptr) throw std::runtime_error("ShaderRecompiler: prepared artifact is missing"); + return handle.source->code; +} + +PreparedShaderInvocation::PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle) : request(request), handle(handle) {} + +std::optional PreparedShaderInvocation::TryCreate(const RecompileRequest& request, const std::shared_ptr& handle) { + if (handle == nullptr || !MatchesPreparedShader(request, *handle)) return std::nullopt; + if (request.shader.stage != ShaderStage::Compute && request.shader.stage != ShaderStage::Fragment) static_cast(RequestInputInfo(request)); + return PreparedShaderInvocation(request, handle); +} + +std::shared_ptr PreparedShaderInvocation::Capture(const SrtRuntime& runtime) const { + auto capture = std::make_shared(); + capture->source = handle->source; + capture->plan = handle->source->plan; + materializeCapture(*capture, runtime); + return capture; +} + +std::shared_ptr PreparedShaderInvocation::Materialize(const ResourceCapture& capture) const { + if (capture.source != handle->source || capture.plan != handle->source->plan) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); + if (request.useCache && ResultMemo()) return materializePreparedMemoized(*handle->source, handle->artifact, request, capture.snapshot, true, nullptr); + auto result = std::make_shared(materializeResult(*handle->artifact, request, capture.snapshot)); + result->cacheHit = true; + return result; } const CompiledShaderArtifact& GetPreparedArtifact(const SourceHandle& handle) { @@ -696,6 +734,7 @@ const CompiledShaderArtifact& GetPreparedArtifact(const SourceHandle& handle) { std::shared_ptr MaterializeShader(const RecompileRequest& request, const ResourceCapture& capture, const SourceHandle& handle) { if (!MatchesPreparedShader(request, handle)) throw std::runtime_error("ShaderRecompiler: prepared artifact does not match the static ABI"); if (capture.source != handle.source || capture.plan != handle.source->plan) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); + if (request.useCache && ResultMemo()) return materializePreparedMemoized(*handle.source, handle.artifact, request, capture.snapshot, true, nullptr); auto result = std::make_shared(materializeResult(*handle.artifact, request, capture.snapshot)); result->cacheHit = true; return result; diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 4e75dc4e9..96528e5c8 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -2,6 +2,7 @@ #include "CacheKey.hpp" #include "ShaderCacheDirectory.hpp" #include "ThreadOwned.hpp" +#include "Optimization/ResourceMaterializer.hpp" #ifdef _WIN32 #ifndef NOMINMAX #define NOMINMAX @@ -70,8 +71,8 @@ static_assert(sizeof(DescriptorBinding) == 448, "DescriptorBinding changed: upda static_assert(sizeof(VertexAttribute) == 28, "VertexAttribute changed: update the attribute encoder"); static_assert(sizeof(VertexInput) == 12, "VertexInput changed: update the vertex input encoder"); static_assert(sizeof(FragmentParameter) == 12, "FragmentParameter changed: update the parameter encoder"); -static_assert(sizeof(CompiledShaderInfo) == 304, "CompiledShaderInfo changed: update the info encoder"); -static_assert(sizeof(ShaderInfo) == 200, "ShaderInfo changed: update the info encoder"); +static_assert(sizeof(CompiledShaderInfo) == 328, "CompiledShaderInfo changed: update the info encoder"); +static_assert(sizeof(ShaderInfo) == 224, "ShaderInfo changed: update the info encoder"); static_assert(sizeof(BufferResource) == 24, "BufferResource changed: update the info encoder"); static_assert(sizeof(ImageResource) == 96, "ImageResource changed: update the info encoder"); static_assert(sizeof(SamplerResource) == 16, "SamplerResource changed: update the info encoder"); @@ -440,6 +441,7 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { out.Value(image.indirectSearchIterations); out.Values(std::span(image.indirectResources)); }); + writer.Value(!info.runtimeImageModes.empty()); writer.List(info.samplers, [](Writer& out, const SamplerResource& sampler) { out.Value(sampler.source); out.Value(sampler.firstUsePc); @@ -532,6 +534,9 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { in.Value(image.indirectSearchIterations); in.Values(image.indirectResources); }); + bool preparedImageModes = false; + reader.Value(preparedImageModes); + if (preparedImageModes) ResourceMaterializer::PrepareImageModes(info); reader.List(info.samplers, 15, [](Reader& in, SamplerResource& sampler) { in.Value(sampler.source); in.Value(sampler.firstUsePc); diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 2af9fbf8b..2ec01b6c4 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -12,7 +12,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 11; +inline constexpr std::uint32_t FormatVersion = 12; enum class LoadStatus { Loaded,