diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index 9fe35d703..dbb81c278 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -49,6 +49,14 @@ void Driver::dispatch(QueueState& queue, std::span packet, packet = resolved; indirectArguments = 0; } + if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) { + pendingDispatchPhases().outcome = DispatchOutcome::FillHle; + return; + } + if (indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) { + pendingDispatchPhases().outcome = DispatchOutcome::CopyHle; + return; + } if (indirectArguments == 0 && (packet[4] & 0x20u) != 0) { const std::array threads{packet[1], packet[2], packet[3]}; for (std::uint32_t axis = 0; axis < 3; ++axis) { @@ -95,15 +103,6 @@ void Driver::dispatch(QueueState& queue, std::span packet, explicit ProbeScope(bool active) : active(active) { if (active) ShaderRecompiler::SetDebugProbeActive(true); } ~ProbeScope() { if (active) ShaderRecompiler::SetDebugProbeActive(false); } } probeScope{probeThis}; - const auto invocation = InvocationFor(snapshot, codeOffset, request); - if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) { - pendingDispatchPhases().outcome = DispatchOutcome::FillHle; - return; - } - if (indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) { - pendingDispatchPhases().outcome = DispatchOutcome::CopyHle; - return; - } const bool noDispatchCache = noDispatchCacheEnv || probeThis; std::uint64_t key = 0xcbf29ce484222325ull; const auto mix = [&](std::uint64_t value) { @@ -178,10 +177,11 @@ void Driver::dispatch(QueueState& queue, std::span packet, previous.key = key; } - if (!stampValidate()) mix(reinterpret_cast(registeredShader.get())); + mix(reinterpret_cast(registeredShader.get())); phaseTiming.Phase(PhaseKey); lookupDispatch(address, submission, key, noDispatchCache, traceCache, profile, memory, phaseTiming, phaseMs, compiledResult, keepVariant, captured, liveWords, dataHit, cached, validated, missedEntry, missedDiffering); if (cached) { + require(keepVariant != nullptr && keepVariant->shader == it->second, "dispatch cache belongs to another registered shader"); captureMs += phaseTiming.Elapsed(); } else { shaderMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); @@ -189,6 +189,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; try { + const auto invocation = InvocationFor(snapshot, codeOffset, request); const auto waitedBefore = traceCapSync() ? Graphics::Recorder::ThreadWaitedMs() : 0.0; forgetAtCapture = GuestMemory::ForgetSerial(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index bb9428fea..556cc619a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -346,6 +346,8 @@ private: // The cached address space this build maps through (its lease pins the ranges); `regions` then // holds only the regions outside it (V#s, snapshots, ranges copied per build). std::shared_ptr space; + mutable std::uint64_t writeTableSerial = 0; + mutable std::shared_ptr> writeTableRanges; // Import registry epoch when `direct` pointers were taken at acquire time; they are reused while // no import was destroyed since. std::uint64_t importsEpoch = 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index cf57d5644..3d622a998 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -86,6 +86,8 @@ struct GuestBufferMemory::AddressSpace { std::vector copied; // The BDA table entries of `base`, in its order. std::vector ranges; + mutable std::mutex tableMutex; + mutable std::map>, std::pair>>> writeTables; }; namespace { @@ -2514,42 +2516,70 @@ std::vector GuestBufferMemory::AddressRanges() std::vector result; result.reserve(regions.size() + (space != nullptr ? space->ranges.size() : 0)); for (const auto& region : regions) result.push_back(addressRange(region)); - if (space != nullptr) result.insert(result.end(), space->ranges.begin(), space->ranges.end()); - std::sort(result.begin(), result.end(), [](const auto& left, const auto& right) { return left.begin < right.begin; }); + if (space != nullptr) { + std::vector combined; + combined.reserve(result.size() + space->ranges.size()); + std::merge(space->ranges.begin(), space->ranges.end(), result.begin(), result.end(), std::back_inserter(combined), [](const auto& left, const auto& right) { return left.begin < right.begin; }); + result = std::move(combined); + } + if (writes.empty()) return result; + auto sortedWrites = writes; + std::sort(sortedWrites.begin(), sortedWrites.end()); + std::vector> intervals; + for (const auto& interval : sortedWrites) { + if (!intervals.empty() && interval.first <= intervals.back().second) intervals.back().second = std::max(intervals.back().second, interval.second); + else intervals.push_back(interval); + } std::vector merged; + merged.reserve(result.size() + intervals.size() * 2u); + std::size_t firstWrite = 0; for (const auto& range : result) { if ((range.permissions & ShaderRecompiler::BdaAbi::Write) != 0u) { merged.push_back(range); continue; } - std::vector boundaries{range.begin, range.end}; - for (const auto& [begin, end] : writes) { - if (begin >= range.end || end <= range.begin) continue; - boundaries.push_back(std::max(begin, range.begin)); - boundaries.push_back(std::min(end, range.end)); - } - std::sort(boundaries.begin(), boundaries.end()); - boundaries.erase(std::unique(boundaries.begin(), boundaries.end()), boundaries.end()); - for (std::size_t index = 1; index < boundaries.size(); ++index) { + while (firstWrite < intervals.size() && intervals[firstWrite].second <= range.begin) ++firstWrite; + auto cursor = range.begin; + const auto append = [&](std::uint64_t end, bool written) { + if (cursor == end) return; auto part = range; - part.begin = boundaries[index - 1]; - part.end = boundaries[index]; - part.deviceAddress += part.begin - range.begin; - if (WritesOverlap(part.begin, static_cast(part.end - part.begin))) { - const auto* region = owner(part.begin); + part.begin = cursor; + part.end = end; + part.deviceAddress += cursor - range.begin; + if (written) { + const auto* region = owner(cursor); Require(region != nullptr && region->writable && (region->mirror == nullptr || region->mirror->writable), "runtime buffer write has no writable GPU owner"); part.permissions |= ShaderRecompiler::BdaAbi::Write; } merged.push_back(part); + cursor = end; + }; + for (auto index = firstWrite; index < intervals.size() && intervals[index].first < range.end; ++index) { + append(std::max(cursor, intervals[index].first), false); + append(std::min(range.end, intervals[index].second), true); } + append(range.end, false); } return merged; } std::optional GuestBufferMemory::CachedAddressTable() const { - if (space == nullptr || !regions.empty() || !writes.empty()) return std::nullopt; + if (space == nullptr || !regions.empty()) return std::nullopt; Require(uploaded && !committed, "guest GPU address ranges are not available"); - return CachedTable{space->serial, &space->ranges}; + if (writes.empty()) return CachedTable{space->serial, &space->ranges}; + if (writeTableRanges != nullptr) return CachedTable{writeTableSerial, writeTableRanges.get()}; + std::lock_guard lock(space->tableMutex); + const auto found = space->writeTables.find(writes); + if (found != space->writeTables.end()) { + writeTableSerial = found->second.first; + writeTableRanges = found->second.second; + } else { + writeTableRanges = std::make_shared>(AddressRanges()); + writeTableSerial = Spaces().serials.fetch_add(1, std::memory_order_relaxed) + 1; + if (space->writeTables.size() >= 64u) space->writeTables.erase(space->writeTables.begin()); + space->writeTables.emplace(writes, std::pair{writeTableSerial, writeTableRanges}); + } + return CachedTable{writeTableSerial, writeTableRanges.get()}; } bool GuestBufferMemory::HasCopiedWrites() const { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 79d63b593..524bb3dac 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -890,23 +890,34 @@ namespace { void ValidateRuntimeResources(const CompiledShader& shader) { Require(shader.program != nullptr, "missing compiled shader"); - const auto& descriptors = shader.program->bindings; - for (const auto& dataBinding : descriptors) { - if (dataBinding.role != ShaderRecompiler::DescriptorRole::ShaderData) continue; - Require(dataBinding.guestDescriptor.size() == ShaderRecompiler::RuntimeAbi::ShaderDataDwords, "invalid runtime resource metadata size"); - ShaderRecompiler::RuntimeAbi::ShaderData data{}; - std::memcpy(&data, dataBinding.guestDescriptor.data(), sizeof(data)); - ShaderRecompiler::RuntimeAbi::RequireVersion(data.version); - Require(data.imageCount <= data.images.size() && data.samplerCount <= data.samplers.size(), "runtime resource metadata capacity exceeded"); - const auto stageBase = dataBinding.binding - dataBinding.binding % static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count); - const auto validate = [&](const ShaderRecompiler::RuntimeAbi::ResourceMetadata& metadata, ShaderRecompiler::DescriptorRole role) { - Require(metadata.binding < static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count), "invalid runtime heap binding"); - const auto found = std::ranges::find_if(descriptors, [&](const ShaderRecompiler::DescriptorBinding& binding) { return binding.binding == stageBase + metadata.binding && binding.role == role; }); - Require(found != descriptors.end(), "runtime metadata references an unbound heap"); - Require(metadata.elementCount != 0u && metadata.firstElement < found->count && metadata.elementCount <= found->count - metadata.firstElement, "runtime metadata exceeds its bound heap"); - }; - for (std::uint32_t i = 0u; i < data.imageCount; ++i) validate(data.images[i], ShaderRecompiler::DescriptorRole::GuestImages); - for (std::uint32_t i = 0u; i < data.samplerCount; ++i) validate(data.samplers[i], ShaderRecompiler::DescriptorRole::GuestSamplers); + const auto& program = *shader.program; + const auto& descriptors = program.bindings; + constexpr auto bindingCount = static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count); + std::array heaps{}; + if (program.runtimeImageCount != 0u) { + for (const auto& binding : descriptors) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages) continue; + auto& heap = heaps[binding.binding % bindingCount]; + Require(heap == nullptr, "duplicate runtime image binding"); + heap = &binding; + } + } + for (const auto& binding : descriptors) { + if (binding.role != ShaderRecompiler::DescriptorRole::ShaderData) continue; + Require(binding.guestDescriptor.size() == program.shaderDataDwords, "invalid compact shader data size"); + const auto words = std::span(binding.guestDescriptor); + constexpr auto metadataWords = sizeof(ShaderRecompiler::RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t); + Require(program.imageMetadataDword <= words.size() && program.runtimeImageCount <= (words.size() - program.imageMetadataDword) / metadataWords, "runtime image metadata exceeds shader data"); + for (const auto index : program.runtimeImageResources) { + Require(index < program.runtimeImageCount, "runtime image resource exceeds its compact layout"); + const auto offset = program.imageMetadataDword + index * metadataWords; + const auto kind = words[offset]; + const auto first = words[offset + 1u]; + const auto count = words[offset + 2u]; + Require(count != 0u, "runtime image metadata has no descriptor elements"); + Require(kind < heaps.size() && heaps[kind] != nullptr, "runtime metadata references an unbound image heap"); + Require(first < heaps[kind]->count && count <= heaps[kind]->count - first, "runtime metadata exceeds its bound heap"); + } } } @@ -924,7 +935,6 @@ void ShaderResources::buildPrepare(std::span shaders, cons if (shaders.size() == 1u) refreshResourceKey = ContentKey(shaders.front(), false, true); std::set occupied; for (const auto& shader : shaders) { - ValidateRuntimeResources(shader); Require(shader.program != nullptr, "missing compiled shader"); ShaderRecompiler::RuntimeAbi::RequireVersion(shader.program->runtimeAbiVersion); const VkShaderStageFlags flags = VulkanStage(shader.stage); @@ -987,14 +997,14 @@ void ShaderResources::buildPrepare(std::span shaders, cons Require(binding.count == 1, "shader data and flattened SRT descriptors must not be arrays"); Require(!binding.guestDescriptor.empty(), "empty shader data descriptor"); if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) { - Require(binding.guestDescriptor.size() == ShaderRecompiler::RuntimeAbi::ShaderDataDwords, "shader data does not match the fixed runtime ABI"); - ShaderRecompiler::RuntimeAbi::RequireVersion(binding.guestDescriptor[0]); + Require(binding.guestDescriptor.size() == shader.program->shaderDataDwords, "shader data does not match the compact runtime layout"); } item.allocations.push_back(addDataBuffer(binding.guestDescriptor)); if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) shaderData = static_cast(item.allocations.back()); } bindings.push_back(std::move(item)); } + ValidateRuntimeResources(shader); for (const auto index : offsetsInData) allocations[index].dataAllocation = shaderData; for (auto index = firstDeferred; index < deferredImages.size(); ++index) { deferredImages[index].firstSampler = firstSampler; @@ -1133,7 +1143,7 @@ void ShaderResources::buildComplete() { if (index == std::numeric_limits::max()) images.push_back({VK_NULL_HANDLE, VK_NULL_HANDLE, VK_IMAGE_LAYOUT_GENERAL}); else images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()}); } - for (auto slot = binding.imageAllocations.size(); slot < binding.layout.descriptorCount; ++slot) images.push_back({VK_NULL_HANDLE, VK_NULL_HANDLE, VK_IMAGE_LAYOUT_GENERAL}); + Require(binding.imageAllocations.size() == binding.layout.descriptorCount, "descriptor allocations disagree with compact binding"); break; case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: write.pImageInfo = images.data() + images.size(); @@ -1141,12 +1151,12 @@ void ShaderResources::buildComplete() { if (index == std::numeric_limits::max()) images.push_back({VK_NULL_HANDLE, VK_NULL_HANDLE, VK_IMAGE_LAYOUT_GENERAL}); else images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL}); } - for (auto slot = binding.imageAllocations.size(); slot < binding.layout.descriptorCount; ++slot) images.push_back({VK_NULL_HANDLE, VK_NULL_HANDLE, VK_IMAGE_LAYOUT_GENERAL}); + Require(binding.imageAllocations.size() == binding.layout.descriptorCount, "descriptor allocations disagree with compact binding"); break; case VK_DESCRIPTOR_TYPE_SAMPLER: write.pImageInfo = images.data() + images.size(); for (const auto index : binding.imageAllocations) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED}); - for (auto slot = binding.imageAllocations.size(); slot < binding.layout.descriptorCount; ++slot) images.push_back({paddingSampler->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED}); + Require(binding.imageAllocations.size() == binding.layout.descriptorCount, "descriptor allocations disagree with compact binding"); break; default: throw std::runtime_error("AGC graphics: ShaderResources encountered an unknown descriptor type while writing the descriptor set"); } @@ -1250,39 +1260,38 @@ void ShaderResources::reportDescriptorCaches() const { AgcDriver::ProfilePrint_nid_no_patch("[descriptors] layouts %llu hits / %llu created, sets %llu from %llu pools and %llu dedicated pools, samplers %llu hits / %llu created\n", static_cast(descriptors.layoutHits), static_cast(descriptors.layoutMisses), static_cast(descriptors.sets), static_cast(descriptors.pools), static_cast(descriptors.dedicatedPools), static_cast(samplerHits), static_cast(samplerMisses)); } -std::vector ShaderResources::ContentKey(const CompiledShader& shader, bool dataWords, bool movableBuffers) { +template +void VisitContentKey(const CompiledShader& shader, bool dataWords, bool movableBuffers, const TVisitor& visit) { Require(shader.program != nullptr, "missing compiled shader"); const auto& program = *shader.program; - std::vector key; - key.reserve(8 + program.bindings.size() * 12); - key.push_back(dataWords ? 1u : 0u); - key.push_back(static_cast(shader.stage)); - key.push_back(static_cast(program.PipelineVariantId())); - key.push_back(static_cast(program.PipelineVariantId() >> 32u)); - key.push_back(static_cast(program.bindings.size())); + visit(dataWords ? 1u : 0u); + visit(static_cast(shader.stage)); + visit(static_cast(program.PipelineVariantId())); + visit(static_cast(program.PipelineVariantId() >> 32u)); + visit(static_cast(program.bindings.size())); const auto packBits = [&](const std::vector& bits) { - key.push_back(static_cast(bits.size())); + visit(static_cast(bits.size())); std::uint32_t word = 0; for (std::size_t i = 0; i < bits.size(); ++i) { if (bits[i]) word |= 1u << (i % 32u); if (i % 32u == 31u || i + 1 == bits.size()) { - key.push_back(word); + visit(word); word = 0; } } }; for (const auto& binding : program.bindings) { - key.insert(key.end(), {static_cast(binding.kind), static_cast(binding.role), binding.descriptorSet, binding.binding, binding.count, binding.readOnly ? 1u : 0u, binding.imageShape.has_value() ? static_cast(*binding.imageShape) + 1u : 0u, static_cast(binding.guestDescriptor.size())}); + for (const auto word : {static_cast(binding.kind), static_cast(binding.role), binding.descriptorSet, binding.binding, binding.count, binding.readOnly ? 1u : 0u, binding.imageShape.has_value() ? static_cast(*binding.imageShape) + 1u : 0u, static_cast(binding.guestDescriptor.size())}) visit(word); if (movableBuffers && binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers && binding.guestDescriptor.size() == static_cast(binding.count) * 4u) { for (std::uint32_t element = 0; element < binding.count; ++element) { const auto* words = binding.guestDescriptor.data() + static_cast(element) * 4u; const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; const bool empty = words[2] == 0 || (words[0] == 0 && (words[1] & 0xffffu) == 0); - if (written || empty) key.insert(key.end(), words, words + 4); - else key.insert(key.end(), {0u, words[1] & 0xffff0000u, 0u, words[3]}); + if (written || empty) { for (std::uint32_t index = 0; index < 4u; ++index) visit(words[index]); } + else { for (const auto word : {0u, words[1] & 0xffff0000u, 0u, words[3]}) visit(word); } } } else if ((dataWords && !movableBuffers) || !DataRole(binding.role)) { - key.insert(key.end(), binding.guestDescriptor.begin(), binding.guestDescriptor.end()); + for (const auto word : binding.guestDescriptor) visit(word); } packBits(binding.imageWritten); packBits(binding.samplerDepthCompare); @@ -1293,12 +1302,29 @@ std::vector ShaderResources::ContentKey(const CompiledShader& sha // Read-only elements are bound without a write set: an object built for one written set // must not serve a build with another (the variant implies it, this makes it explicit). packBits(binding.bufferWritten); - key.push_back(static_cast(binding.imageSamplers.size())); - key.insert(key.end(), binding.imageSamplers.begin(), binding.imageSamplers.end()); + visit(static_cast(binding.imageSamplers.size())); + for (const auto sampler : binding.imageSamplers) visit(sampler); } +} + +std::vector ShaderResources::ContentKey(const CompiledShader& shader, bool dataWords, bool movableBuffers) { + Require(shader.program != nullptr, "missing compiled shader"); + std::vector key; + key.reserve(8u + shader.program->bindings.size() * 12u); + VisitContentKey(shader, dataWords, movableBuffers, [&](std::uint32_t word) { key.push_back(word); }); return key; } +bool MatchesContentKey(const CompiledShader& shader, std::span key) { + std::size_t cursor = 0; + bool equal = true; + VisitContentKey(shader, false, true, [&](std::uint32_t word) { + equal &= cursor < key.size() && key[cursor] == word; + ++cursor; + }); + return equal && cursor == key.size(); +} + namespace { // APS5_PROFILE_DRAW: Revalidate outcomes and time, printed as [rescache] every 10 s next to the @@ -2425,8 +2451,7 @@ bool ShaderResources::DataWordsDiffer(const CompiledShader& shader) const { bool ShaderResources::RefreshData(VkCommandBuffer commands, const CompiledShader& shader, Recorder* recorder) { Require(shader.program != nullptr, "missing compiled shader"); - Require(!refreshResourceKey.empty() && refreshResourceKey == ContentKey(shader, false, true), "runtime data refresh cannot replace bound resources"); - ValidateRuntimeResources(shader); + Require(!refreshResourceKey.empty() && MatchesContentKey(shader, refreshResourceKey), "runtime data refresh cannot replace bound resources"); const auto& program = *shader.program; Require(program.bindings.size() == bindings.size(), "template bindings disagree with the shader"); bool recorded = false; @@ -2442,6 +2467,7 @@ bool ShaderResources::RefreshData(VkCommandBuffer commands, const CompiledShader const auto size = binding.guestDescriptor.size() * sizeof(std::uint32_t); Require(allocation.buffer != nullptr && !allocation.guest && allocation.size == size && size <= MaxRefreshBytes, "template data buffer cannot take the dispatch's words"); if (allocation.dataWords.size() == binding.guestDescriptor.size() && std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), binding.guestDescriptor.begin())) continue; + if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) ValidateRuntimeResources(shader); if (recorder != nullptr && timing == Recorder::NoTiming) timing = recorder->BeginGpuTiming(Recorder::CommandClass::TemplateDataRefresh); writeDataWords(commands, bindings[index].allocations.front(), binding.guestDescriptor); allocation.dataWords.assign(binding.guestDescriptor.begin(), binding.guestDescriptor.end()); @@ -2475,8 +2501,8 @@ void ShaderResources::PrecollectSurfaces() const { void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags) { Require(binding.count != 0, "empty descriptor binding"); const auto heap = static_cast(binding.binding % static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count)); - const auto capacity = ShaderRecompiler::RuntimeAbi::HeapCapacity(heap); - Require(binding.count <= capacity, "typed descriptor heap capacity exceeded"); + const auto capacity = binding.count; + Require(binding.count <= ShaderRecompiler::RuntimeAbi::HeapCapacity(heap), "typed descriptor heap capacity exceeded"); const auto localBinding = static_cast(heap); const bool sampledHeap = localBinding >= ShaderRecompiler::RuntimeAbi::FirstImageBinding && localBinding < ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding; const bool storageHeap = localBinding >= ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding && localBinding < static_cast(ShaderRecompiler::RuntimeAbi::Binding::Samplers); @@ -2513,10 +2539,6 @@ void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& } else { Require(elementWords == 4, "guest sampler descriptor must contain 4 dwords"); Require(capacity <= context.limits.maxPerStageDescriptorSamplers, "shader sampler descriptors exceed per-stage limits"); - if (paddingSampler == nullptr) { - constexpr std::array defaultSampler{}; - paddingSampler = std::make_shared(context, DecodeSamplerResource(defaultSampler)); - } Require(binding.samplerDepthCompare.size() == binding.count, "guest sampler binding is missing depth comparison metadata"); Require(binding.samplerUnnormalized.empty() || binding.samplerUnnormalized.size() == binding.count, "guest sampler binding has unnormalized coordinate metadata of another size"); for (std::uint32_t element = 0; element < binding.count; ++element) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp index 149921fa0..527ef1087 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp @@ -470,7 +470,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const auto heap = static_cast(binding->binding % static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count)); const auto capacity = ShaderRecompiler::RuntimeAbi::HeapCapacity(heap); const auto count = module.constants.find(type[3]); - Require(binding->count != 0u && binding->count <= capacity && count != module.constants.end() && count->second == capacity, "typed heap array length disagrees with runtime ABI"); + Require(binding->count != 0u && binding->count <= capacity && count != module.constants.end() && count->second == binding->count, "typed heap array length disagrees with compact binding"); } else { Require(variable.storage == spv::StorageClassStorageBuffer && decoration.set && decoration.binding, "unsupported or unbound shader resource"); const auto key = std::make_pair(*decoration.set, *decoration.binding); diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index b1e55e42c..b06aa75dc 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -1534,6 +1534,7 @@ void resourceTests() { ShaderRecompiler::RecompileResult fragment; vertex.bindings.push_back(makeBinding(Role::GuestBuffers, 0, 2, join(vsharp(guestFirst.data(), 16), vsharp(guestSecond.data(), 32)))); vertex.bindings.push_back(makeBinding(Role::ShaderData, 5, 1, ShaderDataWords({7, 8, 9}))); + vertex.shaderDataDwords = ShaderRecompiler::RuntimeAbi::ShaderDataDwords; fragment.bindings.push_back(makeBinding(Role::FlattenedSrt, 43, 1, {1, 2})); fragment.bindings.push_back(makeBinding(Role::GuestBuffers, 44, 1, vsharp(guestThird.data(), 8))); AgcDriver::Graphics::ShaderResources resources(context, vertex, fragment, state.color, 0, 0); @@ -1639,8 +1640,8 @@ void resourceTests() { expectSingleFailure(changed([](auto& binding) { binding.count = 2; }), "four DWORDs per array element"); expectSingleFailure(changed([](auto& binding) { binding.role = Role::ShaderData; binding.count = 2; binding.guestDescriptor = {1, 2}; }), "must not be arrays"); expectSingleFailure(changed([](auto& binding) { binding.role = Role::ShaderData; binding.guestDescriptor.clear(); }), "empty shader data descriptor"); - expectSingleFailure(changed([](auto& binding) { binding.role = Role::ShaderData; }), "fixed runtime ABI"); - expectSingleFailure(changed([](auto& binding) { binding.role = Role::ShaderData; binding.guestDescriptor = ShaderDataWords({}); binding.guestDescriptor[0] = 0u; }), "incompatible version"); + expectSingleFailure(changed([](auto& binding) { binding.role = Role::ShaderData; }), "compact runtime layout"); + expectSingleFailure(changed([](auto& binding) { binding.role = Role::ShaderData; binding.guestDescriptor = ShaderDataWords({}); binding.guestDescriptor.pop_back(); }), "compact runtime layout"); expectSingleFailure(changed([](auto& binding) { binding.role = Role::FlattenedSrt; binding.guestDescriptor.clear(); }), "empty shader data descriptor"); expectSingleFailure(changed([](auto& binding) { binding.guestDescriptor[1] |= 0x40000000u; }), "reserved bits"); expectSingleFailure(changed([](auto& binding) { binding.guestDescriptor[3] |= 0x40000000u; }), "unsupported type"); @@ -1730,6 +1731,7 @@ void misalignedShaderDataTests() { ShaderRecompiler::RecompileResult compute; compute.bindings.push_back(makeBinding(Role::GuestBuffers, 0, 2, join(vsharp(guest.data(), 32), vsharp(guest.data() + 1, 8)))); compute.bindings.push_back(makeBinding(Role::ShaderData, 1, 1, ShaderDataWords({0x11, 0x22}))); + compute.shaderDataDwords = ShaderRecompiler::RuntimeAbi::ShaderDataDwords; compute.memoryOffsetDword = ShaderRecompiler::RuntimeAbi::BufferOffsetsDword; auto live = compute; live.bindings[1].guestDescriptor = ShaderDataWords({0x33, 0x44}); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp index 85948a499..39aaef5df 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp @@ -137,7 +137,7 @@ void CheckHeaps() { const auto single = allocate(image, 1u); const auto full = allocate(image, RuntimeAbi::SampledHeapCapacity, RuntimeAbi::SamplerHeapCapacity / 2u); Require(single.layout.ShaderDataDwords() == full.layout.ShaderDataDwords() && single.layout.memoryOffsetDword == full.layout.memoryOffsetDword && !full.layout.UsesPushData(), "runtime layout depends on resource count"); - Require(full.layout.memoryOffsetDword == 132u && full.layout.DispatchThreadLimitDword() == 164u && full.layout.ShaderDataDwords() == 3432u, "runtime layout offsets changed"); + Require(full.layout.memoryOffsetDword == 0u && full.layout.DispatchThreadLimitDword() == 0u && full.layout.ShaderDataDwords() == 0u, "direct image resources allocated runtime metadata"); Reject([&] { allocate(image, RuntimeAbi::SampledHeapCapacity + 1u); }, "heap capacity exceeded"); Reject([&] { allocate(image, 1u, RuntimeAbi::SamplerHeapCapacity + 1u); }, "metadata capacity"); image.resourceClass = ImageResourceClass::Storage; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 0c82c7f0c..de268671b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -54,6 +54,7 @@ void requireSameArtifact(const CompiledShaderArtifact& left, const CompiledShade require(left.bdaAbiVersion == right.bdaAbiVersion, prefix + "BDA ABI version differs"); require(left.runtimeAbiVersion == right.runtimeAbiVersion, prefix + "runtime ABI version differs"); require(left.memoryOffsetDword == right.memoryOffsetDword, prefix + "memory offset differs"); + require(left.shaderDataDwords == right.shaderDataDwords && left.imageMetadataDword == right.imageMetadataDword && left.runtimeImageCount == right.runtimeImageCount && left.runtimeImageResources == right.runtimeImageResources, prefix + "compact data layout differs"); require(left.hostSubgroupSize == right.hostSubgroupSize, prefix + "host subgroup size differs"); require(left.vertexInputs == right.vertexInputs, prefix + "vertex inputs differ"); require(left.vertexInputPatches == right.vertexInputPatches, prefix + "vertex type patches differ"); @@ -637,8 +638,8 @@ void verifyInvocationIsolation() { request.userData = descriptor; const auto changed = Recompile(request.request); require(changed.cacheHit && first.variantId == changed.variantId, "buffer metadata changed the compiled variant"); - requireSameArtifact(first, changed, "runtime buffer artifact"); - require(first.spirv.data() == changed.spirv.data(), "buffer metadata duplicated the SPIR-V"); + const auto repeated = Recompile(request.request); + require(changed.spirv.data() == repeated.spirv.data() && changed.PipelineVariantId() == repeated.PipelineVariantId(), "buffer specialization did not reuse its module"); } request.userData = original; request.userData[3] |= 0x40000000u; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp index d4884f4fc..c63799a9f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp @@ -57,15 +57,13 @@ void Run(AgcDriver::VulkanDevice& device) { RecompileRequest request{{ShaderStage::Compute, reinterpret_cast(code.data()), code, 0u, {}}, {32u, 0u, userData, compute, std::nullopt, std::nullopt, regions}, device.Target(), {0u, 0u, 0u, 128u}}; const auto shader = Recompile(request); if (artifact.variantId == 0u) artifact = shader; - else Require(shader.cacheHit && shader.variantId == artifact.variantId && shader.spirv.data() == artifact.spirv.Words().data(), "bindless table changed the compiled artifact"); + else Require(shader.cacheHit && shader.variantId == artifact.variantId, "bindless table changed the compiled artifact"); if (iteration == 1u) { auto invalid = shader; auto binding = std::ranges::find_if(invalid.bindings, [](const DescriptorBinding& value) { return value.role == DescriptorRole::ShaderData; }); Require(binding != invalid.bindings.end(), "bindless shader has no runtime metadata"); - RuntimeAbi::ShaderData metadata{}; - std::memcpy(&metadata, binding->guestDescriptor.data(), sizeof(metadata)); - metadata.images[0].firstElement = RuntimeAbi::SampledHeapCapacity; - std::memcpy(binding->guestDescriptor.data(), &metadata, sizeof(metadata)); + const auto offset = invalid.imageMetadataDword + invalid.runtimeImageResources.at(0) * (sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)) + offsetof(RuntimeAbi::ResourceMetadata, firstElement) / sizeof(std::uint32_t); + binding->guestDescriptor.at(offset) = RuntimeAbi::SampledHeapCapacity; bool rejected = false; try { device.Dispatch(invalid, 1u, 1u, 1u); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp index 8cd7ff8fd..257349eae 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp @@ -313,7 +313,7 @@ void CheckRuntimeDescriptors(AgcDriver::VulkanDevice& device) { Output.fill(0xdeadbeefu); const auto result = ShaderRecompiler::Recompile(request); if (first.spirv.empty()) first = result; - else Require(result.cacheHit && result.variantId == first.variantId && result.spirv.data() == first.spirv.Words().data(), "runtime V# changed the formatted shader artifact"); + else Require(result.cacheHit && result.variantId == first.variantId, "runtime V# changed the formatted shader artifact"); device.Dispatch(result, 1, 1, 1); device.WaitIdle(); for (std::uint32_t lane = 0; lane < 32u; ++lane) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageLoadPacked.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageLoadPacked.cpp index 901e78b76..93065073d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageLoadPacked.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageLoadPacked.cpp @@ -143,7 +143,7 @@ void Run(AgcDriver::VulkanDevice& device, std::uint32_t format, std::uint32_t sw const auto result = ShaderRecompiler::Recompile(request); static ShaderRecompiler::CompiledShaderArtifact first; if (first.variantId == 0u) first = result; - else Require(result.cacheHit && result.variantId == first.variantId && result.spirv.data() == first.spirv.Words().data(), "runtime image format changed the compiled artifact"); + else Require(result.cacheHit && result.variantId == first.variantId, "runtime image format changed the compiled artifact"); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp index 3ed81fd47..0a3a058a5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp @@ -184,7 +184,7 @@ void Run(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span artifacts; auto& first = artifacts[reinterpret_cast(code.data())]; if (first.variantId == 0u) first = result; - else Require(result.cacheHit && result.variantId == first.variantId && result.spirv.data() == first.spirv.Words().data(), "runtime storage image changed the compiled artifact"); + else Require(result.cacheHit && result.variantId == first.variantId, "runtime storage image changed the compiled artifact"); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); AgcDriver::Graphics::StorageTexture::FlushPending(reinterpret_cast(texels), TexelBytes, nullptr, "test"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedExports.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedExports.cpp index 7e9270b01..ffb80e3fb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedExports.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedExports.cpp @@ -1,3 +1,4 @@ +#include #include "VulkanTestDevice.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" @@ -75,16 +76,20 @@ void Run(AgcDriver::VulkanDevice& device, bool integer) { SrtRuntime runtime{}; const auto capture = invocation.Capture(runtime); const auto pixel = invocation.Materialize(*capture); - Require(pixel->spirv.data() == artifact.spirv.data() && pixel->variantId == artifact.variantId, "export mapping replaced the prepared fragment artifact"); + Require(pixel->variantId == artifact.variantId && pixel->specialization.empty(), "export mapping did not materialize a prepared fragment module"); if (!first) first = pixel; else if (mapping == 0xe4u) Require(pixel == first, "export mapping did not reuse the materialized result"); else Require(pixel != first, "export mapping reused stale materialized data"); for (const bool cached : {true, false}) { request.useCache = cached; const auto materialized = MaterializeShader(request, *capture, *handle); - const auto binding = std::find_if(materialized->bindings.begin(), materialized->bindings.end(), [](const auto& entry) { return entry.role == DescriptorRole::ShaderData; }); - Require(binding != materialized->bindings.end(), "fragment runtime data binding is missing"); - for (unsigned slot = 0; slot < 8; ++slot) Require(binding->guestDescriptor.at(RuntimeAbi::ExportMappingsDword + slot) == request.context.pixel->targetExportMapping[slot], "runtime export mapping is stale"); + Require(materialized->spirv.data() == pixel->spirv.data() && materialized->PipelineVariantId() == pixel->PipelineVariantId(), "export mapping did not reuse its specialized module"); + for (std::size_t cursor = 5; cursor < materialized->spirv.size();) { + const auto count = materialized->spirv[cursor] >> 16u; + Require(count != 0u && count <= materialized->spirv.size() - cursor, "invalid specialized fragment instruction"); + Require((materialized->spirv[cursor] & 0xffffu) != static_cast(spv::OpVectorExtractDynamic), "fragment export retained a dynamic channel selection"); + cursor += count; + } } request.useCache = true; if (integer) continue; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/RuntimeImages.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/RuntimeImages.cpp index 8e3b18113..b2937b0ca 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/RuntimeImages.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/RuntimeImages.cpp @@ -35,7 +35,7 @@ void Run(AgcDriver::VulkanDevice& device) { ShaderRecompiler::RecompileRequest request{{ShaderRecompiler::ShaderStage::Compute, reinterpret_cast(code.data()), code, 0u, {}}, {32u, 0u, userData, compute, std::nullopt, std::nullopt, {}}, device.Target(), {0u, 0u, 0u, 128u}}; const auto shader = ShaderRecompiler::Recompile(request); if (first.variantId == 0u) first = shader; - else Require(shader.cacheHit && shader.variantId == first.variantId && shader.spirv.data() == first.spirv.Words().data(), "T#/S# changed the compiled artifact"); + else Require(shader.cacheHit && shader.variantId == first.variantId, "T#/S# changed the compiled artifact"); device.Dispatch(shader, 1u, 1u, 1u); device.WaitIdle(); const auto pixel = clamped ? 31u : 8u; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/TypedHeaps.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/TypedHeaps.cpp index 60aee3d09..77b95c736 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/TypedHeaps.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/TypedHeaps.cpp @@ -30,13 +30,11 @@ ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std:: const auto instructions = store ? std::span(storeCode) : std::span(code); ShaderRecompiler::RecompileRequest request{{ShaderStage::Compute, reinterpret_cast(instructions.data()), instructions, 0u, {}}, {32u, 0u, userData, compute, std::nullopt, std::nullopt, {}}, device.Target(), {0u, 0u, 0u, 128u}}; auto result = ShaderRecompiler::Recompile(request); - Require(result.pushConstants.empty(), "ShaderData unexpectedly uses a variable push layout"); - const auto dataBinding = std::ranges::find_if(result.bindings, [](const auto& binding) { return binding.role == ShaderRecompiler::DescriptorRole::ShaderData; }); - Require(dataBinding != result.bindings.end() && dataBinding->guestDescriptor.size() == ShaderRecompiler::RuntimeAbi::ShaderDataDwords, "ShaderData does not match the fixed ABI"); - ShaderRecompiler::RuntimeAbi::ShaderData data{}; - std::memcpy(&data, dataBinding->guestDescriptor.data(), sizeof(data)); - Require(data.version == ShaderRecompiler::RuntimeAbi::Version && data.imageCount == 1u, "resource metadata header is invalid"); - Require(data.images[0].elementCount == 1u && data.images[0].firstElement == 0u && (data.images[0].flags & 1u) == (nullImage ? 1u : 0u), "image heap metadata is invalid"); + Require(result.runtimeImageCount == 0u, "direct image allocated runtime image metadata"); + Require(result.shaderDataDwords < ShaderRecompiler::RuntimeAbi::ShaderDataDwords, "shader data was not compacted"); + for (const auto& binding : result.bindings) { + if (binding.role == ShaderRecompiler::DescriptorRole::GuestImages) Require(binding.count == 1u, "single image expanded into unused descriptor slots"); + } return result; } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/VertexFetch.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/VertexFetch.cpp index 52cf32df3..182b63c3e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/VertexFetch.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/VertexFetch.cpp @@ -79,7 +79,7 @@ void Run(AgcDriver::VulkanDevice& device) { Require(vertex.vertexOffsetSgpr == -1 && vertex.instanceOffsetSgpr == -1, "runtime vertex fetch requested draw offset folding"); Require(vertex.bdaAbiVersion != 0u, "runtime vertex fetch has no BDA ABI"); if (first.spirv.empty()) first = vertex; - else Require(vertex.cacheHit && vertex.variantId == first.variantId && vertex.spirv.data() == first.spirv.Words().data(), "runtime vertex descriptor changed the artifact"); + else Require(vertex.cacheHit && vertex.variantId == first.variantId, "runtime vertex descriptor changed the artifact"); const auto pushBytes = static_cast(vertex.pushConstants.size()); ShaderRecompiler::ShaderPixelStageInfo pixelInfo{}; pixelInfo.wave32 = true; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp index de719aa26..91952766e 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp @@ -42,14 +42,18 @@ struct IrBindingLayout { std::uint32_t memoryOffsetDword = 0; std::uint32_t memoryOffsetCount = 0; bool dispatchThreadLimit = false; + std::uint32_t runtimeImageCount = 0; std::vector userDataRegisters; std::vector descriptors; [[nodiscard]] std::uint32_t DispatchThreadLimitDword() const { - return RuntimeAbi::DispatchThreadLimitDword; + return memoryOffsetDword + (memoryOffsetCount + 3u) / 4u; + } + [[nodiscard]] std::uint32_t ImageMetadataDword() const { + return DispatchThreadLimitDword() + (dispatchThreadLimit ? 3u : 0u); } [[nodiscard]] std::uint32_t ShaderDataDwords() const { - return RuntimeAbi::ShaderDataDwords; + return ImageMetadataDword() + runtimeImageCount * (sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)); } [[nodiscard]] bool UsesPushData() const { return pushDataStartDword != PushData::NoStart; diff --git a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp index 469d52b52..7d18b8a79 100644 --- a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp +++ b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp @@ -94,12 +94,14 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin IrBindingLayout next; next.userDataRegisters = collectUserData(program); - next.memoryOffsetDword = RuntimeAbi::BufferOffsetsDword; + next.memoryOffsetDword = static_cast(next.userDataRegisters.size()); next.memoryOffsetCount = static_cast(info.buffers.size()); next.dispatchThreadLimit = info.dispatchThreadLimit; if (info.buffers.size() > RuntimeAbi::BufferCapacity || info.images.size() > RuntimeAbi::ImageCapacity || info.samplers.size() > RuntimeAbi::SamplerHeapCapacity) fail("shader binding layout exceeds runtime metadata capacity"); if (std::ranges::any_of(next.userDataRegisters, [](auto reg) { return reg >= RuntimeAbi::UserDataCapacity; })) fail("shader user data exceeds runtime ABI capacity"); - next.pushDataStartDword = PushData::NoStart; + if (std::ranges::any_of(info.images, [](const auto& image) { return image.indirectRoot != ImageResource::NoIndirectImage; })) next.runtimeImageCount = static_cast(info.images.size()); + const auto pushSize = layout.pushConstantSizeBytes / 4u; + next.pushDataStartDword = next.runtimeImageCount == 0u && next.ShaderDataDwords() != 0u && next.ShaderDataDwords() <= pushSize ? layout.pushConstantOffsetBytes / 4u : PushData::NoStart; if (!info.buffers.empty()) { std::vector resources(info.buffers.size()); diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index 1e5c91244..0fed98c07 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -138,57 +138,38 @@ std::vector GuestSamplersDescriptor(const std::vector ShaderDataDwordsFor(const IrBindingLayout& layout, const ShaderInfo& info, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads, std::span imageModes, std::span exportMappings) { - - RuntimeAbi::ShaderData data{}; - data.version = RuntimeAbi::Version; - if (!exportMappings.empty() && exportMappings.size() != data.exportMappings.size()) fail("invalid runtime export mapping count"); - std::copy(exportMappings.begin(), exportMappings.end(), data.exportMappings.begin()); - if (snapshot.images.size() > data.images.size() || snapshot.samplers.size() > data.samplers.size()) fail("runtime resource metadata capacity exceeded"); - data.imageCount = static_cast(snapshot.images.size()); - data.samplerCount = static_cast(snapshot.samplers.size()); - for (std::size_t i = 0; i < layout.userDataRegisters.size(); i++) { - const std::uint32_t reg = layout.userDataRegisters[i]; - if (reg < userDataBase || reg - userDataBase >= snapshot.userData.size()) { - fail("DescriptorBindingBuilder::Populate user-data register is out of range"); - } - if (reg >= data.userData.size()) fail("runtime user-data capacity exceeded"); - data.userData[reg] = snapshot.userData[reg - userDataBase]; +std::vector ShaderDataDwordsFor(const IrBindingLayout& layout, const ShaderInfo& info, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads, std::span imageModes, const std::vector& selected) { + std::vector result(layout.ShaderDataDwords(), 0u); + for (std::size_t i = 0; i < layout.userDataRegisters.size(); ++i) { + const auto reg = layout.userDataRegisters[i]; + if (reg < userDataBase || reg - userDataBase >= snapshot.userData.size()) fail("shader user-data register is out of range"); + result[i] = snapshot.userData[reg - userDataBase]; } if (layout.dispatchThreadLimit) { - if (partialThreads == std::array{}) { - fail("DescriptorBindingBuilder::Populate partial-group shader has no dispatch size"); - } - std::copy(partialThreads.begin(), partialThreads.end(), data.dispatchThreadLimit.begin()); + if (partialThreads == std::array{}) fail("partial-group shader has no dispatch size"); + std::copy(partialThreads.begin(), partialThreads.end(), result.begin() + layout.DispatchThreadLimitDword()); } - for (const auto& binding : layout.descriptors) { - const bool image = ImageBindingResourceClass(binding.kind) != ImageResourceClass::None; - if (!image && binding.kind != DescriptorBindingKind::Samplers) continue; - if (binding.resources.size() > RuntimeAbi::HeapCapacity(binding.kind)) fail("runtime typed heap capacity exceeded"); + if (layout.runtimeImageCount == 0u) return result; + for (const auto& binding : selected) { + if (ImageBindingResourceClass(binding.kind) == ImageResourceClass::None) continue; for (std::size_t element = 0; element < binding.resources.size(); ++element) { const auto resource = binding.resources[element]; - auto& metadata = image ? data.images.at(resource) : data.samplers.at(resource); - const auto& descriptor = image ? snapshot.images.at(resource) : snapshot.samplers.at(resource); - std::uint32_t modeIndex = 0u; - if (image) { - if (resource >= imageModes.size()) fail("runtime image index exceeds the static capacity"); - modeIndex = imageModes[resource]; - const auto& mode = info.runtimeImageModes.at(resource).at(modeIndex); - if (DescriptorBindingForImage(mode) != binding.kind) continue; - } - if (metadata.elementCount == 0u) { - metadata.binding = static_cast(binding.kind); - metadata.firstElement = static_cast(element); - if (descriptor.dwordCount > metadata.descriptor.size()) fail("runtime resource descriptor width exceeded"); - std::copy_n(descriptor.dwords.begin(), descriptor.dwordCount, metadata.descriptor.begin()); - metadata.flags = image && descriptor.dwords[0] == 0u && (descriptor.dwords[1] & 0xffu) == 0u ? 1u : 0u; - metadata.flags |= modeIndex << 1u; - } - ++metadata.elementCount; + if (info.images.at(resource).indirectRoot == ImageResource::NoIndirectImage) continue; + const auto& mode = info.runtimeImageModes.at(resource).at(imageModes[resource]); + if (DescriptorBindingForImage(mode) != binding.kind) continue; + RuntimeAbi::ResourceMetadata metadata{}; + metadata.binding = static_cast(binding.kind); + metadata.firstElement = static_cast(element); + metadata.elementCount = 1u; + const auto& descriptor = snapshot.images.at(resource); + if (descriptor.dwordCount != metadata.descriptor.size()) fail("invalid runtime image descriptor width"); + std::copy_n(descriptor.dwords.begin(), metadata.descriptor.size(), metadata.descriptor.begin()); + metadata.flags = imageModes[resource] << 1u; + if (descriptor.dwords[0] == 0u && (descriptor.dwords[1] & 0xffu) == 0u) metadata.flags |= 1u; + if (resource >= layout.runtimeImageCount) fail("runtime image metadata exceeds compact layout"); + std::memcpy(result.data() + layout.ImageMetadataDword() + resource * (sizeof(metadata) / sizeof(std::uint32_t)), &metadata, sizeof(metadata)); } } - std::vector result(RuntimeAbi::ShaderDataDwords); - std::memcpy(result.data(), &data, sizeof(data)); return result; } @@ -239,7 +220,77 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con allocation.specialization.push_back({first, imageModes[index]}); for (std::uint32_t component = 0; component < 4u; ++component) allocation.specialization.push_back({first + 1u + component, (snapshot.images.at(index).dwords[3] >> (component * 3u)) & 7u}); } - const std::vector shaderData = ShaderDataDwordsFor(layout, info, userDataBase, snapshot, partialThreads, imageModes, exportMappings); + for (std::uint32_t target = 0; target < exportMappings.size(); ++target) { + for (std::uint32_t component = 0; component < 4u; ++component) allocation.specialization.push_back({PipelineSpecialization::ExportBase + target * 4u + component, (exportMappings[target] >> (component * 2u)) & 3u}); + } + std::vector samplerModes(info.samplers.size(), 0u); + const auto samplerMode = [](const ImageResource& image) { return image.numericClass == IrTextureNumericClass::Sint || image.conversionFormat != IrBufferFormat::Invalid || image.depthBits ? 2u : 1u; }; + for (const auto& pair : info.sampledPairs) { + const auto& image = info.images.at(pair.image); + if (image.indirectRoot == ImageResource::NoIndirectImage) samplerModes.at(pair.sampler) |= samplerMode(info.runtimeImageModes.at(pair.image).at(imageModes[pair.image])); + else { + const auto& root = info.images.at(image.indirectRoot); + for (const auto slot : root.indirectResources) samplerModes.at(pair.sampler) |= samplerMode(info.runtimeImageModes.at(slot).at(imageModes[slot])); + } + } + for (std::uint32_t index = 0; index < info.images.size(); ++index) { + if (info.images[index].mipMode != ImageMipMode::DynamicStorage) continue; + const auto word = snapshot.images.at(index).dwords[3]; + const auto first = (word >> 12u) & 0xfu; + const auto last = (word >> 16u) & 0xfu; + if (last < first || last - first >= RuntimeAbi::StorageHeapCapacity) fail("invalid dynamic storage mip range"); + allocation.specialization.push_back({PipelineSpecialization::MipCountBase + index, last - first + 1u}); + } + for (std::uint32_t resource = 0; resource < info.images.size(); ++resource) { + const auto& image = info.images[resource]; + if (image.indirectRoot == ImageResource::NoIndirectImage) continue; + const auto& modes = info.runtimeImageModes.at(resource); + if (modes.size() > PipelineSpecialization::ImageModeStride) fail("runtime image mode specialization capacity exceeded"); + std::vector active(modes.size(), false); + for (const auto slot : info.images.at(image.indirectRoot).indirectResources) { + const auto mode = ResourceMaterializer::RuntimeImageMode(image, snapshot.images.at(slot), modes); + if (mode != imageModes[slot]) fail("indirect image mode numbering disagrees with its root"); + active[mode] = true; + } + for (std::uint32_t mode = 0; mode < modes.size(); ++mode) allocation.specialization.push_back({PipelineSpecialization::ImageModeBase + resource * PipelineSpecialization::ImageModeStride + mode, active[mode] ? 1u : 0u}); + } + std::vector selected; + std::vector> originalElements; + selected.reserve(layout.descriptors.size()); + for (const auto& logical : layout.descriptors) { + const bool imageHeap = ImageBindingResourceClass(logical.kind) != ImageResourceClass::None; + const bool samplerHeap = logical.kind == DescriptorBindingKind::Samplers; + IrDescriptorBinding compact{logical.kind, {}}; + std::vector originals; + std::uint32_t previous = UINT32_MAX; + std::uint32_t mip = 0; + for (std::uint32_t element = 0; element < logical.resources.size(); ++element) { + const auto resource = logical.resources[element]; + bool active = true; + if (imageHeap) { + const auto& image = info.images.at(resource); + const auto& mode = info.runtimeImageModes.at(resource).at(imageModes[resource]); + mip = previous == resource ? mip + 1u : 0u; + previous = resource; + { + const auto word = snapshot.images.at(resource).dwords[3]; + active = DescriptorBindingForImage(mode) == logical.kind && mode.packedFormat != IrBufferFormat::Fmask8_S2_F1 && (image.mipMode != ImageMipMode::DynamicStorage || mip <= ((word >> 16u) & 0xfu) - ((word >> 12u) & 0xfu)); + } + } else if (samplerHeap) { + active = (samplerModes.at(resource) & (1u << (element & 1u))) != 0u; + } + if (imageHeap || samplerHeap) allocation.specialization.push_back({PipelineSpecialization::DescriptorIndex(static_cast(logical.kind), element), active ? static_cast(compact.resources.size()) : 0u}); + if (active) { + compact.resources.push_back(resource); + originals.push_back(element); + } + } + if (imageHeap || samplerHeap) allocation.specialization.push_back({PipelineSpecialization::HeapCountBase + static_cast(logical.kind), std::max(1u, static_cast(compact.resources.size()))}); + if ((imageHeap || samplerHeap) && compact.resources.empty()) continue; + selected.push_back(std::move(compact)); + originalElements.push_back(std::move(originals)); + } + const std::vector shaderData = ShaderDataDwordsFor(layout, info, userDataBase, snapshot, partialThreads, imageModes, selected); const UnnormalizedProof unnormalized = ProveUnnormalized(info, snapshot); const std::vector samplerElements = SamplerElements(layout, info); @@ -247,7 +298,8 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con bindings.reserve(layout.descriptors.size()); std::size_t writtenHere = 0; std::size_t readOnlyHere = 0; - for (const IrDescriptorBinding& logical : layout.descriptors) { + for (std::size_t bindingIndex = 0; bindingIndex < selected.size(); ++bindingIndex) { + const auto& logical = selected[bindingIndex]; DescriptorBinding physical; physical.descriptorSet = RuntimeAbi::DescriptorSet; physical.binding = NativeBinding(stage, logical.kind); @@ -306,7 +358,7 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con const auto& sampler = info.samplers.at(logical.resources[element]); physical.samplerDepthCompare.push_back(sampler.depthCompare); physical.samplerUnnormalized.push_back(unnormalized.samplers.at(logical.resources[element])); - if ((element & 1u) != 0u) { + if ((originalElements[bindingIndex][element] & 1u) != 0u) { auto& filter = physical.guestDescriptor.at(element * 4u + 2u); filter = PointFilteredSamplerWord(physical.guestDescriptor.at(element * 4u), filter); } diff --git a/core/shader/recompiler/PipelineSpecialization.hpp b/core/shader/recompiler/PipelineSpecialization.hpp index f825ac7a2..f8f54d05b 100644 --- a/core/shader/recompiler/PipelineSpecialization.hpp +++ b/core/shader/recompiler/PipelineSpecialization.hpp @@ -16,10 +16,21 @@ namespace PipelineSpecialization { inline constexpr std::uint32_t BufferBase = 0u; inline constexpr std::uint32_t BufferWords = 3u; -inline constexpr std::uint32_t ImageBase = 128u; +inline constexpr std::uint32_t ImageBase = 1024u; inline constexpr std::uint32_t ImageWords = 5u; -inline constexpr std::uint32_t VertexBase = 512u; +inline constexpr std::uint32_t VertexBase = 4096u; inline constexpr std::uint32_t VertexWords = 6u; +inline constexpr std::uint32_t ExportBase = 8192u; +inline constexpr std::uint32_t HeapCountBase = 8256u; +inline constexpr std::uint32_t MipCountBase = 8320u; +inline constexpr std::uint32_t DescriptorIndexBase = 16384u; +inline constexpr std::uint32_t DescriptorIndexStride = 128u; +inline constexpr std::uint32_t ImageModeBase = 32768u; +inline constexpr std::uint32_t ImageModeStride = 256u; + +inline constexpr std::uint32_t DescriptorIndex(std::uint32_t binding, std::uint32_t element) { + return DescriptorIndexBase + binding * DescriptorIndexStride + element; +} } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 00d581276..3b7af4a36 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -11,6 +11,7 @@ #include "ShaderDiskCache.hpp" #include #include +#include #include #include #include @@ -338,6 +339,12 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra result.bdaAbiVersion = program.Info().usesDma ? request.target.bdaAbiVersion : 0u; result.memoryOffsetDword = bindings.layout.memoryOffsetDword; + result.shaderDataDwords = bindings.layout.ShaderDataDwords(); + result.imageMetadataDword = bindings.layout.ImageMetadataDword(); + result.runtimeImageCount = bindings.layout.runtimeImageCount; + for (std::uint32_t index = 0; index < program.Info().images.size(); ++index) { + if (program.Info().images[index].indirectRoot != ImageResource::NoIndirectImage) result.runtimeImageResources.push_back(index); + } result.hostSubgroupSize = HostSubgroupSize(request); result.vertexOffsetSgpr = program.Info().vertexOffsetSgpr; result.instanceOffsetSgpr = program.Info().instanceOffsetSgpr; @@ -400,14 +407,102 @@ std::uint64_t specializationId(std::uint64_t artifact, std::spansecond; } -SharedSpirv specializeVertexInputs(const CompiledShaderArtifact& artifact, std::span classes) { - if (artifact.vertexInputPatches.empty()) return artifact.spirv; +struct SpecializedModule { + SharedSpirv spirv; + std::vector bindings; + bool pushData = false; +}; + +std::shared_ptr specializeModule(const CompiledShaderArtifact& artifact, std::span classes, std::span constants, std::uint64_t id, const SpirvTarget& target) { static std::mutex mutex; - static std::map>, SharedSpirv> modules; - std::pair key{artifact.variantId, std::vector(classes.begin(), classes.end())}; + static std::map> modules; + { + std::lock_guard lock(mutex); + if (const auto found = modules.find(id); found != modules.end()) return found->second; + } + auto source = SpecializeVertexInputTypes(artifact, classes); + std::map supplied; + for (const auto& constant : constants) { + if (!supplied.emplace(constant.id, constant.value).second) throw std::runtime_error("duplicate prepared specialization ID"); + } + std::map values; + const auto& words = source.Words(); + if (words.size() < 5u || words[0] != spv::MagicNumber) throw std::runtime_error("invalid prepared specialization module"); + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + const auto op = static_cast(words[cursor] & 0xffffu); + if (count == 0u || count > words.size() - cursor) throw std::runtime_error("truncated prepared specialization instruction"); + if (op == spv::OpDecorate && count == 4u && words[cursor + 2u] == spv::DecorationSpecId) { + const auto found = supplied.find(words[cursor + 3u]); + if (found == supplied.end() || !values.emplace(words[cursor + 1u], found->second).second) throw std::runtime_error("missing or duplicate prepared specialization value"); + } + cursor += count; + } + std::vector materialized(words.begin(), words.begin() + 5); + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + const auto op = static_cast(words[cursor] & 0xffffu); + if (op == spv::OpSpecConstant) { + if (count != 4u || !values.contains(words[cursor + 2u])) throw std::runtime_error("invalid prepared specialization constant"); + materialized.insert(materialized.end(), {(4u << 16u) | spv::OpConstant, words[cursor + 1u], words[cursor + 2u], values.at(words[cursor + 2u])}); + } else if (!(op == spv::OpDecorate && count == 4u && words[cursor + 2u] == spv::DecorationSpecId)) { + materialized.insert(materialized.end(), words.begin() + cursor, words.begin() + cursor + count); + } + cursor += count; + } +#if ANYPS5_ENABLE_SPIRV_TOOLS + materialized = ValidateAndOptimizeSpirv(materialized, target.vulkanVersion, target.spirvVersion, target.nonConstantImageOffsets, true, true); +#else + throw std::runtime_error("prepared shader materialization requires SPIRV-Tools"); +#endif + std::map descriptorVariables; + std::set usedDescriptors; + for (std::size_t cursor = 5; cursor < materialized.size();) { + const auto count = materialized[cursor] >> 16u; + const auto op = static_cast(materialized[cursor] & 0xffffu); + if (op == spv::OpDecorate && count == 4u && materialized[cursor + 2u] == spv::DecorationBinding) descriptorVariables.emplace(materialized[cursor + 1u], materialized[cursor + 3u]); + if ((op == spv::OpAccessChain || op == spv::OpInBoundsAccessChain || op == spv::OpPtrAccessChain || op == spv::OpInBoundsPtrAccessChain || op == spv::OpLoad || op == spv::OpCopyObject) && count >= 4u) usedDescriptors.insert(materialized[cursor + 3u]); + cursor += count; + } + const auto unused = [&](std::uint32_t id) { return descriptorVariables.contains(id) && !usedDescriptors.contains(id); }; + std::vector compact(materialized.begin(), materialized.begin() + 5); + for (std::size_t cursor = 5; cursor < materialized.size();) { + const auto count = materialized[cursor] >> 16u; + const auto op = static_cast(materialized[cursor] & 0xffffu); + if (op == spv::OpEntryPoint) { + const auto start = compact.size(); + std::size_t interfaceIndex = 3u; + for (; interfaceIndex < count; ++interfaceIndex) { + const auto word = materialized[cursor + interfaceIndex]; + if ((word & 0xffu) == 0u || (word & 0xff00u) == 0u || (word & 0xff0000u) == 0u || (word & 0xff000000u) == 0u) { ++interfaceIndex; break; } + } + compact.insert(compact.end(), materialized.begin() + cursor, materialized.begin() + cursor + interfaceIndex); + for (auto index = interfaceIndex; index < count; ++index) if (!unused(materialized[cursor + index])) compact.push_back(materialized[cursor + index]); + compact[start] = (static_cast(compact.size() - start) << 16u) | spv::OpEntryPoint; + } else if (!((op == spv::OpVariable && unused(materialized[cursor + 2u])) || ((op == spv::OpDecorate || op == spv::OpName) && unused(materialized[cursor + 1u])))) { + compact.insert(compact.end(), materialized.begin() + cursor, materialized.begin() + cursor + count); + } + cursor += count; + } + materialized = std::move(compact); +#if ANYPS5_ENABLE_SPIRV_TOOLS + materialized = ValidateAndOptimizeSpirv(materialized, target.vulkanVersion, target.spirvVersion, target.nonConstantImageOffsets, false); +#endif + auto module = std::make_shared(); + std::map bindingNumbers; + for (std::size_t cursor = 5; cursor < materialized.size();) { + const auto count = materialized[cursor] >> 16u; + const auto op = static_cast(materialized[cursor] & 0xffffu); + if (op == spv::OpDecorate && count == 4u && materialized[cursor + 2u] == spv::DecorationBinding) bindingNumbers.emplace(materialized[cursor + 1u], materialized[cursor + 3u]); + if (op == spv::OpVariable && count >= 4u) { + if (const auto found = bindingNumbers.find(materialized[cursor + 2u]); found != bindingNumbers.end()) module->bindings.push_back(found->second); + module->pushData |= materialized[cursor + 3u] == spv::StorageClassPushConstant; + } + cursor += count; + } + module->spirv = std::move(materialized); std::lock_guard lock(mutex); - if (const auto found = modules.find(key); found != modules.end()) return found->second; - return modules.emplace(std::move(key), SpecializeVertexInputTypes(artifact, classes)).first->second; + return modules.emplace(id, std::move(module)).first->second; } RecompileResult materializeResult(const CompiledVariant& variant, const RecompileRequest& request, const ResourceSnapshot& snapshot) { @@ -447,8 +542,15 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil result.specialization.push_back({first + 5u, kind}); } } - result.spirv = specializeVertexInputs(variant.artifact, vertexClasses); result.specializationId = specializationId(result.variantId, result.specialization); + if (!result.specialization.empty() || !result.vertexInputPatches.empty()) { + const auto module = specializeModule(variant.artifact, vertexClasses, result.specialization, result.PipelineVariantId(), request.target); + result.spirv = module->spirv; + result.specialization.clear(); + result.vertexInputPatches.clear(); + std::erase_if(result.bindings, [&](const auto& binding) { return std::ranges::find(module->bindings, binding.binding) == module->bindings.end(); }); + if (!result.pushConstants.empty() && !module->pushData) throw std::runtime_error("specialization removed the prepared push constant interface"); + } return result; } diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 00463662d..d1b5bd730 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -423,6 +423,10 @@ struct CompiledShaderArtifact { std::vector vertexInputPatches; SharedSpirv spirv; std::uint32_t memoryOffsetDword = 0; + std::uint32_t shaderDataDwords = 0; + std::uint32_t imageMetadataDword = 0; + std::uint32_t runtimeImageCount = 0; + std::vector runtimeImageResources; std::uint32_t bdaAbiVersion = 0; std::uint32_t runtimeAbiVersion = RuntimeAbi::Version; std::vector vertexInputs; diff --git a/core/shader/recompiler/RuntimeAbi.hpp b/core/shader/recompiler/RuntimeAbi.hpp index 280ea2f30..a4bfc0f35 100644 --- a/core/shader/recompiler/RuntimeAbi.hpp +++ b/core/shader/recompiler/RuntimeAbi.hpp @@ -9,7 +9,7 @@ namespace ShaderRecompiler::RuntimeAbi { -inline constexpr std::uint32_t Version = 6u; +inline constexpr std::uint32_t Version = 7u; inline constexpr std::uint32_t DescriptorSet = 0u; inline constexpr std::uint32_t StageCount = 4u; inline constexpr std::uint32_t PushConstantDwords = 32u; diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 4809b5b5d..fc52ad4be 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -287,6 +287,10 @@ void encodeArtifact(Writer& writer, const CompiledShaderArtifact& result) { writer.Values(std::span(result.spirv.Words())); writer.Value(result.bdaAbiVersion); writer.Value(result.memoryOffsetDword); + writer.Value(result.shaderDataDwords); + writer.Value(result.imageMetadataDword); + writer.Value(result.runtimeImageCount); + writer.Values(std::span(result.runtimeImageResources)); writer.List(result.vertexInputPatches, [](Writer& out, const VertexInputPatch& patch) { out.Value(patch.location); out.Value(patch.word); @@ -322,6 +326,10 @@ void decodeArtifact(Reader& reader, CompiledShaderArtifact& result) { result.spirv = std::move(words); reader.Value(result.bdaAbiVersion); reader.Value(result.memoryOffsetDword); + reader.Value(result.shaderDataDwords); + reader.Value(result.imageMetadataDword); + reader.Value(result.runtimeImageCount); + reader.Values(result.runtimeImageResources); reader.List(result.vertexInputPatches, 20, [](Reader& in, VertexInputPatch& patch) { in.Value(patch.location); in.Value(patch.word); @@ -393,6 +401,7 @@ void encodeLayout(Writer& writer, const IrBindingLayout& layout) { writer.Value(layout.memoryOffsetDword); writer.Value(layout.memoryOffsetCount); writer.Value(layout.dispatchThreadLimit); + writer.Value(layout.runtimeImageCount); writer.Values(std::span(layout.userDataRegisters)); writer.List(layout.descriptors, [](Writer& out, const IrDescriptorBinding& descriptor) { out.Value(descriptor.kind); @@ -405,8 +414,9 @@ void decodeLayout(Reader& reader, IrBindingLayout& layout) { reader.Value(layout.memoryOffsetDword); reader.Value(layout.memoryOffsetCount); reader.Value(layout.dispatchThreadLimit); + reader.Value(layout.runtimeImageCount); reader.Values(layout.userDataRegisters); - reader.List(layout.descriptors, 12, [](Reader& in, IrDescriptorBinding& descriptor) { + reader.List(layout.descriptors, static_cast(RuntimeAbi::Binding::Count), [](Reader& in, IrDescriptorBinding& descriptor) { in.Value(descriptor.kind); in.Values(descriptor.resources); }); diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 83ad09313..e7f9cdd97 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -12,7 +12,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 15; +inline constexpr std::uint32_t FormatVersion = 16; enum class LoadStatus { Loaded, diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvOptimizer.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvOptimizer.hpp index 23f7e561c..661caa053 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvOptimizer.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvOptimizer.hpp @@ -7,7 +7,7 @@ namespace ShaderRecompiler { -[[nodiscard]] std::vector ValidateAndOptimizeSpirv(std::span spirv, std::uint32_t vulkanVersion, std::uint32_t spirvVersion, bool allowOffsetTextureOperand = false, bool optimize = true); +[[nodiscard]] std::vector ValidateAndOptimizeSpirv(std::span spirv, std::uint32_t vulkanVersion, std::uint32_t spirvVersion, bool allowOffsetTextureOperand = false, bool optimize = true, bool specialize = false); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index cd4918b70..a130e350a 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -48,7 +48,7 @@ struct SampleSetup { std::uint32_t RuntimeImageDword(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t member) { const auto index = state.runtimeImageMetadata != 0u ? state.runtimeImageMetadata : ConstantU32(state, resource); - const auto dword = Binary(state, spv::OpIAdd, TypeU32(state), ConstantU32(state, offsetof(RuntimeAbi::ShaderData, images) / sizeof(std::uint32_t) + member), Binary(state, spv::OpIMul, TypeU32(state), index, ConstantU32(state, sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)))); + const auto dword = Binary(state, spv::OpIAdd, TypeU32(state), ConstantU32(state, state.program.Metadata().bindings.ImageMetadataDword() + member), Binary(state, spv::OpIMul, TypeU32(state), index, ConstantU32(state, sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)))); const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.shaderDataStorageVariable, ConstantU32(state, 0u), dword); const auto value = state.module.AllocateId(); @@ -1417,7 +1417,24 @@ void EmitImage(SpirvValueEmitContext& ctx, const IrValue& inst) { std::vector incoming; for (std::uint32_t index = 0u; index < modes.size(); ++index) { EmitLabel(state, labels[index]); + std::uint32_t modeMerge = 0u; + if (base.indirectRoot != ImageResource::NoIndirectImage) { + const auto enabled = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::ImageModeBase + memory.resource * PipelineSpecialization::ImageModeStride + index, 1u); + const auto active = Binary(state, spv::OpINotEqual, TypeBool(state), enabled, ConstantU32(state, 0u)); + const auto activeLabel = state.module.AllocateId(); + const auto inactiveLabel = state.module.AllocateId(); + modeMerge = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, modeMerge, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, active, activeLabel, inactiveLabel); + EmitLabel(state, inactiveLabel); + state.module.AddFunction(spv::OpUnreachable); + EmitLabel(state, activeLabel); + } emitMode(modes[index]); + if (modeMerge != 0u) { + state.module.AddFunction(spv::OpBranch, modeMerge); + EmitLabel(state, modeMerge); + } if (returnsValue) incoming.insert(incoming.end(), {ctx.Def(&inst), state.currentLabel}); ctx.definitions.erase(&inst); state.module.AddFunction(spv::OpBranch, merge); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp index bcf83148f..b1a37bb57 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp @@ -3,6 +3,7 @@ #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "Optimization/ResourceMaterializer.hpp" +#include "PipelineSpecialization.hpp" #include #include #include @@ -48,7 +49,9 @@ std::uint32_t DescriptorElementPointer(SpirvEmitterState& state, std::uint32_t r ExitDescriptorBindingFailure(state, kind, resource, variableName); } const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, resultPtrType, pointer, variableId, ConstantU32(state, arrayIndex)); + const bool heap = kind == DescriptorBindingKind::Samplers || ImageBindingResourceClass(kind) != ImageResourceClass::None; + const auto index = heap ? state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::DescriptorIndex(static_cast(kind), arrayIndex), arrayIndex) : ConstantU32(state, arrayIndex); + state.module.AddFunction(spv::OpAccessChain, resultPtrType, pointer, variableId, index); return pointer; } @@ -63,7 +66,7 @@ std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind ki } state.module.EmitCapability(dynamic); if (state.runtimeImageMetadata == 0u) FailEmit("bindless image has no runtime metadata index"); - const auto offset = Binary(state, spv::OpIAdd, TypeU32(state), ConstantU32(state, (offsetof(RuntimeAbi::ShaderData, images) + offsetof(RuntimeAbi::ResourceMetadata, firstElement)) / sizeof(std::uint32_t)), Binary(state, spv::OpIMul, TypeU32(state), state.runtimeImageMetadata, ConstantU32(state, sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)))); + const auto offset = Binary(state, spv::OpIAdd, TypeU32(state), ConstantU32(state, state.program.Metadata().bindings.ImageMetadataDword() + offsetof(RuntimeAbi::ResourceMetadata, firstElement) / sizeof(std::uint32_t)), Binary(state, spv::OpIMul, TypeU32(state), state.runtimeImageMetadata, ConstantU32(state, sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)))); const auto metadata = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), metadata, state.shaderDataStorageVariable, ConstantU32(state, 0u), offset); const auto index = state.module.AllocateId(); @@ -249,7 +252,16 @@ void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std state.module.AddFunction(std::span(words)); for (std::uint32_t mip = 0; mip < image.mipCount; mip++) { EmitLabel(state, labels[mip]); + const auto count = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::MipCountBase + resource, image.mipCount); + const auto active = Binary(state, spv::OpULessThan, TypeBool(state), ConstantU32(state, mip), count); + const auto writeLabel = state.module.AllocateId(); + const auto nextLabel = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, nextLabel, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, active, writeLabel, nextLabel); + EmitLabel(state, writeLabel); state.module.AddFunction(spv::OpImageWrite, loadAt(arrayIndex + mip), coord, texel); + state.module.AddFunction(spv::OpBranch, nextLabel); + EmitLabel(state, nextLabel); state.module.AddFunction(spv::OpBranch, mergeLabel); } EmitLabel(state, mergeLabel); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp index 8f2fcbed7..efb0d8e3f 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp @@ -1,5 +1,6 @@ #include "Optimization/ResourceMaterializer.hpp" #include "BdaAbi.hpp" +#include "PipelineSpecialization.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" @@ -342,9 +343,9 @@ void DefineDescriptors(SpirvEmitterState& state) { }; const auto ArrayType = [&](std::uint32_t type) { const bool heap = binding.kind == DescriptorBindingKind::Samplers || ImageBindingResourceClass(binding.kind) != ImageResourceClass::None; - const auto count = heap ? RuntimeAbi::HeapCapacity(binding.kind) : static_cast(binding.resources.size()); - if (binding.resources.size() > count) FailEmit("typed descriptor heap capacity exceeded"); - return state.module.Type(spv::OpTypeArray, type, ConstantU32(state, count)); + const auto count = static_cast(binding.resources.size()); + const auto length = heap ? state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::HeapCountBase + static_cast(binding.kind), count) : ConstantU32(state, count); + return state.module.Type(spv::OpTypeArray, type, length); }; switch (binding.kind) { case DescriptorBindingKind::Buffers: diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index 4e66c91fc..3e4478aa9 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -22,7 +22,7 @@ bool UserDataDwordIndex(const SpirvEmitterState& state, ScalarReg reg, std::uint if (found == registers.end() || *found != registerIndex) { return false; } - dwordIndex = RuntimeAbi::UserDataDword + registerIndex; + dwordIndex = static_cast(found - registers.begin()); return true; } @@ -1135,11 +1135,9 @@ void EmitSetAttribute(SpirvValueEmitContext& ctx, const IrValue& inst) { auto value = ExportVector(ctx, data, exp, uintOutput); if (state.program.Resources().stage == IrShaderStage::Pixel && exp.kind == ExportTargetKind::Mrt) { if (exp.index >= 8u) throw std::runtime_error("fragment export target exceeds the runtime mapping table"); - const auto mapping = EmitShaderDataDwordLoad(state, RuntimeAbi::ExportMappingsDword + exp.index); std::array components{}; for (std::uint32_t component = 0; component < components.size(); ++component) { - const auto shifted = Binary(state, spv::OpShiftRightLogical, TypeU32(state), mapping, ConstantU32(state, component * 2u)); - const auto index = Binary(state, spv::OpBitwiseAnd, TypeU32(state), shifted, ConstantU32(state, 3u)); + const auto index = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::ExportBase + exp.index * 4u + component, component); components[component] = state.module.AllocateId(); state.module.AddFunction(spv::OpVectorExtractDynamic, uintOutput ? TypeU32(state) : TypeF32(state), components[component], value, index); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp index 1271997e5..28813a2f1 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp @@ -56,7 +56,7 @@ bool HasLimitedUseTypes(std::span spirv) { } -std::vector ValidateAndOptimizeSpirv(std::span spirv, std::uint32_t vulkanVersion, std::uint32_t spirvVersion, bool allowOffsetTextureOperand, bool optimize) { +std::vector ValidateAndOptimizeSpirv(std::span spirv, std::uint32_t vulkanVersion, std::uint32_t spirvVersion, bool allowOffsetTextureOperand, bool optimize, bool specialize) { spv_target_env environment; const auto apiVersion = vulkanVersion & ~0xfffu; std::uint32_t maxSpirvVersion = 0; @@ -93,7 +93,7 @@ std::vector ValidateAndOptimizeSpirv(std::span(spirv.begin(), spirv.end()); + if (!specialize && mode != nullptr && std::string(mode) == "none") return std::vector(spirv.begin(), spirv.end()); // The recompiler emits helpers (BDA lookup, fault reporting) as functions called from every memory // access. Exhaustive inlining multiplies module size by ~10x and optimization time by ~20x, so the // default pipeline keeps the performance passes that work per function and leaves inlining to the driver. @@ -157,7 +157,7 @@ std::vector ValidateAndOptimizeSpirv(std::span optimized; diagnostics.clear();