From b08c410b9767ce6082f84d0d801d326edcff3df2 Mon Sep 17 00:00:00 2001 From: boykopovar Date: Mon, 5 Oct 2026 09:44:08 +0300 Subject: [PATCH] feat(shader): move buffer accesses to runtime descriptors via bda --- .../Graphics/src/GuestBufferMemory.cpp | 35 +- .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 36 +- .../tests/execution/BufferAtomicCsub.cpp | 2 +- .../tests/execution/BufferAtomics.cpp | 2 +- .../tests/execution/BufferAtomicsFloat.cpp | 2 +- .../tests/execution/BufferAtomicsIncDec64.cpp | 2 +- .../tests/execution/BufferLoadStore.cpp | 40 ++ .../tests/execution/FormatD16.cpp | 4 +- .../tests/execution/FormatD16Store.cpp | 2 +- .../tests/execution/FormattedStore.cpp | 2 +- .../tests/execution/TbufferFormatD16.cpp | 6 +- core/shader/recompiler/BdaAbi.hpp | 2 +- core/shader/recompiler/CMakeLists.txt | 1 + .../IrMetadata/Resources.hpp | 5 - .../Optimization/ResourceMaterializer.hpp | 11 - .../Optimization/src/ResourceMaterializer.cpp | 47 +- .../Optimization/src/ResourceTracker.cpp | 24 +- core/shader/recompiler/ShaderDiskCache.cpp | 24 +- core/shader/recompiler/ShaderDiskCache.hpp | 2 +- .../include/SpirvBackend/SpirvBda.hpp | 2 +- .../SpirvBackend/SpirvEmitterHelpers.hpp | 3 - .../SpirvBackend/SpirvEmitterState.hpp | 1 + .../SpirvMemory/SpirvBufferAccess.hpp | 3 - .../SpirvMemory/SpirvMemoryAccess.hpp | 3 - .../SpirvBackend/SpirvRuntimeBuffer.hpp | 16 + .../SpirvBackend/src/SpirvAnalysis.cpp | 17 +- .../SpirvBackend/src/SpirvBdaLookup.cpp | 3 +- .../SpirvBackend/src/SpirvBdaRead.cpp | 20 +- .../src/SpirvMemory/SpirvBufferAccess.cpp | 39 +- .../SpirvMemory/SpirvMemoryInstructions.cpp | 597 ++---------------- .../SpirvBackend/src/SpirvRuntimeBuffer.cpp | 339 ++++++++++ 31 files changed, 565 insertions(+), 727 deletions(-) create mode 100644 core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvRuntimeBuffer.hpp create mode 100644 core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 3d96f1012..cf57d5644 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -2514,17 +2514,40 @@ std::vector GuestBufferMemory::AddressRanges() std::vector result; result.reserve(regions.size() + (space != nullptr ? space->ranges.size() : 0)); for (const auto& region : regions) result.push_back(addressRange(region)); - if (space == nullptr) return result; - // The table is searched by address (the recompiler's lookup bisects it): both lists are sorted - // and disjoint, so a merge keeps it so. + if (space != nullptr) result.insert(result.end(), space->ranges.begin(), space->ranges.end()); + std::sort(result.begin(), result.end(), [](const auto& left, const auto& right) { return left.begin < right.begin; }); std::vector merged; - merged.reserve(result.size() + space->ranges.size()); - std::merge(space->ranges.begin(), space->ranges.end(), result.begin(), result.end(), std::back_inserter(merged), [](const auto& left, const auto& right) { return left.begin < right.begin; }); + for (const auto& range : result) { + if ((range.permissions & ShaderRecompiler::BdaAbi::Write) != 0u) { + merged.push_back(range); + continue; + } + std::vector boundaries{range.begin, range.end}; + for (const auto& [begin, end] : writes) { + if (begin >= range.end || end <= range.begin) continue; + boundaries.push_back(std::max(begin, range.begin)); + boundaries.push_back(std::min(end, range.end)); + } + std::sort(boundaries.begin(), boundaries.end()); + boundaries.erase(std::unique(boundaries.begin(), boundaries.end()), boundaries.end()); + for (std::size_t index = 1; index < boundaries.size(); ++index) { + auto part = range; + part.begin = boundaries[index - 1]; + part.end = boundaries[index]; + part.deviceAddress += part.begin - range.begin; + if (WritesOverlap(part.begin, static_cast(part.end - part.begin))) { + const auto* region = owner(part.begin); + Require(region != nullptr && region->writable && (region->mirror == nullptr || region->mirror->writable), "runtime buffer write has no writable GPU owner"); + part.permissions |= ShaderRecompiler::BdaAbi::Write; + } + merged.push_back(part); + } + } return merged; } std::optional GuestBufferMemory::CachedAddressTable() const { - if (space == nullptr || !regions.empty()) return std::nullopt; + if (space == nullptr || !regions.empty() || !writes.empty()) return std::nullopt; Require(uploaded && !committed, "guest GPU address ranges are not available"); return CachedTable{space->serial, &space->ranges}; } diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index c96216a18..18009f385 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -155,8 +155,6 @@ CompiledVariant sampleVariant() { buffer.source = 3; buffer.firstUsePc = 0x40; buffer.maxByteExtent = 256; - buffer.packedStride = 16; - buffer.descriptorSwizzle = 0x123u; buffer.imageAlias = 1; buffer.read = true; buffer.atomic = true; @@ -235,7 +233,6 @@ struct SampleRequest { request.target.maxWorkgroupInvocations = 1024; request.target.maxWorkgroupSharedMemoryBytes = 49152; request.layout = {0, 0, 0, 128}; - specialization.buffers = {{16u, IrBufferFormat::Invalid, 0xfacu}}; specialization.images = {ResourceSpecialization::Image{}}; specialization.boundDescriptors = {0, 0}; } @@ -299,7 +296,7 @@ void verifyEntryRoundTrip() { otherKey.back() ^= std::byte{1}; require(ShaderDiskCache::DecodeEntry(file, otherKey, decoded) == ShaderDiskCache::LoadStatus::KeyMismatch, "an entry for another key loads"); auto previousVersion = file; - previousVersion[4] = std::byte{3}; + previousVersion[4] = std::byte{4}; require(ShaderDiskCache::DecodeEntry(previousVersion, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, "an entry with the previous format version loads"); } @@ -368,10 +365,6 @@ void verifyKeySensitivity() { changes("the first binding", [](SampleRequest& sample) { sample.request.layout.firstBinding = 1; }); changes("the push constant offset", [](SampleRequest& sample) { sample.request.layout.pushConstantOffsetBytes = 16; }); changes("the push constant size", [](SampleRequest& sample) { sample.request.layout.pushConstantSizeBytes = 64; }); - changes("a buffer stride", [](SampleRequest& sample) { sample.specialization.buffers[0].packedStride = 32; }); - changes("a buffer format", [](SampleRequest& sample) { sample.specialization.buffers[0].descriptorFormat = static_cast(1); }); - changes("a buffer swizzle", [](SampleRequest& sample) { sample.specialization.buffers[0].descriptorSwizzle = 0xfadu; }); - changes("the buffer count", [](SampleRequest& sample) { sample.specialization.buffers.emplace_back(); }); changes("an image class", [](SampleRequest& sample) { sample.specialization.images[0].numericClass = IrTextureNumericClass::Float; }); changes("an image dimension", [](SampleRequest& sample) { sample.specialization.images[0].dimension = static_cast(1); }); changes("an image mip count", [](SampleRequest& sample) { sample.specialization.images[0].mipCount = 2; }); @@ -428,7 +421,8 @@ void verifyStore() { struct ComputeRequest { std::vector code{0xe0700000u, 0x80000000u, 0xbf810000u}; std::array userData{0x10000000u, 0x00000000u, 0x40u, 0x00027facu}; - std::array capabilities{1u}; + std::array capabilities{1u, 11u, 5347u, 4448u}; + std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; RecompileRequest request{}; explicit ComputeRequest(bool useCache) { @@ -441,6 +435,8 @@ struct ComputeRequest { request.target.spirvVersion = 0x00010300u; request.target.subgroupSize = 32; request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.bdaAbiVersion = 1u; request.layout.pushConstantSizeBytes = 128; request.useCache = useCache; } @@ -574,6 +570,28 @@ void verifyInvocationIsolation() { request.userData[0] -= 0x10000u; const auto restored = Recompile(request.request); requireSameResult(first, restored, "restored invocation"); + const auto original = request.userData; + const std::array, 6> descriptors{{ + {original[0] + 0x20000u, original[1], original[2], original[3]}, + {original[0], original[1], original[2] / 2u, original[3]}, + {original[0], original[1] | (16u << 16u), original[2], original[3]}, + {original[0], original[1], original[2], 0x31004688u}, + {original[0], original[1] | 0x80100000u, original[2], original[3] | 0x00600000u}, + {0u, 0u, 0u, 0u} + }}; + for (const auto& descriptor : descriptors) { + request.userData = descriptor; + const auto changed = Recompile(request.request); + require(changed.cacheHit && first.variantId == changed.variantId, "buffer metadata changed the compiled variant"); + requireSameArtifact(first, changed, "runtime buffer artifact"); + require(first.spirv.data() == changed.spirv.data(), "buffer metadata duplicated the SPIR-V"); + } + request.userData = original; + request.userData[3] |= 0x40000000u; + bool rejected = false; + try { static_cast(Recompile(request.request)); } + catch (const std::runtime_error&) { rejected = true; } + require(rejected, "an unsupported buffer descriptor type was accepted"); } void verifyDefaultDirectory(const char* self) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp index aab4cf664..5a2a267f4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp @@ -79,7 +79,7 @@ std::uint32_t SubtractClamped(std::uint32_t memory, std::uint32_t subtrahend) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp index 2fe922afb..426b177c0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp @@ -51,7 +51,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsFloat.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsFloat.cpp index 38686a9c2..f70162d30 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsFloat.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsFloat.cpp @@ -119,7 +119,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsIncDec64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsIncDec64.cpp index ffaf99d84..022910766 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsIncDec64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsIncDec64.cpp @@ -111,7 +111,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp index d5e82afdf..60f01abfc 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp @@ -288,6 +288,45 @@ void Run(AgcDriver::VulkanDevice& device, std::span code, s device.WaitIdle(); } +void CheckRuntimeDescriptors(AgcDriver::VulkanDevice& device) { + constexpr std::array code{0x34040082u, 0xe0000000u, 0x80000100u, 0xe0701000u, 0x80010102u, 0xbf810000u}; + struct Case { + std::uint32_t stride; + std::uint32_t records; + std::uint32_t format; + std::uint32_t selector; + bool swizzle; + }; + constexpr std::array cases{{{4u, 32u, 20u, 4u, false}, {8u, 32u, 5u, 4u, false}, {8u, 32u, 11u, 4u, true}, {4u, 32u, 18u, 5u, false}, {4u, 16u, 20u, 4u, false}}}; + for (std::uint32_t index = 0; index < Input.size(); ++index) Input[index] = 0x3f000000u + index * 0x201u; + std::array userData{}; + const auto output = BufferDescriptor(Output.data(), 128u); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const ShaderRecompiler::ShaderComputeStageInfo compute{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + ShaderRecompiler::RecompileRequest request{{ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32u, 0u, userData, compute, std::nullopt, std::nullopt, {}}, device.Target(), {0u, 0u, 0u, 128u}}; + ShaderRecompiler::RecompileResult first; + for (const auto& value : cases) { + auto input = BufferDescriptor(Input.data(), value.records); + input[1] |= (value.stride << 16u) | (value.swizzle ? 0x80000000u : 0u); + input[3] = 0x11800000u | (value.format << 12u) | value.selector; + std::copy(input.begin(), input.end(), userData.begin()); + Output.fill(0xdeadbeefu); + const auto result = ShaderRecompiler::Recompile(request); + if (first.spirv.empty()) first = result; + else Require(result.cacheHit && result.variantId == first.variantId && result.spirv.data() == first.spirv.data(), "runtime V# changed the formatted shader artifact"); + device.Dispatch(result, 1, 1, 1); + device.WaitIdle(); + for (std::uint32_t lane = 0; lane < 32u; ++lane) { + const auto index = value.swizzle ? (lane / 8u) * 16u + lane % 8u : lane * value.stride / 4u; + auto expected = lane >= value.records ? 0u : Input[index]; + if (value.format == 5u) expected &= 0xffu; + if (value.format == 11u) expected &= 0xffffu; + if (value.format == 18u) expected = (expected >> 8u) & 0xffu; + Require(Output[lane] == expected, "runtime V# formatted load mismatch at lane " + std::to_string(lane)); + } + } +} + template void Check(const std::uint32_t (&expected)[Lanes][Checked], const char* const (&names)[Checked], const char* run) { for (std::uint32_t tid = 0; tid < Lanes; ++tid) { @@ -310,6 +349,7 @@ int main() { Check(Expected64, Names, "wave64"); Run(*device, Wave64Code, 64, device->ComputeTarget(32)); Check(Expected64, Names, "wave64 split"); + CheckRuntimeDescriptors(*device); std::puts("buffer load store tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16.cpp index 54a17ee50..635afe2fc 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16.cpp @@ -24,7 +24,7 @@ alignas(256) std::array Input{}; alignas(256) std::array Output{}; alignas(256) constexpr std::array Code{ - 0x34020084, 0x34060086, 0xbe900300, 0xbe910301, 0xbe920302, 0xbe9303ff, 0x01001244, 0x7e1402ff, + 0x34020084, 0x34060086, 0xbe900300, 0xbe910301, 0xbe920302, 0xbe9303ff, 0x31001244, 0x7e1402ff, 0xdead0000, 0x7e1602ff, 0xdead0000, 0x7e1802ff, 0xdead0000, 0x7e1a02ff, 0xdead0000, 0x7e1c02ff, 0xdead0000, 0x7e1e02ff, 0xdead0000, 0x7e2002ff, 0xdead0000, 0x7e2202ff, 0xdead0000, 0x7e2402ff, 0xdead0000, 0x7e2602ff, 0xdead0000, 0x7e2802ff, 0xbeefbeef, 0x7e2a02ff, 0xbeefbeef, 0x7e2c02ff, @@ -115,7 +115,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16Store.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16Store.cpp index 33dc26b02..126eb9bc2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16Store.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16Store.cpp @@ -112,7 +112,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStore.cpp index 8c8491eab..c608436a3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStore.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStore.cpp @@ -112,7 +112,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/TbufferFormatD16.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/TbufferFormatD16.cpp index 8367d9e9f..14588e3c6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/TbufferFormatD16.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/TbufferFormatD16.cpp @@ -24,8 +24,8 @@ alignas(256) std::array Input{}; alignas(256) std::array Output{}; alignas(256) constexpr std::array Code{ - 0x34020084, 0x34060086, 0xbe880300, 0xbe890301, 0xbe8a0302, 0xbe8b03ff, 0x0104bfac, 0xbe8c0300, - 0xbe8d0301, 0xbe8e0302, 0xbe8f03ff, 0x0104b80f, 0x7e1402ff, 0xdeadbeef, 0x7e1602ff, 0xdeadbeef, + 0x34020084, 0x34060086, 0xbe880300, 0xbe890301, 0xbe8a0302, 0xbe8b03ff, 0x3104bfac, 0xbe8c0300, + 0xbe8d0301, 0xbe8e0302, 0xbe8f03ff, 0x3104b80f, 0x7e1402ff, 0xdeadbeef, 0x7e1602ff, 0xdeadbeef, 0x7e1802ff, 0xdeadbeef, 0x7e1a02ff, 0xdeadbeef, 0x7e1c02ff, 0xdeadbeef, 0x7e1e02ff, 0xdeadbeef, 0x7e2002ff, 0xdeadbeef, 0x7e2202ff, 0xdeadbeef, 0x7e2402ff, 0xdeadbeef, 0x7e2602ff, 0xdeadbeef, 0x7e2802ff, 0xdeadbeef, 0x7e2a02ff, 0xdeadbeef, 0x7e2c02ff, 0xdeadbeef, 0x7e2e02ff, 0xdeadbeef, @@ -122,7 +122,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/shader/recompiler/BdaAbi.hpp b/core/shader/recompiler/BdaAbi.hpp index 8632c87b9..aa31e43e9 100644 --- a/core/shader/recompiler/BdaAbi.hpp +++ b/core/shader/recompiler/BdaAbi.hpp @@ -27,7 +27,7 @@ struct Range { }; enum class FaultState : std::uint32_t { Empty, Writing, Ready }; -enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle, LoopLimit, Unaligned }; +enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle, LoopLimit, Unaligned, InvalidDescriptor }; struct Fault { FaultState state; diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 0dceb786f..85ee56d4b 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -127,6 +127,7 @@ function(add_shader_recompiler target runtimeTarget) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvBdaRead.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvEmitter.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvBufferFormat.cpp + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvRuntimeBuffer.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvMemory/SpirvConstants.cpp diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp index 4725071cb..59ac1e579 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp @@ -17,17 +17,12 @@ struct BufferResource { std::uint32_t source = 0; std::uint32_t firstUsePc = 0; std::uint32_t maxByteExtent = 0; - std::uint32_t packedStride = 0; - IrBufferFormat descriptorFormat = IrBufferFormat::Invalid; - std::uint32_t descriptorSwizzle = 0x00000facu; std::uint32_t imageAlias = NoImageAlias; bool read = false; bool written = false; bool atomic = false; bool formatted = false; bool scalar = false; - bool empty = false; - std::uint8_t baseMisalignment = 0; bool operator==(const BufferResource& other) const = default; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp index 78a4f9106..b24a47e88 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp @@ -9,16 +9,6 @@ namespace ShaderRecompiler { struct ResourceSpecialization { - struct Buffer { - std::uint32_t packedStride = 0; - IrBufferFormat descriptorFormat = IrBufferFormat::Invalid; - std::uint32_t descriptorSwizzle = ShaderImageIdentitySwizzle; - bool empty = false; - std::uint8_t baseMisalignment = 0; - - bool operator==(const Buffer& other) const; - }; - struct Image { IrTextureNumericClass numericClass = IrTextureNumericClass::Unsupported; RdnaImageDimension dimension = RdnaImageDimension::Unknown; @@ -39,7 +29,6 @@ struct ResourceSpecialization { bool operator==(const Image& other) const; }; - std::vector buffers; std::vector images; bool operator==(const ResourceSpecialization& other) const; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 6f2eaca98..ed8882247 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -574,28 +574,10 @@ std::uint32_t emulatedCompareState(const IrResourcePlan& plan, const ResourceSna void buildResourceSpecialization(const IrResourcePlan& plan, ResourceSnapshot& snapshot, const std::vector& tables, ResourceSpecialization& specialization) { ResourceSpecialization result; - result.buffers.reserve(plan.info.buffers.size()); - for (std::uint32_t i = 0; i < plan.info.buffers.size(); i++) { - const ShaderBufferResource decoded = decodeBufferDescriptor(snapshot.buffers[i]); - if (decoded.Type() != 0u) { - throw std::runtime_error("buffer descriptor uses an unsupported type"); - } - auto packedStride = decoded.PackedStride(); - const auto stride = packedStride & 0x3fffu; - const bool swizzleActive = stride != 0u && ((packedStride >> 14u) & 1u) != 0u; - if (stride == 0u) { - packedStride &= ~((1u << 14u) | (3u << 16u)); - } else if (!swizzleActive) { - packedStride &= ~(3u << 16u); - } - const auto& buffer = plan.info.buffers[i]; - ResourceSpecialization::Buffer entry; - entry.packedStride = packedStride; - entry.descriptorFormat = buffer.formatted ? decoded.Format() : IrBufferFormat::Invalid; - entry.descriptorSwizzle = buffer.formatted ? decoded.DstSelXYZW() : DstSel(4, 5, 6, 7); - entry.empty = decoded.GetSize() == 0u || decoded.Base48() == 0u; - entry.baseMisalignment = entry.empty ? 0u : static_cast(decoded.Base48() & 3u); - result.buffers.push_back(entry); + for (std::uint32_t i = 0; i < plan.info.buffers.size(); ++i) { + const auto decoded = decodeBufferDescriptor(snapshot.buffers.at(i)); + if (decoded.Type() != 0u) throw std::runtime_error("buffer descriptor uses an unsupported type"); + if (plan.stage != IrShaderStage::Compute && decoded.AddTid()) throw std::runtime_error("buffer ADD_TID is only valid for compute shaders"); } result.images.reserve(plan.info.images.size()); @@ -653,8 +635,8 @@ void buildResourceSpecialization(const IrResourcePlan& plan, ResourceSnapshot& s } result.boundDescriptors.clear(); - result.boundDescriptors.reserve(result.buffers.size() + result.images.size()); - for (std::uint32_t index = 0; index < result.buffers.size(); index++) { + result.boundDescriptors.reserve(plan.info.buffers.size() + result.images.size()); + for (std::uint32_t index = 0; index < plan.info.buffers.size(); index++) { result.boundDescriptors.push_back(index); } for (std::uint32_t index = 0; index < result.images.size(); index++) { @@ -670,22 +652,11 @@ void ResourceMaterializer::Apply(IrProgram& program, const ResourceSpecializatio if (!resources.resourceTrackingComplete) { throw std::runtime_error("ResourceMaterializer::Apply requires a completed resource plan"); } - if (resources.info.buffers.size() != specialization.buffers.size()) { - throw std::runtime_error("ResourceMaterializer::Apply buffer count mismatch"); - } if (resources.info.images.size() > specialization.images.size()) { throw std::runtime_error("ResourceMaterializer::Apply image count mismatch"); } auto buffers = resources.info.buffers; - for (std::uint32_t i = 0; i < buffers.size(); i++) { - buffers[i].packedStride = specialization.buffers[i].packedStride; - buffers[i].descriptorFormat = specialization.buffers[i].descriptorFormat; - buffers[i].descriptorSwizzle = specialization.buffers[i].descriptorSwizzle; - buffers[i].empty = specialization.buffers[i].empty; - buffers[i].baseMisalignment = specialization.buffers[i].baseMisalignment; - } - auto images = resources.info.images; images.reserve(specialization.images.size()); for (std::uint32_t index = 0; index < specialization.images.size(); index++) { @@ -1037,16 +1008,12 @@ void ResourceMaterializer::CountBindlessRejection(BindlessRejection reason) { if (reason < BindlessRejection::Count) bindlessCounters().rejected[static_cast(reason)].fetch_add(1, std::memory_order_relaxed); } -bool ResourceSpecialization::Buffer::operator==(const Buffer& other) const { - return packedStride == other.packedStride && descriptorFormat == other.descriptorFormat && descriptorSwizzle == other.descriptorSwizzle && empty == other.empty && baseMisalignment == other.baseMisalignment; -} - bool ResourceSpecialization::Image::operator==(const Image& other) const { return numericClass == other.numericClass && dimension == other.dimension && mipCount == other.mipCount && conversionFormat == other.conversionFormat && shaderSwizzle == other.shaderSwizzle && indirectRoot == other.indirectRoot && indirectMappingOffset == other.indirectMappingOffset && indirectSearchIterations == other.indirectSearchIterations && cube == other.cube && fmask == other.fmask && depthBits == other.depthBits && depthUnorm16 == other.depthUnorm16 && packedFormat == other.packedFormat && emulatedCompare == other.emulatedCompare && srgbDecode == other.srgbDecode; } bool ResourceSpecialization::operator==(const ResourceSpecialization& other) const { - return buffers == other.buffers && images == other.images; + return images == other.images; } } diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index c6839c69f..5e6a7e204 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -798,19 +798,17 @@ private: DescriptorSource descriptor; MakeSource(*handle, 4u, false, false, descriptor); std::uint32_t badDword = 0; - if (ValidateSource(descriptor, badDword)) { - return false; - } - const auto op = inst.Opcode(); - const bool load = op == IrOpcode::LoadBufferU32 || op == IrOpcode::LoadBufferU32x2 || op == IrOpcode::LoadBufferU32x3 || op == IrOpcode::LoadBufferU32x4 || op == IrOpcode::ReadConstBuffer; - const bool store = op == IrOpcode::StoreBufferU32 || op == IrOpcode::StoreBufferU32x2 || op == IrOpcode::StoreBufferU32x3 || op == IrOpcode::StoreBufferU32x4; auto& memory = m_program.Resources().memoryInfo[memoryIndex]; - if ((!load && !store) || memory.formatted || memory.typed || memory.dataBits != 32u) { - return false; + const auto access = BufferAccessOf(inst.Opcode()); + if (ValidateSource(descriptor, badDword)) { + const auto source = InternSource(descriptor); + const auto resource = AddBuffer(source, memory, inst.Opcode(), inst.Flags().pc); + if (resource == std::numeric_limits::max()) fail("buffer resource limit exceeded"); + AddMemoryPatch(memoryIndex, resource, 0u, false); } memory.gpuDescriptor = true; m_info.usesDma = true; - m_info.bdaWrites = m_info.bdaWrites || store; + m_info.bdaWrites = m_info.bdaWrites || access == BufferAccess::Write || access == BufferAccess::Atomic; return true; } @@ -979,13 +977,7 @@ private: std::uint32_t resource = 0; if (buffer != BufferAccess::None) { - if (TakeGpuDescriptor(inst, flags.index)) { - return; - } - GetHandle(inst.Argument(0), IrOpcode::GetBufferResource, 4, handle, source); - resource = AddBuffer(source, memory, op, flags.pc); - AddHandlePatch(handle, resource); - AddMemoryPatch(flags.index, resource, 0, false); + if (!TakeGpuDescriptor(inst, flags.index)) fail("buffer operation requires a four-dword runtime V#"); return; } if (addressInfo.access != AddressAccess::None) { diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 800e472bd..72306b163 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -72,7 +72,7 @@ static_assert(sizeof(VertexInput) == 12, "VertexInput changed: update the vertex static_assert(sizeof(FragmentParameter) == 12, "FragmentParameter changed: update the parameter encoder"); static_assert(sizeof(CompiledShaderInfo) == 304, "CompiledShaderInfo changed: update the info encoder"); static_assert(sizeof(ShaderInfo) == 200, "ShaderInfo changed: update the info encoder"); -static_assert(sizeof(BufferResource) == 36, "BufferResource changed: update the info encoder"); +static_assert(sizeof(BufferResource) == 24, "BufferResource changed: update the info encoder"); static_assert(sizeof(ImageResource) == 96, "ImageResource changed: update the info encoder"); static_assert(sizeof(SamplerResource) == 16, "SamplerResource changed: update the info encoder"); static_assert(sizeof(SampledResourcePair) == 12, "SampledResourcePair changed: update the info encoder"); @@ -82,8 +82,7 @@ static_assert(sizeof(IrBindingLayout) == 64, "IrBindingLayout changed: update th static_assert(sizeof(IrDescriptorBinding) == 32, "IrDescriptorBinding changed: update the layout encoder"); static_assert(sizeof(BindingAllocationResult) == 120, "BindingAllocationResult changed: update the allocation encoder"); static_assert(sizeof(CompiledBindingLayout) == 72, "CompiledBindingLayout changed: update the allocation encoder"); -static_assert(sizeof(ResourceSpecialization) == 72, "ResourceSpecialization changed: update BuildKey"); -static_assert(sizeof(ResourceSpecialization::Buffer) == 16, "ResourceSpecialization::Buffer changed: update BuildKey"); +static_assert(sizeof(ResourceSpecialization) == 48, "ResourceSpecialization changed: update BuildKey"); static_assert(sizeof(ResourceSpecialization::Image) == 48, "ResourceSpecialization::Image changed: update BuildKey"); static_assert(sizeof(BindingLayout) == 16, "BindingLayout changed: update BuildKey"); #endif @@ -404,17 +403,12 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { out.Value(buffer.source); out.Value(buffer.firstUsePc); out.Value(buffer.maxByteExtent); - out.Value(buffer.packedStride); - out.Value(buffer.descriptorFormat); - out.Value(buffer.descriptorSwizzle); out.Value(buffer.imageAlias); out.Value(buffer.read); out.Value(buffer.written); out.Value(buffer.atomic); out.Value(buffer.formatted); out.Value(buffer.scalar); - out.Value(buffer.empty); - out.Value(buffer.baseMisalignment); }); writer.List(info.images, [](Writer& out, const ImageResource& image) { out.Value(image.source); @@ -495,21 +489,16 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { auto& info = compiled.info; reader.Value(info.scratchDwords); reader.Value(info.sharedMemoryBytes); - reader.List(info.buffers, 35, [](Reader& in, BufferResource& buffer) { + reader.List(info.buffers, 21, [](Reader& in, BufferResource& buffer) { in.Value(buffer.source); in.Value(buffer.firstUsePc); in.Value(buffer.maxByteExtent); - in.Value(buffer.packedStride); - in.Value(buffer.descriptorFormat); - in.Value(buffer.descriptorSwizzle); in.Value(buffer.imageAlias); in.Value(buffer.read); in.Value(buffer.written); in.Value(buffer.atomic); in.Value(buffer.formatted); in.Value(buffer.scalar); - in.Value(buffer.empty); - in.Value(buffer.baseMisalignment); }); reader.List(info.images, 63, [](Reader& in, ImageResource& image) { in.Value(image.source); @@ -823,13 +812,6 @@ void BuildKey(const RecompileRequest& request, std::uint32_t hostSubgroupSize, c writer.Value(request.layout.firstBinding); writer.Value(request.layout.pushConstantOffsetBytes); writer.Value(request.layout.pushConstantSizeBytes); - writer.List(specialization.buffers, [](Writer& out, const ResourceSpecialization::Buffer& buffer) { - out.Value(buffer.packedStride); - out.Value(buffer.descriptorFormat); - out.Value(buffer.descriptorSwizzle); - out.Value(buffer.empty); - out.Value(buffer.baseMisalignment); - }); writer.List(specialization.images, [](Writer& out, const ResourceSpecialization::Image& image) { out.Value(image.numericClass); out.Value(image.dimension); diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 485e572c3..29e244b46 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -12,7 +12,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 9; +inline constexpr std::uint32_t FormatVersion = 8; enum class LoadStatus { Loaded, diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp index a1323899b..4b68725c7 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp @@ -23,7 +23,7 @@ void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& targe bool BdaInvocationsMayStop(const IrProgram& program); void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition); std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits); -void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value); +void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u); void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits); std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bytes, const std::function& operation); // Reads the dwords of a 1-4 dword load at address + offset that the program extracts, with one diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index e266e8a90..2e9d0d0d4 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -99,13 +99,10 @@ std::uint32_t EmitBallotLaneActiveBool(SpirvEmitterState& state, std::uint32_t b std::uint32_t EmitSubgroupLaneActiveBool(SpirvEmitterState& state, std::uint32_t lane); std::uint32_t EmitBinaryU32(SpirvEmitterState& state, std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs); std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dwordIndex); -std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem); -IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem); void EmitMemoryOffsets(SpirvEmitterState& state); std::uint32_t LdsDwordCount(const SpirvEmitterState& state); std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state); MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem); -MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType); std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t rawIndex); std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); std::uint32_t EmitMemoryElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index 8a88e4fd7..ae0131609 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -107,6 +107,7 @@ struct SpirvEmitterState { // every read takes the byte path. std::uint32_t bdaProbeFunction = 0; std::uint32_t bdaWritePointerFunction = 0; + std::uint32_t bdaAtomicPointerFunction = 0; std::uint32_t bdaNoteWriteFunction = 0; std::uint32_t bdaFaultFunction = 0; std::array, 2> bdaDwordReadFunctions {}; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp index 2701d0530..ee30b6506 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp @@ -9,12 +9,9 @@ namespace ShaderRecompiler { std::uint32_t EmitBinaryU32(SpirvEmitterState& state, std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs); std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dwordIndex); -std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem); -IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem); void EmitMemoryOffsets(SpirvEmitterState& state); std::uint32_t LdsDwordCount(const SpirvEmitterState& state); std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state); -MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType); MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem); std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t rawIndex); std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvMemoryAccess.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvMemoryAccess.hpp index 1331877d0..4228fbc05 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvMemoryAccess.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvMemoryAccess.hpp @@ -7,13 +7,10 @@ namespace ShaderRecompiler { std::uint32_t LdsDwordCount(const SpirvEmitterState& state); -std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem); -IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem); void EmitMemoryOffsets(SpirvEmitterState& state); MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem); -MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType); std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t rawIndex); std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvRuntimeBuffer.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvRuntimeBuffer.hpp new file mode 100644 index 000000000..bd7fa1a63 --- /dev/null +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvRuntimeBuffer.hpp @@ -0,0 +1,16 @@ +#ifndef CORE_SHADER_RECOMPILER_SPIRVBACKEND_SPIRVRUNTIMEBUFFER_HPP +#define CORE_SHADER_RECOMPILER_SPIRVBACKEND_SPIRVRUNTIMEBUFFER_HPP + +#include "SpirvBackend/SpirvEmitterHelpers.hpp" +#include + +namespace ShaderRecompiler { + +std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components); +void EmitRuntimeBufferStore(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components); +std::uint32_t EmitRuntimeScalarBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction); +std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t bits, std::uint32_t active, const std::function& operation); + +} + +#endif diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp index 9ea921ea8..706711c60 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp @@ -333,20 +333,9 @@ SpirvRequirements AnalyzeProgramRequirements(const IrProgram& program) { const auto& memory = program.Resources().memoryInfo.at(memoryIndex); requirements.coherentBuffers = requirements.coherentBuffers || (memory.coherent && !memory.gpuDescriptor); if (memory.gpuDescriptor) { - if (program.Resources().stage != IrShaderStage::Compute) { - throw std::runtime_error("GPU-selected buffer descriptors outside compute shaders are not implemented"); - } - requirements.subgroupLocalInvocationId = true; - } else if (memory.kind == ResourceKind::Buffer) { - if (memory.resource >= program.Info().buffers.size()) { - throw std::runtime_error("buffer operation has invalid resource metadata"); - } - if ((program.Info().buffers.at(memory.resource).packedStride & (1u << 20u)) != 0u) { - if (program.Resources().stage != IrShaderStage::Compute) { - throw std::runtime_error("buffer ADD_TID is only valid for compute shaders"); - } - requirements.subgroupLocalInvocationId = true; - } + requirements.subgroupLocalInvocationId = requirements.subgroupLocalInvocationId || program.Resources().stage == IrShaderStage::Compute; + } else if (!memory.planningOnly) { + throw std::runtime_error("buffer operation requires runtime descriptor metadata"); } } const auto sharedAccess = SharedAccessOf(inst->Opcode()); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp index 98c1b24d2..e6d905db3 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp @@ -102,7 +102,7 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r fail(binary(spv::OpINotEqual, boolean, BdaLoadWord(state, at(7)), constant(0)), BdaAbi::FaultReason::InvalidTable); fail(binary(spv::OpUGreaterThanEqual, boolean, begin, finish), BdaAbi::FaultReason::InvalidTable); fail(binary(spv::OpUGreaterThan, boolean, end, finish), BdaAbi::FaultReason::Unmapped); - fail(binary(spv::OpIEqual, boolean, binary(spv::OpBitwiseAnd, u32, permissions, constant(permission)), constant(0)), BdaAbi::FaultReason::Permission); + fail(binary(spv::OpINotEqual, boolean, binary(spv::OpBitwiseAnd, u32, permissions, constant(permission)), constant(permission)), BdaAbi::FaultReason::Permission); fail(binary(spv::OpIEqual, boolean, base, BdaConstant(state, 0)), BdaAbi::FaultReason::InvalidTable); const auto offset = binary(spv::OpISub, u64, address, begin); const auto result = binary(spv::OpIAdd, u64, base, offset); @@ -164,6 +164,7 @@ void DefineGetBdaPointer(SpirvEmitterState& state) { DefineBdaDwordReadFunctions(state); if (state.program.Info().bdaWrites) { state.bdaWritePointerFunction = DefineBdaLookup(state, "get_bda_write_pointer", true, BdaAbi::Write); + state.bdaAtomicPointerFunction = DefineBdaLookup(state, "get_bda_atomic_pointer", true, BdaAbi::Read | BdaAbi::Write); state.bdaNoteWriteFunction = DefineBdaNoteWrite(state); } } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index db3fb5db8..60df7b542 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -197,10 +197,28 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std:: return EmitBdaBytes(state, address, bits / 8u, instruction, BdaAccessMask(state, inst)); } -void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value) { +void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits) { auto& state = ctx.state; if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); const auto instruction = ConstantU32(state, inst.Flags().pc); + if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA write width"); + if (bits != 32u) { + EmitBdaOverflowCheck(state, address, bits / 8u, instruction); + for (std::uint32_t byte = 0u; byte < bits / 8u; ++byte) { + const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, BdaConstant(state, byte)); + const auto physical = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaWritePointerFunction, guest, ConstantU32(state, 1u), instruction); + EmitIfCondition(state, Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)), [&] { + const auto byteType = state.module.Type(spv::OpTypeInt, 8u, 0u); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, byteType), pointer, physical); + const auto shifted = Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, ConstantU32(state, byte * 8u)); + state.module.AddFunction(spv::OpStore, pointer, Unary(state, spv::OpUConvert, byteType, shifted), BdaAccessMask(state, inst), 1u); + state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, guest); + }); + } + return; + } const auto unaligned = BdaAddressUnaligned(state, address); EmitIfCondition(state, unaligned, [&] { RecordBdaFault(state, address, ConstantU32(state, 4u), instruction, BdaAbi::FaultReason::Unaligned); }); EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), [&] { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp index 51296ad21..356cb4edf 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp @@ -46,20 +46,6 @@ std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dw return value; } -std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem) { - if (mem.resource >= state.program.Info().buffers.size()) { - ExitDescriptorBindingFailure(state, DescriptorBindingKind::Buffers, mem.resource, "buffer specialization is missing"); - } - return state.program.Info().buffers[mem.resource].packedStride; -} - -IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem) { - if (mem.resource >= state.program.Info().buffers.size()) { - ExitDescriptorBindingFailure(state, DescriptorBindingKind::Buffers, mem.resource, "buffer specialization is missing"); - } - return state.program.Info().buffers[mem.resource].descriptorFormat; -} - void EmitMemoryOffsets(SpirvEmitterState& state) { const IrBindingLayout& layout = state.program.Metadata().bindings; state.memoryByteOffsets.assign(layout.memoryOffsetCount, 0u); @@ -86,26 +72,6 @@ std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state) { return pointer; } -MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType) { - if (variable == 0) { - ExitDescriptorBindingFailure(state, DescriptorBindingKind::Buffers, mem.resource, "storage buffer descriptor array was not emitted"); - } - const auto arrayIndex = ResourceForDescriptor(state, DescriptorBindingKind::Buffers, mem.resource); - MemoryResourceAccess access; - access.kind = mem.kind; - access.objectPointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, pointerType, access.objectPointer, variable, ConstantU32(state, arrayIndex)); - access.byteOffset = state.memoryByteOffsets.at(arrayIndex); - access.memoryAccess = mem.coherent ? spv::MemoryAccessVolatileMask : 0u; - access.length = state.module.AllocateId(); - state.module.AddFunction(spv::OpArrayLength, TypeU32(state), access.length, access.objectPointer, 0u); - if (mem.resource < state.program.Info().buffers.size()) { - if (state.program.Info().buffers[mem.resource].empty) access.length = ConstantU32(state, 0u); - access.misalignment = state.program.Info().buffers[mem.resource].baseMisalignment; - } - return access; -} - MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem) { MemoryResourceAccess access; access.kind = mem.kind; @@ -138,10 +104,7 @@ MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const FailEmit("physical address memory must use the BDA emitter"); case ResourceKind::ScalarBuffer: case ResourceKind::Buffer: - access = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferVariable, TypeStorageBufferPointer(state)); - access.indexOffset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byteOffset, ConstantU32(state, 2u)); - access.addIndexOffset = true; - return access; + FailEmit("buffer memory must use the runtime V# emitter"); default: FailEmit("unsupported memory resource kind " + std::to_string(static_cast(mem.kind))); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index bd9d96280..3a058a038 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -1,4 +1,5 @@ #include "SpirvBackend/SpirvBda.hpp" +#include "SpirvBackend/SpirvRuntimeBuffer.hpp" #include "SpirvBackend/SpirvEmitterInstructions.hpp" #include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp" #include "SpirvBackend/SpirvBufferFormat.hpp" @@ -20,45 +21,6 @@ std::uint32_t AndCondition(SpirvEmitterState& state, std::uint32_t lhs, std::uin return Binary(state, spv::OpLogicalAnd, TypeBool(state), lhs, rhs); } -std::uint32_t BufferByteAddress(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t index, std::uint32_t offset, std::uint32_t soffset) { - auto& state = ctx.state; - const std::uint32_t packed = StorageBufferPackedStride(state, mem); - const std::uint32_t stride = packed & 0x3fffu; - const bool swizzle = stride != 0u && ((packed >> 14u) & 1u) != 0u; - if (((packed >> 20u) & 1u) != 0u) { - const auto lane = Binary(state, spv::OpBitwiseAnd, TypeU32(state), EmitSubgroupLocalInvocationId(state), ConstantU32(state, 63u)); - index = Binary(state, spv::OpIAdd, TypeU32(state), index, lane); - } - if (mem.offset != 0u) { - offset = Binary(state, spv::OpIAdd, TypeU32(state), offset, ConstantU32(state, mem.offset)); - } - std::uint32_t address = 0; - if (!swizzle) { - if (stride == 0u) { - address = offset; - } else { - const auto indexed = stride == 1u ? index : Binary(state, spv::OpIMul, TypeU32(state), index, ConstantU32(state, stride)); - address = Binary(state, spv::OpIAdd, TypeU32(state), indexed, offset); - } - } else { - const std::uint32_t strideEnum = (packed >> 16u) & 3u; - const std::uint32_t indexStride = 8u << strideEnum; - const auto indexMsb = Binary(state, spv::OpShiftRightLogical, TypeU32(state), index, ConstantU32(state, strideEnum + 3u)); - const auto indexLsb = Binary(state, spv::OpBitwiseAnd, TypeU32(state), index, ConstantU32(state, indexStride - 1u)); - const auto offsetMsb = Binary(state, spv::OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, ~3u)); - const auto offsetLsb = Binary(state, spv::OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, 3u)); - const auto indexedMsb = stride == 1u ? indexMsb : Binary(state, spv::OpIMul, TypeU32(state), indexMsb, ConstantU32(state, stride)); - const auto msb = Binary(state, spv::OpIMul, TypeU32(state), Binary(state, spv::OpIAdd, TypeU32(state), indexedMsb, offsetMsb), ConstantU32(state, indexStride)); - const auto lsb = Binary(state, spv::OpIAdd, TypeU32(state), Binary(state, spv::OpShiftLeftLogical, TypeU32(state), indexLsb, ConstantU32(state, 2u)), offsetLsb); - address = Binary(state, spv::OpIAdd, TypeU32(state), msb, lsb); - } - const IrValue* soffsetValue = inst.Argument(3)->Resolve(); - if (soffsetValue->HasImmediate() && soffsetValue->Type() == IrType::U32 && soffsetValue->ImmediateU32() == 0u) { - return address; - } - return Binary(state, spv::OpIAdd, TypeU32(state), address, soffset); -} - struct WordPair { std::uint32_t low = 0; std::uint32_t high = 0; @@ -84,8 +46,6 @@ std::uint32_t ScratchByteAddress(SpirvValueEmitContext& ctx, const MemoryInfo& m std::uint32_t ByteAddress(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { auto& state = ctx.state; switch (mem.kind) { - case ResourceKind::Buffer: - return BufferByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)); case ResourceKind::Lds: case ResourceKind::Gds: if (mem.kind == ResourceKind::Lds) { @@ -219,84 +179,6 @@ std::uint32_t DeviceAddressFromWords(SpirvEmitterState& state, std::uint32_t low return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64); } -std::pair RuntimeBufferByteAddress(SpirvEmitterState& state, std::uint32_t index, std::uint32_t offset, std::uint32_t soffset, std::uint32_t immediate, std::uint32_t stride, std::uint32_t swizzle, std::uint32_t indexStride) { - const auto u32 = TypeU32(state); - const auto add = [&](std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, spv::OpIAdd, u32, lhs, rhs); }; - const auto mul = [&](std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, spv::OpIMul, u32, lhs, rhs); }; - if (immediate != 0u) offset = add(offset, ConstantU32(state, immediate)); - const auto linear = add(mul(index, stride), offset); - const auto indexShift = add(indexStride, ConstantU32(state, 3u)); - const auto indices = Binary(state, spv::OpShiftLeftLogical, u32, ConstantU32(state, 1u), indexShift); - const auto indexMsb = Binary(state, spv::OpShiftRightLogical, u32, index, indexShift); - const auto indexLsb = Binary(state, spv::OpBitwiseAnd, u32, index, Binary(state, spv::OpISub, u32, indices, ConstantU32(state, 1u))); - const auto offsetMsb = Binary(state, spv::OpBitwiseAnd, u32, offset, ConstantU32(state, ~3u)); - const auto offsetLsb = Binary(state, spv::OpBitwiseAnd, u32, offset, ConstantU32(state, 3u)); - const auto swizzled = add(mul(add(mul(indexMsb, stride), offsetMsb), indices), add(Binary(state, spv::OpShiftLeftLogical, u32, indexLsb, ConstantU32(state, 2u)), offsetLsb)); - return {offset, add(Select(state, u32, swizzle, swizzled, linear), soffset)}; -} - -template -void ForEachGpuDescriptorDword(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components, TFunction&& function) { - auto& state = ctx.state; - const IrValue* handle = inst.Argument(0)->Resolve(); - if (handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) { - ctx.Fail(inst, "has no GPU-selected V#"); - } - const auto u32 = TypeU32(state); - const auto boolean = TypeBool(state); - const auto word1 = ctx.Arg(*handle, 1); - const auto records = ctx.Arg(*handle, 2); - const auto word3 = ctx.Arg(*handle, 3); - const auto field = [&](std::uint32_t word, std::uint32_t first, std::uint32_t count) { return EmitBitFieldUExtract(state, word, ConstantU32(state, first), ConstantU32(state, count)); }; - const auto nonzero = [&](std::uint32_t value) { return Binary(state, spv::OpINotEqual, boolean, value, ConstantU32(state, 0u)); }; - const auto hasDword = [&](std::uint32_t offset, std::uint32_t size) { - return AndCondition(state, Binary(state, spv::OpUGreaterThanEqual, boolean, size, ConstantU32(state, 4u)), Binary(state, spv::OpULessThanEqual, boolean, offset, Binary(state, spv::OpISub, u32, size, ConstantU32(state, 4u)))); - }; - const auto stride = field(word1, 16u, 14u); - const auto swizzle = AndCondition(state, nonzero(stride), nonzero(field(word1, 31u, 1u))); - const auto lane = Binary(state, spv::OpBitwiseAnd, u32, EmitSubgroupLocalInvocationId(state), ConstantU32(state, 63u)); - const auto index = Binary(state, spv::OpIAdd, u32, ctx.Arg(inst, 1), Select(state, u32, nonzero(field(word3, 23u, 1u)), lane, ConstantU32(state, 0u))); - const auto soffset = ctx.Arg(inst, 3); - const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), field(word1, 0u, 16u)); - const auto mode = field(word3, 28u, 2u); - const auto indexInBounds = Binary(state, spv::OpULessThan, boolean, index, records); - const auto scalarInBounds = Binary(state, spv::OpULessThanEqual, boolean, soffset, records); - const auto rawRecords = Binary(state, spv::OpISub, u32, records, soffset); - const auto rawIndexInBounds = Binary(state, spv::OpULessThan, boolean, index, rawRecords); - for (std::uint32_t component = 0; component < components; component++) { - const auto [offset, byte] = RuntimeBufferByteAddress(state, index, ctx.Arg(inst, 2), soffset, mem.offset + component * 4u, stride, swizzle, field(word3, 21u, 2u)); - const auto structured = AndCondition(state, indexInBounds, Binary(state, spv::OpULessThan, boolean, offset, stride)); - const auto raw = AndCondition(state, scalarInBounds, Select(state, boolean, swizzle, AndCondition(state, rawIndexInBounds, hasDword(offset, stride)), hasDword(offset, rawRecords))); - auto inBounds = Select(state, boolean, Binary(state, spv::OpIEqual, boolean, mode, ConstantU32(state, 0u)), structured, indexInBounds); - inBounds = Select(state, boolean, Binary(state, spv::OpULessThan, boolean, mode, ConstantU32(state, 2u)), inBounds, Select(state, boolean, Binary(state, spv::OpIEqual, boolean, mode, ConstantU32(state, 2u)), nonzero(records), raw)); - const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), base, Unary(state, spv::OpUConvert, TypeScalarU64(state), byte)); - function(component, guest, AndCondition(state, nonzero(field(word3, 12u, 7u)), inBounds)); - } -} - -std::uint32_t ConstructU32Composite(SpirvEmitterState& state, std::uint32_t components, const std::array& values); - -std::uint32_t LoadGpuDescriptor(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components) { - std::array values{}; - ForEachGpuDescriptorDword(ctx, inst, ctx.Memory(inst), components, [&](std::uint32_t component, std::uint32_t guest, std::uint32_t inBounds) { - values[component] = EmitValueOrZeroIfCondition(ctx.state, inBounds, [&] { return EmitBdaRead(ctx, inst, guest, 32u); }); - }); - return components == 1u ? values[0] : ConstructU32Composite(ctx.state, components, values); -} - -void StoreGpuDescriptor(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components) { - auto& state = ctx.state; - const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u); - ForEachGpuDescriptorDword(ctx, inst, ctx.Memory(inst), components, [&](std::uint32_t component, std::uint32_t guest, std::uint32_t inBounds) { - auto value = data; - if (components != 1u) { - value = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, data, component); - } - EmitIfCondition(state, inBounds, [&] { EmitBdaWrite(ctx, inst, guest, value); }); - }); -} - std::uint32_t GuestAddressBase(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { auto& state = ctx.state; auto low = ctx.Arg(inst, 1); @@ -330,92 +212,6 @@ std::uint32_t LoadBda(SpirvValueEmitContext& ctx, const IrValue& inst, const Mem }); } -IrBufferFormat BufferFormatOf(const SpirvEmitterState& state, const MemoryInfo& mem) { - return mem.typed ? DecodeTBufferFormat(mem.dataFormat, mem.numberFormat) : StorageBufferFormat(state, mem); -} - -SpirvBufferFormatInfo MemoryFormatInfo(const SpirvEmitterState& state, const MemoryInfo& mem) { - return GetFormatInfo(mem.formatted ? BufferFormatOf(state, mem) : IrBufferFormat::Invalid); -} - -MemoryInfo RebaseFormattedComponent(MemoryInfo mem, const SpirvBufferFormatInfo& info, std::uint32_t component) { - mem.offset += GetFormatComponentByteOffset(info, component); - mem.dataDwords = 1u; - mem.componentIndex = component; - return mem; -} - -MemoryInfo RebaseRawComponent(MemoryInfo mem, std::uint32_t component) { - mem.offset += component * 4u; - mem.dataDwords = 1u; - mem.componentIndex = component; - return mem; -} - -SpirvFormattedSource ResolveOutputSource(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const SpirvBufferFormatInfo& info, std::uint32_t outputComponent) { - if (mem.typed) { - if (outputComponent < info.componentCount) { - return {SpirvFormattedSourceKind::Memory, outputComponent}; - } - return {}; - } - const auto swizzle = ctx.state.program.Info().buffers.at(mem.resource).descriptorSwizzle; - return ResolveFormattedSource(info, (swizzle >> (outputComponent * 3u)) & 0x7u); -} - -std::uint32_t FormattedConstant(SpirvValueEmitContext& ctx, const SpirvBufferFormatInfo& info, SpirvFormattedSourceKind kind) { - return ConstantU32(ctx.state, FormattedConstantBits(info, kind)); -} - -template -std::uint32_t LoadFormattedComponent(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const SpirvBufferFormatInfo& info, std::uint32_t outputComponent, TLoadWord&& loadWord, TLoadSubword&& loadSubword) { - auto& state = ctx.state; - const auto source = ResolveOutputSource(ctx, mem, info, outputComponent); - if (source.kind != SpirvFormattedSourceKind::Memory) { - const bool integer = info.type == SpirvFormatComponentType::Uint || info.type == SpirvFormatComponentType::Sint; - if (mem.d16 && source.kind == SpirvFormattedSourceKind::One && !integer) return ConstantU32(state, 0x3c00u); - return FormattedConstant(ctx, info, source.kind); - } - const auto component = source.component; - const auto bits = info.componentBits[component]; - const bool signedType = IsSignedFormatComponent(info.type); - std::uint32_t raw = 0; - if (info.packedBitfield) { - raw = loadWord(component); - const auto type = signedType ? TypeI32(state) : TypeU32(state); - const auto sourceValue = signedType ? Unary(state, spv::OpBitcast, type, raw) : raw; - const auto extracted = state.module.AllocateId(); - state.module.AddFunction(signedType ? spv::OpBitFieldSExtract : spv::OpBitFieldUExtract, type, extracted, sourceValue, ConstantU32(state, info.componentBitOffset[component]), ConstantU32(state, bits)); - raw = signedType ? Unary(state, spv::OpBitcast, TypeU32(state), extracted) : extracted; - } else if (bits == 32u) { - raw = loadWord(component); - } else { - raw = loadSubword(component, bits, signedType); - } - return mem.d16 ? EmitD16FormatComponent(state, info, component, raw) : NormalizeFormatComponent(state, info, component, raw); -} - -std::uint32_t FormattedLoadPrepared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t outputComponent, const MemoryResourceAccess& resource) { - const auto info = MemoryFormatInfo(ctx.state, mem); - if (info.type == SpirvFormatComponentType::Unknown) { - if (mem.d16) ctx.Fail(inst, "is a D16 format load with an unknown buffer format"); - return LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, outputComponent), resource); - } - RequireAlignedFormattedElement(ctx, inst, resource, info); - return LoadFormattedComponent(ctx, mem, info, outputComponent, [&](std::uint32_t component) { - return LoadWordPrepared(ctx, inst, RebaseFormattedComponent(mem, info, component), resource); - }, [&](std::uint32_t component, std::uint32_t bits, bool signExtend) { - return LoadSubwordPrepared(ctx, inst, RebaseFormattedComponent(mem, info, component), resource, bits, signExtend); - }); -} - -std::uint32_t FormattedLoad(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { - return EmitValueOrZeroIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - return FormattedLoadPrepared(ctx, inst, mem, 0u, resource); - }); -} - void StoreWordInBounds(SpirvValueEmitContext& ctx, const MemoryResourceAccess& resource, std::uint32_t index, std::uint32_t data) { auto& state = ctx.state; if (resource.misalignment != 0u) { @@ -486,82 +282,6 @@ void StoreWord(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo }); } -struct PreparedFormattedMemory { - SpirvBufferFormatInfo info; - MemoryResourceAccess resource; - std::array addresses{}; - std::array indices{}; - std::uint32_t inBounds = 0; -}; - -enum class FormattedAccess { - Load, - Store -}; - -PreparedFormattedMemory PrepareFormattedMemory(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource, const SpirvBufferFormatInfo& info, std::uint32_t components, FormattedAccess access) { - auto& state = ctx.state; - RequireAlignedFormattedElement(ctx, inst, resource, info); - PreparedFormattedMemory plan; - plan.info = info; - plan.resource = resource; - std::array required{}; - if (access == FormattedAccess::Load) { - for (std::uint32_t output = 0; output < components; output++) { - const auto source = ResolveOutputSource(ctx, mem, plan.info, output); - if (source.kind == SpirvFormattedSourceKind::Memory) { - required.at(source.component) = true; - } - } - } else { - for (std::uint32_t component = 0; component < plan.info.componentCount; component++) { - required.at(component) = true; - } - } - bool firstBound = true; - for (std::uint32_t component = 0; component < plan.info.componentCount; component++) { - if (!required.at(component)) { - continue; - } - const auto byteOffset = GetFormatComponentByteOffset(plan.info, component); - bool reused = false; - for (std::uint32_t previous = 0; previous < component; previous++) { - if (required.at(previous) && GetFormatComponentByteOffset(plan.info, previous) == byteOffset) { - plan.addresses.at(component) = plan.addresses.at(previous); - plan.indices.at(component) = plan.indices.at(previous); - reused = true; - break; - } - } - if (reused) { - continue; - } - const auto componentMem = RebaseFormattedComponent(mem, plan.info, component); - plan.addresses.at(component) = ByteAddress(ctx, inst, componentMem); - const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), plan.addresses.at(component), ConstantU32(state, 2u)); - plan.indices.at(component) = EmitMemoryElementIndex(state, resource, rawIndex); - const auto componentBound = EmitMemoryElementInBounds(state, resource, plan.indices.at(component)); - if (firstBound) { - plan.inBounds = componentBound; - firstBound = false; - } else { - plan.inBounds = AndCondition(state, plan.inBounds, componentBound); - } - } - if (firstBound) { - plan.inBounds = ConstantBool(state, true); - } - return plan; -} - -std::uint32_t LoadFormattedInBounds(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, std::uint32_t outputComponent) { - return LoadFormattedComponent(ctx, mem, plan.info, outputComponent, [&](std::uint32_t component) { - return LoadWordInBounds(ctx, plan.resource, plan.indices.at(component)); - }, [&](std::uint32_t component, std::uint32_t bits, bool signExtend) { - return LoadSubwordInBounds(ctx, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, signExtend); - }); -} - std::uint32_t ConstructU32Composite(SpirvEmitterState& state, std::uint32_t components, const std::array& values) { const auto result = state.module.AllocateId(); std::vector words{spv::OpCompositeConstruct, TypeU32Composite(state, components), result}; @@ -570,135 +290,6 @@ std::uint32_t ConstructU32Composite(SpirvEmitterState& state, std::uint32_t comp return result; } -std::uint32_t FormattedOutOfBoundsValue(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, std::uint32_t components) { - auto& state = ctx.state; - std::array values{}; - for (std::uint32_t component = 0; component < components; component++) { - const auto source = ResolveOutputSource(ctx, mem, plan.info, component); - values.at(component) = source.kind == SpirvFormattedSourceKind::Memory ? ConstantU32(state, 0u) : FormattedConstant(ctx, plan.info, source.kind); - } - return ConstructU32Composite(state, components, values); -} - -void StoreFormattedElement(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, const std::array& data, std::uint32_t components) { - auto& state = ctx.state; - const auto& info = plan.info; - std::array encoded{}; - for (std::uint32_t component = 0; component < info.componentCount; component++) { - encoded.at(component) = component >= components ? ConstantU32(state, 0u) : mem.d16 ? EmitD16StoreComponent(state, info, component, data.at(component)) : EmitFormatStoreComponent(state, info, component, data.at(component)); - } - if (info.packedBitfield) { - auto word = ConstantU32(state, 0u); - for (std::uint32_t component = 0; component < info.componentCount; component++) { - const std::uint32_t mask = (1u << info.componentBits[component]) - 1u; - word = EmitOrU32(state, word, EmitBinaryU32(state, spv::OpShiftLeftLogical, EmitAndConstant(state, encoded.at(component), mask), ConstantU32(state, info.componentBitOffset[component]))); - } - StoreWordInBounds(ctx, plan.resource, plan.indices.at(0), word); - return; - } - for (std::uint32_t component = 0; component < info.componentCount; component++) { - const auto bits = info.componentBits[component]; - if (bits == 8u || bits == 16u) { - if (plan.resource.misalignment != 0u) StoreMisalignedSubword(ctx, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, encoded.at(component)); - else StoreSubwordInBounds(ctx, mem, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, encoded.at(component)); - } else { - StoreWordInBounds(ctx, plan.resource, plan.indices.at(component), encoded.at(component)); - } - } -} - -void StoreFormattedValues(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource, const SpirvBufferFormatInfo& info, const std::array& data, std::uint32_t components) { - const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, FormattedAccess::Store); - EmitIfCondition(ctx.state, plan.inBounds, [&]() { - StoreFormattedElement(ctx, mem, plan, data, components); - }); -} - -void FormattedStore(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { - EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - const auto info = MemoryFormatInfo(ctx.state, mem); - const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u); - if (info.type == SpirvFormatComponentType::Unknown) { - if (mem.d16) ctx.Fail(inst, "is a D16 format store with an unknown buffer format"); - StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, 0u), resource, data); - return; - } - StoreFormattedValues(ctx, inst, mem, resource, info, {data, 0u, 0u, 0u}, 1u); - }); -} - -std::uint32_t LoadCoherentPair(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource) { - auto& state = ctx.state; - const auto wide = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferU64Variable, TypeStorageBufferU64Pointer(state)); - const auto byteAddress = Binary(state, spv::OpIAdd, TypeU32(state), ByteAddress(ctx, inst, mem), wide.byteOffset); - const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteAddress, ConstantU32(state, 3u)); - const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), byteAddress, ConstantU32(state, 7u)), ConstantU32(state, 0u)); - return EmitValueIfElse(state, AndCondition(state, aligned, EmitMemoryElementInBounds(state, wide, index)), TypeU32Composite(state, 2u), [&]() { - const auto pointer = EmitStorageBufferElementPointer(state, wide, index, TypeStorageBufferU64ElementPointer(state)); - const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpAtomicLoad, TypeScalarU64(state), value, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone)); - const auto high = Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), value, BdaConstant(state, 32u)); - return ConstructU32Composite(state, 2u, {Unary(state, spv::OpUConvert, TypeU32(state), value), Unary(state, spv::OpUConvert, TypeU32(state), high), 0u, 0u}); - }, [&]() { - std::array values{}; - for (std::uint32_t component = 0; component < 2u; component++) { - values.at(component) = LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource); - } - return ConstructU32Composite(state, 2u, values); - }); -} - -std::uint32_t LoadWideBuffer(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) { - auto& state = ctx.state; - return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() { - const auto resource = PrepareMemoryResourceAccess(state, mem); - const auto info = MemoryFormatInfo(state, mem); - if (info.type == SpirvFormatComponentType::Unknown && components == 2u && mem.coherent && state.storageBufferU64Variable != 0u && resource.misalignment == 0u) { - return LoadCoherentPair(ctx, inst, mem, resource); - } - if (info.type != SpirvFormatComponentType::Unknown) { - const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, FormattedAccess::Load); - return EmitValueOrDefaultIfCondition(state, plan.inBounds, TypeU32Composite(state, components), FormattedOutOfBoundsValue(ctx, mem, plan, components), [&]() { - std::array values{}; - for (std::uint32_t component = 0; component < components; component++) { - values.at(component) = LoadFormattedInBounds(ctx, mem, plan, component); - } - return ConstructU32Composite(state, components, values); - }); - } - std::array values{}; - for (std::uint32_t component = 0; component < components; component++) { - values.at(component) = LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource); - } - return ConstructU32Composite(state, components, values); - }); -} - -void StoreWideBuffer(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) { - auto& state = ctx.state; - EmitIfCondition(state, ActiveArgument(ctx, inst), [&]() { - const auto resource = PrepareMemoryResourceAccess(state, mem); - const auto composite = ctx.Arg(inst, inst.ArgumentCount() - 2u); - const auto info = MemoryFormatInfo(state, mem); - if (info.type != SpirvFormatComponentType::Unknown) { - std::array data{}; - for (std::uint32_t component = 0; component < components; component++) { - data.at(component) = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), data.at(component), composite, component); - } - StoreFormattedValues(ctx, inst, mem, resource, info, data, components); - return; - } - if (mem.d16) ctx.Fail(inst, "is a D16 format store with an unknown buffer format"); - for (std::uint32_t component = 0; component < components; component++) { - const auto data = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), data, composite, component); - StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource, data); - } - }); -} - std::uint32_t LoadWideShared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) { auto& state = ctx.state; return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() { @@ -738,27 +329,10 @@ const MemoryInfo& BufferMemory(SpirvValueEmitContext& ctx, const IrValue& inst) if (mem.kind != ResourceKind::Buffer) { ctx.Fail(inst, "must access a buffer resource"); } - if (mem.gpuDescriptor) { - ctx.Fail(inst, "accesses a GPU-selected V# in a way only raw dword loads and stores support"); - } + if (!mem.gpuDescriptor) ctx.Fail(inst, "buffer operation has no runtime V#"); return mem; } -bool EmitGpuDescriptorAccess(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components, bool store) { - if (!ctx.Memory(inst).gpuDescriptor) { - return false; - } - auto& state = ctx.state; - if (store) { - EmitIfCondition(state, ActiveArgument(ctx, inst), [&] { StoreGpuDescriptor(ctx, inst, components); }); - return true; - } - const auto type = components == 1u ? TypeU32(state) : TypeU32Composite(state, components); - const auto zero = components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); - ctx.Define(inst, EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), type, zero, [&] { return LoadGpuDescriptor(ctx, inst, components); })); - return true; -} - const MemoryInfo& SharedMemory(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto& mem = ctx.Memory(inst); if (mem.kind != ResourceKind::Lds && mem.kind != ResourceKind::Gds) { @@ -905,6 +479,7 @@ std::uint32_t EmitAtomicOperation(SpirvValueEmitContext& ctx, const IrValue& ins template std::uint32_t EmitAtomicAccess(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, TOperation&& operation) { + if (mem.gpuDescriptor) return EmitRuntimeBufferAtomic(ctx, inst, 32u, ActiveArgument(ctx, inst), operation); auto& state = ctx.state; return EmitValueOrZeroIfCondition(state, ActiveArgument(ctx, inst), [&]() { const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); @@ -988,53 +563,46 @@ std::uint32_t Atomic32(SpirvValueEmitContext& ctx, const IrValue& inst, const Me if (zeroSkip) active = AndCondition(state, active, nonZero); } if (mem.kind == ResourceKind::Buffer) NoteBufferAtomicSite(zeroSkip, zeroIdentity && !zeroSkip); - return EmitValueOrZeroIfCondition(state, active, [&]() { - const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); - return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access.resource, access.index), [&]() { - const auto pointer = EmitMemoryElementPointer(state, access.resource, access.index); - const auto operation = [&]() { - const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); - if (lds) { - const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; - state.module.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)); - } else { - EmitDeviceAtomicMemoryBarrier(state); - } - return old; - }; - if (!zeroIdentity || zeroSkip) return operation(); - return EmitValueIfElse(state, nonZero, TypeU32(state), operation, [&]() { - // The barrier stays so an "add 0" used as a coherent read keeps its acquire. - const auto current = state.module.AllocateId(); - state.module.AddFunction(spv::OpAtomicLoad, TypeU32(state), current, pointer, ConstantU32(state, scope), ConstantU32(state, spv::MemorySemanticsMaskNone)); + const auto apply = [&](std::uint32_t pointer) { + const auto operation = [&]() { + const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); + if (lds) { + const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; + state.module.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)); + } else { EmitDeviceAtomicMemoryBarrier(state); - return current; - }); + } + return old; + }; + if (!zeroIdentity || zeroSkip) return operation(); + return EmitValueIfElse(state, nonZero, TypeU32(state), operation, [&]() { + const auto current = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicLoad, TypeU32(state), current, pointer, ConstantU32(state, scope), ConstantU32(state, spv::MemorySemanticsMaskNone)); + EmitDeviceAtomicMemoryBarrier(state); + return current; }); + }; + if (mem.gpuDescriptor) return EmitRuntimeBufferAtomic(ctx, inst, 32u, active, apply); + return EmitValueOrZeroIfCondition(state, active, [&] { + const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access.resource, access.index), [&] { return apply(EmitMemoryElementPointer(state, access.resource, access.index)); }); }); } std::uint32_t BufferAtomic64(SpirvValueEmitContext& ctx, const IrValue& inst) { auto& state = ctx.state; - const auto& mem = BufferMemory(ctx, inst); - return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU64(state), ConstantU64(state, 0u), [&]() { - const auto resource = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferU64Variable, TypeStorageBufferU64Pointer(state)); - RequireAlignedAtomic(ctx, resource); - const auto byteAddress = Binary(state, spv::OpIAdd, TypeU32(state), ByteAddress(ctx, inst, mem), resource.byteOffset); - const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteAddress, ConstantU32(state, 3u)); - return EmitValueOrDefaultIfCondition(state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state), ConstantU64(state, 0u), [&]() { - const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 2u)); - const auto old = state.module.AllocateId(); - const auto pointer = EmitStorageBufferElementPointer(state, resource, index, TypeStorageBufferU64ElementPointer(state)); - if (inst.Opcode() == IrOpcode::BufferAtomicCmpSwap64) { - const auto desired = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 3u)); - state.module.AddFunction(spv::OpAtomicCompareExchange, TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), ConstantU32(state, spv::MemorySemanticsMaskNone), desired, value); - } else { - state.module.AddFunction(SpirvAtomicOpcode(inst.Opcode()), TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), value); - } - EmitDeviceAtomicMemoryBarrier(state); - return Unary(state, spv::OpBitcast, TypeU64(state), old); - }); + static_cast(BufferMemory(ctx, inst)); + return EmitRuntimeBufferAtomic(ctx, inst, 64u, ActiveArgument(ctx, inst), [&](std::uint32_t pointer) { + const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 2u)); + const auto old = state.module.AllocateId(); + if (inst.Opcode() == IrOpcode::BufferAtomicCmpSwap64) { + const auto desired = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 3u)); + state.module.AddFunction(spv::OpAtomicCompareExchange, TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), ConstantU32(state, spv::MemorySemanticsMaskNone), desired, value); + } else { + state.module.AddFunction(SpirvAtomicOpcode(inst.Opcode()), TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), value); + } + EmitDeviceAtomicMemoryBarrier(state); + return Unary(state, spv::OpBitcast, TypeU64(state), old); }); } @@ -1095,19 +663,10 @@ std::uint32_t BufferFloatAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, template std::uint32_t BufferAtomic64Update(SpirvValueEmitContext& ctx, const IrValue& inst, TReplacement&& replacement) { auto& state = ctx.state; - const auto& mem = BufferMemory(ctx, inst); - return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU64(state), ConstantU64(state, 0u), [&]() { - const auto resource = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferU64Variable, TypeStorageBufferU64Pointer(state)); - RequireAlignedAtomic(ctx, resource); - const auto byteAddress = Binary(state, spv::OpIAdd, TypeU32(state), ByteAddress(ctx, inst, mem), resource.byteOffset); - const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteAddress, ConstantU32(state, 3u)); - return EmitValueOrDefaultIfCondition(state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state), ConstantU64(state, 0u), [&]() { - const auto pointer = EmitStorageBufferElementPointer(state, resource, index, TypeStorageBufferU64ElementPointer(state)); - const auto old = AtomicUpdateTyped(state, pointer, ResourceKind::Buffer, TypeScalarU64(state), [&](std::uint32_t observed) { - return replacement(observed); - }); - return Unary(state, spv::OpBitcast, TypeU64(state), old); - }); + static_cast(BufferMemory(ctx, inst)); + return EmitRuntimeBufferAtomic(ctx, inst, 64u, ActiveArgument(ctx, inst), [&](std::uint32_t pointer) { + const auto old = AtomicUpdateTyped(state, pointer, ResourceKind::Buffer, TypeScalarU64(state), [&](std::uint32_t observed) { return replacement(observed); }); + return Unary(state, spv::OpBitcast, TypeU64(state), old); }); } @@ -1475,41 +1034,9 @@ std::uint32_t EmitReadConst(SpirvValueEmitContext& ctx, const IrValue& inst) { void EmitReadConstBuffer(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto& mem = ctx.Memory(inst); - if (mem.planningOnly) { - return; - } - if (mem.kind != ResourceKind::ScalarBuffer) { - ctx.Fail(inst, "must read a scalar buffer resource"); - } - auto& state = ctx.state; - const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1), ConstantU32(state, mem.offset)); - const auto noCarry = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address, ctx.Arg(inst, 1)); - if (mem.gpuDescriptor) { - const IrValue* handle = inst.Argument(0)->Resolve(); - if (handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) { - ctx.Fail(inst, "has no GPU-selected V#"); - } - const auto u32 = TypeU32(state); - const auto word1 = ctx.Arg(*handle, 1); - const auto records = ctx.Arg(*handle, 2); - const auto stride = EmitBitFieldUExtract(state, word1, ConstantU32(state, 16u), ConstantU32(state, 14u)); - const auto size = Select(state, u32, Binary(state, spv::OpIEqual, TypeBool(state), stride, ConstantU32(state, 0u)), records, Binary(state, spv::OpIMul, u32, stride, records)); - const auto byte = Binary(state, spv::OpBitwiseAnd, u32, address, ConstantU32(state, ~3u)); - const auto inBounds = AndCondition(state, noCarry, Binary(state, spv::OpULessThan, TypeBool(state), byte, size)); - const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), EmitBitFieldUExtract(state, word1, ConstantU32(state, 0u), ConstantU32(state, 16u))); - const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), base, Unary(state, spv::OpUConvert, TypeScalarU64(state), byte)); - ctx.Define(inst, EmitValueOrZeroIfCondition(state, inBounds, [&] { return EmitBdaRead(ctx, inst, guest, 32u); })); - return; - } - const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); - const auto access = PrepareMemoryResourceAccess(state, mem); - const auto element = EmitMemoryElementIndex(state, access, rawIndex); - const auto inBounds = AndCondition(state, noCarry, EmitMemoryElementInBounds(state, access, element)); - ctx.Define(inst, EmitValueOrZeroIfCondition(state, inBounds, [&]() { - const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, TypeU32(state), value, EmitMemoryElementPointer(state, access, element)); - return value; - })); + if (mem.planningOnly) return; + if (mem.kind != ResourceKind::ScalarBuffer) ctx.Fail(inst, "must read a scalar buffer resource"); + ctx.Define(inst, EmitRuntimeScalarBufferLoad(ctx, inst)); } void EmitGetSrtResource(SpirvValueEmitContext& context) { @@ -1569,65 +1096,51 @@ std::uint32_t EmitAddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst) } void EmitLoadBufferU8(SpirvValueEmitContext& ctx, const IrValue& inst) { - ctx.Define(inst, LoadSubword(ctx, inst, BufferMemory(ctx, inst), 8u)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 1u)); } void EmitLoadBufferU16(SpirvValueEmitContext& ctx, const IrValue& inst) { - ctx.Define(inst, LoadSubword(ctx, inst, BufferMemory(ctx, inst), 16u)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 1u)); } void EmitLoadBufferU32(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 1u, false)) return; - const auto& mem = BufferMemory(ctx, inst); - ctx.Define(inst, mem.formatted ? FormattedLoad(ctx, inst, mem) : LoadWord(ctx, inst, mem)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 1u)); } void EmitLoadBufferU32x2(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 2u, false)) return; - ctx.Define(inst, LoadWideBuffer(ctx, inst, BufferMemory(ctx, inst), 2u)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 2u)); } void EmitLoadBufferU32x3(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 3u, false)) return; - ctx.Define(inst, LoadWideBuffer(ctx, inst, BufferMemory(ctx, inst), 3u)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 3u)); } void EmitLoadBufferU32x4(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 4u, false)) return; - ctx.Define(inst, LoadWideBuffer(ctx, inst, BufferMemory(ctx, inst), 4u)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 4u)); } void EmitStoreBufferU8(SpirvValueEmitContext& ctx, const IrValue& inst) { - StoreSubword(ctx, inst, BufferMemory(ctx, inst), 8u); + EmitRuntimeBufferStore(ctx, inst, 1u); } void EmitStoreBufferU16(SpirvValueEmitContext& ctx, const IrValue& inst) { - StoreSubword(ctx, inst, BufferMemory(ctx, inst), 16u); + EmitRuntimeBufferStore(ctx, inst, 1u); } void EmitStoreBufferU32(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 1u, true)) return; - const auto& mem = BufferMemory(ctx, inst); - if (mem.formatted) { - FormattedStore(ctx, inst, mem); - } else { - StoreWord(ctx, inst, mem); - } + EmitRuntimeBufferStore(ctx, inst, 1u); } void EmitStoreBufferU32x2(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 2u, true)) return; - StoreWideBuffer(ctx, inst, BufferMemory(ctx, inst), 2u); + EmitRuntimeBufferStore(ctx, inst, 2u); } void EmitStoreBufferU32x3(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 3u, true)) return; - StoreWideBuffer(ctx, inst, BufferMemory(ctx, inst), 3u); + EmitRuntimeBufferStore(ctx, inst, 3u); } void EmitStoreBufferU32x4(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 4u, true)) return; - StoreWideBuffer(ctx, inst, BufferMemory(ctx, inst), 4u); + EmitRuntimeBufferStore(ctx, inst, 4u); } std::uint32_t EmitBufferAtomicSwap32(SpirvValueEmitContext& ctx, const IrValue& inst) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp new file mode 100644 index 000000000..78fe0c6a4 --- /dev/null +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -0,0 +1,339 @@ +#include "SpirvBackend/SpirvRuntimeBuffer.hpp" +#include "SpirvBackend/SpirvEmitterInstructions.hpp" +#include "SpirvBackend/SpirvBda.hpp" +#include "SpirvBackend/SpirvBufferFormat.hpp" +#include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp" +#include +#include + +namespace ShaderRecompiler { +namespace { + +struct RuntimeDescriptor { + std::uint32_t base; + std::uint32_t records; + std::uint32_t stride; + std::uint32_t word3; + std::uint32_t swizzle; + std::uint32_t index; + std::uint32_t valid; +}; + +struct RuntimeAddress { + std::uint32_t guest; + std::uint32_t inBounds; +}; + +std::uint32_t field(SpirvEmitterState& state, std::uint32_t word, std::uint32_t offset, std::uint32_t count) { + return EmitBitFieldUExtract(state, word, ConstantU32(state, offset), ConstantU32(state, count)); +} + +std::uint32_t both(SpirvEmitterState& state, std::uint32_t left, std::uint32_t right) { + return Binary(state, spv::OpLogicalAnd, TypeBool(state), left, right); +} + +std::uint32_t equal(SpirvEmitterState& state, std::uint32_t left, std::uint32_t right) { + return Binary(state, spv::OpIEqual, TypeBool(state), left, right); +} + +std::uint32_t nonzero(SpirvEmitterState& state, std::uint32_t value) { + return Binary(state, spv::OpINotEqual, TypeBool(state), value, ConstantU32(state, 0u)); +} + +void faultIf(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, BdaAbi::FaultReason reason) { + auto& state = context.state; + EmitIfCondition(state, condition, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), ConstantU32(state, instruction.Flags().pc), reason); }); + StopBdaInvocationIf(state, condition); +} + +RuntimeDescriptor descriptor(SpirvValueEmitContext& context, const IrValue& instruction) { + auto& state = context.state; + const auto* handle = instruction.Argument(0)->Resolve(); + if (!context.Memory(instruction).gpuDescriptor || handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) context.Fail(instruction, "requires a runtime four-dword V#"); + const auto word1 = context.Arg(*handle, 1u); + const auto word3 = context.Arg(*handle, 3u); + const auto u32 = TypeU32(state); + const auto u64 = TypeScalarU64(state); + const auto high = Binary(state, spv::OpShiftLeftLogical, u64, Unary(state, spv::OpUConvert, u64, field(state, word1, 0u, 16u)), BdaConstant(state, 32u)); + const auto base = Binary(state, spv::OpBitwiseOr, u64, Unary(state, spv::OpUConvert, u64, context.Arg(*handle, 0u)), high); + auto invalid = nonzero(state, field(state, word3, 30u, 2u)); + auto index = ConstantU32(state, 0u); + if (context.Memory(instruction).kind == ResourceKind::Buffer) { + index = context.Arg(instruction, 1u); + const auto addTid = nonzero(state, field(state, word3, 23u, 1u)); + if (state.program.Resources().stage == IrShaderStage::Compute) { + index = Binary(state, spv::OpIAdd, u32, index, Select(state, u32, addTid, EmitSubgroupLocalInvocationId(state), ConstantU32(state, 0u))); + } else { + invalid = Binary(state, spv::OpLogicalOr, TypeBool(state), invalid, addTid); + } + } + faultIf(context, instruction, invalid, base, 0u, BdaAbi::FaultReason::InvalidDescriptor); + const auto stride = field(state, word1, 16u, 14u); + const auto valid = both(state, Unary(state, spv::OpLogicalNot, TypeBool(state), invalid), Binary(state, spv::OpINotEqual, TypeBool(state), base, BdaConstant(state, 0u))); + return {base, context.Arg(*handle, 2u), stride, word3, both(state, nonzero(state, stride), nonzero(state, field(state, word1, 31u, 1u))), index, valid}; +} + +RuntimeAddress address(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t componentOffset, std::uint32_t bytes) { + auto& state = context.state; + const auto u32 = TypeU32(state); + const auto boolean = TypeBool(state); + const auto constant = [&](std::uint32_t value) { return ConstantU32(state, value); }; + const auto add = [&](std::uint32_t left, std::uint32_t right) { return Binary(state, spv::OpIAdd, u32, left, right); }; + const auto mul = [&](std::uint32_t left, std::uint32_t right) { return Binary(state, spv::OpIMul, u32, left, right); }; + const auto less = [&](std::uint32_t left, std::uint32_t right) { return Binary(state, spv::OpULessThan, boolean, left, right); }; + const auto fits = [&](std::uint32_t offset, std::uint32_t size) { + return both(state, Binary(state, spv::OpUGreaterThanEqual, boolean, size, constant(bytes)), Binary(state, spv::OpULessThanEqual, boolean, offset, Binary(state, spv::OpISub, u32, size, constant(bytes)))); + }; + const auto offset = add(context.Arg(instruction, 2u), constant(context.Memory(instruction).offset + componentOffset)); + const auto soffset = context.Arg(instruction, 3u); + const auto indexShift = add(field(state, resource.word3, 21u, 2u), constant(3u)); + const auto indices = Binary(state, spv::OpShiftLeftLogical, u32, constant(1u), indexShift); + const auto indexMsb = Binary(state, spv::OpShiftRightLogical, u32, resource.index, indexShift); + const auto indexLsb = Binary(state, spv::OpBitwiseAnd, u32, resource.index, Binary(state, spv::OpISub, u32, indices, constant(1u))); + const auto offsetMsb = Binary(state, spv::OpBitwiseAnd, u32, offset, constant(~3u)); + const auto offsetLsb = Binary(state, spv::OpBitwiseAnd, u32, offset, constant(3u)); + const auto swizzled = add(mul(add(mul(indexMsb, resource.stride), offsetMsb), indices), add(Binary(state, spv::OpShiftLeftLogical, u32, indexLsb, constant(2u)), offsetLsb)); + const auto linear = add(mul(resource.index, resource.stride), offset); + const auto byte = add(Select(state, u32, resource.swizzle, swizzled, linear), soffset); + const auto mode = field(state, resource.word3, 28u, 2u); + const auto indexInBounds = less(resource.index, resource.records); + const auto structured = both(state, indexInBounds, less(offset, resource.stride)); + const auto rawRecords = Binary(state, spv::OpISub, u32, resource.records, soffset); + const auto raw = both(state, Binary(state, spv::OpULessThanEqual, boolean, soffset, resource.records), Select(state, boolean, resource.swizzle, both(state, less(resource.index, rawRecords), fits(offset, resource.stride)), fits(offset, rawRecords))); + auto inBounds = Select(state, boolean, equal(state, mode, constant(0u)), structured, indexInBounds); + inBounds = Select(state, boolean, less(mode, constant(2u)), inBounds, Select(state, boolean, equal(state, mode, constant(2u)), nonzero(state, resource.records), raw)); + inBounds = both(state, resource.valid, both(state, nonzero(state, field(state, resource.word3, 12u, 7u)), inBounds)); + return {AddBdaAddress(context, instruction, resource.base, Unary(state, spv::OpUConvert, TypeScalarU64(state), byte), false), inBounds}; +} + +std::uint32_t composite(SpirvEmitterState& state, std::uint32_t components, const std::array& values) { + if (components == 1u) return values[0]; + const auto result = state.module.AllocateId(); + std::vector words{spv::OpCompositeConstruct, TypeU32Composite(state, components), result}; + words.insert(words.end(), values.begin(), values.begin() + components); + state.module.AddFunction(words); + return result; +} + +std::array storeValues(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) { + auto& state = context.state; + std::array result{}; + const auto data = context.Arg(instruction, instruction.ArgumentCount() - 2u); + for (std::uint32_t component = 0; component < components; ++component) { + result[component] = data; + if (components != 1u) { + result[component] = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), result[component], data, component); + } + } + return result; +} + +std::uint32_t formatted(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, const SpirvBufferFormatInfo& info, std::uint32_t components, bool store) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + const auto u32 = TypeU32(state); + std::array values{}; + std::array selectors{}; + std::array selected{}; + std::array addresses{}; + auto inBounds = ConstantBool(state, true); + for (std::uint32_t output = 0; output < components; ++output) { + selectors[output] = memory.typed ? ConstantU32(state, output < info.componentCount ? 4u + output : 0u) : field(state, resource.word3, output * 3u, 3u); + selected[output] = Binary(state, spv::OpUMod, u32, Binary(state, spv::OpISub, u32, selectors[output], ConstantU32(state, 4u)), ConstantU32(state, info.componentCount)); + } + std::array required{}; + for (std::uint32_t component = 0; component < info.componentCount; ++component) { + required[component] = ConstantBool(state, store); + if (!store) { + for (std::uint32_t output = 0; output < components; ++output) { + const auto fromMemory = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), selectors[output], ConstantU32(state, 4u)); + required[component] = Binary(state, spv::OpLogicalOr, TypeBool(state), required[component], both(state, fromMemory, equal(state, selected[output], ConstantU32(state, component)))); + } + } + addresses[component] = address(context, instruction, resource, GetFormatComponentByteOffset(info, component), info.packedBitfield ? 4u : info.componentBits[component] / 8u); + inBounds = both(state, inBounds, Binary(state, spv::OpLogicalOr, TypeBool(state), Unary(state, spv::OpLogicalNot, TypeBool(state), required[component]), addresses[component].inBounds)); + } + if (store) { + const auto data = storeValues(context, instruction, components); + EmitIfCondition(state, inBounds, [&] { + auto packed = ConstantU32(state, 0u); + for (std::uint32_t component = 0; component < info.componentCount; ++component) { + const auto encoded = component >= components ? ConstantU32(state, 0u) : memory.d16 ? EmitD16StoreComponent(state, info, component, data[component]) : EmitFormatStoreComponent(state, info, component, data[component]); + if (info.packedBitfield) { + packed = EmitOrU32(state, packed, Binary(state, spv::OpShiftLeftLogical, u32, EmitAndConstant(state, encoded, (1u << info.componentBits[component]) - 1u), ConstantU32(state, info.componentBitOffset[component]))); + } else { + EmitBdaWrite(context, instruction, addresses[component].guest, encoded, info.componentBits[component]); + } + } + if (info.packedBitfield) EmitBdaWrite(context, instruction, addresses[0].guest, packed); + }); + return ConstantU32(state, 0u); + } + for (std::uint32_t component = 0; component < info.componentCount; ++component) { + values[component] = EmitValueOrZeroIfCondition(state, both(state, inBounds, required[component]), [&] { + const auto bits = info.componentBits[component]; + auto raw = EmitBdaRead(context, instruction, addresses[component].guest, info.packedBitfield ? 32u : bits); + if (bits != 32u) { + const auto signedType = IsSignedFormatComponent(info.type); + const auto type = signedType ? TypeI32(state) : u32; + const auto input = signedType ? Unary(state, spv::OpBitcast, type, raw) : raw; + raw = state.module.AllocateId(); + state.module.AddFunction(signedType ? spv::OpBitFieldSExtract : spv::OpBitFieldUExtract, type, raw, input, ConstantU32(state, info.packedBitfield ? info.componentBitOffset[component] : 0u), ConstantU32(state, bits)); + if (signedType) raw = Unary(state, spv::OpBitcast, u32, raw); + } + return memory.d16 ? EmitD16FormatComponent(state, info, component, raw) : NormalizeFormatComponent(state, info, component, raw); + }); + } + const bool integer = info.type == SpirvFormatComponentType::Uint || info.type == SpirvFormatComponentType::Sint; + const auto one = ConstantU32(state, integer ? 1u : memory.d16 ? 0x3c00u : 0x3f800000u); + std::array outputs{}; + for (std::uint32_t output = 0; output < components; ++output) { + auto value = values[0]; + for (std::uint32_t component = 1; component < info.componentCount; ++component) value = Select(state, u32, equal(state, selected[output], ConstantU32(state, component)), values[component], value); + value = Select(state, u32, equal(state, selectors[output], ConstantU32(state, 1u)), one, value); + outputs[output] = Select(state, u32, equal(state, selectors[output], ConstantU32(state, 0u)), ConstantU32(state, 0u), value); + } + return composite(state, components, outputs); +} + +std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + if (memory.typed) { + const auto info = GetFormatInfo(DecodeTBufferFormat(memory.dataFormat, memory.numberFormat)); + if (info.type == SpirvFormatComponentType::Unknown) context.Fail(instruction, "typed buffer instruction has an invalid format"); + return formatted(context, instruction, resource, info, components, store); + } + if (!store) { + auto invalid = ConstantBool(state, false); + for (std::uint32_t component = 0; component < components; ++component) { + const auto selector = field(state, resource.word3, component * 3u, 3u); + invalid = Binary(state, spv::OpLogicalOr, TypeBool(state), invalid, Binary(state, spv::OpLogicalOr, TypeBool(state), equal(state, selector, ConstantU32(state, 2u)), equal(state, selector, ConstantU32(state, 3u)))); + } + faultIf(context, instruction, invalid, resource.base, 0u, BdaAbi::FaultReason::InvalidDescriptor); + } + const auto* word = instruction.Argument(0)->Resolve()->Argument(3)->Resolve(); + if (word->HasImmediate()) { + const auto format = static_cast((word->ImmediateU32() >> 12u) & 0x7fu); + if (format == IrBufferFormat::Invalid) context.Fail(instruction, "formatted buffer instruction has an invalid constant format"); + return formatted(context, instruction, resource, GetFormatInfo(format), components, store); + } + const auto format = field(state, resource.word3, 12u, 7u); + const auto merge = state.module.AllocateId(); + const auto invalid = state.module.AllocateId(); + constexpr std::uint32_t count = static_cast(IrBufferFormat::Format32_32_32_32Float); + std::array labels{}; + std::vector branch{spv::OpSwitch, format, invalid}; + for (std::uint32_t index = 0; index < count; ++index) { + labels[index] = state.module.AllocateId(); + branch.insert(branch.end(), {index + 1u, labels[index]}); + } + const auto type = store || components == 1u ? TypeU32(state) : TypeU32Composite(state, components); + const auto result = state.module.AllocateId(); + std::vector phi{spv::OpPhi, type, result}; + state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + state.module.AddFunction(branch); + for (std::uint32_t index = 0; index < count; ++index) { + EmitLabel(state, labels[index]); + const auto value = formatted(context, instruction, resource, GetFormatInfo(static_cast(index + 1u)), components, store); + const auto exit = state.module.AllocateId(); + state.module.AddFunction(spv::OpBranch, exit); + EmitLabel(state, exit); + state.module.AddFunction(spv::OpBranch, merge); + phi.insert(phi.end(), {value, exit}); + } + EmitLabel(state, invalid); + RecordBdaFault(state, resource.base, ConstantU32(state, 0u), ConstantU32(state, instruction.Flags().pc), BdaAbi::FaultReason::InvalidDescriptor); + const auto zero = store || components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); + const auto invalidExit = state.module.AllocateId(); + state.module.AddFunction(spv::OpBranch, invalidExit); + EmitLabel(state, invalidExit); + state.module.AddFunction(spv::OpBranch, merge); + phi.insert(phi.end(), {zero, invalidExit}); + EmitLabel(state, merge); + state.module.AddFunction(phi); + const auto unsupported = Binary(state, spv::OpLogicalOr, TypeBool(state), equal(state, format, ConstantU32(state, 0u)), Binary(state, spv::OpUGreaterThan, TypeBool(state), format, ConstantU32(state, count))); + StopBdaInvocationIf(state, unsupported); + return result; +} + +} + +std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + const auto type = components == 1u ? TypeU32(state) : TypeU32Composite(state, components); + const auto zero = components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); + return EmitValueOrDefaultIfCondition(state, context.Arg(instruction, instruction.ArgumentCount() - 1u), type, zero, [&] { + const auto resource = descriptor(context, instruction); + if (memory.formatted) return EmitValueOrDefaultIfCondition(state, resource.valid, type, zero, [&] { return formattedDispatch(context, instruction, resource, components, false); }); + std::array values{}; + for (std::uint32_t component = 0; component < components; ++component) { + const auto access = address(context, instruction, resource, component * 4u, memory.dataBits / 8u); + values[component] = EmitValueOrZeroIfCondition(state, access.inBounds, [&] { return EmitBdaRead(context, instruction, access.guest, memory.dataBits); }); + } + return composite(state, components, values); + }); +} + +void EmitRuntimeBufferStore(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + EmitIfCondition(state, context.Arg(instruction, instruction.ArgumentCount() - 1u), [&] { + const auto resource = descriptor(context, instruction); + if (memory.formatted) { + EmitIfCondition(state, resource.valid, [&] { static_cast(formattedDispatch(context, instruction, resource, components, true)); }); + return; + } + const auto values = storeValues(context, instruction, components); + for (std::uint32_t component = 0; component < components; ++component) { + const auto access = address(context, instruction, resource, component * 4u, memory.dataBits / 8u); + EmitIfCondition(state, access.inBounds, [&] { EmitBdaWrite(context, instruction, access.guest, values[component], memory.dataBits); }); + } + }); +} + +std::uint32_t EmitRuntimeScalarBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction) { + auto& state = context.state; + const auto resource = descriptor(context, instruction); + const auto u64 = TypeScalarU64(state); + const auto size = Select(state, u64, equal(state, resource.stride, ConstantU32(state, 0u)), Unary(state, spv::OpUConvert, u64, resource.records), Binary(state, spv::OpIMul, u64, Unary(state, spv::OpUConvert, u64, resource.stride), Unary(state, spv::OpUConvert, u64, resource.records))); + const auto address = EmitAddU32(state, context.Arg(instruction, 1u), ConstantU32(state, context.Memory(instruction).offset)); + const auto noCarry = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address, context.Arg(instruction, 1u)); + const auto byte = EmitAndConstant(state, address, ~3u); + const auto offset = Unary(state, spv::OpUConvert, u64, byte); + const auto inBounds = both(state, noCarry, both(state, resource.valid, Binary(state, spv::OpULessThanEqual, TypeBool(state), Binary(state, spv::OpIAdd, u64, offset, BdaConstant(state, 4u)), size))); + return EmitValueOrZeroIfCondition(state, inBounds, [&] { return EmitBdaRead(context, instruction, AddBdaAddress(context, instruction, resource.base, offset, false), 32u); }); +} + +std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t bits, std::uint32_t active, const std::function& operation) { + auto& state = context.state; + if (bits != 32u && bits != 64u) context.Fail(instruction, "unsupported runtime buffer atomic width"); + if (state.bdaAtomicPointerFunction == 0u || state.bdaNoteWriteFunction == 0u) context.Fail(instruction, "BDA atomic functions are missing"); + const auto type = bits == 64u ? TypeU64(state) : TypeU32(state); + const auto scalar = bits == 64u ? TypeScalarU64(state) : TypeU32(state); + const auto zero = bits == 64u ? ConstantU64(state, 0u) : ConstantU32(state, 0u); + return EmitValueOrDefaultIfCondition(state, active, type, zero, [&] { + const auto resource = descriptor(context, instruction); + const auto access = address(context, instruction, resource, 0u, bits / 8u); + return EmitValueOrDefaultIfCondition(state, access.inBounds, type, zero, [&] { + const auto unaligned = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), access.guest, BdaConstant(state, bits / 8u - 1u)), BdaConstant(state, 0u)); + faultIf(context, instruction, unaligned, access.guest, bits / 8u, BdaAbi::FaultReason::Unaligned); + return EmitValueOrDefaultIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), type, zero, [&] { + const auto physical = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaAtomicPointerFunction, access.guest, ConstantU32(state, bits / 8u), ConstantU32(state, instruction.Flags().pc)); + const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)); + return EmitValueOrDefaultIfCondition(state, mapped, type, zero, [&] { + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, scalar), pointer, physical); + const auto result = operation(pointer); + state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, access.guest); + return result; + }); + }); + }); + }); +} + +}