From 55447e1c7770eaea700da7829466bb2d2fd4e1c9 Mon Sep 17 00:00:00 2001 From: Michon <231849@buas.nl> Date: Mon, 5 Oct 2026 20:20:51 +0200 Subject: [PATCH] fix(recompiler): read a coherent dwordx2 as one 64-bit atomic load A glc buffer_load_dwordx2 is one 64-bit access on the GPU, so it never sees half of a buffer_atomic_swap_x2. It was emitted as two dword loads, which can: the first can read the flag from before the swap and the second the value from after it. GTA V gives its BVH nodes their record indices with a single-pass decoupled look-back prefix sum. Each workgroup publishes a {flag, value} pair for its block with buffer_atomic_swap_x2 and reads the previous 64 blocks' pairs with glc dwordx2 loads. A torn read counts earlier blocks twice, two nodes then share records, the BVH is corrupt and the ray tracing refit hangs. It only happens in scans of more than 64 blocks (the largest BVHs) and depends on timing. In a program that has the u64 storage-buffer view (it uses 64-bit buffer atomics), a coherent raw two-dword buffer load that is 8-byte aligned and in bounds is now read with one device-scope OpAtomicLoad on that view and split with OpUConvert and a shift (not OpBitcast to a uvec2, which crashes NVIDIA's shader compiler). All other loads keep the per-dword path. tests/execution/LookBackScan runs the game's scan kernel: the look-back from given blocks, and 50 whole 125-block scans checked against a CPU prefix sum. Without this change the first whole scan fails. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/execution/LookBackScan.cpp | 190 ++++++++++++++++++ .../SpirvMemory/SpirvMemoryInstructions.cpp | 24 +++ 2 files changed, 214 insertions(+) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/LookBackScan.cpp diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LookBackScan.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LookBackScan.cpp new file mode 100644 index 000000000..3e098baae --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LookBackScan.cpp @@ -0,0 +1,190 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::array ScanCode{ + 0xbfa00003u, 0x878e817eu, 0x8811ff05u, 0x00080000u, 0xbefe040eu, 0xbe900304u, 0xbe92030bu, 0xbe9303ffu, + 0x00016204u, 0x8805ff07u, 0x00040000u, 0xbe840306u, 0xbe86030cu, 0xbe8703ffu, 0x00016204u, 0xbf880003u, + 0x7e020281u, 0xe0c84000u, 0x80000100u, 0xbf8c3f70u, 0xd7600002u, 0x00010101u, 0xbefe04c1u, 0x9003860au, + 0xbf0a0380u, 0x93000a02u, 0x4a240000u, 0x7d88240cu, 0xbf840006u, 0x7e020280u, 0xbefe046au, 0xbf880002u, + 0xe0302000u, 0x80010112u, 0xbefe04c1u, 0xd76d0011u, 0x04018000u, 0xbf0a0381u, 0x7d88220cu, 0xbf840006u, + 0x7e040280u, 0xbefe046au, 0xbf880002u, 0xe0302000u, 0x80010211u, 0xbefe04c1u, 0xd76d0010u, 0x0401fe00u, + 0x00000080u, 0xbf0a0382u, 0x7d88200cu, 0xbf840006u, 0x7e060280u, 0xbefe046au, 0xbf880002u, 0xe0302000u, + 0x80010310u, 0xbefe04c1u, 0xd76d000fu, 0x0401fe00u, 0x000000c0u, 0xbf0a0383u, 0x7d881e0cu, 0xbf840006u, + 0x7e080280u, 0xbefe046au, 0xbf880002u, 0xe0302000u, 0x8001040fu, 0xbefe04c1u, 0xd76d000eu, 0x0401fe00u, + 0x00000100u, 0xbf0a0384u, 0x7d881c0cu, 0xbf840006u, 0x7e0a0280u, 0xbefe046au, 0xbf880002u, 0xe0302000u, + 0x8001050eu, 0xbefe04c1u, 0xd76d000du, 0x0401fe00u, 0x00000140u, 0xbf0a0385u, 0x7d881a0cu, 0xbf840006u, + 0x7e0c0280u, 0xbefe046au, 0xbf880002u, 0xe0302000u, 0x8001060du, 0xbefe04c1u, 0xd76d000cu, 0x0401fe00u, + 0x00000180u, 0xbf0a0386u, 0x7d88180cu, 0xbf840006u, 0x7e0e0280u, 0xbefe046au, 0xbf880002u, 0xe0302000u, + 0x8001070cu, 0xbefe04c1u, 0xd76d000bu, 0x0401fe00u, 0x000001c0u, 0xbf0a0387u, 0x7d88160cu, 0xbf840006u, + 0x7e100280u, 0xbefe046au, 0xbf880002u, 0xe0302000u, 0x8001080bu, 0xbefe04c1u, 0xbe8d0380u, 0xbe8a0380u, + 0xbf0a030du, 0xbf840020u, 0xbefc030du, 0x810d810du, 0xbf8c3f70u, 0x7e148701u, 0x7e12030au, 0xbeea287eu, + 0x02281480u, 0x4a2828fau, 0xff011114u, 0x4a2828fau, 0xff011214u, 0x4a2828fau, 0xff011414u, 0x4a2828fau, + 0xff011814u, 0xd7781013u, 0x03058314u, 0x4a2826fau, 0xaf00e414u, 0x92fea0a0u, 0xd7600000u, 0x00013f14u, + 0x4a282800u, 0xbefe046au, 0x4c121314u, 0x4a14130au, 0x4a12120au, 0xd760006au, 0x00017f0au, 0x7e028509u, + 0x810a6a0au, 0xbf82ffdeu, 0xbefe0481u, 0x7e260202u, 0xbf088002u, 0x7e14020au, 0x856a8281u, 0x7e12026au, + 0xe1402000u, 0x80040913u, 0xbf088002u, 0xbe8d0380u, 0xbefe04c1u, 0xbf840058u, 0xd76d0013u, 0x0401fe02u, + 0xffffffc0u, 0xbe8d0380u, 0x7d0626f9u, 0x06868080u, 0x7e120281u, 0x7e140280u, 0xbefe0400u, 0xbf880002u, + 0xe034e000u, 0x80040913u, 0xbefe04c1u, 0xbf8c3f70u, 0x7d041280u, 0xbf13806au, 0xbf840006u, 0xbefe0400u, + 0xbf880002u, 0xe034e000u, 0x80040913u, 0xbefe04c1u, 0xbf82fff6u, 0x7d0412f9u, 0x06868082u, 0xbf138000u, + 0xbf850016u, 0x4e2626c0u, 0xbeea287eu, 0x02121480u, 0x4a1212fau, 0xff011109u, 0x4a1212fau, 0xff011209u, + 0x4a1212fau, 0xff011409u, 0x4a1212fau, 0xff011809u, 0xd778100au, 0x03058309u, 0x4a121509u, 0xbefe046au, + 0xd7600000u, 0x00013f09u, 0xd7600001u, 0x00017f09u, 0x81140100u, 0x810d140du, 0xbf82ffd3u, 0x7e127201u, + 0x7d8a12c1u, 0x4c1212bfu, 0xbf870002u, 0x7e127200u, 0x4c12129fu, 0x7d881300u, 0x7e000509u, 0x02001480u, + 0xbeea287eu, 0xd7600014u, 0x0000010au, 0x02120080u, 0x4a1212fau, 0xff011109u, 0x4a1212fau, 0xff011209u, + 0x4a1212fau, 0xff011409u, 0x4a1212fau, 0xff011809u, 0xd7781000u, 0x03058309u, 0x4a120109u, 0xbefe0481u, + 0xd7600000u, 0x00013f09u, 0xd7600001u, 0x00017f09u, 0x81150100u, 0x816a1514u, 0x810d0d6au, 0x7e000202u, + 0x4a141af9u, 0x8686060au, 0x7e120282u, 0xe1402000u, 0x80040900u, 0xbefe04c1u, 0x7d86240cu, 0xbf090380u, + 0xbf800000u, 0x8580807eu, 0x8dea006au, 0xbefe046au, 0xbf880004u, 0xbf8c3f70u, 0x4a00020du, 0xe0702000u, + 0x80010012u, 0xbefe04c1u, 0xbf090381u, 0x7d86220cu, 0x8580807eu, 0x8dea006au, 0xbefe046au, 0xbf880004u, + 0xbf8c3f70u, 0x4a00040du, 0xe0702000u, 0x80010011u, 0xbefe04c1u, 0xbf090382u, 0x7d86200cu, 0x8580807eu, + 0x8dea006au, 0xbefe046au, 0xbf880004u, 0xbf8c3f70u, 0x4a00060du, 0xe0702000u, 0x80010010u, 0xbefe04c1u, + 0xbf090383u, 0x7d861e0cu, 0x8580807eu, 0x8dea006au, 0xbefe046au, 0xbf880004u, 0xbf8c3f70u, 0x4a00080du, + 0xe0702000u, 0x8001000fu, 0xbefe04c1u, 0xbf090384u, 0x7d861c0cu, 0x8580807eu, 0x8dea006au, 0xbefe046au, + 0xbf880004u, 0xbf8c3f70u, 0x4a000a0du, 0xe0702000u, 0x8001000eu, 0xbefe04c1u, 0xbf090385u, 0x7d861a0cu, + 0x8580807eu, 0x8dea006au, 0xbefe046au, 0xbf880004u, 0xbf8c3f70u, 0x4a000c0du, 0xe0702000u, 0x8001000du, + 0xbefe04c1u, 0xbf090386u, 0x7d86180cu, 0x8580807eu, 0x8dea006au, 0xbefe046au, 0xbf880004u, 0xbf8c3f70u, + 0x4a000e0du, 0xe0702000u, 0x8001000cu, 0xbefe04c1u, 0xbf090387u, 0x7d86160cu, 0x8580807eu, 0x8dea006au, + 0xbefe046au, 0xbf880004u, 0xbf8c3f70u, 0x4a00100du, 0xe0702000u, 0x8001000bu, 0x816ac10bu, 0xbefe04c1u, + 0xbf066a02u, 0xbf800000u, 0x85ea807eu, 0x87ea0e6au, 0xbefe046au, 0xbf880008u, 0x4a0014f9u, 0x8686060du, + 0xbe8a0381u, 0xbe8b03ffu, 0x00016204u, 0xbe891d92u, 0xe0700000u, 0x80020000u, 0xbf810000u, 0xbf9f0000u, + 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, + 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, + 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, + 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0xbf9f0000u, 0x00000000u, 0x00000000u, + 0x30306c73u, 0x0000013fu, 0x00000156u, 0x00000000u, 0x00102557u, 0x0401a600u, 0x0b100a10u, 0x40100c10u, + 0x4c104b40u, 0x4e104d10u, 0x79606f10u, 0x00107a10u, 0x00006e00u, 0x21200f64u, 0x25402410u, 0x64000010u, + 0x40241021u, 0x28600000u, 0x00102950u, 0x30c9a100u, 0x9e0800c0u, 0xa0841941u, 0x18c08418u, 0x8418c084u, + 0xc08418c0u, 0x18c08418u, 0x8418c084u, 0xe0c01040u, 0x08430820u, 0x08104108u, 0x2334108bu, 0x38838838u, + 0x88388388u, 0x83883883u, 0x80f01040u, 0x98241e05u, 0x0d8442e0u, 0xe1183e0fu, 0x17158644u, 0x46e1985eu, + 0x7e1f1d88u, 0x8a48e218u, 0x989e2725u, 0x2d8c4ae2u, 0xe318be2fu, 0x37358e4cu, 0x4ee398deu, 0xfe3f3d90u, + 0x9250e418u, 0x991e4745u, 0x4f4d92e4u, 0x6a25112eu, 0xe6ab29cau, 0xcb6b32bcu, 0x2afcbb2du, 0x31cc6c33u, + 0xf34b3cbbu, 0xdb36b5cau, 0x6716f378u, 0xddd6739cu, 0x9ede6779u, 0xf9fdf67bu, 0x83a0e067u, 0x687a1e16u, + 0x368ba2e2u, 0xe468fa3eu, 0x5e5693a4u, 0x68a6697au, 0xf7777777u, 0x5554fdddu, 0xcb000205u, 0x70817d02u, + 0x00080019u, 0x003f0146u, 0xa41002a4u, 0x3402a602u, 0xaa000a34u, 0xaa100c02u, 0x3402b002u, 0x1502b631u, + 0x400202bcu, 0x02b60101u, 0x8b2ef007u, 0xd74d0005u, 0xe0828001u, 0x00001601u, 0x00000000u, 0x00000000u, + 0x65726162u, 0x746f6f66u, 0x442e4a87u, 0x00000000u, 0x00000001u, 0x00000618u, 0x00000000u, 0x00000156u, + 0x980d163fu, 0xeddd574eu, 0x00000000u, 0x00000000u, +}; + +constexpr std::uint32_t Lanes = 64; +constexpr std::uint32_t PerBlock = 256; +constexpr std::uint32_t Blocks = 125; +constexpr std::uint32_t Elements = 31860; + +alignas(256) std::array Counter{}; +alignas(256) std::array Pairs{}; +alignas(256) std::array Data{}; +alignas(256) std::array Total{}; + +std::uint32_t Value(std::uint32_t index, std::uint32_t seed) { + return ((index * 2654435761u + seed * 40503u) >> 13u) % 3u; +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t groups) { + alignas(256) static std::array code{}; + code = ScanCode; + const auto address = [](const void* data) { return reinterpret_cast(data); }; + std::vector userData{ + static_cast(address(Counter.data())), static_cast((address(Counter.data()) >> 32u) & 0xffffu) | 0x00040000u, 1u, 0x00005204u, + static_cast(address(Pairs.data())), static_cast(address(Pairs.data()) >> 32u), + static_cast(address(Data.data())), static_cast(address(Data.data()) >> 32u), + static_cast(address(Total.data())), static_cast(address(Total.data()) >> 32u), + PerBlock, Blocks, Elements}; + const std::span words(code); + const std::array memory{{{address(code.data()), std::as_bytes(words)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Lanes, 1, 1}, 0, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, address(code.data()), words, 0, {}}, + {Lanes, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, groups, 1, 1, {}, address(code.data())); + device.WaitIdle(); +} + +std::uint32_t BlockSum(const std::vector& input, std::uint32_t block) { + std::uint32_t sum = 0; + for (std::uint32_t i = block * PerBlock; i < (block + 1) * PerBlock && i < Elements; ++i) sum += input[i]; + return sum; +} + +void CheckLookBack(AgcDriver::VulkanDevice& device, std::uint32_t block) { + const auto name = "look-back from block " + std::to_string(block); + std::vector input(Data.size(), 0u); + for (std::uint32_t i = 0; i < Elements; ++i) input[i] = Value(i, block); + Pairs.fill(0u); + std::uint32_t prefix = 0; + for (std::uint32_t b = 0; b < block; ++b) { + const auto sum = BlockSum(input, b); + prefix += sum; + Pairs[2 * b] = b == 0 ? 2u : 1u; + Pairs[2 * b + 1] = b == 0 ? prefix : sum; + } + Counter.fill(0u); + Counter[0] = block; + std::copy(input.begin(), input.end(), Data.begin()); + Total.fill(0xdeadbeefu); + Run(device, 1); + Require(Counter[0] == block + 1, name + ": the counter is " + std::to_string(Counter[0])); + Require(Pairs[2 * block] == 2u, name + ": its flag is " + std::to_string(Pairs[2 * block]) + ", expected 2"); + Require(Pairs[2 * block + 1] == prefix + BlockSum(input, block), name + ": it published " + std::to_string(Pairs[2 * block + 1]) + ", expected " + std::to_string(prefix + BlockSum(input, block)) + " (prefix " + std::to_string(prefix) + ")"); + std::uint32_t running = prefix; + for (std::uint32_t i = block * PerBlock; i < (block + 1) * PerBlock && i < Elements; ++i) { + Require(Data[i] == running, name + ": element " + std::to_string(i) + " is " + std::to_string(Data[i]) + ", expected " + std::to_string(running) + " (prefix " + std::to_string(prefix) + ")"); + running += input[i]; + } + if (block == Blocks - 1) Require(Total[0] == running, name + ": the total is " + std::to_string(Total[0]) + ", expected " + std::to_string(running)); +} + +void CheckWhole(AgcDriver::VulkanDevice& device, std::uint32_t seed) { + const auto name = "whole scan " + std::to_string(seed); + std::vector input(Data.size(), 0u); + for (std::uint32_t i = 0; i < Elements; ++i) input[i] = Value(i, seed); + Counter.fill(0u); + Pairs.fill(0u); + std::copy(input.begin(), input.end(), Data.begin()); + Total.fill(0xdeadbeefu); + Run(device, Blocks); + std::uint32_t running = 0; + for (std::uint32_t i = 0; i < Elements; ++i) { + Require(Data[i] == running, name + ": element " + std::to_string(i) + " (block " + std::to_string(i / PerBlock) + ") is " + std::to_string(Data[i]) + ", expected " + std::to_string(running)); + running += input[i]; + } + Require(Total[0] == running, name + ": the total is " + std::to_string(Total[0]) + ", expected " + std::to_string(running)); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (device->Target().subgroupSize < 32) { + std::printf("skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize); + return VulkanTestSkipped; + } + for (const std::uint32_t block : {0u, 1u, 50u, 63u, 64u, 65u, 100u, 124u}) CheckLookBack(*device, block); + for (std::uint32_t seed = 0; seed < 50; ++seed) CheckWhole(*device, seed); + std::puts("look-back scan tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index 550803e92..dee96aab9 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -580,11 +580,35 @@ void FormattedStore(SpirvValueEmitContext& ctx, const IrValue& inst, const Memor }); } +std::uint32_t LoadCoherentPair(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource) { + auto& state = ctx.state; + const auto wide = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferU64Variable, TypeStorageBufferU64Pointer(state)); + const auto byteAddress = Binary(state, spv::OpIAdd, TypeU32(state), ByteAddress(ctx, inst, mem), wide.byteOffset); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteAddress, ConstantU32(state, 3u)); + const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), byteAddress, ConstantU32(state, 7u)), ConstantU32(state, 0u)); + return EmitValueIfElse(state, AndCondition(state, aligned, EmitMemoryElementInBounds(state, wide, index)), TypeU32Composite(state, 2u), [&]() { + const auto pointer = EmitStorageBufferElementPointer(state, wide, index, TypeStorageBufferU64ElementPointer(state)); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicLoad, TypeScalarU64(state), value, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone)); + const auto high = Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), value, BdaConstant(state, 32u)); + return ConstructU32Composite(state, 2u, {Unary(state, spv::OpUConvert, TypeU32(state), value), Unary(state, spv::OpUConvert, TypeU32(state), high), 0u, 0u}); + }, [&]() { + std::array values{}; + for (std::uint32_t component = 0; component < 2u; component++) { + values.at(component) = LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource); + } + return ConstructU32Composite(state, 2u, values); + }); +} + std::uint32_t LoadWideBuffer(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) { auto& state = ctx.state; return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() { const auto resource = PrepareMemoryResourceAccess(state, mem); const auto info = MemoryFormatInfo(state, mem); + if (info.type == SpirvFormatComponentType::Unknown && components == 2u && mem.coherent && state.storageBufferU64Variable != 0u) { + return LoadCoherentPair(ctx, inst, mem, resource); + } if (info.type != SpirvFormatComponentType::Unknown) { const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, FormattedAccess::Load); return EmitValueOrDefaultIfCondition(state, plan.inBounds, TypeU32Composite(state, components), FormattedOutOfBoundsValue(ctx, mem, plan, components), [&]() {