From 6a1301f3294d7fcf96bdf43d8719ce3805d0f73b Mon Sep 17 00:00:00 2001 From: Michon <231849@buas.nl> Date: Sun, 4 Oct 2026 19:11:06 +0200 Subject: [PATCH 1/5] feat(recompiler): FLAT addresses in the shared and private apertures reach LDS and scratch A compute FLAT access whose address has the console's shared aperture in its high dword (0x8000xxxx) reads or writes the workgroup's LDS at its low dword, and one in the private aperture (0x7000xxxx) the lane's scratch; other addresses stay guest memory through BDA. GTA V's ray traversal keeps its stack in LDS and spills it to scratch through such addresses; treated as guest memory the pushes were lost and the traversal never ended (device lost). An execution test round-trips both apertures. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/execution/FlatApertures.cpp | 82 ++++++++++++++++++ .../SpirvBackend/src/SpirvAnalysis.cpp | 3 + .../SpirvMemory/SpirvMemoryInstructions.cpp | 86 ++++++++++++++++++- 3 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp new file mode 100644 index 000000000..adb356602 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp @@ -0,0 +1,82 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020082, 0x7e0402ff, 0x80000000, 0x4a0600ff, 0x00001000, 0x7e080288, 0x7e0a02ff, 0x70000000, + 0x4a0c00ff, 0x00002000, 0xdc700000, 0x007d0301, 0xdc700000, 0x007d0604, 0xbf8c0070, 0xdc300000, + 0x077d0001, 0xdc300000, 0x087d0004, 0xbf8c0070, 0xe0701000, 0x80000701, 0xe0701080, 0x80000801, + 0xbf810000, 0xbf810000, 0xbf810000, 0xbf810000, 0xbf810000, 0xbf810000, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Run(AgcDriver::VulkanDevice& device) { + Output.fill(0xdeadbeefu); + std::vector userData(4, 0u); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(output.begin(), output.end(), userData.begin()); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 256u, {false, false, false}, false, 1}; + compute.scratchDwords = 4u; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + Require(Output[tid] == tid + 0x1000u, "flat apertures: lane " + std::to_string(tid) + " read " + Hex(Output[tid]) + " back through the shared aperture, expected " + Hex(tid + 0x1000u)); + Require(Output[Threads + tid] == tid + 0x2000u, "flat apertures: lane " + std::to_string(tid) + " read " + Hex(Output[Threads + tid]) + " back through the private aperture, expected " + Hex(tid + 0x2000u)); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("flat aperture tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp index 91b74f5ec..b49a08668 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp @@ -325,6 +325,9 @@ SpirvRequirements AnalyzeProgramRequirements(const IrProgram& program) { } requirements.functionScratch = true; } + if (program.Resources().memoryInfo.at(memoryIndex).kind == ResourceKind::Flat && program.Resources().stage == IrShaderStage::Compute && program.Info().scratchDwords != 0u) { + requirements.functionScratch = true; + } } if (BufferAccessOf(inst->Opcode()) != BufferAccess::None) { const auto memoryIndex = inst->Flags().index; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index 7faaadbe1..7599ba75b 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -336,11 +336,68 @@ const MemoryInfo& SharedMemory(SpirvValueEmitContext& ctx, const IrValue& inst) return mem; } +constexpr std::uint32_t SharedApertureTop = 0x8000u; +constexpr std::uint32_t PrivateApertureTop = 0x7000u; + +bool RoutesApertures(const SpirvEmitterState& state, const MemoryInfo& mem) { + return mem.kind == ResourceKind::Flat && state.program.Resources().stage == IrShaderStage::Compute; +} + +struct ApertureAddress { + std::uint32_t low = 0; + std::uint32_t shared = 0; + std::uint32_t priv = 0; +}; + +ApertureAddress SplitAperture(SpirvEmitterState& state, std::uint32_t address) { + const auto u32 = TypeU32(state); + const auto high = Unary(state, spv::OpUConvert, u32, Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, 32u))); + const auto top = Binary(state, spv::OpShiftRightLogical, u32, high, ConstantU32(state, 16u)); + return {Unary(state, spv::OpUConvert, u32, address), Binary(state, spv::OpIEqual, TypeBool(state), top, ConstantU32(state, SharedApertureTop)), Binary(state, spv::OpIEqual, TypeBool(state), top, ConstantU32(state, PrivateApertureTop))}; +} + +std::uint32_t LoadApertureElement(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits) { + auto& state = ctx.state; + if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return ConstantU32(state, 0u); + MemoryInfo storage{}; + storage.kind = kind; + const auto resource = PrepareMemoryResourceAccess(state, storage); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteOffset, ConstantU32(state, 2u)); + return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&] { + return bits == 32u ? LoadWordInBounds(ctx, resource, index) : LoadSubwordInBounds(ctx, resource, byteOffset, index, bits, false); + }); +} + +void StoreApertureWord(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t data) { + auto& state = ctx.state; + if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return; + MemoryInfo storage{}; + storage.kind = kind; + const auto resource = PrepareMemoryResourceAccess(state, storage); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteOffset, ConstantU32(state, 2u)); + EmitIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&] { StoreWordInBounds(ctx, resource, index, data); }); +} + +template +std::uint32_t RouteFlatLoad(SpirvEmitterState& state, std::uint32_t address, std::uint32_t type, TAperture&& aperture, TGlobal&& global) { + const auto split = SplitAperture(state, address); + return EmitValueIfElse(state, split.shared, type, [&] { return aperture(ResourceKind::Lds, split.low); }, [&] { + return EmitValueIfElse(state, split.priv, type, [&] { return aperture(ResourceKind::Scratch, split.low); }, global); + }); +} + void LoadAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t bits) { const auto& mem = ctx.Memory(inst); if (bits == 32u && mem.planningOnly) { return; } + if (RoutesApertures(ctx.state, mem)) { + ctx.Define(inst, EmitValueOrZeroIfCondition(ctx.state, ActiveArgument(ctx, inst), [&] { + const auto address = GuestAddress(ctx, inst, mem); + return RouteFlatLoad(ctx.state, address, TypeU32(ctx.state), [&](ResourceKind kind, std::uint32_t low) { return LoadApertureElement(ctx, kind, low, bits); }, [&] { return EmitBdaRead(ctx, inst, address, bits); }); + })); + return; + } switch (mem.kind) { case ResourceKind::Scratch: ctx.Define(inst, bits == 32u ? LoadWord(ctx, inst, mem) : LoadSubword(ctx, inst, mem, bits)); @@ -367,6 +424,22 @@ void LoadAddressWide(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint3 if (mem.planningOnly) { return; } + if (RoutesApertures(ctx.state, mem)) { + auto& state = ctx.state; + const auto type = TypeU32Composite(state, components); + ctx.Define(inst, EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), type, ConstantU32CompositeZero(state, components), [&] { + const auto base = GuestAddressBase(ctx, inst, mem); + const auto address = AddBdaImmediate(ctx, inst, base, static_cast(mem.offset)); + return RouteFlatLoad(state, address, type, [&](ResourceKind kind, std::uint32_t low) { + std::array values{}; + for (std::uint32_t component = 0; component < components; component++) { + values[component] = LoadApertureElement(ctx, kind, Binary(state, spv::OpIAdd, TypeU32(state), low, ConstantU32(state, component * 4u)), 32u); + } + return ConstructU32Composite(state, components, values); + }, [&] { return ConstructU32Composite(state, components, EmitBdaDwordReads(ctx, inst, base, mem.offset, components)); }); + })); + return; + } switch (mem.kind) { case ResourceKind::Flat: case ResourceKind::Global: @@ -390,7 +463,18 @@ void StoreAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t case ResourceKind::Flat: case ResourceKind::Global: EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() { - EmitBdaStore(ctx, inst, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.ArgumentCount() - 2u), bits); + auto& state = ctx.state; + const auto address = GuestAddress(ctx, inst, mem); + const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u); + if (bits != 32u || !RoutesApertures(state, mem)) { + EmitBdaStore(ctx, inst, address, data, bits); + return; + } + const auto split = SplitAperture(state, address); + EmitIfCondition(state, split.shared, [&] { StoreApertureWord(ctx, ResourceKind::Lds, split.low, data); }); + EmitIfCondition(state, split.priv, [&] { StoreApertureWord(ctx, ResourceKind::Scratch, split.low, data); }); + const auto global = Unary(state, spv::OpLogicalNot, TypeBool(state), Binary(state, spv::OpLogicalOr, TypeBool(state), split.shared, split.priv)); + EmitIfCondition(state, global, [&] { EmitBdaStore(ctx, inst, address, data, bits); }); }); return; default: From c35f29b6be811571b84ebe0fe9910b23e29b5a03 Mon Sep 17 00:00:00 2001 From: Michon <231849@buas.nl> Date: Thu, 8 Oct 2026 18:08:17 +0200 Subject: [PATCH 2/5] fix(recompiler): every FLAT access kind routes the shared and private apertures; table test Sub-dword stores and unaligned dword accesses go byte-wise; 32- and 64-bit FLAT atomics route to LDS (64-bit under the workgroup lock), scratch (read-modify-write) or BDA. agc_driver_flat_apertures covers them in 121 rows, with mixed-address waves and wave64. The aperture bases and the unmodelled operands 235-238 are listed as inferred. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/execution/FlatApertures.cpp | 468 ++++++++++++++++-- .../SpirvBackend/src/SpirvAnalysis.cpp | 4 + .../SpirvMemory/SpirvMemoryInstructions.cpp | 259 +++++++++- docs/dev/TechnicalDebt.md | 3 + 4 files changed, 695 insertions(+), 39 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp index adb356602..d73947d4d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp @@ -1,13 +1,27 @@ #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libc/include/GuestAllocations.hpp" #include "Recompiler.hpp" #include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif #include #include #include #include +#include +#include +#include #include +#include +#include +#include #include +#include #include @@ -15,15 +29,69 @@ namespace { using AgcDriver::Graphics::Require; using ShaderRecompiler::ShaderStage; +using Words = std::vector; -constexpr std::uint32_t Threads = 32; -alignas(256) std::array Output{}; +constexpr std::uint32_t SharedHigh = 0x80000000u; +constexpr std::uint32_t PrivateHigh = 0x70000000u; +constexpr std::uint32_t LdsDwords = 256u; +constexpr std::uint32_t ScratchDwords = 4u; +constexpr std::size_t GuestBytes = 65536u; +constexpr std::uint32_t GuestStoreSlot = 0x200u; +constexpr std::uint32_t GuestAtomicSlot = 0x800u; +constexpr std::uint32_t WaitCnt0 = 0xbf8c0070u; +constexpr std::uint32_t EndProgram = 0xbf810000u; +constexpr std::uint32_t ShiftLaneBy5 = 0x34020085u; +constexpr std::uint32_t StoreByteOp = 0x18u; +constexpr std::uint32_t StoreShortOp = 0x1au; +constexpr std::uint32_t StoreDwordOp = 0x1cu; +constexpr std::uint32_t StoreDwordx2Op = 0x1du; +constexpr std::uint32_t LoadUbyteOp = 0x08u; +constexpr std::uint32_t LoadUshortOp = 0x0au; +constexpr std::uint32_t LoadDwordOp = 0x0cu; +constexpr std::uint32_t LoadDwordx2Op = 0x0du; -alignas(256) constexpr std::array Code{ - 0x34020082, 0x7e0402ff, 0x80000000, 0x4a0600ff, 0x00001000, 0x7e080288, 0x7e0a02ff, 0x70000000, - 0x4a0c00ff, 0x00002000, 0xdc700000, 0x007d0301, 0xdc700000, 0x007d0604, 0xbf8c0070, 0xdc300000, - 0x077d0001, 0xdc300000, 0x087d0004, 0xbf8c0070, 0xe0701000, 0x80000701, 0xe0701080, 0x80000801, - 0xbf810000, 0xbf810000, 0xbf810000, 0xbf810000, 0xbf810000, 0xbf810000, +enum class Target : std::uint8_t { Shared, Private, Guest, Mixed }; +enum class AtomicOp : std::uint8_t { Swap, Cmpswap, Add, Sub, Smin, Umin, Smax, Umax, And, Or, Xor, Inc, Dec, Fcmpswap, Fmin, Fmax }; + +constexpr std::size_t Atomic32Ops = 13; +constexpr std::array AtomicOpcodes32{0x30u, 0x31u, 0x32u, 0x33u, 0x35u, 0x36u, 0x37u, 0x38u, 0x39u, 0x3au, 0x3bu, 0x3cu, 0x3du}; +constexpr std::array AtomicOpcodes64{0x50u, 0x51u, 0x52u, 0x53u, 0x55u, 0x56u, 0x57u, 0x58u, 0x59u, 0x5au, 0x5bu, 0x5cu, 0x5du, 0x5eu, 0x5fu, 0x60u}; +constexpr std::array TargetNames{"shared", "private", "guest", "mixed"}; +constexpr std::array OpNames{"swap", "cmpswap", "add", "sub", "smin", "umin", "smax", "umax", "and", "or", "xor", "inc", "dec", "fcmpswap", "fmin", "fmax"}; + +struct Row { + Target target; + bool atomic; + AtomicOp op; + std::uint32_t width; + std::uint32_t offset; + std::uint32_t waveSize; +}; + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, GuestBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(GuestBytes, GuestBytes)); +#endif + Require(block != nullptr, "flat apertures: cannot allocate the guest block"); + std::memset(block, 0, GuestBytes); + GuestAllocations::Mutation().Add(block, GuestBytes, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uintptr_t Address() const { return reinterpret_cast(block); } + +private: + std::uint8_t* block = nullptr; }; std::array BufferDescriptor(const void* data, std::uint32_t bytes) { @@ -37,32 +105,368 @@ std::string Hex(std::uint32_t value) { return text; } -void Run(AgcDriver::VulkanDevice& device) { - Output.fill(0xdeadbeefu); - std::vector userData(4, 0u); - const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); - std::copy(output.begin(), output.end(), userData.begin()); - const std::span code(Code); - const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; - ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 256u, {false, false, false}, false, 1}; - compute.scratchDwords = 4u; - ShaderRecompiler::RecompileRequest request{ +std::string Hex64(std::uint64_t value) { + char text[24]; + std::snprintf(text, sizeof(text), "0x%016llx", static_cast(value)); + return text; +} + +std::uint32_t FlatWord0(std::uint32_t op, bool glc, std::uint32_t offset = 0u) { + return 0xdc000000u | (op << 18u) | (glc ? 0x10000u : 0u) | offset; +} + +std::uint32_t FlatWord1(std::uint32_t vdst, std::uint32_t data, std::uint32_t address) { + return (vdst << 24u) | (0x7du << 16u) | (data << 8u) | address; +} + +void EmitFlat(Words& code, std::uint32_t op, bool glc, std::uint32_t vdst, std::uint32_t data, std::uint32_t address) { + code.push_back(FlatWord0(op, glc)); + code.push_back(FlatWord1(vdst, data, address)); +} + +void EmitTableLoad(Words& code, std::uint32_t vdst, std::uint32_t offset) { + code.push_back(0xe0381000u | offset); + code.push_back(0x80010001u | (vdst << 8u)); +} + +void EmitOutputStore(Words& code, std::uint32_t vdata, std::uint32_t offset) { + code.push_back(0xe0701000u | offset); + code.push_back(0x80000001u | (vdata << 8u)); +} + +std::uint32_t StoreOp(std::uint32_t width) { + return width == 8u ? StoreByteOp : (width == 16u ? StoreShortOp : StoreDwordOp); +} + +std::uint32_t LoadOp(std::uint32_t width) { + return width == 8u ? LoadUbyteOp : (width == 16u ? LoadUshortOp : LoadDwordOp); +} + +Words Kernel32(const Row& row) { + Words code{ShiftLaneBy5}; + EmitTableLoad(code, 2u, 0u); + EmitTableLoad(code, 6u, 16u); + code.push_back(WaitCnt0); + if (row.atomic) { + EmitFlat(code, StoreDwordOp, false, 0u, 6u, 2u); + code.push_back(WaitCnt0); + EmitFlat(code, AtomicOpcodes32[static_cast(row.op)], true, 10u, 4u, 2u); + code.push_back(WaitCnt0); + EmitFlat(code, LoadDwordOp, false, 11u, 0u, 2u); + code.push_back(WaitCnt0); + EmitOutputStore(code, 10u, 0u); + EmitOutputStore(code, 11u, 4u); + } else { + EmitFlat(code, StoreOp(row.width), false, 0u, 4u, 2u); + code.push_back(WaitCnt0); + EmitFlat(code, LoadOp(row.width), false, 10u, 0u, 2u); + EmitFlat(code, LoadUbyteOp, false, 11u, 0u, 2u); + code.push_back(FlatWord0(LoadUbyteOp, false, row.width / 8u - 1u)); + code.push_back(FlatWord1(12u, 0u, 2u)); + code.push_back(WaitCnt0); + EmitOutputStore(code, 10u, 0u); + EmitOutputStore(code, 11u, 4u); + EmitOutputStore(code, 12u, 8u); + } + code.push_back(EndProgram); + return code; +} + +Words Kernel64(const Row& row) { + Words code{ShiftLaneBy5}; + EmitTableLoad(code, 2u, 0u); + EmitTableLoad(code, 6u, 16u); + code.push_back(WaitCnt0); + EmitFlat(code, StoreDwordx2Op, false, 0u, 8u, 2u); + code.push_back(WaitCnt0); + EmitFlat(code, AtomicOpcodes64[static_cast(row.op)], true, 10u, 4u, 2u); + code.push_back(WaitCnt0); + EmitFlat(code, LoadDwordx2Op, false, 12u, 0u, 2u); + code.push_back(WaitCnt0); + EmitOutputStore(code, 10u, 0u); + EmitOutputStore(code, 11u, 4u); + EmitOutputStore(code, 12u, 8u); + EmitOutputStore(code, 13u, 12u); + code.push_back(EndProgram); + return code; +} + +Words Kernel(const Row& row) { + return row.atomic && row.width == 64u ? Kernel64(row) : Kernel32(row); +} + +std::uint32_t OperandA(std::uint32_t lane) { + return 0x9e3779b1u * (lane + 1u) ^ 0x5a5a0f0fu; +} + +std::uint32_t InitValue(std::uint32_t lane) { + return (lane % 2u) != 0u ? (0x80000000u | (lane * 0x101u)) : (lane * 0x35u + 7u); +} + +std::uint32_t CmpValue(std::uint32_t lane) { + return (lane % 2u) != 0u ? InitValue(lane) + 1u : InitValue(lane); +} + +std::uint32_t Apply(AtomicOp op, std::uint32_t init, std::uint32_t a, std::uint32_t b) { + const auto signedInit = static_cast(init); + const auto signedA = static_cast(a); + switch (op) { + case AtomicOp::Swap: return a; + case AtomicOp::Cmpswap: return init == b ? a : init; + case AtomicOp::Add: return init + a; + case AtomicOp::Sub: return init - a; + case AtomicOp::Smin: return signedInit < signedA ? init : a; + case AtomicOp::Umin: return init < a ? init : a; + case AtomicOp::Smax: return signedInit > signedA ? init : a; + case AtomicOp::Umax: return init > a ? init : a; + case AtomicOp::And: return init & a; + case AtomicOp::Or: return init | a; + case AtomicOp::Xor: return init ^ a; + case AtomicOp::Inc: return init >= a ? 0u : init + 1u; + case AtomicOp::Dec: return (init == 0u || init > a) ? a : init - 1u; + default: break; + } + return init; +} + +std::uint64_t Pack(std::uint32_t high, std::uint32_t low) { + return (static_cast(high) << 32u) | low; +} + +std::uint64_t Bits(double value) { + std::uint64_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +bool IsFloatOp(AtomicOp op) { + return op == AtomicOp::Fcmpswap || op == AtomicOp::Fmin || op == AtomicOp::Fmax; +} + +std::uint64_t IntInit64(std::uint32_t lane) { + return Pack((lane % 2u) != 0u ? (0x80000000u | (lane * 0x1357u)) : (lane * 0x0a0bu + 0x11u), InitValue(lane)); +} + +std::uint64_t IntA64(std::uint32_t lane) { + return Pack(0x85ebca6bu * (lane + 1u) ^ 0xc2b2ae35u, OperandA(lane)); +} + +std::uint64_t IntCmp64(std::uint32_t lane) { + return (lane % 2u) != 0u ? IntInit64(lane) + 1u : IntInit64(lane); +} + +double FloatInit(std::uint32_t lane) { + return (lane % 2u) == 0u ? static_cast(lane + 1u) * 3.25 : -static_cast(lane + 1u) * 7.5; +} + +double FloatA(std::uint32_t lane) { + return (lane % 2u) == 0u ? -static_cast(lane + 2u) * 1.5 : static_cast(lane + 3u) * 2.75 - 2.0; +} + +double FloatCmp(std::uint32_t lane) { + return (lane % 2u) == 0u ? FloatInit(lane) : FloatInit(lane) + 1.0; +} + +std::uint64_t InitBits64(AtomicOp op, std::uint32_t lane) { + return IsFloatOp(op) ? Bits(FloatInit(lane)) : IntInit64(lane); +} + +std::uint64_t OperandBits64(AtomicOp op, std::uint32_t lane) { + return IsFloatOp(op) ? Bits(FloatA(lane)) : IntA64(lane); +} + +std::uint64_t CmpBits64(AtomicOp op, std::uint32_t lane) { + return IsFloatOp(op) ? Bits(FloatCmp(lane)) : IntCmp64(lane); +} + +std::uint64_t Apply64(AtomicOp op, std::uint64_t init, std::uint64_t a, std::uint64_t cmp) { + const auto signedInit = static_cast(init); + const auto signedA = static_cast(a); + switch (op) { + case AtomicOp::Swap: return a; + case AtomicOp::Cmpswap: return init == cmp ? a : init; + case AtomicOp::Add: return init + a; + case AtomicOp::Sub: return init - a; + case AtomicOp::Smin: return signedInit < signedA ? init : a; + case AtomicOp::Umin: return init < a ? init : a; + case AtomicOp::Smax: return signedInit > signedA ? init : a; + case AtomicOp::Umax: return init > a ? init : a; + case AtomicOp::And: return init & a; + case AtomicOp::Or: return init | a; + case AtomicOp::Xor: return init ^ a; + case AtomicOp::Inc: return init >= a ? 0u : init + 1u; + case AtomicOp::Dec: return (init == 0u || init > a) ? a : init - 1u; + default: break; + } + return init; +} + +std::uint64_t Expected64(AtomicOp op, std::uint32_t lane) { + if (IsFloatOp(op)) { + const auto init = FloatInit(lane); + const auto a = FloatA(lane); + switch (op) { + case AtomicOp::Fcmpswap: return Bits(init == FloatCmp(lane) ? a : init); + case AtomicOp::Fmin: return Bits(std::min(init, a)); + case AtomicOp::Fmax: return Bits(std::max(init, a)); + default: break; + } + } + return Apply64(op, IntInit64(lane), IntA64(lane), IntCmp64(lane)); +} + +std::uint64_t LaneAddress(const Row& row, std::uint32_t lane, std::uint64_t guestBase) { + Target target = row.target; + if (target == Target::Mixed) { + target = lane % 3u == 0u ? Target::Shared : (lane % 3u == 1u ? Target::Private : Target::Guest); + } + const std::uint32_t privateBase = row.width == 64u ? 8u : 4u; + switch (target) { + case Target::Shared: return (static_cast(SharedHigh) << 32u) | (8u * lane + row.offset); + case Target::Private: return (static_cast(PrivateHigh) << 32u) | (privateBase + row.offset); + case Target::Guest: return guestBase + 8u * lane + row.offset; + default: break; + } + throw std::runtime_error("flat apertures: no address for the row target"); +} + +std::string Describe(const Row& row) { + std::string access; + if (row.atomic) { + access = std::string(row.width == 64u ? "flat_atomic64_" : "flat_atomic_") + OpNames[static_cast(row.op)]; + } else { + access = "flat_store" + std::to_string(row.width); + } + return access + " " + TargetNames[static_cast(row.target)] + " offset " + std::to_string(row.offset) + " wave" + std::to_string(row.waveSize); +} + +void Check(const Row& row, const std::vector& output) { + const std::string name = Describe(row); + for (std::uint32_t lane = 0; lane < row.waveSize; ++lane) { + const auto a = OperandA(lane); + if (row.atomic) { + const auto init = InitValue(lane); + const auto old = output[lane * 8u]; + Require(old == init, name + ": lane " + std::to_string(lane) + " returned " + Hex(old) + ", expected " + Hex(init)); + const auto expected = Apply(row.op, init, a, CmpValue(lane)); + const auto left = output[lane * 8u + 1u]; + Require(left == expected, name + ": lane " + std::to_string(lane) + " left " + Hex(left) + ", expected " + Hex(expected)); + } else { + const auto expected = row.width == 8u ? (a & 0xffu) : (row.width == 16u ? (a & 0xffffu) : a); + const auto read = output[lane * 8u]; + Require(read == expected, name + ": lane " + std::to_string(lane) + " read " + Hex(read) + " back, expected " + Hex(expected)); + const auto first = output[lane * 8u + 1u]; + Require(first == (a & 0xffu), name + ": lane " + std::to_string(lane) + " has first byte " + Hex(first) + " in memory, expected " + Hex(a & 0xffu)); + const auto last = output[lane * 8u + 2u]; + const auto lastExpected = (a >> (8u * (row.width / 8u - 1u))) & 0xffu; + Require(last == lastExpected, name + ": lane " + std::to_string(lane) + " has last byte " + Hex(last) + " in memory, expected " + Hex(lastExpected)); + } + } +} + +void Check64(const Row& row, const std::vector& output) { + const std::string name = Describe(row); + for (std::uint32_t lane = 0; lane < row.waveSize; ++lane) { + const auto old = Pack(output[lane * 8u + 1u], output[lane * 8u]); + const auto init = InitBits64(row.op, lane); + Require(old == init, name + ": lane " + std::to_string(lane) + " returned " + Hex64(old) + ", expected " + Hex64(init)); + const auto left = Pack(output[lane * 8u + 3u], output[lane * 8u + 2u]); + const auto expected = Expected64(row.op, lane); + Require(left == expected, name + ": lane " + std::to_string(lane) + " left " + Hex64(left) + ", expected " + Hex64(expected)); + } +} + +ShaderRecompiler::RecompileRequest MakeRequest(AgcDriver::VulkanDevice& device, std::span code, const std::array& memory, std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, std::uint32_t waveSize) { + return ShaderRecompiler::RecompileRequest{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, - {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory}, device.Target(), {0, 0, 0, 128} }; - request.useCache = false; - const auto result = ShaderRecompiler::Recompile(request); - device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); - device.WaitIdle(); } -void Check() { - for (std::uint32_t tid = 0; tid < Threads; ++tid) { - Require(Output[tid] == tid + 0x1000u, "flat apertures: lane " + std::to_string(tid) + " read " + Hex(Output[tid]) + " back through the shared aperture, expected " + Hex(tid + 0x1000u)); - Require(Output[Threads + tid] == tid + 0x2000u, "flat apertures: lane " + std::to_string(tid) + " read " + Hex(Output[Threads + tid]) + " back through the private aperture, expected " + Hex(tid + 0x2000u)); +void Run(AgcDriver::VulkanDevice& device, const Row& row, const GuestBlock& guest) { + const auto lanes = row.waveSize; + const auto guestSlot = row.atomic ? GuestAtomicSlot : GuestStoreSlot; + const auto guestBase = static_cast(guest.Address()) + guestSlot; + const bool wideAtomic = row.atomic && row.width == 64u; + std::vector table(lanes * 8u, 0u); + for (std::uint32_t lane = 0; lane < lanes; ++lane) { + const auto address = LaneAddress(row, lane, guestBase); + table[lane * 8u] = static_cast(address); + table[lane * 8u + 1u] = static_cast(address >> 32u); + if (wideAtomic) { + const auto operand = OperandBits64(row.op, lane); + const auto cmp = CmpBits64(row.op, lane); + const auto init = InitBits64(row.op, lane); + table[lane * 8u + 2u] = static_cast(operand); + table[lane * 8u + 3u] = static_cast(operand >> 32u); + table[lane * 8u + 4u] = static_cast(cmp); + table[lane * 8u + 5u] = static_cast(cmp >> 32u); + table[lane * 8u + 6u] = static_cast(init); + table[lane * 8u + 7u] = static_cast(init >> 32u); + } else { + table[lane * 8u + 2u] = OperandA(lane); + table[lane * 8u + 3u] = row.atomic ? CmpValue(lane) : 0u; + table[lane * 8u + 4u] = row.atomic ? InitValue(lane) : 0u; + } } + std::vector output(lanes * 8u, 0xdeadbeefu); + std::vector userData(8, 0u); + const auto outputDescriptor = BufferDescriptor(output.data(), static_cast(output.size() * 4u)); + const auto tableDescriptor = BufferDescriptor(table.data(), static_cast(table.size() * 4u)); + std::copy(outputDescriptor.begin(), outputDescriptor.end(), userData.begin()); + std::copy(tableDescriptor.begin(), tableDescriptor.end(), userData.begin() + 4); + const auto code = Kernel(row); + const std::span codeSpan(code); + const std::array memory{{{reinterpret_cast(codeSpan.data()), std::as_bytes(codeSpan)}}}; + ShaderRecompiler::ShaderComputeStageInfo compute{{lanes, 1, 1}, LdsDwords, {false, false, false}, false, 1}; + compute.scratchDwords = ScratchDwords; + auto request = MakeRequest(device, codeSpan, memory, userData, compute, row.waveSize); + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(codeSpan.data())); + device.WaitIdle(); + if (wideAtomic) { + Check64(row, output); + } else { + Check(row, output); + } +} + +std::vector Rows() { + std::vector rows; + const std::array single{Target::Shared, Target::Private, Target::Guest}; + const std::array, 7> stores{{{8u, 0u}, {8u, 3u}, {16u, 0u}, {16u, 1u}, {16u, 3u}, {32u, 0u}, {32u, 2u}}}; + for (const Target target : single) { + for (const auto [width, offset] : stores) { + rows.push_back(Row{target, false, AtomicOp::Swap, width, offset, 32u}); + } + } + rows.push_back(Row{Target::Mixed, false, AtomicOp::Swap, 16u, 3u, 32u}); + rows.push_back(Row{Target::Mixed, false, AtomicOp::Swap, 32u, 2u, 32u}); + for (const Target target : single) { + for (std::uint32_t op = 0; op < Atomic32Ops; ++op) { + rows.push_back(Row{target, true, static_cast(op), 32u, 0u, 32u}); + } + } + for (const Target target : single) { + for (std::uint32_t op = 0; op < OpNames.size(); ++op) { + rows.push_back(Row{target, true, static_cast(op), 64u, 0u, 32u}); + } + } + rows.push_back(Row{Target::Mixed, true, AtomicOp::Add, 32u, 0u, 32u}); + rows.push_back(Row{Target::Mixed, true, AtomicOp::Cmpswap, 32u, 0u, 32u}); + rows.push_back(Row{Target::Mixed, true, AtomicOp::Add, 64u, 0u, 32u}); + rows.push_back(Row{Target::Mixed, true, AtomicOp::Cmpswap, 64u, 0u, 32u}); + rows.push_back(Row{Target::Mixed, true, AtomicOp::Fmax, 64u, 0u, 32u}); + rows.push_back(Row{Target::Mixed, false, AtomicOp::Swap, 16u, 3u, 64u}); + rows.push_back(Row{Target::Mixed, false, AtomicOp::Swap, 32u, 2u, 64u}); + rows.push_back(Row{Target::Mixed, true, AtomicOp::Add, 32u, 0u, 64u}); + rows.push_back(Row{Target::Shared, true, AtomicOp::Cmpswap, 32u, 0u, 64u}); + rows.push_back(Row{Target::Mixed, true, AtomicOp::Add, 64u, 0u, 64u}); + rows.push_back(Row{Target::Shared, true, AtomicOp::Fmin, 64u, 0u, 64u}); + return rows; } } @@ -71,8 +475,16 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - Run(*device); - Check(); + const bool wave64 = device->Target().subgroupSize >= 32u; + GuestBlock guest; + std::size_t checked = 0; + for (const Row& row : Rows()) { + if (row.waveSize == 64u && !wave64) continue; + Run(*device, row, guest); + ++checked; + } + if (!wave64) std::printf("skipped the wave64 rows, subgroup size %u cannot hold a wave64 in two lanes\n", device->Target().subgroupSize); + std::printf("flat aperture rows checked: %zu\n", checked); std::puts("flat aperture tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp index b49a08668..8c6d97555 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp @@ -328,6 +328,10 @@ SpirvRequirements AnalyzeProgramRequirements(const IrProgram& program) { if (program.Resources().memoryInfo.at(memoryIndex).kind == ResourceKind::Flat && program.Resources().stage == IrShaderStage::Compute && program.Info().scratchDwords != 0u) { requirements.functionScratch = true; } + if (program.Resources().memoryInfo.at(memoryIndex).kind == ResourceKind::Flat && program.Resources().stage == IrShaderStage::Compute && addressAccess == AddressAccess::Atomic && inst->Type() == IrType::U64) { + requirements.ldsLock = true; + requirements.subgroupBallot = true; + } } if (BufferAccessOf(inst->Opcode()) != BufferAccess::None) { const auto memoryIndex = inst->Flags().index; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index 7599ba75b..3c96d5066 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -356,7 +356,17 @@ ApertureAddress SplitAperture(SpirvEmitterState& state, std::uint32_t address) { return {Unary(state, spv::OpUConvert, u32, address), Binary(state, spv::OpIEqual, TypeBool(state), top, ConstantU32(state, SharedApertureTop)), Binary(state, spv::OpIEqual, TypeBool(state), top, ConstantU32(state, PrivateApertureTop))}; } -std::uint32_t LoadApertureElement(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits) { +std::uint32_t ApertureByteInBounds(SpirvEmitterState& state, const MemoryResourceAccess& resource, std::uint32_t byteOffset, std::uint32_t offset, std::uint32_t index) { + const auto noWrap = Binary(state, spv::OpULessThanEqual, TypeBool(state), byteOffset, ConstantU32(state, 0xffffffffu - offset)); + return AndCondition(state, noWrap, EmitMemoryElementInBounds(state, resource, index)); +} + +std::uint32_t ApertureContained(SpirvEmitterState& state, std::uint32_t byteOffset, std::uint32_t bits) { + const auto inDword = Binary(state, spv::OpBitwiseAnd, TypeU32(state), byteOffset, ConstantU32(state, 3u)); + return Binary(state, spv::OpULessThanEqual, TypeBool(state), EmitAddU32(state, inDword, ConstantU32(state, bits / 8u)), ConstantU32(state, 4u)); +} + +std::uint32_t LoadApertureDword(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits) { auto& state = ctx.state; if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return ConstantU32(state, 0u); MemoryInfo storage{}; @@ -368,18 +378,78 @@ std::uint32_t LoadApertureElement(SpirvValueEmitContext& ctx, ResourceKind kind, }); } -void StoreApertureWord(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t data) { +std::uint32_t LoadApertureByte(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t offset) { + auto& state = ctx.state; + if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return ConstantU32(state, 0u); + MemoryInfo storage{}; + storage.kind = kind; + const auto resource = PrepareMemoryResourceAccess(state, storage); + const auto address = EmitAddU32(state, byteOffset, ConstantU32(state, offset)); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); + return EmitValueOrZeroIfCondition(state, ApertureByteInBounds(state, resource, byteOffset, offset, index), [&] { + return LoadSubwordInBounds(ctx, resource, address, index, 8u, false); + }); +} + +std::uint32_t LoadApertureElement(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits) { + auto& state = ctx.state; + if (bits == 8u) return LoadApertureByte(ctx, kind, byteOffset, 0u); + return EmitValueIfElse(state, ApertureContained(state, byteOffset, bits), TypeU32(state), [&] { + return LoadApertureDword(ctx, kind, byteOffset, bits); + }, [&] { + std::uint32_t value = ConstantU32(state, 0u); + for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) { + const auto part = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), LoadApertureByte(ctx, kind, byteOffset, byte), ConstantU32(state, byte * 8u)); + value = Binary(state, spv::OpBitwiseOr, TypeU32(state), value, part); + } + return value; + }); +} + +void StoreApertureByte(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t offset, std::uint32_t data) { + auto& state = ctx.state; + if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return; + MemoryInfo storage{}; + storage.kind = kind; + const auto resource = PrepareMemoryResourceAccess(state, storage); + const auto address = EmitAddU32(state, byteOffset, ConstantU32(state, offset)); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); + EmitIfCondition(state, ApertureByteInBounds(state, resource, byteOffset, offset, index), [&] { StoreSubwordInBounds(ctx, storage, resource, address, index, 8u, data); }); +} + +void StoreApertureDword(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits, std::uint32_t data) { auto& state = ctx.state; if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return; MemoryInfo storage{}; storage.kind = kind; const auto resource = PrepareMemoryResourceAccess(state, storage); const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteOffset, ConstantU32(state, 2u)); - EmitIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&] { StoreWordInBounds(ctx, resource, index, data); }); + EmitIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&] { + if (bits == 32u) { + StoreWordInBounds(ctx, resource, index, data); + } else { + StoreSubwordInBounds(ctx, storage, resource, byteOffset, index, bits, data); + } + }); +} + +void StoreApertureElement(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits, std::uint32_t data) { + auto& state = ctx.state; + if (bits == 8u) { + StoreApertureByte(ctx, kind, byteOffset, 0u, data); + return; + } + const auto contained = ApertureContained(state, byteOffset, bits); + EmitIfCondition(state, contained, [&] { StoreApertureDword(ctx, kind, byteOffset, bits, data); }); + EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), contained), [&] { + for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) { + StoreApertureByte(ctx, kind, byteOffset, byte, Binary(state, spv::OpShiftRightLogical, TypeU32(state), data, ConstantU32(state, byte * 8u))); + } + }); } template -std::uint32_t RouteFlatLoad(SpirvEmitterState& state, std::uint32_t address, std::uint32_t type, TAperture&& aperture, TGlobal&& global) { +std::uint32_t RouteFlatAccess(SpirvEmitterState& state, std::uint32_t address, std::uint32_t type, TAperture&& aperture, TGlobal&& global) { const auto split = SplitAperture(state, address); return EmitValueIfElse(state, split.shared, type, [&] { return aperture(ResourceKind::Lds, split.low); }, [&] { return EmitValueIfElse(state, split.priv, type, [&] { return aperture(ResourceKind::Scratch, split.low); }, global); @@ -394,7 +464,7 @@ void LoadAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t if (RoutesApertures(ctx.state, mem)) { ctx.Define(inst, EmitValueOrZeroIfCondition(ctx.state, ActiveArgument(ctx, inst), [&] { const auto address = GuestAddress(ctx, inst, mem); - return RouteFlatLoad(ctx.state, address, TypeU32(ctx.state), [&](ResourceKind kind, std::uint32_t low) { return LoadApertureElement(ctx, kind, low, bits); }, [&] { return EmitBdaRead(ctx, inst, address, bits); }); + return RouteFlatAccess(ctx.state, address, TypeU32(ctx.state), [&](ResourceKind kind, std::uint32_t low) { return LoadApertureElement(ctx, kind, low, bits); }, [&] { return EmitBdaRead(ctx, inst, address, bits); }); })); return; } @@ -430,7 +500,7 @@ void LoadAddressWide(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint3 ctx.Define(inst, EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), type, ConstantU32CompositeZero(state, components), [&] { const auto base = GuestAddressBase(ctx, inst, mem); const auto address = AddBdaImmediate(ctx, inst, base, static_cast(mem.offset)); - return RouteFlatLoad(state, address, type, [&](ResourceKind kind, std::uint32_t low) { + return RouteFlatAccess(state, address, type, [&](ResourceKind kind, std::uint32_t low) { std::array values{}; for (std::uint32_t component = 0; component < components; component++) { values[component] = LoadApertureElement(ctx, kind, Binary(state, spv::OpIAdd, TypeU32(state), low, ConstantU32(state, component * 4u)), 32u); @@ -466,13 +536,13 @@ void StoreAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t auto& state = ctx.state; const auto address = GuestAddress(ctx, inst, mem); const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u); - if (bits != 32u || !RoutesApertures(state, mem)) { + if (!RoutesApertures(state, mem)) { EmitBdaStore(ctx, inst, address, data, bits); return; } const auto split = SplitAperture(state, address); - EmitIfCondition(state, split.shared, [&] { StoreApertureWord(ctx, ResourceKind::Lds, split.low, data); }); - EmitIfCondition(state, split.priv, [&] { StoreApertureWord(ctx, ResourceKind::Scratch, split.low, data); }); + EmitIfCondition(state, split.shared, [&] { StoreApertureElement(ctx, ResourceKind::Lds, split.low, bits, data); }); + EmitIfCondition(state, split.priv, [&] { StoreApertureElement(ctx, ResourceKind::Scratch, split.low, bits, data); }); const auto global = Unary(state, spv::OpLogicalNot, TypeBool(state), Binary(state, spv::OpLogicalOr, TypeBool(state), split.shared, split.priv)); EmitIfCondition(state, global, [&] { EmitBdaStore(ctx, inst, address, data, bits); }); }); @@ -818,6 +888,163 @@ std::uint32_t AddressAtomicOpcode(IrOpcode opcode) { } } +std::uint32_t FlatAtomicNext32(SpirvEmitterState& state, IrOpcode opcode, std::uint32_t old, std::uint32_t value, std::uint32_t comparator) { + const auto u32 = TypeU32(state); + const auto winner = [&](spv::Op compare) { + return Select(state, u32, Binary(state, compare, TypeBool(state), old, value), old, value); + }; + switch (opcode) { + case IrOpcode::AddressAtomicSwap32: + return value; + case IrOpcode::AddressAtomicCmpSwap32: + return Select(state, u32, Binary(state, spv::OpIEqual, TypeBool(state), old, comparator), value, old); + case IrOpcode::AddressAtomicIAdd32: + return Binary(state, spv::OpIAdd, u32, old, value); + case IrOpcode::AddressAtomicISub32: + return Binary(state, spv::OpISub, u32, old, value); + case IrOpcode::AddressAtomicSMin32: + return winner(spv::OpSLessThan); + case IrOpcode::AddressAtomicUMin32: + return winner(spv::OpULessThan); + case IrOpcode::AddressAtomicSMax32: + return winner(spv::OpSGreaterThan); + case IrOpcode::AddressAtomicUMax32: + return winner(spv::OpUGreaterThan); + case IrOpcode::AddressAtomicAnd32: + return Binary(state, spv::OpBitwiseAnd, u32, old, value); + case IrOpcode::AddressAtomicOr32: + return Binary(state, spv::OpBitwiseOr, u32, old, value); + case IrOpcode::AddressAtomicXor32: + return Binary(state, spv::OpBitwiseXor, u32, old, value); + case IrOpcode::AddressAtomicInc32: + return AtomicIncrement(state, old, value); + case IrOpcode::AddressAtomicDec32: + return AtomicDecrement(state, old, value); + case IrOpcode::AddressAtomicUSubSat32: + return AtomicUSubSat(state, old, value); + case IrOpcode::AddressAtomicFMin32: + return AtomicFloatBits{state, false}.minMax(old, value, false); + case IrOpcode::AddressAtomicFMax32: + return AtomicFloatBits{state, false}.minMax(old, value, true); + case IrOpcode::AddressAtomicFCmpSwap32: + return AtomicFloatBits{state, false}.compareSwap(old, comparator, value); + default: + throw std::runtime_error("FlatAtomicNext32: opcode has no aperture lowering"); + } +} + +std::uint32_t ApertureAtomic32(SpirvValueEmitContext& ctx, const IrValue& inst, ResourceKind kind, std::uint32_t address, std::uint32_t byteOffset, std::uint32_t value, std::uint32_t comparator) { + auto& state = ctx.state; + const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto unaligned = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), byteOffset, ConstantU32(state, 3u)), ConstantU32(state, 0u)); + EmitIfCondition(state, unaligned, [&] { RecordBdaFault(state, address, ConstantU32(state, 4u), instruction, BdaAbi::FaultReason::Unaligned); }); + if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return ConstantU32(state, 0u); + MemoryInfo storage{}; + storage.kind = kind; + const auto resource = PrepareMemoryResourceAccess(state, storage); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteOffset, ConstantU32(state, 2u)); + const auto aligned = Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned); + const auto opcode = inst.Opcode(); + return EmitValueOrZeroIfCondition(state, aligned, [&] { + return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&] { + const auto pointer = EmitMemoryElementPointer(state, resource, index); + if (kind == ResourceKind::Lds) { + return AtomicUpdate(state, pointer, kind, [&](std::uint32_t current) { return FlatAtomicNext32(state, opcode, current, value, comparator); }); + } + const auto old = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), old, pointer); + state.module.AddFunction(spv::OpStore, pointer, FlatAtomicNext32(state, opcode, old, value, comparator)); + return old; + }); + }); +} + +std::uint32_t FlatAtomicNext64(SpirvEmitterState& state, IrOpcode opcode, std::uint32_t old, std::uint32_t value, std::uint32_t comparator) { + const auto u64 = TypeScalarU64(state); + const AtomicFloatBits bits{state, true}; + const auto winner = [&](spv::Op compare) { + return Select(state, u64, Binary(state, compare, TypeBool(state), old, value), old, value); + }; + switch (opcode) { + case IrOpcode::AddressAtomicSwap64: + return value; + case IrOpcode::AddressAtomicCmpSwap64: + return Select(state, u64, Binary(state, spv::OpIEqual, TypeBool(state), old, comparator), value, old); + case IrOpcode::AddressAtomicIAdd64: + return Binary(state, spv::OpIAdd, u64, old, value); + case IrOpcode::AddressAtomicISub64: + return Binary(state, spv::OpISub, u64, old, value); + case IrOpcode::AddressAtomicSMin64: + return winner(spv::OpSLessThan); + case IrOpcode::AddressAtomicUMin64: + return winner(spv::OpULessThan); + case IrOpcode::AddressAtomicSMax64: + return winner(spv::OpSGreaterThan); + case IrOpcode::AddressAtomicUMax64: + return winner(spv::OpUGreaterThan); + case IrOpcode::AddressAtomicAnd64: + return Binary(state, spv::OpBitwiseAnd, u64, old, value); + case IrOpcode::AddressAtomicOr64: + return Binary(state, spv::OpBitwiseOr, u64, old, value); + case IrOpcode::AddressAtomicXor64: + return Binary(state, spv::OpBitwiseXor, u64, old, value); + case IrOpcode::AddressAtomicInc64: + return bits.increment(old, value); + case IrOpcode::AddressAtomicDec64: + return bits.decrement(old, value); + case IrOpcode::AddressAtomicFMin64: + return bits.minMax(old, value, false); + case IrOpcode::AddressAtomicFMax64: + return bits.minMax(old, value, true); + case IrOpcode::AddressAtomicFCmpSwap64: + return bits.compareSwap(old, comparator, value); + default: + throw std::runtime_error("FlatAtomicNext64: opcode has no aperture lowering"); + } +} + +template +std::uint32_t LockedLdsUpdate(SpirvEmitterState& state, TUpdate&& update); + +std::uint32_t ApertureAtomic64(SpirvValueEmitContext& ctx, const IrValue& inst, ResourceKind kind, std::uint32_t address, std::uint32_t byteOffset, std::uint32_t value, std::uint32_t comparator) { + auto& state = ctx.state; + const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto unaligned = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), byteOffset, ConstantU32(state, 7u)), ConstantU32(state, 0u)); + EmitIfCondition(state, unaligned, [&] { RecordBdaFault(state, address, ConstantU32(state, 8u), instruction, BdaAbi::FaultReason::Unaligned); }); + if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return ConstantU64(state, 0u); + if (kind == ResourceKind::Lds && !state.requirements.ldsLock) ctx.Fail(inst, "64-bit LDS atomic reached through a flat address without the workgroup lock"); + MemoryInfo storage{}; + storage.kind = kind; + const auto resource = PrepareMemoryResourceAccess(state, storage); + const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteOffset, ConstantU32(state, 2u)); + const auto high = EmitAddU32(state, low, ConstantU32(state, 1u)); + const auto aligned = Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned); + const auto opcode = inst.Opcode(); + return EmitValueOrDefaultIfCondition(state, aligned, TypeU64(state), ConstantU64(state, 0u), [&] { + return EmitValueOrDefaultIfCondition(state, EmitMemoryElementInBounds(state, resource, high), TypeU64(state), ConstantU64(state, 0u), [&] { + const auto update = [&]() { + const auto lowPointer = EmitMemoryElementPointer(state, resource, low); + const auto highPointer = EmitMemoryElementPointer(state, resource, high); + const auto lowValue = state.module.AllocateId(); + const auto highValue = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), lowValue, lowPointer); + state.module.AddFunction(spv::OpLoad, TypeU32(state), highValue, highPointer); + const auto old = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeU64(state), old, lowValue, highValue); + const auto next = Unary(state, spv::OpBitcast, TypeU64(state), FlatAtomicNext64(state, opcode, Unary(state, spv::OpBitcast, TypeScalarU64(state), old), value, comparator)); + const auto nextLow = state.module.AllocateId(); + const auto nextHigh = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), nextLow, next, 0u); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), nextHigh, next, 1u); + state.module.AddFunction(spv::OpStore, lowPointer, nextLow); + state.module.AddFunction(spv::OpStore, highPointer, nextHigh); + return old; + }; + return kind == ResourceKind::Lds ? LockedLdsUpdate(state, update) : update(); + }); + }); +} + std::uint32_t AddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst) { auto& state = ctx.state; const auto& mem = ctx.Memory(inst); @@ -831,7 +1058,7 @@ std::uint32_t AddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto scalarType = wide ? TypeScalarU64(state) : TypeU32(state); const auto scalar = [&](std::uint32_t value) { return wide ? Unary(state, spv::OpBitcast, scalarType, value) : value; }; const auto value = scalar(ctx.Arg(inst, 3)); - const auto old = EmitBdaAtomic(ctx, inst, GuestAddress(ctx, inst, mem), wide ? 8u : 4u, [&](std::uint32_t pointer) { + const auto operation = [&](std::uint32_t pointer) { if (inst.Opcode() == IrOpcode::AddressAtomicInc32 || inst.Opcode() == IrOpcode::AddressAtomicDec32) { const bool increment = inst.Opcode() == IrOpcode::AddressAtomicInc32; return AtomicUpdate(state, pointer, mem.kind, [&](std::uint32_t current) { @@ -873,7 +1100,17 @@ std::uint32_t AddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst) { } EmitDeviceAtomicMemoryBarrier(state); return result; - }); + }; + if (RoutesApertures(state, mem)) { + const auto address = GuestAddress(ctx, inst, mem); + const bool hasComparator = inst.Opcode() == IrOpcode::AddressAtomicCmpSwap32 || inst.Opcode() == IrOpcode::AddressAtomicFCmpSwap32 || inst.Opcode() == IrOpcode::AddressAtomicCmpSwap64 || inst.Opcode() == IrOpcode::AddressAtomicFCmpSwap64; + const auto comparator = hasComparator ? scalar(ctx.Arg(inst, 4)) : 0u; + if (wide) { + return RouteFlatAccess(state, address, type, [&](ResourceKind kind, std::uint32_t low) { return ApertureAtomic64(ctx, inst, kind, address, low, value, comparator); }, [&] { return Unary(state, spv::OpBitcast, type, EmitBdaAtomic(ctx, inst, address, 8u, operation)); }); + } + return RouteFlatAccess(state, address, TypeU32(state), [&](ResourceKind kind, std::uint32_t low) { return ApertureAtomic32(ctx, inst, kind, address, low, value, comparator); }, [&] { return EmitBdaAtomic(ctx, inst, address, 4u, operation); }); + } + const auto old = EmitBdaAtomic(ctx, inst, GuestAddress(ctx, inst, mem), wide ? 8u : 4u, operation); return wide ? Unary(state, spv::OpBitcast, type, old) : old; }); } diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 90a95b787..224256cbf 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -367,6 +367,9 @@ Throughout the project, every function at every stage either **does exactly what - A [V#, T# or S#](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) loaded through a null or unmapped pointer read from memory is zero when the work is recorded, and the capture throws if no flat SRT slot records that read. - [2D accesses through a 1D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) read the texel the 1D addressing selects and ignore the second coordinate, as simonbbby measured on RDNA2 (gfx1036) in #2237 for `image_sample_lz` and mip-less `image_load` with point filtering and wrap, clamp and border addressing; the PS5's gfx1013 is not measured. Other 2D accesses through a 1D T# (storage, depth, packed, BY loads, mips, derivatives, comparison) still throw. - [Emulated comparison of a 2D-array access through a 2D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) samples the 2D image and ignores the slice, as the native modes of a 2D-array access through a 2D T# already do. This follows the T# type deciding the addressing; it is not measured on hardware for comparison samples +- [Scalar aperture base sources](../../core/shader/recompiler/RdnaDecoder/src/RdnaInstructionDecoder.cpp) (core/shader/recompiler) - `src_shared_base`, `src_shared_limit`, `src_private_base` and `src_private_limit` (scalar source codes 235 to 238) are not modelled: decoding throws `unsupported scalar source operand code` with the code and the program counter, so a shader that reads one does not compile. Operand 235 (`src_shared_base`) was reported on [#1182](https://github.com/boykopovar/AnyPS5/pull/1182) from PPSA26344; the flat aperture routing does not read the base, so a shader that builds its own flat addresses from it stops at decode. +- [Flat aperture bases](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (core/shader/recompiler) - the shared base 0x8000 and private base 0x7000 (the top 16 bits of a flat address, `SharedApertureTop` and `PrivateApertureTop`) are inferred from GTA V's traversal addresses. They are not measured on a console and not taken from SH_MEM_BASES. No public source found confirms them: the AMD KFD driver snapshots found (`kfd_device_queue_manager_cik.c`, `kfd_flat_memory.c`, CIK and VI era) place LDS and scratch by one top address nybble as 4 GB windows with scratch directly after LDS, which does not give a 0x7000 private base. The gfx10 and PS5 layout is unknown. +- [Flat accesses outside the aperture routing](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (core/shader/recompiler) - only compute stages route flat accesses, so graphics-stage flat accesses to the shared or private aperture still go to BDA. In compute, a 64-bit flat atomic whose address is in the shared aperture takes the workgroup lock (`ldsLock`) for its two-dword update, and every such atomic requests that lock, so a shader with one pays for it even on global addresses (the path is exercised by `agc_driver_global_atomics_float`). Inferred, not measured on LDS or scratch: an aperture access that is not inside one dword is done byte by byte, an unaligned aperture atomic records the BDA unaligned fault, and LDS or scratch accesses past their end are dropped or read zero. - On a host subgroup narrower than 32 lanes, [`threadBit`](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) reads an EXEC written as a value (`s_mov_b32`/`s_mov_b64`) at bit (subgroup invocation id) of EXEC_LO, so lanes at or above the subgroup size read a lower lane's bit. - [EXEC read as a value](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) (`s_mov_b64 sN, exec`, the `s_*_saveexec` copy, ALU sources) holds only lanes with a host invocation; the console's also holds the wave's other lanes, e.g. 32-63 of a wave64 on 32 host lanes. - [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) and fetch-inactive `v_permlane16_b32`/`v_permlanex16_b32` read undefined data from a lane that the host subgroup does not hold or whose host invocation is inactive; the console reads the lane's VGPR. From 41144bea53191434e655211b6950c0bf2e34f6e0 Mon Sep 17 00:00:00 2001 From: Michon <231849@buas.nl> Date: Sat, 10 Oct 2026 10:16:15 +0200 Subject: [PATCH 3/5] fix(recompiler): multi-dword FLAT stores route the shared and private apertures Main's StoreAddressU32x2..x4 (one BDA lookup per wide store, b95842597) bypassed the aperture routing: a flat_store_dwordx2 into the shared aperture went to BDA at 0x8000000000000000 and LDS kept its old value. The wide store now stores each component through the aperture like the 32-bit store, and keeps the single BDA lookup outside the apertures. The aperture test's V# word 3 is 0x31016fac (OOB_SELECT 3, raw), as in main's other execution tests: with OOB_SELECT 0 its output buffer read as out of range after #639, which was the failure reported on four drivers. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/execution/FlatApertures.cpp | 2 +- .../SpirvMemory/SpirvMemoryInstructions.cpp | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp index d73947d4d..b33d913ee 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp @@ -96,7 +96,7 @@ private: std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index 3c96d5066..d2b7dd85f 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -1440,7 +1440,25 @@ void StoreAddressWide(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint const auto& mem = ctx.Memory(inst); if (mem.kind != ResourceKind::Flat && mem.kind != ResourceKind::Global) ctx.Fail(inst, "must write a physical address resource"); EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() { - EmitBdaDwordWrites(ctx, inst, GuestAddressBase(ctx, inst, mem), mem.offset, components, ctx.Arg(inst, inst.ArgumentCount() - 2u)); + auto& state = ctx.state; + const auto base = GuestAddressBase(ctx, inst, mem); + const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u); + if (!RoutesApertures(state, mem)) { + EmitBdaDwordWrites(ctx, inst, base, mem.offset, components, data); + return; + } + const auto split = SplitAperture(state, AddBdaImmediate(ctx, inst, base, static_cast(mem.offset))); + const auto storeAperture = [&](ResourceKind kind) { + for (std::uint32_t component = 0; component < components; component++) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, data, component); + StoreApertureElement(ctx, kind, Binary(state, spv::OpIAdd, TypeU32(state), split.low, ConstantU32(state, component * 4u)), 32u, value); + } + }; + EmitIfCondition(state, split.shared, [&] { storeAperture(ResourceKind::Lds); }); + EmitIfCondition(state, split.priv, [&] { storeAperture(ResourceKind::Scratch); }); + const auto global = Unary(state, spv::OpLogicalNot, TypeBool(state), Binary(state, spv::OpLogicalOr, TypeBool(state), split.shared, split.priv)); + EmitIfCondition(state, global, [&] { EmitBdaDwordWrites(ctx, inst, base, mem.offset, components, data); }); }); } From f4f05d2e6a99879d987b86d6540a796fb3b9e676 Mon Sep 17 00:00:00 2001 From: Michon <231849@buas.nl> Date: Sat, 10 Oct 2026 11:58:49 +0200 Subject: [PATCH 4/5] fix(recompiler): flat apertures are 4 GB windows; an aperture access past the end is dropped whole Measured on RDNA2 (gfx1035) in review: - the shared and private apertures are 4 GB windows (the gfx9+ layout of kfd_flat_memory.c), so a flat address routes by its whole high dword, 0x80000000 or 0x70000000, not by its top 16 bits; - a dword or short access whose last byte is past the end of LDS reads 0 or writes nothing as a whole; multi-dword accesses are checked per dword. The aperture test gets rows that straddle and pass the end of LDS; the TechnicalDebt line names the public source of the layout and keeps only the PS5's base values as inferred. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/execution/FlatApertures.cpp | 9 +++- .../SpirvMemory/SpirvMemoryInstructions.cpp | 54 ++++++++++++------- docs/dev/TechnicalDebt.md | 2 +- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp index b33d913ee..ef398d101 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp @@ -352,8 +352,12 @@ void Check(const Row& row, const std::vector& output) { const auto left = output[lane * 8u + 1u]; Require(left == expected, name + ": lane " + std::to_string(lane) + " left " + Hex(left) + ", expected " + Hex(expected)); } else { - const auto expected = row.width == 8u ? (a & 0xffu) : (row.width == 16u ? (a & 0xffffu) : a); const auto read = output[lane * 8u]; + if (row.target == Target::Shared && 8u * lane + row.offset + row.width / 8u > LdsDwords * 4u) { + Require(read == 0u, name + ": lane " + std::to_string(lane) + " read " + Hex(read) + " back from an access past the end of LDS, expected 0 (the store is dropped whole)"); + continue; + } + const auto expected = row.width == 8u ? (a & 0xffu) : (row.width == 16u ? (a & 0xffffu) : a); Require(read == expected, name + ": lane " + std::to_string(lane) + " read " + Hex(read) + " back, expected " + Hex(expected)); const auto first = output[lane * 8u + 1u]; Require(first == (a & 0xffu), name + ": lane " + std::to_string(lane) + " has first byte " + Hex(first) + " in memory, expected " + Hex(a & 0xffu)); @@ -445,6 +449,9 @@ std::vector Rows() { } rows.push_back(Row{Target::Mixed, false, AtomicOp::Swap, 16u, 3u, 32u}); rows.push_back(Row{Target::Mixed, false, AtomicOp::Swap, 32u, 2u, 32u}); + for (const auto [width, offset] : std::array, 5>{{{32u, 773u}, {32u, 774u}, {32u, 776u}, {16u, 775u}, {8u, 775u}}}) { + rows.push_back(Row{Target::Shared, false, AtomicOp::Swap, width, offset, 32u}); + } for (const Target target : single) { for (std::uint32_t op = 0; op < Atomic32Ops; ++op) { rows.push_back(Row{target, true, static_cast(op), 32u, 0u, 32u}); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index d2b7dd85f..94b336e13 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -336,8 +336,8 @@ const MemoryInfo& SharedMemory(SpirvValueEmitContext& ctx, const IrValue& inst) return mem; } -constexpr std::uint32_t SharedApertureTop = 0x8000u; -constexpr std::uint32_t PrivateApertureTop = 0x7000u; +constexpr std::uint32_t SharedApertureHigh = 0x80000000u; +constexpr std::uint32_t PrivateApertureHigh = 0x70000000u; bool RoutesApertures(const SpirvEmitterState& state, const MemoryInfo& mem) { return mem.kind == ResourceKind::Flat && state.program.Resources().stage == IrShaderStage::Compute; @@ -352,8 +352,7 @@ struct ApertureAddress { ApertureAddress SplitAperture(SpirvEmitterState& state, std::uint32_t address) { const auto u32 = TypeU32(state); const auto high = Unary(state, spv::OpUConvert, u32, Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, 32u))); - const auto top = Binary(state, spv::OpShiftRightLogical, u32, high, ConstantU32(state, 16u)); - return {Unary(state, spv::OpUConvert, u32, address), Binary(state, spv::OpIEqual, TypeBool(state), top, ConstantU32(state, SharedApertureTop)), Binary(state, spv::OpIEqual, TypeBool(state), top, ConstantU32(state, PrivateApertureTop))}; + return {Unary(state, spv::OpUConvert, u32, address), Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, SharedApertureHigh)), Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, PrivateApertureHigh))}; } std::uint32_t ApertureByteInBounds(SpirvEmitterState& state, const MemoryResourceAccess& resource, std::uint32_t byteOffset, std::uint32_t offset, std::uint32_t index) { @@ -391,18 +390,32 @@ std::uint32_t LoadApertureByte(SpirvValueEmitContext& ctx, ResourceKind kind, st }); } +std::uint32_t ApertureElementInBounds(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits) { + auto& state = ctx.state; + MemoryInfo storage{}; + storage.kind = kind; + const auto resource = PrepareMemoryResourceAccess(state, storage); + const auto last = bits / 8u - 1u; + const auto address = EmitAddU32(state, byteOffset, ConstantU32(state, last)); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); + return ApertureByteInBounds(state, resource, byteOffset, last, index); +} + std::uint32_t LoadApertureElement(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits) { auto& state = ctx.state; if (bits == 8u) return LoadApertureByte(ctx, kind, byteOffset, 0u); - return EmitValueIfElse(state, ApertureContained(state, byteOffset, bits), TypeU32(state), [&] { - return LoadApertureDword(ctx, kind, byteOffset, bits); - }, [&] { - std::uint32_t value = ConstantU32(state, 0u); - for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) { - const auto part = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), LoadApertureByte(ctx, kind, byteOffset, byte), ConstantU32(state, byte * 8u)); - value = Binary(state, spv::OpBitwiseOr, TypeU32(state), value, part); - } - return value; + if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return ConstantU32(state, 0u); + return EmitValueOrZeroIfCondition(state, ApertureElementInBounds(ctx, kind, byteOffset, bits), [&] { + return EmitValueIfElse(state, ApertureContained(state, byteOffset, bits), TypeU32(state), [&] { + return LoadApertureDword(ctx, kind, byteOffset, bits); + }, [&] { + std::uint32_t value = ConstantU32(state, 0u); + for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) { + const auto part = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), LoadApertureByte(ctx, kind, byteOffset, byte), ConstantU32(state, byte * 8u)); + value = Binary(state, spv::OpBitwiseOr, TypeU32(state), value, part); + } + return value; + }); }); } @@ -439,12 +452,15 @@ void StoreApertureElement(SpirvValueEmitContext& ctx, ResourceKind kind, std::ui StoreApertureByte(ctx, kind, byteOffset, 0u, data); return; } - const auto contained = ApertureContained(state, byteOffset, bits); - EmitIfCondition(state, contained, [&] { StoreApertureDword(ctx, kind, byteOffset, bits, data); }); - EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), contained), [&] { - for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) { - StoreApertureByte(ctx, kind, byteOffset, byte, Binary(state, spv::OpShiftRightLogical, TypeU32(state), data, ConstantU32(state, byte * 8u))); - } + if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return; + EmitIfCondition(state, ApertureElementInBounds(ctx, kind, byteOffset, bits), [&] { + const auto contained = ApertureContained(state, byteOffset, bits); + EmitIfCondition(state, contained, [&] { StoreApertureDword(ctx, kind, byteOffset, bits, data); }); + EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), contained), [&] { + for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) { + StoreApertureByte(ctx, kind, byteOffset, byte, Binary(state, spv::OpShiftRightLogical, TypeU32(state), data, ConstantU32(state, byte * 8u))); + } + }); }); } diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 224256cbf..e7b475b0f 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -368,7 +368,7 @@ Throughout the project, every function at every stage either **does exactly what - [2D accesses through a 1D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) read the texel the 1D addressing selects and ignore the second coordinate, as simonbbby measured on RDNA2 (gfx1036) in #2237 for `image_sample_lz` and mip-less `image_load` with point filtering and wrap, clamp and border addressing; the PS5's gfx1013 is not measured. Other 2D accesses through a 1D T# (storage, depth, packed, BY loads, mips, derivatives, comparison) still throw. - [Emulated comparison of a 2D-array access through a 2D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) samples the 2D image and ignores the slice, as the native modes of a 2D-array access through a 2D T# already do. This follows the T# type deciding the addressing; it is not measured on hardware for comparison samples - [Scalar aperture base sources](../../core/shader/recompiler/RdnaDecoder/src/RdnaInstructionDecoder.cpp) (core/shader/recompiler) - `src_shared_base`, `src_shared_limit`, `src_private_base` and `src_private_limit` (scalar source codes 235 to 238) are not modelled: decoding throws `unsupported scalar source operand code` with the code and the program counter, so a shader that reads one does not compile. Operand 235 (`src_shared_base`) was reported on [#1182](https://github.com/boykopovar/AnyPS5/pull/1182) from PPSA26344; the flat aperture routing does not read the base, so a shader that builds its own flat addresses from it stops at decode. -- [Flat aperture bases](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (core/shader/recompiler) - the shared base 0x8000 and private base 0x7000 (the top 16 bits of a flat address, `SharedApertureTop` and `PrivateApertureTop`) are inferred from GTA V's traversal addresses. They are not measured on a console and not taken from SH_MEM_BASES. No public source found confirms them: the AMD KFD driver snapshots found (`kfd_device_queue_manager_cik.c`, `kfd_flat_memory.c`, CIK and VI era) place LDS and scratch by one top address nybble as 4 GB windows with scratch directly after LDS, which does not give a 0x7000 private base. The gfx10 and PS5 layout is unknown. +- [Flat aperture bases](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (core/shader/recompiler) - the layout follows the gfx9+ one in the AMD KFD driver's `kfd_flat_memory.c` (`MAKE_LDS_APP_BASE_V9`, `MAKE_LDS_APP_LIMIT`): each aperture is a 4 GB window whose base is the high dword, so a flat address is routed by its whole high dword, as measured on RDNA2 (gfx1035: `src_shared_base` 0x00010000_00000000, `src_shared_limit` 0x00010000_ffffffff). The PS5's base values, high dwords 0x80000000 (shared, `SharedApertureHigh`) and 0x70000000 (private, `PrivateApertureHigh`), are inferred from GTA V's traversal addresses, not measured on a console or read from SH_MEM_BASES - [Flat accesses outside the aperture routing](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (core/shader/recompiler) - only compute stages route flat accesses, so graphics-stage flat accesses to the shared or private aperture still go to BDA. In compute, a 64-bit flat atomic whose address is in the shared aperture takes the workgroup lock (`ldsLock`) for its two-dword update, and every such atomic requests that lock, so a shader with one pays for it even on global addresses (the path is exercised by `agc_driver_global_atomics_float`). Inferred, not measured on LDS or scratch: an aperture access that is not inside one dword is done byte by byte, an unaligned aperture atomic records the BDA unaligned fault, and LDS or scratch accesses past their end are dropped or read zero. - On a host subgroup narrower than 32 lanes, [`threadBit`](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) reads an EXEC written as a value (`s_mov_b32`/`s_mov_b64`) at bit (subgroup invocation id) of EXEC_LO, so lanes at or above the subgroup size read a lower lane's bit. - [EXEC read as a value](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) (`s_mov_b64 sN, exec`, the `s_*_saveexec` copy, ALU sources) holds only lanes with a host invocation; the console's also holds the wave's other lanes, e.g. 32-63 of a wave64 on 32 host lanes. From df25847fca395f718e29b81d6418fad19aba0f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Sun, 11 Oct 2026 00:06:53 +0200 Subject: [PATCH 5/5] fix(test): pass the gpu argument to GuestAllocations::Add in FlatApertures --- core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp index ef398d101..810f6a16d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatApertures.cpp @@ -78,7 +78,7 @@ public: #endif Require(block != nullptr, "flat apertures: cannot allocate the guest block"); std::memset(block, 0, GuestBytes); - GuestAllocations::Mutation().Add(block, GuestBytes, true, true); + GuestAllocations::Mutation().Add(block, GuestBytes, true, true, true); } ~GuestBlock() {