From c0348a02bfd14627dc5b57dacb717a431a7e8be4 Mon Sep 17 00:00:00 2001 From: raghav314 Date: Sun, 11 Oct 2026 05:07:36 +0530 Subject: [PATCH] fix(agc): note BDA-written pages at full address width and read back only the claimed ones A shader store through the BDA table records its 4 KiB page in the fault buffer, and the driver marks those pages written after the use. The page number was truncated to 32 bits, so a store above 16 TiB marked a page at another address. And the driver scanned all 4096 slots after every use, even when nothing was stored. Slots now hold a 128 KiB chunk (its number plus one, 32 bits for any address below 512 TiB) and a mask of its 4 KiB pages. A shader that claims a slot appends it to a list, with a count. The driver reads the count, decodes only the claimed slots, marks each run of pages once and clears only those slots. A writable BDA range past 512 TiB is refused when the table is built. Co-Authored-By: Claude Opus 5.5 --- .../Graphics/src/BdaResources.cpp | 20 +++--- .../libSceAgcDriver/tests/BdaExecution.cpp | 69 ++++++++++++++++--- .../tests/execution/FlatStore.cpp | 68 ++++++++++++++++-- core/shader/recompiler/BdaAbi.hpp | 40 ++++++++++- .../SpirvBackend/src/SpirvBdaLookup.cpp | 44 +++++++++--- 5 files changed, 206 insertions(+), 35 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp index 605809a50..0b94d5e0c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp @@ -124,7 +124,7 @@ BdaResources::BdaResources(const Context& context) { static_assert(std::endian::native == std::endian::little); Require(ShaderRecompiler::BdaAbi::FaultBufferBytes <= context.limits.maxStorageBufferRange, "BDA fault buffer exceeds storage buffer range limit"); fault = std::make_unique(context, ShaderRecompiler::BdaAbi::FaultBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); - std::memset(fault->Bytes().data(), 0, fault->Bytes().size()); + std::memset(fault->Bytes().data(), 0, ShaderRecompiler::BdaAbi::WrittenListWord * sizeof(std::uint32_t)); } BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memory) : BdaResources(context) { @@ -192,6 +192,7 @@ BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memo ++it; } } + for (const auto& range : ranges) Require((range.permissions & ShaderRecompiler::BdaAbi::Write) == 0 || range.end <= ShaderRecompiler::BdaAbi::WrittenAddressLimit, "a writable BDA range ends past the addresses the written-page record can hold"); table = std::make_shared(context, tableBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); const ShaderRecompiler::BdaAbi::Header header{ShaderRecompiler::BdaAbi::Version, static_cast(ranges.size()), sizeof(ShaderRecompiler::BdaAbi::Range), 0}; std::memcpy(table->Bytes().data(), &header, sizeof(header)); @@ -276,15 +277,16 @@ namespace AgcDriver::Graphics { void BdaResources::markWrittenPages() const { namespace Abi = ShaderRecompiler::BdaAbi; auto* words = reinterpret_cast(fault->Bytes().data()); - Require(words[Abi::WrittenOverflowWord] == 0, "more than " + std::to_string(Abi::WrittenPageSlots) + " pages stored to through the BDA table in one use are not implemented"); - bool any = false; - for (std::uint32_t slot = 0; slot < Abi::WrittenPageSlots; ++slot) { - const auto page = words[Abi::WrittenSlotsWord + slot]; - if (page == 0) continue; - GuestMemory::MarkWritten(static_cast(page - 1u) << Abi::WrittenPageShift, std::size_t{1} << Abi::WrittenPageShift); - any = true; + Require(words[Abi::WrittenOverflowWord] == 0, "more than " + std::to_string(Abi::WrittenPageSlots) + " chunks of " + std::to_string(Abi::WrittenChunkPages) + " pages stored to through the BDA table in one use are not implemented"); + const auto count = words[Abi::WrittenCountWord]; + if (count == 0) return; + Require(Abi::ForEachWrittenRange(words, [](std::uint64_t address, std::size_t bytes) { GuestMemory::MarkWritten(address, bytes); }), "invalid BDA written-page record"); + for (std::uint32_t claim = 0; claim < count; ++claim) { + const auto slot = words[Abi::WrittenListWord + claim]; + words[Abi::WrittenSlotsWord + 2 * slot] = 0; + words[Abi::WrittenSlotsWord + 2 * slot + 1] = 0; } - if (any) std::memset(words + Abi::WrittenSlotsWord, 0, Abi::WrittenPageSlots * sizeof(std::uint32_t)); + words[Abi::WrittenCountWord] = 0; } } diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaExecution.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaExecution.cpp index 7c4c2adc2..bf7674cc4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaExecution.cpp @@ -90,7 +90,46 @@ private: } +static void RunWrittenRangeDecodeTests() { + namespace Abi = ShaderRecompiler::BdaAbi; + std::vector words(Abi::FaultBufferBytes / sizeof(std::uint32_t)); + std::vector> marked; + const auto decode = [&] { + marked.clear(); + return Abi::ForEachWrittenRange(words.data(), [&](std::uint64_t address, std::size_t bytes) { marked.emplace_back(address, bytes); }); + }; + const auto claim = [&](std::uint32_t index, std::uint32_t slot, std::uint64_t address, std::uint32_t mask) { + words[Abi::WrittenListWord + index] = slot; + words[Abi::WrittenSlotsWord + 2 * slot] = Abi::WrittenChunkKey(address); + words[Abi::WrittenSlotsWord + 2 * slot + 1] = mask; + words[Abi::WrittenCountWord] = std::max(words[Abi::WrittenCountWord], index + 1); + }; + constexpr std::uint64_t page = std::uint64_t{1} << Abi::WrittenPageShift; + Require(decode() && marked.empty(), "an empty written-page record marked pages"); + constexpr std::uint64_t top = 0x7ffffffe0000ULL; + claim(0, 5, top, 0b1011u); + Require(decode() && marked == std::vector>{{top, 2 * page}, {top + 3 * page, page}}, "written pages near the top of the address space were not decoded as written"); + std::fill(words.begin(), words.end(), 0u); + constexpr std::uint64_t high = 0x200000001000ULL; + claim(0, Abi::WrittenPageSlots - 1, high, 1u << ((high >> Abi::WrittenPageShift) & (Abi::WrittenChunkPages - 1u))); + Require(decode() && marked == std::vector>{{high, page}}, "a written page above 16 TiB was decoded at another address"); + std::fill(words.begin(), words.end(), 0u); + constexpr std::uint64_t chunk = 0x3e0000000ULL; + claim(0, 7, chunk, 0xffffffffu); + claim(1, 8, chunk + (std::uint64_t{1} << Abi::WrittenChunkShift), 1u); + claim(2, 9, chunk - (std::uint64_t{1} << Abi::WrittenChunkShift), 0x80000000u); + Require(decode() && marked == std::vector>{{chunk, 32 * page}, {chunk + 32 * page, page}, {chunk - page, page}}, "whole, first-page and last-page chunks were not decoded exactly"); + words[Abi::WrittenListWord + 1] = Abi::WrittenPageSlots; + Require(!decode(), "a claim outside the slots was accepted"); + std::fill(words.begin(), words.end(), 0u); + words[Abi::WrittenCountWord] = 1; + Require(!decode(), "a claim of an empty slot was accepted"); + words[Abi::WrittenCountWord] = Abi::WrittenPageSlots + 1; + Require(!decode(), "more claims than slots were accepted"); +} + static void RunBdaDwordWriteTests(const Context& context) { + RunWrittenRangeDecodeTests(); namespace Abi = ShaderRecompiler::BdaAbi; constexpr std::uint64_t guest = 0x7fff12340f80ULL; constexpr std::size_t firstBytes = 160; @@ -114,20 +153,23 @@ static void RunBdaDwordWriteTests(const Context& context) { } return nullptr; }; - const auto noted = [&](std::uint64_t address) { - const auto page = static_cast(address >> Abi::WrittenPageShift) + 1u; - std::vector slots(Abi::WrittenPageSlots); - std::memcpy(slots.data(), fault.Bytes().data() + Abi::WrittenSlotsWord * sizeof(std::uint32_t), slots.size() * sizeof(std::uint32_t)); - return std::find(slots.begin(), slots.end(), page) != slots.end(); + const auto notedBytes = [&](std::uint64_t first, std::uint64_t last) { + std::uint64_t covered = 0, total = 0; + const auto* faultWords = reinterpret_cast(fault.Bytes().data()); + Require(Abi::ForEachWrittenRange(faultWords, [&](std::uint64_t begin, std::size_t bytes) { + total += bytes; + for (const auto address : {first, last}) covered += address >= begin && address < begin + bytes ? 1u : 0u; + }), "a BDA dword write left an invalid written-page record"); + return std::pair{covered, total}; }; - const auto write = [&](std::uint64_t address, std::uint32_t dwords, std::uint32_t storedBytes, Abi::FaultReason reason) { + const auto write = [&](std::uint64_t address, std::uint32_t dwords, std::uint32_t storedBytes, Abi::FaultReason reason, std::uint32_t groups = 1) { const Abi::Header header{Abi::Version, static_cast(ranges.size()), sizeof(Abi::Range), 0}; std::memcpy(table.Bytes().data(), &header, sizeof(header)); std::memcpy(table.Bytes().data() + sizeof(header), ranges.data(), sizeof(ranges)); std::memset(fault.Bytes().data(), 0, fault.Bytes().size()); for (auto* buffer : {&first, &second, &readOnly}) std::memset(buffer->Bytes().data(), 0xa5, buffer->Bytes().size()); Pipeline pipeline(context, MakeBdaDwordWriteTestShader(address, dwords, values.data()), {&table, &fault, &output}); - pipeline.Run(1); + pipeline.Run(groups); Abi::Fault report{}; std::memcpy(&report, fault.Bytes().data(), sizeof(report)); for (std::uint32_t byte = 0; byte < dwords * 4u; ++byte) { @@ -142,7 +184,17 @@ static void RunBdaDwordWriteTests(const Context& context) { } else { Require(report.state == Abi::FaultState::Ready && report.reason == reason && report.instruction == 0x1234, "a BDA dword write did not publish its fault"); } - if (storedBytes != 0) Require(noted(address) && noted(address + storedBytes - 1u), "a BDA dword write did not note its written pages"); + const auto [covered, total] = notedBytes(address, address + (storedBytes != 0 ? storedBytes - 1u : 0u)); + if (storedBytes == 0) { + Require(total == 0, "a BDA dword write that stored nothing noted pages"); + } else { + const auto pages = ((address + storedBytes - 1u) >> Abi::WrittenPageShift) - (address >> Abi::WrittenPageShift) + 1u; + Require(covered == 2 && total == pages << Abi::WrittenPageShift, "a BDA dword write did not note exactly its written pages"); + const auto chunks = ((address + storedBytes - 1u) >> Abi::WrittenChunkShift) - (address >> Abi::WrittenChunkShift) + 1u; + std::uint32_t claims = 0; + std::memcpy(&claims, fault.Bytes().data() + Abi::WrittenCountWord * sizeof(std::uint32_t), sizeof(claims)); + Require(claims == chunks, "a BDA dword write claimed a written-page slot more than once per chunk"); + } }; write(guest + 16, 4, 16, static_cast(0)); write(guest + 20, 4, 16, static_cast(0)); @@ -154,6 +206,7 @@ static void RunBdaDwordWriteTests(const Context& context) { write(guest + 41, 4, 16, static_cast(0)); write(guest + firstBytes - 8, 4, 16, static_cast(0)); write(guest + firstBytes + 12, 4, 4, Abi::FaultReason::Permission); + write(guest + 0x7c, 2, 8, static_cast(0), 64); } void RunBdaExecutionTests(const Context& context) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp index 259929ce0..77db354f6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp @@ -1,6 +1,8 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" #include "Recompiler.hpp" #include "VulkanTestDevice.hpp" #ifdef _WIN32 @@ -8,6 +10,8 @@ #define NOMINMAX #endif #include +#else +#include #endif #include #include @@ -27,6 +31,9 @@ constexpr std::uint32_t MaskedThreads = 16; constexpr std::size_t BlockBytes = 65536; constexpr std::uint32_t CheckedBytes = 0x2000; constexpr std::uint8_t Fill = 0xcd; +#ifdef __linux__ +constexpr std::array GuestWindowAddresses{0x80000000000, 0x200000000000}; +#endif alignas(256) constexpr std::array FlatStoreCode{ 0x34020082, 0x34040084, 0x340c0081, 0x340e0083, 0x4a060200, 0x7e080201, 0x4a1806ff, 0x00000204, @@ -39,28 +46,54 @@ alignas(256) constexpr std::array FlatStoreCode{ class GuestBlock { public: - explicit GuestBlock(bool writable) { + explicit GuestBlock(bool writable, bool watched = false, std::uintptr_t at = 0) : bytes(watched ? 2 * BlockBytes : BlockBytes), watched(watched) { #ifdef _WIN32 - block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Require(!watched, "flat store: watched guest blocks are mapped only on Linux"); + block = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); #else - block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); + if (watched) { +#ifdef __linux__ + void* raw = mmap(reinterpret_cast(at), bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + Require(raw == reinterpret_cast(at), "flat store: cannot map the watched guest block at its address"); + block = static_cast(raw); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes); +#else + Require(false, "flat store: watched guest blocks are mapped only on Linux"); +#endif + } else { + block = static_cast(std::aligned_alloc(BlockBytes, bytes)); + } #endif Require(block != nullptr, "flat store: cannot allocate the guest block"); + Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "flat store: the guest block is not write-watched"); Clear(); - GuestAllocations::Mutation().Add(block, BlockBytes, true, writable, true); + GuestAllocations::Mutation().Add(block, bytes, true, writable, true); } ~GuestBlock() { GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + if (watched) { + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, bytes); + munmap(block, bytes); + } else { + std::free(block); + } +#endif } GuestBlock(const GuestBlock&) = delete; GuestBlock& operator=(const GuestBlock&) = delete; - void Clear() { std::memset(block, Fill, BlockBytes); } + void Clear() { std::memset(block, Fill, bytes); } const std::uint8_t* Data() const { return block; } + std::uint64_t Address() const { return reinterpret_cast(block); } private: + std::size_t bytes; + bool watched; std::uint8_t* block = nullptr; }; @@ -116,6 +149,23 @@ void RunStores(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t } } +#ifdef __linux__ +void RunStampedBlocks(AgcDriver::VulkanDevice& device, GuestBlock& guest) { + namespace GuestMemory = AgcDriver::GuestMemory; + const auto base = guest.Address(); + Dispatch(device, 32, guest.Data()); + for (int use = 0; use < 2; ++use) { + guest.Clear(); + const auto collected = GuestMemory::CollectWrites(base, 2 * BlockBytes); + Require(collected != 0, "flat store: the watched guest blocks are not collected"); + Require(GuestMemory::UnchangedSince(base, 2 * BlockBytes, collected), "flat store: the guest blocks changed before the store"); + Dispatch(device, 32, guest.Data()); + Require(!GuestMemory::UnchangedSince(base, BlockBytes, collected), "flat store: the stored block is not stamped as written"); + Require(GuestMemory::UnchangedSince(base + BlockBytes, BlockBytes, collected), "flat store: a block no store reached is stamped as written"); + } +} +#endif + void RunReadOnly(AgcDriver::VulkanDevice& device, const GuestBlock& guest) { Dispatch(device, 32, guest.Data()); for (std::uint32_t offset = 0; offset < CheckedBytes; ++offset) { @@ -134,6 +184,14 @@ int main() { RunStores(*device, writable, 32); RunStores(*device, writable, 64); RunReadOnly(*device, readOnly); +#ifdef __linux__ + if (AgcDriver::GuestMemory::WriteWatched()) { + for (const auto at : GuestWindowAddresses) { + GuestBlock watched(true, true, at); + RunStampedBlocks(*device, watched); + } + } +#endif std::puts("flat store tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/shader/recompiler/BdaAbi.hpp b/core/shader/recompiler/BdaAbi.hpp index aa31e43e9..f964d7fe9 100644 --- a/core/shader/recompiler/BdaAbi.hpp +++ b/core/shader/recompiler/BdaAbi.hpp @@ -1,6 +1,7 @@ #ifndef CORE_SHADER_RECOMPILER_BDAABI_HPP #define CORE_SHADER_RECOMPILER_BDAABI_HPP +#include #include #include #include @@ -46,12 +47,45 @@ static_assert(std::is_standard_layout_v && std::is_trivially_copyable_v(address >> WrittenChunkShift) + 1u; +} + +template +bool ForEachWrittenRange(const std::uint32_t* words, TMark&& mark) { + const auto count = words[WrittenCountWord]; + if (count > WrittenPageSlots) return false; + for (std::uint32_t claim = 0; claim < count; ++claim) { + const auto slot = words[WrittenListWord + claim]; + if (slot >= WrittenPageSlots || words[WrittenSlotsWord + 2 * slot] == 0) return false; + } + for (std::uint32_t claim = 0; claim < count; ++claim) { + const auto slot = words[WrittenListWord + claim]; + const auto key = words[WrittenSlotsWord + 2 * slot]; + auto mask = words[WrittenSlotsWord + 2 * slot + 1]; + const auto base = static_cast(key - 1u) << WrittenChunkShift; + while (mask != 0) { + const auto first = static_cast(std::countr_zero(mask)); + const auto pages = static_cast(std::countr_one(mask >> first)); + mark(base + (static_cast(first) << WrittenPageShift), static_cast(pages) << WrittenPageShift); + mask = pages + first >= 32u ? 0u : mask & ~((1u << (first + pages)) - 1u); + } + } + return true; +} } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp index 8cc0fa6d2..f4290dcf5 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp @@ -132,10 +132,17 @@ static std::uint32_t DefineBdaNoteWrite(SpirvEmitterState& state) { state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, boolean), done, spv::StorageClassFunction); const auto counter = state.module.AllocateId(); state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, u32), counter, spv::StorageClassFunction); + const auto held = state.module.AllocateId(); + state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, u32), held, spv::StorageClassFunction); state.module.AddFunction(spv::OpStore, done, ConstantBool(state, false)); state.module.AddFunction(spv::OpStore, counter, constant(0u)); - const auto page = binary(spv::OpIAdd, u32, Unary(state, spv::OpUConvert, u32, binary(spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, BdaAbi::WrittenPageShift))), constant(1)); - const auto hash = binary(spv::OpShiftRightLogical, u32, binary(spv::OpIMul, u32, page, constant(0x9e3779b1u)), constant(32u - static_cast(std::countr_zero(BdaAbi::WrittenPageSlots)))); + state.module.AddFunction(spv::OpStore, held, constant(0u)); + const auto u64 = TypeScalarU64(state); + const auto key = binary(spv::OpIAdd, u32, Unary(state, spv::OpUConvert, u32, binary(spv::OpShiftRightLogical, u64, address, BdaConstant(state, BdaAbi::WrittenChunkShift))), constant(1)); + const auto page = binary(spv::OpBitwiseAnd, u32, Unary(state, spv::OpUConvert, u32, binary(spv::OpShiftRightLogical, u64, address, BdaConstant(state, BdaAbi::WrittenPageShift))), constant(BdaAbi::WrittenChunkPages - 1u)); + const auto bit = binary(spv::OpShiftLeftLogical, u32, constant(1u), page); + const auto hash = binary(spv::OpShiftRightLogical, u32, binary(spv::OpIMul, u32, key, constant(0x9e3779b1u)), constant(32u - static_cast(std::countr_zero(BdaAbi::WrittenPageSlots)))); + const auto keyWordOf = [&](std::uint32_t slot) { return binary(spv::OpIAdd, u32, constant(BdaAbi::WrittenSlotsWord), binary(spv::OpShiftLeftLogical, u32, slot, constant(1u))); }; const auto scope = constant(spv::ScopeDevice); const auto relaxed = constant(spv::MemorySemanticsMaskNone); const auto header = state.module.AllocateId(); @@ -152,16 +159,23 @@ static std::uint32_t DefineBdaNoteWrite(SpirvEmitterState& state) { state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone); state.module.AddFunction(spv::OpBranchConditional, searching, body, merge); EmitLabel(state, body); - const auto slot = binary(spv::OpIAdd, u32, constant(BdaAbi::WrittenSlotsWord), binary(spv::OpBitwiseAnd, u32, binary(spv::OpIAdd, u32, hash, probe), constant(BdaAbi::WrittenPageSlots - 1u))); + const auto slot = binary(spv::OpBitwiseAnd, u32, binary(spv::OpIAdd, u32, hash, probe), constant(BdaAbi::WrittenPageSlots - 1u)); + const auto keyWord = keyWordOf(slot); + state.module.AddFunction(spv::OpStore, held, slot); const auto current = state.module.AllocateId(); - state.module.AddFunction(spv::OpAtomicLoad, u32, current, BdaWord(state, state.faultBufferVariable, slot), scope, relaxed); - const auto held = binary(spv::OpIEqual, boolean, current, page); - state.module.AddFunction(spv::OpStore, done, held); - EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, held), [&] { + state.module.AddFunction(spv::OpAtomicLoad, u32, current, BdaWord(state, state.faultBufferVariable, keyWord), scope, relaxed); + const auto same = binary(spv::OpIEqual, boolean, current, key); + state.module.AddFunction(spv::OpStore, done, same); + EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, same), [&] { const auto previous = state.module.AllocateId(); - state.module.AddFunction(spv::OpAtomicCompareExchange, u32, previous, BdaWord(state, state.faultBufferVariable, slot), scope, relaxed, relaxed, page, constant(0u)); - const auto taken = binary(spv::OpLogicalOr, boolean, binary(spv::OpIEqual, boolean, previous, constant(0u)), binary(spv::OpIEqual, boolean, previous, page)); - state.module.AddFunction(spv::OpStore, done, taken); + state.module.AddFunction(spv::OpAtomicCompareExchange, u32, previous, BdaWord(state, state.faultBufferVariable, keyWord), scope, relaxed, relaxed, key, constant(0u)); + const auto claimed = binary(spv::OpIEqual, boolean, previous, constant(0u)); + state.module.AddFunction(spv::OpStore, done, binary(spv::OpLogicalOr, boolean, claimed, binary(spv::OpIEqual, boolean, previous, key))); + EmitIfCondition(state, claimed, [&] { + const auto index = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicIAdd, u32, index, BdaWord(state, state.faultBufferVariable, constant(BdaAbi::WrittenCountWord)), scope, relaxed, constant(1u)); + state.module.AddFunction(spv::OpAtomicStore, BdaWord(state, state.faultBufferVariable, binary(spv::OpIAdd, u32, constant(BdaAbi::WrittenListWord), index)), scope, relaxed, slot); + }); }); state.module.AddFunction(spv::OpBranch, continuation); EmitLabel(state, continuation); @@ -170,6 +184,16 @@ static std::uint32_t DefineBdaNoteWrite(SpirvEmitterState& state) { EmitLabel(state, merge); const auto noted = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, boolean, noted, done); + EmitIfCondition(state, noted, [&] { + const auto heldSlot = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, u32, heldSlot, held); + const auto maskWord = binary(spv::OpIAdd, u32, keyWordOf(heldSlot), constant(1u)); + const auto mask = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicLoad, u32, mask, BdaWord(state, state.faultBufferVariable, maskWord), scope, relaxed); + EmitIfCondition(state, binary(spv::OpIEqual, boolean, binary(spv::OpBitwiseAnd, u32, mask, bit), constant(0u)), [&] { + state.module.AddFunction(spv::OpAtomicOr, u32, state.module.AllocateId(), BdaWord(state, state.faultBufferVariable, maskWord), scope, relaxed, bit); + }); + }); EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, noted), [&] { state.module.AddFunction(spv::OpAtomicStore, BdaWord(state, state.faultBufferVariable, constant(BdaAbi::WrittenOverflowWord)), scope, relaxed, constant(1u)); });