fix(agc): note BDA-written pages at full address width and read back only the claimed ones
A shader store through the BDA table records its 4 KiB page in the fault buffer, and the driver marks those pages written after the use. The page number was truncated to 32 bits, so a store above 16 TiB marked a page at another address. And the driver scanned all 4096 slots after every use, even when nothing was stored. Slots now hold a 128 KiB chunk (its number plus one, 32 bits for any address below 512 TiB) and a mask of its 4 KiB pages. A shader that claims a slot appends it to a list, with a count. The driver reads the count, decodes only the claimed slots, marks each run of pages once and clears only those slots. A writable BDA range past 512 TiB is refused when the table is built. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
raghav314
parent
6968564656
commit
c0348a02bf
|
|
@ -124,7 +124,7 @@ BdaResources::BdaResources(const Context& context) {
|
|||
static_assert(std::endian::native == std::endian::little);
|
||||
Require(ShaderRecompiler::BdaAbi::FaultBufferBytes <= context.limits.maxStorageBufferRange, "BDA fault buffer exceeds storage buffer range limit");
|
||||
fault = std::make_unique<Buffer>(context, ShaderRecompiler::BdaAbi::FaultBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
|
||||
std::memset(fault->Bytes().data(), 0, fault->Bytes().size());
|
||||
std::memset(fault->Bytes().data(), 0, ShaderRecompiler::BdaAbi::WrittenListWord * sizeof(std::uint32_t));
|
||||
}
|
||||
|
||||
BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memory) : BdaResources(context) {
|
||||
|
|
@ -192,6 +192,7 @@ BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memo
|
|||
++it;
|
||||
}
|
||||
}
|
||||
for (const auto& range : ranges) Require((range.permissions & ShaderRecompiler::BdaAbi::Write) == 0 || range.end <= ShaderRecompiler::BdaAbi::WrittenAddressLimit, "a writable BDA range ends past the addresses the written-page record can hold");
|
||||
table = std::make_shared<Buffer>(context, tableBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
|
||||
const ShaderRecompiler::BdaAbi::Header header{ShaderRecompiler::BdaAbi::Version, static_cast<std::uint32_t>(ranges.size()), sizeof(ShaderRecompiler::BdaAbi::Range), 0};
|
||||
std::memcpy(table->Bytes().data(), &header, sizeof(header));
|
||||
|
|
@ -276,15 +277,16 @@ namespace AgcDriver::Graphics {
|
|||
void BdaResources::markWrittenPages() const {
|
||||
namespace Abi = ShaderRecompiler::BdaAbi;
|
||||
auto* words = reinterpret_cast<std::uint32_t*>(fault->Bytes().data());
|
||||
Require(words[Abi::WrittenOverflowWord] == 0, "more than " + std::to_string(Abi::WrittenPageSlots) + " pages stored to through the BDA table in one use are not implemented");
|
||||
bool any = false;
|
||||
for (std::uint32_t slot = 0; slot < Abi::WrittenPageSlots; ++slot) {
|
||||
const auto page = words[Abi::WrittenSlotsWord + slot];
|
||||
if (page == 0) continue;
|
||||
GuestMemory::MarkWritten(static_cast<std::uint64_t>(page - 1u) << Abi::WrittenPageShift, std::size_t{1} << Abi::WrittenPageShift);
|
||||
any = true;
|
||||
Require(words[Abi::WrittenOverflowWord] == 0, "more than " + std::to_string(Abi::WrittenPageSlots) + " chunks of " + std::to_string(Abi::WrittenChunkPages) + " pages stored to through the BDA table in one use are not implemented");
|
||||
const auto count = words[Abi::WrittenCountWord];
|
||||
if (count == 0) return;
|
||||
Require(Abi::ForEachWrittenRange(words, [](std::uint64_t address, std::size_t bytes) { GuestMemory::MarkWritten(address, bytes); }), "invalid BDA written-page record");
|
||||
for (std::uint32_t claim = 0; claim < count; ++claim) {
|
||||
const auto slot = words[Abi::WrittenListWord + claim];
|
||||
words[Abi::WrittenSlotsWord + 2 * slot] = 0;
|
||||
words[Abi::WrittenSlotsWord + 2 * slot + 1] = 0;
|
||||
}
|
||||
if (any) std::memset(words + Abi::WrittenSlotsWord, 0, Abi::WrittenPageSlots * sizeof(std::uint32_t));
|
||||
words[Abi::WrittenCountWord] = 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,7 +90,46 @@ private:
|
|||
|
||||
}
|
||||
|
||||
static void RunWrittenRangeDecodeTests() {
|
||||
namespace Abi = ShaderRecompiler::BdaAbi;
|
||||
std::vector<std::uint32_t> words(Abi::FaultBufferBytes / sizeof(std::uint32_t));
|
||||
std::vector<std::pair<std::uint64_t, std::size_t>> marked;
|
||||
const auto decode = [&] {
|
||||
marked.clear();
|
||||
return Abi::ForEachWrittenRange(words.data(), [&](std::uint64_t address, std::size_t bytes) { marked.emplace_back(address, bytes); });
|
||||
};
|
||||
const auto claim = [&](std::uint32_t index, std::uint32_t slot, std::uint64_t address, std::uint32_t mask) {
|
||||
words[Abi::WrittenListWord + index] = slot;
|
||||
words[Abi::WrittenSlotsWord + 2 * slot] = Abi::WrittenChunkKey(address);
|
||||
words[Abi::WrittenSlotsWord + 2 * slot + 1] = mask;
|
||||
words[Abi::WrittenCountWord] = std::max(words[Abi::WrittenCountWord], index + 1);
|
||||
};
|
||||
constexpr std::uint64_t page = std::uint64_t{1} << Abi::WrittenPageShift;
|
||||
Require(decode() && marked.empty(), "an empty written-page record marked pages");
|
||||
constexpr std::uint64_t top = 0x7ffffffe0000ULL;
|
||||
claim(0, 5, top, 0b1011u);
|
||||
Require(decode() && marked == std::vector<std::pair<std::uint64_t, std::size_t>>{{top, 2 * page}, {top + 3 * page, page}}, "written pages near the top of the address space were not decoded as written");
|
||||
std::fill(words.begin(), words.end(), 0u);
|
||||
constexpr std::uint64_t high = 0x200000001000ULL;
|
||||
claim(0, Abi::WrittenPageSlots - 1, high, 1u << ((high >> Abi::WrittenPageShift) & (Abi::WrittenChunkPages - 1u)));
|
||||
Require(decode() && marked == std::vector<std::pair<std::uint64_t, std::size_t>>{{high, page}}, "a written page above 16 TiB was decoded at another address");
|
||||
std::fill(words.begin(), words.end(), 0u);
|
||||
constexpr std::uint64_t chunk = 0x3e0000000ULL;
|
||||
claim(0, 7, chunk, 0xffffffffu);
|
||||
claim(1, 8, chunk + (std::uint64_t{1} << Abi::WrittenChunkShift), 1u);
|
||||
claim(2, 9, chunk - (std::uint64_t{1} << Abi::WrittenChunkShift), 0x80000000u);
|
||||
Require(decode() && marked == std::vector<std::pair<std::uint64_t, std::size_t>>{{chunk, 32 * page}, {chunk + 32 * page, page}, {chunk - page, page}}, "whole, first-page and last-page chunks were not decoded exactly");
|
||||
words[Abi::WrittenListWord + 1] = Abi::WrittenPageSlots;
|
||||
Require(!decode(), "a claim outside the slots was accepted");
|
||||
std::fill(words.begin(), words.end(), 0u);
|
||||
words[Abi::WrittenCountWord] = 1;
|
||||
Require(!decode(), "a claim of an empty slot was accepted");
|
||||
words[Abi::WrittenCountWord] = Abi::WrittenPageSlots + 1;
|
||||
Require(!decode(), "more claims than slots were accepted");
|
||||
}
|
||||
|
||||
static void RunBdaDwordWriteTests(const Context& context) {
|
||||
RunWrittenRangeDecodeTests();
|
||||
namespace Abi = ShaderRecompiler::BdaAbi;
|
||||
constexpr std::uint64_t guest = 0x7fff12340f80ULL;
|
||||
constexpr std::size_t firstBytes = 160;
|
||||
|
|
@ -114,20 +153,23 @@ static void RunBdaDwordWriteTests(const Context& context) {
|
|||
}
|
||||
return nullptr;
|
||||
};
|
||||
const auto noted = [&](std::uint64_t address) {
|
||||
const auto page = static_cast<std::uint32_t>(address >> Abi::WrittenPageShift) + 1u;
|
||||
std::vector<std::uint32_t> slots(Abi::WrittenPageSlots);
|
||||
std::memcpy(slots.data(), fault.Bytes().data() + Abi::WrittenSlotsWord * sizeof(std::uint32_t), slots.size() * sizeof(std::uint32_t));
|
||||
return std::find(slots.begin(), slots.end(), page) != slots.end();
|
||||
const auto notedBytes = [&](std::uint64_t first, std::uint64_t last) {
|
||||
std::uint64_t covered = 0, total = 0;
|
||||
const auto* faultWords = reinterpret_cast<const std::uint32_t*>(fault.Bytes().data());
|
||||
Require(Abi::ForEachWrittenRange(faultWords, [&](std::uint64_t begin, std::size_t bytes) {
|
||||
total += bytes;
|
||||
for (const auto address : {first, last}) covered += address >= begin && address < begin + bytes ? 1u : 0u;
|
||||
}), "a BDA dword write left an invalid written-page record");
|
||||
return std::pair{covered, total};
|
||||
};
|
||||
const auto write = [&](std::uint64_t address, std::uint32_t dwords, std::uint32_t storedBytes, Abi::FaultReason reason) {
|
||||
const auto write = [&](std::uint64_t address, std::uint32_t dwords, std::uint32_t storedBytes, Abi::FaultReason reason, std::uint32_t groups = 1) {
|
||||
const Abi::Header header{Abi::Version, static_cast<std::uint32_t>(ranges.size()), sizeof(Abi::Range), 0};
|
||||
std::memcpy(table.Bytes().data(), &header, sizeof(header));
|
||||
std::memcpy(table.Bytes().data() + sizeof(header), ranges.data(), sizeof(ranges));
|
||||
std::memset(fault.Bytes().data(), 0, fault.Bytes().size());
|
||||
for (auto* buffer : {&first, &second, &readOnly}) std::memset(buffer->Bytes().data(), 0xa5, buffer->Bytes().size());
|
||||
Pipeline pipeline(context, MakeBdaDwordWriteTestShader(address, dwords, values.data()), {&table, &fault, &output});
|
||||
pipeline.Run(1);
|
||||
pipeline.Run(groups);
|
||||
Abi::Fault report{};
|
||||
std::memcpy(&report, fault.Bytes().data(), sizeof(report));
|
||||
for (std::uint32_t byte = 0; byte < dwords * 4u; ++byte) {
|
||||
|
|
@ -142,7 +184,17 @@ static void RunBdaDwordWriteTests(const Context& context) {
|
|||
} else {
|
||||
Require(report.state == Abi::FaultState::Ready && report.reason == reason && report.instruction == 0x1234, "a BDA dword write did not publish its fault");
|
||||
}
|
||||
if (storedBytes != 0) Require(noted(address) && noted(address + storedBytes - 1u), "a BDA dword write did not note its written pages");
|
||||
const auto [covered, total] = notedBytes(address, address + (storedBytes != 0 ? storedBytes - 1u : 0u));
|
||||
if (storedBytes == 0) {
|
||||
Require(total == 0, "a BDA dword write that stored nothing noted pages");
|
||||
} else {
|
||||
const auto pages = ((address + storedBytes - 1u) >> Abi::WrittenPageShift) - (address >> Abi::WrittenPageShift) + 1u;
|
||||
Require(covered == 2 && total == pages << Abi::WrittenPageShift, "a BDA dword write did not note exactly its written pages");
|
||||
const auto chunks = ((address + storedBytes - 1u) >> Abi::WrittenChunkShift) - (address >> Abi::WrittenChunkShift) + 1u;
|
||||
std::uint32_t claims = 0;
|
||||
std::memcpy(&claims, fault.Bytes().data() + Abi::WrittenCountWord * sizeof(std::uint32_t), sizeof(claims));
|
||||
Require(claims == chunks, "a BDA dword write claimed a written-page slot more than once per chunk");
|
||||
}
|
||||
};
|
||||
write(guest + 16, 4, 16, static_cast<Abi::FaultReason>(0));
|
||||
write(guest + 20, 4, 16, static_cast<Abi::FaultReason>(0));
|
||||
|
|
@ -154,6 +206,7 @@ static void RunBdaDwordWriteTests(const Context& context) {
|
|||
write(guest + 41, 4, 16, static_cast<Abi::FaultReason>(0));
|
||||
write(guest + firstBytes - 8, 4, 16, static_cast<Abi::FaultReason>(0));
|
||||
write(guest + firstBytes + 12, 4, 4, Abi::FaultReason::Permission);
|
||||
write(guest + 0x7c, 2, 8, static_cast<Abi::FaultReason>(0), 64);
|
||||
}
|
||||
|
||||
void RunBdaExecutionTests(const Context& context) {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
|
||||
#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
|
||||
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
|
||||
#include "prx/libc/include/GuestAllocations.hpp"
|
||||
#include "prx/libc/include/GuestWriteWatch.hpp"
|
||||
#include "Recompiler.hpp"
|
||||
#include "VulkanTestDevice.hpp"
|
||||
#ifdef _WIN32
|
||||
|
|
@ -8,6 +10,8 @@
|
|||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
#include <array>
|
||||
#include <cstdio>
|
||||
|
|
@ -27,6 +31,9 @@ constexpr std::uint32_t MaskedThreads = 16;
|
|||
constexpr std::size_t BlockBytes = 65536;
|
||||
constexpr std::uint32_t CheckedBytes = 0x2000;
|
||||
constexpr std::uint8_t Fill = 0xcd;
|
||||
#ifdef __linux__
|
||||
constexpr std::array<std::uintptr_t, 2> GuestWindowAddresses{0x80000000000, 0x200000000000};
|
||||
#endif
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 45> FlatStoreCode{
|
||||
0x34020082, 0x34040084, 0x340c0081, 0x340e0083, 0x4a060200, 0x7e080201, 0x4a1806ff, 0x00000204,
|
||||
|
|
@ -39,28 +46,54 @@ alignas(256) constexpr std::array<std::uint32_t, 45> FlatStoreCode{
|
|||
|
||||
class GuestBlock {
|
||||
public:
|
||||
explicit GuestBlock(bool writable) {
|
||||
explicit GuestBlock(bool writable, bool watched = false, std::uintptr_t at = 0) : bytes(watched ? 2 * BlockBytes : BlockBytes), watched(watched) {
|
||||
#ifdef _WIN32
|
||||
block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
|
||||
Require(!watched, "flat store: watched guest blocks are mapped only on Linux");
|
||||
block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
|
||||
#else
|
||||
block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
|
||||
if (watched) {
|
||||
#ifdef __linux__
|
||||
void* raw = mmap(reinterpret_cast<void*>(at), bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0);
|
||||
Require(raw == reinterpret_cast<void*>(at), "flat store: cannot map the watched guest block at its address");
|
||||
block = static_cast<std::uint8_t*>(raw);
|
||||
GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes);
|
||||
#else
|
||||
Require(false, "flat store: watched guest blocks are mapped only on Linux");
|
||||
#endif
|
||||
} else {
|
||||
block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
|
||||
}
|
||||
#endif
|
||||
Require(block != nullptr, "flat store: cannot allocate the guest block");
|
||||
Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "flat store: the guest block is not write-watched");
|
||||
Clear();
|
||||
GuestAllocations::Mutation().Add(block, BlockBytes, true, writable, true);
|
||||
GuestAllocations::Mutation().Add(block, bytes, true, writable, true);
|
||||
}
|
||||
|
||||
~GuestBlock() {
|
||||
GuestAllocations::Mutation().Remove(block);
|
||||
#ifdef _WIN32
|
||||
VirtualFree(block, 0, MEM_RELEASE);
|
||||
#else
|
||||
if (watched) {
|
||||
GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, bytes);
|
||||
munmap(block, bytes);
|
||||
} else {
|
||||
std::free(block);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
GuestBlock(const GuestBlock&) = delete;
|
||||
GuestBlock& operator=(const GuestBlock&) = delete;
|
||||
|
||||
void Clear() { std::memset(block, Fill, BlockBytes); }
|
||||
void Clear() { std::memset(block, Fill, bytes); }
|
||||
const std::uint8_t* Data() const { return block; }
|
||||
std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(block); }
|
||||
|
||||
private:
|
||||
std::size_t bytes;
|
||||
bool watched;
|
||||
std::uint8_t* block = nullptr;
|
||||
};
|
||||
|
||||
|
|
@ -116,6 +149,23 @@ void RunStores(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t
|
|||
}
|
||||
}
|
||||
|
||||
#ifdef __linux__
|
||||
void RunStampedBlocks(AgcDriver::VulkanDevice& device, GuestBlock& guest) {
|
||||
namespace GuestMemory = AgcDriver::GuestMemory;
|
||||
const auto base = guest.Address();
|
||||
Dispatch(device, 32, guest.Data());
|
||||
for (int use = 0; use < 2; ++use) {
|
||||
guest.Clear();
|
||||
const auto collected = GuestMemory::CollectWrites(base, 2 * BlockBytes);
|
||||
Require(collected != 0, "flat store: the watched guest blocks are not collected");
|
||||
Require(GuestMemory::UnchangedSince(base, 2 * BlockBytes, collected), "flat store: the guest blocks changed before the store");
|
||||
Dispatch(device, 32, guest.Data());
|
||||
Require(!GuestMemory::UnchangedSince(base, BlockBytes, collected), "flat store: the stored block is not stamped as written");
|
||||
Require(GuestMemory::UnchangedSince(base + BlockBytes, BlockBytes, collected), "flat store: a block no store reached is stamped as written");
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
void RunReadOnly(AgcDriver::VulkanDevice& device, const GuestBlock& guest) {
|
||||
Dispatch(device, 32, guest.Data());
|
||||
for (std::uint32_t offset = 0; offset < CheckedBytes; ++offset) {
|
||||
|
|
@ -134,6 +184,14 @@ int main() {
|
|||
RunStores(*device, writable, 32);
|
||||
RunStores(*device, writable, 64);
|
||||
RunReadOnly(*device, readOnly);
|
||||
#ifdef __linux__
|
||||
if (AgcDriver::GuestMemory::WriteWatched()) {
|
||||
for (const auto at : GuestWindowAddresses) {
|
||||
GuestBlock watched(true, true, at);
|
||||
RunStampedBlocks(*device, watched);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
std::puts("flat store tests passed");
|
||||
return 0;
|
||||
} catch (const std::exception& error) {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#ifndef CORE_SHADER_RECOMPILER_BDAABI_HPP
|
||||
#define CORE_SHADER_RECOMPILER_BDAABI_HPP
|
||||
|
||||
#include <bit>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <type_traits>
|
||||
|
|
@ -46,12 +47,45 @@ static_assert(std::is_standard_layout_v<Fault> && std::is_trivially_copyable_v<F
|
|||
static_assert(offsetof(Fault, address) == 8 && offsetof(Fault, instruction) == 24);
|
||||
|
||||
inline constexpr std::uint32_t WrittenPageShift = 12;
|
||||
inline constexpr std::uint32_t WrittenPageSlots = 4096;
|
||||
inline constexpr std::uint32_t WrittenChunkShift = 17;
|
||||
inline constexpr std::uint32_t WrittenChunkPages = 1u << (WrittenChunkShift - WrittenPageShift);
|
||||
inline constexpr std::uint32_t WrittenPageSlots = 2048;
|
||||
inline constexpr std::uint32_t WrittenPageProbes = 8;
|
||||
inline constexpr std::uint64_t WrittenAddressLimit = std::uint64_t{0xffffffffu} << WrittenChunkShift;
|
||||
inline constexpr std::uint32_t WrittenOverflowWord = sizeof(Fault) / sizeof(std::uint32_t);
|
||||
inline constexpr std::uint32_t WrittenSlotsWord = WrittenOverflowWord + 1;
|
||||
inline constexpr std::size_t FaultBufferBytes = (WrittenSlotsWord + WrittenPageSlots) * sizeof(std::uint32_t);
|
||||
inline constexpr std::uint32_t WrittenCountWord = WrittenOverflowWord + 1;
|
||||
inline constexpr std::uint32_t WrittenSlotsWord = WrittenCountWord + 1;
|
||||
inline constexpr std::uint32_t WrittenListWord = WrittenSlotsWord + 2 * WrittenPageSlots;
|
||||
inline constexpr std::size_t FaultBufferBytes = (WrittenListWord + WrittenPageSlots) * sizeof(std::uint32_t);
|
||||
static_assert((WrittenPageSlots & (WrittenPageSlots - 1)) == 0);
|
||||
static_assert(WrittenChunkPages == 32);
|
||||
|
||||
inline constexpr std::uint32_t WrittenChunkKey(std::uint64_t address) {
|
||||
return static_cast<std::uint32_t>(address >> WrittenChunkShift) + 1u;
|
||||
}
|
||||
|
||||
template<typename TMark>
|
||||
bool ForEachWrittenRange(const std::uint32_t* words, TMark&& mark) {
|
||||
const auto count = words[WrittenCountWord];
|
||||
if (count > WrittenPageSlots) return false;
|
||||
for (std::uint32_t claim = 0; claim < count; ++claim) {
|
||||
const auto slot = words[WrittenListWord + claim];
|
||||
if (slot >= WrittenPageSlots || words[WrittenSlotsWord + 2 * slot] == 0) return false;
|
||||
}
|
||||
for (std::uint32_t claim = 0; claim < count; ++claim) {
|
||||
const auto slot = words[WrittenListWord + claim];
|
||||
const auto key = words[WrittenSlotsWord + 2 * slot];
|
||||
auto mask = words[WrittenSlotsWord + 2 * slot + 1];
|
||||
const auto base = static_cast<std::uint64_t>(key - 1u) << WrittenChunkShift;
|
||||
while (mask != 0) {
|
||||
const auto first = static_cast<std::uint32_t>(std::countr_zero(mask));
|
||||
const auto pages = static_cast<std::uint32_t>(std::countr_one(mask >> first));
|
||||
mark(base + (static_cast<std::uint64_t>(first) << WrittenPageShift), static_cast<std::size_t>(pages) << WrittenPageShift);
|
||||
mask = pages + first >= 32u ? 0u : mask & ~((1u << (first + pages)) - 1u);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -132,10 +132,17 @@ static std::uint32_t DefineBdaNoteWrite(SpirvEmitterState& state) {
|
|||
state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, boolean), done, spv::StorageClassFunction);
|
||||
const auto counter = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, u32), counter, spv::StorageClassFunction);
|
||||
const auto held = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, u32), held, spv::StorageClassFunction);
|
||||
state.module.AddFunction(spv::OpStore, done, ConstantBool(state, false));
|
||||
state.module.AddFunction(spv::OpStore, counter, constant(0u));
|
||||
const auto page = binary(spv::OpIAdd, u32, Unary(state, spv::OpUConvert, u32, binary(spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, BdaAbi::WrittenPageShift))), constant(1));
|
||||
const auto hash = binary(spv::OpShiftRightLogical, u32, binary(spv::OpIMul, u32, page, constant(0x9e3779b1u)), constant(32u - static_cast<std::uint32_t>(std::countr_zero(BdaAbi::WrittenPageSlots))));
|
||||
state.module.AddFunction(spv::OpStore, held, constant(0u));
|
||||
const auto u64 = TypeScalarU64(state);
|
||||
const auto key = binary(spv::OpIAdd, u32, Unary(state, spv::OpUConvert, u32, binary(spv::OpShiftRightLogical, u64, address, BdaConstant(state, BdaAbi::WrittenChunkShift))), constant(1));
|
||||
const auto page = binary(spv::OpBitwiseAnd, u32, Unary(state, spv::OpUConvert, u32, binary(spv::OpShiftRightLogical, u64, address, BdaConstant(state, BdaAbi::WrittenPageShift))), constant(BdaAbi::WrittenChunkPages - 1u));
|
||||
const auto bit = binary(spv::OpShiftLeftLogical, u32, constant(1u), page);
|
||||
const auto hash = binary(spv::OpShiftRightLogical, u32, binary(spv::OpIMul, u32, key, constant(0x9e3779b1u)), constant(32u - static_cast<std::uint32_t>(std::countr_zero(BdaAbi::WrittenPageSlots))));
|
||||
const auto keyWordOf = [&](std::uint32_t slot) { return binary(spv::OpIAdd, u32, constant(BdaAbi::WrittenSlotsWord), binary(spv::OpShiftLeftLogical, u32, slot, constant(1u))); };
|
||||
const auto scope = constant(spv::ScopeDevice);
|
||||
const auto relaxed = constant(spv::MemorySemanticsMaskNone);
|
||||
const auto header = state.module.AllocateId();
|
||||
|
|
@ -152,16 +159,23 @@ static std::uint32_t DefineBdaNoteWrite(SpirvEmitterState& state) {
|
|||
state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone);
|
||||
state.module.AddFunction(spv::OpBranchConditional, searching, body, merge);
|
||||
EmitLabel(state, body);
|
||||
const auto slot = binary(spv::OpIAdd, u32, constant(BdaAbi::WrittenSlotsWord), binary(spv::OpBitwiseAnd, u32, binary(spv::OpIAdd, u32, hash, probe), constant(BdaAbi::WrittenPageSlots - 1u)));
|
||||
const auto slot = binary(spv::OpBitwiseAnd, u32, binary(spv::OpIAdd, u32, hash, probe), constant(BdaAbi::WrittenPageSlots - 1u));
|
||||
const auto keyWord = keyWordOf(slot);
|
||||
state.module.AddFunction(spv::OpStore, held, slot);
|
||||
const auto current = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpAtomicLoad, u32, current, BdaWord(state, state.faultBufferVariable, slot), scope, relaxed);
|
||||
const auto held = binary(spv::OpIEqual, boolean, current, page);
|
||||
state.module.AddFunction(spv::OpStore, done, held);
|
||||
EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, held), [&] {
|
||||
state.module.AddFunction(spv::OpAtomicLoad, u32, current, BdaWord(state, state.faultBufferVariable, keyWord), scope, relaxed);
|
||||
const auto same = binary(spv::OpIEqual, boolean, current, key);
|
||||
state.module.AddFunction(spv::OpStore, done, same);
|
||||
EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, same), [&] {
|
||||
const auto previous = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpAtomicCompareExchange, u32, previous, BdaWord(state, state.faultBufferVariable, slot), scope, relaxed, relaxed, page, constant(0u));
|
||||
const auto taken = binary(spv::OpLogicalOr, boolean, binary(spv::OpIEqual, boolean, previous, constant(0u)), binary(spv::OpIEqual, boolean, previous, page));
|
||||
state.module.AddFunction(spv::OpStore, done, taken);
|
||||
state.module.AddFunction(spv::OpAtomicCompareExchange, u32, previous, BdaWord(state, state.faultBufferVariable, keyWord), scope, relaxed, relaxed, key, constant(0u));
|
||||
const auto claimed = binary(spv::OpIEqual, boolean, previous, constant(0u));
|
||||
state.module.AddFunction(spv::OpStore, done, binary(spv::OpLogicalOr, boolean, claimed, binary(spv::OpIEqual, boolean, previous, key)));
|
||||
EmitIfCondition(state, claimed, [&] {
|
||||
const auto index = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpAtomicIAdd, u32, index, BdaWord(state, state.faultBufferVariable, constant(BdaAbi::WrittenCountWord)), scope, relaxed, constant(1u));
|
||||
state.module.AddFunction(spv::OpAtomicStore, BdaWord(state, state.faultBufferVariable, binary(spv::OpIAdd, u32, constant(BdaAbi::WrittenListWord), index)), scope, relaxed, slot);
|
||||
});
|
||||
});
|
||||
state.module.AddFunction(spv::OpBranch, continuation);
|
||||
EmitLabel(state, continuation);
|
||||
|
|
@ -170,6 +184,16 @@ static std::uint32_t DefineBdaNoteWrite(SpirvEmitterState& state) {
|
|||
EmitLabel(state, merge);
|
||||
const auto noted = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpLoad, boolean, noted, done);
|
||||
EmitIfCondition(state, noted, [&] {
|
||||
const auto heldSlot = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpLoad, u32, heldSlot, held);
|
||||
const auto maskWord = binary(spv::OpIAdd, u32, keyWordOf(heldSlot), constant(1u));
|
||||
const auto mask = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpAtomicLoad, u32, mask, BdaWord(state, state.faultBufferVariable, maskWord), scope, relaxed);
|
||||
EmitIfCondition(state, binary(spv::OpIEqual, boolean, binary(spv::OpBitwiseAnd, u32, mask, bit), constant(0u)), [&] {
|
||||
state.module.AddFunction(spv::OpAtomicOr, u32, state.module.AllocateId(), BdaWord(state, state.faultBufferVariable, maskWord), scope, relaxed, bit);
|
||||
});
|
||||
});
|
||||
EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, noted), [&] {
|
||||
state.module.AddFunction(spv::OpAtomicStore, BdaWord(state, state.faultBufferVariable, constant(BdaAbi::WrittenOverflowWord)), scope, relaxed, constant(1u));
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue