feat(shader): move buffer accesses to runtime descriptors via bda
This commit is contained in:
boykopovar
parent
f26c8b6112
commit
b08c410b97
|
|
@ -2514,17 +2514,40 @@ std::vector<ShaderRecompiler::BdaAbi::Range> GuestBufferMemory::AddressRanges()
|
|||
std::vector<ShaderRecompiler::BdaAbi::Range> result;
|
||||
result.reserve(regions.size() + (space != nullptr ? space->ranges.size() : 0));
|
||||
for (const auto& region : regions) result.push_back(addressRange(region));
|
||||
if (space == nullptr) return result;
|
||||
// The table is searched by address (the recompiler's lookup bisects it): both lists are sorted
|
||||
// and disjoint, so a merge keeps it so.
|
||||
if (space != nullptr) result.insert(result.end(), space->ranges.begin(), space->ranges.end());
|
||||
std::sort(result.begin(), result.end(), [](const auto& left, const auto& right) { return left.begin < right.begin; });
|
||||
std::vector<ShaderRecompiler::BdaAbi::Range> merged;
|
||||
merged.reserve(result.size() + space->ranges.size());
|
||||
std::merge(space->ranges.begin(), space->ranges.end(), result.begin(), result.end(), std::back_inserter(merged), [](const auto& left, const auto& right) { return left.begin < right.begin; });
|
||||
for (const auto& range : result) {
|
||||
if ((range.permissions & ShaderRecompiler::BdaAbi::Write) != 0u) {
|
||||
merged.push_back(range);
|
||||
continue;
|
||||
}
|
||||
std::vector<std::uint64_t> boundaries{range.begin, range.end};
|
||||
for (const auto& [begin, end] : writes) {
|
||||
if (begin >= range.end || end <= range.begin) continue;
|
||||
boundaries.push_back(std::max(begin, range.begin));
|
||||
boundaries.push_back(std::min(end, range.end));
|
||||
}
|
||||
std::sort(boundaries.begin(), boundaries.end());
|
||||
boundaries.erase(std::unique(boundaries.begin(), boundaries.end()), boundaries.end());
|
||||
for (std::size_t index = 1; index < boundaries.size(); ++index) {
|
||||
auto part = range;
|
||||
part.begin = boundaries[index - 1];
|
||||
part.end = boundaries[index];
|
||||
part.deviceAddress += part.begin - range.begin;
|
||||
if (WritesOverlap(part.begin, static_cast<std::size_t>(part.end - part.begin))) {
|
||||
const auto* region = owner(part.begin);
|
||||
Require(region != nullptr && region->writable && (region->mirror == nullptr || region->mirror->writable), "runtime buffer write has no writable GPU owner");
|
||||
part.permissions |= ShaderRecompiler::BdaAbi::Write;
|
||||
}
|
||||
merged.push_back(part);
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
std::optional<GuestBufferMemory::CachedTable> GuestBufferMemory::CachedAddressTable() const {
|
||||
if (space == nullptr || !regions.empty()) return std::nullopt;
|
||||
if (space == nullptr || !regions.empty() || !writes.empty()) return std::nullopt;
|
||||
Require(uploaded && !committed, "guest GPU address ranges are not available");
|
||||
return CachedTable{space->serial, &space->ranges};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -155,8 +155,6 @@ CompiledVariant sampleVariant() {
|
|||
buffer.source = 3;
|
||||
buffer.firstUsePc = 0x40;
|
||||
buffer.maxByteExtent = 256;
|
||||
buffer.packedStride = 16;
|
||||
buffer.descriptorSwizzle = 0x123u;
|
||||
buffer.imageAlias = 1;
|
||||
buffer.read = true;
|
||||
buffer.atomic = true;
|
||||
|
|
@ -235,7 +233,6 @@ struct SampleRequest {
|
|||
request.target.maxWorkgroupInvocations = 1024;
|
||||
request.target.maxWorkgroupSharedMemoryBytes = 49152;
|
||||
request.layout = {0, 0, 0, 128};
|
||||
specialization.buffers = {{16u, IrBufferFormat::Invalid, 0xfacu}};
|
||||
specialization.images = {ResourceSpecialization::Image{}};
|
||||
specialization.boundDescriptors = {0, 0};
|
||||
}
|
||||
|
|
@ -299,7 +296,7 @@ void verifyEntryRoundTrip() {
|
|||
otherKey.back() ^= std::byte{1};
|
||||
require(ShaderDiskCache::DecodeEntry(file, otherKey, decoded) == ShaderDiskCache::LoadStatus::KeyMismatch, "an entry for another key loads");
|
||||
auto previousVersion = file;
|
||||
previousVersion[4] = std::byte{3};
|
||||
previousVersion[4] = std::byte{4};
|
||||
require(ShaderDiskCache::DecodeEntry(previousVersion, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, "an entry with the previous format version loads");
|
||||
}
|
||||
|
||||
|
|
@ -368,10 +365,6 @@ void verifyKeySensitivity() {
|
|||
changes("the first binding", [](SampleRequest& sample) { sample.request.layout.firstBinding = 1; });
|
||||
changes("the push constant offset", [](SampleRequest& sample) { sample.request.layout.pushConstantOffsetBytes = 16; });
|
||||
changes("the push constant size", [](SampleRequest& sample) { sample.request.layout.pushConstantSizeBytes = 64; });
|
||||
changes("a buffer stride", [](SampleRequest& sample) { sample.specialization.buffers[0].packedStride = 32; });
|
||||
changes("a buffer format", [](SampleRequest& sample) { sample.specialization.buffers[0].descriptorFormat = static_cast<IrBufferFormat>(1); });
|
||||
changes("a buffer swizzle", [](SampleRequest& sample) { sample.specialization.buffers[0].descriptorSwizzle = 0xfadu; });
|
||||
changes("the buffer count", [](SampleRequest& sample) { sample.specialization.buffers.emplace_back(); });
|
||||
changes("an image class", [](SampleRequest& sample) { sample.specialization.images[0].numericClass = IrTextureNumericClass::Float; });
|
||||
changes("an image dimension", [](SampleRequest& sample) { sample.specialization.images[0].dimension = static_cast<RdnaImageDimension>(1); });
|
||||
changes("an image mip count", [](SampleRequest& sample) { sample.specialization.images[0].mipCount = 2; });
|
||||
|
|
@ -428,7 +421,8 @@ void verifyStore() {
|
|||
struct ComputeRequest {
|
||||
std::vector<std::uint32_t> code{0xe0700000u, 0x80000000u, 0xbf810000u};
|
||||
std::array<std::uint32_t, 4> userData{0x10000000u, 0x00000000u, 0x40u, 0x00027facu};
|
||||
std::array<std::uint32_t, 1> capabilities{1u};
|
||||
std::array<std::uint32_t, 4> capabilities{1u, 11u, 5347u, 4448u};
|
||||
std::array<std::string_view, 2> extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"};
|
||||
RecompileRequest request{};
|
||||
|
||||
explicit ComputeRequest(bool useCache) {
|
||||
|
|
@ -441,6 +435,8 @@ struct ComputeRequest {
|
|||
request.target.spirvVersion = 0x00010300u;
|
||||
request.target.subgroupSize = 32;
|
||||
request.target.supportedCapabilities = capabilities;
|
||||
request.target.supportedExtensions = extensions;
|
||||
request.target.bdaAbiVersion = 1u;
|
||||
request.layout.pushConstantSizeBytes = 128;
|
||||
request.useCache = useCache;
|
||||
}
|
||||
|
|
@ -574,6 +570,28 @@ void verifyInvocationIsolation() {
|
|||
request.userData[0] -= 0x10000u;
|
||||
const auto restored = Recompile(request.request);
|
||||
requireSameResult(first, restored, "restored invocation");
|
||||
const auto original = request.userData;
|
||||
const std::array<std::array<std::uint32_t, 4>, 6> descriptors{{
|
||||
{original[0] + 0x20000u, original[1], original[2], original[3]},
|
||||
{original[0], original[1], original[2] / 2u, original[3]},
|
||||
{original[0], original[1] | (16u << 16u), original[2], original[3]},
|
||||
{original[0], original[1], original[2], 0x31004688u},
|
||||
{original[0], original[1] | 0x80100000u, original[2], original[3] | 0x00600000u},
|
||||
{0u, 0u, 0u, 0u}
|
||||
}};
|
||||
for (const auto& descriptor : descriptors) {
|
||||
request.userData = descriptor;
|
||||
const auto changed = Recompile(request.request);
|
||||
require(changed.cacheHit && first.variantId == changed.variantId, "buffer metadata changed the compiled variant");
|
||||
requireSameArtifact(first, changed, "runtime buffer artifact");
|
||||
require(first.spirv.data() == changed.spirv.data(), "buffer metadata duplicated the SPIR-V");
|
||||
}
|
||||
request.userData = original;
|
||||
request.userData[3] |= 0x40000000u;
|
||||
bool rejected = false;
|
||||
try { static_cast<void>(Recompile(request.request)); }
|
||||
catch (const std::runtime_error&) { rejected = true; }
|
||||
require(rejected, "an unsupported buffer descriptor type was accepted");
|
||||
}
|
||||
|
||||
void verifyDefaultDirectory(const char* self) {
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ std::uint32_t SubtractClamped(std::uint32_t memory, std::uint32_t subtrahend) {
|
|||
|
||||
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(data);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
|
||||
}
|
||||
|
||||
std::string Hex(std::uint32_t value) {
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) {
|
|||
|
||||
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(data);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
|
||||
}
|
||||
|
||||
std::string Hex(std::uint32_t value) {
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) {
|
|||
|
||||
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(data);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
|
||||
}
|
||||
|
||||
std::string Hex(std::uint32_t value) {
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) {
|
|||
|
||||
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(data);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
|
||||
}
|
||||
|
||||
std::string Hex(std::uint32_t value) {
|
||||
|
|
|
|||
|
|
@ -288,6 +288,45 @@ void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, s
|
|||
device.WaitIdle();
|
||||
}
|
||||
|
||||
void CheckRuntimeDescriptors(AgcDriver::VulkanDevice& device) {
|
||||
constexpr std::array<std::uint32_t, 6> code{0x34040082u, 0xe0000000u, 0x80000100u, 0xe0701000u, 0x80010102u, 0xbf810000u};
|
||||
struct Case {
|
||||
std::uint32_t stride;
|
||||
std::uint32_t records;
|
||||
std::uint32_t format;
|
||||
std::uint32_t selector;
|
||||
bool swizzle;
|
||||
};
|
||||
constexpr std::array<Case, 5> cases{{{4u, 32u, 20u, 4u, false}, {8u, 32u, 5u, 4u, false}, {8u, 32u, 11u, 4u, true}, {4u, 32u, 18u, 5u, false}, {4u, 16u, 20u, 4u, false}}};
|
||||
for (std::uint32_t index = 0; index < Input.size(); ++index) Input[index] = 0x3f000000u + index * 0x201u;
|
||||
std::array<std::uint32_t, 8> userData{};
|
||||
const auto output = BufferDescriptor(Output.data(), 128u);
|
||||
std::copy(output.begin(), output.end(), userData.begin() + 4);
|
||||
const ShaderRecompiler::ShaderComputeStageInfo compute{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
|
||||
ShaderRecompiler::RecompileRequest request{{ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}}, {32u, 0u, userData, compute, std::nullopt, std::nullopt, {}}, device.Target(), {0u, 0u, 0u, 128u}};
|
||||
ShaderRecompiler::RecompileResult first;
|
||||
for (const auto& value : cases) {
|
||||
auto input = BufferDescriptor(Input.data(), value.records);
|
||||
input[1] |= (value.stride << 16u) | (value.swizzle ? 0x80000000u : 0u);
|
||||
input[3] = 0x11800000u | (value.format << 12u) | value.selector;
|
||||
std::copy(input.begin(), input.end(), userData.begin());
|
||||
Output.fill(0xdeadbeefu);
|
||||
const auto result = ShaderRecompiler::Recompile(request);
|
||||
if (first.spirv.empty()) first = result;
|
||||
else Require(result.cacheHit && result.variantId == first.variantId && result.spirv.data() == first.spirv.data(), "runtime V# changed the formatted shader artifact");
|
||||
device.Dispatch(result, 1, 1, 1);
|
||||
device.WaitIdle();
|
||||
for (std::uint32_t lane = 0; lane < 32u; ++lane) {
|
||||
const auto index = value.swizzle ? (lane / 8u) * 16u + lane % 8u : lane * value.stride / 4u;
|
||||
auto expected = lane >= value.records ? 0u : Input[index];
|
||||
if (value.format == 5u) expected &= 0xffu;
|
||||
if (value.format == 11u) expected &= 0xffffu;
|
||||
if (value.format == 18u) expected = (expected >> 8u) & 0xffu;
|
||||
Require(Output[lane] == expected, "runtime V# formatted load mismatch at lane " + std::to_string(lane));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <std::size_t Lanes>
|
||||
void Check(const std::uint32_t (&expected)[Lanes][Checked], const char* const (&names)[Checked], const char* run) {
|
||||
for (std::uint32_t tid = 0; tid < Lanes; ++tid) {
|
||||
|
|
@ -310,6 +349,7 @@ int main() {
|
|||
Check(Expected64, Names, "wave64");
|
||||
Run(*device, Wave64Code, 64, device->ComputeTarget(32));
|
||||
Check(Expected64, Names, "wave64 split");
|
||||
CheckRuntimeDescriptors(*device);
|
||||
std::puts("buffer load store tests passed");
|
||||
return 0;
|
||||
} catch (const std::exception& error) {
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
|
|||
alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 101> Code{
|
||||
0x34020084, 0x34060086, 0xbe900300, 0xbe910301, 0xbe920302, 0xbe9303ff, 0x01001244, 0x7e1402ff,
|
||||
0x34020084, 0x34060086, 0xbe900300, 0xbe910301, 0xbe920302, 0xbe9303ff, 0x31001244, 0x7e1402ff,
|
||||
0xdead0000, 0x7e1602ff, 0xdead0000, 0x7e1802ff, 0xdead0000, 0x7e1a02ff, 0xdead0000, 0x7e1c02ff,
|
||||
0xdead0000, 0x7e1e02ff, 0xdead0000, 0x7e2002ff, 0xdead0000, 0x7e2202ff, 0xdead0000, 0x7e2402ff,
|
||||
0xdead0000, 0x7e2602ff, 0xdead0000, 0x7e2802ff, 0xbeefbeef, 0x7e2a02ff, 0xbeefbeef, 0x7e2c02ff,
|
||||
|
|
@ -115,7 +115,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) {
|
|||
|
||||
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(data);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
|
||||
}
|
||||
|
||||
std::string Hex(std::uint32_t value) {
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) {
|
|||
|
||||
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(data);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
|
||||
}
|
||||
|
||||
std::string Hex(std::uint32_t value) {
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) {
|
|||
|
||||
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(data);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
|
||||
}
|
||||
|
||||
std::string Hex(std::uint32_t value) {
|
||||
|
|
|
|||
|
|
@ -24,8 +24,8 @@ alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
|
|||
alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 70> Code{
|
||||
0x34020084, 0x34060086, 0xbe880300, 0xbe890301, 0xbe8a0302, 0xbe8b03ff, 0x0104bfac, 0xbe8c0300,
|
||||
0xbe8d0301, 0xbe8e0302, 0xbe8f03ff, 0x0104b80f, 0x7e1402ff, 0xdeadbeef, 0x7e1602ff, 0xdeadbeef,
|
||||
0x34020084, 0x34060086, 0xbe880300, 0xbe890301, 0xbe8a0302, 0xbe8b03ff, 0x3104bfac, 0xbe8c0300,
|
||||
0xbe8d0301, 0xbe8e0302, 0xbe8f03ff, 0x3104b80f, 0x7e1402ff, 0xdeadbeef, 0x7e1602ff, 0xdeadbeef,
|
||||
0x7e1802ff, 0xdeadbeef, 0x7e1a02ff, 0xdeadbeef, 0x7e1c02ff, 0xdeadbeef, 0x7e1e02ff, 0xdeadbeef,
|
||||
0x7e2002ff, 0xdeadbeef, 0x7e2202ff, 0xdeadbeef, 0x7e2402ff, 0xdeadbeef, 0x7e2602ff, 0xdeadbeef,
|
||||
0x7e2802ff, 0xdeadbeef, 0x7e2a02ff, 0xdeadbeef, 0x7e2c02ff, 0xdeadbeef, 0x7e2e02ff, 0xdeadbeef,
|
||||
|
|
@ -122,7 +122,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) {
|
|||
|
||||
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(data);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
|
||||
}
|
||||
|
||||
std::string Hex(std::uint32_t value) {
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ struct Range {
|
|||
};
|
||||
|
||||
enum class FaultState : std::uint32_t { Empty, Writing, Ready };
|
||||
enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle, LoopLimit, Unaligned };
|
||||
enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle, LoopLimit, Unaligned, InvalidDescriptor };
|
||||
|
||||
struct Fault {
|
||||
FaultState state;
|
||||
|
|
|
|||
|
|
@ -127,6 +127,7 @@ function(add_shader_recompiler target runtimeTarget)
|
|||
${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvBdaRead.cpp
|
||||
${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvEmitter.cpp
|
||||
${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvBufferFormat.cpp
|
||||
${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvRuntimeBuffer.cpp
|
||||
|
||||
${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp
|
||||
${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvMemory/SpirvConstants.cpp
|
||||
|
|
|
|||
|
|
@ -17,17 +17,12 @@ struct BufferResource {
|
|||
std::uint32_t source = 0;
|
||||
std::uint32_t firstUsePc = 0;
|
||||
std::uint32_t maxByteExtent = 0;
|
||||
std::uint32_t packedStride = 0;
|
||||
IrBufferFormat descriptorFormat = IrBufferFormat::Invalid;
|
||||
std::uint32_t descriptorSwizzle = 0x00000facu;
|
||||
std::uint32_t imageAlias = NoImageAlias;
|
||||
bool read = false;
|
||||
bool written = false;
|
||||
bool atomic = false;
|
||||
bool formatted = false;
|
||||
bool scalar = false;
|
||||
bool empty = false;
|
||||
std::uint8_t baseMisalignment = 0;
|
||||
|
||||
bool operator==(const BufferResource& other) const = default;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -9,16 +9,6 @@
|
|||
namespace ShaderRecompiler {
|
||||
|
||||
struct ResourceSpecialization {
|
||||
struct Buffer {
|
||||
std::uint32_t packedStride = 0;
|
||||
IrBufferFormat descriptorFormat = IrBufferFormat::Invalid;
|
||||
std::uint32_t descriptorSwizzle = ShaderImageIdentitySwizzle;
|
||||
bool empty = false;
|
||||
std::uint8_t baseMisalignment = 0;
|
||||
|
||||
bool operator==(const Buffer& other) const;
|
||||
};
|
||||
|
||||
struct Image {
|
||||
IrTextureNumericClass numericClass = IrTextureNumericClass::Unsupported;
|
||||
RdnaImageDimension dimension = RdnaImageDimension::Unknown;
|
||||
|
|
@ -39,7 +29,6 @@ struct ResourceSpecialization {
|
|||
bool operator==(const Image& other) const;
|
||||
};
|
||||
|
||||
std::vector<Buffer> buffers;
|
||||
std::vector<Image> images;
|
||||
|
||||
bool operator==(const ResourceSpecialization& other) const;
|
||||
|
|
|
|||
|
|
@ -574,28 +574,10 @@ std::uint32_t emulatedCompareState(const IrResourcePlan& plan, const ResourceSna
|
|||
|
||||
void buildResourceSpecialization(const IrResourcePlan& plan, ResourceSnapshot& snapshot, const std::vector<TableResolution>& tables, ResourceSpecialization& specialization) {
|
||||
ResourceSpecialization result;
|
||||
result.buffers.reserve(plan.info.buffers.size());
|
||||
for (std::uint32_t i = 0; i < plan.info.buffers.size(); i++) {
|
||||
const ShaderBufferResource decoded = decodeBufferDescriptor(snapshot.buffers[i]);
|
||||
if (decoded.Type() != 0u) {
|
||||
throw std::runtime_error("buffer descriptor uses an unsupported type");
|
||||
}
|
||||
auto packedStride = decoded.PackedStride();
|
||||
const auto stride = packedStride & 0x3fffu;
|
||||
const bool swizzleActive = stride != 0u && ((packedStride >> 14u) & 1u) != 0u;
|
||||
if (stride == 0u) {
|
||||
packedStride &= ~((1u << 14u) | (3u << 16u));
|
||||
} else if (!swizzleActive) {
|
||||
packedStride &= ~(3u << 16u);
|
||||
}
|
||||
const auto& buffer = plan.info.buffers[i];
|
||||
ResourceSpecialization::Buffer entry;
|
||||
entry.packedStride = packedStride;
|
||||
entry.descriptorFormat = buffer.formatted ? decoded.Format() : IrBufferFormat::Invalid;
|
||||
entry.descriptorSwizzle = buffer.formatted ? decoded.DstSelXYZW() : DstSel(4, 5, 6, 7);
|
||||
entry.empty = decoded.GetSize() == 0u || decoded.Base48() == 0u;
|
||||
entry.baseMisalignment = entry.empty ? 0u : static_cast<std::uint8_t>(decoded.Base48() & 3u);
|
||||
result.buffers.push_back(entry);
|
||||
for (std::uint32_t i = 0; i < plan.info.buffers.size(); ++i) {
|
||||
const auto decoded = decodeBufferDescriptor(snapshot.buffers.at(i));
|
||||
if (decoded.Type() != 0u) throw std::runtime_error("buffer descriptor uses an unsupported type");
|
||||
if (plan.stage != IrShaderStage::Compute && decoded.AddTid()) throw std::runtime_error("buffer ADD_TID is only valid for compute shaders");
|
||||
}
|
||||
|
||||
result.images.reserve(plan.info.images.size());
|
||||
|
|
@ -653,8 +635,8 @@ void buildResourceSpecialization(const IrResourcePlan& plan, ResourceSnapshot& s
|
|||
}
|
||||
|
||||
result.boundDescriptors.clear();
|
||||
result.boundDescriptors.reserve(result.buffers.size() + result.images.size());
|
||||
for (std::uint32_t index = 0; index < result.buffers.size(); index++) {
|
||||
result.boundDescriptors.reserve(plan.info.buffers.size() + result.images.size());
|
||||
for (std::uint32_t index = 0; index < plan.info.buffers.size(); index++) {
|
||||
result.boundDescriptors.push_back(index);
|
||||
}
|
||||
for (std::uint32_t index = 0; index < result.images.size(); index++) {
|
||||
|
|
@ -670,22 +652,11 @@ void ResourceMaterializer::Apply(IrProgram& program, const ResourceSpecializatio
|
|||
if (!resources.resourceTrackingComplete) {
|
||||
throw std::runtime_error("ResourceMaterializer::Apply requires a completed resource plan");
|
||||
}
|
||||
if (resources.info.buffers.size() != specialization.buffers.size()) {
|
||||
throw std::runtime_error("ResourceMaterializer::Apply buffer count mismatch");
|
||||
}
|
||||
if (resources.info.images.size() > specialization.images.size()) {
|
||||
throw std::runtime_error("ResourceMaterializer::Apply image count mismatch");
|
||||
}
|
||||
|
||||
auto buffers = resources.info.buffers;
|
||||
for (std::uint32_t i = 0; i < buffers.size(); i++) {
|
||||
buffers[i].packedStride = specialization.buffers[i].packedStride;
|
||||
buffers[i].descriptorFormat = specialization.buffers[i].descriptorFormat;
|
||||
buffers[i].descriptorSwizzle = specialization.buffers[i].descriptorSwizzle;
|
||||
buffers[i].empty = specialization.buffers[i].empty;
|
||||
buffers[i].baseMisalignment = specialization.buffers[i].baseMisalignment;
|
||||
}
|
||||
|
||||
auto images = resources.info.images;
|
||||
images.reserve(specialization.images.size());
|
||||
for (std::uint32_t index = 0; index < specialization.images.size(); index++) {
|
||||
|
|
@ -1037,16 +1008,12 @@ void ResourceMaterializer::CountBindlessRejection(BindlessRejection reason) {
|
|||
if (reason < BindlessRejection::Count) bindlessCounters().rejected[static_cast<std::size_t>(reason)].fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
bool ResourceSpecialization::Buffer::operator==(const Buffer& other) const {
|
||||
return packedStride == other.packedStride && descriptorFormat == other.descriptorFormat && descriptorSwizzle == other.descriptorSwizzle && empty == other.empty && baseMisalignment == other.baseMisalignment;
|
||||
}
|
||||
|
||||
bool ResourceSpecialization::Image::operator==(const Image& other) const {
|
||||
return numericClass == other.numericClass && dimension == other.dimension && mipCount == other.mipCount && conversionFormat == other.conversionFormat && shaderSwizzle == other.shaderSwizzle && indirectRoot == other.indirectRoot && indirectMappingOffset == other.indirectMappingOffset && indirectSearchIterations == other.indirectSearchIterations && cube == other.cube && fmask == other.fmask && depthBits == other.depthBits && depthUnorm16 == other.depthUnorm16 && packedFormat == other.packedFormat && emulatedCompare == other.emulatedCompare && srgbDecode == other.srgbDecode;
|
||||
}
|
||||
|
||||
bool ResourceSpecialization::operator==(const ResourceSpecialization& other) const {
|
||||
return buffers == other.buffers && images == other.images;
|
||||
return images == other.images;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -798,19 +798,17 @@ private:
|
|||
DescriptorSource descriptor;
|
||||
MakeSource(*handle, 4u, false, false, descriptor);
|
||||
std::uint32_t badDword = 0;
|
||||
if (ValidateSource(descriptor, badDword)) {
|
||||
return false;
|
||||
}
|
||||
const auto op = inst.Opcode();
|
||||
const bool load = op == IrOpcode::LoadBufferU32 || op == IrOpcode::LoadBufferU32x2 || op == IrOpcode::LoadBufferU32x3 || op == IrOpcode::LoadBufferU32x4 || op == IrOpcode::ReadConstBuffer;
|
||||
const bool store = op == IrOpcode::StoreBufferU32 || op == IrOpcode::StoreBufferU32x2 || op == IrOpcode::StoreBufferU32x3 || op == IrOpcode::StoreBufferU32x4;
|
||||
auto& memory = m_program.Resources().memoryInfo[memoryIndex];
|
||||
if ((!load && !store) || memory.formatted || memory.typed || memory.dataBits != 32u) {
|
||||
return false;
|
||||
const auto access = BufferAccessOf(inst.Opcode());
|
||||
if (ValidateSource(descriptor, badDword)) {
|
||||
const auto source = InternSource(descriptor);
|
||||
const auto resource = AddBuffer(source, memory, inst.Opcode(), inst.Flags<MemoryFlags>().pc);
|
||||
if (resource == std::numeric_limits<std::uint32_t>::max()) fail("buffer resource limit exceeded");
|
||||
AddMemoryPatch(memoryIndex, resource, 0u, false);
|
||||
}
|
||||
memory.gpuDescriptor = true;
|
||||
m_info.usesDma = true;
|
||||
m_info.bdaWrites = m_info.bdaWrites || store;
|
||||
m_info.bdaWrites = m_info.bdaWrites || access == BufferAccess::Write || access == BufferAccess::Atomic;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -979,13 +977,7 @@ private:
|
|||
std::uint32_t resource = 0;
|
||||
|
||||
if (buffer != BufferAccess::None) {
|
||||
if (TakeGpuDescriptor(inst, flags.index)) {
|
||||
return;
|
||||
}
|
||||
GetHandle(inst.Argument(0), IrOpcode::GetBufferResource, 4, handle, source);
|
||||
resource = AddBuffer(source, memory, op, flags.pc);
|
||||
AddHandlePatch(handle, resource);
|
||||
AddMemoryPatch(flags.index, resource, 0, false);
|
||||
if (!TakeGpuDescriptor(inst, flags.index)) fail("buffer operation requires a four-dword runtime V#");
|
||||
return;
|
||||
}
|
||||
if (addressInfo.access != AddressAccess::None) {
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ static_assert(sizeof(VertexInput) == 12, "VertexInput changed: update the vertex
|
|||
static_assert(sizeof(FragmentParameter) == 12, "FragmentParameter changed: update the parameter encoder");
|
||||
static_assert(sizeof(CompiledShaderInfo) == 304, "CompiledShaderInfo changed: update the info encoder");
|
||||
static_assert(sizeof(ShaderInfo) == 200, "ShaderInfo changed: update the info encoder");
|
||||
static_assert(sizeof(BufferResource) == 36, "BufferResource changed: update the info encoder");
|
||||
static_assert(sizeof(BufferResource) == 24, "BufferResource changed: update the info encoder");
|
||||
static_assert(sizeof(ImageResource) == 96, "ImageResource changed: update the info encoder");
|
||||
static_assert(sizeof(SamplerResource) == 16, "SamplerResource changed: update the info encoder");
|
||||
static_assert(sizeof(SampledResourcePair) == 12, "SampledResourcePair changed: update the info encoder");
|
||||
|
|
@ -82,8 +82,7 @@ static_assert(sizeof(IrBindingLayout) == 64, "IrBindingLayout changed: update th
|
|||
static_assert(sizeof(IrDescriptorBinding) == 32, "IrDescriptorBinding changed: update the layout encoder");
|
||||
static_assert(sizeof(BindingAllocationResult) == 120, "BindingAllocationResult changed: update the allocation encoder");
|
||||
static_assert(sizeof(CompiledBindingLayout) == 72, "CompiledBindingLayout changed: update the allocation encoder");
|
||||
static_assert(sizeof(ResourceSpecialization) == 72, "ResourceSpecialization changed: update BuildKey");
|
||||
static_assert(sizeof(ResourceSpecialization::Buffer) == 16, "ResourceSpecialization::Buffer changed: update BuildKey");
|
||||
static_assert(sizeof(ResourceSpecialization) == 48, "ResourceSpecialization changed: update BuildKey");
|
||||
static_assert(sizeof(ResourceSpecialization::Image) == 48, "ResourceSpecialization::Image changed: update BuildKey");
|
||||
static_assert(sizeof(BindingLayout) == 16, "BindingLayout changed: update BuildKey");
|
||||
#endif
|
||||
|
|
@ -404,17 +403,12 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) {
|
|||
out.Value(buffer.source);
|
||||
out.Value(buffer.firstUsePc);
|
||||
out.Value(buffer.maxByteExtent);
|
||||
out.Value(buffer.packedStride);
|
||||
out.Value(buffer.descriptorFormat);
|
||||
out.Value(buffer.descriptorSwizzle);
|
||||
out.Value(buffer.imageAlias);
|
||||
out.Value(buffer.read);
|
||||
out.Value(buffer.written);
|
||||
out.Value(buffer.atomic);
|
||||
out.Value(buffer.formatted);
|
||||
out.Value(buffer.scalar);
|
||||
out.Value(buffer.empty);
|
||||
out.Value(buffer.baseMisalignment);
|
||||
});
|
||||
writer.List(info.images, [](Writer& out, const ImageResource& image) {
|
||||
out.Value(image.source);
|
||||
|
|
@ -495,21 +489,16 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) {
|
|||
auto& info = compiled.info;
|
||||
reader.Value(info.scratchDwords);
|
||||
reader.Value(info.sharedMemoryBytes);
|
||||
reader.List(info.buffers, 35, [](Reader& in, BufferResource& buffer) {
|
||||
reader.List(info.buffers, 21, [](Reader& in, BufferResource& buffer) {
|
||||
in.Value(buffer.source);
|
||||
in.Value(buffer.firstUsePc);
|
||||
in.Value(buffer.maxByteExtent);
|
||||
in.Value(buffer.packedStride);
|
||||
in.Value(buffer.descriptorFormat);
|
||||
in.Value(buffer.descriptorSwizzle);
|
||||
in.Value(buffer.imageAlias);
|
||||
in.Value(buffer.read);
|
||||
in.Value(buffer.written);
|
||||
in.Value(buffer.atomic);
|
||||
in.Value(buffer.formatted);
|
||||
in.Value(buffer.scalar);
|
||||
in.Value(buffer.empty);
|
||||
in.Value(buffer.baseMisalignment);
|
||||
});
|
||||
reader.List(info.images, 63, [](Reader& in, ImageResource& image) {
|
||||
in.Value(image.source);
|
||||
|
|
@ -823,13 +812,6 @@ void BuildKey(const RecompileRequest& request, std::uint32_t hostSubgroupSize, c
|
|||
writer.Value(request.layout.firstBinding);
|
||||
writer.Value(request.layout.pushConstantOffsetBytes);
|
||||
writer.Value(request.layout.pushConstantSizeBytes);
|
||||
writer.List(specialization.buffers, [](Writer& out, const ResourceSpecialization::Buffer& buffer) {
|
||||
out.Value(buffer.packedStride);
|
||||
out.Value(buffer.descriptorFormat);
|
||||
out.Value(buffer.descriptorSwizzle);
|
||||
out.Value(buffer.empty);
|
||||
out.Value(buffer.baseMisalignment);
|
||||
});
|
||||
writer.List(specialization.images, [](Writer& out, const ResourceSpecialization::Image& image) {
|
||||
out.Value(image.numericClass);
|
||||
out.Value(image.dimension);
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
|
||||
namespace ShaderRecompiler::ShaderDiskCache {
|
||||
|
||||
inline constexpr std::uint32_t FormatVersion = 9;
|
||||
inline constexpr std::uint32_t FormatVersion = 8;
|
||||
|
||||
enum class LoadStatus {
|
||||
Loaded,
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& targe
|
|||
bool BdaInvocationsMayStop(const IrProgram& program);
|
||||
void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition);
|
||||
std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits);
|
||||
void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value);
|
||||
void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u);
|
||||
void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits);
|
||||
std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bytes, const std::function<std::uint32_t(std::uint32_t)>& operation);
|
||||
// Reads the dwords of a 1-4 dword load at address + offset that the program extracts, with one
|
||||
|
|
|
|||
|
|
@ -99,13 +99,10 @@ std::uint32_t EmitBallotLaneActiveBool(SpirvEmitterState& state, std::uint32_t b
|
|||
std::uint32_t EmitSubgroupLaneActiveBool(SpirvEmitterState& state, std::uint32_t lane);
|
||||
std::uint32_t EmitBinaryU32(SpirvEmitterState& state, std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs);
|
||||
std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dwordIndex);
|
||||
std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem);
|
||||
IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem);
|
||||
void EmitMemoryOffsets(SpirvEmitterState& state);
|
||||
std::uint32_t LdsDwordCount(const SpirvEmitterState& state);
|
||||
std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state);
|
||||
MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem);
|
||||
MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType);
|
||||
std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t rawIndex);
|
||||
std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index);
|
||||
std::uint32_t EmitMemoryElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index);
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ struct SpirvEmitterState {
|
|||
// every read takes the byte path.
|
||||
std::uint32_t bdaProbeFunction = 0;
|
||||
std::uint32_t bdaWritePointerFunction = 0;
|
||||
std::uint32_t bdaAtomicPointerFunction = 0;
|
||||
std::uint32_t bdaNoteWriteFunction = 0;
|
||||
std::uint32_t bdaFaultFunction = 0;
|
||||
std::array<std::array<std::uint32_t, 2>, 2> bdaDwordReadFunctions {};
|
||||
|
|
|
|||
|
|
@ -9,12 +9,9 @@ namespace ShaderRecompiler {
|
|||
|
||||
std::uint32_t EmitBinaryU32(SpirvEmitterState& state, std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs);
|
||||
std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dwordIndex);
|
||||
std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem);
|
||||
IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem);
|
||||
void EmitMemoryOffsets(SpirvEmitterState& state);
|
||||
std::uint32_t LdsDwordCount(const SpirvEmitterState& state);
|
||||
std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state);
|
||||
MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType);
|
||||
MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem);
|
||||
std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t rawIndex);
|
||||
std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index);
|
||||
|
|
|
|||
|
|
@ -7,13 +7,10 @@
|
|||
namespace ShaderRecompiler {
|
||||
|
||||
std::uint32_t LdsDwordCount(const SpirvEmitterState& state);
|
||||
std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem);
|
||||
IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem);
|
||||
|
||||
void EmitMemoryOffsets(SpirvEmitterState& state);
|
||||
|
||||
MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem);
|
||||
MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType);
|
||||
|
||||
std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t rawIndex);
|
||||
std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
#ifndef CORE_SHADER_RECOMPILER_SPIRVBACKEND_SPIRVRUNTIMEBUFFER_HPP
|
||||
#define CORE_SHADER_RECOMPILER_SPIRVBACKEND_SPIRVRUNTIMEBUFFER_HPP
|
||||
|
||||
#include "SpirvBackend/SpirvEmitterHelpers.hpp"
|
||||
#include <functional>
|
||||
|
||||
namespace ShaderRecompiler {
|
||||
|
||||
std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components);
|
||||
void EmitRuntimeBufferStore(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components);
|
||||
std::uint32_t EmitRuntimeScalarBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction);
|
||||
std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t bits, std::uint32_t active, const std::function<std::uint32_t(std::uint32_t)>& operation);
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -333,20 +333,9 @@ SpirvRequirements AnalyzeProgramRequirements(const IrProgram& program) {
|
|||
const auto& memory = program.Resources().memoryInfo.at(memoryIndex);
|
||||
requirements.coherentBuffers = requirements.coherentBuffers || (memory.coherent && !memory.gpuDescriptor);
|
||||
if (memory.gpuDescriptor) {
|
||||
if (program.Resources().stage != IrShaderStage::Compute) {
|
||||
throw std::runtime_error("GPU-selected buffer descriptors outside compute shaders are not implemented");
|
||||
}
|
||||
requirements.subgroupLocalInvocationId = true;
|
||||
} else if (memory.kind == ResourceKind::Buffer) {
|
||||
if (memory.resource >= program.Info().buffers.size()) {
|
||||
throw std::runtime_error("buffer operation has invalid resource metadata");
|
||||
}
|
||||
if ((program.Info().buffers.at(memory.resource).packedStride & (1u << 20u)) != 0u) {
|
||||
if (program.Resources().stage != IrShaderStage::Compute) {
|
||||
throw std::runtime_error("buffer ADD_TID is only valid for compute shaders");
|
||||
}
|
||||
requirements.subgroupLocalInvocationId = true;
|
||||
}
|
||||
requirements.subgroupLocalInvocationId = requirements.subgroupLocalInvocationId || program.Resources().stage == IrShaderStage::Compute;
|
||||
} else if (!memory.planningOnly) {
|
||||
throw std::runtime_error("buffer operation requires runtime descriptor metadata");
|
||||
}
|
||||
}
|
||||
const auto sharedAccess = SharedAccessOf(inst->Opcode());
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r
|
|||
fail(binary(spv::OpINotEqual, boolean, BdaLoadWord(state, at(7)), constant(0)), BdaAbi::FaultReason::InvalidTable);
|
||||
fail(binary(spv::OpUGreaterThanEqual, boolean, begin, finish), BdaAbi::FaultReason::InvalidTable);
|
||||
fail(binary(spv::OpUGreaterThan, boolean, end, finish), BdaAbi::FaultReason::Unmapped);
|
||||
fail(binary(spv::OpIEqual, boolean, binary(spv::OpBitwiseAnd, u32, permissions, constant(permission)), constant(0)), BdaAbi::FaultReason::Permission);
|
||||
fail(binary(spv::OpINotEqual, boolean, binary(spv::OpBitwiseAnd, u32, permissions, constant(permission)), constant(permission)), BdaAbi::FaultReason::Permission);
|
||||
fail(binary(spv::OpIEqual, boolean, base, BdaConstant(state, 0)), BdaAbi::FaultReason::InvalidTable);
|
||||
const auto offset = binary(spv::OpISub, u64, address, begin);
|
||||
const auto result = binary(spv::OpIAdd, u64, base, offset);
|
||||
|
|
@ -164,6 +164,7 @@ void DefineGetBdaPointer(SpirvEmitterState& state) {
|
|||
DefineBdaDwordReadFunctions(state);
|
||||
if (state.program.Info().bdaWrites) {
|
||||
state.bdaWritePointerFunction = DefineBdaLookup(state, "get_bda_write_pointer", true, BdaAbi::Write);
|
||||
state.bdaAtomicPointerFunction = DefineBdaLookup(state, "get_bda_atomic_pointer", true, BdaAbi::Read | BdaAbi::Write);
|
||||
state.bdaNoteWriteFunction = DefineBdaNoteWrite(state);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -197,10 +197,28 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::
|
|||
return EmitBdaBytes(state, address, bits / 8u, instruction, BdaAccessMask(state, inst));
|
||||
}
|
||||
|
||||
void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value) {
|
||||
void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits) {
|
||||
auto& state = ctx.state;
|
||||
if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing");
|
||||
const auto instruction = ConstantU32(state, inst.Flags<MemoryFlags>().pc);
|
||||
if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA write width");
|
||||
if (bits != 32u) {
|
||||
EmitBdaOverflowCheck(state, address, bits / 8u, instruction);
|
||||
for (std::uint32_t byte = 0u; byte < bits / 8u; ++byte) {
|
||||
const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, BdaConstant(state, byte));
|
||||
const auto physical = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaWritePointerFunction, guest, ConstantU32(state, 1u), instruction);
|
||||
EmitIfCondition(state, Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)), [&] {
|
||||
const auto byteType = state.module.Type(spv::OpTypeInt, 8u, 0u);
|
||||
const auto pointer = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, byteType), pointer, physical);
|
||||
const auto shifted = Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, ConstantU32(state, byte * 8u));
|
||||
state.module.AddFunction(spv::OpStore, pointer, Unary(state, spv::OpUConvert, byteType, shifted), BdaAccessMask(state, inst), 1u);
|
||||
state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, guest);
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
const auto unaligned = BdaAddressUnaligned(state, address);
|
||||
EmitIfCondition(state, unaligned, [&] { RecordBdaFault(state, address, ConstantU32(state, 4u), instruction, BdaAbi::FaultReason::Unaligned); });
|
||||
EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), [&] {
|
||||
|
|
|
|||
|
|
@ -46,20 +46,6 @@ std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dw
|
|||
return value;
|
||||
}
|
||||
|
||||
std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem) {
|
||||
if (mem.resource >= state.program.Info().buffers.size()) {
|
||||
ExitDescriptorBindingFailure(state, DescriptorBindingKind::Buffers, mem.resource, "buffer specialization is missing");
|
||||
}
|
||||
return state.program.Info().buffers[mem.resource].packedStride;
|
||||
}
|
||||
|
||||
IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem) {
|
||||
if (mem.resource >= state.program.Info().buffers.size()) {
|
||||
ExitDescriptorBindingFailure(state, DescriptorBindingKind::Buffers, mem.resource, "buffer specialization is missing");
|
||||
}
|
||||
return state.program.Info().buffers[mem.resource].descriptorFormat;
|
||||
}
|
||||
|
||||
void EmitMemoryOffsets(SpirvEmitterState& state) {
|
||||
const IrBindingLayout& layout = state.program.Metadata().bindings;
|
||||
state.memoryByteOffsets.assign(layout.memoryOffsetCount, 0u);
|
||||
|
|
@ -86,26 +72,6 @@ std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state) {
|
|||
return pointer;
|
||||
}
|
||||
|
||||
MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType) {
|
||||
if (variable == 0) {
|
||||
ExitDescriptorBindingFailure(state, DescriptorBindingKind::Buffers, mem.resource, "storage buffer descriptor array was not emitted");
|
||||
}
|
||||
const auto arrayIndex = ResourceForDescriptor(state, DescriptorBindingKind::Buffers, mem.resource);
|
||||
MemoryResourceAccess access;
|
||||
access.kind = mem.kind;
|
||||
access.objectPointer = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpAccessChain, pointerType, access.objectPointer, variable, ConstantU32(state, arrayIndex));
|
||||
access.byteOffset = state.memoryByteOffsets.at(arrayIndex);
|
||||
access.memoryAccess = mem.coherent ? spv::MemoryAccessVolatileMask : 0u;
|
||||
access.length = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpArrayLength, TypeU32(state), access.length, access.objectPointer, 0u);
|
||||
if (mem.resource < state.program.Info().buffers.size()) {
|
||||
if (state.program.Info().buffers[mem.resource].empty) access.length = ConstantU32(state, 0u);
|
||||
access.misalignment = state.program.Info().buffers[mem.resource].baseMisalignment;
|
||||
}
|
||||
return access;
|
||||
}
|
||||
|
||||
MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem) {
|
||||
MemoryResourceAccess access;
|
||||
access.kind = mem.kind;
|
||||
|
|
@ -138,10 +104,7 @@ MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const
|
|||
FailEmit("physical address memory must use the BDA emitter");
|
||||
case ResourceKind::ScalarBuffer:
|
||||
case ResourceKind::Buffer:
|
||||
access = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferVariable, TypeStorageBufferPointer(state));
|
||||
access.indexOffset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byteOffset, ConstantU32(state, 2u));
|
||||
access.addIndexOffset = true;
|
||||
return access;
|
||||
FailEmit("buffer memory must use the runtime V# emitter");
|
||||
default:
|
||||
FailEmit("unsupported memory resource kind " + std::to_string(static_cast<std::uint32_t>(mem.kind)));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
#include "SpirvBackend/SpirvBda.hpp"
|
||||
#include "SpirvBackend/SpirvRuntimeBuffer.hpp"
|
||||
#include "SpirvBackend/SpirvEmitterInstructions.hpp"
|
||||
#include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp"
|
||||
#include "SpirvBackend/SpirvBufferFormat.hpp"
|
||||
|
|
@ -20,45 +21,6 @@ std::uint32_t AndCondition(SpirvEmitterState& state, std::uint32_t lhs, std::uin
|
|||
return Binary(state, spv::OpLogicalAnd, TypeBool(state), lhs, rhs);
|
||||
}
|
||||
|
||||
std::uint32_t BufferByteAddress(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t index, std::uint32_t offset, std::uint32_t soffset) {
|
||||
auto& state = ctx.state;
|
||||
const std::uint32_t packed = StorageBufferPackedStride(state, mem);
|
||||
const std::uint32_t stride = packed & 0x3fffu;
|
||||
const bool swizzle = stride != 0u && ((packed >> 14u) & 1u) != 0u;
|
||||
if (((packed >> 20u) & 1u) != 0u) {
|
||||
const auto lane = Binary(state, spv::OpBitwiseAnd, TypeU32(state), EmitSubgroupLocalInvocationId(state), ConstantU32(state, 63u));
|
||||
index = Binary(state, spv::OpIAdd, TypeU32(state), index, lane);
|
||||
}
|
||||
if (mem.offset != 0u) {
|
||||
offset = Binary(state, spv::OpIAdd, TypeU32(state), offset, ConstantU32(state, mem.offset));
|
||||
}
|
||||
std::uint32_t address = 0;
|
||||
if (!swizzle) {
|
||||
if (stride == 0u) {
|
||||
address = offset;
|
||||
} else {
|
||||
const auto indexed = stride == 1u ? index : Binary(state, spv::OpIMul, TypeU32(state), index, ConstantU32(state, stride));
|
||||
address = Binary(state, spv::OpIAdd, TypeU32(state), indexed, offset);
|
||||
}
|
||||
} else {
|
||||
const std::uint32_t strideEnum = (packed >> 16u) & 3u;
|
||||
const std::uint32_t indexStride = 8u << strideEnum;
|
||||
const auto indexMsb = Binary(state, spv::OpShiftRightLogical, TypeU32(state), index, ConstantU32(state, strideEnum + 3u));
|
||||
const auto indexLsb = Binary(state, spv::OpBitwiseAnd, TypeU32(state), index, ConstantU32(state, indexStride - 1u));
|
||||
const auto offsetMsb = Binary(state, spv::OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, ~3u));
|
||||
const auto offsetLsb = Binary(state, spv::OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, 3u));
|
||||
const auto indexedMsb = stride == 1u ? indexMsb : Binary(state, spv::OpIMul, TypeU32(state), indexMsb, ConstantU32(state, stride));
|
||||
const auto msb = Binary(state, spv::OpIMul, TypeU32(state), Binary(state, spv::OpIAdd, TypeU32(state), indexedMsb, offsetMsb), ConstantU32(state, indexStride));
|
||||
const auto lsb = Binary(state, spv::OpIAdd, TypeU32(state), Binary(state, spv::OpShiftLeftLogical, TypeU32(state), indexLsb, ConstantU32(state, 2u)), offsetLsb);
|
||||
address = Binary(state, spv::OpIAdd, TypeU32(state), msb, lsb);
|
||||
}
|
||||
const IrValue* soffsetValue = inst.Argument(3)->Resolve();
|
||||
if (soffsetValue->HasImmediate() && soffsetValue->Type() == IrType::U32 && soffsetValue->ImmediateU32() == 0u) {
|
||||
return address;
|
||||
}
|
||||
return Binary(state, spv::OpIAdd, TypeU32(state), address, soffset);
|
||||
}
|
||||
|
||||
struct WordPair {
|
||||
std::uint32_t low = 0;
|
||||
std::uint32_t high = 0;
|
||||
|
|
@ -84,8 +46,6 @@ std::uint32_t ScratchByteAddress(SpirvValueEmitContext& ctx, const MemoryInfo& m
|
|||
std::uint32_t ByteAddress(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) {
|
||||
auto& state = ctx.state;
|
||||
switch (mem.kind) {
|
||||
case ResourceKind::Buffer:
|
||||
return BufferByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3));
|
||||
case ResourceKind::Lds:
|
||||
case ResourceKind::Gds:
|
||||
if (mem.kind == ResourceKind::Lds) {
|
||||
|
|
@ -219,84 +179,6 @@ std::uint32_t DeviceAddressFromWords(SpirvEmitterState& state, std::uint32_t low
|
|||
return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
|
||||
}
|
||||
|
||||
std::pair<std::uint32_t, std::uint32_t> RuntimeBufferByteAddress(SpirvEmitterState& state, std::uint32_t index, std::uint32_t offset, std::uint32_t soffset, std::uint32_t immediate, std::uint32_t stride, std::uint32_t swizzle, std::uint32_t indexStride) {
|
||||
const auto u32 = TypeU32(state);
|
||||
const auto add = [&](std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, spv::OpIAdd, u32, lhs, rhs); };
|
||||
const auto mul = [&](std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, spv::OpIMul, u32, lhs, rhs); };
|
||||
if (immediate != 0u) offset = add(offset, ConstantU32(state, immediate));
|
||||
const auto linear = add(mul(index, stride), offset);
|
||||
const auto indexShift = add(indexStride, ConstantU32(state, 3u));
|
||||
const auto indices = Binary(state, spv::OpShiftLeftLogical, u32, ConstantU32(state, 1u), indexShift);
|
||||
const auto indexMsb = Binary(state, spv::OpShiftRightLogical, u32, index, indexShift);
|
||||
const auto indexLsb = Binary(state, spv::OpBitwiseAnd, u32, index, Binary(state, spv::OpISub, u32, indices, ConstantU32(state, 1u)));
|
||||
const auto offsetMsb = Binary(state, spv::OpBitwiseAnd, u32, offset, ConstantU32(state, ~3u));
|
||||
const auto offsetLsb = Binary(state, spv::OpBitwiseAnd, u32, offset, ConstantU32(state, 3u));
|
||||
const auto swizzled = add(mul(add(mul(indexMsb, stride), offsetMsb), indices), add(Binary(state, spv::OpShiftLeftLogical, u32, indexLsb, ConstantU32(state, 2u)), offsetLsb));
|
||||
return {offset, add(Select(state, u32, swizzle, swizzled, linear), soffset)};
|
||||
}
|
||||
|
||||
template <typename TFunction>
|
||||
void ForEachGpuDescriptorDword(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components, TFunction&& function) {
|
||||
auto& state = ctx.state;
|
||||
const IrValue* handle = inst.Argument(0)->Resolve();
|
||||
if (handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) {
|
||||
ctx.Fail(inst, "has no GPU-selected V#");
|
||||
}
|
||||
const auto u32 = TypeU32(state);
|
||||
const auto boolean = TypeBool(state);
|
||||
const auto word1 = ctx.Arg(*handle, 1);
|
||||
const auto records = ctx.Arg(*handle, 2);
|
||||
const auto word3 = ctx.Arg(*handle, 3);
|
||||
const auto field = [&](std::uint32_t word, std::uint32_t first, std::uint32_t count) { return EmitBitFieldUExtract(state, word, ConstantU32(state, first), ConstantU32(state, count)); };
|
||||
const auto nonzero = [&](std::uint32_t value) { return Binary(state, spv::OpINotEqual, boolean, value, ConstantU32(state, 0u)); };
|
||||
const auto hasDword = [&](std::uint32_t offset, std::uint32_t size) {
|
||||
return AndCondition(state, Binary(state, spv::OpUGreaterThanEqual, boolean, size, ConstantU32(state, 4u)), Binary(state, spv::OpULessThanEqual, boolean, offset, Binary(state, spv::OpISub, u32, size, ConstantU32(state, 4u))));
|
||||
};
|
||||
const auto stride = field(word1, 16u, 14u);
|
||||
const auto swizzle = AndCondition(state, nonzero(stride), nonzero(field(word1, 31u, 1u)));
|
||||
const auto lane = Binary(state, spv::OpBitwiseAnd, u32, EmitSubgroupLocalInvocationId(state), ConstantU32(state, 63u));
|
||||
const auto index = Binary(state, spv::OpIAdd, u32, ctx.Arg(inst, 1), Select(state, u32, nonzero(field(word3, 23u, 1u)), lane, ConstantU32(state, 0u)));
|
||||
const auto soffset = ctx.Arg(inst, 3);
|
||||
const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), field(word1, 0u, 16u));
|
||||
const auto mode = field(word3, 28u, 2u);
|
||||
const auto indexInBounds = Binary(state, spv::OpULessThan, boolean, index, records);
|
||||
const auto scalarInBounds = Binary(state, spv::OpULessThanEqual, boolean, soffset, records);
|
||||
const auto rawRecords = Binary(state, spv::OpISub, u32, records, soffset);
|
||||
const auto rawIndexInBounds = Binary(state, spv::OpULessThan, boolean, index, rawRecords);
|
||||
for (std::uint32_t component = 0; component < components; component++) {
|
||||
const auto [offset, byte] = RuntimeBufferByteAddress(state, index, ctx.Arg(inst, 2), soffset, mem.offset + component * 4u, stride, swizzle, field(word3, 21u, 2u));
|
||||
const auto structured = AndCondition(state, indexInBounds, Binary(state, spv::OpULessThan, boolean, offset, stride));
|
||||
const auto raw = AndCondition(state, scalarInBounds, Select(state, boolean, swizzle, AndCondition(state, rawIndexInBounds, hasDword(offset, stride)), hasDword(offset, rawRecords)));
|
||||
auto inBounds = Select(state, boolean, Binary(state, spv::OpIEqual, boolean, mode, ConstantU32(state, 0u)), structured, indexInBounds);
|
||||
inBounds = Select(state, boolean, Binary(state, spv::OpULessThan, boolean, mode, ConstantU32(state, 2u)), inBounds, Select(state, boolean, Binary(state, spv::OpIEqual, boolean, mode, ConstantU32(state, 2u)), nonzero(records), raw));
|
||||
const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), base, Unary(state, spv::OpUConvert, TypeScalarU64(state), byte));
|
||||
function(component, guest, AndCondition(state, nonzero(field(word3, 12u, 7u)), inBounds));
|
||||
}
|
||||
}
|
||||
|
||||
std::uint32_t ConstructU32Composite(SpirvEmitterState& state, std::uint32_t components, const std::array<std::uint32_t, 4>& values);
|
||||
|
||||
std::uint32_t LoadGpuDescriptor(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components) {
|
||||
std::array<std::uint32_t, 4> values{};
|
||||
ForEachGpuDescriptorDword(ctx, inst, ctx.Memory(inst), components, [&](std::uint32_t component, std::uint32_t guest, std::uint32_t inBounds) {
|
||||
values[component] = EmitValueOrZeroIfCondition(ctx.state, inBounds, [&] { return EmitBdaRead(ctx, inst, guest, 32u); });
|
||||
});
|
||||
return components == 1u ? values[0] : ConstructU32Composite(ctx.state, components, values);
|
||||
}
|
||||
|
||||
void StoreGpuDescriptor(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components) {
|
||||
auto& state = ctx.state;
|
||||
const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u);
|
||||
ForEachGpuDescriptorDword(ctx, inst, ctx.Memory(inst), components, [&](std::uint32_t component, std::uint32_t guest, std::uint32_t inBounds) {
|
||||
auto value = data;
|
||||
if (components != 1u) {
|
||||
value = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, data, component);
|
||||
}
|
||||
EmitIfCondition(state, inBounds, [&] { EmitBdaWrite(ctx, inst, guest, value); });
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t GuestAddressBase(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) {
|
||||
auto& state = ctx.state;
|
||||
auto low = ctx.Arg(inst, 1);
|
||||
|
|
@ -330,92 +212,6 @@ std::uint32_t LoadBda(SpirvValueEmitContext& ctx, const IrValue& inst, const Mem
|
|||
});
|
||||
}
|
||||
|
||||
IrBufferFormat BufferFormatOf(const SpirvEmitterState& state, const MemoryInfo& mem) {
|
||||
return mem.typed ? DecodeTBufferFormat(mem.dataFormat, mem.numberFormat) : StorageBufferFormat(state, mem);
|
||||
}
|
||||
|
||||
SpirvBufferFormatInfo MemoryFormatInfo(const SpirvEmitterState& state, const MemoryInfo& mem) {
|
||||
return GetFormatInfo(mem.formatted ? BufferFormatOf(state, mem) : IrBufferFormat::Invalid);
|
||||
}
|
||||
|
||||
MemoryInfo RebaseFormattedComponent(MemoryInfo mem, const SpirvBufferFormatInfo& info, std::uint32_t component) {
|
||||
mem.offset += GetFormatComponentByteOffset(info, component);
|
||||
mem.dataDwords = 1u;
|
||||
mem.componentIndex = component;
|
||||
return mem;
|
||||
}
|
||||
|
||||
MemoryInfo RebaseRawComponent(MemoryInfo mem, std::uint32_t component) {
|
||||
mem.offset += component * 4u;
|
||||
mem.dataDwords = 1u;
|
||||
mem.componentIndex = component;
|
||||
return mem;
|
||||
}
|
||||
|
||||
SpirvFormattedSource ResolveOutputSource(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const SpirvBufferFormatInfo& info, std::uint32_t outputComponent) {
|
||||
if (mem.typed) {
|
||||
if (outputComponent < info.componentCount) {
|
||||
return {SpirvFormattedSourceKind::Memory, outputComponent};
|
||||
}
|
||||
return {};
|
||||
}
|
||||
const auto swizzle = ctx.state.program.Info().buffers.at(mem.resource).descriptorSwizzle;
|
||||
return ResolveFormattedSource(info, (swizzle >> (outputComponent * 3u)) & 0x7u);
|
||||
}
|
||||
|
||||
std::uint32_t FormattedConstant(SpirvValueEmitContext& ctx, const SpirvBufferFormatInfo& info, SpirvFormattedSourceKind kind) {
|
||||
return ConstantU32(ctx.state, FormattedConstantBits(info, kind));
|
||||
}
|
||||
|
||||
template<typename TLoadWord, typename TLoadSubword>
|
||||
std::uint32_t LoadFormattedComponent(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const SpirvBufferFormatInfo& info, std::uint32_t outputComponent, TLoadWord&& loadWord, TLoadSubword&& loadSubword) {
|
||||
auto& state = ctx.state;
|
||||
const auto source = ResolveOutputSource(ctx, mem, info, outputComponent);
|
||||
if (source.kind != SpirvFormattedSourceKind::Memory) {
|
||||
const bool integer = info.type == SpirvFormatComponentType::Uint || info.type == SpirvFormatComponentType::Sint;
|
||||
if (mem.d16 && source.kind == SpirvFormattedSourceKind::One && !integer) return ConstantU32(state, 0x3c00u);
|
||||
return FormattedConstant(ctx, info, source.kind);
|
||||
}
|
||||
const auto component = source.component;
|
||||
const auto bits = info.componentBits[component];
|
||||
const bool signedType = IsSignedFormatComponent(info.type);
|
||||
std::uint32_t raw = 0;
|
||||
if (info.packedBitfield) {
|
||||
raw = loadWord(component);
|
||||
const auto type = signedType ? TypeI32(state) : TypeU32(state);
|
||||
const auto sourceValue = signedType ? Unary(state, spv::OpBitcast, type, raw) : raw;
|
||||
const auto extracted = state.module.AllocateId();
|
||||
state.module.AddFunction(signedType ? spv::OpBitFieldSExtract : spv::OpBitFieldUExtract, type, extracted, sourceValue, ConstantU32(state, info.componentBitOffset[component]), ConstantU32(state, bits));
|
||||
raw = signedType ? Unary(state, spv::OpBitcast, TypeU32(state), extracted) : extracted;
|
||||
} else if (bits == 32u) {
|
||||
raw = loadWord(component);
|
||||
} else {
|
||||
raw = loadSubword(component, bits, signedType);
|
||||
}
|
||||
return mem.d16 ? EmitD16FormatComponent(state, info, component, raw) : NormalizeFormatComponent(state, info, component, raw);
|
||||
}
|
||||
|
||||
std::uint32_t FormattedLoadPrepared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t outputComponent, const MemoryResourceAccess& resource) {
|
||||
const auto info = MemoryFormatInfo(ctx.state, mem);
|
||||
if (info.type == SpirvFormatComponentType::Unknown) {
|
||||
if (mem.d16) ctx.Fail(inst, "is a D16 format load with an unknown buffer format");
|
||||
return LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, outputComponent), resource);
|
||||
}
|
||||
RequireAlignedFormattedElement(ctx, inst, resource, info);
|
||||
return LoadFormattedComponent(ctx, mem, info, outputComponent, [&](std::uint32_t component) {
|
||||
return LoadWordPrepared(ctx, inst, RebaseFormattedComponent(mem, info, component), resource);
|
||||
}, [&](std::uint32_t component, std::uint32_t bits, bool signExtend) {
|
||||
return LoadSubwordPrepared(ctx, inst, RebaseFormattedComponent(mem, info, component), resource, bits, signExtend);
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t FormattedLoad(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) {
|
||||
return EmitValueOrZeroIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() {
|
||||
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
|
||||
return FormattedLoadPrepared(ctx, inst, mem, 0u, resource);
|
||||
});
|
||||
}
|
||||
|
||||
void StoreWordInBounds(SpirvValueEmitContext& ctx, const MemoryResourceAccess& resource, std::uint32_t index, std::uint32_t data) {
|
||||
auto& state = ctx.state;
|
||||
if (resource.misalignment != 0u) {
|
||||
|
|
@ -486,82 +282,6 @@ void StoreWord(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo
|
|||
});
|
||||
}
|
||||
|
||||
struct PreparedFormattedMemory {
|
||||
SpirvBufferFormatInfo info;
|
||||
MemoryResourceAccess resource;
|
||||
std::array<std::uint32_t, 4> addresses{};
|
||||
std::array<std::uint32_t, 4> indices{};
|
||||
std::uint32_t inBounds = 0;
|
||||
};
|
||||
|
||||
enum class FormattedAccess {
|
||||
Load,
|
||||
Store
|
||||
};
|
||||
|
||||
PreparedFormattedMemory PrepareFormattedMemory(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource, const SpirvBufferFormatInfo& info, std::uint32_t components, FormattedAccess access) {
|
||||
auto& state = ctx.state;
|
||||
RequireAlignedFormattedElement(ctx, inst, resource, info);
|
||||
PreparedFormattedMemory plan;
|
||||
plan.info = info;
|
||||
plan.resource = resource;
|
||||
std::array<bool, 4> required{};
|
||||
if (access == FormattedAccess::Load) {
|
||||
for (std::uint32_t output = 0; output < components; output++) {
|
||||
const auto source = ResolveOutputSource(ctx, mem, plan.info, output);
|
||||
if (source.kind == SpirvFormattedSourceKind::Memory) {
|
||||
required.at(source.component) = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (std::uint32_t component = 0; component < plan.info.componentCount; component++) {
|
||||
required.at(component) = true;
|
||||
}
|
||||
}
|
||||
bool firstBound = true;
|
||||
for (std::uint32_t component = 0; component < plan.info.componentCount; component++) {
|
||||
if (!required.at(component)) {
|
||||
continue;
|
||||
}
|
||||
const auto byteOffset = GetFormatComponentByteOffset(plan.info, component);
|
||||
bool reused = false;
|
||||
for (std::uint32_t previous = 0; previous < component; previous++) {
|
||||
if (required.at(previous) && GetFormatComponentByteOffset(plan.info, previous) == byteOffset) {
|
||||
plan.addresses.at(component) = plan.addresses.at(previous);
|
||||
plan.indices.at(component) = plan.indices.at(previous);
|
||||
reused = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (reused) {
|
||||
continue;
|
||||
}
|
||||
const auto componentMem = RebaseFormattedComponent(mem, plan.info, component);
|
||||
plan.addresses.at(component) = ByteAddress(ctx, inst, componentMem);
|
||||
const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), plan.addresses.at(component), ConstantU32(state, 2u));
|
||||
plan.indices.at(component) = EmitMemoryElementIndex(state, resource, rawIndex);
|
||||
const auto componentBound = EmitMemoryElementInBounds(state, resource, plan.indices.at(component));
|
||||
if (firstBound) {
|
||||
plan.inBounds = componentBound;
|
||||
firstBound = false;
|
||||
} else {
|
||||
plan.inBounds = AndCondition(state, plan.inBounds, componentBound);
|
||||
}
|
||||
}
|
||||
if (firstBound) {
|
||||
plan.inBounds = ConstantBool(state, true);
|
||||
}
|
||||
return plan;
|
||||
}
|
||||
|
||||
std::uint32_t LoadFormattedInBounds(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, std::uint32_t outputComponent) {
|
||||
return LoadFormattedComponent(ctx, mem, plan.info, outputComponent, [&](std::uint32_t component) {
|
||||
return LoadWordInBounds(ctx, plan.resource, plan.indices.at(component));
|
||||
}, [&](std::uint32_t component, std::uint32_t bits, bool signExtend) {
|
||||
return LoadSubwordInBounds(ctx, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, signExtend);
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t ConstructU32Composite(SpirvEmitterState& state, std::uint32_t components, const std::array<std::uint32_t, 4>& values) {
|
||||
const auto result = state.module.AllocateId();
|
||||
std::vector<std::uint32_t> words{spv::OpCompositeConstruct, TypeU32Composite(state, components), result};
|
||||
|
|
@ -570,135 +290,6 @@ std::uint32_t ConstructU32Composite(SpirvEmitterState& state, std::uint32_t comp
|
|||
return result;
|
||||
}
|
||||
|
||||
std::uint32_t FormattedOutOfBoundsValue(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, std::uint32_t components) {
|
||||
auto& state = ctx.state;
|
||||
std::array<std::uint32_t, 4> values{};
|
||||
for (std::uint32_t component = 0; component < components; component++) {
|
||||
const auto source = ResolveOutputSource(ctx, mem, plan.info, component);
|
||||
values.at(component) = source.kind == SpirvFormattedSourceKind::Memory ? ConstantU32(state, 0u) : FormattedConstant(ctx, plan.info, source.kind);
|
||||
}
|
||||
return ConstructU32Composite(state, components, values);
|
||||
}
|
||||
|
||||
void StoreFormattedElement(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, const std::array<std::uint32_t, 4>& data, std::uint32_t components) {
|
||||
auto& state = ctx.state;
|
||||
const auto& info = plan.info;
|
||||
std::array<std::uint32_t, 4> encoded{};
|
||||
for (std::uint32_t component = 0; component < info.componentCount; component++) {
|
||||
encoded.at(component) = component >= components ? ConstantU32(state, 0u) : mem.d16 ? EmitD16StoreComponent(state, info, component, data.at(component)) : EmitFormatStoreComponent(state, info, component, data.at(component));
|
||||
}
|
||||
if (info.packedBitfield) {
|
||||
auto word = ConstantU32(state, 0u);
|
||||
for (std::uint32_t component = 0; component < info.componentCount; component++) {
|
||||
const std::uint32_t mask = (1u << info.componentBits[component]) - 1u;
|
||||
word = EmitOrU32(state, word, EmitBinaryU32(state, spv::OpShiftLeftLogical, EmitAndConstant(state, encoded.at(component), mask), ConstantU32(state, info.componentBitOffset[component])));
|
||||
}
|
||||
StoreWordInBounds(ctx, plan.resource, plan.indices.at(0), word);
|
||||
return;
|
||||
}
|
||||
for (std::uint32_t component = 0; component < info.componentCount; component++) {
|
||||
const auto bits = info.componentBits[component];
|
||||
if (bits == 8u || bits == 16u) {
|
||||
if (plan.resource.misalignment != 0u) StoreMisalignedSubword(ctx, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, encoded.at(component));
|
||||
else StoreSubwordInBounds(ctx, mem, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, encoded.at(component));
|
||||
} else {
|
||||
StoreWordInBounds(ctx, plan.resource, plan.indices.at(component), encoded.at(component));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void StoreFormattedValues(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource, const SpirvBufferFormatInfo& info, const std::array<std::uint32_t, 4>& data, std::uint32_t components) {
|
||||
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, FormattedAccess::Store);
|
||||
EmitIfCondition(ctx.state, plan.inBounds, [&]() {
|
||||
StoreFormattedElement(ctx, mem, plan, data, components);
|
||||
});
|
||||
}
|
||||
|
||||
void FormattedStore(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) {
|
||||
EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() {
|
||||
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
|
||||
const auto info = MemoryFormatInfo(ctx.state, mem);
|
||||
const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u);
|
||||
if (info.type == SpirvFormatComponentType::Unknown) {
|
||||
if (mem.d16) ctx.Fail(inst, "is a D16 format store with an unknown buffer format");
|
||||
StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, 0u), resource, data);
|
||||
return;
|
||||
}
|
||||
StoreFormattedValues(ctx, inst, mem, resource, info, {data, 0u, 0u, 0u}, 1u);
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t LoadCoherentPair(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource) {
|
||||
auto& state = ctx.state;
|
||||
const auto wide = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferU64Variable, TypeStorageBufferU64Pointer(state));
|
||||
const auto byteAddress = Binary(state, spv::OpIAdd, TypeU32(state), ByteAddress(ctx, inst, mem), wide.byteOffset);
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteAddress, ConstantU32(state, 3u));
|
||||
const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), byteAddress, ConstantU32(state, 7u)), ConstantU32(state, 0u));
|
||||
return EmitValueIfElse(state, AndCondition(state, aligned, EmitMemoryElementInBounds(state, wide, index)), TypeU32Composite(state, 2u), [&]() {
|
||||
const auto pointer = EmitStorageBufferElementPointer(state, wide, index, TypeStorageBufferU64ElementPointer(state));
|
||||
const auto value = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpAtomicLoad, TypeScalarU64(state), value, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone));
|
||||
const auto high = Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), value, BdaConstant(state, 32u));
|
||||
return ConstructU32Composite(state, 2u, {Unary(state, spv::OpUConvert, TypeU32(state), value), Unary(state, spv::OpUConvert, TypeU32(state), high), 0u, 0u});
|
||||
}, [&]() {
|
||||
std::array<std::uint32_t, 4> values{};
|
||||
for (std::uint32_t component = 0; component < 2u; component++) {
|
||||
values.at(component) = LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource);
|
||||
}
|
||||
return ConstructU32Composite(state, 2u, values);
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t LoadWideBuffer(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) {
|
||||
auto& state = ctx.state;
|
||||
return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() {
|
||||
const auto resource = PrepareMemoryResourceAccess(state, mem);
|
||||
const auto info = MemoryFormatInfo(state, mem);
|
||||
if (info.type == SpirvFormatComponentType::Unknown && components == 2u && mem.coherent && state.storageBufferU64Variable != 0u && resource.misalignment == 0u) {
|
||||
return LoadCoherentPair(ctx, inst, mem, resource);
|
||||
}
|
||||
if (info.type != SpirvFormatComponentType::Unknown) {
|
||||
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, FormattedAccess::Load);
|
||||
return EmitValueOrDefaultIfCondition(state, plan.inBounds, TypeU32Composite(state, components), FormattedOutOfBoundsValue(ctx, mem, plan, components), [&]() {
|
||||
std::array<std::uint32_t, 4> values{};
|
||||
for (std::uint32_t component = 0; component < components; component++) {
|
||||
values.at(component) = LoadFormattedInBounds(ctx, mem, plan, component);
|
||||
}
|
||||
return ConstructU32Composite(state, components, values);
|
||||
});
|
||||
}
|
||||
std::array<std::uint32_t, 4> values{};
|
||||
for (std::uint32_t component = 0; component < components; component++) {
|
||||
values.at(component) = LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource);
|
||||
}
|
||||
return ConstructU32Composite(state, components, values);
|
||||
});
|
||||
}
|
||||
|
||||
void StoreWideBuffer(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) {
|
||||
auto& state = ctx.state;
|
||||
EmitIfCondition(state, ActiveArgument(ctx, inst), [&]() {
|
||||
const auto resource = PrepareMemoryResourceAccess(state, mem);
|
||||
const auto composite = ctx.Arg(inst, inst.ArgumentCount() - 2u);
|
||||
const auto info = MemoryFormatInfo(state, mem);
|
||||
if (info.type != SpirvFormatComponentType::Unknown) {
|
||||
std::array<std::uint32_t, 4> data{};
|
||||
for (std::uint32_t component = 0; component < components; component++) {
|
||||
data.at(component) = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), data.at(component), composite, component);
|
||||
}
|
||||
StoreFormattedValues(ctx, inst, mem, resource, info, data, components);
|
||||
return;
|
||||
}
|
||||
if (mem.d16) ctx.Fail(inst, "is a D16 format store with an unknown buffer format");
|
||||
for (std::uint32_t component = 0; component < components; component++) {
|
||||
const auto data = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), data, composite, component);
|
||||
StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource, data);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t LoadWideShared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) {
|
||||
auto& state = ctx.state;
|
||||
return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() {
|
||||
|
|
@ -738,27 +329,10 @@ const MemoryInfo& BufferMemory(SpirvValueEmitContext& ctx, const IrValue& inst)
|
|||
if (mem.kind != ResourceKind::Buffer) {
|
||||
ctx.Fail(inst, "must access a buffer resource");
|
||||
}
|
||||
if (mem.gpuDescriptor) {
|
||||
ctx.Fail(inst, "accesses a GPU-selected V# in a way only raw dword loads and stores support");
|
||||
}
|
||||
if (!mem.gpuDescriptor) ctx.Fail(inst, "buffer operation has no runtime V#");
|
||||
return mem;
|
||||
}
|
||||
|
||||
bool EmitGpuDescriptorAccess(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components, bool store) {
|
||||
if (!ctx.Memory(inst).gpuDescriptor) {
|
||||
return false;
|
||||
}
|
||||
auto& state = ctx.state;
|
||||
if (store) {
|
||||
EmitIfCondition(state, ActiveArgument(ctx, inst), [&] { StoreGpuDescriptor(ctx, inst, components); });
|
||||
return true;
|
||||
}
|
||||
const auto type = components == 1u ? TypeU32(state) : TypeU32Composite(state, components);
|
||||
const auto zero = components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components);
|
||||
ctx.Define(inst, EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), type, zero, [&] { return LoadGpuDescriptor(ctx, inst, components); }));
|
||||
return true;
|
||||
}
|
||||
|
||||
const MemoryInfo& SharedMemory(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
const auto& mem = ctx.Memory(inst);
|
||||
if (mem.kind != ResourceKind::Lds && mem.kind != ResourceKind::Gds) {
|
||||
|
|
@ -905,6 +479,7 @@ std::uint32_t EmitAtomicOperation(SpirvValueEmitContext& ctx, const IrValue& ins
|
|||
|
||||
template<typename TOperation>
|
||||
std::uint32_t EmitAtomicAccess(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, TOperation&& operation) {
|
||||
if (mem.gpuDescriptor) return EmitRuntimeBufferAtomic(ctx, inst, 32u, ActiveArgument(ctx, inst), operation);
|
||||
auto& state = ctx.state;
|
||||
return EmitValueOrZeroIfCondition(state, ActiveArgument(ctx, inst), [&]() {
|
||||
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
|
||||
|
|
@ -988,53 +563,46 @@ std::uint32_t Atomic32(SpirvValueEmitContext& ctx, const IrValue& inst, const Me
|
|||
if (zeroSkip) active = AndCondition(state, active, nonZero);
|
||||
}
|
||||
if (mem.kind == ResourceKind::Buffer) NoteBufferAtomicSite(zeroSkip, zeroIdentity && !zeroSkip);
|
||||
return EmitValueOrZeroIfCondition(state, active, [&]() {
|
||||
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
|
||||
return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access.resource, access.index), [&]() {
|
||||
const auto pointer = EmitMemoryElementPointer(state, access.resource, access.index);
|
||||
const auto operation = [&]() {
|
||||
const auto old = EmitAtomicOperation(ctx, inst, pointer, scope);
|
||||
if (lds) {
|
||||
const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
|
||||
state.module.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics));
|
||||
} else {
|
||||
EmitDeviceAtomicMemoryBarrier(state);
|
||||
}
|
||||
return old;
|
||||
};
|
||||
if (!zeroIdentity || zeroSkip) return operation();
|
||||
return EmitValueIfElse(state, nonZero, TypeU32(state), operation, [&]() {
|
||||
// The barrier stays so an "add 0" used as a coherent read keeps its acquire.
|
||||
const auto current = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpAtomicLoad, TypeU32(state), current, pointer, ConstantU32(state, scope), ConstantU32(state, spv::MemorySemanticsMaskNone));
|
||||
const auto apply = [&](std::uint32_t pointer) {
|
||||
const auto operation = [&]() {
|
||||
const auto old = EmitAtomicOperation(ctx, inst, pointer, scope);
|
||||
if (lds) {
|
||||
const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
|
||||
state.module.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics));
|
||||
} else {
|
||||
EmitDeviceAtomicMemoryBarrier(state);
|
||||
return current;
|
||||
});
|
||||
}
|
||||
return old;
|
||||
};
|
||||
if (!zeroIdentity || zeroSkip) return operation();
|
||||
return EmitValueIfElse(state, nonZero, TypeU32(state), operation, [&]() {
|
||||
const auto current = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpAtomicLoad, TypeU32(state), current, pointer, ConstantU32(state, scope), ConstantU32(state, spv::MemorySemanticsMaskNone));
|
||||
EmitDeviceAtomicMemoryBarrier(state);
|
||||
return current;
|
||||
});
|
||||
};
|
||||
if (mem.gpuDescriptor) return EmitRuntimeBufferAtomic(ctx, inst, 32u, active, apply);
|
||||
return EmitValueOrZeroIfCondition(state, active, [&] {
|
||||
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
|
||||
return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access.resource, access.index), [&] { return apply(EmitMemoryElementPointer(state, access.resource, access.index)); });
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t BufferAtomic64(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
auto& state = ctx.state;
|
||||
const auto& mem = BufferMemory(ctx, inst);
|
||||
return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU64(state), ConstantU64(state, 0u), [&]() {
|
||||
const auto resource = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferU64Variable, TypeStorageBufferU64Pointer(state));
|
||||
RequireAlignedAtomic(ctx, resource);
|
||||
const auto byteAddress = Binary(state, spv::OpIAdd, TypeU32(state), ByteAddress(ctx, inst, mem), resource.byteOffset);
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteAddress, ConstantU32(state, 3u));
|
||||
return EmitValueOrDefaultIfCondition(state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state), ConstantU64(state, 0u), [&]() {
|
||||
const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 2u));
|
||||
const auto old = state.module.AllocateId();
|
||||
const auto pointer = EmitStorageBufferElementPointer(state, resource, index, TypeStorageBufferU64ElementPointer(state));
|
||||
if (inst.Opcode() == IrOpcode::BufferAtomicCmpSwap64) {
|
||||
const auto desired = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 3u));
|
||||
state.module.AddFunction(spv::OpAtomicCompareExchange, TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), ConstantU32(state, spv::MemorySemanticsMaskNone), desired, value);
|
||||
} else {
|
||||
state.module.AddFunction(SpirvAtomicOpcode(inst.Opcode()), TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), value);
|
||||
}
|
||||
EmitDeviceAtomicMemoryBarrier(state);
|
||||
return Unary(state, spv::OpBitcast, TypeU64(state), old);
|
||||
});
|
||||
static_cast<void>(BufferMemory(ctx, inst));
|
||||
return EmitRuntimeBufferAtomic(ctx, inst, 64u, ActiveArgument(ctx, inst), [&](std::uint32_t pointer) {
|
||||
const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 2u));
|
||||
const auto old = state.module.AllocateId();
|
||||
if (inst.Opcode() == IrOpcode::BufferAtomicCmpSwap64) {
|
||||
const auto desired = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 3u));
|
||||
state.module.AddFunction(spv::OpAtomicCompareExchange, TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), ConstantU32(state, spv::MemorySemanticsMaskNone), desired, value);
|
||||
} else {
|
||||
state.module.AddFunction(SpirvAtomicOpcode(inst.Opcode()), TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), value);
|
||||
}
|
||||
EmitDeviceAtomicMemoryBarrier(state);
|
||||
return Unary(state, spv::OpBitcast, TypeU64(state), old);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -1095,19 +663,10 @@ std::uint32_t BufferFloatAtomic(SpirvValueEmitContext& ctx, const IrValue& inst,
|
|||
template<typename TReplacement>
|
||||
std::uint32_t BufferAtomic64Update(SpirvValueEmitContext& ctx, const IrValue& inst, TReplacement&& replacement) {
|
||||
auto& state = ctx.state;
|
||||
const auto& mem = BufferMemory(ctx, inst);
|
||||
return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU64(state), ConstantU64(state, 0u), [&]() {
|
||||
const auto resource = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferU64Variable, TypeStorageBufferU64Pointer(state));
|
||||
RequireAlignedAtomic(ctx, resource);
|
||||
const auto byteAddress = Binary(state, spv::OpIAdd, TypeU32(state), ByteAddress(ctx, inst, mem), resource.byteOffset);
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteAddress, ConstantU32(state, 3u));
|
||||
return EmitValueOrDefaultIfCondition(state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state), ConstantU64(state, 0u), [&]() {
|
||||
const auto pointer = EmitStorageBufferElementPointer(state, resource, index, TypeStorageBufferU64ElementPointer(state));
|
||||
const auto old = AtomicUpdateTyped(state, pointer, ResourceKind::Buffer, TypeScalarU64(state), [&](std::uint32_t observed) {
|
||||
return replacement(observed);
|
||||
});
|
||||
return Unary(state, spv::OpBitcast, TypeU64(state), old);
|
||||
});
|
||||
static_cast<void>(BufferMemory(ctx, inst));
|
||||
return EmitRuntimeBufferAtomic(ctx, inst, 64u, ActiveArgument(ctx, inst), [&](std::uint32_t pointer) {
|
||||
const auto old = AtomicUpdateTyped(state, pointer, ResourceKind::Buffer, TypeScalarU64(state), [&](std::uint32_t observed) { return replacement(observed); });
|
||||
return Unary(state, spv::OpBitcast, TypeU64(state), old);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -1475,41 +1034,9 @@ std::uint32_t EmitReadConst(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
|||
|
||||
void EmitReadConstBuffer(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
const auto& mem = ctx.Memory(inst);
|
||||
if (mem.planningOnly) {
|
||||
return;
|
||||
}
|
||||
if (mem.kind != ResourceKind::ScalarBuffer) {
|
||||
ctx.Fail(inst, "must read a scalar buffer resource");
|
||||
}
|
||||
auto& state = ctx.state;
|
||||
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1), ConstantU32(state, mem.offset));
|
||||
const auto noCarry = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address, ctx.Arg(inst, 1));
|
||||
if (mem.gpuDescriptor) {
|
||||
const IrValue* handle = inst.Argument(0)->Resolve();
|
||||
if (handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) {
|
||||
ctx.Fail(inst, "has no GPU-selected V#");
|
||||
}
|
||||
const auto u32 = TypeU32(state);
|
||||
const auto word1 = ctx.Arg(*handle, 1);
|
||||
const auto records = ctx.Arg(*handle, 2);
|
||||
const auto stride = EmitBitFieldUExtract(state, word1, ConstantU32(state, 16u), ConstantU32(state, 14u));
|
||||
const auto size = Select(state, u32, Binary(state, spv::OpIEqual, TypeBool(state), stride, ConstantU32(state, 0u)), records, Binary(state, spv::OpIMul, u32, stride, records));
|
||||
const auto byte = Binary(state, spv::OpBitwiseAnd, u32, address, ConstantU32(state, ~3u));
|
||||
const auto inBounds = AndCondition(state, noCarry, Binary(state, spv::OpULessThan, TypeBool(state), byte, size));
|
||||
const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), EmitBitFieldUExtract(state, word1, ConstantU32(state, 0u), ConstantU32(state, 16u)));
|
||||
const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), base, Unary(state, spv::OpUConvert, TypeScalarU64(state), byte));
|
||||
ctx.Define(inst, EmitValueOrZeroIfCondition(state, inBounds, [&] { return EmitBdaRead(ctx, inst, guest, 32u); }));
|
||||
return;
|
||||
}
|
||||
const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u));
|
||||
const auto access = PrepareMemoryResourceAccess(state, mem);
|
||||
const auto element = EmitMemoryElementIndex(state, access, rawIndex);
|
||||
const auto inBounds = AndCondition(state, noCarry, EmitMemoryElementInBounds(state, access, element));
|
||||
ctx.Define(inst, EmitValueOrZeroIfCondition(state, inBounds, [&]() {
|
||||
const auto value = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpLoad, TypeU32(state), value, EmitMemoryElementPointer(state, access, element));
|
||||
return value;
|
||||
}));
|
||||
if (mem.planningOnly) return;
|
||||
if (mem.kind != ResourceKind::ScalarBuffer) ctx.Fail(inst, "must read a scalar buffer resource");
|
||||
ctx.Define(inst, EmitRuntimeScalarBufferLoad(ctx, inst));
|
||||
}
|
||||
|
||||
void EmitGetSrtResource(SpirvValueEmitContext& context) {
|
||||
|
|
@ -1569,65 +1096,51 @@ std::uint32_t EmitAddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst)
|
|||
}
|
||||
|
||||
void EmitLoadBufferU8(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
ctx.Define(inst, LoadSubword(ctx, inst, BufferMemory(ctx, inst), 8u));
|
||||
ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 1u));
|
||||
}
|
||||
|
||||
void EmitLoadBufferU16(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
ctx.Define(inst, LoadSubword(ctx, inst, BufferMemory(ctx, inst), 16u));
|
||||
ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 1u));
|
||||
}
|
||||
|
||||
void EmitLoadBufferU32(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
if (EmitGpuDescriptorAccess(ctx, inst, 1u, false)) return;
|
||||
const auto& mem = BufferMemory(ctx, inst);
|
||||
ctx.Define(inst, mem.formatted ? FormattedLoad(ctx, inst, mem) : LoadWord(ctx, inst, mem));
|
||||
ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 1u));
|
||||
}
|
||||
|
||||
void EmitLoadBufferU32x2(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
if (EmitGpuDescriptorAccess(ctx, inst, 2u, false)) return;
|
||||
ctx.Define(inst, LoadWideBuffer(ctx, inst, BufferMemory(ctx, inst), 2u));
|
||||
ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 2u));
|
||||
}
|
||||
|
||||
void EmitLoadBufferU32x3(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
if (EmitGpuDescriptorAccess(ctx, inst, 3u, false)) return;
|
||||
ctx.Define(inst, LoadWideBuffer(ctx, inst, BufferMemory(ctx, inst), 3u));
|
||||
ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 3u));
|
||||
}
|
||||
|
||||
void EmitLoadBufferU32x4(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
if (EmitGpuDescriptorAccess(ctx, inst, 4u, false)) return;
|
||||
ctx.Define(inst, LoadWideBuffer(ctx, inst, BufferMemory(ctx, inst), 4u));
|
||||
ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 4u));
|
||||
}
|
||||
|
||||
void EmitStoreBufferU8(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
StoreSubword(ctx, inst, BufferMemory(ctx, inst), 8u);
|
||||
EmitRuntimeBufferStore(ctx, inst, 1u);
|
||||
}
|
||||
|
||||
void EmitStoreBufferU16(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
StoreSubword(ctx, inst, BufferMemory(ctx, inst), 16u);
|
||||
EmitRuntimeBufferStore(ctx, inst, 1u);
|
||||
}
|
||||
|
||||
void EmitStoreBufferU32(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
if (EmitGpuDescriptorAccess(ctx, inst, 1u, true)) return;
|
||||
const auto& mem = BufferMemory(ctx, inst);
|
||||
if (mem.formatted) {
|
||||
FormattedStore(ctx, inst, mem);
|
||||
} else {
|
||||
StoreWord(ctx, inst, mem);
|
||||
}
|
||||
EmitRuntimeBufferStore(ctx, inst, 1u);
|
||||
}
|
||||
|
||||
void EmitStoreBufferU32x2(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
if (EmitGpuDescriptorAccess(ctx, inst, 2u, true)) return;
|
||||
StoreWideBuffer(ctx, inst, BufferMemory(ctx, inst), 2u);
|
||||
EmitRuntimeBufferStore(ctx, inst, 2u);
|
||||
}
|
||||
|
||||
void EmitStoreBufferU32x3(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
if (EmitGpuDescriptorAccess(ctx, inst, 3u, true)) return;
|
||||
StoreWideBuffer(ctx, inst, BufferMemory(ctx, inst), 3u);
|
||||
EmitRuntimeBufferStore(ctx, inst, 3u);
|
||||
}
|
||||
|
||||
void EmitStoreBufferU32x4(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
if (EmitGpuDescriptorAccess(ctx, inst, 4u, true)) return;
|
||||
StoreWideBuffer(ctx, inst, BufferMemory(ctx, inst), 4u);
|
||||
EmitRuntimeBufferStore(ctx, inst, 4u);
|
||||
}
|
||||
|
||||
std::uint32_t EmitBufferAtomicSwap32(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,339 @@
|
|||
#include "SpirvBackend/SpirvRuntimeBuffer.hpp"
|
||||
#include "SpirvBackend/SpirvEmitterInstructions.hpp"
|
||||
#include "SpirvBackend/SpirvBda.hpp"
|
||||
#include "SpirvBackend/SpirvBufferFormat.hpp"
|
||||
#include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp"
|
||||
#include <array>
|
||||
#include <vector>
|
||||
|
||||
namespace ShaderRecompiler {
|
||||
namespace {
|
||||
|
||||
struct RuntimeDescriptor {
|
||||
std::uint32_t base;
|
||||
std::uint32_t records;
|
||||
std::uint32_t stride;
|
||||
std::uint32_t word3;
|
||||
std::uint32_t swizzle;
|
||||
std::uint32_t index;
|
||||
std::uint32_t valid;
|
||||
};
|
||||
|
||||
struct RuntimeAddress {
|
||||
std::uint32_t guest;
|
||||
std::uint32_t inBounds;
|
||||
};
|
||||
|
||||
std::uint32_t field(SpirvEmitterState& state, std::uint32_t word, std::uint32_t offset, std::uint32_t count) {
|
||||
return EmitBitFieldUExtract(state, word, ConstantU32(state, offset), ConstantU32(state, count));
|
||||
}
|
||||
|
||||
std::uint32_t both(SpirvEmitterState& state, std::uint32_t left, std::uint32_t right) {
|
||||
return Binary(state, spv::OpLogicalAnd, TypeBool(state), left, right);
|
||||
}
|
||||
|
||||
std::uint32_t equal(SpirvEmitterState& state, std::uint32_t left, std::uint32_t right) {
|
||||
return Binary(state, spv::OpIEqual, TypeBool(state), left, right);
|
||||
}
|
||||
|
||||
std::uint32_t nonzero(SpirvEmitterState& state, std::uint32_t value) {
|
||||
return Binary(state, spv::OpINotEqual, TypeBool(state), value, ConstantU32(state, 0u));
|
||||
}
|
||||
|
||||
void faultIf(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, BdaAbi::FaultReason reason) {
|
||||
auto& state = context.state;
|
||||
EmitIfCondition(state, condition, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), ConstantU32(state, instruction.Flags<MemoryFlags>().pc), reason); });
|
||||
StopBdaInvocationIf(state, condition);
|
||||
}
|
||||
|
||||
RuntimeDescriptor descriptor(SpirvValueEmitContext& context, const IrValue& instruction) {
|
||||
auto& state = context.state;
|
||||
const auto* handle = instruction.Argument(0)->Resolve();
|
||||
if (!context.Memory(instruction).gpuDescriptor || handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) context.Fail(instruction, "requires a runtime four-dword V#");
|
||||
const auto word1 = context.Arg(*handle, 1u);
|
||||
const auto word3 = context.Arg(*handle, 3u);
|
||||
const auto u32 = TypeU32(state);
|
||||
const auto u64 = TypeScalarU64(state);
|
||||
const auto high = Binary(state, spv::OpShiftLeftLogical, u64, Unary(state, spv::OpUConvert, u64, field(state, word1, 0u, 16u)), BdaConstant(state, 32u));
|
||||
const auto base = Binary(state, spv::OpBitwiseOr, u64, Unary(state, spv::OpUConvert, u64, context.Arg(*handle, 0u)), high);
|
||||
auto invalid = nonzero(state, field(state, word3, 30u, 2u));
|
||||
auto index = ConstantU32(state, 0u);
|
||||
if (context.Memory(instruction).kind == ResourceKind::Buffer) {
|
||||
index = context.Arg(instruction, 1u);
|
||||
const auto addTid = nonzero(state, field(state, word3, 23u, 1u));
|
||||
if (state.program.Resources().stage == IrShaderStage::Compute) {
|
||||
index = Binary(state, spv::OpIAdd, u32, index, Select(state, u32, addTid, EmitSubgroupLocalInvocationId(state), ConstantU32(state, 0u)));
|
||||
} else {
|
||||
invalid = Binary(state, spv::OpLogicalOr, TypeBool(state), invalid, addTid);
|
||||
}
|
||||
}
|
||||
faultIf(context, instruction, invalid, base, 0u, BdaAbi::FaultReason::InvalidDescriptor);
|
||||
const auto stride = field(state, word1, 16u, 14u);
|
||||
const auto valid = both(state, Unary(state, spv::OpLogicalNot, TypeBool(state), invalid), Binary(state, spv::OpINotEqual, TypeBool(state), base, BdaConstant(state, 0u)));
|
||||
return {base, context.Arg(*handle, 2u), stride, word3, both(state, nonzero(state, stride), nonzero(state, field(state, word1, 31u, 1u))), index, valid};
|
||||
}
|
||||
|
||||
RuntimeAddress address(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t componentOffset, std::uint32_t bytes) {
|
||||
auto& state = context.state;
|
||||
const auto u32 = TypeU32(state);
|
||||
const auto boolean = TypeBool(state);
|
||||
const auto constant = [&](std::uint32_t value) { return ConstantU32(state, value); };
|
||||
const auto add = [&](std::uint32_t left, std::uint32_t right) { return Binary(state, spv::OpIAdd, u32, left, right); };
|
||||
const auto mul = [&](std::uint32_t left, std::uint32_t right) { return Binary(state, spv::OpIMul, u32, left, right); };
|
||||
const auto less = [&](std::uint32_t left, std::uint32_t right) { return Binary(state, spv::OpULessThan, boolean, left, right); };
|
||||
const auto fits = [&](std::uint32_t offset, std::uint32_t size) {
|
||||
return both(state, Binary(state, spv::OpUGreaterThanEqual, boolean, size, constant(bytes)), Binary(state, spv::OpULessThanEqual, boolean, offset, Binary(state, spv::OpISub, u32, size, constant(bytes))));
|
||||
};
|
||||
const auto offset = add(context.Arg(instruction, 2u), constant(context.Memory(instruction).offset + componentOffset));
|
||||
const auto soffset = context.Arg(instruction, 3u);
|
||||
const auto indexShift = add(field(state, resource.word3, 21u, 2u), constant(3u));
|
||||
const auto indices = Binary(state, spv::OpShiftLeftLogical, u32, constant(1u), indexShift);
|
||||
const auto indexMsb = Binary(state, spv::OpShiftRightLogical, u32, resource.index, indexShift);
|
||||
const auto indexLsb = Binary(state, spv::OpBitwiseAnd, u32, resource.index, Binary(state, spv::OpISub, u32, indices, constant(1u)));
|
||||
const auto offsetMsb = Binary(state, spv::OpBitwiseAnd, u32, offset, constant(~3u));
|
||||
const auto offsetLsb = Binary(state, spv::OpBitwiseAnd, u32, offset, constant(3u));
|
||||
const auto swizzled = add(mul(add(mul(indexMsb, resource.stride), offsetMsb), indices), add(Binary(state, spv::OpShiftLeftLogical, u32, indexLsb, constant(2u)), offsetLsb));
|
||||
const auto linear = add(mul(resource.index, resource.stride), offset);
|
||||
const auto byte = add(Select(state, u32, resource.swizzle, swizzled, linear), soffset);
|
||||
const auto mode = field(state, resource.word3, 28u, 2u);
|
||||
const auto indexInBounds = less(resource.index, resource.records);
|
||||
const auto structured = both(state, indexInBounds, less(offset, resource.stride));
|
||||
const auto rawRecords = Binary(state, spv::OpISub, u32, resource.records, soffset);
|
||||
const auto raw = both(state, Binary(state, spv::OpULessThanEqual, boolean, soffset, resource.records), Select(state, boolean, resource.swizzle, both(state, less(resource.index, rawRecords), fits(offset, resource.stride)), fits(offset, rawRecords)));
|
||||
auto inBounds = Select(state, boolean, equal(state, mode, constant(0u)), structured, indexInBounds);
|
||||
inBounds = Select(state, boolean, less(mode, constant(2u)), inBounds, Select(state, boolean, equal(state, mode, constant(2u)), nonzero(state, resource.records), raw));
|
||||
inBounds = both(state, resource.valid, both(state, nonzero(state, field(state, resource.word3, 12u, 7u)), inBounds));
|
||||
return {AddBdaAddress(context, instruction, resource.base, Unary(state, spv::OpUConvert, TypeScalarU64(state), byte), false), inBounds};
|
||||
}
|
||||
|
||||
std::uint32_t composite(SpirvEmitterState& state, std::uint32_t components, const std::array<std::uint32_t, 4>& values) {
|
||||
if (components == 1u) return values[0];
|
||||
const auto result = state.module.AllocateId();
|
||||
std::vector<std::uint32_t> words{spv::OpCompositeConstruct, TypeU32Composite(state, components), result};
|
||||
words.insert(words.end(), values.begin(), values.begin() + components);
|
||||
state.module.AddFunction(words);
|
||||
return result;
|
||||
}
|
||||
|
||||
std::array<std::uint32_t, 4> storeValues(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) {
|
||||
auto& state = context.state;
|
||||
std::array<std::uint32_t, 4> result{};
|
||||
const auto data = context.Arg(instruction, instruction.ArgumentCount() - 2u);
|
||||
for (std::uint32_t component = 0; component < components; ++component) {
|
||||
result[component] = data;
|
||||
if (components != 1u) {
|
||||
result[component] = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), result[component], data, component);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::uint32_t formatted(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, const SpirvBufferFormatInfo& info, std::uint32_t components, bool store) {
|
||||
auto& state = context.state;
|
||||
const auto& memory = context.Memory(instruction);
|
||||
const auto u32 = TypeU32(state);
|
||||
std::array<std::uint32_t, 4> values{};
|
||||
std::array<std::uint32_t, 4> selectors{};
|
||||
std::array<std::uint32_t, 4> selected{};
|
||||
std::array<RuntimeAddress, 4> addresses{};
|
||||
auto inBounds = ConstantBool(state, true);
|
||||
for (std::uint32_t output = 0; output < components; ++output) {
|
||||
selectors[output] = memory.typed ? ConstantU32(state, output < info.componentCount ? 4u + output : 0u) : field(state, resource.word3, output * 3u, 3u);
|
||||
selected[output] = Binary(state, spv::OpUMod, u32, Binary(state, spv::OpISub, u32, selectors[output], ConstantU32(state, 4u)), ConstantU32(state, info.componentCount));
|
||||
}
|
||||
std::array<std::uint32_t, 4> required{};
|
||||
for (std::uint32_t component = 0; component < info.componentCount; ++component) {
|
||||
required[component] = ConstantBool(state, store);
|
||||
if (!store) {
|
||||
for (std::uint32_t output = 0; output < components; ++output) {
|
||||
const auto fromMemory = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), selectors[output], ConstantU32(state, 4u));
|
||||
required[component] = Binary(state, spv::OpLogicalOr, TypeBool(state), required[component], both(state, fromMemory, equal(state, selected[output], ConstantU32(state, component))));
|
||||
}
|
||||
}
|
||||
addresses[component] = address(context, instruction, resource, GetFormatComponentByteOffset(info, component), info.packedBitfield ? 4u : info.componentBits[component] / 8u);
|
||||
inBounds = both(state, inBounds, Binary(state, spv::OpLogicalOr, TypeBool(state), Unary(state, spv::OpLogicalNot, TypeBool(state), required[component]), addresses[component].inBounds));
|
||||
}
|
||||
if (store) {
|
||||
const auto data = storeValues(context, instruction, components);
|
||||
EmitIfCondition(state, inBounds, [&] {
|
||||
auto packed = ConstantU32(state, 0u);
|
||||
for (std::uint32_t component = 0; component < info.componentCount; ++component) {
|
||||
const auto encoded = component >= components ? ConstantU32(state, 0u) : memory.d16 ? EmitD16StoreComponent(state, info, component, data[component]) : EmitFormatStoreComponent(state, info, component, data[component]);
|
||||
if (info.packedBitfield) {
|
||||
packed = EmitOrU32(state, packed, Binary(state, spv::OpShiftLeftLogical, u32, EmitAndConstant(state, encoded, (1u << info.componentBits[component]) - 1u), ConstantU32(state, info.componentBitOffset[component])));
|
||||
} else {
|
||||
EmitBdaWrite(context, instruction, addresses[component].guest, encoded, info.componentBits[component]);
|
||||
}
|
||||
}
|
||||
if (info.packedBitfield) EmitBdaWrite(context, instruction, addresses[0].guest, packed);
|
||||
});
|
||||
return ConstantU32(state, 0u);
|
||||
}
|
||||
for (std::uint32_t component = 0; component < info.componentCount; ++component) {
|
||||
values[component] = EmitValueOrZeroIfCondition(state, both(state, inBounds, required[component]), [&] {
|
||||
const auto bits = info.componentBits[component];
|
||||
auto raw = EmitBdaRead(context, instruction, addresses[component].guest, info.packedBitfield ? 32u : bits);
|
||||
if (bits != 32u) {
|
||||
const auto signedType = IsSignedFormatComponent(info.type);
|
||||
const auto type = signedType ? TypeI32(state) : u32;
|
||||
const auto input = signedType ? Unary(state, spv::OpBitcast, type, raw) : raw;
|
||||
raw = state.module.AllocateId();
|
||||
state.module.AddFunction(signedType ? spv::OpBitFieldSExtract : spv::OpBitFieldUExtract, type, raw, input, ConstantU32(state, info.packedBitfield ? info.componentBitOffset[component] : 0u), ConstantU32(state, bits));
|
||||
if (signedType) raw = Unary(state, spv::OpBitcast, u32, raw);
|
||||
}
|
||||
return memory.d16 ? EmitD16FormatComponent(state, info, component, raw) : NormalizeFormatComponent(state, info, component, raw);
|
||||
});
|
||||
}
|
||||
const bool integer = info.type == SpirvFormatComponentType::Uint || info.type == SpirvFormatComponentType::Sint;
|
||||
const auto one = ConstantU32(state, integer ? 1u : memory.d16 ? 0x3c00u : 0x3f800000u);
|
||||
std::array<std::uint32_t, 4> outputs{};
|
||||
for (std::uint32_t output = 0; output < components; ++output) {
|
||||
auto value = values[0];
|
||||
for (std::uint32_t component = 1; component < info.componentCount; ++component) value = Select(state, u32, equal(state, selected[output], ConstantU32(state, component)), values[component], value);
|
||||
value = Select(state, u32, equal(state, selectors[output], ConstantU32(state, 1u)), one, value);
|
||||
outputs[output] = Select(state, u32, equal(state, selectors[output], ConstantU32(state, 0u)), ConstantU32(state, 0u), value);
|
||||
}
|
||||
return composite(state, components, outputs);
|
||||
}
|
||||
|
||||
std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store) {
|
||||
auto& state = context.state;
|
||||
const auto& memory = context.Memory(instruction);
|
||||
if (memory.typed) {
|
||||
const auto info = GetFormatInfo(DecodeTBufferFormat(memory.dataFormat, memory.numberFormat));
|
||||
if (info.type == SpirvFormatComponentType::Unknown) context.Fail(instruction, "typed buffer instruction has an invalid format");
|
||||
return formatted(context, instruction, resource, info, components, store);
|
||||
}
|
||||
if (!store) {
|
||||
auto invalid = ConstantBool(state, false);
|
||||
for (std::uint32_t component = 0; component < components; ++component) {
|
||||
const auto selector = field(state, resource.word3, component * 3u, 3u);
|
||||
invalid = Binary(state, spv::OpLogicalOr, TypeBool(state), invalid, Binary(state, spv::OpLogicalOr, TypeBool(state), equal(state, selector, ConstantU32(state, 2u)), equal(state, selector, ConstantU32(state, 3u))));
|
||||
}
|
||||
faultIf(context, instruction, invalid, resource.base, 0u, BdaAbi::FaultReason::InvalidDescriptor);
|
||||
}
|
||||
const auto* word = instruction.Argument(0)->Resolve()->Argument(3)->Resolve();
|
||||
if (word->HasImmediate()) {
|
||||
const auto format = static_cast<IrBufferFormat>((word->ImmediateU32() >> 12u) & 0x7fu);
|
||||
if (format == IrBufferFormat::Invalid) context.Fail(instruction, "formatted buffer instruction has an invalid constant format");
|
||||
return formatted(context, instruction, resource, GetFormatInfo(format), components, store);
|
||||
}
|
||||
const auto format = field(state, resource.word3, 12u, 7u);
|
||||
const auto merge = state.module.AllocateId();
|
||||
const auto invalid = state.module.AllocateId();
|
||||
constexpr std::uint32_t count = static_cast<std::uint32_t>(IrBufferFormat::Format32_32_32_32Float);
|
||||
std::array<std::uint32_t, count> labels{};
|
||||
std::vector<std::uint32_t> branch{spv::OpSwitch, format, invalid};
|
||||
for (std::uint32_t index = 0; index < count; ++index) {
|
||||
labels[index] = state.module.AllocateId();
|
||||
branch.insert(branch.end(), {index + 1u, labels[index]});
|
||||
}
|
||||
const auto type = store || components == 1u ? TypeU32(state) : TypeU32Composite(state, components);
|
||||
const auto result = state.module.AllocateId();
|
||||
std::vector<std::uint32_t> phi{spv::OpPhi, type, result};
|
||||
state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
|
||||
state.module.AddFunction(branch);
|
||||
for (std::uint32_t index = 0; index < count; ++index) {
|
||||
EmitLabel(state, labels[index]);
|
||||
const auto value = formatted(context, instruction, resource, GetFormatInfo(static_cast<IrBufferFormat>(index + 1u)), components, store);
|
||||
const auto exit = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpBranch, exit);
|
||||
EmitLabel(state, exit);
|
||||
state.module.AddFunction(spv::OpBranch, merge);
|
||||
phi.insert(phi.end(), {value, exit});
|
||||
}
|
||||
EmitLabel(state, invalid);
|
||||
RecordBdaFault(state, resource.base, ConstantU32(state, 0u), ConstantU32(state, instruction.Flags<MemoryFlags>().pc), BdaAbi::FaultReason::InvalidDescriptor);
|
||||
const auto zero = store || components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components);
|
||||
const auto invalidExit = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpBranch, invalidExit);
|
||||
EmitLabel(state, invalidExit);
|
||||
state.module.AddFunction(spv::OpBranch, merge);
|
||||
phi.insert(phi.end(), {zero, invalidExit});
|
||||
EmitLabel(state, merge);
|
||||
state.module.AddFunction(phi);
|
||||
const auto unsupported = Binary(state, spv::OpLogicalOr, TypeBool(state), equal(state, format, ConstantU32(state, 0u)), Binary(state, spv::OpUGreaterThan, TypeBool(state), format, ConstantU32(state, count)));
|
||||
StopBdaInvocationIf(state, unsupported);
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) {
|
||||
auto& state = context.state;
|
||||
const auto& memory = context.Memory(instruction);
|
||||
const auto type = components == 1u ? TypeU32(state) : TypeU32Composite(state, components);
|
||||
const auto zero = components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components);
|
||||
return EmitValueOrDefaultIfCondition(state, context.Arg(instruction, instruction.ArgumentCount() - 1u), type, zero, [&] {
|
||||
const auto resource = descriptor(context, instruction);
|
||||
if (memory.formatted) return EmitValueOrDefaultIfCondition(state, resource.valid, type, zero, [&] { return formattedDispatch(context, instruction, resource, components, false); });
|
||||
std::array<std::uint32_t, 4> values{};
|
||||
for (std::uint32_t component = 0; component < components; ++component) {
|
||||
const auto access = address(context, instruction, resource, component * 4u, memory.dataBits / 8u);
|
||||
values[component] = EmitValueOrZeroIfCondition(state, access.inBounds, [&] { return EmitBdaRead(context, instruction, access.guest, memory.dataBits); });
|
||||
}
|
||||
return composite(state, components, values);
|
||||
});
|
||||
}
|
||||
|
||||
void EmitRuntimeBufferStore(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) {
|
||||
auto& state = context.state;
|
||||
const auto& memory = context.Memory(instruction);
|
||||
EmitIfCondition(state, context.Arg(instruction, instruction.ArgumentCount() - 1u), [&] {
|
||||
const auto resource = descriptor(context, instruction);
|
||||
if (memory.formatted) {
|
||||
EmitIfCondition(state, resource.valid, [&] { static_cast<void>(formattedDispatch(context, instruction, resource, components, true)); });
|
||||
return;
|
||||
}
|
||||
const auto values = storeValues(context, instruction, components);
|
||||
for (std::uint32_t component = 0; component < components; ++component) {
|
||||
const auto access = address(context, instruction, resource, component * 4u, memory.dataBits / 8u);
|
||||
EmitIfCondition(state, access.inBounds, [&] { EmitBdaWrite(context, instruction, access.guest, values[component], memory.dataBits); });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t EmitRuntimeScalarBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction) {
|
||||
auto& state = context.state;
|
||||
const auto resource = descriptor(context, instruction);
|
||||
const auto u64 = TypeScalarU64(state);
|
||||
const auto size = Select(state, u64, equal(state, resource.stride, ConstantU32(state, 0u)), Unary(state, spv::OpUConvert, u64, resource.records), Binary(state, spv::OpIMul, u64, Unary(state, spv::OpUConvert, u64, resource.stride), Unary(state, spv::OpUConvert, u64, resource.records)));
|
||||
const auto address = EmitAddU32(state, context.Arg(instruction, 1u), ConstantU32(state, context.Memory(instruction).offset));
|
||||
const auto noCarry = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address, context.Arg(instruction, 1u));
|
||||
const auto byte = EmitAndConstant(state, address, ~3u);
|
||||
const auto offset = Unary(state, spv::OpUConvert, u64, byte);
|
||||
const auto inBounds = both(state, noCarry, both(state, resource.valid, Binary(state, spv::OpULessThanEqual, TypeBool(state), Binary(state, spv::OpIAdd, u64, offset, BdaConstant(state, 4u)), size)));
|
||||
return EmitValueOrZeroIfCondition(state, inBounds, [&] { return EmitBdaRead(context, instruction, AddBdaAddress(context, instruction, resource.base, offset, false), 32u); });
|
||||
}
|
||||
|
||||
std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t bits, std::uint32_t active, const std::function<std::uint32_t(std::uint32_t)>& operation) {
|
||||
auto& state = context.state;
|
||||
if (bits != 32u && bits != 64u) context.Fail(instruction, "unsupported runtime buffer atomic width");
|
||||
if (state.bdaAtomicPointerFunction == 0u || state.bdaNoteWriteFunction == 0u) context.Fail(instruction, "BDA atomic functions are missing");
|
||||
const auto type = bits == 64u ? TypeU64(state) : TypeU32(state);
|
||||
const auto scalar = bits == 64u ? TypeScalarU64(state) : TypeU32(state);
|
||||
const auto zero = bits == 64u ? ConstantU64(state, 0u) : ConstantU32(state, 0u);
|
||||
return EmitValueOrDefaultIfCondition(state, active, type, zero, [&] {
|
||||
const auto resource = descriptor(context, instruction);
|
||||
const auto access = address(context, instruction, resource, 0u, bits / 8u);
|
||||
return EmitValueOrDefaultIfCondition(state, access.inBounds, type, zero, [&] {
|
||||
const auto unaligned = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), access.guest, BdaConstant(state, bits / 8u - 1u)), BdaConstant(state, 0u));
|
||||
faultIf(context, instruction, unaligned, access.guest, bits / 8u, BdaAbi::FaultReason::Unaligned);
|
||||
return EmitValueOrDefaultIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), type, zero, [&] {
|
||||
const auto physical = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaAtomicPointerFunction, access.guest, ConstantU32(state, bits / 8u), ConstantU32(state, instruction.Flags<MemoryFlags>().pc));
|
||||
const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u));
|
||||
return EmitValueOrDefaultIfCondition(state, mapped, type, zero, [&] {
|
||||
const auto pointer = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, scalar), pointer, physical);
|
||||
const auto result = operation(pointer);
|
||||
state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, access.guest);
|
||||
return result;
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue