shifathosn
/
AnyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(shader): store multi-dword global writes with one BDA lookup 

A global_store_dwordx2/x3/x4 was split into one StoreAddressU32 per
dword, each with its own write-pointer lookup and its own 4-byte store.
Across a wave those are strided partial writes, which are very slow to
host memory: RADV's clear_htile_mask kernel (one dwordx4 load and store
per thread over 640 KiB) took ~9 ms. Multi-dword stores are now one
StoreAddressU32xN; the emitter probes the whole span once and, when it
is mapped writable and 4-byte aligned, stores it as one vector (16- or
8-byte aligned) or as consecutive dwords, noting each written page.
Anything else takes the per-dword path as before, so faults and
range-spanning stores behave as they did. APS5_BDA_DWORD_STORES=1
restores per-dword stores.

AI-assisted (Claude Code).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Scott Schuster 2026-10-09 09:47:07 -05:00
parent 001124896d
commit b958425977
14 changed files with 325 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
core/libs/prx/libSceAgcDriver/tests/BdaExecution.cpp
View File

@ -1,9 +1,12 @@
#include "BdaShader.hpp"
#include "BdaAbi.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
#include <algorithm>
#include <array>
#include <cstring>
#include <limits>
#include <string>
#include <vector>
namespace {
@ -87,6 +90,72 @@ private:
}
static void RunBdaDwordWriteTests(const Context& context) {
namespace Abi = ShaderRecompiler::BdaAbi;
constexpr std::uint64_t guest = 0x7fff12340f80ULL;
constexpr std::size_t firstBytes = 160;
Buffer first(context, firstBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
Buffer second(context, 16, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
Buffer readOnly(context, 16, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
const std::array<Abi::Range, 3> ranges{{
{guest, guest + firstBytes, first.DeviceAddress(), Abi::Read | Abi::Write, 0},
{guest + firstBytes, guest + firstBytes + 16, second.DeviceAddress(), Abi::Read | Abi::Write, 0},
{guest + firstBytes + 16, guest + firstBytes + 32, readOnly.DeviceAddress(), Abi::Read, 0},
}};
Buffer table(context, sizeof(Abi::Header) + sizeof(ranges), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
Buffer fault(context, Abi::FaultBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
Buffer output(context, 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
const std::array<std::uint32_t, 4> values{0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u};
const auto bytesAt = [&](std::uint64_t address) -> std::byte* {
for (std::size_t index = 0; index < ranges.size(); ++index) {
if (address < ranges[index].begin || address >= ranges[index].end) continue;
auto& buffer = index == 0 ? first : index == 1 ? second : readOnly;
return buffer.Bytes().data() + (address - ranges[index].begin);
}
return nullptr;
};
const auto noted = [&](std::uint64_t address) {
const auto page = static_cast<std::uint32_t>(address >> Abi::WrittenPageShift) + 1u;
std::vector<std::uint32_t> slots(Abi::WrittenPageSlots);
std::memcpy(slots.data(), fault.Bytes().data() + Abi::WrittenSlotsWord * sizeof(std::uint32_t), slots.size() * sizeof(std::uint32_t));
return std::find(slots.begin(), slots.end(), page) != slots.end();
};
const auto write = [&](std::uint64_t address, std::uint32_t dwords, std::uint32_t storedBytes, Abi::FaultReason reason) {
const Abi::Header header{Abi::Version, static_cast<std::uint32_t>(ranges.size()), sizeof(Abi::Range), 0};
std::memcpy(table.Bytes().data(), &header, sizeof(header));
std::memcpy(table.Bytes().data() + sizeof(header), ranges.data(), sizeof(ranges));
std::memset(fault.Bytes().data(), 0, fault.Bytes().size());
for (auto* buffer : {&first, &second, &readOnly}) std::memset(buffer->Bytes().data(), 0xa5, buffer->Bytes().size());
Pipeline pipeline(context, MakeBdaDwordWriteTestShader(address, dwords, values.data()), {&table, &fault, &output});
pipeline.Run(1);
Abi::Fault report{};
std::memcpy(&report, fault.Bytes().data(), sizeof(report));
for (std::uint32_t byte = 0; byte < dwords * 4u; ++byte) {
const auto* stored = bytesAt(address + byte);
const auto expected = byte < storedBytes ? static_cast<std::byte>(values[byte / 4u] >> ((byte % 4u) * 8u)) : std::byte{0xa5};
Require(stored != nullptr && *stored == expected, "BDA dword write stored incorrect bytes at offset " + std::to_string(byte) + " of guest + " + std::to_string(address - guest));
}
Require(*bytesAt(address - 1) == std::byte{0xa5}, "BDA dword write touched the byte before it");
if (address + dwords * 4u < guest + firstBytes + 32) Require(*bytesAt(address + dwords * 4u) == std::byte{0xa5}, "BDA dword write touched the byte after it");
if (static_cast<std::uint32_t>(reason) == 0) {
Require(report.state == Abi::FaultState::Empty, "a mapped BDA dword write published a fault");
} else {
Require(report.state == Abi::FaultState::Ready && report.reason == reason && report.instruction == 0x1234, "a BDA dword write did not publish its fault");
}
if (storedBytes != 0) Require(noted(address) && noted(address + storedBytes - 1u), "a BDA dword write did not note its written pages");
};
write(guest + 16, 4, 16, static_cast<Abi::FaultReason>(0));
write(guest + 20, 4, 16, static_cast<Abi::FaultReason>(0));
write(guest + 8, 2, 8, static_cast<Abi::FaultReason>(0));
write(guest + 12, 2, 8, static_cast<Abi::FaultReason>(0));
write(guest + 32, 3, 12, static_cast<Abi::FaultReason>(0));
write(guest + 0x78, 4, 16, static_cast<Abi::FaultReason>(0));
write(guest + 0x7c, 2, 8, static_cast<Abi::FaultReason>(0));
write(guest + 41, 4, 16, static_cast<Abi::FaultReason>(0));
write(guest + firstBytes - 8, 4, 16, static_cast<Abi::FaultReason>(0));
write(guest + firstBytes + 12, 4, 4, Abi::FaultReason::Permission);
}
void RunBdaExecutionTests(const Context& context) {
namespace Abi = ShaderRecompiler::BdaAbi;
Buffer first(context, 3, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
@ -206,4 +275,5 @@ void RunBdaExecutionTests(const Context& context) {
}
ranges[0].permissions = 0;
run(guest, 8, 0, Abi::FaultReason::Permission);
RunBdaDwordWriteTests(context);
}

18
core/libs/prx/libSceAgcDriver/tests/BdaShader.cpp
View File

@ -7,11 +7,12 @@
namespace {
template<typename TEmit>
std::vector<std::uint32_t> MakeShader(std::uint64_t address, bool coherent, bool stops, const TEmit& emit, std::uint32_t extracted = 0) {
std::vector<std::uint32_t> MakeShader(std::uint64_t address, bool coherent, bool stops, const TEmit& emit, std::uint32_t extracted = 0, bool writes = false) {
using namespace ShaderRecompiler;
IrProgram program;
program.Resources().stage = IrShaderStage::Compute;
program.Info().usesDma = true;
program.Info().bdaWrites = writes;
if (coherent) {
MemoryInfo memory;
memory.coherent = true;
@ -83,6 +84,21 @@ std::vector<std::uint32_t> MakeBdaSpanReadTestShader(std::uint64_t address, std:
}, extracted);
}
std::vector<std::uint32_t> MakeBdaDwordWriteTestShader(std::uint64_t address, std::uint32_t dwords, const std::uint32_t* values) {
using namespace ShaderRecompiler;
return MakeShader(address, false, true, [&](SpirvValueEmitContext& ctx, const IrValue& instruction, std::uint32_t base) {
auto& state = ctx.state;
std::vector<std::uint32_t> parts;
for (std::uint32_t dword = 0; dword < dwords; ++dword) parts.push_back(ConstantU32(state, values[dword]));
const auto composite = state.module.AllocateId();
if (dwords == 2u) state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 2u), composite, parts[0], parts[1]);
else if (dwords == 3u) state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 3u), composite, parts[0], parts[1], parts[2]);
else state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4u), composite, parts[0], parts[1], parts[2], parts[3]);
EmitBdaDwordWrites(ctx, instruction, base, 0u, dwords, composite);
return std::vector<std::uint32_t>{ConstantU32(state, 1u)};
}, 0u, true);
}
std::vector<std::uint32_t> MakeBdaDwordReadTestShader(std::uint64_t address, std::uint32_t dwords, bool coherent, bool stops) {
using namespace ShaderRecompiler;
return MakeShader(address, coherent, stops, [&](SpirvValueEmitContext& ctx, const IrValue& instruction, std::uint32_t base) {

1
core/libs/prx/libSceAgcDriver/tests/BdaShader.hpp
View File

@ -8,6 +8,7 @@
std::vector<std::uint32_t> MakeBdaTestShader(std::uint64_t address, std::uint32_t bits, std::int64_t offset = 0);
std::vector<std::uint32_t> MakeBdaDwordReadTestShader(std::uint64_t address, std::uint32_t dwords, bool coherent, bool stops);
std::vector<std::uint32_t> MakeBdaSpanReadTestShader(std::uint64_t address, std::uint32_t offset, std::uint32_t extracted, bool coherent, bool stops);
std::vector<std::uint32_t> MakeBdaDwordWriteTestShader(std::uint64_t address, std::uint32_t dwords, const std::uint32_t* values);
void RunBdaExecutionTests(const AgcDriver::Graphics::Context& context);
void RunBdaContractTests();

94
core/libs/prx/libSceAgcDriver/tests/execution/MaskedGlobalStore.cpp Normal file
View File

@ -0,0 +1,94 @@
#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
#include "prx/libc/include/GuestAllocations.hpp"
#include "Recompiler.hpp"
#include "VulkanTestDevice.hpp"
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#endif
#include <array>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
namespace {
using AgcDriver::Graphics::Require;
using ShaderRecompiler::ShaderStage;
constexpr std::size_t BlockBytes = 64 * 1024;
alignas(256) constexpr std::array<std::uint32_t, 17> MaskedStoreCode{0xd7460000u, 0x04010c06u, 0x34000084u, 0xdc388000u, 0x04020000u, 0xbf8c3f70u, 0xd7710008u, 0x00120805u, 0xd7710009u, 0x00120a05u, 0xd771000au, 0x00120c05u, 0xd771000bu, 0x00120e05u, 0xdc788000u, 0x00020800u, 0xbf810000u};
class GuestBlock {
public:
GuestBlock() {
#ifdef _WIN32
block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
#else
block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
#endif
Require(block != nullptr, "masked global store: cannot allocate the guest block");
GuestAllocations::Mutation().Add(block, BlockBytes, true, true);
}
~GuestBlock() { GuestAllocations::Mutation().Remove(block); }
GuestBlock(const GuestBlock&) = delete;
GuestBlock& operator=(const GuestBlock&) = delete;
std::uint8_t* Data() { return block; }
private:
std::uint8_t* block = nullptr;
};
void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t waveSize, std::uint32_t value, std::uint32_t keep) {
constexpr std::uint32_t groups = 64;
constexpr std::size_t words = groups * 64u * 4u;
auto* data = reinterpret_cast<std::uint32_t*>(guest.Data());
for (std::size_t index = 0; index < BlockBytes / 4u; ++index) data[index] = static_cast<std::uint32_t>(index * 0x9e3779b1u);
const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(guest.Data()));
const std::vector<std::uint32_t> userData{0, 0, static_cast<std::uint32_t>(address), static_cast<std::uint32_t>(address >> 32u), value, keep};
const std::span<const std::uint32_t> code(MaskedStoreCode);
const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
const ShaderRecompiler::ShaderComputeStageInfo compute{{64, 1, 1}, 0, {true, false, false}, false, 1};
ShaderRecompiler::RecompileRequest request{
{ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
{waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
device.Target(),
{0, 0, 0, 128}
};
request.useCache = false;
const auto result = ShaderRecompiler::Recompile(request);
device.Dispatch(result, groups, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
device.WaitIdle();
for (std::size_t index = 0; index < BlockBytes / 4u; ++index) {
const auto original = static_cast<std::uint32_t>(index * 0x9e3779b1u);
const auto expected = index < words ? (original & keep) | value : original;
Require(data[index] == expected, "a masked global_store_dwordx4 left dword " + std::to_string(index) + " incorrect");
}
}
}
int main() {
try {
const auto device = OpenVulkanTestDevice();
if (!device) return VulkanTestSkipped;
GuestBlock guest;
for (const auto waveSize : {64u, 32u}) {
Run(*device, guest, waveSize, 0u, 0x00000ff0u);
Run(*device, guest, waveSize, 0xfffff00fu, 0x00000ff0u);
}
std::puts("masked global store tests passed");
return 0;
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
}

3
core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp
View File

@ -293,6 +293,9 @@ enum class IrOpcode : std::uint16_t {
StoreAddressU8,
StoreAddressU16,
StoreAddressU32,
StoreAddressU32x2,
StoreAddressU32x3,
StoreAddressU32x4,
AddressAtomicSwap32,
AddressAtomicCmpSwap32,
AddressAtomicIAdd32,

9
core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp
View File

@ -334,6 +334,9 @@ constexpr std::array<OpcodeMeta, static_cast<std::size_t>(IrOpcode::Count)> Meta
makeMeta("StoreAddressU8", Void, AddressResource, U32, U32, U8, U1),
makeMeta("StoreAddressU16", Void, AddressResource, U32, U32, U16, U1),
makeMeta("StoreAddressU32", Void, AddressResource, U32, U32, U32, U1),
makeMeta("StoreAddressU32x2", Void, AddressResource, U32, U32, U32x2, U1),
makeMeta("StoreAddressU32x3", Void, AddressResource, U32, U32, U32x3, U1),
makeMeta("StoreAddressU32x4", Void, AddressResource, U32, U32, U32x4, U1),
makeMeta("AddressAtomicSwap32", U32, AddressResource, U32, U32, U32, U1),
makeMeta("AddressAtomicCmpSwap32", U32, AddressResource, U32, U32, U32, U32, U1),
makeMeta("AddressAtomicIAdd32", U32, AddressResource, U32, U32, U32, U1),
@ -747,6 +750,12 @@ AddressOpcodeInfo AddressOpcodeInfoOf(IrOpcode opcode) {
return {AddressAccess::Write, 16u};
case IrOpcode::StoreAddressU32:
return {AddressAccess::Write, 32u};
case IrOpcode::StoreAddressU32x2:
return {AddressAccess::Write, 32u, 2u};
case IrOpcode::StoreAddressU32x3:
return {AddressAccess::Write, 32u, 3u};
case IrOpcode::StoreAddressU32x4:
return {AddressAccess::Write, 32u, 4u};
case IrOpcode::AddressAtomicSwap32:
case IrOpcode::AddressAtomicCmpSwap32:
case IrOpcode::AddressAtomicIAdd32:

1
core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp
View File

@ -28,6 +28,7 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::
void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u);
void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits);
void DefineBdaByteWriteFunctions(SpirvEmitterState& state);
void EmitBdaDwordWrites(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, std::uint32_t dwords, std::uint32_t value);
std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bytes, const std::function<std::uint32_t(std::uint32_t)>& operation);
// Reads the dwords of a 1-4 dword load at address + offset that the program extracts, with one
// table lookup for the whole span; the per-byte lookups of EmitBdaRead remain the fallback (and

3
core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp
View File

@ -243,6 +243,9 @@ void EmitLoadAddressU32x4(SpirvValueEmitContext& ctx, const IrValue& inst);
void EmitStoreAddressU8(SpirvValueEmitContext& ctx, const IrValue& inst);
void EmitStoreAddressU16(SpirvValueEmitContext& ctx, const IrValue& inst);
void EmitStoreAddressU32(SpirvValueEmitContext& ctx, const IrValue& inst);
void EmitStoreAddressU32x2(SpirvValueEmitContext& ctx, const IrValue& inst);
void EmitStoreAddressU32x3(SpirvValueEmitContext& ctx, const IrValue& inst);
void EmitStoreAddressU32x4(SpirvValueEmitContext& ctx, const IrValue& inst);
std::uint32_t EmitAddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst);
void EmitLoadBufferU8(SpirvValueEmitContext& ctx, const IrValue& inst);
void EmitLoadBufferU16(SpirvValueEmitContext& ctx, const IrValue& inst);

1
core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp
View File

@ -109,6 +109,7 @@ struct SpirvEmitterState {
// every read takes the byte path.
std::uint32_t bdaProbeFunction = 0;
std::uint32_t bdaWritePointerFunction = 0;
std::uint32_t bdaWriteProbeFunction = 0;
std::uint32_t bdaAtomicPointerFunction = 0;
std::uint32_t bdaNoteWriteFunction = 0;
std::array<std::uint32_t, 2> bdaByteWriteFunctions {};

1
core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp
View File

@ -187,6 +187,7 @@ void DefineGetBdaPointer(SpirvEmitterState& state) {
DefineBdaSpanReadFunctions(state);
if (state.program.Info().bdaWrites) {
state.bdaWritePointerFunction = DefineBdaLookup(state, "get_bda_write_pointer", true, BdaAbi::Write);
if (!BdaByteReadsForced()) state.bdaWriteProbeFunction = DefineBdaLookup(state, "probe_bda_write_pointer", false, BdaAbi::Write);
state.bdaAtomicPointerFunction = DefineBdaLookup(state, "get_bda_atomic_pointer", true, BdaAbi::Read | BdaAbi::Write);
state.bdaNoteWriteFunction = DefineBdaNoteWrite(state);
DefineBdaByteWriteFunctions(state);

72
core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp
View File

@ -422,6 +422,78 @@ void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t
});
}
void EmitBdaDwordWrites(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, std::uint32_t dwords, std::uint32_t value) {
auto& state = ctx.state;
if (dwords < 2u || dwords > 4u) ctx.Fail(inst, "unsupported BDA dword store count");
if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing");
const auto u32 = TypeU32(state);
const auto u64 = TypeScalarU64(state);
const auto boolean = TypeBool(state);
std::array<std::uint32_t, 4> values{};
std::array<std::uint32_t, 4> addresses{};
for (std::uint32_t dword = 0; dword < dwords; ++dword) {
values[dword] = state.module.AllocateId();
state.module.AddFunction(spv::OpCompositeExtract, u32, values[dword], value, dword);
addresses[dword] = AddBdaImmediate(ctx, inst, address, static_cast<std::int32_t>(offset + dword * 4u));
}
const auto storeDwords = [&] {
for (std::uint32_t dword = 0; dword < dwords; ++dword) EmitBdaStore(ctx, inst, addresses[dword], values[dword], 32u);
};
if (state.bdaWriteProbeFunction == 0) {
storeDwords();
return;
}
const auto instruction = ConstantU32(state, inst.Flags<MemoryFlags>().pc);
const auto accessMask = BdaAccessMask(state, inst);
const auto physical = state.module.AllocateId();
state.module.AddFunction(spv::OpFunctionCall, u64, physical, state.bdaWriteProbeFunction, addresses[0], ConstantU32(state, dwords * 4u), instruction);
const auto mapped = Binary(state, spv::OpINotEqual, boolean, physical, BdaConstant(state, 0u));
const auto aligned = Binary(state, spv::OpLogicalAnd, boolean, Unary(state, spv::OpLogicalNot, boolean, BdaAddressUnaligned(state, addresses[0])), Unary(state, spv::OpLogicalNot, boolean, BdaAddressUnaligned(state, physical)));
const auto wide = Binary(state, spv::OpLogicalAnd, boolean, mapped, aligned);
const auto wideLabel = state.module.AllocateId();
const auto dwordLabel = state.module.AllocateId();
const auto merge = state.module.AllocateId();
state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
state.module.AddFunction(spv::OpBranchConditional, wide, wideLabel, dwordLabel);
EmitLabel(state, wideLabel);
const auto vectorBytes = dwords == 3u ? 4u : dwords * 4u;
const auto separate = [&] {
for (std::uint32_t dword = 0; dword < dwords; ++dword) {
const auto element = dword == 0u ? physical : Binary(state, spv::OpIAdd, u64, physical, BdaConstant(state, dword * 4u));
const auto pointer = state.module.AllocateId();
state.module.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, element);
state.module.AddFunction(spv::OpStore, pointer, values[dword], accessMask, 4u);
}
};
if (dwords == 3u) {
separate();
} else {
const auto vectorAligned = Binary(state, spv::OpIEqual, boolean, Binary(state, spv::OpBitwiseAnd, u64, physical, BdaConstant(state, vectorBytes - 1u)), BdaConstant(state, 0u));
const auto vectorType = TypeU32Vector(state, dwords);
const auto vector = state.module.AllocateId();
if (dwords == 2u) state.module.AddFunction(spv::OpCompositeConstruct, vectorType, vector, values[0], values[1]);
else state.module.AddFunction(spv::OpCompositeConstruct, vectorType, vector, values[0], values[1], values[2], values[3]);
EmitIfCondition(state, vectorAligned, [&] {
const auto pointer = state.module.AllocateId();
state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, vectorType), pointer, physical);
state.module.AddFunction(spv::OpStore, pointer, vector, accessMask, vectorBytes);
});
EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, vectorAligned), separate);
}
const auto last = addresses[dwords - 1u];
state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, addresses[0]);
const auto pageShift = BdaConstant(state, BdaAbi::WrittenPageShift);
const auto crosses = Binary(state, spv::OpINotEqual, boolean, Binary(state, spv::OpShiftRightLogical, u64, addresses[0], pageShift), Binary(state, spv::OpShiftRightLogical, u64, last, pageShift));
EmitIfCondition(state, crosses, [&] {
state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, last);
});
state.module.AddFunction(spv::OpBranch, merge);
EmitLabel(state, dwordLabel);
storeDwords();
state.module.AddFunction(spv::OpBranch, merge);
EmitLabel(state, merge);
}
std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bytes, const std::function<std::uint32_t(std::uint32_t)>& operation) {
auto& state = ctx.state;
if (bytes != 4u && bytes != 8u) ctx.Fail(inst, "unsupported BDA atomic width");

3
core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp
View File

@ -508,6 +508,9 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) {
case IrOpcode::StoreAddressU8: return Invoke(EmitStoreAddressU8, ctx, inst);
case IrOpcode::StoreAddressU16: return Invoke(EmitStoreAddressU16, ctx, inst);
case IrOpcode::StoreAddressU32: return Invoke(EmitStoreAddressU32, ctx, inst);
case IrOpcode::StoreAddressU32x2: return Invoke(EmitStoreAddressU32x2, ctx, inst);
case IrOpcode::StoreAddressU32x3: return Invoke(EmitStoreAddressU32x3, ctx, inst);
case IrOpcode::StoreAddressU32x4: return Invoke(EmitStoreAddressU32x4, ctx, inst);
case IrOpcode::AddressAtomicSwap32: return Invoke(EmitAddressAtomic, ctx, inst);
case IrOpcode::AddressAtomicCmpSwap32: return Invoke(EmitAddressAtomic, ctx, inst);
case IrOpcode::AddressAtomicIAdd32: return Invoke(EmitAddressAtomic, ctx, inst);

20
core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp
View File

@ -1121,6 +1121,26 @@ void EmitStoreAddressU32(SpirvValueEmitContext& ctx, const IrValue& inst) {
StoreAddress(ctx, inst, 32u);
}
void StoreAddressWide(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components) {
const auto& mem = ctx.Memory(inst);
if (mem.kind != ResourceKind::Flat && mem.kind != ResourceKind::Global) ctx.Fail(inst, "must write a physical address resource");
EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() {
EmitBdaDwordWrites(ctx, inst, GuestAddressBase(ctx, inst, mem), mem.offset, components, ctx.Arg(inst, inst.ArgumentCount() - 2u));
});
}
void EmitStoreAddressU32x2(SpirvValueEmitContext& ctx, const IrValue& inst) {
StoreAddressWide(ctx, inst, 2u);
}
void EmitStoreAddressU32x3(SpirvValueEmitContext& ctx, const IrValue& inst) {
StoreAddressWide(ctx, inst, 3u);
}
void EmitStoreAddressU32x4(SpirvValueEmitContext& ctx, const IrValue& inst) {
StoreAddressWide(ctx, inst, 4u);
}
std::uint32_t EmitAddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst) {
return AddressAtomic(ctx, inst);
}

30
core/shader/recompiler/Translation/src/MemoryInstructions/FlatInstructions.cpp
View File

@ -1,6 +1,7 @@
#include "Translation/MemoryInstructions.hpp"
#include "Translation/TranslationContext.hpp"
#include <algorithm>
#include <array>
#include <cstdlib>
#include <stdexcept>
@ -15,6 +16,22 @@ bool wideAddressLoadsEnabled() {
return !byteReads;
}
bool wideAddressStoresEnabled() {
static const bool byteWrites = std::getenv("APS5_BDA_DWORD_STORES") != nullptr;
return !byteWrites;
}
IrOpcode wideAddressStoreOpcode(std::uint32_t dwords) {
switch (dwords) {
case 2u:
return IrOpcode::StoreAddressU32x2;
case 3u:
return IrOpcode::StoreAddressU32x3;
default:
return IrOpcode::StoreAddressU32x4;
}
}
IrOpcode wideAddressLoadOpcode(std::uint32_t dwords) {
switch (dwords) {
case 2u:
@ -136,6 +153,19 @@ bool TranslationContext::flatStore(const RdnaInstruction& inst) {
const AddressOperands address = readAddressOperands(inst, 0u);
IrValue& active = ir.GetExec();
const std::uint32_t count = memory.dataBits == 32u ? memory.dataDwords : 1u;
if (count > 1u && count <= 4u && memory.kind != ResourceKind::Scratch && wideAddressStoresEnabled()) {
MemoryInfo group = memory;
group.dataDwords = count;
group.componentCount = count;
group.componentIndex = 0u;
std::array<IrValue*, 4> parts{};
for (std::uint32_t index = 0u; index < count; ++index) parts[index] = &readU32(offsetOperand(inst.destination, index)).Value();
IrValue& data = count == 2u ? ir.Emit(IrOpcode::CompositeConstructU32x2, IrOpcodeType(IrOpcode::CompositeConstructU32x2), {parts[0], parts[1]})
: count == 3u ? ir.Emit(IrOpcode::CompositeConstructU32x3, IrOpcodeType(IrOpcode::CompositeConstructU32x3), {parts[0], parts[1], parts[2]})
: ir.Emit(IrOpcode::CompositeConstructU32x4, IrOpcodeType(IrOpcode::CompositeConstructU32x4), {parts[0], parts[1], parts[2], parts[3]});
(void)ir.Emit(wideAddressStoreOpcode(count), IrType::Void, {address.resource, address.low, address.high, &data, &active}, addMemoryInfo(group, inst.programCounter));
return true;
}
for (std::uint32_t index = 0u; index < count; ++index) {
MemoryInfo component = memory;
component.offset += index * 4u;