Merge pull request #1190 from MichonGoddijn231849/feat/flat-shared-private-apertures
feat(recompiler): FLAT addresses in the shared and private apertures reach LDS and scratch
This commit is contained in:
Adrià
GitHub
commit
b65d6b71be
|
|
@ -0,0 +1,501 @@
|
|||
#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
|
||||
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
|
||||
#include "prx/libc/include/GuestAllocations.hpp"
|
||||
#include "Recompiler.hpp"
|
||||
#include "VulkanTestDevice.hpp"
|
||||
#ifdef _WIN32
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#endif
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <exception>
|
||||
#include <iostream>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace {
|
||||
|
||||
using AgcDriver::Graphics::Require;
|
||||
using ShaderRecompiler::ShaderStage;
|
||||
using Words = std::vector<std::uint32_t>;
|
||||
|
||||
constexpr std::uint32_t SharedHigh = 0x80000000u;
|
||||
constexpr std::uint32_t PrivateHigh = 0x70000000u;
|
||||
constexpr std::uint32_t LdsDwords = 256u;
|
||||
constexpr std::uint32_t ScratchDwords = 4u;
|
||||
constexpr std::size_t GuestBytes = 65536u;
|
||||
constexpr std::uint32_t GuestStoreSlot = 0x200u;
|
||||
constexpr std::uint32_t GuestAtomicSlot = 0x800u;
|
||||
constexpr std::uint32_t WaitCnt0 = 0xbf8c0070u;
|
||||
constexpr std::uint32_t EndProgram = 0xbf810000u;
|
||||
constexpr std::uint32_t ShiftLaneBy5 = 0x34020085u;
|
||||
constexpr std::uint32_t StoreByteOp = 0x18u;
|
||||
constexpr std::uint32_t StoreShortOp = 0x1au;
|
||||
constexpr std::uint32_t StoreDwordOp = 0x1cu;
|
||||
constexpr std::uint32_t StoreDwordx2Op = 0x1du;
|
||||
constexpr std::uint32_t LoadUbyteOp = 0x08u;
|
||||
constexpr std::uint32_t LoadUshortOp = 0x0au;
|
||||
constexpr std::uint32_t LoadDwordOp = 0x0cu;
|
||||
constexpr std::uint32_t LoadDwordx2Op = 0x0du;
|
||||
|
||||
enum class Target : std::uint8_t { Shared, Private, Guest, Mixed };
|
||||
enum class AtomicOp : std::uint8_t { Swap, Cmpswap, Add, Sub, Smin, Umin, Smax, Umax, And, Or, Xor, Inc, Dec, Fcmpswap, Fmin, Fmax };
|
||||
|
||||
constexpr std::size_t Atomic32Ops = 13;
|
||||
constexpr std::array<std::uint32_t, 13> AtomicOpcodes32{0x30u, 0x31u, 0x32u, 0x33u, 0x35u, 0x36u, 0x37u, 0x38u, 0x39u, 0x3au, 0x3bu, 0x3cu, 0x3du};
|
||||
constexpr std::array<std::uint32_t, 16> AtomicOpcodes64{0x50u, 0x51u, 0x52u, 0x53u, 0x55u, 0x56u, 0x57u, 0x58u, 0x59u, 0x5au, 0x5bu, 0x5cu, 0x5du, 0x5eu, 0x5fu, 0x60u};
|
||||
constexpr std::array<const char*, 4> TargetNames{"shared", "private", "guest", "mixed"};
|
||||
constexpr std::array<const char*, 16> OpNames{"swap", "cmpswap", "add", "sub", "smin", "umin", "smax", "umax", "and", "or", "xor", "inc", "dec", "fcmpswap", "fmin", "fmax"};
|
||||
|
||||
struct Row {
|
||||
Target target;
|
||||
bool atomic;
|
||||
AtomicOp op;
|
||||
std::uint32_t width;
|
||||
std::uint32_t offset;
|
||||
std::uint32_t waveSize;
|
||||
};
|
||||
|
||||
class GuestBlock {
|
||||
public:
|
||||
GuestBlock() {
|
||||
#ifdef _WIN32
|
||||
block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, GuestBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
|
||||
#else
|
||||
block = static_cast<std::uint8_t*>(std::aligned_alloc(GuestBytes, GuestBytes));
|
||||
#endif
|
||||
Require(block != nullptr, "flat apertures: cannot allocate the guest block");
|
||||
std::memset(block, 0, GuestBytes);
|
||||
GuestAllocations::Mutation().Add(block, GuestBytes, true, true, true);
|
||||
}
|
||||
|
||||
~GuestBlock() {
|
||||
GuestAllocations::Mutation().Remove(block);
|
||||
}
|
||||
|
||||
GuestBlock(const GuestBlock&) = delete;
|
||||
GuestBlock& operator=(const GuestBlock&) = delete;
|
||||
|
||||
std::uintptr_t Address() const { return reinterpret_cast<std::uintptr_t>(block); }
|
||||
|
||||
private:
|
||||
std::uint8_t* block = nullptr;
|
||||
};
|
||||
|
||||
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(data);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
|
||||
}
|
||||
|
||||
std::string Hex(std::uint32_t value) {
|
||||
char text[16];
|
||||
std::snprintf(text, sizeof(text), "0x%08x", value);
|
||||
return text;
|
||||
}
|
||||
|
||||
std::string Hex64(std::uint64_t value) {
|
||||
char text[24];
|
||||
std::snprintf(text, sizeof(text), "0x%016llx", static_cast<unsigned long long>(value));
|
||||
return text;
|
||||
}
|
||||
|
||||
std::uint32_t FlatWord0(std::uint32_t op, bool glc, std::uint32_t offset = 0u) {
|
||||
return 0xdc000000u | (op << 18u) | (glc ? 0x10000u : 0u) | offset;
|
||||
}
|
||||
|
||||
std::uint32_t FlatWord1(std::uint32_t vdst, std::uint32_t data, std::uint32_t address) {
|
||||
return (vdst << 24u) | (0x7du << 16u) | (data << 8u) | address;
|
||||
}
|
||||
|
||||
void EmitFlat(Words& code, std::uint32_t op, bool glc, std::uint32_t vdst, std::uint32_t data, std::uint32_t address) {
|
||||
code.push_back(FlatWord0(op, glc));
|
||||
code.push_back(FlatWord1(vdst, data, address));
|
||||
}
|
||||
|
||||
void EmitTableLoad(Words& code, std::uint32_t vdst, std::uint32_t offset) {
|
||||
code.push_back(0xe0381000u | offset);
|
||||
code.push_back(0x80010001u | (vdst << 8u));
|
||||
}
|
||||
|
||||
void EmitOutputStore(Words& code, std::uint32_t vdata, std::uint32_t offset) {
|
||||
code.push_back(0xe0701000u | offset);
|
||||
code.push_back(0x80000001u | (vdata << 8u));
|
||||
}
|
||||
|
||||
std::uint32_t StoreOp(std::uint32_t width) {
|
||||
return width == 8u ? StoreByteOp : (width == 16u ? StoreShortOp : StoreDwordOp);
|
||||
}
|
||||
|
||||
std::uint32_t LoadOp(std::uint32_t width) {
|
||||
return width == 8u ? LoadUbyteOp : (width == 16u ? LoadUshortOp : LoadDwordOp);
|
||||
}
|
||||
|
||||
Words Kernel32(const Row& row) {
|
||||
Words code{ShiftLaneBy5};
|
||||
EmitTableLoad(code, 2u, 0u);
|
||||
EmitTableLoad(code, 6u, 16u);
|
||||
code.push_back(WaitCnt0);
|
||||
if (row.atomic) {
|
||||
EmitFlat(code, StoreDwordOp, false, 0u, 6u, 2u);
|
||||
code.push_back(WaitCnt0);
|
||||
EmitFlat(code, AtomicOpcodes32[static_cast<std::size_t>(row.op)], true, 10u, 4u, 2u);
|
||||
code.push_back(WaitCnt0);
|
||||
EmitFlat(code, LoadDwordOp, false, 11u, 0u, 2u);
|
||||
code.push_back(WaitCnt0);
|
||||
EmitOutputStore(code, 10u, 0u);
|
||||
EmitOutputStore(code, 11u, 4u);
|
||||
} else {
|
||||
EmitFlat(code, StoreOp(row.width), false, 0u, 4u, 2u);
|
||||
code.push_back(WaitCnt0);
|
||||
EmitFlat(code, LoadOp(row.width), false, 10u, 0u, 2u);
|
||||
EmitFlat(code, LoadUbyteOp, false, 11u, 0u, 2u);
|
||||
code.push_back(FlatWord0(LoadUbyteOp, false, row.width / 8u - 1u));
|
||||
code.push_back(FlatWord1(12u, 0u, 2u));
|
||||
code.push_back(WaitCnt0);
|
||||
EmitOutputStore(code, 10u, 0u);
|
||||
EmitOutputStore(code, 11u, 4u);
|
||||
EmitOutputStore(code, 12u, 8u);
|
||||
}
|
||||
code.push_back(EndProgram);
|
||||
return code;
|
||||
}
|
||||
|
||||
Words Kernel64(const Row& row) {
|
||||
Words code{ShiftLaneBy5};
|
||||
EmitTableLoad(code, 2u, 0u);
|
||||
EmitTableLoad(code, 6u, 16u);
|
||||
code.push_back(WaitCnt0);
|
||||
EmitFlat(code, StoreDwordx2Op, false, 0u, 8u, 2u);
|
||||
code.push_back(WaitCnt0);
|
||||
EmitFlat(code, AtomicOpcodes64[static_cast<std::size_t>(row.op)], true, 10u, 4u, 2u);
|
||||
code.push_back(WaitCnt0);
|
||||
EmitFlat(code, LoadDwordx2Op, false, 12u, 0u, 2u);
|
||||
code.push_back(WaitCnt0);
|
||||
EmitOutputStore(code, 10u, 0u);
|
||||
EmitOutputStore(code, 11u, 4u);
|
||||
EmitOutputStore(code, 12u, 8u);
|
||||
EmitOutputStore(code, 13u, 12u);
|
||||
code.push_back(EndProgram);
|
||||
return code;
|
||||
}
|
||||
|
||||
Words Kernel(const Row& row) {
|
||||
return row.atomic && row.width == 64u ? Kernel64(row) : Kernel32(row);
|
||||
}
|
||||
|
||||
std::uint32_t OperandA(std::uint32_t lane) {
|
||||
return 0x9e3779b1u * (lane + 1u) ^ 0x5a5a0f0fu;
|
||||
}
|
||||
|
||||
std::uint32_t InitValue(std::uint32_t lane) {
|
||||
return (lane % 2u) != 0u ? (0x80000000u | (lane * 0x101u)) : (lane * 0x35u + 7u);
|
||||
}
|
||||
|
||||
std::uint32_t CmpValue(std::uint32_t lane) {
|
||||
return (lane % 2u) != 0u ? InitValue(lane) + 1u : InitValue(lane);
|
||||
}
|
||||
|
||||
std::uint32_t Apply(AtomicOp op, std::uint32_t init, std::uint32_t a, std::uint32_t b) {
|
||||
const auto signedInit = static_cast<std::int32_t>(init);
|
||||
const auto signedA = static_cast<std::int32_t>(a);
|
||||
switch (op) {
|
||||
case AtomicOp::Swap: return a;
|
||||
case AtomicOp::Cmpswap: return init == b ? a : init;
|
||||
case AtomicOp::Add: return init + a;
|
||||
case AtomicOp::Sub: return init - a;
|
||||
case AtomicOp::Smin: return signedInit < signedA ? init : a;
|
||||
case AtomicOp::Umin: return init < a ? init : a;
|
||||
case AtomicOp::Smax: return signedInit > signedA ? init : a;
|
||||
case AtomicOp::Umax: return init > a ? init : a;
|
||||
case AtomicOp::And: return init & a;
|
||||
case AtomicOp::Or: return init | a;
|
||||
case AtomicOp::Xor: return init ^ a;
|
||||
case AtomicOp::Inc: return init >= a ? 0u : init + 1u;
|
||||
case AtomicOp::Dec: return (init == 0u || init > a) ? a : init - 1u;
|
||||
default: break;
|
||||
}
|
||||
return init;
|
||||
}
|
||||
|
||||
std::uint64_t Pack(std::uint32_t high, std::uint32_t low) {
|
||||
return (static_cast<std::uint64_t>(high) << 32u) | low;
|
||||
}
|
||||
|
||||
std::uint64_t Bits(double value) {
|
||||
std::uint64_t bits = 0;
|
||||
std::memcpy(&bits, &value, sizeof(bits));
|
||||
return bits;
|
||||
}
|
||||
|
||||
bool IsFloatOp(AtomicOp op) {
|
||||
return op == AtomicOp::Fcmpswap || op == AtomicOp::Fmin || op == AtomicOp::Fmax;
|
||||
}
|
||||
|
||||
std::uint64_t IntInit64(std::uint32_t lane) {
|
||||
return Pack((lane % 2u) != 0u ? (0x80000000u | (lane * 0x1357u)) : (lane * 0x0a0bu + 0x11u), InitValue(lane));
|
||||
}
|
||||
|
||||
std::uint64_t IntA64(std::uint32_t lane) {
|
||||
return Pack(0x85ebca6bu * (lane + 1u) ^ 0xc2b2ae35u, OperandA(lane));
|
||||
}
|
||||
|
||||
std::uint64_t IntCmp64(std::uint32_t lane) {
|
||||
return (lane % 2u) != 0u ? IntInit64(lane) + 1u : IntInit64(lane);
|
||||
}
|
||||
|
||||
double FloatInit(std::uint32_t lane) {
|
||||
return (lane % 2u) == 0u ? static_cast<double>(lane + 1u) * 3.25 : -static_cast<double>(lane + 1u) * 7.5;
|
||||
}
|
||||
|
||||
double FloatA(std::uint32_t lane) {
|
||||
return (lane % 2u) == 0u ? -static_cast<double>(lane + 2u) * 1.5 : static_cast<double>(lane + 3u) * 2.75 - 2.0;
|
||||
}
|
||||
|
||||
double FloatCmp(std::uint32_t lane) {
|
||||
return (lane % 2u) == 0u ? FloatInit(lane) : FloatInit(lane) + 1.0;
|
||||
}
|
||||
|
||||
std::uint64_t InitBits64(AtomicOp op, std::uint32_t lane) {
|
||||
return IsFloatOp(op) ? Bits(FloatInit(lane)) : IntInit64(lane);
|
||||
}
|
||||
|
||||
std::uint64_t OperandBits64(AtomicOp op, std::uint32_t lane) {
|
||||
return IsFloatOp(op) ? Bits(FloatA(lane)) : IntA64(lane);
|
||||
}
|
||||
|
||||
std::uint64_t CmpBits64(AtomicOp op, std::uint32_t lane) {
|
||||
return IsFloatOp(op) ? Bits(FloatCmp(lane)) : IntCmp64(lane);
|
||||
}
|
||||
|
||||
std::uint64_t Apply64(AtomicOp op, std::uint64_t init, std::uint64_t a, std::uint64_t cmp) {
|
||||
const auto signedInit = static_cast<std::int64_t>(init);
|
||||
const auto signedA = static_cast<std::int64_t>(a);
|
||||
switch (op) {
|
||||
case AtomicOp::Swap: return a;
|
||||
case AtomicOp::Cmpswap: return init == cmp ? a : init;
|
||||
case AtomicOp::Add: return init + a;
|
||||
case AtomicOp::Sub: return init - a;
|
||||
case AtomicOp::Smin: return signedInit < signedA ? init : a;
|
||||
case AtomicOp::Umin: return init < a ? init : a;
|
||||
case AtomicOp::Smax: return signedInit > signedA ? init : a;
|
||||
case AtomicOp::Umax: return init > a ? init : a;
|
||||
case AtomicOp::And: return init & a;
|
||||
case AtomicOp::Or: return init | a;
|
||||
case AtomicOp::Xor: return init ^ a;
|
||||
case AtomicOp::Inc: return init >= a ? 0u : init + 1u;
|
||||
case AtomicOp::Dec: return (init == 0u || init > a) ? a : init - 1u;
|
||||
default: break;
|
||||
}
|
||||
return init;
|
||||
}
|
||||
|
||||
std::uint64_t Expected64(AtomicOp op, std::uint32_t lane) {
|
||||
if (IsFloatOp(op)) {
|
||||
const auto init = FloatInit(lane);
|
||||
const auto a = FloatA(lane);
|
||||
switch (op) {
|
||||
case AtomicOp::Fcmpswap: return Bits(init == FloatCmp(lane) ? a : init);
|
||||
case AtomicOp::Fmin: return Bits(std::min(init, a));
|
||||
case AtomicOp::Fmax: return Bits(std::max(init, a));
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
return Apply64(op, IntInit64(lane), IntA64(lane), IntCmp64(lane));
|
||||
}
|
||||
|
||||
std::uint64_t LaneAddress(const Row& row, std::uint32_t lane, std::uint64_t guestBase) {
|
||||
Target target = row.target;
|
||||
if (target == Target::Mixed) {
|
||||
target = lane % 3u == 0u ? Target::Shared : (lane % 3u == 1u ? Target::Private : Target::Guest);
|
||||
}
|
||||
const std::uint32_t privateBase = row.width == 64u ? 8u : 4u;
|
||||
switch (target) {
|
||||
case Target::Shared: return (static_cast<std::uint64_t>(SharedHigh) << 32u) | (8u * lane + row.offset);
|
||||
case Target::Private: return (static_cast<std::uint64_t>(PrivateHigh) << 32u) | (privateBase + row.offset);
|
||||
case Target::Guest: return guestBase + 8u * lane + row.offset;
|
||||
default: break;
|
||||
}
|
||||
throw std::runtime_error("flat apertures: no address for the row target");
|
||||
}
|
||||
|
||||
std::string Describe(const Row& row) {
|
||||
std::string access;
|
||||
if (row.atomic) {
|
||||
access = std::string(row.width == 64u ? "flat_atomic64_" : "flat_atomic_") + OpNames[static_cast<std::size_t>(row.op)];
|
||||
} else {
|
||||
access = "flat_store" + std::to_string(row.width);
|
||||
}
|
||||
return access + " " + TargetNames[static_cast<std::size_t>(row.target)] + " offset " + std::to_string(row.offset) + " wave" + std::to_string(row.waveSize);
|
||||
}
|
||||
|
||||
void Check(const Row& row, const std::vector<std::uint32_t>& output) {
|
||||
const std::string name = Describe(row);
|
||||
for (std::uint32_t lane = 0; lane < row.waveSize; ++lane) {
|
||||
const auto a = OperandA(lane);
|
||||
if (row.atomic) {
|
||||
const auto init = InitValue(lane);
|
||||
const auto old = output[lane * 8u];
|
||||
Require(old == init, name + ": lane " + std::to_string(lane) + " returned " + Hex(old) + ", expected " + Hex(init));
|
||||
const auto expected = Apply(row.op, init, a, CmpValue(lane));
|
||||
const auto left = output[lane * 8u + 1u];
|
||||
Require(left == expected, name + ": lane " + std::to_string(lane) + " left " + Hex(left) + ", expected " + Hex(expected));
|
||||
} else {
|
||||
const auto read = output[lane * 8u];
|
||||
if (row.target == Target::Shared && 8u * lane + row.offset + row.width / 8u > LdsDwords * 4u) {
|
||||
Require(read == 0u, name + ": lane " + std::to_string(lane) + " read " + Hex(read) + " back from an access past the end of LDS, expected 0 (the store is dropped whole)");
|
||||
continue;
|
||||
}
|
||||
const auto expected = row.width == 8u ? (a & 0xffu) : (row.width == 16u ? (a & 0xffffu) : a);
|
||||
Require(read == expected, name + ": lane " + std::to_string(lane) + " read " + Hex(read) + " back, expected " + Hex(expected));
|
||||
const auto first = output[lane * 8u + 1u];
|
||||
Require(first == (a & 0xffu), name + ": lane " + std::to_string(lane) + " has first byte " + Hex(first) + " in memory, expected " + Hex(a & 0xffu));
|
||||
const auto last = output[lane * 8u + 2u];
|
||||
const auto lastExpected = (a >> (8u * (row.width / 8u - 1u))) & 0xffu;
|
||||
Require(last == lastExpected, name + ": lane " + std::to_string(lane) + " has last byte " + Hex(last) + " in memory, expected " + Hex(lastExpected));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Check64(const Row& row, const std::vector<std::uint32_t>& output) {
|
||||
const std::string name = Describe(row);
|
||||
for (std::uint32_t lane = 0; lane < row.waveSize; ++lane) {
|
||||
const auto old = Pack(output[lane * 8u + 1u], output[lane * 8u]);
|
||||
const auto init = InitBits64(row.op, lane);
|
||||
Require(old == init, name + ": lane " + std::to_string(lane) + " returned " + Hex64(old) + ", expected " + Hex64(init));
|
||||
const auto left = Pack(output[lane * 8u + 3u], output[lane * 8u + 2u]);
|
||||
const auto expected = Expected64(row.op, lane);
|
||||
Require(left == expected, name + ": lane " + std::to_string(lane) + " left " + Hex64(left) + ", expected " + Hex64(expected));
|
||||
}
|
||||
}
|
||||
|
||||
ShaderRecompiler::RecompileRequest MakeRequest(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const std::array<ShaderRecompiler::MemoryRegion, 1>& memory, std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, std::uint32_t waveSize) {
|
||||
return ShaderRecompiler::RecompileRequest{
|
||||
{ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
|
||||
{waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
|
||||
device.Target(),
|
||||
{0, 0, 0, 128}
|
||||
};
|
||||
}
|
||||
|
||||
void Run(AgcDriver::VulkanDevice& device, const Row& row, const GuestBlock& guest) {
|
||||
const auto lanes = row.waveSize;
|
||||
const auto guestSlot = row.atomic ? GuestAtomicSlot : GuestStoreSlot;
|
||||
const auto guestBase = static_cast<std::uint64_t>(guest.Address()) + guestSlot;
|
||||
const bool wideAtomic = row.atomic && row.width == 64u;
|
||||
std::vector<std::uint32_t> table(lanes * 8u, 0u);
|
||||
for (std::uint32_t lane = 0; lane < lanes; ++lane) {
|
||||
const auto address = LaneAddress(row, lane, guestBase);
|
||||
table[lane * 8u] = static_cast<std::uint32_t>(address);
|
||||
table[lane * 8u + 1u] = static_cast<std::uint32_t>(address >> 32u);
|
||||
if (wideAtomic) {
|
||||
const auto operand = OperandBits64(row.op, lane);
|
||||
const auto cmp = CmpBits64(row.op, lane);
|
||||
const auto init = InitBits64(row.op, lane);
|
||||
table[lane * 8u + 2u] = static_cast<std::uint32_t>(operand);
|
||||
table[lane * 8u + 3u] = static_cast<std::uint32_t>(operand >> 32u);
|
||||
table[lane * 8u + 4u] = static_cast<std::uint32_t>(cmp);
|
||||
table[lane * 8u + 5u] = static_cast<std::uint32_t>(cmp >> 32u);
|
||||
table[lane * 8u + 6u] = static_cast<std::uint32_t>(init);
|
||||
table[lane * 8u + 7u] = static_cast<std::uint32_t>(init >> 32u);
|
||||
} else {
|
||||
table[lane * 8u + 2u] = OperandA(lane);
|
||||
table[lane * 8u + 3u] = row.atomic ? CmpValue(lane) : 0u;
|
||||
table[lane * 8u + 4u] = row.atomic ? InitValue(lane) : 0u;
|
||||
}
|
||||
}
|
||||
std::vector<std::uint32_t> output(lanes * 8u, 0xdeadbeefu);
|
||||
std::vector<std::uint32_t> userData(8, 0u);
|
||||
const auto outputDescriptor = BufferDescriptor(output.data(), static_cast<std::uint32_t>(output.size() * 4u));
|
||||
const auto tableDescriptor = BufferDescriptor(table.data(), static_cast<std::uint32_t>(table.size() * 4u));
|
||||
std::copy(outputDescriptor.begin(), outputDescriptor.end(), userData.begin());
|
||||
std::copy(tableDescriptor.begin(), tableDescriptor.end(), userData.begin() + 4);
|
||||
const auto code = Kernel(row);
|
||||
const std::span<const std::uint32_t> codeSpan(code);
|
||||
const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(codeSpan.data()), std::as_bytes(codeSpan)}}};
|
||||
ShaderRecompiler::ShaderComputeStageInfo compute{{lanes, 1, 1}, LdsDwords, {false, false, false}, false, 1};
|
||||
compute.scratchDwords = ScratchDwords;
|
||||
auto request = MakeRequest(device, codeSpan, memory, userData, compute, row.waveSize);
|
||||
request.useCache = false;
|
||||
const auto result = ShaderRecompiler::Recompile(request);
|
||||
device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(codeSpan.data()));
|
||||
device.WaitIdle();
|
||||
if (wideAtomic) {
|
||||
Check64(row, output);
|
||||
} else {
|
||||
Check(row, output);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Row> Rows() {
|
||||
std::vector<Row> rows;
|
||||
const std::array<Target, 3> single{Target::Shared, Target::Private, Target::Guest};
|
||||
const std::array<std::pair<std::uint32_t, std::uint32_t>, 7> stores{{{8u, 0u}, {8u, 3u}, {16u, 0u}, {16u, 1u}, {16u, 3u}, {32u, 0u}, {32u, 2u}}};
|
||||
for (const Target target : single) {
|
||||
for (const auto [width, offset] : stores) {
|
||||
rows.push_back(Row{target, false, AtomicOp::Swap, width, offset, 32u});
|
||||
}
|
||||
}
|
||||
rows.push_back(Row{Target::Mixed, false, AtomicOp::Swap, 16u, 3u, 32u});
|
||||
rows.push_back(Row{Target::Mixed, false, AtomicOp::Swap, 32u, 2u, 32u});
|
||||
for (const auto [width, offset] : std::array<std::pair<std::uint32_t, std::uint32_t>, 5>{{{32u, 773u}, {32u, 774u}, {32u, 776u}, {16u, 775u}, {8u, 775u}}}) {
|
||||
rows.push_back(Row{Target::Shared, false, AtomicOp::Swap, width, offset, 32u});
|
||||
}
|
||||
for (const Target target : single) {
|
||||
for (std::uint32_t op = 0; op < Atomic32Ops; ++op) {
|
||||
rows.push_back(Row{target, true, static_cast<AtomicOp>(op), 32u, 0u, 32u});
|
||||
}
|
||||
}
|
||||
for (const Target target : single) {
|
||||
for (std::uint32_t op = 0; op < OpNames.size(); ++op) {
|
||||
rows.push_back(Row{target, true, static_cast<AtomicOp>(op), 64u, 0u, 32u});
|
||||
}
|
||||
}
|
||||
rows.push_back(Row{Target::Mixed, true, AtomicOp::Add, 32u, 0u, 32u});
|
||||
rows.push_back(Row{Target::Mixed, true, AtomicOp::Cmpswap, 32u, 0u, 32u});
|
||||
rows.push_back(Row{Target::Mixed, true, AtomicOp::Add, 64u, 0u, 32u});
|
||||
rows.push_back(Row{Target::Mixed, true, AtomicOp::Cmpswap, 64u, 0u, 32u});
|
||||
rows.push_back(Row{Target::Mixed, true, AtomicOp::Fmax, 64u, 0u, 32u});
|
||||
rows.push_back(Row{Target::Mixed, false, AtomicOp::Swap, 16u, 3u, 64u});
|
||||
rows.push_back(Row{Target::Mixed, false, AtomicOp::Swap, 32u, 2u, 64u});
|
||||
rows.push_back(Row{Target::Mixed, true, AtomicOp::Add, 32u, 0u, 64u});
|
||||
rows.push_back(Row{Target::Shared, true, AtomicOp::Cmpswap, 32u, 0u, 64u});
|
||||
rows.push_back(Row{Target::Mixed, true, AtomicOp::Add, 64u, 0u, 64u});
|
||||
rows.push_back(Row{Target::Shared, true, AtomicOp::Fmin, 64u, 0u, 64u});
|
||||
return rows;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main() {
|
||||
try {
|
||||
const auto device = OpenVulkanTestDevice();
|
||||
if (!device) return VulkanTestSkipped;
|
||||
const bool wave64 = device->Target().subgroupSize >= 32u;
|
||||
GuestBlock guest;
|
||||
std::size_t checked = 0;
|
||||
for (const Row& row : Rows()) {
|
||||
if (row.waveSize == 64u && !wave64) continue;
|
||||
Run(*device, row, guest);
|
||||
++checked;
|
||||
}
|
||||
if (!wave64) std::printf("skipped the wave64 rows, subgroup size %u cannot hold a wave64 in two lanes\n", device->Target().subgroupSize);
|
||||
std::printf("flat aperture rows checked: %zu\n", checked);
|
||||
std::puts("flat aperture tests passed");
|
||||
return 0;
|
||||
} catch (const std::exception& error) {
|
||||
std::cerr << error.what() << '\n';
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -325,6 +325,13 @@ SpirvRequirements AnalyzeProgramRequirements(const IrProgram& program) {
|
|||
}
|
||||
requirements.functionScratch = true;
|
||||
}
|
||||
if (program.Resources().memoryInfo.at(memoryIndex).kind == ResourceKind::Flat && program.Resources().stage == IrShaderStage::Compute && program.Info().scratchDwords != 0u) {
|
||||
requirements.functionScratch = true;
|
||||
}
|
||||
if (program.Resources().memoryInfo.at(memoryIndex).kind == ResourceKind::Flat && program.Resources().stage == IrShaderStage::Compute && addressAccess == AddressAccess::Atomic && inst->Type() == IrType::U64) {
|
||||
requirements.ldsLock = true;
|
||||
requirements.subgroupBallot = true;
|
||||
}
|
||||
}
|
||||
if (BufferAccessOf(inst->Opcode()) != BufferAccess::None) {
|
||||
const auto memoryIndex = inst->Flags<MemoryFlags>().index;
|
||||
|
|
|
|||
|
|
@ -336,11 +336,154 @@ const MemoryInfo& SharedMemory(SpirvValueEmitContext& ctx, const IrValue& inst)
|
|||
return mem;
|
||||
}
|
||||
|
||||
constexpr std::uint32_t SharedApertureHigh = 0x80000000u;
|
||||
constexpr std::uint32_t PrivateApertureHigh = 0x70000000u;
|
||||
|
||||
bool RoutesApertures(const SpirvEmitterState& state, const MemoryInfo& mem) {
|
||||
return mem.kind == ResourceKind::Flat && state.program.Resources().stage == IrShaderStage::Compute;
|
||||
}
|
||||
|
||||
struct ApertureAddress {
|
||||
std::uint32_t low = 0;
|
||||
std::uint32_t shared = 0;
|
||||
std::uint32_t priv = 0;
|
||||
};
|
||||
|
||||
ApertureAddress SplitAperture(SpirvEmitterState& state, std::uint32_t address) {
|
||||
const auto u32 = TypeU32(state);
|
||||
const auto high = Unary(state, spv::OpUConvert, u32, Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, 32u)));
|
||||
return {Unary(state, spv::OpUConvert, u32, address), Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, SharedApertureHigh)), Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, PrivateApertureHigh))};
|
||||
}
|
||||
|
||||
std::uint32_t ApertureByteInBounds(SpirvEmitterState& state, const MemoryResourceAccess& resource, std::uint32_t byteOffset, std::uint32_t offset, std::uint32_t index) {
|
||||
const auto noWrap = Binary(state, spv::OpULessThanEqual, TypeBool(state), byteOffset, ConstantU32(state, 0xffffffffu - offset));
|
||||
return AndCondition(state, noWrap, EmitMemoryElementInBounds(state, resource, index));
|
||||
}
|
||||
|
||||
std::uint32_t ApertureContained(SpirvEmitterState& state, std::uint32_t byteOffset, std::uint32_t bits) {
|
||||
const auto inDword = Binary(state, spv::OpBitwiseAnd, TypeU32(state), byteOffset, ConstantU32(state, 3u));
|
||||
return Binary(state, spv::OpULessThanEqual, TypeBool(state), EmitAddU32(state, inDword, ConstantU32(state, bits / 8u)), ConstantU32(state, 4u));
|
||||
}
|
||||
|
||||
std::uint32_t LoadApertureDword(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits) {
|
||||
auto& state = ctx.state;
|
||||
if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return ConstantU32(state, 0u);
|
||||
MemoryInfo storage{};
|
||||
storage.kind = kind;
|
||||
const auto resource = PrepareMemoryResourceAccess(state, storage);
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteOffset, ConstantU32(state, 2u));
|
||||
return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&] {
|
||||
return bits == 32u ? LoadWordInBounds(ctx, resource, index) : LoadSubwordInBounds(ctx, resource, byteOffset, index, bits, false);
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t LoadApertureByte(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t offset) {
|
||||
auto& state = ctx.state;
|
||||
if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return ConstantU32(state, 0u);
|
||||
MemoryInfo storage{};
|
||||
storage.kind = kind;
|
||||
const auto resource = PrepareMemoryResourceAccess(state, storage);
|
||||
const auto address = EmitAddU32(state, byteOffset, ConstantU32(state, offset));
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u));
|
||||
return EmitValueOrZeroIfCondition(state, ApertureByteInBounds(state, resource, byteOffset, offset, index), [&] {
|
||||
return LoadSubwordInBounds(ctx, resource, address, index, 8u, false);
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t ApertureElementInBounds(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits) {
|
||||
auto& state = ctx.state;
|
||||
MemoryInfo storage{};
|
||||
storage.kind = kind;
|
||||
const auto resource = PrepareMemoryResourceAccess(state, storage);
|
||||
const auto last = bits / 8u - 1u;
|
||||
const auto address = EmitAddU32(state, byteOffset, ConstantU32(state, last));
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u));
|
||||
return ApertureByteInBounds(state, resource, byteOffset, last, index);
|
||||
}
|
||||
|
||||
std::uint32_t LoadApertureElement(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits) {
|
||||
auto& state = ctx.state;
|
||||
if (bits == 8u) return LoadApertureByte(ctx, kind, byteOffset, 0u);
|
||||
if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return ConstantU32(state, 0u);
|
||||
return EmitValueOrZeroIfCondition(state, ApertureElementInBounds(ctx, kind, byteOffset, bits), [&] {
|
||||
return EmitValueIfElse(state, ApertureContained(state, byteOffset, bits), TypeU32(state), [&] {
|
||||
return LoadApertureDword(ctx, kind, byteOffset, bits);
|
||||
}, [&] {
|
||||
std::uint32_t value = ConstantU32(state, 0u);
|
||||
for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) {
|
||||
const auto part = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), LoadApertureByte(ctx, kind, byteOffset, byte), ConstantU32(state, byte * 8u));
|
||||
value = Binary(state, spv::OpBitwiseOr, TypeU32(state), value, part);
|
||||
}
|
||||
return value;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void StoreApertureByte(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t offset, std::uint32_t data) {
|
||||
auto& state = ctx.state;
|
||||
if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return;
|
||||
MemoryInfo storage{};
|
||||
storage.kind = kind;
|
||||
const auto resource = PrepareMemoryResourceAccess(state, storage);
|
||||
const auto address = EmitAddU32(state, byteOffset, ConstantU32(state, offset));
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u));
|
||||
EmitIfCondition(state, ApertureByteInBounds(state, resource, byteOffset, offset, index), [&] { StoreSubwordInBounds(ctx, storage, resource, address, index, 8u, data); });
|
||||
}
|
||||
|
||||
void StoreApertureDword(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits, std::uint32_t data) {
|
||||
auto& state = ctx.state;
|
||||
if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return;
|
||||
MemoryInfo storage{};
|
||||
storage.kind = kind;
|
||||
const auto resource = PrepareMemoryResourceAccess(state, storage);
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteOffset, ConstantU32(state, 2u));
|
||||
EmitIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&] {
|
||||
if (bits == 32u) {
|
||||
StoreWordInBounds(ctx, resource, index, data);
|
||||
} else {
|
||||
StoreSubwordInBounds(ctx, storage, resource, byteOffset, index, bits, data);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void StoreApertureElement(SpirvValueEmitContext& ctx, ResourceKind kind, std::uint32_t byteOffset, std::uint32_t bits, std::uint32_t data) {
|
||||
auto& state = ctx.state;
|
||||
if (bits == 8u) {
|
||||
StoreApertureByte(ctx, kind, byteOffset, 0u, data);
|
||||
return;
|
||||
}
|
||||
if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return;
|
||||
EmitIfCondition(state, ApertureElementInBounds(ctx, kind, byteOffset, bits), [&] {
|
||||
const auto contained = ApertureContained(state, byteOffset, bits);
|
||||
EmitIfCondition(state, contained, [&] { StoreApertureDword(ctx, kind, byteOffset, bits, data); });
|
||||
EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), contained), [&] {
|
||||
for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) {
|
||||
StoreApertureByte(ctx, kind, byteOffset, byte, Binary(state, spv::OpShiftRightLogical, TypeU32(state), data, ConstantU32(state, byte * 8u)));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
template<typename TAperture, typename TGlobal>
|
||||
std::uint32_t RouteFlatAccess(SpirvEmitterState& state, std::uint32_t address, std::uint32_t type, TAperture&& aperture, TGlobal&& global) {
|
||||
const auto split = SplitAperture(state, address);
|
||||
return EmitValueIfElse(state, split.shared, type, [&] { return aperture(ResourceKind::Lds, split.low); }, [&] {
|
||||
return EmitValueIfElse(state, split.priv, type, [&] { return aperture(ResourceKind::Scratch, split.low); }, global);
|
||||
});
|
||||
}
|
||||
|
||||
void LoadAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t bits) {
|
||||
const auto& mem = ctx.Memory(inst);
|
||||
if (bits == 32u && mem.planningOnly) {
|
||||
return;
|
||||
}
|
||||
if (RoutesApertures(ctx.state, mem)) {
|
||||
ctx.Define(inst, EmitValueOrZeroIfCondition(ctx.state, ActiveArgument(ctx, inst), [&] {
|
||||
const auto address = GuestAddress(ctx, inst, mem);
|
||||
return RouteFlatAccess(ctx.state, address, TypeU32(ctx.state), [&](ResourceKind kind, std::uint32_t low) { return LoadApertureElement(ctx, kind, low, bits); }, [&] { return EmitBdaRead(ctx, inst, address, bits); });
|
||||
}));
|
||||
return;
|
||||
}
|
||||
switch (mem.kind) {
|
||||
case ResourceKind::Scratch:
|
||||
ctx.Define(inst, bits == 32u ? LoadWord(ctx, inst, mem) : LoadSubword(ctx, inst, mem, bits));
|
||||
|
|
@ -367,6 +510,22 @@ void LoadAddressWide(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint3
|
|||
if (mem.planningOnly) {
|
||||
return;
|
||||
}
|
||||
if (RoutesApertures(ctx.state, mem)) {
|
||||
auto& state = ctx.state;
|
||||
const auto type = TypeU32Composite(state, components);
|
||||
ctx.Define(inst, EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), type, ConstantU32CompositeZero(state, components), [&] {
|
||||
const auto base = GuestAddressBase(ctx, inst, mem);
|
||||
const auto address = AddBdaImmediate(ctx, inst, base, static_cast<std::int32_t>(mem.offset));
|
||||
return RouteFlatAccess(state, address, type, [&](ResourceKind kind, std::uint32_t low) {
|
||||
std::array<std::uint32_t, 4> values{};
|
||||
for (std::uint32_t component = 0; component < components; component++) {
|
||||
values[component] = LoadApertureElement(ctx, kind, Binary(state, spv::OpIAdd, TypeU32(state), low, ConstantU32(state, component * 4u)), 32u);
|
||||
}
|
||||
return ConstructU32Composite(state, components, values);
|
||||
}, [&] { return ConstructU32Composite(state, components, EmitBdaDwordReads(ctx, inst, base, mem.offset, components)); });
|
||||
}));
|
||||
return;
|
||||
}
|
||||
switch (mem.kind) {
|
||||
case ResourceKind::Flat:
|
||||
case ResourceKind::Global:
|
||||
|
|
@ -390,7 +549,18 @@ void StoreAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t
|
|||
case ResourceKind::Flat:
|
||||
case ResourceKind::Global:
|
||||
EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() {
|
||||
EmitBdaStore(ctx, inst, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.ArgumentCount() - 2u), bits);
|
||||
auto& state = ctx.state;
|
||||
const auto address = GuestAddress(ctx, inst, mem);
|
||||
const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u);
|
||||
if (!RoutesApertures(state, mem)) {
|
||||
EmitBdaStore(ctx, inst, address, data, bits);
|
||||
return;
|
||||
}
|
||||
const auto split = SplitAperture(state, address);
|
||||
EmitIfCondition(state, split.shared, [&] { StoreApertureElement(ctx, ResourceKind::Lds, split.low, bits, data); });
|
||||
EmitIfCondition(state, split.priv, [&] { StoreApertureElement(ctx, ResourceKind::Scratch, split.low, bits, data); });
|
||||
const auto global = Unary(state, spv::OpLogicalNot, TypeBool(state), Binary(state, spv::OpLogicalOr, TypeBool(state), split.shared, split.priv));
|
||||
EmitIfCondition(state, global, [&] { EmitBdaStore(ctx, inst, address, data, bits); });
|
||||
});
|
||||
return;
|
||||
default:
|
||||
|
|
@ -734,6 +904,163 @@ std::uint32_t AddressAtomicOpcode(IrOpcode opcode) {
|
|||
}
|
||||
}
|
||||
|
||||
std::uint32_t FlatAtomicNext32(SpirvEmitterState& state, IrOpcode opcode, std::uint32_t old, std::uint32_t value, std::uint32_t comparator) {
|
||||
const auto u32 = TypeU32(state);
|
||||
const auto winner = [&](spv::Op compare) {
|
||||
return Select(state, u32, Binary(state, compare, TypeBool(state), old, value), old, value);
|
||||
};
|
||||
switch (opcode) {
|
||||
case IrOpcode::AddressAtomicSwap32:
|
||||
return value;
|
||||
case IrOpcode::AddressAtomicCmpSwap32:
|
||||
return Select(state, u32, Binary(state, spv::OpIEqual, TypeBool(state), old, comparator), value, old);
|
||||
case IrOpcode::AddressAtomicIAdd32:
|
||||
return Binary(state, spv::OpIAdd, u32, old, value);
|
||||
case IrOpcode::AddressAtomicISub32:
|
||||
return Binary(state, spv::OpISub, u32, old, value);
|
||||
case IrOpcode::AddressAtomicSMin32:
|
||||
return winner(spv::OpSLessThan);
|
||||
case IrOpcode::AddressAtomicUMin32:
|
||||
return winner(spv::OpULessThan);
|
||||
case IrOpcode::AddressAtomicSMax32:
|
||||
return winner(spv::OpSGreaterThan);
|
||||
case IrOpcode::AddressAtomicUMax32:
|
||||
return winner(spv::OpUGreaterThan);
|
||||
case IrOpcode::AddressAtomicAnd32:
|
||||
return Binary(state, spv::OpBitwiseAnd, u32, old, value);
|
||||
case IrOpcode::AddressAtomicOr32:
|
||||
return Binary(state, spv::OpBitwiseOr, u32, old, value);
|
||||
case IrOpcode::AddressAtomicXor32:
|
||||
return Binary(state, spv::OpBitwiseXor, u32, old, value);
|
||||
case IrOpcode::AddressAtomicInc32:
|
||||
return AtomicIncrement(state, old, value);
|
||||
case IrOpcode::AddressAtomicDec32:
|
||||
return AtomicDecrement(state, old, value);
|
||||
case IrOpcode::AddressAtomicUSubSat32:
|
||||
return AtomicUSubSat(state, old, value);
|
||||
case IrOpcode::AddressAtomicFMin32:
|
||||
return AtomicFloatBits{state, false}.minMax(old, value, false);
|
||||
case IrOpcode::AddressAtomicFMax32:
|
||||
return AtomicFloatBits{state, false}.minMax(old, value, true);
|
||||
case IrOpcode::AddressAtomicFCmpSwap32:
|
||||
return AtomicFloatBits{state, false}.compareSwap(old, comparator, value);
|
||||
default:
|
||||
throw std::runtime_error("FlatAtomicNext32: opcode has no aperture lowering");
|
||||
}
|
||||
}
|
||||
|
||||
std::uint32_t ApertureAtomic32(SpirvValueEmitContext& ctx, const IrValue& inst, ResourceKind kind, std::uint32_t address, std::uint32_t byteOffset, std::uint32_t value, std::uint32_t comparator) {
|
||||
auto& state = ctx.state;
|
||||
const auto instruction = ConstantU32(state, inst.Flags<MemoryFlags>().pc);
|
||||
const auto unaligned = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), byteOffset, ConstantU32(state, 3u)), ConstantU32(state, 0u));
|
||||
EmitIfCondition(state, unaligned, [&] { RecordBdaFault(state, address, ConstantU32(state, 4u), instruction, BdaAbi::FaultReason::Unaligned); });
|
||||
if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return ConstantU32(state, 0u);
|
||||
MemoryInfo storage{};
|
||||
storage.kind = kind;
|
||||
const auto resource = PrepareMemoryResourceAccess(state, storage);
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteOffset, ConstantU32(state, 2u));
|
||||
const auto aligned = Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned);
|
||||
const auto opcode = inst.Opcode();
|
||||
return EmitValueOrZeroIfCondition(state, aligned, [&] {
|
||||
return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&] {
|
||||
const auto pointer = EmitMemoryElementPointer(state, resource, index);
|
||||
if (kind == ResourceKind::Lds) {
|
||||
return AtomicUpdate(state, pointer, kind, [&](std::uint32_t current) { return FlatAtomicNext32(state, opcode, current, value, comparator); });
|
||||
}
|
||||
const auto old = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpLoad, TypeU32(state), old, pointer);
|
||||
state.module.AddFunction(spv::OpStore, pointer, FlatAtomicNext32(state, opcode, old, value, comparator));
|
||||
return old;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t FlatAtomicNext64(SpirvEmitterState& state, IrOpcode opcode, std::uint32_t old, std::uint32_t value, std::uint32_t comparator) {
|
||||
const auto u64 = TypeScalarU64(state);
|
||||
const AtomicFloatBits bits{state, true};
|
||||
const auto winner = [&](spv::Op compare) {
|
||||
return Select(state, u64, Binary(state, compare, TypeBool(state), old, value), old, value);
|
||||
};
|
||||
switch (opcode) {
|
||||
case IrOpcode::AddressAtomicSwap64:
|
||||
return value;
|
||||
case IrOpcode::AddressAtomicCmpSwap64:
|
||||
return Select(state, u64, Binary(state, spv::OpIEqual, TypeBool(state), old, comparator), value, old);
|
||||
case IrOpcode::AddressAtomicIAdd64:
|
||||
return Binary(state, spv::OpIAdd, u64, old, value);
|
||||
case IrOpcode::AddressAtomicISub64:
|
||||
return Binary(state, spv::OpISub, u64, old, value);
|
||||
case IrOpcode::AddressAtomicSMin64:
|
||||
return winner(spv::OpSLessThan);
|
||||
case IrOpcode::AddressAtomicUMin64:
|
||||
return winner(spv::OpULessThan);
|
||||
case IrOpcode::AddressAtomicSMax64:
|
||||
return winner(spv::OpSGreaterThan);
|
||||
case IrOpcode::AddressAtomicUMax64:
|
||||
return winner(spv::OpUGreaterThan);
|
||||
case IrOpcode::AddressAtomicAnd64:
|
||||
return Binary(state, spv::OpBitwiseAnd, u64, old, value);
|
||||
case IrOpcode::AddressAtomicOr64:
|
||||
return Binary(state, spv::OpBitwiseOr, u64, old, value);
|
||||
case IrOpcode::AddressAtomicXor64:
|
||||
return Binary(state, spv::OpBitwiseXor, u64, old, value);
|
||||
case IrOpcode::AddressAtomicInc64:
|
||||
return bits.increment(old, value);
|
||||
case IrOpcode::AddressAtomicDec64:
|
||||
return bits.decrement(old, value);
|
||||
case IrOpcode::AddressAtomicFMin64:
|
||||
return bits.minMax(old, value, false);
|
||||
case IrOpcode::AddressAtomicFMax64:
|
||||
return bits.minMax(old, value, true);
|
||||
case IrOpcode::AddressAtomicFCmpSwap64:
|
||||
return bits.compareSwap(old, comparator, value);
|
||||
default:
|
||||
throw std::runtime_error("FlatAtomicNext64: opcode has no aperture lowering");
|
||||
}
|
||||
}
|
||||
|
||||
template<typename TUpdate>
|
||||
std::uint32_t LockedLdsUpdate(SpirvEmitterState& state, TUpdate&& update);
|
||||
|
||||
std::uint32_t ApertureAtomic64(SpirvValueEmitContext& ctx, const IrValue& inst, ResourceKind kind, std::uint32_t address, std::uint32_t byteOffset, std::uint32_t value, std::uint32_t comparator) {
|
||||
auto& state = ctx.state;
|
||||
const auto instruction = ConstantU32(state, inst.Flags<MemoryFlags>().pc);
|
||||
const auto unaligned = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), byteOffset, ConstantU32(state, 7u)), ConstantU32(state, 0u));
|
||||
EmitIfCondition(state, unaligned, [&] { RecordBdaFault(state, address, ConstantU32(state, 8u), instruction, BdaAbi::FaultReason::Unaligned); });
|
||||
if (kind == ResourceKind::Scratch && state.program.Info().scratchDwords == 0u) return ConstantU64(state, 0u);
|
||||
if (kind == ResourceKind::Lds && !state.requirements.ldsLock) ctx.Fail(inst, "64-bit LDS atomic reached through a flat address without the workgroup lock");
|
||||
MemoryInfo storage{};
|
||||
storage.kind = kind;
|
||||
const auto resource = PrepareMemoryResourceAccess(state, storage);
|
||||
const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteOffset, ConstantU32(state, 2u));
|
||||
const auto high = EmitAddU32(state, low, ConstantU32(state, 1u));
|
||||
const auto aligned = Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned);
|
||||
const auto opcode = inst.Opcode();
|
||||
return EmitValueOrDefaultIfCondition(state, aligned, TypeU64(state), ConstantU64(state, 0u), [&] {
|
||||
return EmitValueOrDefaultIfCondition(state, EmitMemoryElementInBounds(state, resource, high), TypeU64(state), ConstantU64(state, 0u), [&] {
|
||||
const auto update = [&]() {
|
||||
const auto lowPointer = EmitMemoryElementPointer(state, resource, low);
|
||||
const auto highPointer = EmitMemoryElementPointer(state, resource, high);
|
||||
const auto lowValue = state.module.AllocateId();
|
||||
const auto highValue = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpLoad, TypeU32(state), lowValue, lowPointer);
|
||||
state.module.AddFunction(spv::OpLoad, TypeU32(state), highValue, highPointer);
|
||||
const auto old = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpCompositeConstruct, TypeU64(state), old, lowValue, highValue);
|
||||
const auto next = Unary(state, spv::OpBitcast, TypeU64(state), FlatAtomicNext64(state, opcode, Unary(state, spv::OpBitcast, TypeScalarU64(state), old), value, comparator));
|
||||
const auto nextLow = state.module.AllocateId();
|
||||
const auto nextHigh = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), nextLow, next, 0u);
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), nextHigh, next, 1u);
|
||||
state.module.AddFunction(spv::OpStore, lowPointer, nextLow);
|
||||
state.module.AddFunction(spv::OpStore, highPointer, nextHigh);
|
||||
return old;
|
||||
};
|
||||
return kind == ResourceKind::Lds ? LockedLdsUpdate(state, update) : update();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t AddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
||||
auto& state = ctx.state;
|
||||
const auto& mem = ctx.Memory(inst);
|
||||
|
|
@ -747,7 +1074,7 @@ std::uint32_t AddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
|||
const auto scalarType = wide ? TypeScalarU64(state) : TypeU32(state);
|
||||
const auto scalar = [&](std::uint32_t value) { return wide ? Unary(state, spv::OpBitcast, scalarType, value) : value; };
|
||||
const auto value = scalar(ctx.Arg(inst, 3));
|
||||
const auto old = EmitBdaAtomic(ctx, inst, GuestAddress(ctx, inst, mem), wide ? 8u : 4u, [&](std::uint32_t pointer) {
|
||||
const auto operation = [&](std::uint32_t pointer) {
|
||||
if (inst.Opcode() == IrOpcode::AddressAtomicInc32 || inst.Opcode() == IrOpcode::AddressAtomicDec32) {
|
||||
const bool increment = inst.Opcode() == IrOpcode::AddressAtomicInc32;
|
||||
return AtomicUpdate(state, pointer, mem.kind, [&](std::uint32_t current) {
|
||||
|
|
@ -789,7 +1116,17 @@ std::uint32_t AddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst) {
|
|||
}
|
||||
EmitDeviceAtomicMemoryBarrier(state);
|
||||
return result;
|
||||
});
|
||||
};
|
||||
if (RoutesApertures(state, mem)) {
|
||||
const auto address = GuestAddress(ctx, inst, mem);
|
||||
const bool hasComparator = inst.Opcode() == IrOpcode::AddressAtomicCmpSwap32 || inst.Opcode() == IrOpcode::AddressAtomicFCmpSwap32 || inst.Opcode() == IrOpcode::AddressAtomicCmpSwap64 || inst.Opcode() == IrOpcode::AddressAtomicFCmpSwap64;
|
||||
const auto comparator = hasComparator ? scalar(ctx.Arg(inst, 4)) : 0u;
|
||||
if (wide) {
|
||||
return RouteFlatAccess(state, address, type, [&](ResourceKind kind, std::uint32_t low) { return ApertureAtomic64(ctx, inst, kind, address, low, value, comparator); }, [&] { return Unary(state, spv::OpBitcast, type, EmitBdaAtomic(ctx, inst, address, 8u, operation)); });
|
||||
}
|
||||
return RouteFlatAccess(state, address, TypeU32(state), [&](ResourceKind kind, std::uint32_t low) { return ApertureAtomic32(ctx, inst, kind, address, low, value, comparator); }, [&] { return EmitBdaAtomic(ctx, inst, address, 4u, operation); });
|
||||
}
|
||||
const auto old = EmitBdaAtomic(ctx, inst, GuestAddress(ctx, inst, mem), wide ? 8u : 4u, operation);
|
||||
return wide ? Unary(state, spv::OpBitcast, type, old) : old;
|
||||
});
|
||||
}
|
||||
|
|
@ -1119,7 +1456,25 @@ void StoreAddressWide(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint
|
|||
const auto& mem = ctx.Memory(inst);
|
||||
if (mem.kind != ResourceKind::Flat && mem.kind != ResourceKind::Global) ctx.Fail(inst, "must write a physical address resource");
|
||||
EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() {
|
||||
EmitBdaDwordWrites(ctx, inst, GuestAddressBase(ctx, inst, mem), mem.offset, components, ctx.Arg(inst, inst.ArgumentCount() - 2u));
|
||||
auto& state = ctx.state;
|
||||
const auto base = GuestAddressBase(ctx, inst, mem);
|
||||
const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u);
|
||||
if (!RoutesApertures(state, mem)) {
|
||||
EmitBdaDwordWrites(ctx, inst, base, mem.offset, components, data);
|
||||
return;
|
||||
}
|
||||
const auto split = SplitAperture(state, AddBdaImmediate(ctx, inst, base, static_cast<std::int32_t>(mem.offset)));
|
||||
const auto storeAperture = [&](ResourceKind kind) {
|
||||
for (std::uint32_t component = 0; component < components; component++) {
|
||||
const auto value = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, data, component);
|
||||
StoreApertureElement(ctx, kind, Binary(state, spv::OpIAdd, TypeU32(state), split.low, ConstantU32(state, component * 4u)), 32u, value);
|
||||
}
|
||||
};
|
||||
EmitIfCondition(state, split.shared, [&] { storeAperture(ResourceKind::Lds); });
|
||||
EmitIfCondition(state, split.priv, [&] { storeAperture(ResourceKind::Scratch); });
|
||||
const auto global = Unary(state, spv::OpLogicalNot, TypeBool(state), Binary(state, spv::OpLogicalOr, TypeBool(state), split.shared, split.priv));
|
||||
EmitIfCondition(state, global, [&] { EmitBdaDwordWrites(ctx, inst, base, mem.offset, components, data); });
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -368,6 +368,9 @@ Throughout the project, every function at every stage either **does exactly what
|
|||
- A [V#, T# or S#](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) loaded through a null or unmapped pointer read from memory is zero when the work is recorded, and the capture throws if no flat SRT slot records that read.
|
||||
- [2D accesses through a 1D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) read the texel the 1D addressing selects and ignore the second coordinate, as simonbbby measured on RDNA2 (gfx1036) in #2237 for `image_sample_lz` and mip-less `image_load` with point filtering and wrap, clamp and border addressing; the PS5's gfx1013 is not measured. Other 2D accesses through a 1D T# (storage, depth, packed, BY loads, mips, derivatives, comparison) still throw.
|
||||
- [Emulated comparison of a 2D-array access through a 2D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) samples the 2D image and ignores the slice, as the native modes of a 2D-array access through a 2D T# already do. This follows the T# type deciding the addressing; it is not measured on hardware for comparison samples
|
||||
- [Scalar aperture base sources](../../core/shader/recompiler/RdnaDecoder/src/RdnaInstructionDecoder.cpp) (core/shader/recompiler) - `src_shared_base`, `src_shared_limit`, `src_private_base` and `src_private_limit` (scalar source codes 235 to 238) are not modelled: decoding throws `unsupported scalar source operand code` with the code and the program counter, so a shader that reads one does not compile. Operand 235 (`src_shared_base`) was reported on [#1182](https://github.com/boykopovar/AnyPS5/pull/1182) from PPSA26344; the flat aperture routing does not read the base, so a shader that builds its own flat addresses from it stops at decode.
|
||||
- [Flat aperture bases](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (core/shader/recompiler) - the layout follows the gfx9+ one in the AMD KFD driver's `kfd_flat_memory.c` (`MAKE_LDS_APP_BASE_V9`, `MAKE_LDS_APP_LIMIT`): each aperture is a 4 GB window whose base is the high dword, so a flat address is routed by its whole high dword, as measured on RDNA2 (gfx1035: `src_shared_base` 0x00010000_00000000, `src_shared_limit` 0x00010000_ffffffff). The PS5's base values, high dwords 0x80000000 (shared, `SharedApertureHigh`) and 0x70000000 (private, `PrivateApertureHigh`), are inferred from GTA V's traversal addresses, not measured on a console or read from SH_MEM_BASES
|
||||
- [Flat accesses outside the aperture routing](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (core/shader/recompiler) - only compute stages route flat accesses, so graphics-stage flat accesses to the shared or private aperture still go to BDA. In compute, a 64-bit flat atomic whose address is in the shared aperture takes the workgroup lock (`ldsLock`) for its two-dword update, and every such atomic requests that lock, so a shader with one pays for it even on global addresses (the path is exercised by `agc_driver_global_atomics_float`). Inferred, not measured on LDS or scratch: an aperture access that is not inside one dword is done byte by byte, an unaligned aperture atomic records the BDA unaligned fault, and LDS or scratch accesses past their end are dropped or read zero.
|
||||
- On a host subgroup narrower than 32 lanes, [`threadBit`](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) reads an EXEC written as a value (`s_mov_b32`/`s_mov_b64`) at bit (subgroup invocation id) of EXEC_LO, so lanes at or above the subgroup size read a lower lane's bit.
|
||||
- [EXEC read as a value](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) (`s_mov_b64 sN, exec`, the `s_*_saveexec` copy, ALU sources) holds only lanes with a host invocation; the console's also holds the wave's other lanes, e.g. 32-63 of a wave64 on 32 host lanes.
|
||||
- [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) and fetch-inactive `v_permlane16_b32`/`v_permlanex16_b32` read undefined data from a lane that the host subgroup does not hold or whose host invocation is inactive; the console reads the lane's VGPR.
|
||||
|
|
|
|||
Loading…
Reference in New Issue