fix(shader): convert formatted buffer stores to the element format
buffer_store_format_* and tbuffer_store_format_* wrote the register bits as they were: an 8_UNORM store of 1.0f wrote 0x00 (the low byte of 0x3f800000), packed formats such as 2_10_10_10 overwrote the whole word with each component, and the components the instruction does not provide were left untouched. Each component is now encoded as the hardware does: unorm/snorm (and uscaled/sscaled, which store the same way) round x*M exactly to nearest even after clamping, with NaN as 0; uint/sint saturate; f16 and the 11/10-bit floats truncate toward zero and saturate. The whole element is written, with the missing components as 0, and packed formats are composed into one word. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Adrià Franch
parent
5c8c5d2578
commit
4d90c0bc8e
|
|
@ -0,0 +1,173 @@
|
|||
#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
|
||||
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
|
||||
#include "Recompiler.hpp"
|
||||
#include "VulkanTestDevice.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <bit>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
using AgcDriver::Graphics::Require;
|
||||
using ShaderRecompiler::ShaderStage;
|
||||
|
||||
constexpr std::uint32_t Threads = 32;
|
||||
constexpr std::uint32_t Inputs = 4;
|
||||
constexpr std::uint32_t Results = 16;
|
||||
alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
|
||||
alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 77> Code{
|
||||
0x34020084, 0x34060086, 0xe0301000, 0x80000401, 0xe0301004, 0x80000501, 0xe0301008, 0x80000601,
|
||||
0xe030100c, 0x80000701, 0xbf8c3f70, 0x7e3c02ff, 0xaaaaaaaa, 0xe0701000, 0x80011e03, 0xe0701004,
|
||||
0x80011e03, 0xe0701008, 0x80011e03, 0xe070100c, 0x80011e03, 0xe0701010, 0x80011e03, 0xe0701014,
|
||||
0x80011e03, 0xe0701018, 0x80011e03, 0xe070101c, 0x80011e03, 0xe0701020, 0x80011e03, 0xe0701024,
|
||||
0x80011e03, 0xe0701028, 0x80011e03, 0xe070102c, 0x80011e03, 0xe0701030, 0x80011e03, 0xe0701034,
|
||||
0x80011e03, 0xe0701038, 0x80011e03, 0xe070103c, 0x80011e03, 0xbf8c3f70, 0xe80c1000, 0x80010403,
|
||||
0xe8141004, 0x80010403, 0xe83c1008, 0x80010403, 0xe844100c, 0x80010403, 0xe81c1010, 0x80010403,
|
||||
0xe8541014, 0x80010403, 0xe82c1018, 0x80010503, 0xe864101c, 0x80010503, 0xe86c1020, 0x80010403,
|
||||
0xe9241024, 0x80010403, 0xe9971028, 0x80010403, 0xe8ed102c, 0x80010603, 0xe9f41030, 0x80010503,
|
||||
0xe9ee1038, 0x80010503, 0xe99d103c, 0x80010603, 0xbf810000,
|
||||
};
|
||||
|
||||
constexpr std::uint32_t Rows[32][4] = {
|
||||
{0x3f800000u, 0x00000000u, 0xbf80a8ceu, 0x3f11bdb2u},
|
||||
{0x3f000000u, 0x00000001u, 0x3ecf5c23u, 0xbd80681du},
|
||||
{0xbf800000u, 0x000000ffu, 0xbf5a19dau, 0x3f60be0fu},
|
||||
{0xbf000000u, 0x00000100u, 0x3f6c6e9eu, 0x3d190ff1u},
|
||||
{0x3e800000u, 0x0000ffffu, 0x3c79e575u, 0xbf4ab56eu},
|
||||
{0x40000000u, 0x00010000u, 0xbfbec755u, 0xbec61281u},
|
||||
{0x437f0000u, 0x7fffffffu, 0x3e831de6u, 0xbfa566ffu},
|
||||
{0x477ff000u, 0x80000000u, 0x3f6199cdu, 0xbf4da142u},
|
||||
{0x80000000u, 0xffffffffu, 0xbf4d4a50u, 0xbfafbe42u},
|
||||
{0x3b808081u, 0xffff8000u, 0x3fbf1042u, 0x3f374fecu},
|
||||
{0x3b008081u, 0x00007fffu, 0x3f9003e3u, 0x3eb201d6u},
|
||||
{0x3bc0c0c1u, 0xffffff80u, 0xbfb2eac7u, 0xbf0361dcu},
|
||||
{0x0da24260u, 0x00000080u, 0xbbf09defu, 0xbf93876fu},
|
||||
{0x7f800000u, 0x00008000u, 0x3fad8de1u, 0xbeca10e3u},
|
||||
{0xff800000u, 0x27a1d402u, 0x3f391acfu, 0xbf9e07a5u},
|
||||
{0x7fc00000u, 0xcf014130u, 0x3fa45498u, 0xbe0ffdf6u},
|
||||
{0x7f800001u, 0x8d500f76u, 0x3f2f74afu, 0xbe97896au},
|
||||
{0xffc12345u, 0x26ae54eeu, 0xbf70743eu, 0xbfb1bd09u},
|
||||
{0x00000001u, 0xddaaad70u, 0x3f75def3u, 0xbfa3583au},
|
||||
{0xbf9af15au, 0xac7dc96bu, 0x3fb401deu, 0x3f97486bu},
|
||||
{0x3f827428u, 0xc34660fcu, 0x3f98994bu, 0x3fbbb4e1u},
|
||||
{0xbfdbaf7bu, 0x8f643855u, 0xbd9174f1u, 0x3f7b265fu},
|
||||
{0xbf990c1eu, 0xec54b3b3u, 0xbf87e0c3u, 0x3ddc18c9u},
|
||||
{0xbf550148u, 0xf45aa8b6u, 0xbe158a3bu, 0xbeaf720bu},
|
||||
{0x3ffd5c07u, 0x62f7c6f9u, 0xbf33f058u, 0x3f804d65u},
|
||||
{0xbfae9eedu, 0xb74ba47du, 0x3fbd3e83u, 0xbc441430u},
|
||||
{0xbd0a9d77u, 0x8ce90a07u, 0x3e98a50bu, 0x3d883addu},
|
||||
{0x3e927841u, 0x51927773u, 0x3e2d872fu, 0x3ebc5937u},
|
||||
{0xbfa37e42u, 0x2b8b23a4u, 0x3ec24cacu, 0x3f801ffau},
|
||||
{0xbf2ba50du, 0xc308bd98u, 0x3e955afau, 0xbf14853eu},
|
||||
{0xbfcb1fc5u, 0x10ce0a84u, 0x3fa27b59u, 0xbf7a5aa6u},
|
||||
{0xbe91fdb4u, 0x28ee3a6du, 0x3f0cd06eu, 0x3f0717b3u},
|
||||
};
|
||||
constexpr std::uint32_t Expected[32][16] = {
|
||||
{0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa3c00u, 0x000003c0u, 0x800003ffu, 0x388dbc05u, 0x00000000u, 0x00000000u, 0x007f8000u, 0x00048e01u},
|
||||
{0xaaaaaa80u, 0xaaaaaa40u, 0xaaaa8000u, 0xaaaa4000u, 0xaaaaaa80u, 0xaaaa4000u, 0xaaaaaa01u, 0xaaaa0001u, 0xaaaa3800u, 0x00000380u, 0x19e00200u, 0xac03367au, 0x00000001u, 0x00000000u, 0x00807f01u, 0x000f80cfu},
|
||||
{0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa00ffu, 0xaaaabc00u, 0x00000000u, 0xc0000000u, 0x3b05bad0u, 0x000000ffu, 0x00000000u, 0x007f807fu, 0x0007064du},
|
||||
{0xaaaaaa00u, 0xaaaaaac0u, 0xaaaa0000u, 0xaaaac000u, 0xaaaaaa00u, 0xaaaac000u, 0xaaaaaaffu, 0xaaaa0100u, 0xaaaab800u, 0x00000000u, 0x3b100000u, 0x28c83b63u, 0x00000100u, 0x00000000u, 0x007f7f7fu, 0x00004dd8u},
|
||||
{0xaaaaaa40u, 0xaaaaaa20u, 0xaaaa4000u, 0xaaaa2000u, 0xaaaaaa40u, 0xaaaa2000u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa3400u, 0x00000340u, 0x01000100u, 0xba5523cfu, 0x0000ffffu, 0x00000000u, 0x00807f7fu, 0x0009ac08u},
|
||||
{0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa4000u, 0x00000400u, 0x000003ffu, 0xb630bdf6u, 0x00010000u, 0x00000000u, 0x0080807fu, 0x000cea01u},
|
||||
{0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa5bf8u, 0x000005bfu, 0x106003ffu, 0xbd2b3418u, 0x7fffffffu, 0x00000000u, 0x00807f7fu, 0x00080483u},
|
||||
{0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa7bffu, 0x000007bfu, 0x386003ffu, 0xba6d3b0cu, 0x80000000u, 0x00000000u, 0x00807f80u, 0x000999c2u},
|
||||
{0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaaffu, 0xaaaaffffu, 0xaaaa8000u, 0x00000000u, 0x00000000u, 0xbd7dba6au, 0xffffffffu, 0x00000000u, 0x008080ffu, 0x00080666u},
|
||||
{0xaaaaaa01u, 0xaaaaaa00u, 0xaaaa0101u, 0xaaaa0080u, 0xaaaaaa01u, 0xaaaa0080u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa1c04u, 0x000001c0u, 0xbff00004u, 0x39ba3df8u, 0xffff8000u, 0x00000000u, 0x007f7f80u, 0x0005b9ffu},
|
||||
{0xaaaaaa01u, 0xaaaaaa00u, 0xaaaa0081u, 0xaaaa0040u, 0xaaaaaa01u, 0xaaaa0040u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa1804u, 0x00000180u, 0x7ff00002u, 0x35903c80u, 0x00007fffu, 0x00000000u, 0x007f7f7fu, 0x0002c9ffu},
|
||||
{0xaaaaaa02u, 0xaaaaaa01u, 0xaaaa0182u, 0xaaaa00c1u, 0xaaaaaa02u, 0xaaaa00c1u, 0xaaaaaaffu, 0xaaaaff80u, 0xaaaa1e06u, 0x000001e0u, 0x00000006u, 0xb81bbd97u, 0xffffff80u, 0x00000000u, 0x00808080u, 0x000bea01u},
|
||||
{0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaa80u, 0xaaaa0080u, 0xaaaa0000u, 0x00000000u, 0x00000000u, 0xbc9c9f84u, 0x00000080u, 0x00000000u, 0x0080807fu, 0x000807fcu},
|
||||
{0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa7c00u, 0x000007c0u, 0x3ff003ffu, 0xb6503d6cu, 0x00008000u, 0x00000000u, 0x00807f7fu, 0x000cd9ffu},
|
||||
{0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaafc00u, 0x00000000u, 0x2e400000u, 0xbcf039c8u, 0x27a1d402u, 0x00000000u, 0x00807f7fu, 0x00080571u},
|
||||
{0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa7e00u, 0x000007e0u, 0x3ff00000u, 0xb07f3d22u, 0xcf014130u, 0x00000000u, 0x00807f80u, 0x000ee1ffu},
|
||||
{0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa7c01u, 0x000007c1u, 0x2bd00000u, 0xb4bc397bu, 0x8d500f76u, 0x00000000u, 0x00807f80u, 0x000da55eu},
|
||||
{0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaafe09u, 0x000007e0u, 0x00000000u, 0xbd8dbb83u, 0x26ae54eeu, 0x00000000u, 0x0080807fu, 0x00080620u},
|
||||
{0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa0000u, 0x00000000u, 0x3d700000u, 0xbd1a3baeu, 0xddaaad70u, 0x00000000u, 0x00807f80u, 0x000805ebu},
|
||||
{0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaabcd7u, 0x00000000u, 0xfff00000u, 0x3cba3da0u, 0xac7dc96bu, 0x00000000u, 0x007f7f80u, 0x0007fdffu},
|
||||
{0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa3c13u, 0x000003c1u, 0xfff003ffu, 0x3ddd3cc4u, 0xc34660fcu, 0x00000000u, 0x007f7f80u, 0x0007fdffu},
|
||||
{0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaabeddu, 0x00000000u, 0xc0000000u, 0x3bd9ac8bu, 0x8f643855u, 0x00000000u, 0x007f8080u, 0x0007d7dcu},
|
||||
{0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaabcc8u, 0x00000000u, 0x00000000u, 0x2ee0bc3fu, 0xec54b3b3u, 0x00000000u, 0x007f8080u, 0x0000de01u},
|
||||
{0xaaaaaa00u, 0xaaaaaa96u, 0xaaaa0000u, 0xaaaa9580u, 0xaaaaaa00u, 0xaaaa9580u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaabaa8u, 0x00000000u, 0x00000000u, 0xb57bb0acu, 0xf45aa8b6u, 0x00000000u, 0x00808080u, 0x000d47b5u},
|
||||
{0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa3feau, 0x000003feu, 0xc00fffffu, 0x3c02b99fu, 0x62f7c6f9u, 0x00000000u, 0x007f807fu, 0x0007fe99u},
|
||||
{0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaabd74u, 0x00000000u, 0x3ff00000u, 0xa2203de9u, 0xb74ba47du, 0x00000000u, 0x00807f80u, 0x000fe9ffu},
|
||||
{0xaaaaaa00u, 0xaaaaaafcu, 0xaaaa0000u, 0xaaaafbabu, 0xaaaaaa00u, 0xaaaafbabu, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaaa854u, 0x00000000u, 0x13100000u, 0x2c4134c5u, 0x8ce90a07u, 0x00000000u, 0x007f7f80u, 0x00008898u},
|
||||
{0xaaaaaa49u, 0xaaaaaa24u, 0xaaaa493cu, 0xaaaa249eu, 0xaaaaaa49u, 0xaaaa249eu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa3493u, 0x00000349u, 0x4adffd25u, 0x35e2316cu, 0x51927773u, 0x00000000u, 0x007f7f7fu, 0x0002f057u},
|
||||
{0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaabd1bu, 0x00000000u, 0xd8400000u, 0x3c003612u, 0x2b8b23a4u, 0x00000000u, 0x007f7f7fu, 0x0007fcc2u},
|
||||
{0xaaaaaa00u, 0xaaaaaaabu, 0xaaaa0000u, 0xaaaaaa2eu, 0xaaaaaa00u, 0xaaaaaa2eu, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaab95du, 0x00000000u, 0x12a00000u, 0xb8a434aau, 0xc308bd98u, 0x00000000u, 0x00807f80u, 0x000b6095u},
|
||||
{0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaabe58u, 0x00000000u, 0x3ff00000u, 0xbbd23d13u, 0x10ce0a84u, 0x00000000u, 0x00807f7fu, 0x000831ffu},
|
||||
{0xaaaaaa00u, 0xaaaaaadcu, 0xaaaa0000u, 0xaaaadb81u, 0xaaaaaa00u, 0xaaaadb81u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaab48fu, 0x00000000u, 0xa3300000u, 0x38383866u, 0x28ee3a6du, 0x00000000u, 0x007f7f7fu, 0x00043919u},
|
||||
};
|
||||
constexpr const char* Names[16] = {"8unorm", "8snorm", "16unorm", "16snorm", "8uscaled", "16sscaled", "8uint", "16sint", "16float", "uf11_x", "unorm1010102_xyzw", "f16x2_xy", "uint32x2_x", "uint32x2_y", "sint8888_xyz", "snorm1010102_xy"};
|
||||
|
||||
void Fill(std::uint32_t tid, std::uint32_t* words) {
|
||||
std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words);
|
||||
}
|
||||
|
||||
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(data);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
|
||||
}
|
||||
|
||||
std::string Hex(std::uint32_t value) {
|
||||
char text[16];
|
||||
std::snprintf(text, sizeof(text), "0x%08x", value);
|
||||
return text;
|
||||
}
|
||||
|
||||
void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
|
||||
Require(actual == expected, std::string("formatted store: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
|
||||
}
|
||||
|
||||
void Run(AgcDriver::VulkanDevice& device) {
|
||||
for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
|
||||
Output.fill(0xdeadbeefu);
|
||||
std::vector<std::uint32_t> userData(8, 0u);
|
||||
const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
|
||||
const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
|
||||
std::copy(input.begin(), input.end(), userData.begin());
|
||||
std::copy(output.begin(), output.end(), userData.begin() + 4);
|
||||
const std::span<const std::uint32_t> code(Code);
|
||||
const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
|
||||
const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
|
||||
ShaderRecompiler::RecompileRequest request{
|
||||
{ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
|
||||
{32, 0, userData, compute, std::nullopt, std::nullopt, memory},
|
||||
device.Target(),
|
||||
{0, 0, 0, 128}
|
||||
};
|
||||
request.useCache = false;
|
||||
const auto result = ShaderRecompiler::Recompile(request);
|
||||
device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
|
||||
device.WaitIdle();
|
||||
}
|
||||
|
||||
void Check() {
|
||||
for (std::uint32_t tid = 0; tid < Threads; ++tid) {
|
||||
const std::uint32_t* in = &Input[tid * Inputs];
|
||||
const std::uint32_t* out = &Output[tid * Results];
|
||||
for (std::uint32_t index = 0; index < 16; ++index) Expect(tid, out[index], Expected[tid][index], Names[index]);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main() {
|
||||
try {
|
||||
const auto device = OpenVulkanTestDevice();
|
||||
if (!device) return VulkanTestSkipped;
|
||||
Run(*device);
|
||||
Check();
|
||||
std::puts("formatted store tests passed");
|
||||
return 0;
|
||||
} catch (const std::exception& error) {
|
||||
std::cerr << error.what() << '\n';
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -107,6 +107,7 @@ bool IsSignedFormatComponent(SpirvFormatComponentType type);
|
|||
std::uint32_t EmitUFloatToF32Bits(SpirvEmitterState& state, std::uint32_t raw, std::uint32_t bits);
|
||||
std::uint32_t NormalizeFormatComponent(SpirvEmitterState& state, const SpirvBufferFormatInfo& info, std::uint32_t component, std::uint32_t raw);
|
||||
std::uint32_t EmitD16FormatComponent(SpirvEmitterState& state, const SpirvBufferFormatInfo& info, std::uint32_t component, std::uint32_t raw);
|
||||
std::uint32_t EmitFormatStoreComponent(SpirvEmitterState& state, const SpirvBufferFormatInfo& info, std::uint32_t component, std::uint32_t data);
|
||||
void EmitDeviceAtomicMemoryBarrier(SpirvEmitterState& state);
|
||||
std::uint32_t EmitFloatAtomicReplacement(SpirvEmitterState& state, std::uint32_t old, std::uint32_t source, bool maxValue);
|
||||
std::uint32_t EmitDsSwizzleTargetLane(SpirvEmitterState& state, std::uint32_t subid, std::uint32_t control);
|
||||
|
|
|
|||
|
|
@ -164,28 +164,74 @@ std::uint32_t IntegerToF16Bits(SpirvEmitterState& state, std::uint32_t value) {
|
|||
return EmitSelectValueU32(state, zero, ConstantU32(state, 0u), Ext(state, TypeU32(state), GLSLstd450UMin, bits, ConstantU32(state, 0x7bffu)));
|
||||
}
|
||||
|
||||
std::uint32_t F32BitsToF16BitsRtz(SpirvEmitterState& state, std::uint32_t bits) {
|
||||
const auto sign = EmitAndConstant(state, EmitBinaryU32(state, spv::OpShiftRightLogical, bits, ConstantU32(state, 16u)), 0x8000u);
|
||||
std::uint32_t F32BitsToSmallFloatRtz(SpirvEmitterState& state, std::uint32_t bits, std::uint32_t mantissaBits, bool signedFormat) {
|
||||
const std::uint32_t infinity = 0x1fu << mantissaBits;
|
||||
const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, bits, 0x80000000u), 0u);
|
||||
const auto sign = signedFormat ? EmitSelectValueU32(state, negative, ConstantU32(state, 1u << (5u + mantissaBits)), ConstantU32(state, 0u)) : ConstantU32(state, 0u);
|
||||
const auto exponent = EmitAndConstant(state, EmitBinaryU32(state, spv::OpShiftRightLogical, bits, ConstantU32(state, 23u)), 0xffu);
|
||||
const auto mantissa = EmitAndConstant(state, bits, 0x7fffffu);
|
||||
const auto top = EmitBinaryU32(state, spv::OpShiftRightLogical, mantissa, ConstantU32(state, 13u));
|
||||
const auto top = EmitBinaryU32(state, spv::OpShiftRightLogical, mantissa, ConstantU32(state, 23u - mantissaBits));
|
||||
const auto lostPayload = EmitLogicalAndBool(state, EmitCompareU32Constant(state, spv::OpIEqual, top, 0u), EmitCompareU32Constant(state, spv::OpINotEqual, mantissa, 0u));
|
||||
const auto special = EmitOrU32(state, ConstantU32(state, 0x7c00u), EmitSelectValueU32(state, lostPayload, ConstantU32(state, 1u), top));
|
||||
const auto special = EmitOrU32(state, ConstantU32(state, infinity), EmitSelectValueU32(state, lostPayload, ConstantU32(state, 1u), top));
|
||||
const auto biased = Binary(state, spv::OpISub, TypeI32(state), EmitTBufferBitcastU32ToI32(state, exponent), EmitTBufferBitcastU32ToI32(state, ConstantU32(state, 112u)));
|
||||
const auto atLeast = [&](std::uint32_t limit) {
|
||||
return Binary(state, spv::OpSGreaterThanEqual, TypeBool(state), biased, EmitTBufferBitcastU32ToI32(state, ConstantU32(state, limit)));
|
||||
};
|
||||
const auto biasedU = Unary(state, spv::OpBitcast, TypeU32(state), biased);
|
||||
const auto normal = EmitOrU32(state, EmitBinaryU32(state, spv::OpShiftLeftLogical, biasedU, ConstantU32(state, 10u)), top);
|
||||
const auto subnormalShift = EmitBinaryU32(state, spv::OpISub, ConstantU32(state, 14u), biasedU);
|
||||
const auto normal = EmitOrU32(state, EmitBinaryU32(state, spv::OpShiftLeftLogical, biasedU, ConstantU32(state, mantissaBits)), top);
|
||||
const auto subnormalShift = EmitBinaryU32(state, spv::OpISub, ConstantU32(state, 24u - mantissaBits), biasedU);
|
||||
const auto subnormal = EmitBinaryU32(state, spv::OpShiftRightLogical, EmitOrU32(state, mantissa, ConstantU32(state, 0x800000u)), Ext(state, TypeU32(state), GLSLstd450UMin, subnormalShift, ConstantU32(state, 31u)));
|
||||
auto magnitude = EmitSelectValueU32(state, atLeast(static_cast<std::uint32_t>(-10)), subnormal, ConstantU32(state, 0u));
|
||||
auto magnitude = EmitSelectValueU32(state, atLeast(static_cast<std::uint32_t>(-static_cast<std::int32_t>(mantissaBits))), subnormal, ConstantU32(state, 0u));
|
||||
magnitude = EmitSelectValueU32(state, atLeast(1u), normal, magnitude);
|
||||
magnitude = EmitSelectValueU32(state, atLeast(31u), ConstantU32(state, 0x7bffu), magnitude);
|
||||
magnitude = EmitSelectValueU32(state, EmitCompareU32Constant(state, spv::OpIEqual, exponent, 0xffu), special, magnitude);
|
||||
magnitude = EmitSelectValueU32(state, atLeast(31u), ConstantU32(state, infinity - 1u), magnitude);
|
||||
const auto isSpecial = EmitCompareU32Constant(state, spv::OpIEqual, exponent, 0xffu);
|
||||
magnitude = EmitSelectValueU32(state, isSpecial, special, magnitude);
|
||||
if (!signedFormat) {
|
||||
const auto nan = EmitLogicalAndBool(state, isSpecial, EmitCompareU32Constant(state, spv::OpINotEqual, mantissa, 0u));
|
||||
magnitude = EmitSelectValueU32(state, EmitLogicalAndBool(state, negative, EmitLogicalNotBool(state, nan)), ConstantU32(state, 0u), magnitude);
|
||||
}
|
||||
return EmitOrU32(state, sign, magnitude);
|
||||
}
|
||||
|
||||
std::uint32_t NormStoreBits(SpirvEmitterState& state, std::uint32_t bits, std::uint32_t maximum, bool signedFormat) {
|
||||
const auto value = EmitBitcastU32ToF32(state, bits);
|
||||
const auto magnitudeBits = EmitAndConstant(state, bits, 0x7fffffffu);
|
||||
const auto magnitude = EmitBitcastU32ToF32(state, magnitudeBits);
|
||||
const auto exponent = EmitAndConstant(state, EmitBinaryU32(state, spv::OpShiftRightLogical, bits, ConstantU32(state, 23u)), 0xffu);
|
||||
const auto denormal = EmitCompareU32Constant(state, spv::OpIEqual, exponent, 0u);
|
||||
const auto mantissa = EmitOrU32(state, EmitAndConstant(state, bits, 0x7fffffu), EmitSelectValueU32(state, denormal, ConstantU32(state, 0u), ConstantU32(state, 0x800000u)));
|
||||
const auto shift = EmitSelectValueU32(state, denormal, ConstantU32(state, 149u), EmitBinaryU32(state, spv::OpISub, ConstantU32(state, 150u), exponent));
|
||||
const auto extra = EmitBinaryU32(state, spv::OpISub, shift, ConstantU32(state, 24u));
|
||||
const auto product = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpUMulExtended, state.module.Type(spv::OpTypeStruct, TypeU32(state), TypeU32(state)), product, mantissa, ConstantU32(state, maximum));
|
||||
const auto low = state.module.AllocateId();
|
||||
const auto high = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, product, 0u);
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), high, product, 1u);
|
||||
const auto integer = EmitOrU32(state, EmitBinaryU32(state, spv::OpShiftRightLogical, low, ConstantU32(state, 24u)), EmitBinaryU32(state, spv::OpShiftLeftLogical, high, ConstantU32(state, 8u)));
|
||||
const auto fraction = EmitAndConstant(state, low, 0xffffffu);
|
||||
const auto fixed = EmitOrU32(state, EmitBinaryU32(state, spv::OpShiftLeftLogical, integer, ConstantU32(state, 16u)), EmitBinaryU32(state, spv::OpShiftRightLogical, fraction, ConstantU32(state, 8u)));
|
||||
const auto sticky = EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, fraction, 0xffu), 0u);
|
||||
const auto safeExtra = Ext(state, TypeU32(state), GLSLstd450UMin, extra, ConstantU32(state, 16u));
|
||||
const auto quotient = EmitBinaryU32(state, spv::OpShiftRightLogical, EmitBinaryU32(state, spv::OpShiftRightLogical, fixed, ConstantU32(state, 16u)), safeExtra);
|
||||
const auto remainder = EmitBinaryU32(state, spv::OpISub, fixed, EmitBinaryU32(state, spv::OpShiftLeftLogical, EmitBinaryU32(state, spv::OpShiftLeftLogical, quotient, ConstantU32(state, 16u)), safeExtra));
|
||||
const auto half = EmitBinaryU32(state, spv::OpShiftLeftLogical, ConstantU32(state, 1u << 15u), safeExtra);
|
||||
const auto odd = EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, quotient, 1u), 0u);
|
||||
const auto tie = EmitLogicalAndBool(state, Binary(state, spv::OpIEqual, TypeBool(state), remainder, half), EmitLogicalOrBool(state, sticky, odd));
|
||||
const auto up = EmitLogicalOrBool(state, UnsignedGreater(state, remainder, half), tie);
|
||||
const auto rounded = EmitAddU32(state, quotient, EmitSelectValueU32(state, up, ConstantU32(state, 1u), ConstantU32(state, 0u)));
|
||||
auto result = EmitSelectValueU32(state, UnsignedGreater(state, extra, ConstantU32(state, 16u)), ConstantU32(state, 0u), rounded);
|
||||
const auto full = Binary(state, spv::OpFOrdGreaterThanEqual, TypeBool(state), magnitude, ConstantF32Value(state, 1.0f));
|
||||
result = EmitSelectValueU32(state, full, ConstantU32(state, maximum), result);
|
||||
const auto positive = Binary(state, spv::OpFOrdGreaterThan, TypeBool(state), signedFormat ? magnitude : value, ConstantF32Value(state, 0.0f));
|
||||
result = EmitSelectValueU32(state, positive, result, ConstantU32(state, 0u));
|
||||
if (!signedFormat) {
|
||||
return result;
|
||||
}
|
||||
const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, bits, 0x80000000u), 0u);
|
||||
return EmitSelectValueU32(state, negative, Unary(state, spv::OpSNegate, TypeU32(state), result), result);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::uint32_t EmitD16FormatComponent(SpirvEmitterState& state, const SpirvBufferFormatInfo& info, std::uint32_t component, std::uint32_t raw) {
|
||||
|
|
@ -219,7 +265,36 @@ std::uint32_t EmitD16FormatComponent(SpirvEmitterState& state, const SpirvBuffer
|
|||
}
|
||||
case SpirvFormatComponentType::Float:
|
||||
if (bits == 16u) return EmitAndConstant(state, raw, 0xffffu);
|
||||
return F32BitsToF16BitsRtz(state, NormalizeFormatComponent(state, info, component, raw));
|
||||
return F32BitsToSmallFloatRtz(state, NormalizeFormatComponent(state, info, component, raw), 10u, true);
|
||||
default:
|
||||
FailEmit("buffer format component type is not supported");
|
||||
}
|
||||
}
|
||||
|
||||
std::uint32_t EmitFormatStoreComponent(SpirvEmitterState& state, const SpirvBufferFormatInfo& info, std::uint32_t component, std::uint32_t data) {
|
||||
const auto bits = info.componentBits[component];
|
||||
const std::uint32_t mask = bits >= 32u ? 0xffffffffu : (1u << bits) - 1u;
|
||||
switch (info.type) {
|
||||
case SpirvFormatComponentType::Uint:
|
||||
return bits >= 32u ? data : Ext(state, TypeU32(state), GLSLstd450UMin, data, ConstantU32(state, mask));
|
||||
case SpirvFormatComponentType::Sint: {
|
||||
if (bits >= 32u) return data;
|
||||
const auto low = EmitTBufferBitcastU32ToI32(state, ConstantU32(state, ~(mask >> 1u)));
|
||||
const auto high = EmitTBufferBitcastU32ToI32(state, ConstantU32(state, mask >> 1u));
|
||||
const auto clamped = Ext(state, TypeI32(state), GLSLstd450SMin, Ext(state, TypeI32(state), GLSLstd450SMax, EmitTBufferBitcastU32ToI32(state, data), low), high);
|
||||
return EmitAndConstant(state, Unary(state, spv::OpBitcast, TypeU32(state), clamped), mask);
|
||||
}
|
||||
case SpirvFormatComponentType::Unorm:
|
||||
case SpirvFormatComponentType::Uscaled:
|
||||
return NormStoreBits(state, data, mask, false);
|
||||
case SpirvFormatComponentType::Snorm:
|
||||
case SpirvFormatComponentType::Sscaled:
|
||||
return EmitAndConstant(state, NormStoreBits(state, data, mask >> 1u, true), mask);
|
||||
case SpirvFormatComponentType::Float:
|
||||
if (bits == 32u) return data;
|
||||
if (bits == 16u) return F32BitsToSmallFloatRtz(state, data, 10u, true);
|
||||
if (bits == 11u || bits == 10u) return F32BitsToSmallFloatRtz(state, data, bits - 5u, false);
|
||||
FailEmit("buffer float format component width is not supported for stores");
|
||||
default:
|
||||
FailEmit("buffer format component type is not supported");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -434,31 +434,6 @@ void StoreWord(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo
|
|||
});
|
||||
}
|
||||
|
||||
void FormattedStorePrepared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t component, const MemoryResourceAccess& resource, std::uint32_t data) {
|
||||
const auto info = MemoryFormatInfo(ctx.state, mem);
|
||||
if (info.type == SpirvFormatComponentType::Unknown) {
|
||||
StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource, data);
|
||||
return;
|
||||
}
|
||||
if (component >= info.componentCount) {
|
||||
return;
|
||||
}
|
||||
const auto bits = info.componentBits[component];
|
||||
const auto componentMem = RebaseFormattedComponent(mem, info, component);
|
||||
if (bits == 8u || bits == 16u) {
|
||||
StoreSubwordPrepared(ctx, inst, componentMem, resource, bits, data);
|
||||
} else {
|
||||
StoreWordPrepared(ctx, inst, componentMem, resource, data);
|
||||
}
|
||||
}
|
||||
|
||||
void FormattedStore(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) {
|
||||
EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() {
|
||||
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
|
||||
FormattedStorePrepared(ctx, inst, mem, 0u, resource, ctx.Arg(inst, inst.ArgumentCount() - 2u));
|
||||
});
|
||||
}
|
||||
|
||||
struct PreparedFormattedMemory {
|
||||
SpirvBufferFormatInfo info;
|
||||
MemoryResourceAccess resource;
|
||||
|
|
@ -486,7 +461,7 @@ PreparedFormattedMemory PrepareFormattedMemory(SpirvValueEmitContext& ctx, const
|
|||
}
|
||||
}
|
||||
} else {
|
||||
for (std::uint32_t component = 0; component < std::min(components, plan.info.componentCount); component++) {
|
||||
for (std::uint32_t component = 0; component < plan.info.componentCount; component++) {
|
||||
required.at(component) = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -552,18 +527,52 @@ std::uint32_t FormattedOutOfBoundsValue(SpirvValueEmitContext& ctx, const Memory
|
|||
return ConstructU32Composite(state, components, values);
|
||||
}
|
||||
|
||||
void StoreFormattedInBounds(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, std::uint32_t component, std::uint32_t data) {
|
||||
if (component >= plan.info.componentCount) {
|
||||
void StoreFormattedElement(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, const std::array<std::uint32_t, 4>& data, std::uint32_t components) {
|
||||
auto& state = ctx.state;
|
||||
const auto& info = plan.info;
|
||||
std::array<std::uint32_t, 4> encoded{};
|
||||
for (std::uint32_t component = 0; component < info.componentCount; component++) {
|
||||
encoded.at(component) = component < components ? EmitFormatStoreComponent(state, info, component, data.at(component)) : ConstantU32(state, 0u);
|
||||
}
|
||||
if (info.packedBitfield) {
|
||||
auto word = ConstantU32(state, 0u);
|
||||
for (std::uint32_t component = 0; component < info.componentCount; component++) {
|
||||
const std::uint32_t mask = (1u << info.componentBits[component]) - 1u;
|
||||
word = EmitOrU32(state, word, EmitBinaryU32(state, spv::OpShiftLeftLogical, EmitAndConstant(state, encoded.at(component), mask), ConstantU32(state, info.componentBitOffset[component])));
|
||||
}
|
||||
StoreWordInBounds(ctx, plan.resource, plan.indices.at(0), word);
|
||||
return;
|
||||
}
|
||||
const auto bits = plan.info.componentBits[component];
|
||||
if (bits == 8u || bits == 16u) {
|
||||
StoreSubwordInBounds(ctx, mem, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, data);
|
||||
} else {
|
||||
StoreWordInBounds(ctx, plan.resource, plan.indices.at(component), data);
|
||||
for (std::uint32_t component = 0; component < info.componentCount; component++) {
|
||||
const auto bits = info.componentBits[component];
|
||||
if (bits == 8u || bits == 16u) {
|
||||
StoreSubwordInBounds(ctx, mem, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, encoded.at(component));
|
||||
} else {
|
||||
StoreWordInBounds(ctx, plan.resource, plan.indices.at(component), encoded.at(component));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void StoreFormattedValues(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource, const SpirvBufferFormatInfo& info, const std::array<std::uint32_t, 4>& data, std::uint32_t components) {
|
||||
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, FormattedAccess::Store);
|
||||
EmitIfCondition(ctx.state, plan.inBounds, [&]() {
|
||||
StoreFormattedElement(ctx, mem, plan, data, components);
|
||||
});
|
||||
}
|
||||
|
||||
void FormattedStore(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) {
|
||||
EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() {
|
||||
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
|
||||
const auto info = MemoryFormatInfo(ctx.state, mem);
|
||||
const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u);
|
||||
if (info.type == SpirvFormatComponentType::Unknown) {
|
||||
StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, 0u), resource, data);
|
||||
return;
|
||||
}
|
||||
StoreFormattedValues(ctx, inst, mem, resource, info, {data, 0u, 0u, 0u}, 1u);
|
||||
});
|
||||
}
|
||||
|
||||
std::uint32_t LoadWideBuffer(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) {
|
||||
auto& state = ctx.state;
|
||||
return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() {
|
||||
|
|
@ -594,14 +603,12 @@ void StoreWideBuffer(SpirvValueEmitContext& ctx, const IrValue& inst, const Memo
|
|||
const auto composite = ctx.Arg(inst, inst.ArgumentCount() - 2u);
|
||||
const auto info = MemoryFormatInfo(state, mem);
|
||||
if (info.type != SpirvFormatComponentType::Unknown) {
|
||||
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, FormattedAccess::Store);
|
||||
EmitIfCondition(state, plan.inBounds, [&]() {
|
||||
for (std::uint32_t component = 0; component < components; component++) {
|
||||
const auto data = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), data, composite, component);
|
||||
StoreFormattedInBounds(ctx, mem, plan, component, data);
|
||||
}
|
||||
});
|
||||
std::array<std::uint32_t, 4> data{};
|
||||
for (std::uint32_t component = 0; component < components; component++) {
|
||||
data.at(component) = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), data.at(component), composite, component);
|
||||
}
|
||||
StoreFormattedValues(ctx, inst, mem, resource, info, data, components);
|
||||
return;
|
||||
}
|
||||
for (std::uint32_t component = 0; component < components; component++) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue