diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStore.cpp new file mode 100644 index 000000000..8c8491eab --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStore.cpp @@ -0,0 +1,173 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084, 0x34060086, 0xe0301000, 0x80000401, 0xe0301004, 0x80000501, 0xe0301008, 0x80000601, + 0xe030100c, 0x80000701, 0xbf8c3f70, 0x7e3c02ff, 0xaaaaaaaa, 0xe0701000, 0x80011e03, 0xe0701004, + 0x80011e03, 0xe0701008, 0x80011e03, 0xe070100c, 0x80011e03, 0xe0701010, 0x80011e03, 0xe0701014, + 0x80011e03, 0xe0701018, 0x80011e03, 0xe070101c, 0x80011e03, 0xe0701020, 0x80011e03, 0xe0701024, + 0x80011e03, 0xe0701028, 0x80011e03, 0xe070102c, 0x80011e03, 0xe0701030, 0x80011e03, 0xe0701034, + 0x80011e03, 0xe0701038, 0x80011e03, 0xe070103c, 0x80011e03, 0xbf8c3f70, 0xe80c1000, 0x80010403, + 0xe8141004, 0x80010403, 0xe83c1008, 0x80010403, 0xe844100c, 0x80010403, 0xe81c1010, 0x80010403, + 0xe8541014, 0x80010403, 0xe82c1018, 0x80010503, 0xe864101c, 0x80010503, 0xe86c1020, 0x80010403, + 0xe9241024, 0x80010403, 0xe9971028, 0x80010403, 0xe8ed102c, 0x80010603, 0xe9f41030, 0x80010503, + 0xe9ee1038, 0x80010503, 0xe99d103c, 0x80010603, 0xbf810000, +}; + +constexpr std::uint32_t Rows[32][4] = { + {0x3f800000u, 0x00000000u, 0xbf80a8ceu, 0x3f11bdb2u}, + {0x3f000000u, 0x00000001u, 0x3ecf5c23u, 0xbd80681du}, + {0xbf800000u, 0x000000ffu, 0xbf5a19dau, 0x3f60be0fu}, + {0xbf000000u, 0x00000100u, 0x3f6c6e9eu, 0x3d190ff1u}, + {0x3e800000u, 0x0000ffffu, 0x3c79e575u, 0xbf4ab56eu}, + {0x40000000u, 0x00010000u, 0xbfbec755u, 0xbec61281u}, + {0x437f0000u, 0x7fffffffu, 0x3e831de6u, 0xbfa566ffu}, + {0x477ff000u, 0x80000000u, 0x3f6199cdu, 0xbf4da142u}, + {0x80000000u, 0xffffffffu, 0xbf4d4a50u, 0xbfafbe42u}, + {0x3b808081u, 0xffff8000u, 0x3fbf1042u, 0x3f374fecu}, + {0x3b008081u, 0x00007fffu, 0x3f9003e3u, 0x3eb201d6u}, + {0x3bc0c0c1u, 0xffffff80u, 0xbfb2eac7u, 0xbf0361dcu}, + {0x0da24260u, 0x00000080u, 0xbbf09defu, 0xbf93876fu}, + {0x7f800000u, 0x00008000u, 0x3fad8de1u, 0xbeca10e3u}, + {0xff800000u, 0x27a1d402u, 0x3f391acfu, 0xbf9e07a5u}, + {0x7fc00000u, 0xcf014130u, 0x3fa45498u, 0xbe0ffdf6u}, + {0x7f800001u, 0x8d500f76u, 0x3f2f74afu, 0xbe97896au}, + {0xffc12345u, 0x26ae54eeu, 0xbf70743eu, 0xbfb1bd09u}, + {0x00000001u, 0xddaaad70u, 0x3f75def3u, 0xbfa3583au}, + {0xbf9af15au, 0xac7dc96bu, 0x3fb401deu, 0x3f97486bu}, + {0x3f827428u, 0xc34660fcu, 0x3f98994bu, 0x3fbbb4e1u}, + {0xbfdbaf7bu, 0x8f643855u, 0xbd9174f1u, 0x3f7b265fu}, + {0xbf990c1eu, 0xec54b3b3u, 0xbf87e0c3u, 0x3ddc18c9u}, + {0xbf550148u, 0xf45aa8b6u, 0xbe158a3bu, 0xbeaf720bu}, + {0x3ffd5c07u, 0x62f7c6f9u, 0xbf33f058u, 0x3f804d65u}, + {0xbfae9eedu, 0xb74ba47du, 0x3fbd3e83u, 0xbc441430u}, + {0xbd0a9d77u, 0x8ce90a07u, 0x3e98a50bu, 0x3d883addu}, + {0x3e927841u, 0x51927773u, 0x3e2d872fu, 0x3ebc5937u}, + {0xbfa37e42u, 0x2b8b23a4u, 0x3ec24cacu, 0x3f801ffau}, + {0xbf2ba50du, 0xc308bd98u, 0x3e955afau, 0xbf14853eu}, + {0xbfcb1fc5u, 0x10ce0a84u, 0x3fa27b59u, 0xbf7a5aa6u}, + {0xbe91fdb4u, 0x28ee3a6du, 0x3f0cd06eu, 0x3f0717b3u}, +}; +constexpr std::uint32_t Expected[32][16] = { + {0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa3c00u, 0x000003c0u, 0x800003ffu, 0x388dbc05u, 0x00000000u, 0x00000000u, 0x007f8000u, 0x00048e01u}, + {0xaaaaaa80u, 0xaaaaaa40u, 0xaaaa8000u, 0xaaaa4000u, 0xaaaaaa80u, 0xaaaa4000u, 0xaaaaaa01u, 0xaaaa0001u, 0xaaaa3800u, 0x00000380u, 0x19e00200u, 0xac03367au, 0x00000001u, 0x00000000u, 0x00807f01u, 0x000f80cfu}, + {0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa00ffu, 0xaaaabc00u, 0x00000000u, 0xc0000000u, 0x3b05bad0u, 0x000000ffu, 0x00000000u, 0x007f807fu, 0x0007064du}, + {0xaaaaaa00u, 0xaaaaaac0u, 0xaaaa0000u, 0xaaaac000u, 0xaaaaaa00u, 0xaaaac000u, 0xaaaaaaffu, 0xaaaa0100u, 0xaaaab800u, 0x00000000u, 0x3b100000u, 0x28c83b63u, 0x00000100u, 0x00000000u, 0x007f7f7fu, 0x00004dd8u}, + {0xaaaaaa40u, 0xaaaaaa20u, 0xaaaa4000u, 0xaaaa2000u, 0xaaaaaa40u, 0xaaaa2000u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa3400u, 0x00000340u, 0x01000100u, 0xba5523cfu, 0x0000ffffu, 0x00000000u, 0x00807f7fu, 0x0009ac08u}, + {0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa4000u, 0x00000400u, 0x000003ffu, 0xb630bdf6u, 0x00010000u, 0x00000000u, 0x0080807fu, 0x000cea01u}, + {0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa5bf8u, 0x000005bfu, 0x106003ffu, 0xbd2b3418u, 0x7fffffffu, 0x00000000u, 0x00807f7fu, 0x00080483u}, + {0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa7bffu, 0x000007bfu, 0x386003ffu, 0xba6d3b0cu, 0x80000000u, 0x00000000u, 0x00807f80u, 0x000999c2u}, + {0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaaffu, 0xaaaaffffu, 0xaaaa8000u, 0x00000000u, 0x00000000u, 0xbd7dba6au, 0xffffffffu, 0x00000000u, 0x008080ffu, 0x00080666u}, + {0xaaaaaa01u, 0xaaaaaa00u, 0xaaaa0101u, 0xaaaa0080u, 0xaaaaaa01u, 0xaaaa0080u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa1c04u, 0x000001c0u, 0xbff00004u, 0x39ba3df8u, 0xffff8000u, 0x00000000u, 0x007f7f80u, 0x0005b9ffu}, + {0xaaaaaa01u, 0xaaaaaa00u, 0xaaaa0081u, 0xaaaa0040u, 0xaaaaaa01u, 0xaaaa0040u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa1804u, 0x00000180u, 0x7ff00002u, 0x35903c80u, 0x00007fffu, 0x00000000u, 0x007f7f7fu, 0x0002c9ffu}, + {0xaaaaaa02u, 0xaaaaaa01u, 0xaaaa0182u, 0xaaaa00c1u, 0xaaaaaa02u, 0xaaaa00c1u, 0xaaaaaaffu, 0xaaaaff80u, 0xaaaa1e06u, 0x000001e0u, 0x00000006u, 0xb81bbd97u, 0xffffff80u, 0x00000000u, 0x00808080u, 0x000bea01u}, + {0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaa80u, 0xaaaa0080u, 0xaaaa0000u, 0x00000000u, 0x00000000u, 0xbc9c9f84u, 0x00000080u, 0x00000000u, 0x0080807fu, 0x000807fcu}, + {0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa7c00u, 0x000007c0u, 0x3ff003ffu, 0xb6503d6cu, 0x00008000u, 0x00000000u, 0x00807f7fu, 0x000cd9ffu}, + {0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaafc00u, 0x00000000u, 0x2e400000u, 0xbcf039c8u, 0x27a1d402u, 0x00000000u, 0x00807f7fu, 0x00080571u}, + {0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa7e00u, 0x000007e0u, 0x3ff00000u, 0xb07f3d22u, 0xcf014130u, 0x00000000u, 0x00807f80u, 0x000ee1ffu}, + {0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa7c01u, 0x000007c1u, 0x2bd00000u, 0xb4bc397bu, 0x8d500f76u, 0x00000000u, 0x00807f80u, 0x000da55eu}, + {0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaafe09u, 0x000007e0u, 0x00000000u, 0xbd8dbb83u, 0x26ae54eeu, 0x00000000u, 0x0080807fu, 0x00080620u}, + {0xaaaaaa00u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaa0000u, 0xaaaaaa00u, 0xaaaa0000u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa0000u, 0x00000000u, 0x3d700000u, 0xbd1a3baeu, 0xddaaad70u, 0x00000000u, 0x00807f80u, 0x000805ebu}, + {0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaabcd7u, 0x00000000u, 0xfff00000u, 0x3cba3da0u, 0xac7dc96bu, 0x00000000u, 0x007f7f80u, 0x0007fdffu}, + {0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaa3c13u, 0x000003c1u, 0xfff003ffu, 0x3ddd3cc4u, 0xc34660fcu, 0x00000000u, 0x007f7f80u, 0x0007fdffu}, + {0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaabeddu, 0x00000000u, 0xc0000000u, 0x3bd9ac8bu, 0x8f643855u, 0x00000000u, 0x007f8080u, 0x0007d7dcu}, + {0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaabcc8u, 0x00000000u, 0x00000000u, 0x2ee0bc3fu, 0xec54b3b3u, 0x00000000u, 0x007f8080u, 0x0000de01u}, + {0xaaaaaa00u, 0xaaaaaa96u, 0xaaaa0000u, 0xaaaa9580u, 0xaaaaaa00u, 0xaaaa9580u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaabaa8u, 0x00000000u, 0x00000000u, 0xb57bb0acu, 0xf45aa8b6u, 0x00000000u, 0x00808080u, 0x000d47b5u}, + {0xaaaaaaffu, 0xaaaaaa7fu, 0xaaaaffffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa3feau, 0x000003feu, 0xc00fffffu, 0x3c02b99fu, 0x62f7c6f9u, 0x00000000u, 0x007f807fu, 0x0007fe99u}, + {0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaabd74u, 0x00000000u, 0x3ff00000u, 0xa2203de9u, 0xb74ba47du, 0x00000000u, 0x00807f80u, 0x000fe9ffu}, + {0xaaaaaa00u, 0xaaaaaafcu, 0xaaaa0000u, 0xaaaafbabu, 0xaaaaaa00u, 0xaaaafbabu, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaaa854u, 0x00000000u, 0x13100000u, 0x2c4134c5u, 0x8ce90a07u, 0x00000000u, 0x007f7f80u, 0x00008898u}, + {0xaaaaaa49u, 0xaaaaaa24u, 0xaaaa493cu, 0xaaaa249eu, 0xaaaaaa49u, 0xaaaa249eu, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaa3493u, 0x00000349u, 0x4adffd25u, 0x35e2316cu, 0x51927773u, 0x00000000u, 0x007f7f7fu, 0x0002f057u}, + {0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaabd1bu, 0x00000000u, 0xd8400000u, 0x3c003612u, 0x2b8b23a4u, 0x00000000u, 0x007f7f7fu, 0x0007fcc2u}, + {0xaaaaaa00u, 0xaaaaaaabu, 0xaaaa0000u, 0xaaaaaa2eu, 0xaaaaaa00u, 0xaaaaaa2eu, 0xaaaaaaffu, 0xaaaa8000u, 0xaaaab95du, 0x00000000u, 0x12a00000u, 0xb8a434aau, 0xc308bd98u, 0x00000000u, 0x00807f80u, 0x000b6095u}, + {0xaaaaaa00u, 0xaaaaaa81u, 0xaaaa0000u, 0xaaaa8001u, 0xaaaaaa00u, 0xaaaa8001u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaabe58u, 0x00000000u, 0x3ff00000u, 0xbbd23d13u, 0x10ce0a84u, 0x00000000u, 0x00807f7fu, 0x000831ffu}, + {0xaaaaaa00u, 0xaaaaaadcu, 0xaaaa0000u, 0xaaaadb81u, 0xaaaaaa00u, 0xaaaadb81u, 0xaaaaaaffu, 0xaaaa7fffu, 0xaaaab48fu, 0x00000000u, 0xa3300000u, 0x38383866u, 0x28ee3a6du, 0x00000000u, 0x007f7f7fu, 0x00043919u}, +}; +constexpr const char* Names[16] = {"8unorm", "8snorm", "16unorm", "16snorm", "8uscaled", "16sscaled", "8uint", "16sint", "16float", "uf11_x", "unorm1010102_xyzw", "f16x2_xy", "uint32x2_x", "uint32x2_y", "sint8888_xyz", "snorm1010102_xy"}; + +void Fill(std::uint32_t tid, std::uint32_t* words) { + std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words); +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) { + Require(actual == expected, std::string("formatted store: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* in = &Input[tid * Inputs]; + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t index = 0; index < 16; ++index) Expect(tid, out[index], Expected[tid][index], Names[index]); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("formatted store tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index 91d3a2cd7..c3864584d 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -107,6 +107,7 @@ bool IsSignedFormatComponent(SpirvFormatComponentType type); std::uint32_t EmitUFloatToF32Bits(SpirvEmitterState& state, std::uint32_t raw, std::uint32_t bits); std::uint32_t NormalizeFormatComponent(SpirvEmitterState& state, const SpirvBufferFormatInfo& info, std::uint32_t component, std::uint32_t raw); std::uint32_t EmitD16FormatComponent(SpirvEmitterState& state, const SpirvBufferFormatInfo& info, std::uint32_t component, std::uint32_t raw); +std::uint32_t EmitFormatStoreComponent(SpirvEmitterState& state, const SpirvBufferFormatInfo& info, std::uint32_t component, std::uint32_t data); void EmitDeviceAtomicMemoryBarrier(SpirvEmitterState& state); std::uint32_t EmitFloatAtomicReplacement(SpirvEmitterState& state, std::uint32_t old, std::uint32_t source, bool maxValue); std::uint32_t EmitDsSwizzleTargetLane(SpirvEmitterState& state, std::uint32_t subid, std::uint32_t control); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvFormatConvert.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvFormatConvert.cpp index 1937ade6d..f370c8a4c 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvFormatConvert.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvFormatConvert.cpp @@ -164,28 +164,74 @@ std::uint32_t IntegerToF16Bits(SpirvEmitterState& state, std::uint32_t value) { return EmitSelectValueU32(state, zero, ConstantU32(state, 0u), Ext(state, TypeU32(state), GLSLstd450UMin, bits, ConstantU32(state, 0x7bffu))); } -std::uint32_t F32BitsToF16BitsRtz(SpirvEmitterState& state, std::uint32_t bits) { - const auto sign = EmitAndConstant(state, EmitBinaryU32(state, spv::OpShiftRightLogical, bits, ConstantU32(state, 16u)), 0x8000u); +std::uint32_t F32BitsToSmallFloatRtz(SpirvEmitterState& state, std::uint32_t bits, std::uint32_t mantissaBits, bool signedFormat) { + const std::uint32_t infinity = 0x1fu << mantissaBits; + const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, bits, 0x80000000u), 0u); + const auto sign = signedFormat ? EmitSelectValueU32(state, negative, ConstantU32(state, 1u << (5u + mantissaBits)), ConstantU32(state, 0u)) : ConstantU32(state, 0u); const auto exponent = EmitAndConstant(state, EmitBinaryU32(state, spv::OpShiftRightLogical, bits, ConstantU32(state, 23u)), 0xffu); const auto mantissa = EmitAndConstant(state, bits, 0x7fffffu); - const auto top = EmitBinaryU32(state, spv::OpShiftRightLogical, mantissa, ConstantU32(state, 13u)); + const auto top = EmitBinaryU32(state, spv::OpShiftRightLogical, mantissa, ConstantU32(state, 23u - mantissaBits)); const auto lostPayload = EmitLogicalAndBool(state, EmitCompareU32Constant(state, spv::OpIEqual, top, 0u), EmitCompareU32Constant(state, spv::OpINotEqual, mantissa, 0u)); - const auto special = EmitOrU32(state, ConstantU32(state, 0x7c00u), EmitSelectValueU32(state, lostPayload, ConstantU32(state, 1u), top)); + const auto special = EmitOrU32(state, ConstantU32(state, infinity), EmitSelectValueU32(state, lostPayload, ConstantU32(state, 1u), top)); const auto biased = Binary(state, spv::OpISub, TypeI32(state), EmitTBufferBitcastU32ToI32(state, exponent), EmitTBufferBitcastU32ToI32(state, ConstantU32(state, 112u))); const auto atLeast = [&](std::uint32_t limit) { return Binary(state, spv::OpSGreaterThanEqual, TypeBool(state), biased, EmitTBufferBitcastU32ToI32(state, ConstantU32(state, limit))); }; const auto biasedU = Unary(state, spv::OpBitcast, TypeU32(state), biased); - const auto normal = EmitOrU32(state, EmitBinaryU32(state, spv::OpShiftLeftLogical, biasedU, ConstantU32(state, 10u)), top); - const auto subnormalShift = EmitBinaryU32(state, spv::OpISub, ConstantU32(state, 14u), biasedU); + const auto normal = EmitOrU32(state, EmitBinaryU32(state, spv::OpShiftLeftLogical, biasedU, ConstantU32(state, mantissaBits)), top); + const auto subnormalShift = EmitBinaryU32(state, spv::OpISub, ConstantU32(state, 24u - mantissaBits), biasedU); const auto subnormal = EmitBinaryU32(state, spv::OpShiftRightLogical, EmitOrU32(state, mantissa, ConstantU32(state, 0x800000u)), Ext(state, TypeU32(state), GLSLstd450UMin, subnormalShift, ConstantU32(state, 31u))); - auto magnitude = EmitSelectValueU32(state, atLeast(static_cast(-10)), subnormal, ConstantU32(state, 0u)); + auto magnitude = EmitSelectValueU32(state, atLeast(static_cast(-static_cast(mantissaBits))), subnormal, ConstantU32(state, 0u)); magnitude = EmitSelectValueU32(state, atLeast(1u), normal, magnitude); - magnitude = EmitSelectValueU32(state, atLeast(31u), ConstantU32(state, 0x7bffu), magnitude); - magnitude = EmitSelectValueU32(state, EmitCompareU32Constant(state, spv::OpIEqual, exponent, 0xffu), special, magnitude); + magnitude = EmitSelectValueU32(state, atLeast(31u), ConstantU32(state, infinity - 1u), magnitude); + const auto isSpecial = EmitCompareU32Constant(state, spv::OpIEqual, exponent, 0xffu); + magnitude = EmitSelectValueU32(state, isSpecial, special, magnitude); + if (!signedFormat) { + const auto nan = EmitLogicalAndBool(state, isSpecial, EmitCompareU32Constant(state, spv::OpINotEqual, mantissa, 0u)); + magnitude = EmitSelectValueU32(state, EmitLogicalAndBool(state, negative, EmitLogicalNotBool(state, nan)), ConstantU32(state, 0u), magnitude); + } return EmitOrU32(state, sign, magnitude); } +std::uint32_t NormStoreBits(SpirvEmitterState& state, std::uint32_t bits, std::uint32_t maximum, bool signedFormat) { + const auto value = EmitBitcastU32ToF32(state, bits); + const auto magnitudeBits = EmitAndConstant(state, bits, 0x7fffffffu); + const auto magnitude = EmitBitcastU32ToF32(state, magnitudeBits); + const auto exponent = EmitAndConstant(state, EmitBinaryU32(state, spv::OpShiftRightLogical, bits, ConstantU32(state, 23u)), 0xffu); + const auto denormal = EmitCompareU32Constant(state, spv::OpIEqual, exponent, 0u); + const auto mantissa = EmitOrU32(state, EmitAndConstant(state, bits, 0x7fffffu), EmitSelectValueU32(state, denormal, ConstantU32(state, 0u), ConstantU32(state, 0x800000u))); + const auto shift = EmitSelectValueU32(state, denormal, ConstantU32(state, 149u), EmitBinaryU32(state, spv::OpISub, ConstantU32(state, 150u), exponent)); + const auto extra = EmitBinaryU32(state, spv::OpISub, shift, ConstantU32(state, 24u)); + const auto product = state.module.AllocateId(); + state.module.AddFunction(spv::OpUMulExtended, state.module.Type(spv::OpTypeStruct, TypeU32(state), TypeU32(state)), product, mantissa, ConstantU32(state, maximum)); + const auto low = state.module.AllocateId(); + const auto high = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, product, 0u); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), high, product, 1u); + const auto integer = EmitOrU32(state, EmitBinaryU32(state, spv::OpShiftRightLogical, low, ConstantU32(state, 24u)), EmitBinaryU32(state, spv::OpShiftLeftLogical, high, ConstantU32(state, 8u))); + const auto fraction = EmitAndConstant(state, low, 0xffffffu); + const auto fixed = EmitOrU32(state, EmitBinaryU32(state, spv::OpShiftLeftLogical, integer, ConstantU32(state, 16u)), EmitBinaryU32(state, spv::OpShiftRightLogical, fraction, ConstantU32(state, 8u))); + const auto sticky = EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, fraction, 0xffu), 0u); + const auto safeExtra = Ext(state, TypeU32(state), GLSLstd450UMin, extra, ConstantU32(state, 16u)); + const auto quotient = EmitBinaryU32(state, spv::OpShiftRightLogical, EmitBinaryU32(state, spv::OpShiftRightLogical, fixed, ConstantU32(state, 16u)), safeExtra); + const auto remainder = EmitBinaryU32(state, spv::OpISub, fixed, EmitBinaryU32(state, spv::OpShiftLeftLogical, EmitBinaryU32(state, spv::OpShiftLeftLogical, quotient, ConstantU32(state, 16u)), safeExtra)); + const auto half = EmitBinaryU32(state, spv::OpShiftLeftLogical, ConstantU32(state, 1u << 15u), safeExtra); + const auto odd = EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, quotient, 1u), 0u); + const auto tie = EmitLogicalAndBool(state, Binary(state, spv::OpIEqual, TypeBool(state), remainder, half), EmitLogicalOrBool(state, sticky, odd)); + const auto up = EmitLogicalOrBool(state, UnsignedGreater(state, remainder, half), tie); + const auto rounded = EmitAddU32(state, quotient, EmitSelectValueU32(state, up, ConstantU32(state, 1u), ConstantU32(state, 0u))); + auto result = EmitSelectValueU32(state, UnsignedGreater(state, extra, ConstantU32(state, 16u)), ConstantU32(state, 0u), rounded); + const auto full = Binary(state, spv::OpFOrdGreaterThanEqual, TypeBool(state), magnitude, ConstantF32Value(state, 1.0f)); + result = EmitSelectValueU32(state, full, ConstantU32(state, maximum), result); + const auto positive = Binary(state, spv::OpFOrdGreaterThan, TypeBool(state), signedFormat ? magnitude : value, ConstantF32Value(state, 0.0f)); + result = EmitSelectValueU32(state, positive, result, ConstantU32(state, 0u)); + if (!signedFormat) { + return result; + } + const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, bits, 0x80000000u), 0u); + return EmitSelectValueU32(state, negative, Unary(state, spv::OpSNegate, TypeU32(state), result), result); +} + } std::uint32_t EmitD16FormatComponent(SpirvEmitterState& state, const SpirvBufferFormatInfo& info, std::uint32_t component, std::uint32_t raw) { @@ -219,7 +265,36 @@ std::uint32_t EmitD16FormatComponent(SpirvEmitterState& state, const SpirvBuffer } case SpirvFormatComponentType::Float: if (bits == 16u) return EmitAndConstant(state, raw, 0xffffu); - return F32BitsToF16BitsRtz(state, NormalizeFormatComponent(state, info, component, raw)); + return F32BitsToSmallFloatRtz(state, NormalizeFormatComponent(state, info, component, raw), 10u, true); + default: + FailEmit("buffer format component type is not supported"); + } +} + +std::uint32_t EmitFormatStoreComponent(SpirvEmitterState& state, const SpirvBufferFormatInfo& info, std::uint32_t component, std::uint32_t data) { + const auto bits = info.componentBits[component]; + const std::uint32_t mask = bits >= 32u ? 0xffffffffu : (1u << bits) - 1u; + switch (info.type) { + case SpirvFormatComponentType::Uint: + return bits >= 32u ? data : Ext(state, TypeU32(state), GLSLstd450UMin, data, ConstantU32(state, mask)); + case SpirvFormatComponentType::Sint: { + if (bits >= 32u) return data; + const auto low = EmitTBufferBitcastU32ToI32(state, ConstantU32(state, ~(mask >> 1u))); + const auto high = EmitTBufferBitcastU32ToI32(state, ConstantU32(state, mask >> 1u)); + const auto clamped = Ext(state, TypeI32(state), GLSLstd450SMin, Ext(state, TypeI32(state), GLSLstd450SMax, EmitTBufferBitcastU32ToI32(state, data), low), high); + return EmitAndConstant(state, Unary(state, spv::OpBitcast, TypeU32(state), clamped), mask); + } + case SpirvFormatComponentType::Unorm: + case SpirvFormatComponentType::Uscaled: + return NormStoreBits(state, data, mask, false); + case SpirvFormatComponentType::Snorm: + case SpirvFormatComponentType::Sscaled: + return EmitAndConstant(state, NormStoreBits(state, data, mask >> 1u, true), mask); + case SpirvFormatComponentType::Float: + if (bits == 32u) return data; + if (bits == 16u) return F32BitsToSmallFloatRtz(state, data, 10u, true); + if (bits == 11u || bits == 10u) return F32BitsToSmallFloatRtz(state, data, bits - 5u, false); + FailEmit("buffer float format component width is not supported for stores"); default: FailEmit("buffer format component type is not supported"); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index f9e2031ae..688e5491f 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -434,31 +434,6 @@ void StoreWord(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo }); } -void FormattedStorePrepared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t component, const MemoryResourceAccess& resource, std::uint32_t data) { - const auto info = MemoryFormatInfo(ctx.state, mem); - if (info.type == SpirvFormatComponentType::Unknown) { - StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource, data); - return; - } - if (component >= info.componentCount) { - return; - } - const auto bits = info.componentBits[component]; - const auto componentMem = RebaseFormattedComponent(mem, info, component); - if (bits == 8u || bits == 16u) { - StoreSubwordPrepared(ctx, inst, componentMem, resource, bits, data); - } else { - StoreWordPrepared(ctx, inst, componentMem, resource, data); - } -} - -void FormattedStore(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { - EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - FormattedStorePrepared(ctx, inst, mem, 0u, resource, ctx.Arg(inst, inst.ArgumentCount() - 2u)); - }); -} - struct PreparedFormattedMemory { SpirvBufferFormatInfo info; MemoryResourceAccess resource; @@ -486,7 +461,7 @@ PreparedFormattedMemory PrepareFormattedMemory(SpirvValueEmitContext& ctx, const } } } else { - for (std::uint32_t component = 0; component < std::min(components, plan.info.componentCount); component++) { + for (std::uint32_t component = 0; component < plan.info.componentCount; component++) { required.at(component) = true; } } @@ -552,18 +527,52 @@ std::uint32_t FormattedOutOfBoundsValue(SpirvValueEmitContext& ctx, const Memory return ConstructU32Composite(state, components, values); } -void StoreFormattedInBounds(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, std::uint32_t component, std::uint32_t data) { - if (component >= plan.info.componentCount) { +void StoreFormattedElement(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, const std::array& data, std::uint32_t components) { + auto& state = ctx.state; + const auto& info = plan.info; + std::array encoded{}; + for (std::uint32_t component = 0; component < info.componentCount; component++) { + encoded.at(component) = component < components ? EmitFormatStoreComponent(state, info, component, data.at(component)) : ConstantU32(state, 0u); + } + if (info.packedBitfield) { + auto word = ConstantU32(state, 0u); + for (std::uint32_t component = 0; component < info.componentCount; component++) { + const std::uint32_t mask = (1u << info.componentBits[component]) - 1u; + word = EmitOrU32(state, word, EmitBinaryU32(state, spv::OpShiftLeftLogical, EmitAndConstant(state, encoded.at(component), mask), ConstantU32(state, info.componentBitOffset[component]))); + } + StoreWordInBounds(ctx, plan.resource, plan.indices.at(0), word); return; } - const auto bits = plan.info.componentBits[component]; - if (bits == 8u || bits == 16u) { - StoreSubwordInBounds(ctx, mem, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, data); - } else { - StoreWordInBounds(ctx, plan.resource, plan.indices.at(component), data); + for (std::uint32_t component = 0; component < info.componentCount; component++) { + const auto bits = info.componentBits[component]; + if (bits == 8u || bits == 16u) { + StoreSubwordInBounds(ctx, mem, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, encoded.at(component)); + } else { + StoreWordInBounds(ctx, plan.resource, plan.indices.at(component), encoded.at(component)); + } } } +void StoreFormattedValues(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource, const SpirvBufferFormatInfo& info, const std::array& data, std::uint32_t components) { + const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, FormattedAccess::Store); + EmitIfCondition(ctx.state, plan.inBounds, [&]() { + StoreFormattedElement(ctx, mem, plan, data, components); + }); +} + +void FormattedStore(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { + EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() { + const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); + const auto info = MemoryFormatInfo(ctx.state, mem); + const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u); + if (info.type == SpirvFormatComponentType::Unknown) { + StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, 0u), resource, data); + return; + } + StoreFormattedValues(ctx, inst, mem, resource, info, {data, 0u, 0u, 0u}, 1u); + }); +} + std::uint32_t LoadWideBuffer(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) { auto& state = ctx.state; return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() { @@ -594,14 +603,12 @@ void StoreWideBuffer(SpirvValueEmitContext& ctx, const IrValue& inst, const Memo const auto composite = ctx.Arg(inst, inst.ArgumentCount() - 2u); const auto info = MemoryFormatInfo(state, mem); if (info.type != SpirvFormatComponentType::Unknown) { - const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, FormattedAccess::Store); - EmitIfCondition(state, plan.inBounds, [&]() { - for (std::uint32_t component = 0; component < components; component++) { - const auto data = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), data, composite, component); - StoreFormattedInBounds(ctx, mem, plan, component, data); - } - }); + std::array data{}; + for (std::uint32_t component = 0; component < components; component++) { + data.at(component) = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), data.at(component), composite, component); + } + StoreFormattedValues(ctx, inst, mem, resource, info, data, components); return; } for (std::uint32_t component = 0; component < components; component++) {