From 6de064f6d318ebce342e09661e07fcf837730320 Mon Sep 17 00:00:00 2001 From: Jordan Hans Date: Fri, 9 Oct 2026 04:18:22 +0000 Subject: [PATCH] fix(shader): round the f16 fused multiply-adds once v_fma_f16, v_fmac_f16, v_fmamk_f16, v_fmaak_f16, v_pk_fma_f16 and v_pk_fmac_f16 rounded an f32 fused multiply-add to f16, rounding twice. The product of two f16 values is exact in f32, so the sum is now rounded to odd in f32 with an exact error term and then rounded to nearest even in f16, which rounds the exact result once. --- .../tests/execution/Float16FmaRounding.cpp | 232 ++++++++++++++++++ .../Translation/TranslationContext.hpp | 1 + .../Translation/src/FloatInstructions.cpp | 29 ++- docs/dev/TechnicalDebt.md | 1 - 4 files changed, 258 insertions(+), 5 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/Float16FmaRounding.cpp diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16FmaRounding.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16FmaRounding.cpp new file mode 100644 index 000000000..424059a6c --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16FmaRounding.cpp @@ -0,0 +1,232 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 64; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 8; +constexpr std::uint32_t Checks = 7; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084, 0x34060085, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xd74b000a, 0x041a0b04, 0xd74b380b, + 0x041a0b04, 0x7e180306, 0x6c180b04, 0xcc0e400d, 0x1c1a0b04, 0x7e1c0306, 0x781c0b04, 0x701e0f04, + 0x00000001, 0x6e200f04, 0x00003d55, 0x7e220280, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, + 0xbf810000, +}; + +constexpr std::uint32_t Rows[Threads][Inputs] = { + {0x800062e9u, 0x3c00d159u, 0x800023ecu, 0x0000f9a3u}, + {0x0000d4e0u, 0xbc005ae0u, 0x0000881fu, 0x00001e01u}, + {0x4200c1b4u, 0x4400f580u, 0xca008544u, 0x00003700u}, + {0x7c005af7u, 0x4000bde3u, 0x3c00772cu, 0x000027e9u}, + {0x3c0053e4u, 0x3c005900u, 0xfc000955u, 0x00002500u}, + {0x5bff3464u, 0x5bff5542u, 0x3c0012ffu, 0x00006d19u}, + {0x3c00c6d2u, 0x3c01064eu, 0x800129d8u, 0x00003200u}, + {0x1400e568u, 0x14004b80u, 0x00000891u, 0x000069d9u}, + {0xba18f442u, 0x70c0b900u, 0x88810546u, 0x00000700u}, + {0xbba84092u, 0x6a804788u, 0x814512ffu, 0x00008ba1u}, + {0xbc2cd70cu, 0x6d3ad700u, 0xa7f90ee0u, 0x00002100u}, + {0xbf00562cu, 0x72cc3774u, 0x8497ee8cu, 0x0000c716u}, + {0x56b4e294u, 0xdf004f00u, 0x15fb0683u, 0x00001700u}, + {0x1d0dd0e0u, 0x0a56cea0u, 0x1828837fu, 0x00001a01u}, + {0x5a40f006u, 0xcad01f55u, 0x80e6ea0eu, 0x00000900u}, + {0xcb3cc300u, 0x9615e4bcu, 0xc4a80843u, 0x0000d0fbu}, + {0x9e945d6eu, 0x95793cf9u, 0x998179b2u, 0x00001a00u}, + {0x603451cdu, 0xcf80d341u, 0x022c2276u, 0x00009041u}, + {0x06fbc6d0u, 0xf697ed40u, 0x5ea10c65u, 0x000030c0u}, + {0xd416668bu, 0x0a5d0272u, 0xc2d759b3u, 0x000032c9u}, + {0x6836db60u, 0x4900c9e0u, 0x806a02f8u, 0x00001ca0u}, + {0x02637ad4u, 0xe2b43100u, 0xd7ad0608u, 0x00002007u}, + {0xb4544d00u, 0x70806a2eu, 0x84828f49u, 0x00002a74u}, + {0x560003e6u, 0xd8453e28u, 0x0245af54u, 0x00005100u}, + {0x9b9bd580u, 0x08356004u, 0x14eb1222u, 0x000021e8u}, + {0x9aec64f8u, 0x67e64440u, 0x01018892u, 0x0000ad81u}, + {0x4366c00eu, 0x6e00eddcu, 0x0c16abffu, 0x00003900u}, + {0x002142a6u, 0xc7c266dbu, 0x28409b9du, 0x00000f11u}, + {0x45e74af4u, 0x015b6900u, 0xb5681505u, 0x00002d00u}, + {0x18eb7a00u, 0x6bc32ce7u, 0x804202f8u, 0x00008001u}, + {0xe990f8f2u, 0xc1201b57u, 0x06f71bc1u, 0x00008280u}, + {0xd9001c4du, 0xddeaf293u, 0x165316e1u, 0x00000e2bu}, + {0x532aee50u, 0x4e7c41c0u, 0xa97e11aau, 0x000009c0u}, + {0x066e7100u, 0xf4fa45aeu, 0xeee58a84u, 0x00003274u}, + {0x4dabed80u, 0x143c493cu, 0xcc6d8b1au, 0x000009e8u}, + {0xbd10d940u, 0x6d205d18u, 0x87670b12u, 0x00002001u}, + {0x68704f00u, 0x4e60e80au, 0x11f601a7u, 0x0000289cu}, + {0x5c805cd0u, 0xd9d45820u, 0x977a112bu, 0x00008357u}, + {0xf3003d96u, 0x34946e00u, 0x07660459u, 0x00003a00u}, + {0x363c73c0u, 0xa5223f70u, 0xb7e30d44u, 0x0000083au}, + {0x73de9e78u, 0xbe00262fu, 0x8872ac8eu, 0x00005a80u}, + {0xe6d7ccfeu, 0xb8f8e900u, 0xa88286b3u, 0x00001561u}, + {0x2824d550u, 0xf1805ac0u, 0x00860ec2u, 0x000022c0u}, + {0x36442fcau, 0x770019c7u, 0x8c9f254au, 0x00005fd8u}, + {0x6e192033u, 0xa3ae06abu, 0x84228eadu, 0x00005a00u}, + {0xad87d9b0u, 0x51cad8e0u, 0xf3f304cbu, 0x0000f6e3u}, + {0x39da80abu, 0xbad699fdu, 0xdc3c807du, 0x000012a2u}, + {0x199a8735u, 0x724992a9u, 0x9b4f82d6u, 0x0000a863u}, + {0xe56f0101u, 0x22719dfau, 0xe53e820au, 0x00000e86u}, + {0x66400f97u, 0x4c280a53u, 0x14008394u, 0x00004c5cu}, + {0x54409989u, 0xce7880b9u, 0x0151825eu, 0x0000066eu}, + {0x54b00f7cu, 0x506091e1u, 0x00a08146u, 0x00008201u}, + {0x50d689e7u, 0x4237096cu, 0x0dfa831eu, 0x00006495u}, + {0x04288b6fu, 0x0fb3084eu, 0x01e9037fu, 0x0000a90cu}, + {0xa1eb1a63u, 0x853d01e1u, 0x015a03b9u, 0x00003ea2u}, + {0x8af80d78u, 0x98e105dau, 0x02ef8286u, 0x0000294bu}, + {0x216c08b8u, 0x86d30ac8u, 0x8336022fu, 0x00006f80u}, + {0x87f41c46u, 0x0c0688eau, 0x82e782efu, 0x0000c6b4u}, + {0x0a215be0u, 0x183e5c10u, 0x83618001u, 0x00001c60u}, + {0x03dcdbe0u, 0x96385c10u, 0x02650001u, 0x000092e8u}, + {0x13496ff2u, 0x92dd5328u, 0x0270cb02u, 0x00008a00u}, + {0x8b276d1au, 0x9f695348u, 0x8397f1e6u, 0x000048c4u}, + {0x9560ee89u, 0x817d02f3u, 0x0331d633u, 0x0000de07u}, + {0x0fb3d541u, 0x0a3c089du, 0x822a503fu, 0x00003c3du} +}; +constexpr std::uint32_t Expected[Threads][Checks] = { + {0xf89fu, 0x8000u, 0xf89fu, 0x8000f89fu, 0x8000f89fu, 0xfc00u, 0xf97eu}, + {0xf431u, 0x0000u, 0xf431u, 0x0000f431u, 0x0000f431u, 0xb751u, 0xd67fu}, + {0x7bd7u, 0x0000u, 0x7bd7u, 0x00007bd7u, 0x00007bd7u, 0xbcfdu, 0xc2bau}, + {0x7717u, 0x7c00u, 0x7717u, 0x7c007717u, 0x7c007717u, 0x46e3u, 0x5ca5u}, + {0x70efu, 0xfc00u, 0x70efu, 0xfc0070efu, 0xfc0070efu, 0x3cefu, 0x5543u}, + {0x4dc5u, 0x7bfeu, 0x4dc5u, 0x7bfe4dc5u, 0x7bfe4dc5u, 0x6598u, 0x6d19u}, + {0x29c3u, 0x3c01u, 0x29c3u, 0x3c0129c3u, 0x3c0129c3u, 0xbd1du, 0xc874u}, + {0xf511u, 0x0010u, 0xf511u, 0x0010f511u, 0x0010f511u, 0xfc00u, 0x647du}, + {0x7153u, 0xef3du, 0x7153u, 0xef3d7153u, 0xef3d7153u, 0xbf73u, 0xf5adu}, + {0x4c4du, 0xea39u, 0x4c4du, 0xea394c4du, 0xea394c4du, 0x905cu, 0x4217u}, + {0x722bu, 0xed73u, 0x722bu, 0xed73722bu, 0xed73722bu, 0xbc67u, 0xd8b2u}, + {0xee81u, 0xf5f3u, 0xee81u, 0xf5f3ee81u, 0xf5f3ee81u, 0xe177u, 0x57c9u}, + {0xf5c1u, 0xf9ddu, 0xf5c1u, 0xf9ddf5c1u, 0xf9ddf5c1u, 0xbdc1u, 0xe462u}, + {0x6409u, 0x1829u, 0x6409u, 0x18296409u, 0x18296409u, 0xaf51u, 0xd27fu}, + {0xea2bu, 0xe953u, 0xea2bu, 0xe953ea2bu, 0xe953ea2bu, 0xbd07u, 0xf15du}, + {0x6c25u, 0xc4a3u, 0x6c25u, 0xc4a36c25u, 0xc4a36c25u, 0x585cu, 0xd190u}, + {0x79bfu, 0x997du, 0x79bfu, 0x997d79bfu, 0x997d79bfu, 0x3c13u, 0x5f3du}, + {0xe943u, 0xf3e1u, 0xe943u, 0xf3e1e943u, 0xf3e1e943u, 0xa62bu, 0x53bbu}, + {0x7879u, 0x5e95u, 0x7879u, 0x5e957879u, 0x5e957879u, 0xbc0bu, 0xc877u}, + {0x59b3u, 0xc2ddu, 0x59b3u, 0xc2dd59b3u, 0xc2dd59b3u, 0x5d8du, 0x685du}, + {0x696bu, 0x7543u, 0x696bu, 0x7543696bu, 0x7543696bu, 0xbc43u, 0xdceau}, + {0x7045u, 0xd7adu, 0x7045u, 0xd7ad7045u, 0xd7ad7045u, 0x5ee0u, 0x7c00u}, + {0x7bb9u, 0xe8dfu, 0x7bb9u, 0xe8df7bb9u, 0xe8df7bb9u, 0x3c09u, 0x4eadu}, + {0xaf53u, 0xf267u, 0xaf53u, 0xf267af53u, 0xf267af53u, 0x18e0u, 0x5100u}, + {0xf985u, 0x14ebu, 0xf985u, 0x14ebf985u, 0x14ebf985u, 0xbc0fu, 0xd755u}, + {0x6d47u, 0xc6d5u, 0x6d47u, 0xc6d56d47u, 0xc6d56d47u, 0xd6d6u, 0x669fu}, + {0x71f1u, 0x758du, 0x71f1u, 0x758d71f1u, 0x758d71f1u, 0xbd11u, 0xc028u}, + {0x6db3u, 0x283fu, 0x6db3u, 0x283f6db3u, 0x283f6db3u, 0x15dfu, 0x446fu}, + {0x7859u, 0xb567u, 0x7859u, 0xb5677859u, 0xb5677859u, 0x3c59u, 0x4ca7u}, + {0x6b5bu, 0x48c5u, 0x6b5bu, 0x48c56b5bu, 0x48c56b5bu, 0x9a00u, 0x7bffu}, + {0xd889u, 0x6f21u, 0xd889u, 0x6f21d889u, 0x6f21d889u, 0x3e2fu, 0xfa98u}, + {0xd311u, 0x7b65u, 0xd311u, 0x7b65d311u, 0x7b65d311u, 0x001cu, 0x1e1eu}, + {0xf489u, 0x65cfu, 0xf489u, 0x65cff489u, 0x65cff489u, 0xbc89u, 0xf035u}, + {0x7b19u, 0xeee5u, 0x7b19u, 0xeee57b19u, 0xeee57b19u, 0x6809u, 0x72aau}, + {0xfb33u, 0xcc6bu, 0xfb33u, 0xcc6bfb33u, 0xcc6bfb33u, 0xbc0fu, 0xef55u}, + {0xfaafu, 0xee7du, 0xfaafu, 0xee7dfaafu, 0xee7dfaafu, 0xbd41u, 0xdaffu}, + {0xfb11u, 0x7b13u, 0xfb11u, 0x7b13fb11u, 0x7b13fb11u, 0x3c09u, 0x50acu}, + {0x78f7u, 0xfa8fu, 0x78f7u, 0xfa8f78f7u, 0xfa8f78f7u, 0xa405u, 0x5e6au}, + {0x7031u, 0xec01u, 0x7031u, 0xec017031u, 0xec017031u, 0x3c31u, 0x4139u}, + {0x7735u, 0xb801u, 0x7735u, 0xb8017735u, 0xb8017735u, 0x4018u, 0x752au}, + {0xac91u, 0xf5e7u, 0xac91u, 0xf5e7ac91u, 0xf5e7ac91u, 0xbd41u, 0x5a80u}, + {0x7a3du, 0x643fu, 0x7a3du, 0x643f7a3du, 0x643f7a3du, 0xa6b7u, 0xcea7u}, + {0xf47bu, 0xddb1u, 0xf47bu, 0xddb1f47bu, 0xddb1f47bu, 0xbc7bu, 0xd715u}, + {0x2561u, 0x717bu, 0x2561u, 0x717b2561u, 0x717b2561u, 0x53a3u, 0x5fd9u}, + {0x8ea9u, 0xd5dbu, 0x8ea9u, 0xd5db8ea9u, 0xd5db8ea9u, 0x3e4du, 0x5a00u}, + {0x76efu, 0xf3f3u, 0x76efu, 0xf3f376efu, 0xf3f376efu, 0x7c00u, 0xf6f2u}, + {0x807du, 0xdc3fu, 0x807du, 0xdc3f807du, 0xdc3f807du, 0x0001u, 0x1286u}, + {0x82d5u, 0x5067u, 0x82d5u, 0x506782d5u, 0x506782d5u, 0x0040u, 0xa868u}, + {0x820bu, 0xe54fu, 0x820bu, 0xe54f820bu, 0xe54f820bu, 0x0001u, 0x0edcu}, + {0x8393u, 0x767fu, 0x8393u, 0x767f8393u, 0x767f8393u, 0x2023u, 0x4c5cu}, + {0x825du, 0xe6dfu, 0x825du, 0xe6df825du, 0xe6df825du, 0x8003u, 0x9b2du}, + {0x814bu, 0x6921u, 0x814bu, 0x6921814bu, 0x6921814bu, 0x0001u, 0x10bdu}, + {0x831fu, 0x5783u, 0x831fu, 0x5783831fu, 0x5783831fu, 0xb2c3u, 0x6495u}, + {0x037fu, 0x01e9u, 0x037fu, 0x01e9037fu, 0x01e9037fu, 0x0097u, 0xa916u}, + {0x03bbu, 0x0169u, 0x03bbu, 0x016903bbu, 0x016903bbu, 0x1d4cu, 0x3ea6u}, + {0x8285u, 0x02f7u, 0x8285u, 0x02f78285u, 0x02f78285u, 0x00e9u, 0x295au}, + {0x022fu, 0x8349u, 0x022fu, 0x8349022fu, 0x8349022fu, 0x3c6du, 0x6f80u}, + {0x82f9u, 0x82e7u, 0x82f9u, 0x82e782f9u, 0x82e782f9u, 0xa729u, 0xc6b3u}, + {0x7bffu, 0x835bu, 0x7bffu, 0x835b7bffu, 0x835b7bffu, 0x3c4fu, 0x5d40u}, + {0xfbffu, 0x0263u, 0xfbffu, 0x0263fbffu, 0x0263fbffu, 0x32ccu, 0xdd40u}, + {0x7c00u, 0x0263u, 0x7c00u, 0x02637c00u, 0x02637c00u, 0xbdf5u, 0x714cu}, + {0x7c00u, 0x837du, 0x7c00u, 0x837d7c00u, 0x837d7c00u, 0x7a14u, 0x6ecfu}, + {0xd638u, 0x0331u, 0xd638u, 0x0331d638u, 0x0331d638u, 0x7c00u, 0xf08bu}, + {0x503fu, 0x8229u, 0x503fu, 0x8229503fu, 0x8229503fu, 0xd591u, 0xd6f0u} +}; +constexpr std::uint32_t Masks[Checks] = {0xffffu, 0xffffu, 0xffffu, 0xffffffffu, 0xffffffffu, 0xffffu, 0xffffu}; +constexpr const char* Names[Checks] = { + "v_fma_f16 v10, v4, v5, v6", + "v_fma_f16 v11, v4, v5, v6 op_sel:[1,1,1,0]", + "v_fmac_f16 v12, v4, v5", + "v_pk_fma_f16 v13, v4, v5, v6", + "v_pk_fmac_f16 v14, v4, v5", + "v_fmaak_f16 v15, v4, v7, 0x0001", + "v_fmamk_f16 v16, v4, 0x3d55, v7", +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t index = 0; index < Checks; ++index) { + const std::uint32_t actual = Output[tid * Results + index] & Masks[index]; + Require(actual == Expected[tid][index], std::string("f16 fma rounding: lane ") + std::to_string(tid) + " " + Names[index] + " is " + Hex(actual) + ", expected " + Hex(Expected[tid][index])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("f16 fma rounding tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index bf5a58d69..0543afc24 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -177,6 +177,7 @@ private: bool vDivFixupF16(const RdnaInstruction& inst); bool float16Binary(const RdnaInstruction& inst, IrOpcode opcode, bool reverse); bool float16Ternary(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool mix); + IrF32 fmaF16RoundedToOdd(IrF32 lhs, IrF32 rhs, IrF32 addend); bool floatUnary(const RdnaInstruction& inst, IrOpcode opcode); bool floatBinary(const RdnaInstruction& inst, IrOpcode opcode, bool reverse); bool floatTernary(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool mix); diff --git a/core/shader/recompiler/Translation/src/FloatInstructions.cpp b/core/shader/recompiler/Translation/src/FloatInstructions.cpp index 502e601a3..52c7607ab 100644 --- a/core/shader/recompiler/Translation/src/FloatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/FloatInstructions.cpp @@ -17,13 +17,15 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc const auto translateLane = [&](bool high) -> IrF32 { const IrF32 lhs = readF16LaneAsF32(sourceAt(inst, 0u), high, true); const IrF32 rhs = readF16LaneAsF32(sourceAt(inst, 1u), high, true); + const auto ternary = [&](const IrF32& third) -> IrF32 { + if (opcode == IrOpcode::FPFma32) return fmaF16RoundedToOdd(lhs, rhs, third); + return IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &third.Value()})); + }; if (accumulator) { - const IrF32 acc = readF16LaneAsF32(accumulatorOperand(inst), high, true); - return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &acc.Value()}))); + return applyF32ResultModifiers(inst.destination, ternary(readF16LaneAsF32(accumulatorOperand(inst), high, true))); } if (inst.sourceCount == 3u) { - const IrF32 third = readF16LaneAsF32(sourceAt(inst, 2u), high, true); - return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &third.Value()}))); + return applyF32ResultModifiers(inst.destination, ternary(readF16LaneAsF32(sourceAt(inst, 2u), high, true))); } return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value()}))); }; @@ -213,10 +215,29 @@ bool TranslationContext::float16Ternary(const RdnaInstruction& inst, IrOpcode op const RdnaOperand& operand = accumulator && index == 2u ? accumulatorOperand(inst) : sourceAt(inst, index); args[index] = mix ? &readMixF32(operand).Value() : &readF16AsF32(operand).Value(); } + if (opcode == IrOpcode::FPFma32 && !mix) { + writeF16(inst.destination, fmaF16RoundedToOdd(IrF32(*args[0]), IrF32(*args[1]), IrF32(*args[2]))); + return true; + } writeF16(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {args[0], args[1], args[2]}))); return true; } +IrF32 TranslationContext::fmaF16RoundedToOdd(IrF32 lhs, IrF32 rhs, IrF32 addend) { + IrValue& product = ir.Emit(IrOpcode::FPMul32, IrType::F32, {&lhs.Value(), &rhs.Value()}); + IrValue& sum = ir.Emit(IrOpcode::FPAdd32, IrType::F32, {&product, &addend.Value()}); + IrValue& addendPart = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&sum, &product}); + IrValue& productPart = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&sum, &addendPart}); + IrValue& productError = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&product, &productPart}); + IrValue& addendError = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&addend.Value(), &addendPart}); + IrValue& error = ir.Emit(IrOpcode::FPAdd32, IrType::F32, {&productError, &addendError}); + IrValue& inexact = ir.Emit(IrOpcode::FPOrdNotEqual32, IrType::U1, {&error, &ir.ConstantF32(0.0f)}); + IrValue& sumBits = ir.BitCastU32(sum); + IrValue& sameSign = ir.IEqual(ir.BitwiseAnd(ir.BitwiseXor(sumBits, ir.BitCastU32(error)), ir.Constant(0x80000000u)), ir.Constant(0u)); + IrValue& towardZero = ir.Select(sameSign, sumBits, ir.ISub(sumBits, ir.Constant(1u))); + return IrF32(ir.BitCastF32(ir.Select(inexact, ir.BitwiseOr(towardZero, ir.Constant(1u)), sumBits))); +} + IrU32 TranslationContext::readF16Bits(const RdnaOperand& operand) { const IrU32 source = readF16SourceBits(operand); IrU32 bits(ir.BitwiseAnd((operand.opSel ? ir.ShiftRightLogical(source.Value(), ir.Constant(16u)) : source.Value()), ir.Constant(0xffffu))); diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index b5a4d0cef..b9e4f86f7 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -295,7 +295,6 @@ Throughout the project, every function at every stage either **does exactly what - The [shader float mode](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (`FLOAT_MODE`, `DX10_CLAMP`, `IEEE_MODE` and `FP16_OVFL` of `COMPUTE_PGM_RSRC1` and `SPI_SHADER_PGM_RSRC1_PS/GS/HS`, bit positions from Mesa's `gfx10.json` and `gfx103.json`) is decoded into `GuestContext::floatMode` and is part of the shader cache keys and the serialized request (#981). `IEEE_MODE` is read by `v_min_f32`, `v_max_f32`, `v_min3_f32`, `v_max3_f32` and `v_med3_f32`, `IEEE_MODE` and the `FLOAT_MODE` output denormal bits by the output modifier, `DX10_CLAMP` by the clamp modifier, and the whole mode by the 16-bit interpolation instructions (see their entries). With IEEE=1 a signaling NaN operand returns that NaN quieted (the first operand checked first, min3/max3 as two nested pairs, med3 as min3 when any operand is NaN), measured on RDNA2 (gfx1035) over every combination of ±0, ±1, ±inf, denormals, quiet and signaling NaNs with f32 denormals kept and flushed. IEEE=0 and unknown modes keep the previous lowering, which matches IEEE=0. The f16 and f64 min/max/med3, NaN quieting in f16 arithmetic (the unary instructions and the f32 arithmetic follow it, see below), `FP16_OVFL` and `FLOAT_MODE` (f32 denormals flushed before the min/max compare in Astro's Playroom's 0xc0) still follow a single mode, as the entries below record. The mode comes from the registered shader header's `sh_registers`, the same source at registration and at draw/dispatch so the prepared artifact keys match. A title that rewrites RSRC1 at run time, or a raw compute program with no header, compiles with that mode or none. - [v_mullit_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the hardware measured with the MODE register at IEEE=1 and denormals kept (a signaling NaN src0 is quieted only with IEEE=1, as measured at IEEE=0 too; a positive denormal src2 counts as positive): other denormal settings are not modelled. - [v_fma_legacy_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (VOP3 0x140) is assumed to be fused, which has not been measured on hardware. LLVM decodes this encoding as `v_mad_legacy_f32` for gfx1013, and `v_mac_legacy_f32` (VOP2 0x06) was measured to round its product before the add, so 0x140 may do the same. -- [v_fma_f16, v_pk_fma_f16 and v_pk_fmac_f16](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) are an f32 FMA rounded to f16, which rounds twice: 0x62e9 * 0xd159 + 0x23ec gives 0xf89e, the hardware 0xf89f. - [Signaling NaN inputs](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) of the f32 and f16 unary instructions (`v_sqrt/rcp/rsq/log/exp/sin/cos/fract/floor/ceil/trunc/rndne/rcp_iflag/frexp_mant`), `v_ldexp_f32/f16`, `v_frexp_mant_f16`, `v_div_fixup_f32/f16` and `v_mullit_f32` are returned quieted only with IEEE=1 and unchanged with IEEE=0, measured on RDNA2 (gfx1035) in both modes; shaders without a known float mode follow IEEE=0. `v_add/sub/subrev/mul/fma/fmac/fmamk/fmaak/mul_legacy/fma_legacy_f32` and `v_fma_mix_f32` with f32 sources choose their NaN result on the bits, measured over 1000 operand triples in both modes: the first NaN source in operand order after its modifiers (the subtrahend negated, as the hardware adds its negation), quieted only with IEEE=1, or the default NaN 0xffc00000 for an invalid operation; in the fma forms an invalid product (inf * 0) gives the default NaN even when src2 is a NaN, and in the legacy forms a zero factor makes the product 0 even when the other factor is a NaN (a NaN src2 of `v_fma_legacy_f32` still wins). The f16 arithmetic, `v_fma_mix_f32` with f16 sources, `v_cubema`, `v_mac_legacy_f32` and the separately rounded mad/mac forms (not measurable on RDNA2) follow the same rule on hardware but their NaN bits are left to the driver. `v_cvt_f16_f32` and `v_dot2_f32_f16` always quiet and `v_div_scale_f32` never does. - [Packed f16 NaN results](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (`v_pk_add/mul/fma/min/max_f16`) follow the hardware measured on RDNA2 (gfx1035) at IEEE=0 and IEEE=1 with f16 round-to-nearest-even and f16 denormals kept: with IEEE=1 a signaling NaN input is quieted and min/max return the other operand only for a quiet NaN; with IEEE=0 NaN inputs pass through unquieted and min/max ignore signaling. In `v_pk_fma_f16` an invalid product (inf * 0) gives the default NaN even when src2 is a NaN. A shader without a known float mode is translated as IEEE=0, the mode titles use; a mode with another f16 rounding or that flushes f16 denormals throws. - [f32 denormals](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follow the shader's `FLOAT_MODE` f32 denormal field (bits 5:4) when it flushes them (0, and shaders without a known float mode, as in the 0xc0 most titles use), measured on RDNA2 (gfx1035) over 1444 rows per instruction at IEEE=0 and IEEE=1. `v_add/sub/subrev/mul/mul_legacy/fma/fmac/fmamk/fmaak/min/max/min3/max3/med3/fract/floor/ceil/frexp_mant/sin/cos_f32` flush their f32 inputs and their result to a zero of the same sign; the f32 compares, `v_cvt_i32/u32/rpi_i32/flr_i32_f32`, `v_frexp_exp_i32_f32`, `v_cvt_f64_f32`, `v_cvt_f16_f32`, `v_cvt_pkrtz_f16_f32`, `v_cvt_pknorm_i16/u16_f32` and `v_cvt_pk_u8_f32` flush their f32 inputs. Tininess is detected before rounding: a product whose exact value is below 2^-126 gives zero even when it rounds to 2^-126, computed exactly from the significands for `v_mul_f32`, `v_mul_legacy_f32`, and for `v_fma_f32`, `v_fmac_f32`, `v_fmamk_f32`, `v_fmaak_f32` and `v_fma_mix_f32` when the addend is zero; with a nonzero addend the fused result is kept, so an exact result just below 2^-126 that rounds to it is not flushed. `v_fma_mix_f32` was measured with f32 sources only; its f16 sources are not flushed. Denormal modes 1 and 2 throw. Not covered, and not measurable on RDNA2 (no such opcodes) or not measured: `v_mac_f32`, `v_mad_f32`, `v_madak_f32` and `v_madmk_f32` (rounded separately), `v_mac_legacy_f32`, `v_dot2_f32_f16`, the division steps (on hardware `v_div_fixup_f32` flushes only src1 and src2, `v_div_fmas_f32` only its result), f32 interpolation and LDS float atomics keep the previous lowering. With f32 denormals kept (3) the results depend on the driver's default denormal behaviour, as no `DenormPreserve` execution mode is requested, and `v_ldexp_f32` flushes in every mode.