diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16FmaRounding.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16FmaRounding.cpp new file mode 100644 index 000000000..424059a6c --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16FmaRounding.cpp @@ -0,0 +1,232 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 64; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 8; +constexpr std::uint32_t Checks = 7; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084, 0x34060085, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xd74b000a, 0x041a0b04, 0xd74b380b, + 0x041a0b04, 0x7e180306, 0x6c180b04, 0xcc0e400d, 0x1c1a0b04, 0x7e1c0306, 0x781c0b04, 0x701e0f04, + 0x00000001, 0x6e200f04, 0x00003d55, 0x7e220280, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, + 0xbf810000, +}; + +constexpr std::uint32_t Rows[Threads][Inputs] = { + {0x800062e9u, 0x3c00d159u, 0x800023ecu, 0x0000f9a3u}, + {0x0000d4e0u, 0xbc005ae0u, 0x0000881fu, 0x00001e01u}, + {0x4200c1b4u, 0x4400f580u, 0xca008544u, 0x00003700u}, + {0x7c005af7u, 0x4000bde3u, 0x3c00772cu, 0x000027e9u}, + {0x3c0053e4u, 0x3c005900u, 0xfc000955u, 0x00002500u}, + {0x5bff3464u, 0x5bff5542u, 0x3c0012ffu, 0x00006d19u}, + {0x3c00c6d2u, 0x3c01064eu, 0x800129d8u, 0x00003200u}, + {0x1400e568u, 0x14004b80u, 0x00000891u, 0x000069d9u}, + {0xba18f442u, 0x70c0b900u, 0x88810546u, 0x00000700u}, + {0xbba84092u, 0x6a804788u, 0x814512ffu, 0x00008ba1u}, + {0xbc2cd70cu, 0x6d3ad700u, 0xa7f90ee0u, 0x00002100u}, + {0xbf00562cu, 0x72cc3774u, 0x8497ee8cu, 0x0000c716u}, + {0x56b4e294u, 0xdf004f00u, 0x15fb0683u, 0x00001700u}, + {0x1d0dd0e0u, 0x0a56cea0u, 0x1828837fu, 0x00001a01u}, + {0x5a40f006u, 0xcad01f55u, 0x80e6ea0eu, 0x00000900u}, + {0xcb3cc300u, 0x9615e4bcu, 0xc4a80843u, 0x0000d0fbu}, + {0x9e945d6eu, 0x95793cf9u, 0x998179b2u, 0x00001a00u}, + {0x603451cdu, 0xcf80d341u, 0x022c2276u, 0x00009041u}, + {0x06fbc6d0u, 0xf697ed40u, 0x5ea10c65u, 0x000030c0u}, + {0xd416668bu, 0x0a5d0272u, 0xc2d759b3u, 0x000032c9u}, + {0x6836db60u, 0x4900c9e0u, 0x806a02f8u, 0x00001ca0u}, + {0x02637ad4u, 0xe2b43100u, 0xd7ad0608u, 0x00002007u}, + {0xb4544d00u, 0x70806a2eu, 0x84828f49u, 0x00002a74u}, + {0x560003e6u, 0xd8453e28u, 0x0245af54u, 0x00005100u}, + {0x9b9bd580u, 0x08356004u, 0x14eb1222u, 0x000021e8u}, + {0x9aec64f8u, 0x67e64440u, 0x01018892u, 0x0000ad81u}, + {0x4366c00eu, 0x6e00eddcu, 0x0c16abffu, 0x00003900u}, + {0x002142a6u, 0xc7c266dbu, 0x28409b9du, 0x00000f11u}, + {0x45e74af4u, 0x015b6900u, 0xb5681505u, 0x00002d00u}, + {0x18eb7a00u, 0x6bc32ce7u, 0x804202f8u, 0x00008001u}, + {0xe990f8f2u, 0xc1201b57u, 0x06f71bc1u, 0x00008280u}, + {0xd9001c4du, 0xddeaf293u, 0x165316e1u, 0x00000e2bu}, + {0x532aee50u, 0x4e7c41c0u, 0xa97e11aau, 0x000009c0u}, + {0x066e7100u, 0xf4fa45aeu, 0xeee58a84u, 0x00003274u}, + {0x4dabed80u, 0x143c493cu, 0xcc6d8b1au, 0x000009e8u}, + {0xbd10d940u, 0x6d205d18u, 0x87670b12u, 0x00002001u}, + {0x68704f00u, 0x4e60e80au, 0x11f601a7u, 0x0000289cu}, + {0x5c805cd0u, 0xd9d45820u, 0x977a112bu, 0x00008357u}, + {0xf3003d96u, 0x34946e00u, 0x07660459u, 0x00003a00u}, + {0x363c73c0u, 0xa5223f70u, 0xb7e30d44u, 0x0000083au}, + {0x73de9e78u, 0xbe00262fu, 0x8872ac8eu, 0x00005a80u}, + {0xe6d7ccfeu, 0xb8f8e900u, 0xa88286b3u, 0x00001561u}, + {0x2824d550u, 0xf1805ac0u, 0x00860ec2u, 0x000022c0u}, + {0x36442fcau, 0x770019c7u, 0x8c9f254au, 0x00005fd8u}, + {0x6e192033u, 0xa3ae06abu, 0x84228eadu, 0x00005a00u}, + {0xad87d9b0u, 0x51cad8e0u, 0xf3f304cbu, 0x0000f6e3u}, + {0x39da80abu, 0xbad699fdu, 0xdc3c807du, 0x000012a2u}, + {0x199a8735u, 0x724992a9u, 0x9b4f82d6u, 0x0000a863u}, + {0xe56f0101u, 0x22719dfau, 0xe53e820au, 0x00000e86u}, + {0x66400f97u, 0x4c280a53u, 0x14008394u, 0x00004c5cu}, + {0x54409989u, 0xce7880b9u, 0x0151825eu, 0x0000066eu}, + {0x54b00f7cu, 0x506091e1u, 0x00a08146u, 0x00008201u}, + {0x50d689e7u, 0x4237096cu, 0x0dfa831eu, 0x00006495u}, + {0x04288b6fu, 0x0fb3084eu, 0x01e9037fu, 0x0000a90cu}, + {0xa1eb1a63u, 0x853d01e1u, 0x015a03b9u, 0x00003ea2u}, + {0x8af80d78u, 0x98e105dau, 0x02ef8286u, 0x0000294bu}, + {0x216c08b8u, 0x86d30ac8u, 0x8336022fu, 0x00006f80u}, + {0x87f41c46u, 0x0c0688eau, 0x82e782efu, 0x0000c6b4u}, + {0x0a215be0u, 0x183e5c10u, 0x83618001u, 0x00001c60u}, + {0x03dcdbe0u, 0x96385c10u, 0x02650001u, 0x000092e8u}, + {0x13496ff2u, 0x92dd5328u, 0x0270cb02u, 0x00008a00u}, + {0x8b276d1au, 0x9f695348u, 0x8397f1e6u, 0x000048c4u}, + {0x9560ee89u, 0x817d02f3u, 0x0331d633u, 0x0000de07u}, + {0x0fb3d541u, 0x0a3c089du, 0x822a503fu, 0x00003c3du} +}; +constexpr std::uint32_t Expected[Threads][Checks] = { + {0xf89fu, 0x8000u, 0xf89fu, 0x8000f89fu, 0x8000f89fu, 0xfc00u, 0xf97eu}, + {0xf431u, 0x0000u, 0xf431u, 0x0000f431u, 0x0000f431u, 0xb751u, 0xd67fu}, + {0x7bd7u, 0x0000u, 0x7bd7u, 0x00007bd7u, 0x00007bd7u, 0xbcfdu, 0xc2bau}, + {0x7717u, 0x7c00u, 0x7717u, 0x7c007717u, 0x7c007717u, 0x46e3u, 0x5ca5u}, + {0x70efu, 0xfc00u, 0x70efu, 0xfc0070efu, 0xfc0070efu, 0x3cefu, 0x5543u}, + {0x4dc5u, 0x7bfeu, 0x4dc5u, 0x7bfe4dc5u, 0x7bfe4dc5u, 0x6598u, 0x6d19u}, + {0x29c3u, 0x3c01u, 0x29c3u, 0x3c0129c3u, 0x3c0129c3u, 0xbd1du, 0xc874u}, + {0xf511u, 0x0010u, 0xf511u, 0x0010f511u, 0x0010f511u, 0xfc00u, 0x647du}, + {0x7153u, 0xef3du, 0x7153u, 0xef3d7153u, 0xef3d7153u, 0xbf73u, 0xf5adu}, + {0x4c4du, 0xea39u, 0x4c4du, 0xea394c4du, 0xea394c4du, 0x905cu, 0x4217u}, + {0x722bu, 0xed73u, 0x722bu, 0xed73722bu, 0xed73722bu, 0xbc67u, 0xd8b2u}, + {0xee81u, 0xf5f3u, 0xee81u, 0xf5f3ee81u, 0xf5f3ee81u, 0xe177u, 0x57c9u}, + {0xf5c1u, 0xf9ddu, 0xf5c1u, 0xf9ddf5c1u, 0xf9ddf5c1u, 0xbdc1u, 0xe462u}, + {0x6409u, 0x1829u, 0x6409u, 0x18296409u, 0x18296409u, 0xaf51u, 0xd27fu}, + {0xea2bu, 0xe953u, 0xea2bu, 0xe953ea2bu, 0xe953ea2bu, 0xbd07u, 0xf15du}, + {0x6c25u, 0xc4a3u, 0x6c25u, 0xc4a36c25u, 0xc4a36c25u, 0x585cu, 0xd190u}, + {0x79bfu, 0x997du, 0x79bfu, 0x997d79bfu, 0x997d79bfu, 0x3c13u, 0x5f3du}, + {0xe943u, 0xf3e1u, 0xe943u, 0xf3e1e943u, 0xf3e1e943u, 0xa62bu, 0x53bbu}, + {0x7879u, 0x5e95u, 0x7879u, 0x5e957879u, 0x5e957879u, 0xbc0bu, 0xc877u}, + {0x59b3u, 0xc2ddu, 0x59b3u, 0xc2dd59b3u, 0xc2dd59b3u, 0x5d8du, 0x685du}, + {0x696bu, 0x7543u, 0x696bu, 0x7543696bu, 0x7543696bu, 0xbc43u, 0xdceau}, + {0x7045u, 0xd7adu, 0x7045u, 0xd7ad7045u, 0xd7ad7045u, 0x5ee0u, 0x7c00u}, + {0x7bb9u, 0xe8dfu, 0x7bb9u, 0xe8df7bb9u, 0xe8df7bb9u, 0x3c09u, 0x4eadu}, + {0xaf53u, 0xf267u, 0xaf53u, 0xf267af53u, 0xf267af53u, 0x18e0u, 0x5100u}, + {0xf985u, 0x14ebu, 0xf985u, 0x14ebf985u, 0x14ebf985u, 0xbc0fu, 0xd755u}, + {0x6d47u, 0xc6d5u, 0x6d47u, 0xc6d56d47u, 0xc6d56d47u, 0xd6d6u, 0x669fu}, + {0x71f1u, 0x758du, 0x71f1u, 0x758d71f1u, 0x758d71f1u, 0xbd11u, 0xc028u}, + {0x6db3u, 0x283fu, 0x6db3u, 0x283f6db3u, 0x283f6db3u, 0x15dfu, 0x446fu}, + {0x7859u, 0xb567u, 0x7859u, 0xb5677859u, 0xb5677859u, 0x3c59u, 0x4ca7u}, + {0x6b5bu, 0x48c5u, 0x6b5bu, 0x48c56b5bu, 0x48c56b5bu, 0x9a00u, 0x7bffu}, + {0xd889u, 0x6f21u, 0xd889u, 0x6f21d889u, 0x6f21d889u, 0x3e2fu, 0xfa98u}, + {0xd311u, 0x7b65u, 0xd311u, 0x7b65d311u, 0x7b65d311u, 0x001cu, 0x1e1eu}, + {0xf489u, 0x65cfu, 0xf489u, 0x65cff489u, 0x65cff489u, 0xbc89u, 0xf035u}, + {0x7b19u, 0xeee5u, 0x7b19u, 0xeee57b19u, 0xeee57b19u, 0x6809u, 0x72aau}, + {0xfb33u, 0xcc6bu, 0xfb33u, 0xcc6bfb33u, 0xcc6bfb33u, 0xbc0fu, 0xef55u}, + {0xfaafu, 0xee7du, 0xfaafu, 0xee7dfaafu, 0xee7dfaafu, 0xbd41u, 0xdaffu}, + {0xfb11u, 0x7b13u, 0xfb11u, 0x7b13fb11u, 0x7b13fb11u, 0x3c09u, 0x50acu}, + {0x78f7u, 0xfa8fu, 0x78f7u, 0xfa8f78f7u, 0xfa8f78f7u, 0xa405u, 0x5e6au}, + {0x7031u, 0xec01u, 0x7031u, 0xec017031u, 0xec017031u, 0x3c31u, 0x4139u}, + {0x7735u, 0xb801u, 0x7735u, 0xb8017735u, 0xb8017735u, 0x4018u, 0x752au}, + {0xac91u, 0xf5e7u, 0xac91u, 0xf5e7ac91u, 0xf5e7ac91u, 0xbd41u, 0x5a80u}, + {0x7a3du, 0x643fu, 0x7a3du, 0x643f7a3du, 0x643f7a3du, 0xa6b7u, 0xcea7u}, + {0xf47bu, 0xddb1u, 0xf47bu, 0xddb1f47bu, 0xddb1f47bu, 0xbc7bu, 0xd715u}, + {0x2561u, 0x717bu, 0x2561u, 0x717b2561u, 0x717b2561u, 0x53a3u, 0x5fd9u}, + {0x8ea9u, 0xd5dbu, 0x8ea9u, 0xd5db8ea9u, 0xd5db8ea9u, 0x3e4du, 0x5a00u}, + {0x76efu, 0xf3f3u, 0x76efu, 0xf3f376efu, 0xf3f376efu, 0x7c00u, 0xf6f2u}, + {0x807du, 0xdc3fu, 0x807du, 0xdc3f807du, 0xdc3f807du, 0x0001u, 0x1286u}, + {0x82d5u, 0x5067u, 0x82d5u, 0x506782d5u, 0x506782d5u, 0x0040u, 0xa868u}, + {0x820bu, 0xe54fu, 0x820bu, 0xe54f820bu, 0xe54f820bu, 0x0001u, 0x0edcu}, + {0x8393u, 0x767fu, 0x8393u, 0x767f8393u, 0x767f8393u, 0x2023u, 0x4c5cu}, + {0x825du, 0xe6dfu, 0x825du, 0xe6df825du, 0xe6df825du, 0x8003u, 0x9b2du}, + {0x814bu, 0x6921u, 0x814bu, 0x6921814bu, 0x6921814bu, 0x0001u, 0x10bdu}, + {0x831fu, 0x5783u, 0x831fu, 0x5783831fu, 0x5783831fu, 0xb2c3u, 0x6495u}, + {0x037fu, 0x01e9u, 0x037fu, 0x01e9037fu, 0x01e9037fu, 0x0097u, 0xa916u}, + {0x03bbu, 0x0169u, 0x03bbu, 0x016903bbu, 0x016903bbu, 0x1d4cu, 0x3ea6u}, + {0x8285u, 0x02f7u, 0x8285u, 0x02f78285u, 0x02f78285u, 0x00e9u, 0x295au}, + {0x022fu, 0x8349u, 0x022fu, 0x8349022fu, 0x8349022fu, 0x3c6du, 0x6f80u}, + {0x82f9u, 0x82e7u, 0x82f9u, 0x82e782f9u, 0x82e782f9u, 0xa729u, 0xc6b3u}, + {0x7bffu, 0x835bu, 0x7bffu, 0x835b7bffu, 0x835b7bffu, 0x3c4fu, 0x5d40u}, + {0xfbffu, 0x0263u, 0xfbffu, 0x0263fbffu, 0x0263fbffu, 0x32ccu, 0xdd40u}, + {0x7c00u, 0x0263u, 0x7c00u, 0x02637c00u, 0x02637c00u, 0xbdf5u, 0x714cu}, + {0x7c00u, 0x837du, 0x7c00u, 0x837d7c00u, 0x837d7c00u, 0x7a14u, 0x6ecfu}, + {0xd638u, 0x0331u, 0xd638u, 0x0331d638u, 0x0331d638u, 0x7c00u, 0xf08bu}, + {0x503fu, 0x8229u, 0x503fu, 0x8229503fu, 0x8229503fu, 0xd591u, 0xd6f0u} +}; +constexpr std::uint32_t Masks[Checks] = {0xffffu, 0xffffu, 0xffffu, 0xffffffffu, 0xffffffffu, 0xffffu, 0xffffu}; +constexpr const char* Names[Checks] = { + "v_fma_f16 v10, v4, v5, v6", + "v_fma_f16 v11, v4, v5, v6 op_sel:[1,1,1,0]", + "v_fmac_f16 v12, v4, v5", + "v_pk_fma_f16 v13, v4, v5, v6", + "v_pk_fmac_f16 v14, v4, v5", + "v_fmaak_f16 v15, v4, v7, 0x0001", + "v_fmamk_f16 v16, v4, 0x3d55, v7", +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t index = 0; index < Checks; ++index) { + const std::uint32_t actual = Output[tid * Results + index] & Masks[index]; + Require(actual == Expected[tid][index], std::string("f16 fma rounding: lane ") + std::to_string(tid) + " " + Names[index] + " is " + Hex(actual) + ", expected " + Hex(Expected[tid][index])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("f16 fma rounding tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index bf5a58d69..0543afc24 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -177,6 +177,7 @@ private: bool vDivFixupF16(const RdnaInstruction& inst); bool float16Binary(const RdnaInstruction& inst, IrOpcode opcode, bool reverse); bool float16Ternary(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool mix); + IrF32 fmaF16RoundedToOdd(IrF32 lhs, IrF32 rhs, IrF32 addend); bool floatUnary(const RdnaInstruction& inst, IrOpcode opcode); bool floatBinary(const RdnaInstruction& inst, IrOpcode opcode, bool reverse); bool floatTernary(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool mix); diff --git a/core/shader/recompiler/Translation/src/FloatInstructions.cpp b/core/shader/recompiler/Translation/src/FloatInstructions.cpp index 79beb3d88..b16a8ceaf 100644 --- a/core/shader/recompiler/Translation/src/FloatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/FloatInstructions.cpp @@ -17,13 +17,15 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc const auto translateLane = [&](bool high) -> IrF32 { const IrF32 lhs = readF16LaneAsF32(sourceAt(inst, 0u), high, true); const IrF32 rhs = readF16LaneAsF32(sourceAt(inst, 1u), high, true); + const auto ternary = [&](const IrF32& third) -> IrF32 { + if (opcode == IrOpcode::FPFma32) return fmaF16RoundedToOdd(lhs, rhs, third); + return IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &third.Value()})); + }; if (accumulator) { - const IrF32 acc = readF16LaneAsF32(accumulatorOperand(inst), high, true); - return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &acc.Value()}))); + return applyF32ResultModifiers(inst.destination, ternary(readF16LaneAsF32(accumulatorOperand(inst), high, true))); } if (inst.sourceCount == 3u) { - const IrF32 third = readF16LaneAsF32(sourceAt(inst, 2u), high, true); - return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &third.Value()}))); + return applyF32ResultModifiers(inst.destination, ternary(readF16LaneAsF32(sourceAt(inst, 2u), high, true))); } return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value()}))); }; @@ -214,10 +216,29 @@ bool TranslationContext::float16Ternary(const RdnaInstruction& inst, IrOpcode op const RdnaOperand& operand = accumulator && index == 2u ? accumulatorOperand(inst) : sourceAt(inst, index); args[index] = mix ? &readMixF32(operand).Value() : &readF16AsF32(operand).Value(); } + if (opcode == IrOpcode::FPFma32 && !mix) { + writeF16(inst.destination, fmaF16RoundedToOdd(IrF32(*args[0]), IrF32(*args[1]), IrF32(*args[2]))); + return true; + } writeF16(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {args[0], args[1], args[2]}))); return true; } +IrF32 TranslationContext::fmaF16RoundedToOdd(IrF32 lhs, IrF32 rhs, IrF32 addend) { + IrValue& product = ir.Emit(IrOpcode::FPMul32, IrType::F32, {&lhs.Value(), &rhs.Value()}); + IrValue& sum = ir.Emit(IrOpcode::FPAdd32, IrType::F32, {&product, &addend.Value()}); + IrValue& addendPart = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&sum, &product}); + IrValue& productPart = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&sum, &addendPart}); + IrValue& productError = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&product, &productPart}); + IrValue& addendError = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&addend.Value(), &addendPart}); + IrValue& error = ir.Emit(IrOpcode::FPAdd32, IrType::F32, {&productError, &addendError}); + IrValue& inexact = ir.Emit(IrOpcode::FPOrdNotEqual32, IrType::U1, {&error, &ir.ConstantF32(0.0f)}); + IrValue& sumBits = ir.BitCastU32(sum); + IrValue& sameSign = ir.IEqual(ir.BitwiseAnd(ir.BitwiseXor(sumBits, ir.BitCastU32(error)), ir.Constant(0x80000000u)), ir.Constant(0u)); + IrValue& towardZero = ir.Select(sameSign, sumBits, ir.ISub(sumBits, ir.Constant(1u))); + return IrF32(ir.BitCastF32(ir.Select(inexact, ir.BitwiseOr(towardZero, ir.Constant(1u)), sumBits))); +} + IrU32 TranslationContext::readF16Bits(const RdnaOperand& operand) { const IrU32 source = readF16SourceBits(operand); IrU32 bits(ir.BitwiseAnd((operand.opSel ? ir.ShiftRightLogical(source.Value(), ir.Constant(16u)) : source.Value()), ir.Constant(0xffffu))); diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 187c57d8d..3672e743d 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -329,7 +329,6 @@ Throughout the project, every function at every stage either **does exactly what - The [shader float mode](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (`FLOAT_MODE`, `DX10_CLAMP`, `IEEE_MODE` and `FP16_OVFL` of `COMPUTE_PGM_RSRC1` and `SPI_SHADER_PGM_RSRC1_PS/GS/HS`, bit positions from Mesa's `gfx10.json` and `gfx103.json`) is decoded into `GuestContext::floatMode` and is part of the shader cache keys and the serialized request (#981). `IEEE_MODE` is read by `v_min_f32`, `v_max_f32`, `v_min3_f32`, `v_max3_f32` and `v_med3_f32`, `IEEE_MODE` and the `FLOAT_MODE` output denormal bits by the output modifier, `DX10_CLAMP` by the clamp modifier, and the whole mode by the 16-bit interpolation instructions (see their entries). With IEEE=1 a signaling NaN operand returns that NaN quieted (the first operand checked first, min3/max3 as two nested pairs, med3 as min3 when any operand is NaN), measured on RDNA2 (gfx1035) over every combination of ±0, ±1, ±inf, denormals, quiet and signaling NaNs with f32 denormals kept and flushed. IEEE=0 and unknown modes keep the previous lowering, which matches IEEE=0. The f16 and f64 min/max/med3, NaN quieting in f16 arithmetic (the unary instructions and the f32 arithmetic follow it, see below), `FP16_OVFL` and `FLOAT_MODE` (f32 denormals flushed before the min/max compare in Astro's Playroom's 0xc0) still follow a single mode, as the entries below record. The mode comes from the registered shader header's `sh_registers`, the same source at registration and at draw/dispatch so the prepared artifact keys match. A title that rewrites RSRC1 at run time, or a raw compute program with no header, compiles with that mode or none. - [v_mullit_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the hardware measured with the MODE register at IEEE=1 and denormals kept (a signaling NaN src0 is quieted only with IEEE=1, as measured at IEEE=0 too; a positive denormal src2 counts as positive): other denormal settings are not modelled. - [v_fma_legacy_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (VOP3 0x140) is assumed to be fused, which has not been measured on hardware. LLVM decodes this encoding as `v_mad_legacy_f32` for gfx1013, and `v_mac_legacy_f32` (VOP2 0x06) was measured to round its product before the add, so 0x140 may do the same. -- [v_fma_f16, v_pk_fma_f16 and v_pk_fmac_f16](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) are an f32 FMA rounded to f16, which rounds twice: 0x62e9 * 0xd159 + 0x23ec gives 0xf89e, the hardware 0xf89f. - [Signaling NaN inputs](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) of the f32 and f16 unary instructions (`v_sqrt/rcp/rsq/log/exp/sin/cos/fract/floor/ceil/trunc/rndne/rcp_iflag/frexp_mant`), `v_ldexp_f32/f16`, `v_frexp_mant_f16`, `v_div_fixup_f32/f16` and `v_mullit_f32` are returned quieted only with IEEE=1 and unchanged with IEEE=0, measured on RDNA2 (gfx1035) in both modes; shaders without a known float mode follow IEEE=0. `v_add/sub/subrev/mul/fma/fmac/fmamk/fmaak/mul_legacy/fma_legacy_f32` and `v_fma_mix_f32` with f32 sources choose their NaN result on the bits, measured over 1000 operand triples in both modes: the first NaN source in operand order after its modifiers (the subtrahend negated, as the hardware adds its negation), quieted only with IEEE=1, or the default NaN 0xffc00000 for an invalid operation; in the fma forms an invalid product (inf * 0) gives the default NaN even when src2 is a NaN, and in the legacy forms a zero factor makes the product 0 even when the other factor is a NaN (a NaN src2 of `v_fma_legacy_f32` still wins). The f16 arithmetic, `v_fma_mix_f32` with f16 sources, `v_cubema`, `v_mac_legacy_f32` and the separately rounded mad/mac forms (not measurable on RDNA2) follow the same rule on hardware but their NaN bits are left to the driver. `v_cvt_f16_f32` and `v_dot2_f32_f16` always quiet and `v_div_scale_f32` never does. - [Packed f16 NaN results](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (`v_pk_add/mul/fma/min/max_f16`) follow the hardware measured on RDNA2 (gfx1035) at IEEE=0 and IEEE=1 with f16 round-to-nearest-even and f16 denormals kept: with IEEE=1 a signaling NaN input is quieted and min/max return the other operand only for a quiet NaN; with IEEE=0 NaN inputs pass through unquieted and min/max ignore signaling. In `v_pk_fma_f16` an invalid product (inf * 0) gives the default NaN even when src2 is a NaN. A shader without a known float mode is translated as IEEE=0, the mode titles use; a mode with another f16 rounding or that flushes f16 denormals throws. - [f32 denormals](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follow the shader's `FLOAT_MODE` f32 denormal field (bits 5:4) when it flushes them (0, and shaders without a known float mode, as in the 0xc0 most titles use), measured on RDNA2 (gfx1035) over 1444 rows per instruction at IEEE=0 and IEEE=1. `v_add/sub/subrev/mul/mul_legacy/fma/fmac/fmamk/fmaak/min/max/min3/max3/med3/fract/floor/ceil/frexp_mant/sin/cos_f32` flush their f32 inputs and their result to a zero of the same sign; the f32 compares, `v_cvt_i32/u32/rpi_i32/flr_i32_f32`, `v_frexp_exp_i32_f32`, `v_cvt_f64_f32`, `v_cvt_f16_f32`, `v_cvt_pkrtz_f16_f32`, `v_cvt_pknorm_i16/u16_f32` and `v_cvt_pk_u8_f32` flush their f32 inputs. Tininess is detected before rounding: a product whose exact value is below 2^-126 gives zero even when it rounds to 2^-126, computed exactly from the significands for `v_mul_f32`, `v_mul_legacy_f32`, and for `v_fma_f32`, `v_fmac_f32`, `v_fmamk_f32`, `v_fmaak_f32` and `v_fma_mix_f32` when the addend is zero; with a nonzero addend the fused result is kept, so an exact result just below 2^-126 that rounds to it is not flushed. `v_fma_mix_f32` was measured with f32 sources only; its f16 sources are not flushed. Denormal modes 1 and 2 throw. Not covered, and not measurable on RDNA2 (no such opcodes) or not measured: `v_mac_f32`, `v_mad_f32`, `v_madak_f32` and `v_madmk_f32` (rounded separately), `v_mac_legacy_f32`, `v_dot2_f32_f16`, the division steps (on hardware `v_div_fixup_f32` flushes only src1 and src2, `v_div_fmas_f32` only its result), f32 interpolation and LDS float atomics keep the previous lowering. With f32 denormals kept (3) the results depend on the driver's default denormal behaviour, as no `DenormPreserve` execution mode is requested, and `v_ldexp_f32` flushes in every mode.