Merge pull request #2000 from hansjm10/fix/shader-f16-fma-single-rounding
fix(shader): round the f16 fused multiply-adds once
This commit is contained in:
Adrià
GitHub
commit
0f84feff1d
|
|
@ -0,0 +1,232 @@
|
|||
#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
|
||||
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
|
||||
#include "Recompiler.hpp"
|
||||
#include "VulkanTestDevice.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
using AgcDriver::Graphics::Require;
|
||||
using ShaderRecompiler::ShaderStage;
|
||||
|
||||
constexpr std::uint32_t Threads = 64;
|
||||
constexpr std::uint32_t Inputs = 4;
|
||||
constexpr std::uint32_t Results = 8;
|
||||
constexpr std::uint32_t Checks = 7;
|
||||
alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
|
||||
alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
|
||||
|
||||
alignas(256) constexpr std::array<std::uint32_t, 25> Code{
|
||||
0x34020084, 0x34060085, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xd74b000a, 0x041a0b04, 0xd74b380b,
|
||||
0x041a0b04, 0x7e180306, 0x6c180b04, 0xcc0e400d, 0x1c1a0b04, 0x7e1c0306, 0x781c0b04, 0x701e0f04,
|
||||
0x00000001, 0x6e200f04, 0x00003d55, 0x7e220280, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03,
|
||||
0xbf810000,
|
||||
};
|
||||
|
||||
constexpr std::uint32_t Rows[Threads][Inputs] = {
|
||||
{0x800062e9u, 0x3c00d159u, 0x800023ecu, 0x0000f9a3u},
|
||||
{0x0000d4e0u, 0xbc005ae0u, 0x0000881fu, 0x00001e01u},
|
||||
{0x4200c1b4u, 0x4400f580u, 0xca008544u, 0x00003700u},
|
||||
{0x7c005af7u, 0x4000bde3u, 0x3c00772cu, 0x000027e9u},
|
||||
{0x3c0053e4u, 0x3c005900u, 0xfc000955u, 0x00002500u},
|
||||
{0x5bff3464u, 0x5bff5542u, 0x3c0012ffu, 0x00006d19u},
|
||||
{0x3c00c6d2u, 0x3c01064eu, 0x800129d8u, 0x00003200u},
|
||||
{0x1400e568u, 0x14004b80u, 0x00000891u, 0x000069d9u},
|
||||
{0xba18f442u, 0x70c0b900u, 0x88810546u, 0x00000700u},
|
||||
{0xbba84092u, 0x6a804788u, 0x814512ffu, 0x00008ba1u},
|
||||
{0xbc2cd70cu, 0x6d3ad700u, 0xa7f90ee0u, 0x00002100u},
|
||||
{0xbf00562cu, 0x72cc3774u, 0x8497ee8cu, 0x0000c716u},
|
||||
{0x56b4e294u, 0xdf004f00u, 0x15fb0683u, 0x00001700u},
|
||||
{0x1d0dd0e0u, 0x0a56cea0u, 0x1828837fu, 0x00001a01u},
|
||||
{0x5a40f006u, 0xcad01f55u, 0x80e6ea0eu, 0x00000900u},
|
||||
{0xcb3cc300u, 0x9615e4bcu, 0xc4a80843u, 0x0000d0fbu},
|
||||
{0x9e945d6eu, 0x95793cf9u, 0x998179b2u, 0x00001a00u},
|
||||
{0x603451cdu, 0xcf80d341u, 0x022c2276u, 0x00009041u},
|
||||
{0x06fbc6d0u, 0xf697ed40u, 0x5ea10c65u, 0x000030c0u},
|
||||
{0xd416668bu, 0x0a5d0272u, 0xc2d759b3u, 0x000032c9u},
|
||||
{0x6836db60u, 0x4900c9e0u, 0x806a02f8u, 0x00001ca0u},
|
||||
{0x02637ad4u, 0xe2b43100u, 0xd7ad0608u, 0x00002007u},
|
||||
{0xb4544d00u, 0x70806a2eu, 0x84828f49u, 0x00002a74u},
|
||||
{0x560003e6u, 0xd8453e28u, 0x0245af54u, 0x00005100u},
|
||||
{0x9b9bd580u, 0x08356004u, 0x14eb1222u, 0x000021e8u},
|
||||
{0x9aec64f8u, 0x67e64440u, 0x01018892u, 0x0000ad81u},
|
||||
{0x4366c00eu, 0x6e00eddcu, 0x0c16abffu, 0x00003900u},
|
||||
{0x002142a6u, 0xc7c266dbu, 0x28409b9du, 0x00000f11u},
|
||||
{0x45e74af4u, 0x015b6900u, 0xb5681505u, 0x00002d00u},
|
||||
{0x18eb7a00u, 0x6bc32ce7u, 0x804202f8u, 0x00008001u},
|
||||
{0xe990f8f2u, 0xc1201b57u, 0x06f71bc1u, 0x00008280u},
|
||||
{0xd9001c4du, 0xddeaf293u, 0x165316e1u, 0x00000e2bu},
|
||||
{0x532aee50u, 0x4e7c41c0u, 0xa97e11aau, 0x000009c0u},
|
||||
{0x066e7100u, 0xf4fa45aeu, 0xeee58a84u, 0x00003274u},
|
||||
{0x4dabed80u, 0x143c493cu, 0xcc6d8b1au, 0x000009e8u},
|
||||
{0xbd10d940u, 0x6d205d18u, 0x87670b12u, 0x00002001u},
|
||||
{0x68704f00u, 0x4e60e80au, 0x11f601a7u, 0x0000289cu},
|
||||
{0x5c805cd0u, 0xd9d45820u, 0x977a112bu, 0x00008357u},
|
||||
{0xf3003d96u, 0x34946e00u, 0x07660459u, 0x00003a00u},
|
||||
{0x363c73c0u, 0xa5223f70u, 0xb7e30d44u, 0x0000083au},
|
||||
{0x73de9e78u, 0xbe00262fu, 0x8872ac8eu, 0x00005a80u},
|
||||
{0xe6d7ccfeu, 0xb8f8e900u, 0xa88286b3u, 0x00001561u},
|
||||
{0x2824d550u, 0xf1805ac0u, 0x00860ec2u, 0x000022c0u},
|
||||
{0x36442fcau, 0x770019c7u, 0x8c9f254au, 0x00005fd8u},
|
||||
{0x6e192033u, 0xa3ae06abu, 0x84228eadu, 0x00005a00u},
|
||||
{0xad87d9b0u, 0x51cad8e0u, 0xf3f304cbu, 0x0000f6e3u},
|
||||
{0x39da80abu, 0xbad699fdu, 0xdc3c807du, 0x000012a2u},
|
||||
{0x199a8735u, 0x724992a9u, 0x9b4f82d6u, 0x0000a863u},
|
||||
{0xe56f0101u, 0x22719dfau, 0xe53e820au, 0x00000e86u},
|
||||
{0x66400f97u, 0x4c280a53u, 0x14008394u, 0x00004c5cu},
|
||||
{0x54409989u, 0xce7880b9u, 0x0151825eu, 0x0000066eu},
|
||||
{0x54b00f7cu, 0x506091e1u, 0x00a08146u, 0x00008201u},
|
||||
{0x50d689e7u, 0x4237096cu, 0x0dfa831eu, 0x00006495u},
|
||||
{0x04288b6fu, 0x0fb3084eu, 0x01e9037fu, 0x0000a90cu},
|
||||
{0xa1eb1a63u, 0x853d01e1u, 0x015a03b9u, 0x00003ea2u},
|
||||
{0x8af80d78u, 0x98e105dau, 0x02ef8286u, 0x0000294bu},
|
||||
{0x216c08b8u, 0x86d30ac8u, 0x8336022fu, 0x00006f80u},
|
||||
{0x87f41c46u, 0x0c0688eau, 0x82e782efu, 0x0000c6b4u},
|
||||
{0x0a215be0u, 0x183e5c10u, 0x83618001u, 0x00001c60u},
|
||||
{0x03dcdbe0u, 0x96385c10u, 0x02650001u, 0x000092e8u},
|
||||
{0x13496ff2u, 0x92dd5328u, 0x0270cb02u, 0x00008a00u},
|
||||
{0x8b276d1au, 0x9f695348u, 0x8397f1e6u, 0x000048c4u},
|
||||
{0x9560ee89u, 0x817d02f3u, 0x0331d633u, 0x0000de07u},
|
||||
{0x0fb3d541u, 0x0a3c089du, 0x822a503fu, 0x00003c3du}
|
||||
};
|
||||
constexpr std::uint32_t Expected[Threads][Checks] = {
|
||||
{0xf89fu, 0x8000u, 0xf89fu, 0x8000f89fu, 0x8000f89fu, 0xfc00u, 0xf97eu},
|
||||
{0xf431u, 0x0000u, 0xf431u, 0x0000f431u, 0x0000f431u, 0xb751u, 0xd67fu},
|
||||
{0x7bd7u, 0x0000u, 0x7bd7u, 0x00007bd7u, 0x00007bd7u, 0xbcfdu, 0xc2bau},
|
||||
{0x7717u, 0x7c00u, 0x7717u, 0x7c007717u, 0x7c007717u, 0x46e3u, 0x5ca5u},
|
||||
{0x70efu, 0xfc00u, 0x70efu, 0xfc0070efu, 0xfc0070efu, 0x3cefu, 0x5543u},
|
||||
{0x4dc5u, 0x7bfeu, 0x4dc5u, 0x7bfe4dc5u, 0x7bfe4dc5u, 0x6598u, 0x6d19u},
|
||||
{0x29c3u, 0x3c01u, 0x29c3u, 0x3c0129c3u, 0x3c0129c3u, 0xbd1du, 0xc874u},
|
||||
{0xf511u, 0x0010u, 0xf511u, 0x0010f511u, 0x0010f511u, 0xfc00u, 0x647du},
|
||||
{0x7153u, 0xef3du, 0x7153u, 0xef3d7153u, 0xef3d7153u, 0xbf73u, 0xf5adu},
|
||||
{0x4c4du, 0xea39u, 0x4c4du, 0xea394c4du, 0xea394c4du, 0x905cu, 0x4217u},
|
||||
{0x722bu, 0xed73u, 0x722bu, 0xed73722bu, 0xed73722bu, 0xbc67u, 0xd8b2u},
|
||||
{0xee81u, 0xf5f3u, 0xee81u, 0xf5f3ee81u, 0xf5f3ee81u, 0xe177u, 0x57c9u},
|
||||
{0xf5c1u, 0xf9ddu, 0xf5c1u, 0xf9ddf5c1u, 0xf9ddf5c1u, 0xbdc1u, 0xe462u},
|
||||
{0x6409u, 0x1829u, 0x6409u, 0x18296409u, 0x18296409u, 0xaf51u, 0xd27fu},
|
||||
{0xea2bu, 0xe953u, 0xea2bu, 0xe953ea2bu, 0xe953ea2bu, 0xbd07u, 0xf15du},
|
||||
{0x6c25u, 0xc4a3u, 0x6c25u, 0xc4a36c25u, 0xc4a36c25u, 0x585cu, 0xd190u},
|
||||
{0x79bfu, 0x997du, 0x79bfu, 0x997d79bfu, 0x997d79bfu, 0x3c13u, 0x5f3du},
|
||||
{0xe943u, 0xf3e1u, 0xe943u, 0xf3e1e943u, 0xf3e1e943u, 0xa62bu, 0x53bbu},
|
||||
{0x7879u, 0x5e95u, 0x7879u, 0x5e957879u, 0x5e957879u, 0xbc0bu, 0xc877u},
|
||||
{0x59b3u, 0xc2ddu, 0x59b3u, 0xc2dd59b3u, 0xc2dd59b3u, 0x5d8du, 0x685du},
|
||||
{0x696bu, 0x7543u, 0x696bu, 0x7543696bu, 0x7543696bu, 0xbc43u, 0xdceau},
|
||||
{0x7045u, 0xd7adu, 0x7045u, 0xd7ad7045u, 0xd7ad7045u, 0x5ee0u, 0x7c00u},
|
||||
{0x7bb9u, 0xe8dfu, 0x7bb9u, 0xe8df7bb9u, 0xe8df7bb9u, 0x3c09u, 0x4eadu},
|
||||
{0xaf53u, 0xf267u, 0xaf53u, 0xf267af53u, 0xf267af53u, 0x18e0u, 0x5100u},
|
||||
{0xf985u, 0x14ebu, 0xf985u, 0x14ebf985u, 0x14ebf985u, 0xbc0fu, 0xd755u},
|
||||
{0x6d47u, 0xc6d5u, 0x6d47u, 0xc6d56d47u, 0xc6d56d47u, 0xd6d6u, 0x669fu},
|
||||
{0x71f1u, 0x758du, 0x71f1u, 0x758d71f1u, 0x758d71f1u, 0xbd11u, 0xc028u},
|
||||
{0x6db3u, 0x283fu, 0x6db3u, 0x283f6db3u, 0x283f6db3u, 0x15dfu, 0x446fu},
|
||||
{0x7859u, 0xb567u, 0x7859u, 0xb5677859u, 0xb5677859u, 0x3c59u, 0x4ca7u},
|
||||
{0x6b5bu, 0x48c5u, 0x6b5bu, 0x48c56b5bu, 0x48c56b5bu, 0x9a00u, 0x7bffu},
|
||||
{0xd889u, 0x6f21u, 0xd889u, 0x6f21d889u, 0x6f21d889u, 0x3e2fu, 0xfa98u},
|
||||
{0xd311u, 0x7b65u, 0xd311u, 0x7b65d311u, 0x7b65d311u, 0x001cu, 0x1e1eu},
|
||||
{0xf489u, 0x65cfu, 0xf489u, 0x65cff489u, 0x65cff489u, 0xbc89u, 0xf035u},
|
||||
{0x7b19u, 0xeee5u, 0x7b19u, 0xeee57b19u, 0xeee57b19u, 0x6809u, 0x72aau},
|
||||
{0xfb33u, 0xcc6bu, 0xfb33u, 0xcc6bfb33u, 0xcc6bfb33u, 0xbc0fu, 0xef55u},
|
||||
{0xfaafu, 0xee7du, 0xfaafu, 0xee7dfaafu, 0xee7dfaafu, 0xbd41u, 0xdaffu},
|
||||
{0xfb11u, 0x7b13u, 0xfb11u, 0x7b13fb11u, 0x7b13fb11u, 0x3c09u, 0x50acu},
|
||||
{0x78f7u, 0xfa8fu, 0x78f7u, 0xfa8f78f7u, 0xfa8f78f7u, 0xa405u, 0x5e6au},
|
||||
{0x7031u, 0xec01u, 0x7031u, 0xec017031u, 0xec017031u, 0x3c31u, 0x4139u},
|
||||
{0x7735u, 0xb801u, 0x7735u, 0xb8017735u, 0xb8017735u, 0x4018u, 0x752au},
|
||||
{0xac91u, 0xf5e7u, 0xac91u, 0xf5e7ac91u, 0xf5e7ac91u, 0xbd41u, 0x5a80u},
|
||||
{0x7a3du, 0x643fu, 0x7a3du, 0x643f7a3du, 0x643f7a3du, 0xa6b7u, 0xcea7u},
|
||||
{0xf47bu, 0xddb1u, 0xf47bu, 0xddb1f47bu, 0xddb1f47bu, 0xbc7bu, 0xd715u},
|
||||
{0x2561u, 0x717bu, 0x2561u, 0x717b2561u, 0x717b2561u, 0x53a3u, 0x5fd9u},
|
||||
{0x8ea9u, 0xd5dbu, 0x8ea9u, 0xd5db8ea9u, 0xd5db8ea9u, 0x3e4du, 0x5a00u},
|
||||
{0x76efu, 0xf3f3u, 0x76efu, 0xf3f376efu, 0xf3f376efu, 0x7c00u, 0xf6f2u},
|
||||
{0x807du, 0xdc3fu, 0x807du, 0xdc3f807du, 0xdc3f807du, 0x0001u, 0x1286u},
|
||||
{0x82d5u, 0x5067u, 0x82d5u, 0x506782d5u, 0x506782d5u, 0x0040u, 0xa868u},
|
||||
{0x820bu, 0xe54fu, 0x820bu, 0xe54f820bu, 0xe54f820bu, 0x0001u, 0x0edcu},
|
||||
{0x8393u, 0x767fu, 0x8393u, 0x767f8393u, 0x767f8393u, 0x2023u, 0x4c5cu},
|
||||
{0x825du, 0xe6dfu, 0x825du, 0xe6df825du, 0xe6df825du, 0x8003u, 0x9b2du},
|
||||
{0x814bu, 0x6921u, 0x814bu, 0x6921814bu, 0x6921814bu, 0x0001u, 0x10bdu},
|
||||
{0x831fu, 0x5783u, 0x831fu, 0x5783831fu, 0x5783831fu, 0xb2c3u, 0x6495u},
|
||||
{0x037fu, 0x01e9u, 0x037fu, 0x01e9037fu, 0x01e9037fu, 0x0097u, 0xa916u},
|
||||
{0x03bbu, 0x0169u, 0x03bbu, 0x016903bbu, 0x016903bbu, 0x1d4cu, 0x3ea6u},
|
||||
{0x8285u, 0x02f7u, 0x8285u, 0x02f78285u, 0x02f78285u, 0x00e9u, 0x295au},
|
||||
{0x022fu, 0x8349u, 0x022fu, 0x8349022fu, 0x8349022fu, 0x3c6du, 0x6f80u},
|
||||
{0x82f9u, 0x82e7u, 0x82f9u, 0x82e782f9u, 0x82e782f9u, 0xa729u, 0xc6b3u},
|
||||
{0x7bffu, 0x835bu, 0x7bffu, 0x835b7bffu, 0x835b7bffu, 0x3c4fu, 0x5d40u},
|
||||
{0xfbffu, 0x0263u, 0xfbffu, 0x0263fbffu, 0x0263fbffu, 0x32ccu, 0xdd40u},
|
||||
{0x7c00u, 0x0263u, 0x7c00u, 0x02637c00u, 0x02637c00u, 0xbdf5u, 0x714cu},
|
||||
{0x7c00u, 0x837du, 0x7c00u, 0x837d7c00u, 0x837d7c00u, 0x7a14u, 0x6ecfu},
|
||||
{0xd638u, 0x0331u, 0xd638u, 0x0331d638u, 0x0331d638u, 0x7c00u, 0xf08bu},
|
||||
{0x503fu, 0x8229u, 0x503fu, 0x8229503fu, 0x8229503fu, 0xd591u, 0xd6f0u}
|
||||
};
|
||||
constexpr std::uint32_t Masks[Checks] = {0xffffu, 0xffffu, 0xffffu, 0xffffffffu, 0xffffffffu, 0xffffu, 0xffffu};
|
||||
constexpr const char* Names[Checks] = {
|
||||
"v_fma_f16 v10, v4, v5, v6",
|
||||
"v_fma_f16 v11, v4, v5, v6 op_sel:[1,1,1,0]",
|
||||
"v_fmac_f16 v12, v4, v5",
|
||||
"v_pk_fma_f16 v13, v4, v5, v6",
|
||||
"v_pk_fmac_f16 v14, v4, v5",
|
||||
"v_fmaak_f16 v15, v4, v7, 0x0001",
|
||||
"v_fmamk_f16 v16, v4, 0x3d55, v7",
|
||||
};
|
||||
|
||||
std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(data);
|
||||
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
|
||||
}
|
||||
|
||||
std::string Hex(std::uint32_t value) {
|
||||
char text[16];
|
||||
std::snprintf(text, sizeof(text), "0x%08x", value);
|
||||
return text;
|
||||
}
|
||||
|
||||
void Run(AgcDriver::VulkanDevice& device) {
|
||||
for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]);
|
||||
Output.fill(0xdeadbeefu);
|
||||
std::vector<std::uint32_t> userData(8, 0u);
|
||||
const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
|
||||
const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
|
||||
std::copy(input.begin(), input.end(), userData.begin());
|
||||
std::copy(output.begin(), output.end(), userData.begin() + 4);
|
||||
const std::span<const std::uint32_t> code(Code);
|
||||
const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
|
||||
const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
|
||||
ShaderRecompiler::RecompileRequest request{
|
||||
{ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
|
||||
{32, 0, userData, compute, std::nullopt, std::nullopt, memory},
|
||||
device.Target(),
|
||||
{0, 0, 0, 128}
|
||||
};
|
||||
request.useCache = false;
|
||||
const auto result = ShaderRecompiler::Recompile(request);
|
||||
device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
|
||||
device.WaitIdle();
|
||||
}
|
||||
|
||||
void Check() {
|
||||
for (std::uint32_t tid = 0; tid < Threads; ++tid) {
|
||||
for (std::uint32_t index = 0; index < Checks; ++index) {
|
||||
const std::uint32_t actual = Output[tid * Results + index] & Masks[index];
|
||||
Require(actual == Expected[tid][index], std::string("f16 fma rounding: lane ") + std::to_string(tid) + " " + Names[index] + " is " + Hex(actual) + ", expected " + Hex(Expected[tid][index]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main() {
|
||||
try {
|
||||
const auto device = OpenVulkanTestDevice();
|
||||
if (!device) return VulkanTestSkipped;
|
||||
Run(*device);
|
||||
Check();
|
||||
std::puts("f16 fma rounding tests passed");
|
||||
return 0;
|
||||
} catch (const std::exception& error) {
|
||||
std::cerr << error.what() << '\n';
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -177,6 +177,7 @@ private:
|
|||
bool vDivFixupF16(const RdnaInstruction& inst);
|
||||
bool float16Binary(const RdnaInstruction& inst, IrOpcode opcode, bool reverse);
|
||||
bool float16Ternary(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool mix);
|
||||
IrF32 fmaF16RoundedToOdd(IrF32 lhs, IrF32 rhs, IrF32 addend);
|
||||
bool floatUnary(const RdnaInstruction& inst, IrOpcode opcode);
|
||||
bool floatBinary(const RdnaInstruction& inst, IrOpcode opcode, bool reverse);
|
||||
bool floatTernary(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool mix);
|
||||
|
|
|
|||
|
|
@ -17,13 +17,15 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc
|
|||
const auto translateLane = [&](bool high) -> IrF32 {
|
||||
const IrF32 lhs = readF16LaneAsF32(sourceAt(inst, 0u), high, true);
|
||||
const IrF32 rhs = readF16LaneAsF32(sourceAt(inst, 1u), high, true);
|
||||
const auto ternary = [&](const IrF32& third) -> IrF32 {
|
||||
if (opcode == IrOpcode::FPFma32) return fmaF16RoundedToOdd(lhs, rhs, third);
|
||||
return IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &third.Value()}));
|
||||
};
|
||||
if (accumulator) {
|
||||
const IrF32 acc = readF16LaneAsF32(accumulatorOperand(inst), high, true);
|
||||
return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &acc.Value()})));
|
||||
return applyF32ResultModifiers(inst.destination, ternary(readF16LaneAsF32(accumulatorOperand(inst), high, true)));
|
||||
}
|
||||
if (inst.sourceCount == 3u) {
|
||||
const IrF32 third = readF16LaneAsF32(sourceAt(inst, 2u), high, true);
|
||||
return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &third.Value()})));
|
||||
return applyF32ResultModifiers(inst.destination, ternary(readF16LaneAsF32(sourceAt(inst, 2u), high, true)));
|
||||
}
|
||||
return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value()})));
|
||||
};
|
||||
|
|
@ -214,10 +216,29 @@ bool TranslationContext::float16Ternary(const RdnaInstruction& inst, IrOpcode op
|
|||
const RdnaOperand& operand = accumulator && index == 2u ? accumulatorOperand(inst) : sourceAt(inst, index);
|
||||
args[index] = mix ? &readMixF32(operand).Value() : &readF16AsF32(operand).Value();
|
||||
}
|
||||
if (opcode == IrOpcode::FPFma32 && !mix) {
|
||||
writeF16(inst.destination, fmaF16RoundedToOdd(IrF32(*args[0]), IrF32(*args[1]), IrF32(*args[2])));
|
||||
return true;
|
||||
}
|
||||
writeF16(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {args[0], args[1], args[2]})));
|
||||
return true;
|
||||
}
|
||||
|
||||
IrF32 TranslationContext::fmaF16RoundedToOdd(IrF32 lhs, IrF32 rhs, IrF32 addend) {
|
||||
IrValue& product = ir.Emit(IrOpcode::FPMul32, IrType::F32, {&lhs.Value(), &rhs.Value()});
|
||||
IrValue& sum = ir.Emit(IrOpcode::FPAdd32, IrType::F32, {&product, &addend.Value()});
|
||||
IrValue& addendPart = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&sum, &product});
|
||||
IrValue& productPart = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&sum, &addendPart});
|
||||
IrValue& productError = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&product, &productPart});
|
||||
IrValue& addendError = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&addend.Value(), &addendPart});
|
||||
IrValue& error = ir.Emit(IrOpcode::FPAdd32, IrType::F32, {&productError, &addendError});
|
||||
IrValue& inexact = ir.Emit(IrOpcode::FPOrdNotEqual32, IrType::U1, {&error, &ir.ConstantF32(0.0f)});
|
||||
IrValue& sumBits = ir.BitCastU32(sum);
|
||||
IrValue& sameSign = ir.IEqual(ir.BitwiseAnd(ir.BitwiseXor(sumBits, ir.BitCastU32(error)), ir.Constant(0x80000000u)), ir.Constant(0u));
|
||||
IrValue& towardZero = ir.Select(sameSign, sumBits, ir.ISub(sumBits, ir.Constant(1u)));
|
||||
return IrF32(ir.BitCastF32(ir.Select(inexact, ir.BitwiseOr(towardZero, ir.Constant(1u)), sumBits)));
|
||||
}
|
||||
|
||||
IrU32 TranslationContext::readF16Bits(const RdnaOperand& operand) {
|
||||
const IrU32 source = readF16SourceBits(operand);
|
||||
IrU32 bits(ir.BitwiseAnd((operand.opSel ? ir.ShiftRightLogical(source.Value(), ir.Constant(16u)) : source.Value()), ir.Constant(0xffffu)));
|
||||
|
|
|
|||
|
|
@ -329,7 +329,6 @@ Throughout the project, every function at every stage either **does exactly what
|
|||
- The [shader float mode](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (`FLOAT_MODE`, `DX10_CLAMP`, `IEEE_MODE` and `FP16_OVFL` of `COMPUTE_PGM_RSRC1` and `SPI_SHADER_PGM_RSRC1_PS/GS/HS`, bit positions from Mesa's `gfx10.json` and `gfx103.json`) is decoded into `GuestContext::floatMode` and is part of the shader cache keys and the serialized request (#981). `IEEE_MODE` is read by `v_min_f32`, `v_max_f32`, `v_min3_f32`, `v_max3_f32` and `v_med3_f32`, `IEEE_MODE` and the `FLOAT_MODE` output denormal bits by the output modifier, `DX10_CLAMP` by the clamp modifier, and the whole mode by the 16-bit interpolation instructions (see their entries). With IEEE=1 a signaling NaN operand returns that NaN quieted (the first operand checked first, min3/max3 as two nested pairs, med3 as min3 when any operand is NaN), measured on RDNA2 (gfx1035) over every combination of ±0, ±1, ±inf, denormals, quiet and signaling NaNs with f32 denormals kept and flushed. IEEE=0 and unknown modes keep the previous lowering, which matches IEEE=0. The f16 and f64 min/max/med3, NaN quieting in f16 arithmetic (the unary instructions and the f32 arithmetic follow it, see below), `FP16_OVFL` and `FLOAT_MODE` (f32 denormals flushed before the min/max compare in Astro's Playroom's 0xc0) still follow a single mode, as the entries below record. The mode comes from the registered shader header's `sh_registers`, the same source at registration and at draw/dispatch so the prepared artifact keys match. A title that rewrites RSRC1 at run time, or a raw compute program with no header, compiles with that mode or none.
|
||||
- [v_mullit_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the hardware measured with the MODE register at IEEE=1 and denormals kept (a signaling NaN src0 is quieted only with IEEE=1, as measured at IEEE=0 too; a positive denormal src2 counts as positive): other denormal settings are not modelled.
|
||||
- [v_fma_legacy_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (VOP3 0x140) is assumed to be fused, which has not been measured on hardware. LLVM decodes this encoding as `v_mad_legacy_f32` for gfx1013, and `v_mac_legacy_f32` (VOP2 0x06) was measured to round its product before the add, so 0x140 may do the same.
|
||||
- [v_fma_f16, v_pk_fma_f16 and v_pk_fmac_f16](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) are an f32 FMA rounded to f16, which rounds twice: 0x62e9 * 0xd159 + 0x23ec gives 0xf89e, the hardware 0xf89f.
|
||||
- [Signaling NaN inputs](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) of the f32 and f16 unary instructions (`v_sqrt/rcp/rsq/log/exp/sin/cos/fract/floor/ceil/trunc/rndne/rcp_iflag/frexp_mant`), `v_ldexp_f32/f16`, `v_frexp_mant_f16`, `v_div_fixup_f32/f16` and `v_mullit_f32` are returned quieted only with IEEE=1 and unchanged with IEEE=0, measured on RDNA2 (gfx1035) in both modes; shaders without a known float mode follow IEEE=0. `v_add/sub/subrev/mul/fma/fmac/fmamk/fmaak/mul_legacy/fma_legacy_f32` and `v_fma_mix_f32` with f32 sources choose their NaN result on the bits, measured over 1000 operand triples in both modes: the first NaN source in operand order after its modifiers (the subtrahend negated, as the hardware adds its negation), quieted only with IEEE=1, or the default NaN 0xffc00000 for an invalid operation; in the fma forms an invalid product (inf * 0) gives the default NaN even when src2 is a NaN, and in the legacy forms a zero factor makes the product 0 even when the other factor is a NaN (a NaN src2 of `v_fma_legacy_f32` still wins). The f16 arithmetic, `v_fma_mix_f32` with f16 sources, `v_cubema`, `v_mac_legacy_f32` and the separately rounded mad/mac forms (not measurable on RDNA2) follow the same rule on hardware but their NaN bits are left to the driver. `v_cvt_f16_f32` and `v_dot2_f32_f16` always quiet and `v_div_scale_f32` never does.
|
||||
- [Packed f16 NaN results](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (`v_pk_add/mul/fma/min/max_f16`) follow the hardware measured on RDNA2 (gfx1035) at IEEE=0 and IEEE=1 with f16 round-to-nearest-even and f16 denormals kept: with IEEE=1 a signaling NaN input is quieted and min/max return the other operand only for a quiet NaN; with IEEE=0 NaN inputs pass through unquieted and min/max ignore signaling. In `v_pk_fma_f16` an invalid product (inf * 0) gives the default NaN even when src2 is a NaN. A shader without a known float mode is translated as IEEE=0, the mode titles use; a mode with another f16 rounding or that flushes f16 denormals throws.
|
||||
- [f32 denormals](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follow the shader's `FLOAT_MODE` f32 denormal field (bits 5:4) when it flushes them (0, and shaders without a known float mode, as in the 0xc0 most titles use), measured on RDNA2 (gfx1035) over 1444 rows per instruction at IEEE=0 and IEEE=1. `v_add/sub/subrev/mul/mul_legacy/fma/fmac/fmamk/fmaak/min/max/min3/max3/med3/fract/floor/ceil/frexp_mant/sin/cos_f32` flush their f32 inputs and their result to a zero of the same sign; the f32 compares, `v_cvt_i32/u32/rpi_i32/flr_i32_f32`, `v_frexp_exp_i32_f32`, `v_cvt_f64_f32`, `v_cvt_f16_f32`, `v_cvt_pkrtz_f16_f32`, `v_cvt_pknorm_i16/u16_f32` and `v_cvt_pk_u8_f32` flush their f32 inputs. Tininess is detected before rounding: a product whose exact value is below 2^-126 gives zero even when it rounds to 2^-126, computed exactly from the significands for `v_mul_f32`, `v_mul_legacy_f32`, and for `v_fma_f32`, `v_fmac_f32`, `v_fmamk_f32`, `v_fmaak_f32` and `v_fma_mix_f32` when the addend is zero; with a nonzero addend the fused result is kept, so an exact result just below 2^-126 that rounds to it is not flushed. `v_fma_mix_f32` was measured with f32 sources only; its f16 sources are not flushed. Denormal modes 1 and 2 throw. Not covered, and not measurable on RDNA2 (no such opcodes) or not measured: `v_mac_f32`, `v_mad_f32`, `v_madak_f32` and `v_madmk_f32` (rounded separately), `v_mac_legacy_f32`, `v_dot2_f32_f16`, the division steps (on hardware `v_div_fixup_f32` flushes only src1 and src2, `v_div_fmas_f32` only its result), f32 interpolation and LDS float atomics keep the previous lowering. With f32 denormals kept (3) the results depend on the driver's default denormal behaviour, as no `DenormPreserve` execution mode is requested, and `v_ldexp_f32` flushes in every mode.
|
||||
|
|
|
|||
Loading…
Reference in New Issue