From e4f212b2e2e0fcfb6fa4608e0d1107ef090700dc Mon Sep 17 00:00:00 2001 From: Karan Vora <52519664+karanvora2599@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:14:38 -0400 Subject: [PATCH] perf(shader): convert f16 operands with UnpackHalf2x16 where f16 denormals are kept Since #1463, every f16 operand read decodes sign, exponent and mantissa with integer operations, 8 to 12 instructions more per read than UnpackHalf2x16 on NVIDIA. The decode works around an AMD Windows driver that folds the mask of a byte-selected half into UnpackHalf2x16. A new target flag, float16DenormPreserve, is set when the device reports shaderDenormPreserveFloat16 and denormBehaviorIndependence ALL, the same rule as float64DenormPreserve. With it, EmitF16BitsToF32 uses UnpackHalf2x16 and declares DenormPreserve for 16-bit floats, under which SPV_KHR_float_controls keeps unpacked subnormal halves. NVIDIA returns the default NaN for every NaN half, so the NaN bits are rebuilt from the half: its sign at bit 31, its payload at bits 22:13. The DenormPreserve 16 mode is declared once, shared with the native f32 to f16 conversion. Without the flag (lavapipe; RADV reports 32_BIT_ONLY) the integer decode stays. Halves of fields narrower than 16 bits, such as SDWA byte selects, keep the integer decode on every device. The flag is serialized (request version 17) and part of the cache key. New test agc_driver_f16_to_f32_unpack converts all 65,536 halves, with nonzero high bits, through v_cvt_f32_f16, and byte-selected halves through v_add_f16 SDWA, with each decode the device supports, and compares the bits with an exact decode. agc_driver_f16_to_f32_unpack_spirv checks which decode the SPIR-V uses with and without the flag. --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 5 + .../Execution/src/VulkanDevice.cpp | 5 + .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 1 + .../libSceAgcDriver/tests/ShaderMemory.cpp | 11 +- .../tests/execution/F16ToF32Unpack.cpp | 179 ++++++++++++++++++ core/shader/recompiler/CacheKey.hpp | 1 + .../ControlFlow/src/RequestSerializer.cpp | 6 +- core/shader/recompiler/Recompiler.cpp | 1 + core/shader/recompiler/Recompiler.hpp | 1 + .../include/SpirvBackend/SpirvEmitter.hpp | 1 + .../SpirvBackend/SpirvEmitterHelpers.hpp | 2 + .../SpirvBackend/SpirvEmitterInstructions.hpp | 2 +- .../SpirvBackend/SpirvEmitterState.hpp | 2 + .../src/SpirvAlu/SpirvAluEmitterMath.cpp | 14 +- .../SpirvBackend/src/SpirvEmitter.cpp | 1 + .../SpirvBackend/src/SpirvModuleEmitter.cpp | 21 ++ 16 files changed, 243 insertions(+), 10 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/F16ToF32Unpack.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 371ccc8c2..ed8fddcc2 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1232,6 +1232,11 @@ if(BUILD_TESTING) if(WIN32) set_tests_properties(agc_driver_f64_denorm_preserve_spirv PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test(NAME agc_driver_f16_to_f32_unpack_spirv COMMAND agc_driver_f16_to_f32_unpack_tests --verify-spirv) + set_tests_properties(agc_driver_f16_to_f32_unpack_spirv PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_driver_f16_to_f32_unpack_spirv PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() add_test(NAME agc_structurizer_intersection_off COMMAND agc_structurizer_tests) set_tests_properties(agc_structurizer_intersection_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_DOM_INTERSECTION=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_structurizer_iteration_selection_off COMMAND agc_structurizer_tests) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index acf0d2b6c..0935de780 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -273,6 +273,7 @@ struct VulkanDevice::State { bool shaderClock = false; bool narrowSubgroupClock = false; bool float64DenormPreserve = false; + bool float16DenormPreserve = false; // VK_EXT_descriptor_indexing with non-uniform image array indexing (bindless image tables in // graphics stages, and compute workgroups wider than a wave). bool descriptorIndexing = false; @@ -1167,6 +1168,9 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->capabilities.push_back(spv::CapabilityRoundingModeRTE); state->capabilities.push_back(spv::CapabilityDenormPreserve); } + state->float16DenormPreserve = floatControls.shaderDenormPreserveFloat16 == VK_TRUE && + floatControls.denormBehaviorIndependence == VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_ALL; + if (state->float16DenormPreserve && std::find(state->capabilities.begin(), state->capabilities.end(), static_cast(spv::CapabilityDenormPreserve)) == state->capabilities.end()) state->capabilities.push_back(spv::CapabilityDenormPreserve); deviceExtensions.push_back(VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME); deviceExtensions.push_back(VK_KHR_8BIT_STORAGE_EXTENSION_NAME); state->capabilities.push_back(4448); @@ -3023,6 +3027,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const { target.nonConstantImageOffsets = state->maintenance8; target.narrowSubgroupClock = state->narrowSubgroupClock; target.float64DenormPreserve = state->float64DenormPreserve; + target.float16DenormPreserve = state->float16DenormPreserve; target.fixedPushSlots = state->graphicsPipelineLibrary; target.registrationModules = state->graphicsPipelineLibrary; target.srgbDecodeFormats = state->srgbDecodeFormats; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index f3b6b8c48..6ab145881 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -415,6 +415,7 @@ void verifyKeySensitivity() { changes("the sRGB formats decoded in the shader", [](SampleRequest& sample) { sample.request.target.srgbDecodeFormats = 2u; }); changes("the narrow subgroup clock", [](SampleRequest& sample) { sample.request.target.narrowSubgroupClock = true; }); changes("f64 denormal preservation", [](SampleRequest& sample) { sample.request.target.float64DenormPreserve = true; }); + changes("f16 denormal preservation", [](SampleRequest& sample) { sample.request.target.float16DenormPreserve = true; }); changes("the workgroup size limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSize[2] = 128; }); changes("the invocation limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupInvocations = 512; }); changes("the shared memory limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSharedMemoryBytes = 32768; }); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 66dd9926c..578ab7659 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1826,13 +1826,13 @@ void verifyPixelRequestSerialization() { minimal.context.pixel = ShaderPixelStageInfo{}; minimal.context.pixel->rasterPrimitive = RasterPrimitive::Rectangle; const auto encoded = serializer.Serialize(minimal); - require(requestPrefix(encoded, 8u) == "NVNQQRQAAAA=", "new requests did not use serialization version 20"); + require(requestPrefix(encoded, 8u) == "NVNQQRUAAAA=", "new requests did not use serialization version 21"); require(serializer.Deserialize(encoded).request.context.pixel->rasterPrimitive == RasterPrimitive::Rectangle, "the pixel primitive class did not round-trip"); constexpr std::size_t mappingOffset = 8u + 37u + 18u + 163u; - for (std::size_t bytes = 0; bytes < 20u; ++bytes) { + for (std::size_t bytes = 0; bytes < 21u; ++bytes) { expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source, alpha-to-coverage, stencil reference export flag or primitive class was accepted"); } - for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRUAAAA="}) { + for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRYAAAA="}) { expectFailure([&] { static_cast(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted"); } static constexpr std::string_view version18Request{ @@ -2034,7 +2034,7 @@ void verifyLegacyPixelRequests() { require(!pixel.orderedPixelShader, "a legacy request became a primitive-ordered pixel shader"); require(request.shader.stage == ShaderStage::Fragment && request.shader.code.size() == 1u && request.shader.code[0] == 0xbf810000u && !request.context.vertex.has_value() && request.context.memory.empty(), "legacy guest context was misaligned"); require(request.target.vulkanVersion == 0x00401000u && request.target.spirvVersion == 0x00010300u && request.target.subgroupSize == 64u && request.layout.firstBinding == 11u && request.layout.pushConstantSizeBytes == 128u, "legacy target or binding layout was misaligned"); - require(request.useCache == (version == 1u) && request.target.nonConstantImageOffsets == (version >= 6u) && request.target.srgbDecodeFormats == 0u && !request.target.narrowSubgroupClock && !request.target.float64DenormPreserve && !request.target.registrationModules, "legacy request trailer was misread"); + require(request.useCache == (version == 1u) && request.target.nonConstantImageOffsets == (version >= 6u) && request.target.srgbDecodeFormats == 0u && !request.target.narrowSubgroupClock && !request.target.float64DenormPreserve && !request.target.registrationModules && !request.target.float16DenormPreserve, "legacy request trailer was misread"); const auto upgraded = serializer.Deserialize(serializer.Serialize(request)); require(upgraded.request.context.pixel->targetExportMapping == pixel.targetExportMapping && RecompileCacheKey::ContextHash(upgraded.request) == RecompileCacheKey::ContextHash(request), "upgrading a legacy capture changed its pixel mapping"); } @@ -3642,6 +3642,9 @@ int main(int argc, char** argv) { RecompileCacheKey::Build(uncached, plainKey); RecompileCacheKey::Build(registrationModules, registrationKey); require(plainKey != registrationKey, "registration modules did not change the cache key"); + auto float16Denormals = uncached; + float16Denormals.target.float16DenormPreserve = true; + require(RequestSerializer{}.Deserialize(RequestSerializer{}.Serialize(float16Denormals)).request.target.float16DenormPreserve, "f16 denormal preservation was lost in serialization"); auto changedLayout = request; changedLayout.layout.pushConstantSizeBytes = 64; require(!Recompile(changedLayout).cacheHit, "binding layout change reused an incompatible variant"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F16ToF32Unpack.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F16ToF32Unpack.cpp new file mode 100644 index 000000000..7c2567a68 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F16ToF32Unpack.cpp @@ -0,0 +1,179 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Halves = 1u << 16u; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array ConvertCode{ + 0x34020082, 0x8f098708, 0x4a020209, 0xe0301000, 0x80000401, 0xbf8c3f70, 0x7e0a1704, 0xe0701000, + 0x80010501, 0xbf810000, +}; + +alignas(256) constexpr std::array ByteCode{ + 0x34020082, 0x8f098708, 0x4a020209, 0xe0301000, 0x80000401, 0x7e0e02ff, 0x00008000, 0xbf8c3f70, + 0x640c0ef9, 0x06010604, 0xe0701000, 0x80010601, 0xbf810000, +}; + +std::uint32_t HalfToFloatBits(std::uint32_t half) { + const std::uint32_t sign = (half & 0x8000u) << 16u; + const std::uint32_t exponent = (half >> 10u) & 0x1fu; + const std::uint32_t mantissa = half & 0x3ffu; + if (exponent == 0x1fu) return sign | 0x7f800000u | (mantissa << 13u); + if (exponent != 0u) return sign | ((exponent + 112u) << 23u) | (mantissa << 13u); + return sign | std::bit_cast(static_cast(mantissa) * 0x1p-24f); +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +std::vector UserData() { + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + return userData; +} + +ShaderRecompiler::RecompileRequest Request(std::span code, const ShaderRecompiler::SpirvTarget& target, std::span userData, std::span memory) { + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {true, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + target, + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}; + return request; +} + +struct F16Decode { + std::uint32_t unpacks = 0; + bool denormPreserve16 = false; +}; + +F16Decode ScanF16Decode(std::span words) { + F16Decode decode; + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + const auto opcode = words[cursor] & 0xffffu; + Require(count != 0 && cursor + count <= words.size(), "f16 to f32 unpack: malformed SPIR-V"); + if (opcode == spv::OpExecutionMode && count == 4 && words[cursor + 2] == spv::ExecutionModeDenormPreserve && words[cursor + 3] == 16u) decode.denormPreserve16 = true; + if (opcode == spv::OpExtInst && count >= 5 && words[cursor + 4] == GLSLstd450UnpackHalf2x16) ++decode.unpacks; + cursor += count; + } + return decode; +} + +void VerifySpirv() { + constexpr std::array capabilities{ + spv::CapabilityShader, spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, + spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, + spv::CapabilityDenormPreserve, spv::CapabilitySignedZeroInfNanPreserve, + }; + constexpr std::array extensions{ + "SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage", "SPV_KHR_float_controls", + "SPV_KHR_storage_buffer_storage_class", + }; + const auto userData = UserData(); + ShaderRecompiler::SpirvTarget target{}; + target.vulkanVersion = 0x00403000u; + target.spirvVersion = 0x00010600u; + target.subgroupSize = 32; + target.bdaAbiVersion = 1; + target.supportedCapabilities = capabilities; + target.supportedExtensions = extensions; + target.maxWorkgroupSize = {1024, 1024, 64}; + target.maxWorkgroupInvocations = 1024; + target.maxWorkgroupSharedMemoryBytes = 49152; + for (const std::span code : {std::span(ConvertCode), std::span(ByteCode)}) { + const std::string program = code.size() == ConvertCode.size() ? "v_cvt_f32_f16" : "v_add_f16 src0_sel:BYTE_1"; + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + for (const bool preserve : {false, true}) { + target.float16DenormPreserve = preserve; + const std::string flag = std::string(" with float16DenormPreserve=") + (preserve ? "true" : "false"); + const auto decode = ScanF16Decode(ShaderRecompiler::Recompile(Request(code, target, userData, memory)).spirv.Words()); + Require(decode.denormPreserve16 == preserve, "f16 to f32 unpack: " + program + (preserve ? " lacks" : " has") + " DenormPreserve 16" + flag); + Require(decode.unpacks == (preserve ? 1u : 0u), "f16 to f32 unpack: " + program + " has " + std::to_string(decode.unpacks) + " UnpackHalf2x16" + flag); + } + } +} + +void Dispatch(AgcDriver::VulkanDevice& device, std::span code, const ShaderRecompiler::SpirvTarget& target) { + const auto userData = UserData(); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const auto result = ShaderRecompiler::Recompile(Request(code, target, userData, memory)); + Output.fill(0xdeadbeefu); + device.Dispatch(result, Halves / Threads, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Run(AgcDriver::VulkanDevice& device, const ShaderRecompiler::SpirvTarget& target, const char* name) { + Dispatch(device, ConvertCode, target); + for (std::uint32_t lane = 0; lane < Halves; ++lane) { + Require(Output[lane] == HalfToFloatBits(lane), std::string("f16 to f32 unpack (") + name + "): v_cvt_f32_f16 of " + Hex(Input[lane]) + " is " + Hex(Output[lane]) + ", expected " + Hex(HalfToFloatBits(lane))); + } + Dispatch(device, ByteCode, target); + for (std::uint32_t lane = 0; lane < Halves; ++lane) { + const std::uint32_t selected = (Input[lane] >> 8u) & 0xffu; + Require((Output[lane] & 0xffffu) == selected, std::string("f16 to f32 unpack (") + name + "): v_add_f16 src0_sel:BYTE_1 of " + Hex(Input[lane]) + " and -0 is " + Hex(Output[lane]) + ", expected " + Hex(selected)); + } +} + +} + +int main(int argc, char** argv) { + try { + if (argc == 2 && std::string_view(argv[1]) == "--verify-spirv") { + VerifySpirv(); + std::puts("f16 to f32 conversions use UnpackHalf2x16 only under DenormPreserve 16"); + return 0; + } + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + for (std::uint32_t lane = 0; lane < Halves; ++lane) Input[lane] = lane | ((lane ^ 0xa5a5u) << 16u); + auto target = device->Target(); + const bool preserve = target.float16DenormPreserve; + target.float16DenormPreserve = false; + Run(*device, target, "integer decode"); + if (preserve) { + target.float16DenormPreserve = true; + Run(*device, target, "UnpackHalf2x16"); + } + std::printf("f16 to f32 unpack tests passed (%u halves, %s)\n", Halves, preserve ? "both decodes" : "integer decode only"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index 19c4d46bb..89c6cf331 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -245,6 +245,7 @@ private: append(key, value.registrationModules); append(key, value.registrationWordLimit); append(key, value.vertexSubgroups); + append(key, value.float16DenormPreserve); } }; diff --git a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp index 46cc45c52..d2ae965bb 100644 --- a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp +++ b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp @@ -748,7 +748,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const std::string buffer; Writer writer(buffer); writer.WriteU32(0x41505335u); - writer.WriteU32(20u); + writer.WriteU32(21u); writeShaderBinary(writer, request.shader); writeGuestContext(writer, request.context); writeSpirvTarget(writer, request.target); @@ -771,6 +771,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const writer.WriteBool(request.target.registrationModules); writer.WriteU32(request.target.registrationWordLimit); writer.WriteBool(request.target.vertexSubgroups); + writer.WriteBool(request.target.float16DenormPreserve); return base64Encode(buffer); } @@ -779,7 +780,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const Reader reader(decoded); if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature"); const auto version = reader.ReadU32(); - if (version < 1u || version > 20u) throw std::runtime_error("unsupported recompile request serialization version"); + if (version < 1u || version > 21u) throw std::runtime_error("unsupported recompile request serialization version"); DeserializedRequest result{}; result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader); result.request.context = readGuestContext(reader, result, version); @@ -803,6 +804,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const if (version >= 17u) result.request.target.registrationModules = reader.ReadBool(); if (version >= 17u) result.request.target.registrationWordLimit = reader.ReadU32(); if (version >= 18u) result.request.target.vertexSubgroups = reader.ReadBool(); + if (version >= 21u) result.request.target.float16DenormPreserve = reader.ReadBool(); return result; } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index c66b7726b..e66bc8460 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -438,6 +438,7 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra targetOptions.float64DenormPreserve = request.target.float64DenormPreserve; targetOptions.registrationModules = registration; targetOptions.fragmentDescriptorSet = request.target.registrationModules; + targetOptions.float16DenormPreserve = request.target.float16DenormPreserve; constexpr SpirvEmitter spirvEmitter; CompiledShaderArtifact result; diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 4ec0fc73f..4c6c6130a 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -242,6 +242,7 @@ struct SpirvTarget { bool registrationModules = false; std::uint32_t registrationWordLimit = 200000u; bool vertexSubgroups = true; + bool float16DenormPreserve = false; }; struct BindingLayout { diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp index 65cb02117..9d22da588 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp @@ -23,6 +23,7 @@ struct SpirvTargetOptions { bool float64DenormPreserve = false; bool registrationModules = false; bool fragmentDescriptorSet = false; + bool float16DenormPreserve = false; }; class SpirvEmitter { diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index eceb3b3d6..7d861c62c 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -133,7 +133,9 @@ std::uint32_t EmitClassMaskF32(SpirvEmitterState& state, std::uint32_t value, st std::uint32_t EmitMinMaxF32Value(SpirvEmitterState& state, std::uint32_t lhs, std::uint32_t rhs, bool maxValue); std::uint32_t EmitFlushF32DenormToSignedZero(SpirvEmitterState& state, std::uint32_t value); std::uint32_t EmitTrigCycleF32(SpirvEmitterState& state, std::uint32_t src, bool preserveSignedZero); +void EmitF16DenormPreserve(SpirvEmitterState& state); std::uint32_t EmitF16BitsToF32(SpirvEmitterState& state, std::uint32_t bits); +std::uint32_t EmitF16BitsToF32Decoded(SpirvEmitterState& state, std::uint32_t bits); void EmitProgram(SpirvEmitterState& state); void DefineGetBdaPointer(SpirvEmitterState& state); void DefineDot2F32F16Function(SpirvEmitterState& state); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp index 3c5550ac8..0ad1aefdb 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp @@ -211,7 +211,7 @@ std::uint32_t EmitBitCastF16U16(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitConvertU32U16(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitConvertU32U8(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitBitCastU32F32(SpirvEmitterState& state, std::uint32_t arg0); -std::uint32_t EmitConvertF32F16(SpirvEmitterState& state, std::uint32_t bits); +std::uint32_t EmitConvertF32F16(SpirvEmitterState& state, const IrValue& inst, std::uint32_t bits); std::uint32_t EmitCompositeExtractU32x2(SpirvEmitterState& state, std::uint32_t arg0, const IrValue* arg1); std::uint32_t EmitCompositeExtractU32x3(SpirvEmitterState& state, std::uint32_t arg0, const IrValue* arg1); std::uint32_t EmitCompositeExtractU32x4(SpirvEmitterState& state, std::uint32_t arg0, const IrValue* arg1); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index a26c20a54..7ffafec22 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -93,6 +93,7 @@ struct SpirvEmitterState { bool float64DenormPreserve = false; bool registrationModules = false; bool fragmentDescriptorSet = false; + bool float16DenormPreserve = false; std::unordered_set sharedLaneValues; std::uint32_t laneHalf = 0; // The target's SPIR-V version and what the device accepts, for capabilities an emitter adds @@ -123,6 +124,7 @@ struct SpirvEmitterState { std::array bdaByteWriteFunctions {}; std::uint32_t bdaFaultFunction = 0; bool nativeF16ModesEmitted = false; + bool f16DenormPreserveEmitted = false; std::uint32_t dot2F32F16Function = 0; std::array, 2> bdaDwordReadFunctions {}; std::array, 2> bdaSpanReadFunctions {}; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp index 9936d3199..6b4e31728 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp @@ -1412,7 +1412,7 @@ static void EmitNativeF16Modes(SpirvEmitterState& state) { state.module.EmitCapability(spv::CapabilityDenormPreserve); if (!state.nativeF16ModesEmitted) { state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeRoundingModeRTE, 16u); - state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeDenormPreserve, 16u); + EmitF16DenormPreserve(state); state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeSignedZeroInfNanPreserve, 16u); state.nativeF16ModesEmitted = true; } @@ -1544,8 +1544,16 @@ std::uint32_t EmitBitCastU32F32(SpirvEmitterState& state, std::uint32_t arg0) { return EmitBitCastU16F16(state, arg0); } -std::uint32_t EmitConvertF32F16(SpirvEmitterState& state, std::uint32_t bits) { - return EmitF16BitsToF32(state, bits); +static bool NarrowF16Field(const IrValue& half) { + const IrValue* value = half.Resolve(); + while (value->Opcode() == IrOpcode::BitCastF16U16 || value->Opcode() == IrOpcode::ConvertU16U32) value = value->Argument(0)->Resolve(); + if (value->Opcode() != IrOpcode::BitFieldUExtract && value->Opcode() != IrOpcode::BitFieldSExtract) return false; + const IrValue* count = value->Argument(2)->Resolve(); + return !count->HasImmediate() || count->ImmediateU32() < 16u; +} + +std::uint32_t EmitConvertF32F16(SpirvEmitterState& state, const IrValue& inst, std::uint32_t bits) { + return NarrowF16Field(*inst.Argument(0)) ? EmitF16BitsToF32Decoded(state, bits) : EmitF16BitsToF32(state, bits); } std::uint32_t EmitCompositeExtractU32x2(SpirvEmitterState& state, std::uint32_t arg0, const IrValue* arg1) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index 3db76e6e7..2bd877635 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -252,6 +252,7 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh state.float64DenormPreserve = target.float64DenormPreserve; state.registrationModules = target.registrationModules; state.fragmentDescriptorSet = target.fragmentDescriptorSet; + state.float16DenormPreserve = target.float16DenormPreserve; state.hostSubgroupSize = target.subgroupSize; if (HostSubgroupSingleLane(state)) { state.requirements.subgroupBallot = false; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index 2356103ee..a037b5626 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -988,7 +988,28 @@ std::uint32_t EmitTrigCycleF32(SpirvEmitterState& state, std::uint32_t src, bool return result; } +void EmitF16DenormPreserve(SpirvEmitterState& state) { + if (state.f16DenormPreserveEmitted) return; + state.module.EmitCapability(spv::CapabilityDenormPreserve); + state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeDenormPreserve, 16u); + state.f16DenormPreserveEmitted = true; +} + std::uint32_t EmitF16BitsToF32(SpirvEmitterState& state, std::uint32_t bits) { + if (!state.float16DenormPreserve) return EmitF16BitsToF32Decoded(state, bits); + EmitF16DenormPreserve(state); + const auto u32 = TypeU32(state); + const auto f32 = TypeF32(state); + const auto unpacked = state.module.AllocateId(); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), GLSLstd450UnpackHalf2x16, bits); + state.module.AddFunction(spv::OpCompositeExtract, f32, value, unpacked, 0u); + const auto signAndPayload = Binary(state, spv::OpShiftRightArithmetic, u32, Binary(state, spv::OpShiftLeftLogical, u32, bits, ConstantU32(state, 16u)), ConstantU32(state, 3u)); + const auto nan = Unary(state, spv::OpBitcast, f32, Binary(state, spv::OpBitwiseOr, u32, signAndPayload, ConstantU32(state, 0x7f800000u))); + return Select(state, f32, Unary(state, spv::OpIsNan, TypeBool(state), value), nan, value); +} + +std::uint32_t EmitF16BitsToF32Decoded(SpirvEmitterState& state, std::uint32_t bits) { const auto u32 = TypeU32(state); const auto f32 = TypeF32(state); const auto op = [&](spv::Op opcode, std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, opcode, u32, lhs, rhs); };