From 066a8fbe3076fadae3d31d3948168db45ff5601d Mon Sep 17 00:00:00 2001 From: Shahul Mubeen A <174251444+shahulmubeen-a@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:55:14 +0530 Subject: [PATCH] perf(shader): store 8- and 16-bit buffer components as bytes in compute A formatted store of an 8- or 16-bit component into a bound buffer patched the dword around it with a compare-exchange loop (writeBuffer in SpirvRuntimeBuffer.cpp). Ghost of Yotei's light-culling compute pass (program 0x800038be00, 240x135 groups per frame) stores millions of such components per dispatch and took 426 ms of GPU time per dispatch on an RTX 4070 Ti SUPER. Compute stores now go through a byte view of the bound buffers (buffers_u8, an aliased declaration of the buffers binding, defined on the first narrow store): one OpStore per byte under StorageBuffer8BitAccess, which the shader device profile already requires. Without the capability, and in every other stage, the compare-exchange path remains: the draw-time validator accepts only u32 and u64 aliased views. Loads, dword stores and stores through GPU-selected descriptors (BDA) are unchanged. Ghost of Yotei (PPSA26344 02.002.000), Windows 11, RTX 4070 Ti SUPER, logo phase: that dispatch 426 -> 10.7 ms of GPU time, 1.9 -> 11 fps. agc_formatted_buffer_byte_stores compiles a formatted 8_8_8_8 store as compute with and without StorageBuffer8BitAccess and as a vertex shader with it, and checks the byte view, the byte stores, the compare-exchange loops and the fallbacks. agc_driver_formatted_store_unaligned executes 8- and 16-bit formatted stores at misaligned offsets, including stores that cross a dword, and checks every byte of the output including the untouched neighbours. Co-Authored-By: Claude Sonnet 5.5 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 7 +- .../tests/FormattedBufferByteStores.cpp | 119 +++++++++++++++ .../execution/FormattedStoreUnaligned.cpp | 135 ++++++++++++++++++ .../SpirvBackend/SpirvEmitterHelpers.hpp | 4 + .../SpirvBackend/SpirvEmitterState.hpp | 1 + .../SpirvBackend/SpirvMemory/SpirvTypes.hpp | 4 + .../src/SpirvMemory/SpirvTypes.cpp | 24 ++++ .../SpirvBackend/src/SpirvRuntimeBuffer.cpp | 40 ++++++ 8 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/FormattedBufferByteStores.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/FormattedStoreUnaligned.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index a4102bea2..b167474f0 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -733,6 +733,11 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_formatted_buffer_byte_stores_tests EXCLUDE_FROM_ALL tests/FormattedBufferByteStores.cpp) +target_link_libraries(agc_formatted_buffer_byte_stores_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_formatted_buffer_byte_stores_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_formatted_buffer_byte_stores_tests) + add_test_executable(agc_shader_disk_cache_tests EXCLUDE_FROM_ALL tests/ShaderDiskCache.cpp) target_include_directories(agc_shader_disk_cache_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_shader_disk_cache_tests PRIVATE shader_recompiler_agc_driver libc) @@ -1212,7 +1217,7 @@ if(BUILD_TESTING) if(WIN32) set_tests_properties(agc_shader_bindless agc_shader_srt_read_patching agc_shader_frontend_pair agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() - foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests}) + foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests}) if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests") continue() endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/FormattedBufferByteStores.cpp b/core/libs/prx/libSceAgcDriver/tests/FormattedBufferByteStores.cpp new file mode 100644 index 000000000..0383aa0ba --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/FormattedBufferByteStores.cpp @@ -0,0 +1,119 @@ +#include "BdaAbi.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; + +void Check(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +SpirvTarget Target(bool byteAccess) { + static constexpr std::array withBytes{ + spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, + }; + static constexpr std::array withoutBytes{spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses}; + static constexpr std::array extensions{ + "SPV_EXT_descriptor_indexing", "SPV_KHR_8bit_storage", "SPV_KHR_float_controls", "SPV_KHR_physical_storage_buffer", + }; + return {0x00401000u, 0x00010300u, 32, BdaAbi::Version, byteAccess ? std::span(withBytes) : std::span(withoutBytes), extensions, false, + {1024, 1024, 64}, 1024, 32768, {}, {}}; +} + +struct Counts { + std::size_t compareExchanges = 0; + std::size_t byteStores = 0; + std::size_t byteTypes = 0; + bool byteCapability = false; + bool byteBuffers = false; +}; + +Counts Count(std::span words) { + Check(words.size() >= 5 && words[0] == spv::MagicNumber, "formatted buffer byte stores: missing SPIR-V module"); + Counts counts; + std::uint32_t byteType = 0; + std::vector bytePointers; + for (std::size_t offset = 5; offset < words.size();) { + const auto count = words[offset] >> spv::WordCountShift; + Check(count != 0 && count <= words.size() - offset, "formatted buffer byte stores: malformed SPIR-V instruction"); + const auto opcode = words[offset] & spv::OpCodeMask; + if (opcode == spv::OpAtomicCompareExchange) ++counts.compareExchanges; + if (opcode == spv::OpCapability && words[offset + 1] == spv::CapabilityStorageBuffer8BitAccess) counts.byteCapability = true; + if (opcode == spv::OpTypeInt && words[offset + 2] == 8u) { + ++counts.byteTypes; + byteType = words[offset + 1]; + } + if (opcode == spv::OpTypePointer && words[offset + 2] == spv::StorageClassStorageBuffer && words[offset + 3] == byteType && byteType != 0) bytePointers.push_back(words[offset + 1]); + if (opcode == spv::OpName) { + const auto name = std::string_view(reinterpret_cast(&words[offset + 2])); + if (name == "buffers_u8") counts.byteBuffers = true; + } + offset += count; + } + std::vector byteChains; + for (std::size_t offset = 5; offset < words.size();) { + const auto count = words[offset] >> spv::WordCountShift; + const auto opcode = words[offset] & spv::OpCodeMask; + if (opcode == spv::OpAccessChain) { + for (const auto pointer : bytePointers) { + if (words[offset + 1] == pointer) byteChains.push_back(words[offset + 2]); + } + } + if (opcode == spv::OpStore) { + for (const auto chain : byteChains) { + if (words[offset + 1] == chain) ++counts.byteStores; + } + } + offset += count; + } + return counts; +} + +Counts Compile(bool byteAccess, bool vertex = false) { + std::vector code{0xe0000000u, 0x80020a00u, 0xbf8c3f70u, 0xe0100000u, 0x80020a00u}; + if (vertex) code.insert(code.end(), {0xf80008cfu, 0x03020100u}); + code.push_back(0xbf810000u); + const std::span words(code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(words)}}}; + const std::array userData{0u, 0u, 0u, 0u, 0x10000000u, 0u, 0x1000u, 0x31016facu, 0x20000000u, 0u, 0x1000u, 0x31038facu}; + RecompileRequest request{}; + request.shader = {vertex ? ShaderStage::Vertex : ShaderStage::Compute, reinterpret_cast(code.data()), words, 0, {}}; + request.target = Target(byteAccess); + request.layout = {0, 0, 0, 128}; + request.context.waveSize = 32; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + if (vertex) request.context.vertex = ShaderVertexStageInfo{}; + else request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.context.memory = memory; + request.useCache = false; + return Count(Recompile(request).spirv.Words()); +} + +} + +int main() { + try { + const auto bytes = Compile(true); + Check(bytes.compareExchanges == 0, "formatted buffer byte stores: a store still patches its dword with " + std::to_string(bytes.compareExchanges) + " compare-exchange loops"); + Check(bytes.byteBuffers && bytes.byteCapability && bytes.byteTypes == 1, "formatted buffer byte stores: the byte view of the buffers is not declared under StorageBuffer8BitAccess"); + Check(bytes.byteStores >= 4, "formatted buffer byte stores: the 8-bit components are stored with " + std::to_string(bytes.byteStores) + " byte stores, expected at least 4"); + const auto dwords = Compile(false); + Check(dwords.compareExchanges > 0 && !dwords.byteBuffers && !dwords.byteCapability, "formatted buffer byte stores: without StorageBuffer8BitAccess the stores do not fall back to the dword patch"); + std::printf("formatted buffer byte stores tests passed (%zu byte stores, %zu compare-exchange loops without the capability)\n", bytes.byteStores, dwords.compareExchanges); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStoreUnaligned.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStoreUnaligned.cpp new file mode 100644 index 000000000..a39cd01e8 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStoreUnaligned.cpp @@ -0,0 +1,135 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t LaneBytes = 64; +constexpr std::uint32_t Fill = 0xaaaaaaaau; + +struct Store { + std::uint32_t format; + std::uint32_t width; + std::uint32_t channels; + std::uint32_t components; + std::uint32_t offset; +}; + +constexpr std::array Stores{{ + {5, 1, 1, 1, 1}, + {11, 2, 1, 1, 3}, + {60, 1, 4, 4, 6}, + {60, 1, 4, 3, 14}, + {11, 2, 1, 1, 19}, + {27, 2, 2, 2, 26}, + {69, 2, 4, 4, 33}, + {18, 1, 2, 2, 45}, +}}; + +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +std::vector Program() { + std::vector code{0x34020084, 0x34060086}; + for (std::uint32_t component = 0; component < Inputs; ++component) code.insert(code.end(), {0xe0301000u | (component * 4u), 0x80000401u | (component << 8u)}); + code.push_back(0xbf8c3f70); + code.insert(code.end(), {0x7e3c02ff, Fill}); + for (std::uint32_t offset = 0; offset < LaneBytes; offset += 4) code.insert(code.end(), {0xe0701000u | offset, 0x80011e03}); + code.push_back(0xbf8c3f70); + for (const auto& store : Stores) { + const std::uint32_t op = 3u + store.components; + code.insert(code.end(), {0xe8001000u | (store.format << 19u) | (op << 16u) | store.offset, 0x80010403}); + } + code.push_back(0xbf810000); + return code; +} + +std::uint32_t Value(std::uint32_t tid, std::uint32_t component) { + const auto mixed = (tid + 1u) * 2654435761u + component * 40503u; + return tid % 5u == 3u ? mixed | 0x10000u : mixed >> 3u & 0xffffu; +} + +std::array Expected(std::uint32_t tid) { + std::array bytes{}; + bytes.fill(static_cast(Fill & 0xffu)); + for (const auto& store : Stores) { + const std::uint32_t width = store.width; + const std::uint32_t limit = width == 1u ? 0xffu : 0xffffu; + for (std::uint32_t component = 0; component < store.channels; ++component) { + const auto value = component < store.components ? std::min(Value(tid, component), limit) : 0u; + for (std::uint32_t byte = 0; byte < width; ++byte) bytes[store.offset + component * width + byte] = static_cast(value >> (8u * byte)); + } + } + return bytes; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t component = 0; component < Inputs; ++component) Input[tid * Inputs + component] = Value(tid, component); + } + Output.fill(0xdeu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size())); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + static const auto code = Program(); + const std::span words(code); + const std::array memory{{{reinterpret_cast(words.data()), std::as_bytes(words)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(words.data()), words, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(words.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto expected = Expected(tid); + for (std::uint32_t byte = 0; byte < LaneBytes; ++byte) { + const auto actual = Output[tid * LaneBytes + byte]; + Require(actual == expected[byte], "formatted store unaligned: lane " + std::to_string(tid) + " byte " + std::to_string(byte) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[byte])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("formatted store unaligned tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index 25813bb5f..546f49828 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -16,6 +16,7 @@ const RdnaImageDimensionInfo& RdnaImageDimensionInfoFor(RdnaImageDimension dimen std::uint32_t TypeVoid(SpirvEmitterState& state); std::uint32_t TypeBool(SpirvEmitterState& state); std::uint32_t TypeBoolVector(SpirvEmitterState& state, std::uint32_t components); +std::uint32_t TypeU8(SpirvEmitterState& state); std::uint32_t TypeU32(SpirvEmitterState& state); std::uint32_t TypeU64(SpirvEmitterState& state); std::uint32_t TypeScalarU64(SpirvEmitterState& state); @@ -33,6 +34,9 @@ std::uint32_t TypeStorageBufferPointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferElementPointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferU64Pointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferU64ElementPointer(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8Block(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8Pointer(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8ElementPointer(SpirvEmitterState& state); std::uint32_t TypePhysicalU32Pointer(SpirvEmitterState& state); std::uint32_t TypePushConstantElementPointer(SpirvEmitterState& state); std::uint32_t TypeU32ArrayPointer(SpirvEmitterState& state, std::uint32_t storageClass, std::uint32_t dwords); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index f13e27c4d..f5d6a95d9 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -107,6 +107,7 @@ struct SpirvEmitterState { std::uint32_t storageBufferVariable = 0; std::uint32_t storageBufferU64Variable = 0; std::uint32_t storageBufferCoherentVariable = 0; + std::uint32_t storageBufferU8Variable = 0; std::vector memoryByteOffsets; std::map, std::uint32_t> formattedBufferFunctions; std::map, std::uint32_t> formattedGpuBufferFunctions; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvTypes.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvTypes.hpp index c1fb1ae9c..bf9ea5769 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvTypes.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvTypes.hpp @@ -9,6 +9,7 @@ namespace ShaderRecompiler { std::uint32_t TypeVoid(SpirvEmitterState& state); std::uint32_t TypeBool(SpirvEmitterState& state); std::uint32_t TypeBoolVector(SpirvEmitterState& state, std::uint32_t components); +std::uint32_t TypeU8(SpirvEmitterState& state); std::uint32_t TypeU32(SpirvEmitterState& state); std::uint32_t TypeU64(SpirvEmitterState& state); std::uint32_t TypeScalarU64(SpirvEmitterState& state); @@ -26,6 +27,9 @@ std::uint32_t TypeStorageBufferPointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferElementPointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferU64Pointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferU64ElementPointer(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8Block(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8Pointer(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8ElementPointer(SpirvEmitterState& state); std::uint32_t TypePhysicalU32Pointer(SpirvEmitterState& state); std::uint32_t TypePushConstantElementPointer(SpirvEmitterState& state); std::uint32_t TypeU32ArrayPointer(SpirvEmitterState& state, std::uint32_t storageClass, std::uint32_t dwords); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp index bdc7ced72..1265fd1b0 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp @@ -25,6 +25,14 @@ std::uint32_t StorageBufferU64Type(SpirvEmitterState& state) { array); } +std::uint32_t StorageBufferU8Type(SpirvEmitterState& state) { + const auto array = state.module.DecoratedType(spv::OpTypeRuntimeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, 1u}}}, TypeU8(state)); + return state.module.DecoratedType(spv::OpTypeStruct, + {{spv::OpDecorate, {spv::DecorationBlock}}, + {spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}}}, + array); +} + } @@ -40,6 +48,10 @@ std::uint32_t TypeBoolVector(SpirvEmitterState& state, std::uint32_t components) return state.module.Type(spv::OpTypeVector, TypeBool(state), components); } +std::uint32_t TypeU8(SpirvEmitterState& state) { + return state.module.Type(spv::OpTypeInt, 8u, 0u); +} + std::uint32_t TypeU32(SpirvEmitterState& state) { return state.module.Type(spv::OpTypeInt, 32u, 0u); } @@ -113,6 +125,18 @@ std::uint32_t TypeStorageBufferU64ElementPointer(SpirvEmitterState& state) { return TypePointer(state, spv::StorageClassStorageBuffer, TypeScalarU64(state)); } +std::uint32_t TypeStorageBufferU8Block(SpirvEmitterState& state) { + return StorageBufferU8Type(state); +} + +std::uint32_t TypeStorageBufferU8Pointer(SpirvEmitterState& state) { + return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferU8Type(state)); +} + +std::uint32_t TypeStorageBufferU8ElementPointer(SpirvEmitterState& state) { + return TypePointer(state, spv::StorageClassStorageBuffer, TypeU8(state)); +} + std::uint32_t TypePhysicalU32Pointer(SpirvEmitterState& state) { return TypePointer(state, spv::StorageClassPhysicalStorageBuffer, TypeU32(state)); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp index 8c65294c3..7f0ae8c42 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -184,6 +184,28 @@ std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruct }); } +std::uint32_t byteBuffers(SpirvEmitterState& state) { + if (state.storageBufferU8Variable != 0u) return state.storageBufferU8Variable; + if (state.program.Resources().stage != IrShaderStage::Compute) return 0u; + if (std::find(state.supportedCapabilities.begin(), state.supportedCapabilities.end(), static_cast(spv::CapabilityStorageBuffer8BitAccess)) == state.supportedCapabilities.end()) return 0u; + const IrBindingLayout& layout = state.program.Metadata().bindings; + const auto binding = std::find_if(layout.descriptors.begin(), layout.descriptors.end(), [](const IrDescriptorBinding& descriptor) { return descriptor.kind == DescriptorBindingKind::Buffers; }); + if (binding == layout.descriptors.end() || state.storageBufferVariable == 0u) return 0u; + state.module.EmitCapability(spv::CapabilityStorageBuffer8BitAccess); + state.module.EmitExtension("SPV_KHR_8bit_storage"); + const auto count = ConstantU32(state, static_cast(binding->resources.size())); + const auto type = state.module.Type(spv::OpTypeArray, TypeStorageBufferU8Block(state), count); + const auto variable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassStorageBuffer, type), spv::StorageClassStorageBuffer); + state.module.AddName(variable, "buffers_u8"); + state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationDescriptorSet, 0u); + state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBinding, NativeBinding(state.program.Resources().stage, DescriptorBindingKind::Buffers)); + state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased); + if (state.storageBufferU64Variable == 0u && state.storageBufferCoherentVariable == 0u) state.module.AddAnnotation(spv::OpDecorate, state.storageBufferVariable, spv::DecorationAliased); + if (state.requirements.coherentBuffers) state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationCoherent); + state.storageBufferU8Variable = variable; + return variable; +} + void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u) { if (context.Memory(instruction).gpuDescriptor) { EmitBdaWrite(context, instruction, address, value, bits); @@ -197,6 +219,24 @@ void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std const auto unaligned = nonzero(state, misalignment); const auto next = EmitAddU32(state, index, ConstantU32(state, 1u)); const auto last = Select(state, u32, unaligned, next, index); + const auto bytes = bits != 32u ? byteBuffers(state) : 0u; + if (bytes != 0u) { + const auto element = ResourceForDescriptor(state, DescriptorBindingKind::Buffers, context.Memory(instruction).resource); + const auto block = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferU8Pointer(state), block, bytes, ConstantU32(state, element)); + EmitIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] { + const auto first = EmitAddU32(state, EmitAddU32(state, Binary(state, spv::OpShiftLeftLogical, u32, index, ConstantU32(state, 2u)), misalignment), EmitAndConstant(state, address, 3u)); + for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) { + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferU8ElementPointer(state), pointer, block, ConstantU32(state, 0u), EmitAddU32(state, first, ConstantU32(state, byte))); + const auto shifted = byte == 0u ? value : Binary(state, spv::OpShiftRightLogical, u32, value, ConstantU32(state, byte * 8u)); + const auto stored = Unary(state, spv::OpUConvert, TypeU8(state), shifted); + if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpStore, pointer, stored, access.memoryAccess); + else state.module.AddFunction(spv::OpStore, pointer, stored); + } + }); + return; + } EmitIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] { auto mask = ConstantU32(state, bits == 32u ? 0xffffffffu : (1u << bits) - 1u); auto masked = Binary(state, spv::OpBitwiseAnd, u32, value, mask);