diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 1789f76d5..8b5909b8e 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -733,6 +733,11 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_formatted_buffer_byte_stores_tests EXCLUDE_FROM_ALL tests/FormattedBufferByteStores.cpp) +target_link_libraries(agc_formatted_buffer_byte_stores_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_formatted_buffer_byte_stores_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_formatted_buffer_byte_stores_tests) + add_test_executable(agc_shader_disk_cache_tests EXCLUDE_FROM_ALL tests/ShaderDiskCache.cpp) target_include_directories(agc_shader_disk_cache_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_shader_disk_cache_tests PRIVATE shader_recompiler_agc_driver libc) @@ -1212,7 +1217,7 @@ if(BUILD_TESTING) if(WIN32) set_tests_properties(agc_shader_bindless agc_shader_srt_read_patching agc_shader_frontend_pair agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() - foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests}) + foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests}) if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests") continue() endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/FormattedBufferByteStores.cpp b/core/libs/prx/libSceAgcDriver/tests/FormattedBufferByteStores.cpp new file mode 100644 index 000000000..0383aa0ba --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/FormattedBufferByteStores.cpp @@ -0,0 +1,119 @@ +#include "BdaAbi.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; + +void Check(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +SpirvTarget Target(bool byteAccess) { + static constexpr std::array withBytes{ + spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, + }; + static constexpr std::array withoutBytes{spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses}; + static constexpr std::array extensions{ + "SPV_EXT_descriptor_indexing", "SPV_KHR_8bit_storage", "SPV_KHR_float_controls", "SPV_KHR_physical_storage_buffer", + }; + return {0x00401000u, 0x00010300u, 32, BdaAbi::Version, byteAccess ? std::span(withBytes) : std::span(withoutBytes), extensions, false, + {1024, 1024, 64}, 1024, 32768, {}, {}}; +} + +struct Counts { + std::size_t compareExchanges = 0; + std::size_t byteStores = 0; + std::size_t byteTypes = 0; + bool byteCapability = false; + bool byteBuffers = false; +}; + +Counts Count(std::span words) { + Check(words.size() >= 5 && words[0] == spv::MagicNumber, "formatted buffer byte stores: missing SPIR-V module"); + Counts counts; + std::uint32_t byteType = 0; + std::vector bytePointers; + for (std::size_t offset = 5; offset < words.size();) { + const auto count = words[offset] >> spv::WordCountShift; + Check(count != 0 && count <= words.size() - offset, "formatted buffer byte stores: malformed SPIR-V instruction"); + const auto opcode = words[offset] & spv::OpCodeMask; + if (opcode == spv::OpAtomicCompareExchange) ++counts.compareExchanges; + if (opcode == spv::OpCapability && words[offset + 1] == spv::CapabilityStorageBuffer8BitAccess) counts.byteCapability = true; + if (opcode == spv::OpTypeInt && words[offset + 2] == 8u) { + ++counts.byteTypes; + byteType = words[offset + 1]; + } + if (opcode == spv::OpTypePointer && words[offset + 2] == spv::StorageClassStorageBuffer && words[offset + 3] == byteType && byteType != 0) bytePointers.push_back(words[offset + 1]); + if (opcode == spv::OpName) { + const auto name = std::string_view(reinterpret_cast(&words[offset + 2])); + if (name == "buffers_u8") counts.byteBuffers = true; + } + offset += count; + } + std::vector byteChains; + for (std::size_t offset = 5; offset < words.size();) { + const auto count = words[offset] >> spv::WordCountShift; + const auto opcode = words[offset] & spv::OpCodeMask; + if (opcode == spv::OpAccessChain) { + for (const auto pointer : bytePointers) { + if (words[offset + 1] == pointer) byteChains.push_back(words[offset + 2]); + } + } + if (opcode == spv::OpStore) { + for (const auto chain : byteChains) { + if (words[offset + 1] == chain) ++counts.byteStores; + } + } + offset += count; + } + return counts; +} + +Counts Compile(bool byteAccess, bool vertex = false) { + std::vector code{0xe0000000u, 0x80020a00u, 0xbf8c3f70u, 0xe0100000u, 0x80020a00u}; + if (vertex) code.insert(code.end(), {0xf80008cfu, 0x03020100u}); + code.push_back(0xbf810000u); + const std::span words(code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(words)}}}; + const std::array userData{0u, 0u, 0u, 0u, 0x10000000u, 0u, 0x1000u, 0x31016facu, 0x20000000u, 0u, 0x1000u, 0x31038facu}; + RecompileRequest request{}; + request.shader = {vertex ? ShaderStage::Vertex : ShaderStage::Compute, reinterpret_cast(code.data()), words, 0, {}}; + request.target = Target(byteAccess); + request.layout = {0, 0, 0, 128}; + request.context.waveSize = 32; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + if (vertex) request.context.vertex = ShaderVertexStageInfo{}; + else request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.context.memory = memory; + request.useCache = false; + return Count(Recompile(request).spirv.Words()); +} + +} + +int main() { + try { + const auto bytes = Compile(true); + Check(bytes.compareExchanges == 0, "formatted buffer byte stores: a store still patches its dword with " + std::to_string(bytes.compareExchanges) + " compare-exchange loops"); + Check(bytes.byteBuffers && bytes.byteCapability && bytes.byteTypes == 1, "formatted buffer byte stores: the byte view of the buffers is not declared under StorageBuffer8BitAccess"); + Check(bytes.byteStores >= 4, "formatted buffer byte stores: the 8-bit components are stored with " + std::to_string(bytes.byteStores) + " byte stores, expected at least 4"); + const auto dwords = Compile(false); + Check(dwords.compareExchanges > 0 && !dwords.byteBuffers && !dwords.byteCapability, "formatted buffer byte stores: without StorageBuffer8BitAccess the stores do not fall back to the dword patch"); + std::printf("formatted buffer byte stores tests passed (%zu byte stores, %zu compare-exchange loops without the capability)\n", bytes.byteStores, dwords.compareExchanges); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStoreUnaligned.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStoreUnaligned.cpp new file mode 100644 index 000000000..a39cd01e8 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStoreUnaligned.cpp @@ -0,0 +1,135 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t LaneBytes = 64; +constexpr std::uint32_t Fill = 0xaaaaaaaau; + +struct Store { + std::uint32_t format; + std::uint32_t width; + std::uint32_t channels; + std::uint32_t components; + std::uint32_t offset; +}; + +constexpr std::array Stores{{ + {5, 1, 1, 1, 1}, + {11, 2, 1, 1, 3}, + {60, 1, 4, 4, 6}, + {60, 1, 4, 3, 14}, + {11, 2, 1, 1, 19}, + {27, 2, 2, 2, 26}, + {69, 2, 4, 4, 33}, + {18, 1, 2, 2, 45}, +}}; + +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +std::vector Program() { + std::vector code{0x34020084, 0x34060086}; + for (std::uint32_t component = 0; component < Inputs; ++component) code.insert(code.end(), {0xe0301000u | (component * 4u), 0x80000401u | (component << 8u)}); + code.push_back(0xbf8c3f70); + code.insert(code.end(), {0x7e3c02ff, Fill}); + for (std::uint32_t offset = 0; offset < LaneBytes; offset += 4) code.insert(code.end(), {0xe0701000u | offset, 0x80011e03}); + code.push_back(0xbf8c3f70); + for (const auto& store : Stores) { + const std::uint32_t op = 3u + store.components; + code.insert(code.end(), {0xe8001000u | (store.format << 19u) | (op << 16u) | store.offset, 0x80010403}); + } + code.push_back(0xbf810000); + return code; +} + +std::uint32_t Value(std::uint32_t tid, std::uint32_t component) { + const auto mixed = (tid + 1u) * 2654435761u + component * 40503u; + return tid % 5u == 3u ? mixed | 0x10000u : mixed >> 3u & 0xffffu; +} + +std::array Expected(std::uint32_t tid) { + std::array bytes{}; + bytes.fill(static_cast(Fill & 0xffu)); + for (const auto& store : Stores) { + const std::uint32_t width = store.width; + const std::uint32_t limit = width == 1u ? 0xffu : 0xffffu; + for (std::uint32_t component = 0; component < store.channels; ++component) { + const auto value = component < store.components ? std::min(Value(tid, component), limit) : 0u; + for (std::uint32_t byte = 0; byte < width; ++byte) bytes[store.offset + component * width + byte] = static_cast(value >> (8u * byte)); + } + } + return bytes; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t component = 0; component < Inputs; ++component) Input[tid * Inputs + component] = Value(tid, component); + } + Output.fill(0xdeu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size())); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + static const auto code = Program(); + const std::span words(code); + const std::array memory{{{reinterpret_cast(words.data()), std::as_bytes(words)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(words.data()), words, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(words.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto expected = Expected(tid); + for (std::uint32_t byte = 0; byte < LaneBytes; ++byte) { + const auto actual = Output[tid * LaneBytes + byte]; + Require(actual == expected[byte], "formatted store unaligned: lane " + std::to_string(tid) + " byte " + std::to_string(byte) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[byte])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("formatted store unaligned tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index 7321f46db..bbc32567d 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -16,6 +16,7 @@ const RdnaImageDimensionInfo& RdnaImageDimensionInfoFor(RdnaImageDimension dimen std::uint32_t TypeVoid(SpirvEmitterState& state); std::uint32_t TypeBool(SpirvEmitterState& state); std::uint32_t TypeBoolVector(SpirvEmitterState& state, std::uint32_t components); +std::uint32_t TypeU8(SpirvEmitterState& state); std::uint32_t TypeU32(SpirvEmitterState& state); std::uint32_t TypeU64(SpirvEmitterState& state); std::uint32_t TypeScalarU64(SpirvEmitterState& state); @@ -33,6 +34,9 @@ std::uint32_t TypeStorageBufferPointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferElementPointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferU64Pointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferU64ElementPointer(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8Block(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8Pointer(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8ElementPointer(SpirvEmitterState& state); std::uint32_t TypePhysicalU32Pointer(SpirvEmitterState& state); std::uint32_t TypePushConstantElementPointer(SpirvEmitterState& state); std::uint32_t TypeU32ArrayPointer(SpirvEmitterState& state, std::uint32_t storageClass, std::uint32_t dwords); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index f13e27c4d..f5d6a95d9 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -107,6 +107,7 @@ struct SpirvEmitterState { std::uint32_t storageBufferVariable = 0; std::uint32_t storageBufferU64Variable = 0; std::uint32_t storageBufferCoherentVariable = 0; + std::uint32_t storageBufferU8Variable = 0; std::vector memoryByteOffsets; std::map, std::uint32_t> formattedBufferFunctions; std::map, std::uint32_t> formattedGpuBufferFunctions; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvTypes.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvTypes.hpp index c1fb1ae9c..bf9ea5769 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvTypes.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvTypes.hpp @@ -9,6 +9,7 @@ namespace ShaderRecompiler { std::uint32_t TypeVoid(SpirvEmitterState& state); std::uint32_t TypeBool(SpirvEmitterState& state); std::uint32_t TypeBoolVector(SpirvEmitterState& state, std::uint32_t components); +std::uint32_t TypeU8(SpirvEmitterState& state); std::uint32_t TypeU32(SpirvEmitterState& state); std::uint32_t TypeU64(SpirvEmitterState& state); std::uint32_t TypeScalarU64(SpirvEmitterState& state); @@ -26,6 +27,9 @@ std::uint32_t TypeStorageBufferPointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferElementPointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferU64Pointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferU64ElementPointer(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8Block(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8Pointer(SpirvEmitterState& state); +std::uint32_t TypeStorageBufferU8ElementPointer(SpirvEmitterState& state); std::uint32_t TypePhysicalU32Pointer(SpirvEmitterState& state); std::uint32_t TypePushConstantElementPointer(SpirvEmitterState& state); std::uint32_t TypeU32ArrayPointer(SpirvEmitterState& state, std::uint32_t storageClass, std::uint32_t dwords); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp index bdc7ced72..1265fd1b0 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp @@ -25,6 +25,14 @@ std::uint32_t StorageBufferU64Type(SpirvEmitterState& state) { array); } +std::uint32_t StorageBufferU8Type(SpirvEmitterState& state) { + const auto array = state.module.DecoratedType(spv::OpTypeRuntimeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, 1u}}}, TypeU8(state)); + return state.module.DecoratedType(spv::OpTypeStruct, + {{spv::OpDecorate, {spv::DecorationBlock}}, + {spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}}}, + array); +} + } @@ -40,6 +48,10 @@ std::uint32_t TypeBoolVector(SpirvEmitterState& state, std::uint32_t components) return state.module.Type(spv::OpTypeVector, TypeBool(state), components); } +std::uint32_t TypeU8(SpirvEmitterState& state) { + return state.module.Type(spv::OpTypeInt, 8u, 0u); +} + std::uint32_t TypeU32(SpirvEmitterState& state) { return state.module.Type(spv::OpTypeInt, 32u, 0u); } @@ -113,6 +125,18 @@ std::uint32_t TypeStorageBufferU64ElementPointer(SpirvEmitterState& state) { return TypePointer(state, spv::StorageClassStorageBuffer, TypeScalarU64(state)); } +std::uint32_t TypeStorageBufferU8Block(SpirvEmitterState& state) { + return StorageBufferU8Type(state); +} + +std::uint32_t TypeStorageBufferU8Pointer(SpirvEmitterState& state) { + return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferU8Type(state)); +} + +std::uint32_t TypeStorageBufferU8ElementPointer(SpirvEmitterState& state) { + return TypePointer(state, spv::StorageClassStorageBuffer, TypeU8(state)); +} + std::uint32_t TypePhysicalU32Pointer(SpirvEmitterState& state) { return TypePointer(state, spv::StorageClassPhysicalStorageBuffer, TypeU32(state)); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp index 8c65294c3..7f0ae8c42 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -184,6 +184,28 @@ std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruct }); } +std::uint32_t byteBuffers(SpirvEmitterState& state) { + if (state.storageBufferU8Variable != 0u) return state.storageBufferU8Variable; + if (state.program.Resources().stage != IrShaderStage::Compute) return 0u; + if (std::find(state.supportedCapabilities.begin(), state.supportedCapabilities.end(), static_cast(spv::CapabilityStorageBuffer8BitAccess)) == state.supportedCapabilities.end()) return 0u; + const IrBindingLayout& layout = state.program.Metadata().bindings; + const auto binding = std::find_if(layout.descriptors.begin(), layout.descriptors.end(), [](const IrDescriptorBinding& descriptor) { return descriptor.kind == DescriptorBindingKind::Buffers; }); + if (binding == layout.descriptors.end() || state.storageBufferVariable == 0u) return 0u; + state.module.EmitCapability(spv::CapabilityStorageBuffer8BitAccess); + state.module.EmitExtension("SPV_KHR_8bit_storage"); + const auto count = ConstantU32(state, static_cast(binding->resources.size())); + const auto type = state.module.Type(spv::OpTypeArray, TypeStorageBufferU8Block(state), count); + const auto variable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassStorageBuffer, type), spv::StorageClassStorageBuffer); + state.module.AddName(variable, "buffers_u8"); + state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationDescriptorSet, 0u); + state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBinding, NativeBinding(state.program.Resources().stage, DescriptorBindingKind::Buffers)); + state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased); + if (state.storageBufferU64Variable == 0u && state.storageBufferCoherentVariable == 0u) state.module.AddAnnotation(spv::OpDecorate, state.storageBufferVariable, spv::DecorationAliased); + if (state.requirements.coherentBuffers) state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationCoherent); + state.storageBufferU8Variable = variable; + return variable; +} + void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u) { if (context.Memory(instruction).gpuDescriptor) { EmitBdaWrite(context, instruction, address, value, bits); @@ -197,6 +219,24 @@ void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std const auto unaligned = nonzero(state, misalignment); const auto next = EmitAddU32(state, index, ConstantU32(state, 1u)); const auto last = Select(state, u32, unaligned, next, index); + const auto bytes = bits != 32u ? byteBuffers(state) : 0u; + if (bytes != 0u) { + const auto element = ResourceForDescriptor(state, DescriptorBindingKind::Buffers, context.Memory(instruction).resource); + const auto block = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferU8Pointer(state), block, bytes, ConstantU32(state, element)); + EmitIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] { + const auto first = EmitAddU32(state, EmitAddU32(state, Binary(state, spv::OpShiftLeftLogical, u32, index, ConstantU32(state, 2u)), misalignment), EmitAndConstant(state, address, 3u)); + for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) { + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferU8ElementPointer(state), pointer, block, ConstantU32(state, 0u), EmitAddU32(state, first, ConstantU32(state, byte))); + const auto shifted = byte == 0u ? value : Binary(state, spv::OpShiftRightLogical, u32, value, ConstantU32(state, byte * 8u)); + const auto stored = Unary(state, spv::OpUConvert, TypeU8(state), shifted); + if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpStore, pointer, stored, access.memoryAccess); + else state.module.AddFunction(spv::OpStore, pointer, stored); + } + }); + return; + } EmitIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] { auto mask = ConstantU32(state, bits == 32u ? 0xffffffffu : (1u << bits) - 1u); auto masked = Binary(state, spv::OpBitwiseAnd, u32, value, mask);