Merge pull request #4250 from shahulmubeen-a/perf/shader-byte-buffer-stores

perf(shader): store 8- and 16-bit buffer components as bytes

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -733,6 +733,11 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_formatted_buffer_byte_stores_tests EXCLUDE_FROM_ALL tests/FormattedBufferByteStores.cpp)
+target_link_libraries(agc_formatted_buffer_byte_stores_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_formatted_buffer_byte_stores_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_formatted_buffer_byte_stores_tests)
+
 add_test_executable(agc_shader_disk_cache_tests EXCLUDE_FROM_ALL tests/ShaderDiskCache.cpp)
 target_include_directories(agc_shader_disk_cache_tests PRIVATE ${LIBS_INCLUDE_DIR})
 target_link_libraries(agc_shader_disk_cache_tests PRIVATE shader_recompiler_agc_driver libc)
@@ -1212,7 +1217,7 @@ if(BUILD_TESTING)
     if(WIN32)
         set_tests_properties(agc_shader_bindless agc_shader_srt_read_patching agc_shader_frontend_pair agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
-    foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests})
+    foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests})
         if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests")
             continue()
         endif()

core/libs/prx/libSceAgcDriver/tests/FormattedBufferByteStores.cpp

@@ -0,0 +1,119 @@
+#include "BdaAbi.hpp"
+#include "Recompiler.hpp"
+#include <array>
+#include <cstdio>
+#include <exception>
+#include <span>
+#include <spirv/unified1/spirv.hpp>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace {
+
+using namespace ShaderRecompiler;
+
+void Check(bool condition, const std::string& message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+SpirvTarget Target(bool byteAccess) {
+    static constexpr std::array<std::uint32_t, 3> withBytes{
+        spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess,
+    };
+    static constexpr std::array<std::uint32_t, 2> withoutBytes{spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses};
+    static constexpr std::array<std::string_view, 4> extensions{
+        "SPV_EXT_descriptor_indexing", "SPV_KHR_8bit_storage", "SPV_KHR_float_controls", "SPV_KHR_physical_storage_buffer",
+    };
+    return {0x00401000u, 0x00010300u, 32, BdaAbi::Version, byteAccess ? std::span<const std::uint32_t>(withBytes) : std::span<const std::uint32_t>(withoutBytes), extensions, false,
+            {1024, 1024, 64}, 1024, 32768, {}, {}};
+}
+
+struct Counts {
+    std::size_t compareExchanges = 0;
+    std::size_t byteStores = 0;
+    std::size_t byteTypes = 0;
+    bool byteCapability = false;
+    bool byteBuffers = false;
+};
+
+Counts Count(std::span<const std::uint32_t> words) {
+    Check(words.size() >= 5 && words[0] == spv::MagicNumber, "formatted buffer byte stores: missing SPIR-V module");
+    Counts counts;
+    std::uint32_t byteType = 0;
+    std::vector<std::uint32_t> bytePointers;
+    for (std::size_t offset = 5; offset < words.size();) {
+        const auto count = words[offset] >> spv::WordCountShift;
+        Check(count != 0 && count <= words.size() - offset, "formatted buffer byte stores: malformed SPIR-V instruction");
+        const auto opcode = words[offset] & spv::OpCodeMask;
+        if (opcode == spv::OpAtomicCompareExchange) ++counts.compareExchanges;
+        if (opcode == spv::OpCapability && words[offset + 1] == spv::CapabilityStorageBuffer8BitAccess) counts.byteCapability = true;
+        if (opcode == spv::OpTypeInt && words[offset + 2] == 8u) {
+            ++counts.byteTypes;
+            byteType = words[offset + 1];
+        }
+        if (opcode == spv::OpTypePointer && words[offset + 2] == spv::StorageClassStorageBuffer && words[offset + 3] == byteType && byteType != 0) bytePointers.push_back(words[offset + 1]);
+        if (opcode == spv::OpName) {
+            const auto name = std::string_view(reinterpret_cast<const char*>(&words[offset + 2]));
+            if (name == "buffers_u8") counts.byteBuffers = true;
+        }
+        offset += count;
+    }
+    std::vector<std::uint32_t> byteChains;
+    for (std::size_t offset = 5; offset < words.size();) {
+        const auto count = words[offset] >> spv::WordCountShift;
+        const auto opcode = words[offset] & spv::OpCodeMask;
+        if (opcode == spv::OpAccessChain) {
+            for (const auto pointer : bytePointers) {
+                if (words[offset + 1] == pointer) byteChains.push_back(words[offset + 2]);
+            }
+        }
+        if (opcode == spv::OpStore) {
+            for (const auto chain : byteChains) {
+                if (words[offset + 1] == chain) ++counts.byteStores;
+            }
+        }
+        offset += count;
+    }
+    return counts;
+}
+
+Counts Compile(bool byteAccess, bool vertex = false) {
+    std::vector<std::uint32_t> code{0xe0000000u, 0x80020a00u, 0xbf8c3f70u, 0xe0100000u, 0x80020a00u};
+    if (vertex) code.insert(code.end(), {0xf80008cfu, 0x03020100u});
+    code.push_back(0xbf810000u);
+    const std::span<const std::uint32_t> words(code);
+    const std::array<MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(words)}}};
+    const std::array<std::uint32_t, 12> userData{0u, 0u, 0u, 0u, 0x10000000u, 0u, 0x1000u, 0x31016facu, 0x20000000u, 0u, 0x1000u, 0x31038facu};
+    RecompileRequest request{};
+    request.shader = {vertex ? ShaderStage::Vertex : ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), words, 0, {}};
+    request.target = Target(byteAccess);
+    request.layout = {0, 0, 0, 128};
+    request.context.waveSize = 32;
+    request.context.userDataBaseRegister = 0;
+    request.context.userData = userData;
+    if (vertex) request.context.vertex = ShaderVertexStageInfo{};
+    else request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
+    request.context.memory = memory;
+    request.useCache = false;
+    return Count(Recompile(request).spirv.Words());
+}
+
+}
+
+int main() {
+    try {
+        const auto bytes = Compile(true);
+        Check(bytes.compareExchanges == 0, "formatted buffer byte stores: a store still patches its dword with " + std::to_string(bytes.compareExchanges) + " compare-exchange loops");
+        Check(bytes.byteBuffers && bytes.byteCapability && bytes.byteTypes == 1, "formatted buffer byte stores: the byte view of the buffers is not declared under StorageBuffer8BitAccess");
+        Check(bytes.byteStores >= 4, "formatted buffer byte stores: the 8-bit components are stored with " + std::to_string(bytes.byteStores) + " byte stores, expected at least 4");
+        const auto dwords = Compile(false);
+        Check(dwords.compareExchanges > 0 && !dwords.byteBuffers && !dwords.byteCapability, "formatted buffer byte stores: without StorageBuffer8BitAccess the stores do not fall back to the dword patch");
+        std::printf("formatted buffer byte stores tests passed (%zu byte stores, %zu compare-exchange loops without the capability)\n", bytes.byteStores, dwords.compareExchanges);
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "%s\n", error.what());
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/FormattedStoreUnaligned.cpp

@@ -0,0 +1,135 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t LaneBytes = 64;
+constexpr std::uint32_t Fill = 0xaaaaaaaau;
+
+struct Store {
+    std::uint32_t format;
+    std::uint32_t width;
+    std::uint32_t channels;
+    std::uint32_t components;
+    std::uint32_t offset;
+};
+
+constexpr std::array<Store, 8> Stores{{
+    {5, 1, 1, 1, 1},
+    {11, 2, 1, 1, 3},
+    {60, 1, 4, 4, 6},
+    {60, 1, 4, 3, 14},
+    {11, 2, 1, 1, 19},
+    {27, 2, 2, 2, 26},
+    {69, 2, 4, 4, 33},
+    {18, 1, 2, 2, 45},
+}};
+
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint8_t, Threads * LaneBytes> Output{};
+
+std::vector<std::uint32_t> Program() {
+    std::vector<std::uint32_t> code{0x34020084, 0x34060086};
+    for (std::uint32_t component = 0; component < Inputs; ++component) code.insert(code.end(), {0xe0301000u | (component * 4u), 0x80000401u | (component << 8u)});
+    code.push_back(0xbf8c3f70);
+    code.insert(code.end(), {0x7e3c02ff, Fill});
+    for (std::uint32_t offset = 0; offset < LaneBytes; offset += 4) code.insert(code.end(), {0xe0701000u | offset, 0x80011e03});
+    code.push_back(0xbf8c3f70);
+    for (const auto& store : Stores) {
+        const std::uint32_t op = 3u + store.components;
+        code.insert(code.end(), {0xe8001000u | (store.format << 19u) | (op << 16u) | store.offset, 0x80010403});
+    }
+    code.push_back(0xbf810000);
+    return code;
+}
+
+std::uint32_t Value(std::uint32_t tid, std::uint32_t component) {
+    const auto mixed = (tid + 1u) * 2654435761u + component * 40503u;
+    return tid % 5u == 3u ? mixed | 0x10000u : mixed >> 3u & 0xffffu;
+}
+
+std::array<std::uint8_t, LaneBytes> Expected(std::uint32_t tid) {
+    std::array<std::uint8_t, LaneBytes> bytes{};
+    bytes.fill(static_cast<std::uint8_t>(Fill & 0xffu));
+    for (const auto& store : Stores) {
+        const std::uint32_t width = store.width;
+        const std::uint32_t limit = width == 1u ? 0xffu : 0xffffu;
+        for (std::uint32_t component = 0; component < store.channels; ++component) {
+            const auto value = component < store.components ? std::min(Value(tid, component), limit) : 0u;
+            for (std::uint32_t byte = 0; byte < width; ++byte) bytes[store.offset + component * width + byte] = static_cast<std::uint8_t>(value >> (8u * byte));
+        }
+    }
+    return bytes;
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        for (std::uint32_t component = 0; component < Inputs; ++component) Input[tid * Inputs + component] = Value(tid, component);
+    }
+    Output.fill(0xdeu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size()));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    static const auto code = Program();
+    const std::span<const std::uint32_t> words(code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(words.data()), std::as_bytes(words)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(words.data()), words, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(words.data()));
+    device.WaitIdle();
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const auto expected = Expected(tid);
+        for (std::uint32_t byte = 0; byte < LaneBytes; ++byte) {
+            const auto actual = Output[tid * LaneBytes + byte];
+            Require(actual == expected[byte], "formatted store unaligned: lane " + std::to_string(tid) + " byte " + std::to_string(byte) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[byte]));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device);
+        Check();
+        std::puts("formatted store unaligned tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp

@@ -16,6 +16,7 @@ const RdnaImageDimensionInfo& RdnaImageDimensionInfoFor(RdnaImageDimension dimen
 std::uint32_t TypeVoid(SpirvEmitterState& state);
 std::uint32_t TypeBool(SpirvEmitterState& state);
 std::uint32_t TypeBoolVector(SpirvEmitterState& state, std::uint32_t components);
+std::uint32_t TypeU8(SpirvEmitterState& state);
 std::uint32_t TypeU32(SpirvEmitterState& state);
 std::uint32_t TypeU64(SpirvEmitterState& state);
 std::uint32_t TypeScalarU64(SpirvEmitterState& state);
@@ -33,6 +34,9 @@ std::uint32_t TypeStorageBufferPointer(SpirvEmitterState& state);
 std::uint32_t TypeStorageBufferElementPointer(SpirvEmitterState& state);
 std::uint32_t TypeStorageBufferU64Pointer(SpirvEmitterState& state);
 std::uint32_t TypeStorageBufferU64ElementPointer(SpirvEmitterState& state);
+std::uint32_t TypeStorageBufferU8Block(SpirvEmitterState& state);
+std::uint32_t TypeStorageBufferU8Pointer(SpirvEmitterState& state);
+std::uint32_t TypeStorageBufferU8ElementPointer(SpirvEmitterState& state);
 std::uint32_t TypePhysicalU32Pointer(SpirvEmitterState& state);
 std::uint32_t TypePushConstantElementPointer(SpirvEmitterState& state);
 std::uint32_t TypeU32ArrayPointer(SpirvEmitterState& state, std::uint32_t storageClass, std::uint32_t dwords);

core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp

@@ -107,6 +107,7 @@ struct SpirvEmitterState {
     std::uint32_t storageBufferVariable = 0;
     std::uint32_t storageBufferU64Variable = 0;
     std::uint32_t storageBufferCoherentVariable = 0;
+    std::uint32_t storageBufferU8Variable = 0;
     std::vector<std::uint32_t> memoryByteOffsets;
     std::map<std::array<std::uint32_t, 6>, std::uint32_t> formattedBufferFunctions;
     std::map<std::array<std::uint32_t, 6>, std::uint32_t> formattedGpuBufferFunctions;

core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvTypes.hpp

@@ -9,6 +9,7 @@ namespace ShaderRecompiler {
 std::uint32_t TypeVoid(SpirvEmitterState& state);
 std::uint32_t TypeBool(SpirvEmitterState& state);
 std::uint32_t TypeBoolVector(SpirvEmitterState& state, std::uint32_t components);
+std::uint32_t TypeU8(SpirvEmitterState& state);
 std::uint32_t TypeU32(SpirvEmitterState& state);
 std::uint32_t TypeU64(SpirvEmitterState& state);
 std::uint32_t TypeScalarU64(SpirvEmitterState& state);
@@ -26,6 +27,9 @@ std::uint32_t TypeStorageBufferPointer(SpirvEmitterState& state);
 std::uint32_t TypeStorageBufferElementPointer(SpirvEmitterState& state);
 std::uint32_t TypeStorageBufferU64Pointer(SpirvEmitterState& state);
 std::uint32_t TypeStorageBufferU64ElementPointer(SpirvEmitterState& state);
+std::uint32_t TypeStorageBufferU8Block(SpirvEmitterState& state);
+std::uint32_t TypeStorageBufferU8Pointer(SpirvEmitterState& state);
+std::uint32_t TypeStorageBufferU8ElementPointer(SpirvEmitterState& state);
 std::uint32_t TypePhysicalU32Pointer(SpirvEmitterState& state);
 std::uint32_t TypePushConstantElementPointer(SpirvEmitterState& state);
 std::uint32_t TypeU32ArrayPointer(SpirvEmitterState& state, std::uint32_t storageClass, std::uint32_t dwords);

core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp

@@ -25,6 +25,14 @@ std::uint32_t StorageBufferU64Type(SpirvEmitterState& state) {
         array);
 }
 
+std::uint32_t StorageBufferU8Type(SpirvEmitterState& state) {
+    const auto array = state.module.DecoratedType(spv::OpTypeRuntimeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, 1u}}}, TypeU8(state));
+    return state.module.DecoratedType(spv::OpTypeStruct,
+        {{spv::OpDecorate, {spv::DecorationBlock}},
+         {spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}}},
+        array);
+}
+
 }
 
 
@@ -40,6 +48,10 @@ std::uint32_t TypeBoolVector(SpirvEmitterState& state, std::uint32_t components)
     return state.module.Type(spv::OpTypeVector, TypeBool(state), components);
 }
 
+std::uint32_t TypeU8(SpirvEmitterState& state) {
+    return state.module.Type(spv::OpTypeInt, 8u, 0u);
+}
+
 std::uint32_t TypeU32(SpirvEmitterState& state) {
     return state.module.Type(spv::OpTypeInt, 32u, 0u);
 }
@@ -113,6 +125,18 @@ std::uint32_t TypeStorageBufferU64ElementPointer(SpirvEmitterState& state) {
     return TypePointer(state, spv::StorageClassStorageBuffer, TypeScalarU64(state));
 }
 
+std::uint32_t TypeStorageBufferU8Block(SpirvEmitterState& state) {
+    return StorageBufferU8Type(state);
+}
+
+std::uint32_t TypeStorageBufferU8Pointer(SpirvEmitterState& state) {
+    return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferU8Type(state));
+}
+
+std::uint32_t TypeStorageBufferU8ElementPointer(SpirvEmitterState& state) {
+    return TypePointer(state, spv::StorageClassStorageBuffer, TypeU8(state));
+}
+
 std::uint32_t TypePhysicalU32Pointer(SpirvEmitterState& state) {
     return TypePointer(state, spv::StorageClassPhysicalStorageBuffer, TypeU32(state));
 }

core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp

@@ -184,6 +184,28 @@ std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruct
     });
 }
 
+std::uint32_t byteBuffers(SpirvEmitterState& state) {
+    if (state.storageBufferU8Variable != 0u) return state.storageBufferU8Variable;
+    if (state.program.Resources().stage != IrShaderStage::Compute) return 0u;
+    if (std::find(state.supportedCapabilities.begin(), state.supportedCapabilities.end(), static_cast<std::uint32_t>(spv::CapabilityStorageBuffer8BitAccess)) == state.supportedCapabilities.end()) return 0u;
+    const IrBindingLayout& layout = state.program.Metadata().bindings;
+    const auto binding = std::find_if(layout.descriptors.begin(), layout.descriptors.end(), [](const IrDescriptorBinding& descriptor) { return descriptor.kind == DescriptorBindingKind::Buffers; });
+    if (binding == layout.descriptors.end() || state.storageBufferVariable == 0u) return 0u;
+    state.module.EmitCapability(spv::CapabilityStorageBuffer8BitAccess);
+    state.module.EmitExtension("SPV_KHR_8bit_storage");
+    const auto count = ConstantU32(state, static_cast<std::uint32_t>(binding->resources.size()));
+    const auto type = state.module.Type(spv::OpTypeArray, TypeStorageBufferU8Block(state), count);
+    const auto variable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassStorageBuffer, type), spv::StorageClassStorageBuffer);
+    state.module.AddName(variable, "buffers_u8");
+    state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationDescriptorSet, 0u);
+    state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBinding, NativeBinding(state.program.Resources().stage, DescriptorBindingKind::Buffers));
+    state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased);
+    if (state.storageBufferU64Variable == 0u && state.storageBufferCoherentVariable == 0u) state.module.AddAnnotation(spv::OpDecorate, state.storageBufferVariable, spv::DecorationAliased);
+    if (state.requirements.coherentBuffers) state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationCoherent);
+    state.storageBufferU8Variable = variable;
+    return variable;
+}
+
 void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u) {
     if (context.Memory(instruction).gpuDescriptor) {
         EmitBdaWrite(context, instruction, address, value, bits);
@@ -197,6 +219,24 @@ void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std
     const auto unaligned = nonzero(state, misalignment);
     const auto next = EmitAddU32(state, index, ConstantU32(state, 1u));
     const auto last = Select(state, u32, unaligned, next, index);
+    const auto bytes = bits != 32u ? byteBuffers(state) : 0u;
+    if (bytes != 0u) {
+        const auto element = ResourceForDescriptor(state, DescriptorBindingKind::Buffers, context.Memory(instruction).resource);
+        const auto block = state.module.AllocateId();
+        state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferU8Pointer(state), block, bytes, ConstantU32(state, element));
+        EmitIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] {
+            const auto first = EmitAddU32(state, EmitAddU32(state, Binary(state, spv::OpShiftLeftLogical, u32, index, ConstantU32(state, 2u)), misalignment), EmitAndConstant(state, address, 3u));
+            for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) {
+                const auto pointer = state.module.AllocateId();
+                state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferU8ElementPointer(state), pointer, block, ConstantU32(state, 0u), EmitAddU32(state, first, ConstantU32(state, byte)));
+                const auto shifted = byte == 0u ? value : Binary(state, spv::OpShiftRightLogical, u32, value, ConstantU32(state, byte * 8u));
+                const auto stored = Unary(state, spv::OpUConvert, TypeU8(state), shifted);
+                if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpStore, pointer, stored, access.memoryAccess);
+                else state.module.AddFunction(spv::OpStore, pointer, stored);
+            }
+        });
+        return;
+    }
     EmitIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] {
         auto mask = ConstantU32(state, bits == 32u ? 0xffffffffu : (1u << bits) - 1u);
         auto masked = Binary(state, spv::OpBitwiseAnd, u32, value, mask);