diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index e2ddd56df..c5f97e5d5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -175,6 +175,7 @@ CompiledVariant sampleVariant() { buffer.scalar = true; buffer.descriptorFormatted = true; buffer.formattedReadMask = 5u; + buffer.typedAlignment = 2u; info.info.buffers = {buffer, BufferResource{}}; ImageResource image{}; image.source = 5; @@ -628,8 +629,11 @@ void verifyInvocationIsolation() { const auto restored = Recompile(request.request); requireSameResult(first, restored, "restored invocation"); const auto original = request.userData; - const std::array, 7> descriptors{{ + const std::array, 10> descriptors{{ {original[0] + 0x20000u, original[1], original[2], original[3]}, + {original[0] + 1u, original[1], original[2], original[3]}, + {original[0] + 2u, original[1], original[2], original[3]}, + {original[0] + 3u, original[1], original[2], original[3]}, {original[0], original[1] | 1u, original[2], original[3]}, {original[0], original[1], original[2] / 2u, original[3]}, {original[0], original[1] | (16u << 16u), original[2], original[3]}, @@ -641,7 +645,7 @@ void verifyInvocationIsolation() { request.userData = descriptor; const auto changed = Recompile(request.request); require(changed.cacheHit && first.variantId == changed.variantId, "buffer metadata changed the compiled variant"); - if ((descriptor[1] & 0xffff0000u) == (original[1] & 0xffff0000u) && descriptor[3] == original[3] && (descriptor[0] != 0u || (descriptor[1] & 0xffffu) != 0u)) { + if ((descriptor[0] & 3u) == (original[0] & 3u) && (descriptor[1] & 0xffff0000u) == (original[1] & 0xffff0000u) && descriptor[3] == original[3] && (descriptor[0] != 0u || (descriptor[1] & 0xffffu) != 0u)) { require(first.spirv.data() == changed.spirv.data() && first.PipelineVariantId() == changed.PipelineVariantId(), "buffer address or size rebuilt the specialized module"); } ComputeRequest fresh(false); @@ -658,6 +662,38 @@ void verifyInvocationIsolation() { require(rejected, "an unsupported buffer descriptor type was accepted"); } +void verifyBufferAlignmentSpecialization() { + ComputeRequest request(true); + request.request.context.waveSize = 32u; + request.request.context.compute->numThreads = {32u, 1u, 1u}; + request.userData[3] = 0x31027facu; + static_cast(PrepareShader(request.request)); + const auto misses = diskMisses(); + const auto aligned = Recompile(request.request); + const auto countAtomics = [](const RecompileResult& result) { + std::size_t atomics = 0; + for (std::size_t cursor = 5; cursor < result.spirv.size();) { + const auto count = result.spirv[cursor] >> 16u; + require(count != 0u && count <= result.spirv.size() - cursor, "invalid specialized buffer instruction"); + if ((result.spirv[cursor] & 0xffffu) == spv::OpAtomicCompareExchange) ++atomics; + cursor += count; + } + return atomics; + }; + require(countAtomics(aligned) == 0u, "aligned DWORD store retained unaligned atomic updates"); + for (std::uint32_t offset = 1; offset < 4u; ++offset) { + request.userData[0] = 0x10000000u + offset; + const auto unaligned = Recompile(request.request); + require(unaligned.variantId == aligned.variantId && diskMisses() == misses, "base alignment repeated static shader preparation"); + require(countAtomics(unaligned) == 2u, "unaligned DWORD store did not split into two atomic updates"); + request.userData[0] += 0x10000u; + const auto relocated = Recompile(request.request); + require(unaligned.spirv.data() == relocated.spirv.data(), "relocation with unchanged alignment rebuilt the buffer specialization"); + } + request.userData[0] = 0x10000000u; + require(Recompile(request.request).spirv.data() == aligned.spirv.data(), "unaligned buffers replaced the aligned specialization"); +} + void verifyBindingPlanSelection() { ShaderInfo info{}; info.buffers.resize(2); @@ -703,6 +739,31 @@ void verifyBindingPlanSelection() { try { builder.Populate(populated, compiled, selected, 0u, snapshot, {}); } catch (const std::runtime_error&) { rejected = true; } require(rejected, "selected plan accepted an invalid live buffer descriptor"); + snapshot.buffers[0].dwordCount = 4u; + const auto rejectsAlignment = [&](const char* diagnostic) { + try { static_cast(builder.Prepare(compiled.layout, info, IrShaderStage::Compute, snapshot)); } + catch (const std::runtime_error& error) { + require(std::string_view(error.what()).find(diagnostic) != std::string_view::npos, "alignment check returned an unrelated failure"); + return; + } + throw std::runtime_error("binding plan accepted an unsupported buffer alignment"); + }; + snapshot.buffers[0].dwords[0] |= 2u; + rejectsAlignment("buffer atomic on a V# whose base is not DWORD aligned"); + snapshot.buffers[0].dwords[0] &= ~3u; + info.buffers[1].typedAlignment = 2u; + snapshot.buffers[1].dwords[0] |= 2u; + static_cast(builder.Prepare(compiled.layout, info, IrShaderStage::Compute, snapshot)); + snapshot.buffers[1].dwords[0] |= 1u; + rejectsAlignment("base is not aligned to its element"); + info.buffers[1].typedAlignment = 1u; + info.buffers[1].descriptorFormatted = true; + snapshot.buffers[1].dwords[3] = 0x3100bfacu; + rejectsAlignment("base is not aligned to its element"); + snapshot.buffers[1].dwords[0] &= ~1u; + static_cast(builder.Prepare(compiled.layout, info, IrShaderStage::Compute, snapshot)); + snapshot.buffers[1].dwords[3] = 0x31014facu; + rejectsAlignment("base is not aligned to its element"); } void verifyVertexTypeSpecialization() { @@ -936,6 +997,7 @@ int main(int argc, char** argv) { verifyEmissionFailureMemo(); verifyFailureMemoSwitch(argv[0]); verifyInvocationIsolation(); + verifyBufferAlignmentSpecialization(); verifyBindingPlanSelection(); verifyVertexTypeSpecialization(); verifyBuiltinSpecialization(); diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp index 8986f563b..aa631636b 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp @@ -25,6 +25,7 @@ struct BufferResource { bool descriptorFormatted = false; std::uint32_t formattedReadMask = 0; bool scalar = false; + std::uint8_t typedAlignment = 1; bool operator==(const BufferResource& other) const = default; }; diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index 30bd4ed27..0005d4c46 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -1,6 +1,7 @@ #include "Optimization/DescriptorBindingBuilder.hpp" #include "Optimization/ResourceMaterializer.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" +#include "SpirvBackend/SpirvBufferFormat.hpp" #include #include #include @@ -264,12 +265,18 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l if (descriptor.dwordCount != 4u) fail("buffer specialization requires four descriptor words"); const auto& buffer = info.buffers[index]; const auto format = (descriptor.dwords[3] >> 12u) & 0x7fu; - const bool present = descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u; + const bool present = descriptor.dwords[2] != 0u && (descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u); if (buffer.descriptorFormatted && (format > static_cast(IrBufferFormat::Format32_32_32_32Float) || (present && format == 0u))) fail("buffer specialization has an unsupported format"); + const auto misalignment = present ? descriptor.dwords[0] & 3u : 0u; + if (buffer.atomic && misalignment != 0u) fail("buffer atomic on a V# whose base is not DWORD aligned"); + auto alignment = static_cast(buffer.typedAlignment); + if (buffer.descriptorFormatted && format != 0u) alignment = std::max(alignment, std::min(GetFormatInfo(static_cast(format)).byteSize, 4u)); + if (alignment == 0u || misalignment % alignment != 0u) fail("is a formatted buffer access through a V# whose base is not aligned to its element"); const auto first = PipelineSpecialization::BufferBase + index * PipelineSpecialization::BufferWords; plan.specialization.push_back({first, descriptor.dwords[1] & 0xffff0000u}); plan.specialization.push_back({first + 1u, descriptor.dwords[3]}); plan.specialization.push_back({first + 2u, present ? 1u : 0u}); + plan.specialization.push_back({first + 3u, misalignment}); if (buffer.formattedReadMask != 0u) { for (std::uint32_t component = 0; component < 4u; ++component) { if ((buffer.formattedReadMask & (1u << component)) == 0u) continue; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index c5ff6e9c1..801043f8c 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -170,9 +170,10 @@ DecodedImage decodeImageDescriptor(const DescriptorValue& descriptor, const Imag decoded.packedFormat = format; } decoded.conversionFormat = RemapTextureFormat(format) != format ? format : IrBufferFormat::Invalid; - if (format == IrBufferFormat::Format11_11_10UNorm) { - if (!base.srgbDecodeCompatible) throw std::runtime_error("sampling, gathering or querying LOD of a converted unorm image is not implemented"); - if (!base.depthBitsCompatible) throw std::runtime_error("reads or writes a converted unorm image with 16-bit data, which is not implemented"); + if (format == IrBufferFormat::Format11_11_10UNorm || format == IrBufferFormat::Format10_11_11Float) { + const bool floating = format == IrBufferFormat::Format10_11_11Float; + if (!base.srgbDecodeCompatible) throw std::runtime_error(floating ? "samples or gathers a converted float image, or queries its LOD, which is not implemented" : "sampling, gathering or querying LOD of a converted unorm image is not implemented"); + if (!base.depthBitsCompatible) throw std::runtime_error(floating ? "reads or writes a converted float image with 16-bit data, which is not implemented" : "reads or writes a converted unorm image with 16-bit data, which is not implemented"); for (std::uint32_t component = 0; component < 4u; ++component) { if (((descriptorImageSwizzle(descriptor) >> (component * 3u)) & 7u) == 7u) throw std::runtime_error("selects a channel the converted image format does not have"); } @@ -594,7 +595,7 @@ std::vector ResourceMaterializer::RuntimeImageModes(const ImageRe mode.cube = false; mode.mipCount = mode.mipMode == ImageMipMode::DynamicStorage ? RuntimeAbi::StorageHeapCapacity : 1u; mode.shaderSwizzle = ShaderImageIdentitySwizzle; - if (conversion == IrBufferFormat::Format11_11_10UNorm) mode.shaderSwizzle = 0x2acu; + if (conversion == IrBufferFormat::Format11_11_10UNorm || conversion == IrBufferFormat::Format10_11_11Float) mode.shaderSwizzle = 0x2acu; modes.push_back(mode); if (image.dimension == RdnaImageDimension::Dim1DArray || image.dimension == RdnaImageDimension::Dim2DArray || image.dimension == RdnaImageDimension::Dim2DMsaaArray) { auto plain = mode; @@ -629,7 +630,10 @@ std::vector ResourceMaterializer::RuntimeImageModes(const ImageRe append(IrTextureNumericClass::Uint, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false); if (!storage) append(IrTextureNumericClass::Sint, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false); append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UInt, IrBufferFormat::Invalid, false, false); - if (image.srgbDecodeCompatible && image.depthBitsCompatible) append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UNorm, IrBufferFormat::Invalid, false, false); + if (image.srgbDecodeCompatible && image.depthBitsCompatible) { + append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UNorm, IrBufferFormat::Invalid, false, false); + append(IrTextureNumericClass::Uint, IrBufferFormat::Format10_11_11Float, IrBufferFormat::Invalid, false, false); + } if (!storage) { if (image.depthBitsCompatible) { append(IrTextureNumericClass::Float, IrBufferFormat::Invalid, IrBufferFormat::Invalid, true, false); diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index b59a50f0d..32a8b5230 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -1,6 +1,7 @@ #include "Optimization/ResourceTracker.hpp" #include "Optimization/SrtWalker.hpp" #include "Optimization/ResourceMaterializer.hpp" +#include "SpirvBackend/SpirvBufferFormat.hpp" #include "IntermediateRepresentation/IrBuilder.hpp" #include "Optimization/SrtWalker/SrtInstructionPredicates.hpp" @@ -851,6 +852,11 @@ private: resource.atomic = resource.atomic || atomic; resource.formatted = resource.formatted || memory.formatted; resource.descriptorFormatted = resource.descriptorFormatted || (memory.formatted && !memory.typed); + if (memory.formatted && memory.typed) { + const auto format = GetFormatInfo(DecodeTBufferFormat(memory.dataFormat, memory.numberFormat)); + if (format.byteSize == 0u) fail("typed buffer instruction has an invalid format"); + resource.typedAlignment = std::max(resource.typedAlignment, static_cast(std::min(format.byteSize, 4u))); + } if (memory.formatted && !memory.typed && !write) resource.formattedReadMask |= (1u << std::min(memory.dataDwords, 4u)) - 1u; resource.scalar = resource.scalar || op == IrOpcode::ReadConstBuffer || memory.kind == ResourceKind::ScalarBuffer; } diff --git a/core/shader/recompiler/PipelineSpecialization.hpp b/core/shader/recompiler/PipelineSpecialization.hpp index a6e5a7bc9..4b7fea929 100644 --- a/core/shader/recompiler/PipelineSpecialization.hpp +++ b/core/shader/recompiler/PipelineSpecialization.hpp @@ -17,7 +17,7 @@ namespace PipelineSpecialization { inline constexpr std::uint32_t CompareBase = 49152u; inline constexpr std::uint32_t CompareWords = 6u; inline constexpr std::uint32_t BufferBase = 0u; -inline constexpr std::uint32_t BufferWords = 3u; +inline constexpr std::uint32_t BufferWords = 4u; inline constexpr std::uint32_t ImageBase = 1024u; inline constexpr std::uint32_t ImageWords = 5u; inline constexpr std::uint32_t VertexBase = 4096u; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index a415f2ec9..a2870e10f 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -517,7 +517,8 @@ std::shared_ptr preparedBindingPlan(const CompiledVariant& key.push_back(descriptor.dwordCount); key.push_back(descriptor.dwords[1] & 0xffff0000u); key.push_back(descriptor.dwords[3]); - key.push_back(descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u); + const bool present = descriptor.dwords[2] != 0u && (descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u); + key.push_back(present ? 4u | (descriptor.dwords[0] & 3u) : 0u); } for (std::size_t index = 0; index < variant.info.info.images.size(); ++index) { const auto& descriptor = snapshot.images.at(index); diff --git a/core/shader/recompiler/RuntimeAbi.hpp b/core/shader/recompiler/RuntimeAbi.hpp index 0da695ba8..66f7f8a2f 100644 --- a/core/shader/recompiler/RuntimeAbi.hpp +++ b/core/shader/recompiler/RuntimeAbi.hpp @@ -9,7 +9,7 @@ namespace ShaderRecompiler::RuntimeAbi { -inline constexpr std::uint32_t Version = 9u; +inline constexpr std::uint32_t Version = 10u; inline constexpr std::uint32_t DescriptorSet = 0u; inline constexpr std::uint32_t StageCount = 4u; inline constexpr std::uint32_t PushConstantDwords = 32u; diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index fa840dcae..31a514a13 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -73,7 +73,7 @@ static_assert(sizeof(VertexInput) == 12, "VertexInput changed: update the vertex static_assert(sizeof(FragmentParameter) == 12, "FragmentParameter changed: update the parameter encoder"); static_assert(sizeof(CompiledShaderInfo) == 328, "CompiledShaderInfo changed: update the info encoder"); static_assert(sizeof(ShaderInfo) == 224, "ShaderInfo changed: update the info encoder"); -static_assert(sizeof(BufferResource) == 24, "BufferResource changed: update the info encoder"); +static_assert(sizeof(BufferResource) == 32, "BufferResource changed: update the info encoder"); static_assert(sizeof(ImageResource) == 104, "ImageResource changed: update the info encoder"); static_assert(sizeof(SamplerResource) == 16, "SamplerResource changed: update the info encoder"); static_assert(sizeof(SampledResourcePair) == 12, "SampledResourcePair changed: update the info encoder"); @@ -445,6 +445,7 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { out.Value(buffer.descriptorFormatted); out.Value(buffer.formattedReadMask); out.Value(buffer.scalar); + out.Value(buffer.typedAlignment); }); writer.List(info.images, [](Writer& out, const ImageResource& image) { out.Value(image.source); @@ -530,7 +531,7 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { auto& info = compiled.info; reader.Value(info.scratchDwords); reader.Value(info.sharedMemoryBytes); - reader.List(info.buffers, 26, [](Reader& in, BufferResource& buffer) { + reader.List(info.buffers, 27, [](Reader& in, BufferResource& buffer) { in.Value(buffer.source); in.Value(buffer.firstUsePc); in.Value(buffer.maxByteExtent); @@ -542,6 +543,7 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { in.Value(buffer.descriptorFormatted); in.Value(buffer.formattedReadMask); in.Value(buffer.scalar); + in.Value(buffer.typedAlignment); }); reader.List(info.images, 72, [](Reader& in, ImageResource& image) { in.Value(image.source); diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 6b52e3a99..d76dbde14 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -12,7 +12,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 18; +inline constexpr std::uint32_t FormatVersion = 19; enum class LoadStatus { Loaded, diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index b6520ab26..4ef487bfc 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -117,10 +117,6 @@ void RequireAlignedAtomic(SpirvValueEmitContext& ctx, const MemoryResourceAccess if (resource.misalignment != 0u) ctx.Fail("buffer atomic on a V# whose base is not DWORD aligned"); } -void RequireAlignedFormattedElement(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryResourceAccess& resource, const SpirvBufferFormatInfo& info) { - if (resource.misalignment % std::min(info.byteSize, 4u) != 0u) ctx.Fail(inst, "is a formatted buffer access through a V# whose base is not aligned to its element"); -} - std::uint32_t LoadSubwordInBounds(SpirvValueEmitContext& ctx, const MemoryResourceAccess& resource, std::uint32_t address, std::uint32_t index, std::uint32_t bits, bool signExtend) { auto& state = ctx.state; if (bits != 8u && bits != 16u) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp index a68f99c0e..08a20dd32 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -131,28 +131,48 @@ MemoryResourceAccess directAccess(SpirvValueEmitContext& context, const IrValue& return access; } +std::uint32_t baseMisalignment(SpirvValueEmitContext& context, const IrValue& instruction) { + const auto& memory = context.Memory(instruction); + if (memory.kind == ResourceKind::ScalarBuffer) return ConstantU32(context.state, 0u); + const auto first = PipelineSpecialization::BufferBase + memory.resource * PipelineSpecialization::BufferWords; + return context.state.module.SpecializationConstant(TypeU32(context.state), first + 3u, 0u); +} + +std::uint32_t directWordIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t address) { + const auto relative = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); + const auto base = Binary(state, spv::OpShiftRightLogical, TypeU32(state), access.byteOffset, ConstantU32(state, 2u)); + return EmitAddU32(state, relative, base); +} + std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t bits) { if (context.Memory(instruction).gpuDescriptor) return EmitBdaRead(context, instruction, address, bits); auto& state = context.state; const auto access = directAccess(context, instruction); - const auto byte = EmitAddU32(state, address, access.byteOffset); - const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte, ConstantU32(state, 2u)); - const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), EmitAndConstant(state, byte, 3u), ConstantU32(state, 3u)); - const auto load = [&](std::uint32_t word) { - return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access, word), [&] { + const auto index = directWordIndex(state, access, address); + const auto misalignment = baseMisalignment(context, instruction); + const auto unaligned = nonzero(state, misalignment); + const auto next = EmitAddU32(state, index, ConstantU32(state, 1u)); + const auto last = Select(state, TypeU32(state), unaligned, next, index); + return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] { + const auto load = [&](std::uint32_t word) { const auto value = state.module.AllocateId(); const auto pointer = EmitMemoryElementPointer(state, access, word); if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, access.memoryAccess); else state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); return value; + }; + const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), misalignment, ConstantU32(state, 3u)); + const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(index), shift); + const auto high = EmitValueOrZeroIfCondition(state, unaligned, [&] { + return Binary(state, spv::OpShiftLeftLogical, TypeU32(state), load(next), Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32u), shift)); }); - }; - const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(index), shift); - const auto crosses = Binary(state, spv::OpUGreaterThan, TypeBool(state), shift, ConstantU32(state, 32u - bits)); - const auto high = EmitValueOrZeroIfCondition(state, crosses, [&] { - return Binary(state, spv::OpShiftLeftLogical, TypeU32(state), load(EmitAddU32(state, index, ConstantU32(state, 1u))), Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32u), shift)); + auto value = EmitOrU32(state, low, high); + if (bits != 32u) { + const auto subwordShift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), EmitAndConstant(state, address, 3u), ConstantU32(state, 3u)); + value = Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, subwordShift); + } + return EmitAndConstant(state, value, bits == 32u ? 0xffffffffu : (1u << bits) - 1u); }); - return EmitAndConstant(state, EmitOrU32(state, low, high), bits == 32u ? 0xffffffffu : (1u << bits) - 1u); } void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u) { @@ -163,33 +183,38 @@ void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std auto& state = context.state; const auto access = directAccess(context, instruction); const auto u32 = TypeU32(state); - const auto byte = EmitAddU32(state, address, access.byteOffset); - const auto index = Binary(state, spv::OpShiftRightLogical, u32, byte, ConstantU32(state, 2u)); - const auto shift = Binary(state, spv::OpShiftLeftLogical, u32, EmitAndConstant(state, byte, 3u), ConstantU32(state, 3u)); - const auto mask = ConstantU32(state, bits == 32u ? 0xffffffffu : (1u << bits) - 1u); - const auto masked = Binary(state, spv::OpBitwiseAnd, u32, value, mask); - const auto update = [&](std::uint32_t word, std::uint32_t wordMask, std::uint32_t replacement) { - EmitIfCondition(state, EmitMemoryElementInBounds(state, access, word), [&] { - const auto pointer = EmitMemoryElementPointer(state, access, word); - AtomicUpdate(state, pointer, access.kind, [&](std::uint32_t old) { + const auto index = directWordIndex(state, access, address); + const auto misalignment = baseMisalignment(context, instruction); + const auto unaligned = nonzero(state, misalignment); + const auto next = EmitAddU32(state, index, ConstantU32(state, 1u)); + const auto last = Select(state, u32, unaligned, next, index); + EmitIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] { + auto mask = ConstantU32(state, bits == 32u ? 0xffffffffu : (1u << bits) - 1u); + auto masked = Binary(state, spv::OpBitwiseAnd, u32, value, mask); + if (bits != 32u) { + const auto subwordShift = Binary(state, spv::OpShiftLeftLogical, u32, EmitAndConstant(state, address, 3u), ConstantU32(state, 3u)); + mask = Binary(state, spv::OpShiftLeftLogical, u32, mask, subwordShift); + masked = Binary(state, spv::OpShiftLeftLogical, u32, masked, subwordShift); + } + const auto update = [&](std::uint32_t word, std::uint32_t wordMask, std::uint32_t replacement) { + AtomicUpdate(state, EmitMemoryElementPointer(state, access, word), access.kind, [&](std::uint32_t old) { return EmitOrU32(state, Binary(state, spv::OpBitwiseAnd, u32, old, Unary(state, spv::OpNot, u32, wordMask)), replacement); }); - }); - }; - const auto unaligned = nonzero(state, shift); - if (bits == 32u) { - EmitIfCondition(state, both(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), EmitMemoryElementInBounds(state, access, index)), [&] { - const auto pointer = EmitMemoryElementPointer(state, access, index); - if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpStore, pointer, value, access.memoryAccess); - else state.module.AddFunction(spv::OpStore, pointer, value); - }); - } - EmitIfCondition(state, bits == 32u ? unaligned : ConstantBool(state, true), [&] { - update(index, Binary(state, spv::OpShiftLeftLogical, u32, mask, shift), Binary(state, spv::OpShiftLeftLogical, u32, masked, shift)); - const auto crosses = Binary(state, spv::OpUGreaterThan, TypeBool(state), shift, ConstantU32(state, 32u - bits)); - EmitIfCondition(state, crosses, [&] { - const auto remaining = Binary(state, spv::OpISub, u32, ConstantU32(state, 32u), shift); - update(EmitAddU32(state, index, ConstantU32(state, 1u)), Binary(state, spv::OpShiftRightLogical, u32, mask, remaining), Binary(state, spv::OpShiftRightLogical, u32, masked, remaining)); + }; + if (bits == 32u) { + EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), [&] { + const auto pointer = EmitMemoryElementPointer(state, access, index); + if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpStore, pointer, value, access.memoryAccess); + else state.module.AddFunction(spv::OpStore, pointer, value); + }); + } + EmitIfCondition(state, bits == 32u ? unaligned : ConstantBool(state, true), [&] { + const auto shift = Binary(state, spv::OpShiftLeftLogical, u32, misalignment, ConstantU32(state, 3u)); + update(index, Binary(state, spv::OpShiftLeftLogical, u32, mask, shift), Binary(state, spv::OpShiftLeftLogical, u32, masked, shift)); + EmitIfCondition(state, unaligned, [&] { + const auto remaining = Binary(state, spv::OpISub, u32, ConstantU32(state, 32u), shift); + update(next, Binary(state, spv::OpShiftRightLogical, u32, mask, remaining), Binary(state, spv::OpShiftRightLogical, u32, masked, remaining)); + }); }); }); } @@ -369,7 +394,7 @@ std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValu const auto wide = directAccess(context, instruction, true); const auto byte = EmitAddU32(state, runtime.guest, wide.byteOffset); const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte, ConstantU32(state, 3u)); - const auto aligned = equal(state, EmitAndConstant(state, byte, 7u), ConstantU32(state, 0u)); + const auto aligned = both(state, equal(state, baseMisalignment(context, instruction), ConstantU32(state, 0u)), equal(state, EmitAndConstant(state, byte, 7u), ConstantU32(state, 0u))); const auto inBounds = both(state, runtime.inBounds, EmitMemoryElementInBounds(state, wide, index)); return EmitValueIfElse(state, both(state, aligned, inBounds), type, [&] { const auto pointer = EmitStorageBufferElementPointer(state, wide, index, TypeStorageBufferU64ElementPointer(state));