shifathosn
/
AnyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(shader): preserve main buffer alignment and image formats in prepared shaders

This commit is contained in:
boykopovar 2026-10-08 17:51:05 +03:00
parent 082f371758
commit eb68ad3b02
12 changed files with 159 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp
View File

@ -175,6 +175,7 @@ CompiledVariant sampleVariant() {
buffer.scalar = true;
buffer.descriptorFormatted = true;
buffer.formattedReadMask = 5u;
buffer.typedAlignment = 2u;
info.info.buffers = {buffer, BufferResource{}};
ImageResource image{};
image.source = 5;
@ -628,8 +629,11 @@ void verifyInvocationIsolation() {
const auto restored = Recompile(request.request);
requireSameResult(first, restored, "restored invocation");
const auto original = request.userData;
const std::array<std::array<std::uint32_t, 4>, 7> descriptors{{
const std::array<std::array<std::uint32_t, 4>, 10> descriptors{{
{original[0] + 0x20000u, original[1], original[2], original[3]},
{original[0] + 1u, original[1], original[2], original[3]},
{original[0] + 2u, original[1], original[2], original[3]},
{original[0] + 3u, original[1], original[2], original[3]},
{original[0], original[1] | 1u, original[2], original[3]},
{original[0], original[1], original[2] / 2u, original[3]},
{original[0], original[1] | (16u << 16u), original[2], original[3]},
@ -641,7 +645,7 @@ void verifyInvocationIsolation() {
request.userData = descriptor;
const auto changed = Recompile(request.request);
require(changed.cacheHit && first.variantId == changed.variantId, "buffer metadata changed the compiled variant");
if ((descriptor[1] & 0xffff0000u) == (original[1] & 0xffff0000u) && descriptor[3] == original[3] && (descriptor[0] != 0u || (descriptor[1] & 0xffffu) != 0u)) {
if ((descriptor[0] & 3u) == (original[0] & 3u) && (descriptor[1] & 0xffff0000u) == (original[1] & 0xffff0000u) && descriptor[3] == original[3] && (descriptor[0] != 0u || (descriptor[1] & 0xffffu) != 0u)) {
require(first.spirv.data() == changed.spirv.data() && first.PipelineVariantId() == changed.PipelineVariantId(), "buffer address or size rebuilt the specialized module");
}
ComputeRequest fresh(false);
@ -658,6 +662,38 @@ void verifyInvocationIsolation() {
require(rejected, "an unsupported buffer descriptor type was accepted");
}
void verifyBufferAlignmentSpecialization() {
ComputeRequest request(true);
request.request.context.waveSize = 32u;
request.request.context.compute->numThreads = {32u, 1u, 1u};
request.userData[3] = 0x31027facu;
static_cast<void>(PrepareShader(request.request));
const auto misses = diskMisses();
const auto aligned = Recompile(request.request);
const auto countAtomics = [](const RecompileResult& result) {
std::size_t atomics = 0;
for (std::size_t cursor = 5; cursor < result.spirv.size();) {
const auto count = result.spirv[cursor] >> 16u;
require(count != 0u && count <= result.spirv.size() - cursor, "invalid specialized buffer instruction");
if ((result.spirv[cursor] & 0xffffu) == spv::OpAtomicCompareExchange) ++atomics;
cursor += count;
}
return atomics;
};
require(countAtomics(aligned) == 0u, "aligned DWORD store retained unaligned atomic updates");
for (std::uint32_t offset = 1; offset < 4u; ++offset) {
request.userData[0] = 0x10000000u + offset;
const auto unaligned = Recompile(request.request);
require(unaligned.variantId == aligned.variantId && diskMisses() == misses, "base alignment repeated static shader preparation");
require(countAtomics(unaligned) == 2u, "unaligned DWORD store did not split into two atomic updates");
request.userData[0] += 0x10000u;
const auto relocated = Recompile(request.request);
require(unaligned.spirv.data() == relocated.spirv.data(), "relocation with unchanged alignment rebuilt the buffer specialization");
}
request.userData[0] = 0x10000000u;
require(Recompile(request.request).spirv.data() == aligned.spirv.data(), "unaligned buffers replaced the aligned specialization");
}
void verifyBindingPlanSelection() {
ShaderInfo info{};
info.buffers.resize(2);
@ -703,6 +739,31 @@ void verifyBindingPlanSelection() {
try { builder.Populate(populated, compiled, selected, 0u, snapshot, {}); }
catch (const std::runtime_error&) { rejected = true; }
require(rejected, "selected plan accepted an invalid live buffer descriptor");
snapshot.buffers[0].dwordCount = 4u;
const auto rejectsAlignment = [&](const char* diagnostic) {
try { static_cast<void>(builder.Prepare(compiled.layout, info, IrShaderStage::Compute, snapshot)); }
catch (const std::runtime_error& error) {
require(std::string_view(error.what()).find(diagnostic) != std::string_view::npos, "alignment check returned an unrelated failure");
return;
}
throw std::runtime_error("binding plan accepted an unsupported buffer alignment");
};
snapshot.buffers[0].dwords[0] |= 2u;
rejectsAlignment("buffer atomic on a V# whose base is not DWORD aligned");
snapshot.buffers[0].dwords[0] &= ~3u;
info.buffers[1].typedAlignment = 2u;
snapshot.buffers[1].dwords[0] |= 2u;
static_cast<void>(builder.Prepare(compiled.layout, info, IrShaderStage::Compute, snapshot));
snapshot.buffers[1].dwords[0] |= 1u;
rejectsAlignment("base is not aligned to its element");
info.buffers[1].typedAlignment = 1u;
info.buffers[1].descriptorFormatted = true;
snapshot.buffers[1].dwords[3] = 0x3100bfacu;
rejectsAlignment("base is not aligned to its element");
snapshot.buffers[1].dwords[0] &= ~1u;
static_cast<void>(builder.Prepare(compiled.layout, info, IrShaderStage::Compute, snapshot));
snapshot.buffers[1].dwords[3] = 0x31014facu;
rejectsAlignment("base is not aligned to its element");
}
void verifyVertexTypeSpecialization() {
@ -936,6 +997,7 @@ int main(int argc, char** argv) {
verifyEmissionFailureMemo();
verifyFailureMemoSwitch(argv[0]);
verifyInvocationIsolation();
verifyBufferAlignmentSpecialization();
verifyBindingPlanSelection();
verifyVertexTypeSpecialization();
verifyBuiltinSpecialization();

1
core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp
View File

@ -25,6 +25,7 @@ struct BufferResource {
bool descriptorFormatted = false;
std::uint32_t formattedReadMask = 0;
bool scalar = false;
std::uint8_t typedAlignment = 1;
bool operator==(const BufferResource& other) const = default;
};

9
core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp
View File

@ -1,6 +1,7 @@
#include "Optimization/DescriptorBindingBuilder.hpp"
#include "Optimization/ResourceMaterializer.hpp"
#include "SpirvBackend/SpirvEmitterHelpers.hpp"
#include "SpirvBackend/SpirvBufferFormat.hpp"
#include <spirv/unified1/spirv.hpp>
#include <algorithm>
#include <atomic>
@ -264,12 +265,18 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l
if (descriptor.dwordCount != 4u) fail("buffer specialization requires four descriptor words");
const auto& buffer = info.buffers[index];
const auto format = (descriptor.dwords[3] >> 12u) & 0x7fu;
const bool present = descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u;
const bool present = descriptor.dwords[2] != 0u && (descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u);
if (buffer.descriptorFormatted && (format > static_cast<std::uint32_t>(IrBufferFormat::Format32_32_32_32Float) || (present && format == 0u))) fail("buffer specialization has an unsupported format");
const auto misalignment = present ? descriptor.dwords[0] & 3u : 0u;
if (buffer.atomic && misalignment != 0u) fail("buffer atomic on a V# whose base is not DWORD aligned");
auto alignment = static_cast<std::uint32_t>(buffer.typedAlignment);
if (buffer.descriptorFormatted && format != 0u) alignment = std::max(alignment, std::min(GetFormatInfo(static_cast<IrBufferFormat>(format)).byteSize, 4u));
if (alignment == 0u || misalignment % alignment != 0u) fail("is a formatted buffer access through a V# whose base is not aligned to its element");
const auto first = PipelineSpecialization::BufferBase + index * PipelineSpecialization::BufferWords;
plan.specialization.push_back({first, descriptor.dwords[1] & 0xffff0000u});
plan.specialization.push_back({first + 1u, descriptor.dwords[3]});
plan.specialization.push_back({first + 2u, present ? 1u : 0u});
plan.specialization.push_back({first + 3u, misalignment});
if (buffer.formattedReadMask != 0u) {
for (std::uint32_t component = 0; component < 4u; ++component) {
if ((buffer.formattedReadMask & (1u << component)) == 0u) continue;

14
core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp
View File

@ -170,9 +170,10 @@ DecodedImage decodeImageDescriptor(const DescriptorValue& descriptor, const Imag
decoded.packedFormat = format;
}
decoded.conversionFormat = RemapTextureFormat(format) != format ? format : IrBufferFormat::Invalid;
if (format == IrBufferFormat::Format11_11_10UNorm) {
if (!base.srgbDecodeCompatible) throw std::runtime_error("sampling, gathering or querying LOD of a converted unorm image is not implemented");
if (!base.depthBitsCompatible) throw std::runtime_error("reads or writes a converted unorm image with 16-bit data, which is not implemented");
if (format == IrBufferFormat::Format11_11_10UNorm || format == IrBufferFormat::Format10_11_11Float) {
const bool floating = format == IrBufferFormat::Format10_11_11Float;
if (!base.srgbDecodeCompatible) throw std::runtime_error(floating ? "samples or gathers a converted float image, or queries its LOD, which is not implemented" : "sampling, gathering or querying LOD of a converted unorm image is not implemented");
if (!base.depthBitsCompatible) throw std::runtime_error(floating ? "reads or writes a converted float image with 16-bit data, which is not implemented" : "reads or writes a converted unorm image with 16-bit data, which is not implemented");
for (std::uint32_t component = 0; component < 4u; ++component) {
if (((descriptorImageSwizzle(descriptor) >> (component * 3u)) & 7u) == 7u) throw std::runtime_error("selects a channel the converted image format does not have");
}
@ -594,7 +595,7 @@ std::vector<ImageResource> ResourceMaterializer::RuntimeImageModes(const ImageRe
mode.cube = false;
mode.mipCount = mode.mipMode == ImageMipMode::DynamicStorage ? RuntimeAbi::StorageHeapCapacity : 1u;
mode.shaderSwizzle = ShaderImageIdentitySwizzle;
if (conversion == IrBufferFormat::Format11_11_10UNorm) mode.shaderSwizzle = 0x2acu;
if (conversion == IrBufferFormat::Format11_11_10UNorm || conversion == IrBufferFormat::Format10_11_11Float) mode.shaderSwizzle = 0x2acu;
modes.push_back(mode);
if (image.dimension == RdnaImageDimension::Dim1DArray || image.dimension == RdnaImageDimension::Dim2DArray || image.dimension == RdnaImageDimension::Dim2DMsaaArray) {
auto plain = mode;
@ -629,7 +630,10 @@ std::vector<ImageResource> ResourceMaterializer::RuntimeImageModes(const ImageRe
append(IrTextureNumericClass::Uint, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false);
if (!storage) append(IrTextureNumericClass::Sint, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false);
append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UInt, IrBufferFormat::Invalid, false, false);
if (image.srgbDecodeCompatible && image.depthBitsCompatible) append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UNorm, IrBufferFormat::Invalid, false, false);
if (image.srgbDecodeCompatible && image.depthBitsCompatible) {
append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UNorm, IrBufferFormat::Invalid, false, false);
append(IrTextureNumericClass::Uint, IrBufferFormat::Format10_11_11Float, IrBufferFormat::Invalid, false, false);
}
if (!storage) {
if (image.depthBitsCompatible) {
append(IrTextureNumericClass::Float, IrBufferFormat::Invalid, IrBufferFormat::Invalid, true, false);

6
core/shader/recompiler/Optimization/src/ResourceTracker.cpp
View File

@ -1,6 +1,7 @@
#include "Optimization/ResourceTracker.hpp"
#include "Optimization/SrtWalker.hpp"
#include "Optimization/ResourceMaterializer.hpp"
#include "SpirvBackend/SpirvBufferFormat.hpp"
#include "IntermediateRepresentation/IrBuilder.hpp"
#include "Optimization/SrtWalker/SrtInstructionPredicates.hpp"
@ -851,6 +852,11 @@ private:
resource.atomic = resource.atomic || atomic;
resource.formatted = resource.formatted || memory.formatted;
resource.descriptorFormatted = resource.descriptorFormatted || (memory.formatted && !memory.typed);
if (memory.formatted && memory.typed) {
const auto format = GetFormatInfo(DecodeTBufferFormat(memory.dataFormat, memory.numberFormat));
if (format.byteSize == 0u) fail("typed buffer instruction has an invalid format");
resource.typedAlignment = std::max(resource.typedAlignment, static_cast<std::uint8_t>(std::min(format.byteSize, 4u)));
}
if (memory.formatted && !memory.typed && !write) resource.formattedReadMask |= (1u << std::min(memory.dataDwords, 4u)) - 1u;
resource.scalar = resource.scalar || op == IrOpcode::ReadConstBuffer || memory.kind == ResourceKind::ScalarBuffer;
}

2
core/shader/recompiler/PipelineSpecialization.hpp
View File

@ -17,7 +17,7 @@ namespace PipelineSpecialization {
inline constexpr std::uint32_t CompareBase = 49152u;
inline constexpr std::uint32_t CompareWords = 6u;
inline constexpr std::uint32_t BufferBase = 0u;
inline constexpr std::uint32_t BufferWords = 3u;
inline constexpr std::uint32_t BufferWords = 4u;
inline constexpr std::uint32_t ImageBase = 1024u;
inline constexpr std::uint32_t ImageWords = 5u;
inline constexpr std::uint32_t VertexBase = 4096u;

3
core/shader/recompiler/Recompiler.cpp
View File

@ -517,7 +517,8 @@ std::shared_ptr<PreparedBindingPlan> preparedBindingPlan(const CompiledVariant&
key.push_back(descriptor.dwordCount);
key.push_back(descriptor.dwords[1] & 0xffff0000u);
key.push_back(descriptor.dwords[3]);
key.push_back(descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u);
const bool present = descriptor.dwords[2] != 0u && (descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u);
key.push_back(present ? 4u | (descriptor.dwords[0] & 3u) : 0u);
}
for (std::size_t index = 0; index < variant.info.info.images.size(); ++index) {
const auto& descriptor = snapshot.images.at(index);

2
core/shader/recompiler/RuntimeAbi.hpp
View File

@ -9,7 +9,7 @@
namespace ShaderRecompiler::RuntimeAbi {
inline constexpr std::uint32_t Version = 9u;
inline constexpr std::uint32_t Version = 10u;
inline constexpr std::uint32_t DescriptorSet = 0u;
inline constexpr std::uint32_t StageCount = 4u;
inline constexpr std::uint32_t PushConstantDwords = 32u;

6
core/shader/recompiler/ShaderDiskCache.cpp
View File

@ -73,7 +73,7 @@ static_assert(sizeof(VertexInput) == 12, "VertexInput changed: update the vertex
static_assert(sizeof(FragmentParameter) == 12, "FragmentParameter changed: update the parameter encoder");
static_assert(sizeof(CompiledShaderInfo) == 328, "CompiledShaderInfo changed: update the info encoder");
static_assert(sizeof(ShaderInfo) == 224, "ShaderInfo changed: update the info encoder");
static_assert(sizeof(BufferResource) == 24, "BufferResource changed: update the info encoder");
static_assert(sizeof(BufferResource) == 32, "BufferResource changed: update the info encoder");
static_assert(sizeof(ImageResource) == 104, "ImageResource changed: update the info encoder");
static_assert(sizeof(SamplerResource) == 16, "SamplerResource changed: update the info encoder");
static_assert(sizeof(SampledResourcePair) == 12, "SampledResourcePair changed: update the info encoder");
@ -445,6 +445,7 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) {
out.Value(buffer.descriptorFormatted);
out.Value(buffer.formattedReadMask);
out.Value(buffer.scalar);
out.Value(buffer.typedAlignment);
});
writer.List(info.images, [](Writer& out, const ImageResource& image) {
out.Value(image.source);
@ -530,7 +531,7 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) {
auto& info = compiled.info;
reader.Value(info.scratchDwords);
reader.Value(info.sharedMemoryBytes);
reader.List(info.buffers, 26, [](Reader& in, BufferResource& buffer) {
reader.List(info.buffers, 27, [](Reader& in, BufferResource& buffer) {
in.Value(buffer.source);
in.Value(buffer.firstUsePc);
in.Value(buffer.maxByteExtent);
@ -542,6 +543,7 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) {
in.Value(buffer.descriptorFormatted);
in.Value(buffer.formattedReadMask);
in.Value(buffer.scalar);
in.Value(buffer.typedAlignment);
});
reader.List(info.images, 72, [](Reader& in, ImageResource& image) {
in.Value(image.source);

2
core/shader/recompiler/ShaderDiskCache.hpp
View File

@ -12,7 +12,7 @@
namespace ShaderRecompiler::ShaderDiskCache {
inline constexpr std::uint32_t FormatVersion = 18;
inline constexpr std::uint32_t FormatVersion = 19;
enum class LoadStatus {
Loaded,

4
core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp
View File

@ -117,10 +117,6 @@ void RequireAlignedAtomic(SpirvValueEmitContext& ctx, const MemoryResourceAccess
if (resource.misalignment != 0u) ctx.Fail("buffer atomic on a V# whose base is not DWORD aligned");
}
void RequireAlignedFormattedElement(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryResourceAccess& resource, const SpirvBufferFormatInfo& info) {
if (resource.misalignment % std::min(info.byteSize, 4u) != 0u) ctx.Fail(inst, "is a formatted buffer access through a V# whose base is not aligned to its element");
}
std::uint32_t LoadSubwordInBounds(SpirvValueEmitContext& ctx, const MemoryResourceAccess& resource, std::uint32_t address, std::uint32_t index, std::uint32_t bits, bool signExtend) {
auto& state = ctx.state;
if (bits != 8u && bits != 16u) {

99
core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp
View File

@ -131,28 +131,48 @@ MemoryResourceAccess directAccess(SpirvValueEmitContext& context, const IrValue&
return access;
}
std::uint32_t baseMisalignment(SpirvValueEmitContext& context, const IrValue& instruction) {
const auto& memory = context.Memory(instruction);
if (memory.kind == ResourceKind::ScalarBuffer) return ConstantU32(context.state, 0u);
const auto first = PipelineSpecialization::BufferBase + memory.resource * PipelineSpecialization::BufferWords;
return context.state.module.SpecializationConstant(TypeU32(context.state), first + 3u, 0u);
}
std::uint32_t directWordIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t address) {
const auto relative = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u));
const auto base = Binary(state, spv::OpShiftRightLogical, TypeU32(state), access.byteOffset, ConstantU32(state, 2u));
return EmitAddU32(state, relative, base);
}
std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t bits) {
if (context.Memory(instruction).gpuDescriptor) return EmitBdaRead(context, instruction, address, bits);
auto& state = context.state;
const auto access = directAccess(context, instruction);
const auto byte = EmitAddU32(state, address, access.byteOffset);
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte, ConstantU32(state, 2u));
const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), EmitAndConstant(state, byte, 3u), ConstantU32(state, 3u));
const auto load = [&](std::uint32_t word) {
return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access, word), [&] {
const auto index = directWordIndex(state, access, address);
const auto misalignment = baseMisalignment(context, instruction);
const auto unaligned = nonzero(state, misalignment);
const auto next = EmitAddU32(state, index, ConstantU32(state, 1u));
const auto last = Select(state, TypeU32(state), unaligned, next, index);
return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] {
const auto load = [&](std::uint32_t word) {
const auto value = state.module.AllocateId();
const auto pointer = EmitMemoryElementPointer(state, access, word);
if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, access.memoryAccess);
else state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
return value;
};
const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), misalignment, ConstantU32(state, 3u));
const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(index), shift);
const auto high = EmitValueOrZeroIfCondition(state, unaligned, [&] {
return Binary(state, spv::OpShiftLeftLogical, TypeU32(state), load(next), Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32u), shift));
});
};
const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(index), shift);
const auto crosses = Binary(state, spv::OpUGreaterThan, TypeBool(state), shift, ConstantU32(state, 32u - bits));
const auto high = EmitValueOrZeroIfCondition(state, crosses, [&] {
return Binary(state, spv::OpShiftLeftLogical, TypeU32(state), load(EmitAddU32(state, index, ConstantU32(state, 1u))), Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32u), shift));
auto value = EmitOrU32(state, low, high);
if (bits != 32u) {
const auto subwordShift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), EmitAndConstant(state, address, 3u), ConstantU32(state, 3u));
value = Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, subwordShift);
}
return EmitAndConstant(state, value, bits == 32u ? 0xffffffffu : (1u << bits) - 1u);
});
return EmitAndConstant(state, EmitOrU32(state, low, high), bits == 32u ? 0xffffffffu : (1u << bits) - 1u);
}
void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u) {
@ -163,33 +183,38 @@ void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std
auto& state = context.state;
const auto access = directAccess(context, instruction);
const auto u32 = TypeU32(state);
const auto byte = EmitAddU32(state, address, access.byteOffset);
const auto index = Binary(state, spv::OpShiftRightLogical, u32, byte, ConstantU32(state, 2u));
const auto shift = Binary(state, spv::OpShiftLeftLogical, u32, EmitAndConstant(state, byte, 3u), ConstantU32(state, 3u));
const auto mask = ConstantU32(state, bits == 32u ? 0xffffffffu : (1u << bits) - 1u);
const auto masked = Binary(state, spv::OpBitwiseAnd, u32, value, mask);
const auto update = [&](std::uint32_t word, std::uint32_t wordMask, std::uint32_t replacement) {
EmitIfCondition(state, EmitMemoryElementInBounds(state, access, word), [&] {
const auto pointer = EmitMemoryElementPointer(state, access, word);
AtomicUpdate(state, pointer, access.kind, [&](std::uint32_t old) {
const auto index = directWordIndex(state, access, address);
const auto misalignment = baseMisalignment(context, instruction);
const auto unaligned = nonzero(state, misalignment);
const auto next = EmitAddU32(state, index, ConstantU32(state, 1u));
const auto last = Select(state, u32, unaligned, next, index);
EmitIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] {
auto mask = ConstantU32(state, bits == 32u ? 0xffffffffu : (1u << bits) - 1u);
auto masked = Binary(state, spv::OpBitwiseAnd, u32, value, mask);
if (bits != 32u) {
const auto subwordShift = Binary(state, spv::OpShiftLeftLogical, u32, EmitAndConstant(state, address, 3u), ConstantU32(state, 3u));
mask = Binary(state, spv::OpShiftLeftLogical, u32, mask, subwordShift);
masked = Binary(state, spv::OpShiftLeftLogical, u32, masked, subwordShift);
}
const auto update = [&](std::uint32_t word, std::uint32_t wordMask, std::uint32_t replacement) {
AtomicUpdate(state, EmitMemoryElementPointer(state, access, word), access.kind, [&](std::uint32_t old) {
return EmitOrU32(state, Binary(state, spv::OpBitwiseAnd, u32, old, Unary(state, spv::OpNot, u32, wordMask)), replacement);
});
});
};
const auto unaligned = nonzero(state, shift);
if (bits == 32u) {
EmitIfCondition(state, both(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), EmitMemoryElementInBounds(state, access, index)), [&] {
const auto pointer = EmitMemoryElementPointer(state, access, index);
if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpStore, pointer, value, access.memoryAccess);
else state.module.AddFunction(spv::OpStore, pointer, value);
});
}
EmitIfCondition(state, bits == 32u ? unaligned : ConstantBool(state, true), [&] {
update(index, Binary(state, spv::OpShiftLeftLogical, u32, mask, shift), Binary(state, spv::OpShiftLeftLogical, u32, masked, shift));
const auto crosses = Binary(state, spv::OpUGreaterThan, TypeBool(state), shift, ConstantU32(state, 32u - bits));
EmitIfCondition(state, crosses, [&] {
const auto remaining = Binary(state, spv::OpISub, u32, ConstantU32(state, 32u), shift);
update(EmitAddU32(state, index, ConstantU32(state, 1u)), Binary(state, spv::OpShiftRightLogical, u32, mask, remaining), Binary(state, spv::OpShiftRightLogical, u32, masked, remaining));
};
if (bits == 32u) {
EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), [&] {
const auto pointer = EmitMemoryElementPointer(state, access, index);
if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpStore, pointer, value, access.memoryAccess);
else state.module.AddFunction(spv::OpStore, pointer, value);
});
}
EmitIfCondition(state, bits == 32u ? unaligned : ConstantBool(state, true), [&] {
const auto shift = Binary(state, spv::OpShiftLeftLogical, u32, misalignment, ConstantU32(state, 3u));
update(index, Binary(state, spv::OpShiftLeftLogical, u32, mask, shift), Binary(state, spv::OpShiftLeftLogical, u32, masked, shift));
EmitIfCondition(state, unaligned, [&] {
const auto remaining = Binary(state, spv::OpISub, u32, ConstantU32(state, 32u), shift);
update(next, Binary(state, spv::OpShiftRightLogical, u32, mask, remaining), Binary(state, spv::OpShiftRightLogical, u32, masked, remaining));
});
});
});
}
@ -369,7 +394,7 @@ std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValu
const auto wide = directAccess(context, instruction, true);
const auto byte = EmitAddU32(state, runtime.guest, wide.byteOffset);
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte, ConstantU32(state, 3u));
const auto aligned = equal(state, EmitAndConstant(state, byte, 7u), ConstantU32(state, 0u));
const auto aligned = both(state, equal(state, baseMisalignment(context, instruction), ConstantU32(state, 0u)), equal(state, EmitAndConstant(state, byte, 7u), ConstantU32(state, 0u)));
const auto inBounds = both(state, runtime.inBounds, EmitMemoryElementInBounds(state, wide, index));
return EmitValueIfElse(state, both(state, aligned, inBounds), type, [&] {
const auto pointer = EmitStorageBufferElementPointer(state, wide, index, TypeStorageBufferU64ElementPointer(state));