diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index 1c1327cb..ac750555 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -1408,24 +1408,40 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address if (!selected) { return false; } - auto& image = m_slot_images[selected]; - vk::ClearColorValue color_clear {}; - float depth_clear = 0.0f; - uint8_t stencil_clear = 0; + auto& image = m_slot_images[selected]; + vk::ClearValue clear {}; if (aspect == vk::ImageAspectFlagBits::eColor) { - if (!DecodePackedColorClear(image.info.pixel_format, packed_clear, color_clear)) { + if (!DecodePackedColorClear(image.info.pixel_format, packed_clear, clear.color)) { return false; } } else { + uint8_t stencil_clear = 0; if ((aspect == vk::ImageAspectFlagBits::eDepth && - !DecodePackedDepthClear(image.info.pixel_format, packed_clear, depth_clear)) || + !DecodePackedDepthClear(image.info.pixel_format, packed_clear, clear.depthStencil.depth)) || (aspect == vk::ImageAspectFlagBits::eStencil && !DecodePackedStencilClear(packed_clear, stencil_clear))) { return false; } + clear.depthStencil.stencil = stencil_clear; } + ClearImage(command, selected, {aspect, 0, image.info.resources.levels, 0, image.backing.layers}, + clear); + return true; +} + +void TextureCache::ClearImage(CommandBuffer& command, ImageId id, + const vk::ImageSubresourceRange& range, const vk::ClearValue& clear) { + auto& image = m_slot_images[id]; + const auto aspects = image.info.IsDepth() ? ImageViewOps::DepthAspectMask(image.backing.format) + : vk::ImageAspectFlagBits::eColor; + EXIT_IF(command.IsInvalid() || image.depth_id || !range.aspectMask || + range.levelCount == 0 || range.baseMipLevel >= image.info.resources.levels || + range.levelCount > image.info.resources.levels - range.baseMipLevel || + range.layerCount == 0 || range.baseArrayLayer >= image.backing.layers || + range.layerCount > image.backing.layers - range.baseArrayLayer || + (range.aspectMask & aspects) != range.aspectMask); if (image.IsBufferModified() || image.IsCpuDirty()) { - InitializeImage(selected); + InitializeImage(id); if (image.info.samples == 1 && (image.IsBufferModified() || image.IsCpuDirty())) { EXIT("TextureCache: image clear retained guest ownership\n"); } @@ -1433,18 +1449,14 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address command.EndRendering(); image.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command.Handle()); - const vk::ImageSubresourceRange range {aspect, 0, VK_REMAINING_MIP_LEVELS, 0, - image.backing.layers}; - if (aspect == vk::ImageAspectFlagBits::eColor) { + if (range.aspectMask == vk::ImageAspectFlagBits::eColor) { command.Handle().clearColorImage(image.backing.image, vk::ImageLayout::eTransferDstOptimal, - &color_clear, 1, &range); + &clear.color, 1, &range); } else { - const vk::ClearDepthStencilValue clear {depth_clear, stencil_clear}; command.Handle().clearDepthStencilImage( - image.backing.image, vk::ImageLayout::eTransferDstOptimal, &clear, 1, &range); + image.backing.image, vk::ImageLayout::eTransferDstOptimal, &clear.depthStencil, 1, &range); } CommitGpuWrite(image); - return true; } void TextureCache::InvalidateMemory(uint64_t address, uint64_t size) { diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.h b/src/graphics/host_gpu/renderer/cache/textureCache.h index 72651d28..44ee6283 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.h +++ b/src/graphics/host_gpu/renderer/cache/textureCache.h @@ -147,6 +147,9 @@ private: uint64_t destination_size, DownloadPlan plan); void DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset); void CommitGpuWrite(Image& image); + // Caller holds m_lock. Clears only the selected aspects and subresources. + void ClearImage(CommandBuffer& command, ImageId id, const vk::ImageSubresourceRange& range, + const vk::ClearValue& clear); void PrepareImageCopy(Image& image); void RefreshCopySource(ImageId id); [[nodiscard]] bool CopyD16(Image& destination, Image& source); diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index 3a548612..81a12d30 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -204,6 +204,9 @@ private: void ResetBindings(); [[nodiscard]] bool TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input, const CommandBuffer& buffer); + [[nodiscard]] bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, + CommandBuffer& command, uint32_t group_x, + uint32_t group_y, uint32_t group_z, uint32_t mode); RenderContext& m_context; std::vector m_bound_images; diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index cc31c2a1..5a08f229 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,19 @@ static uint64_t BufferDescriptorSize(const ShaderBufferResource& descriptor) { return stride == 0 ? records : records * stride; } +static bool FillSourcesDisjoint(std::span sources, + GuestRange destination, uint32_t output_buffer = UINT32_MAX) { + for (uint32_t i = 0; i < sources.size(); ++i) { + if (i == output_buffer) continue; + const auto source = DecodeNativeDescriptor(sources[i]); + const auto bytes = BufferDescriptorSize(source); + if (source.Base48() > UINT64_MAX - bytes || + (source.Base48() < destination.End() && + destination.address < source.Base48() + bytes)) return false; + } + return true; +} + bool RenderExecutor::TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input, const CommandBuffer& buffer) { const auto& program = *input.stage.program; @@ -83,27 +97,28 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou uint64_t& resolved_size) { const auto& program = *input.stage.program; const auto& resources = input.stage.resources; - const auto& fill = resources.buffer_fill; - if (fill.element_size == 0 || fill.element_size > 16 || fill.element_size % 4 != 0 || - fill.buffer >= resources.buffers.size() || + const auto& fill = resources.uniform_fill; + if (fill.kind != ShaderRecompiler::IR::UniformFillKind::Buffer || fill.words == 0 || + fill.words > 4 || fill.resource >= resources.buffers.size() || resources.buffers.size() != program.info.buffers.size()) { return false; } - const auto& resource = program.info.buffers[fill.buffer]; - const auto& raw = resources.buffers[fill.buffer]; + const auto element_size = fill.words * sizeof(uint32_t); + const auto& resource = program.info.buffers[fill.resource]; + const auto& raw = resources.buffers[fill.resource]; const auto descriptor = DecodeNativeDescriptor(raw); constexpr std::array formats { Prospero::BufferFormat::k32UInt, Prospero::BufferFormat::k32_32UInt, Prospero::BufferFormat::k32_32_32UInt, Prospero::BufferFormat::k32_32_32_32UInt}; if (!resource.formatted || !resource.written || resource.read || resource.atomic || - resource.scalar || descriptor.Stride() != fill.element_size || - descriptor.Format() != formats[fill.element_size / 4 - 1] || descriptor.SwizzleEnabled() || + resource.scalar || descriptor.Stride() != element_size || + descriptor.Format() != formats[fill.words - 1] || descriptor.SwizzleEnabled() || descriptor.IndexStride() != 0 || descriptor.AddTid() || resource.packed_stride != descriptor.PackedStride() || raw.dword_count != 4 || descriptor.Base48() == 0) { return false; } - if (input.threads_num[0] == 0 || input.threads_num[0] != fill.group_stride || + if (input.threads_num[0] == 0 || input.threads_num[0] != fill.group_stride[0] || input.threads_num[1] != 1 || input.threads_num[2] != 1 || group_x == 0 || group_y != 1 || group_z != 1 || mode != (input.dispatch_thread_dimensions ? 0x61u : 0x41u)) { return false; @@ -119,25 +134,71 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou input.dispatch_threads_num[1] != 1 || input.dispatch_threads_num[2] != 1))) { return false; } - for (uint32_t i = 0; i < resources.buffers.size(); ++i) { - if (i == fill.buffer) continue; - const auto source = DecodeNativeDescriptor(resources.buffers[i]); - const auto bytes = BufferDescriptorSize(source); - if (source.Base48() > UINT64_MAX - bytes || - (source.Base48() < descriptor.Base48() + size && - descriptor.Base48() < source.Base48() + bytes)) { - return false; - } - } + if (!FillSourcesDisjoint(resources.buffers, {descriptor.Base48(), size}, fill.resource)) + return false; resolved_descriptor = descriptor; resolved_clear = fill.value; resolved_size = size; return true; } -static bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, CommandBuffer& command, - uint32_t group_x, uint32_t group_y, uint32_t group_z, - uint32_t mode) { +bool RenderExecutor::TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, + CommandBuffer& command, uint32_t group_x, + uint32_t group_y, uint32_t group_z, uint32_t mode) { + const auto& program = *input.stage.program; + const auto& resources = input.stage.resources; + const auto& fill = resources.uniform_fill; + auto& cache = command.GetContext().GetTextureCache(); + if (fill.kind == ShaderRecompiler::IR::UniformFillKind::Image) { + if (mode != 0x41u || input.dispatch_thread_dimensions || fill.words != 1 || fill.value > 255 || + resources.images.size() != 1 || program.info.images.size() != 1 || fill.resource != 0 || + resources.images[0].dword_count != 8 || input.threads_num[2] != 1 || + fill.group_stride[2] != 1) return false; + const auto descriptor = DecodeNativeDescriptor(resources.images[0]); + const auto& resource = program.info.images[0]; + if (descriptor.IsNull() || descriptor.Format() != Prospero::BufferFormat::k8UInt || + descriptor.Type() != Prospero::ImageType::kColor2DArray || + descriptor.MetaCompress() || descriptor.WriteCompress() || + descriptor.BaseLevel() > descriptor.LastLevel() || + descriptor.BaseLevel() > descriptor.MaxMip() || + descriptor.BaseArray5() > descriptor.Depth() || + descriptor.DstSelX() != 4 || (resource.shader_swizzle & 7u) != 4 || + resource.numeric_class != Prospero::TextureNumericClass::Uint || + resource.dimension != ShaderRecompiler::Decoder::ImageDimension::Dim2DArray || + resource.read || resource.atomic || !resource.written || + resource.mip_mode != ShaderRecompiler::IR::ImageMipMode::None) return false; + const std::array extents { + std::max(1u, (descriptor.Width5() + 1u) >> descriptor.BaseLevel()), + std::max(1u, (descriptor.Height5() + 1u) >> descriptor.BaseLevel()), + descriptor.Depth() - descriptor.BaseArray5() + 1u}; + const std::array groups {group_x, group_y, group_z}; + for (uint32_t axis = 0; axis < 3; ++axis) { + const uint64_t threads = input.threads_num[axis]; + // Guest image writes outside the descriptor dimensions are discarded. Only the + // final workgroup may extend beyond the selected image view. + if (threads == 0 || threads != fill.group_stride[axis] || + groups[axis] != (extents[axis] + threads - 1) / threads || + groups[axis] * threads > UINT32_MAX) return false; + } + const auto binding = ResolveTexture(resource, resources.images[0]); + const auto& destination = binding.desc.info.data; + if (!FillSourcesDisjoint(resources.buffers, destination)) return false; + std::scoped_lock lock {cache.m_lock}; + const auto& image = cache.GetImage(binding.image_id); + const auto& view = binding.desc.view_info; + if (image.backing.format != vk::Format::eD32SfloatS8Uint || image.info.samples != 1 || + image.info.stencil != destination || view.base_level >= image.backing.mip_levels || + view.base_layer >= image.backing.layers || view.layer_count != extents[2] || + view.layer_count > image.backing.layers - view.base_layer || + std::max(1u, image.info.extent.width >> view.base_level) != extents[0] || + std::max(1u, image.info.extent.height >> view.base_level) != extents[1]) return false; + const vk::ImageSubresourceRange range {vk::ImageAspectFlagBits::eStencil, view.base_level, + 1, view.base_layer, view.layer_count}; + vk::ClearValue clear {}; + clear.depthStencil = vk::ClearDepthStencilValue {0.0f, fill.value}; + cache.ClearImage(command, binding.image_id, range, clear); + return true; + } ShaderBufferResource descriptor; uint32_t packed_clear = 0; uint64_t size = 0; @@ -145,7 +206,6 @@ static bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, Com size)) { return false; } - auto& cache = command.GetContext().GetTextureCache(); if (!cache.ClearImageFromBuffer(command, descriptor.Base48(), size, packed_clear)) { // Recognized metadata-fill shaders access DCC as an ordinary storage buffer and may run // before the render target is bound. TryConsumeDccFill either consumes registered state diff --git a/src/graphics/shader/recompiler/ir/ResourceSnapshot.h b/src/graphics/shader/recompiler/ir/ResourceSnapshot.h index db08b3ec..4734fa07 100644 --- a/src/graphics/shader/recompiler/ir/ResourceSnapshot.h +++ b/src/graphics/shader/recompiler/ir/ResourceSnapshot.h @@ -16,13 +16,16 @@ struct DescriptorValue { } }; -struct BufferFill { - uint32_t buffer = 0; - uint32_t group_stride = 0; - uint32_t element_size = 0; - uint32_t value = 0; +enum class UniformFillKind { None, Buffer, Image }; - bool operator==(const BufferFill&) const = default; +struct UniformFill { + UniformFillKind kind = UniformFillKind::None; + uint32_t resource = 0; + std::array group_stride {}; + uint32_t words = 0; + uint32_t value = 0; + + bool operator==(const UniformFill&) const = default; }; struct ResourceSnapshot { @@ -31,7 +34,7 @@ struct ResourceSnapshot { std::vector samplers; std::vector flattened_srt; std::vector user_data; - BufferFill buffer_fill; + UniformFill uniform_fill; }; } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 61703b4e..969d2206 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -492,8 +492,8 @@ struct CompiledShaderInfo { BindingLayout bindings; }; -struct BufferFillPlan { - BufferFill fill; +struct UniformFillPlan { + UniformFill fill; std::array values; }; @@ -523,7 +523,7 @@ struct ResourcePlan { bool srt_plan_complete = false; bool resource_tracking_complete = false; ShaderInfo info; - BufferFillPlan buffer_fill; + UniformFillPlan uniform_fill; }; struct Program: ResourcePlan { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 569bce60..aab04bd7 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -300,16 +300,16 @@ static bool MaterializeSnapshot(const ResourcePlan& program, const SrtRuntime& r } auto& next = snapshot.resources; - const auto& fill = program.buffer_fill; - const auto words = fill.fill.element_size / sizeof(uint32_t); + const auto& fill = program.uniform_fill; + const auto words = fill.fill.words; std::array stored {}; if (words != 0 && EvaluateUniformValues(program, std::span(fill.values).first(words), runtime, std::span(stored).first(words)) && std::all_of(stored.begin(), stored.begin() + words, [&](uint32_t value) { return value == stored[0]; })) { - next.buffer_fill = fill.fill; - next.buffer_fill.value = stored[0]; + next.uniform_fill = fill.fill; + next.uniform_fill.value = stored[0]; } auto cursor = values.begin(); next.buffers.assign(cursor, cursor + program.info.buffers.size()); @@ -724,9 +724,10 @@ static std::vector ResourceControlFlow(const Program& program) { return blocks; } -// Nonnegative affine coefficients for constant, local X and workgroup X. Reject modular +// Nonnegative affine coefficients for constant, local and workgroup coordinates. Reject modular // arithmetic that could wrap; runtime coverage also bounds the largest invocation index. -static std::optional> FillIndex(Value value, uint32_t depth = 0) { +static std::optional> FillIndex(Value value, uint32_t axis, + uint32_t depth = 0) { value = value.Resolve(); if (depth > 32 || value.GetType() != Type::U32) { return {}; @@ -739,7 +740,7 @@ static std::optional> FillIndex(Value value, uint32_t de return {}; } const auto op = inst->GetOpcode(); - if (op == ValueOpcode::GetBuiltin && inst->Arg(1) == Value(0u)) { + if (op == ValueOpcode::GetBuiltin && inst->Arg(1) == Value(axis)) { if (inst->Arg(0) == Value(static_cast(StageInputKind::LocalInvocationId))) { return std::array {0, 1, 0}; } @@ -751,8 +752,8 @@ static std::optional> FillIndex(Value value, uint32_t de op != ValueOpcode::ShiftLeftLogical32) { return {}; } - auto left = FillIndex(inst->Arg(0), depth + 1); - auto right = FillIndex(inst->Arg(1), depth + 1); + auto left = FillIndex(inst->Arg(0), axis, depth + 1); + auto right = FillIndex(inst->Arg(1), axis, depth + 1); if (!left || !right) { return {}; } @@ -774,10 +775,10 @@ static std::optional> FillIndex(Value value, uint32_t de return left; } -static BufferFillPlan AnalyzeBufferFill(const Program& program) { +static UniformFillPlan AnalyzeUniformFill(const Program& program) { if (program.stage != ShaderType::Compute || program.blocks.empty() || program.blocks.size() != program.block_info.size() || program.info.uses_dma || - !program.info.images.empty() || !program.info.samplers.empty()) { + !program.info.samplers.empty()) { return {}; } std::unordered_set visited; @@ -788,7 +789,8 @@ static BufferFillPlan AnalyzeBufferFill(const Program& program) { for (const auto& inst: *program.blocks[index]) { if (AddressOpcodeInfoOf(inst.GetOpcode()).access != AddressAccess::None) return {}; if (!inst.MayHaveSideEffects()) continue; - if (store != nullptr || BufferAccessOf(inst.GetOpcode()) != BufferAccess::Write) + if (store != nullptr || (BufferAccessOf(inst.GetOpcode()) != BufferAccess::Write && + inst.GetOpcode() != ValueOpcode::ImageWrite)) return {}; store = &inst; } @@ -800,36 +802,64 @@ static BufferFillPlan AnalyzeBufferFill(const Program& program) { index = static_cast(next - program.block_info.begin()); } if (store == nullptr || visited.size() != program.blocks.size()) return {}; - const auto op = store->GetOpcode(); - constexpr std::array stores {ValueOpcode::StoreBufferU32, ValueOpcode::StoreBufferU32x2, - ValueOpcode::StoreBufferU32x3, ValueOpcode::StoreBufferU32x4}; - const auto store_op = std::ranges::find(stores, op); - if (store_op == stores.end() || store->Arg(2).Resolve() != Value(0u) || - store->Arg(3).Resolve() != Value(0u) || store->Arg(5).Resolve() != Value(true)) - return {}; - const auto& memory = program.memory_info.at(store->Flags().index); - if (!memory.formatted || memory.typed || !memory.idxen || memory.offen || memory.offset != 0 || - memory.data_bits != 32 || - memory.data_dwords != static_cast(store_op - stores.begin() + 1)) - return {}; - const auto address = FillIndex(store->Arg(1)); - if (!address || (*address)[0] != 0 || (*address)[1] != 1 || (*address)[2] == 0) return {}; for (const auto& buffer: program.info.buffers) { if (buffer.read && (!buffer.scalar || buffer.written)) return {}; } - BufferFillPlan result; - result.fill = {memory.resource, static_cast((*address)[2]), memory.data_dwords * 4, - 0}; - const auto data = store->Arg(4).Resolve(); + const auto& memory = program.memory_info.at(store->Flags().index); + UniformFillPlan result; + result.fill.resource = memory.resource; + Value data; + if (store->GetOpcode() == ValueOpcode::ImageWrite) { + if (program.info.images.size() != 1 || memory.dmask != 1 || memory.data_bits != 32 || + memory.image_has_mip || memory.image_sample_flags != 0 || memory.image_r128 || + memory.image_dimension != Decoder::ImageDimension::Dim2DArray || + store->Arg(3).Resolve() != Value(true)) return {}; + const auto& image = program.info.images[memory.resource]; + if (image.read || image.atomic || image.mip_mode != ImageMipMode::None) return {}; + const auto* address = store->Arg(1).ResolveInstruction(); + if (address == nullptr || address->GetOpcode() != ValueOpcode::MakeImageAddress) return {}; + for (uint32_t axis = 0; axis < 3; ++axis) { + const auto index = FillIndex(address->Arg(axis), axis); + if (!index || (*index)[0] != 0 || (*index)[1] != (axis < 2 ? 1u : 0u) || + (*index)[2] == 0 || (axis == 2 && (*index)[2] != 1)) return {}; + result.fill.group_stride[axis] = static_cast((*index)[2]); + } + const auto* values = store->Arg(2).ResolveInstruction(); + if (values == nullptr || values->GetOpcode() != ValueOpcode::CompositeConstructU32x4) + return {}; + result.fill.kind = UniformFillKind::Image; + result.fill.words = 1; + data = values->Arg(0); + } else { + if (!program.info.images.empty()) return {}; + const auto op = store->GetOpcode(); + constexpr std::array stores {ValueOpcode::StoreBufferU32, ValueOpcode::StoreBufferU32x2, + ValueOpcode::StoreBufferU32x3, ValueOpcode::StoreBufferU32x4}; + const auto store_op = std::ranges::find(stores, op); + if (store_op == stores.end() || store->Arg(2).Resolve() != Value(0u) || + store->Arg(3).Resolve() != Value(0u) || store->Arg(5).Resolve() != Value(true)) + return {}; + if (!memory.formatted || memory.typed || !memory.idxen || memory.offen || memory.offset != 0 || + memory.data_bits != 32 || + memory.data_dwords != static_cast(store_op - stores.begin() + 1)) + return {}; + const auto address = FillIndex(store->Arg(1), 0); + if (!address || (*address)[0] != 0 || (*address)[1] != 1 || (*address)[2] == 0) return {}; + result.fill.kind = UniformFillKind::Buffer; + result.fill.group_stride[0] = static_cast((*address)[2]); + result.fill.words = memory.data_dwords; + data = store->Arg(4); + } + data = data.Resolve(); const auto* vector = data.TryInstruction(); constexpr std::array composites {ValueOpcode::CompositeConstructU32x2, ValueOpcode::CompositeConstructU32x3, ValueOpcode::CompositeConstructU32x4}; - for (uint32_t i = 0; i < memory.data_dwords; ++i) { - if (memory.data_dwords > 1 && - (vector == nullptr || vector->GetOpcode() != composites[memory.data_dwords - 2])) + for (uint32_t i = 0; i < result.fill.words; ++i) { + if (result.fill.words > 1 && + (vector == nullptr || vector->GetOpcode() != composites[result.fill.words - 2])) return {}; - const auto word = memory.data_dwords == 1 ? data : vector->Arg(i); + const auto word = result.fill.words == 1 ? data : vector->Arg(i); if (word.GetType() != Type::U32 || !UniformIntegerValue(program, word)) return {}; result.values[i] = word; } @@ -895,9 +925,9 @@ ResourcePlan ExtractResourcePlan(const Program& program) { for (auto& block: plan.control_flow) { block.condition = Clone(block.condition); } - plan.buffer_fill = AnalyzeBufferFill(program); - for (uint32_t i = 0; i < plan.buffer_fill.fill.element_size / sizeof(uint32_t); ++i) { - plan.buffer_fill.values[i] = Clone(plan.buffer_fill.values[i]); + plan.uniform_fill = AnalyzeUniformFill(program); + for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) { + plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]); } plan.materialization_sources.reserve(plan.info.buffers.size() + plan.info.images.size() + plan.info.samplers.size()); diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index ca9ad335..566c36fa 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -35,7 +35,7 @@ bool SameResourceSnapshot(const ResourceSnapshot &lhs, return lhs.buffers == rhs.buffers && lhs.images == rhs.images && lhs.samplers == rhs.samplers && lhs.flattened_srt == rhs.flattened_srt && - lhs.user_data == rhs.user_data && lhs.buffer_fill == rhs.buffer_fill; + lhs.user_data == rhs.user_data && lhs.uniform_fill == rhs.uniform_fill; } template @@ -543,12 +543,12 @@ void TestComputeBufferFill() { const bool expected = !options.conditional && !options.shifted && !options.extra_store && !options.branch && (!options.scalar || options.clean); - Check((snapshot.buffer_fill.element_size != 0) == expected, + Check((snapshot.uniform_fill.words != 0) == expected, "fill proof accepted an unsafe store or missed the real GTA3 clear"); if (expected) { - Check(snapshot.buffer_fill.element_size == 4 && - snapshot.buffer_fill.group_stride == 64 && - snapshot.buffer_fill.value == + Check(snapshot.uniform_fill.words == 1 && + snapshot.uniform_fill.group_stride[0] == 64 && + snapshot.uniform_fill.value == (options.scalar ? 0x40404040u : 0u), "fill proof lost address coverage or the actual stored scalar"); } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index ccb854e5..5634e996 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -211,6 +211,13 @@ struct TextureCacheTestAccess { return std::unique_lock(cache.m_lock); } + static void ClearImage(TextureCache &cache, CommandBuffer &command, ImageId id, + const vk::ImageSubresourceRange &range, + const vk::ClearValue &clear) { + auto lock = Lock(cache); + cache.ClearImage(command, id, range, clear); + } + static void ConfigureGarbageCollection(TextureCache &cache, std::span oldest, uint64_t tick, uint64_t pressure) { @@ -368,6 +375,12 @@ struct TextureCacheTestAccess { }; struct RenderExecutorTestAccess { + static bool TryConsumeComputeImageClear(RenderExecutor &executor, + const ShaderComputeInputInfo &input, CommandBuffer &command, + uint32_t x, uint32_t y, uint32_t z, uint32_t mode) { + return executor.TryConsumeComputeImageClear(input, command, x, y, z, mode); + } + static bool TryConsumeComputeMetaClear(RenderExecutor &executor, const ShaderComputeInputInfo &input, const CommandBuffer &buffer) { @@ -5327,6 +5340,72 @@ public: "cleared image could not be read or written after ownership " "transfer"); + auto stencil_clear_desc = MakeLinearDesc( + base + 0x27e0000, 0x1000, vk::Format::eD32SfloatS8Uint, + Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D, + {13, 11, 1}, 2, 4, 1); + stencil_clear_desc.info.stencil = {base + 0x27e2000, 0x1000}; + stencil_clear_desc.info.resources.levels = 2; + stencil_clear_desc.info.mip_layout[0] = {0, 0x800, 13, 11}; + stencil_clear_desc.info.mip_layout[1] = {0x800, 0x800, 6, 5}; + stencil_clear_desc.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + const auto stencil_clear_id = texture_cache.FindImage(stencil_clear_desc); + auto &stencil_clear_image = texture_cache.GetImage(stencil_clear_id); + stencil_clear_image.Transit(vk::ImageLayout::eTransferDstOptimal, + vk::AccessFlagBits2::eTransferWrite, {}, command.Handle()); + const vk::ClearDepthStencilValue initial_depth_stencil{0.75f, 0x91}; + const vk::ImageSubresourceRange all_depth_stencil{ + stencil_clear_desc.view_info.aspect, 0, 2, 0, 2}; + command.Handle().clearDepthStencilImage(stencil_clear_image.backing.image, + vk::ImageLayout::eTransferDstOptimal, &initial_depth_stencil, 1, + &all_depth_stencil); + texture_cache.MarkGpuWritten(stencil_clear_id); + vk::ClearValue stencil_value{}; + stencil_value.depthStencil = vk::ClearDepthStencilValue{0.f, 0x35}; + TextureCacheTestAccess::ClearImage(texture_cache, command, stencil_clear_id, + {vk::ImageAspectFlagBits::eStencil, 1, 1, 1, 1}, stencil_value); + + // Read every mip/layer of both aspects to prove that the stencil clear is local. + auto stencil_readback = CreateHostBuffer(name, 0x4000, + vk::BufferUsageFlagBits::eTransferDst, {}); + std::vector stencil_copies; + for (uint32_t mip = 0; mip < 2; mip++) { + for (uint32_t layer = 0; layer < 2; layer++) { + for (const auto aspect : {vk::ImageAspectFlagBits::eDepth, + vk::ImageAspectFlagBits::eStencil}) { + vk::BufferImageCopy copy{}; + copy.bufferOffset = stencil_copies.size() * 0x800; + copy.imageSubresource = {aspect, mip, layer, 1}; + copy.imageExtent = {13u >> mip, 11u >> mip, 1}; + stencil_copies.push_back(copy); + } + } + } + stencil_clear_image.Download(stencil_copies, stencil_readback.buffer, 0, + stencil_readback.size); + HostReadBarrier(stencil_readback.buffer, stencil_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + scheduler.Finish(); + const auto stencil_words = ReadBuffer(name, stencil_readback, 0x1000); + const auto *stencil_bytes = reinterpret_cast(stencil_words.data()); + for (const auto © : stencil_copies) { + const auto &sub = copy.imageSubresource; + const auto count = copy.imageExtent.width * copy.imageExtent.height; + const auto *bytes = stencil_bytes + copy.bufferOffset; + const bool depth = sub.aspectMask == vk::ImageAspectFlagBits::eDepth; + for (uint32_t pixel = 0; pixel < count; pixel++) { + uint32_t actual = bytes[pixel]; + if (depth) std::memcpy(&actual, bytes + pixel * 4, 4); + const uint32_t expected = depth ? std::bit_cast(0.75f) + : sub.mipLevel == 1 && sub.baseArrayLayer == 1 ? 0x35u : 0x91u; + Require(name, "stencil clear preserves depth and neighbors", actual == expected, + "selected stencil clear changed depth, another mip, or another layer"); + } + } + DestroyBuffer(&stencil_readback); + constexpr uint64_t partial_image_offset = 0xa000; constexpr uint64_t partial_buffer_offset = 0xa010; constexpr uint64_t partial_clean_offset = 0xa020; @@ -9725,35 +9804,100 @@ public: "RebindImages did not acquire the associated depth owner"); RenderExecutorTestAccess::ResetBindings(executor); - ShaderRecompiler::IR::CompiledShaderInfo stencil_storage_program{}; - stencil_storage_program.stage = ShaderType::Compute; - auto stencil_storage_resource = stencil_resource; - stencil_storage_resource.resource_class = - ShaderRecompiler::IR::ImageResourceClass::Storage; - stencil_storage_resource.read = false; - stencil_storage_resource.written = true; - stencil_storage_program.info.images.push_back(stencil_storage_resource); + // Execute the captured IMAGE_STORE clear through production fill recognition. + const std::array stencil_shader{ + 0xd7460000u, 0x0401060cu, 0xf4201a84u, 0xfa000000u, + 0xbf8cc07fu, 0x7e06026au, 0xd7460001u, 0x0405060du, + 0x7e04020eu, 0xf0200128u, 0x00000300u, 0xbf810000u}; + uint32_t stencil_byte = 0x35; + const auto value_address = reinterpret_cast(&stencil_byte); auto storage_stencil = stencil; + storage_stencil.fields[3] = (storage_stencil.fields[3] & 0x0fffffffu) | + (static_cast(Prospero::ImageType::kColor2DArray) << 28u); storage_stencil.fields[5] = 0x00700000u; - ShaderRecompiler::IR::DescriptorValue stencil_storage_descriptor{}; - std::copy(std::begin(storage_stencil.fields), - std::end(storage_stencil.fields), - stencil_storage_descriptor.dwords.begin()); - stencil_storage_descriptor.dword_count = 8; - ShaderRecompiler::IR::ResourceSnapshot stencil_storage_snapshot{}; - stencil_storage_snapshot.images.push_back(stencil_storage_descriptor); - ShaderStageRuntime stencil_storage_runtime{ - &stencil_storage_program, std::move(stencil_storage_snapshot)}; - auto storage_redirected = - executor.PrepareBindings(stencil_storage_runtime); - executor.RebindImages(storage_redirected); - Require( - name, "storage stencil acquisition", - storage_redirected.resources.images[0].image_id == depth_id && - storage_redirected.resources.images[0].image_view != nullptr && - texture_cache.GetImage(depth_id).usage.storage, - "storage stencil binding did not acquire the associated depth owner"); - RenderExecutorTestAccess::ResetBindings(executor); + std::array stencil_userdata{}; + std::copy_n(storage_stencil.fields, 8, stencil_userdata.begin()); + stencil_userdata[8] = static_cast(value_address); + stencil_userdata[9] = static_cast(value_address >> 32) | (4u << 16); + stencil_userdata[10] = 1; + stencil_userdata[11] = 0x14204u; + ShaderComputeInputInfo stencil_compute{}; + stencil_compute.threads_num[0] = stencil_compute.threads_num[1] = 8; + stencil_compute.threads_num[2] = 1; + stencil_compute.workgroup_register = 12; + stencil_compute.thread_ids_num = 2; + std::fill_n(stencil_compute.group_id, 3, true); + ShaderRecompiler::CompileOptions stencil_options; + stencil_options.stage = ShaderType::Compute; + stencil_options.user_data = stencil_userdata; + stencil_options.input_info.compute = &stencil_compute; + auto stencil_translated = ShaderRecompiler::TranslateProgram(stencil_shader, stencil_options); + auto stencil_plan = ShaderRecompiler::IR::ExtractResourcePlan(stencil_translated.program); + ShaderRecompiler::IR::ResourceSpecialization stencil_specialization; + Require(name, "stencil fill materialization", + ShaderRecompiler::IR::MaterializeResources(stencil_plan, + {.user_data = stencil_userdata, .userdata = &stencil_byte, + .read_specialization_memory = +[](void *data, uint64_t address, uint32_t *word) { + if (address != reinterpret_cast(data)) return false; + *word = *static_cast(data); + return true; + }}, stencil_compute.stage.resources, stencil_specialization), + "captured stencil shader could not resolve its clear byte"); + ShaderRecompiler::IR::ApplyResourceSpecialization(stencil_translated.program, + stencil_specialization); + ShaderRecompiler::IR::CompiledShaderInfo stencil_program{}; + stencil_program.stage = ShaderType::Compute; + stencil_program.info = std::move(stencil_translated.program.info); + stencil_compute.stage.program = &stencil_program; + vk::ClearValue stencil_initial{}; + stencil_initial.depthStencil = vk::ClearDepthStencilValue{0.625f, 0x91}; + TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), depth_id, + {vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1}, + stencil_initial); + Require(name, "stencil fill coverage", + !RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute, + scheduler.Current(), 0, 1, 1, 0x41u) && + !RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute, + scheduler.Current(), 2, 1, 1, 0x41u), + "incomplete or excessive stencil dispatch was consumed"); + auto &stencil_source = stencil_compute.stage.resources.buffers[0]; + const auto clean_source = stencil_source; + stencil_source.dwords[0] = static_cast(stencil_address); + stencil_source.dwords[1] = static_cast(stencil_address >> 32) | (4u << 16); + Require(name, "stencil fill source alias", + !RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute, + scheduler.Current(), 1, 1, 1, 0x41u), + "an aliased scalar load was consumed as a uniform image clear"); + stencil_source = clean_source; + Require(name, "stencil IMAGE_STORE clear", + RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute, + scheduler.Current(), 1, 1, 1, 0x41u), + "captured clear attempted an unsupported depth/stencil storage binding"); + auto stencil_readback = CreateHostBuffer(name, 8, + vk::BufferUsageFlagBits::eTransferDst, {}); + const std::array stencil_copies{{ + {0, 0, 0, {vk::ImageAspectFlagBits::eDepth, 0, 0, 1}, {}, {1, 1, 1}}, + {4, 0, 0, {vk::ImageAspectFlagBits::eStencil, 0, 0, 1}, {}, {1, 1, 1}}}}; + texture_cache.GetImage(depth_id).Download(stencil_copies, stencil_readback.buffer, 0, 8); + vk::MemoryBarrier2 stencil_host_barrier{}; + stencil_host_barrier.sType = vk::StructureType::eMemoryBarrier2; + stencil_host_barrier.srcStageMask = vk::PipelineStageFlagBits2::eTransfer; + stencil_host_barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite; + stencil_host_barrier.dstStageMask = vk::PipelineStageFlagBits2::eHost; + stencil_host_barrier.dstAccessMask = vk::AccessFlagBits2::eHostRead; + vk::DependencyInfo stencil_dependency{}; + stencil_dependency.sType = vk::StructureType::eDependencyInfo; + stencil_dependency.memoryBarrierCount = 1; + stencil_dependency.pMemoryBarriers = &stencil_host_barrier; + scheduler.Current().Handle().pipelineBarrier2(stencil_dependency); + scheduler.Finish(); + const auto stencil_result = ReadBuffer(name, stencil_readback, 2); + Require(name, "stencil clear retains depth", + stencil_result[0] == std::bit_cast(0.625f) && + (stencil_result[1] & 0xffu) == 0x35 && + !texture_cache.GetImage(depth_id).usage.storage, + "stencil clear altered depth, lost its scalar value, or used a storage view"); + DestroyBuffer(&stencil_readback); resources.UnmapMemory(base, allocation_size); scheduler.Finish(); RenderExecutorTestAccess::DestroyDescriptorPipelines( diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 66c83ed9..94ae8021 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -11017,6 +11017,92 @@ void TestTypedDescriptorRealWideMoveTranslation() { } } +void TestComputeImageFill() { + using namespace ShaderRecompiler::IR; + // Captured GTA3 stencil clear: one scalar load and one IMAGE_STORE, with + // x/y = local + 8 * group and array layer = group.z. + const uint32_t shader[] = { + 0xd7460000u, 0x0401060cu, 0xf4201a84u, 0xfa000000u, + 0xbf8cc07fu, 0x7e06026au, 0xd7460001u, 0x0405060du, + 0x7e04020eu, 0xf0200128u, 0x00000300u, 0xbf810000u, + }; + uint32_t value = 0x7bu; + const auto address = reinterpret_cast(&value); + const std::array userdata = { + 0x20021000u, 0xc0500000u, 0x01d8c347u, 0xd1800204u, + 0u, 0x00700000u, 0u, 0u, + static_cast(address), + static_cast(address >> 32) | (4u << 16), 1u, 0x14204u, + }; + ShaderComputeInputInfo compute{}; + compute.threads_num[0] = compute.threads_num[1] = 8; + compute.threads_num[2] = 1; + compute.workgroup_register = 12; + compute.thread_ids_num = 2; + compute.group_id[0] = compute.group_id[1] = compute.group_id[2] = true; + auto options = MakeCompileOptions(ShaderType::Compute); + options.input_info.compute = &compute; + options.user_data = userdata; + enum class Mutation { None, Offset, WrongAxis, VaryingValue, Predicate, ExtraStore, A16 }; + const auto Run = [&](Mutation mutation, bool clean = true) { + auto translated = ShaderRecompiler::TranslateProgram(shader, options); + auto &program = translated.program; + for (auto *block : program.blocks) { + const auto found = std::ranges::find(*block, ValueOpcode::ImageWrite, &Inst::GetOpcode); + if (found == block->end()) continue; + auto &store = *found; + auto *coordinates = store.Arg(1).ResolveInstruction(); + auto *values = store.Arg(2).ResolveInstruction(); + Check(coordinates && values, "stencil fill fixture lost its image operands"); + const auto x = coordinates->Arg(0); + switch (mutation) { + case Mutation::None: break; + case Mutation::Offset: + coordinates->SetArg(0, Value(&*block->PrependNewInst(found, ValueOpcode::IAdd32, + {x, Value(1u)}))); + break; + case Mutation::WrongAxis: coordinates->SetArg(1, x); break; + case Mutation::VaryingValue: values->SetArg(0, x); break; + case Mutation::Predicate: + store.SetArg(3, Value(&*block->PrependNewInst(found, ValueOpcode::ULessThan32, + {x, Value(32u)}))); + break; + case Mutation::ExtraStore: + block->PrependNewInst(found, ValueOpcode::ImageWrite, + {store.Arg(0), store.Arg(1), store.Arg(2), store.Arg(3)}, store.Flags()); + break; + case Mutation::A16: + program.memory_info[store.Flags().index].image_sample_flags |= + ShaderRecompiler::Decoder::ImageSampleFlagA16; + break; + } + break; + } + auto plan = ExtractResourcePlan(program); + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + Check(MaterializeResources(plan, + {.user_data = userdata, .read_memory = ReadHostTestMemory, + .read_specialization_memory = clean ? ReadHostTestMemory : nullptr}, + snapshot, specialization), + "captured stencil clear did not materialize"); + const auto &fill = snapshot.uniform_fill; + const bool expected = mutation == Mutation::None && clean; + Check((fill.kind == UniformFillKind::Image) == expected, + "image fill proof accepted an unsafe store or missed the captured clear"); + if (expected) { + Check(fill.resource == 0 && fill.words == 1 && fill.value == value && + fill.group_stride == std::array{8, 8, 1}, + "image fill lost its scalar value or axis coverage"); + } + }; + Run(Mutation::None); + Run(Mutation::None, false); + for (const auto mutation : {Mutation::Offset, Mutation::WrongAxis, Mutation::VaryingValue, + Mutation::Predicate, Mutation::ExtraStore, Mutation::A16}) + Run(mutation); +} + void TestTypedDescriptorRealCarryAndScalarLoads() { const uint32_t carry_shader[] = { EncodeSop1(0x1f, 0, 0), // s_getpc_b64 s[0:1] @@ -12238,6 +12324,7 @@ int main() { TestRenderTargetReverseExportMapping(); TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled(); TestTypedDescriptorRealWideMoveTranslation(); + TestComputeImageFill(); TestTypedDescriptorRealCarryAndScalarLoads(); TestSrtWalkerRealSmemTranslation(); TestSrtWalkerVccBaseTranslation();