From c32270c39249ada403c518c4bb28e8f807f00e1a Mon Sep 17 00:00:00 2001 From: Bruno <56844494+fab918@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:27:26 +0200 Subject: [PATCH] Fix guest states rendering (#391) * Improve rendering accuracy Handle a few guest states that were not rendered correctly. * Refine buffer and shader handling --- .../backend/spirv/emit_spirv_special.cpp | 11 +- .../backend/spirv/spirv_emit_context.cpp | 12 +- .../ir_opt/texture_pass.cpp | 13 +- src/yuzu_shader_recompiler/runtime_info.h | 3 + .../buffer_cache/buffer_base.h | 3 + .../buffer_cache/buffer_cache.h | 368 +++++++++++++++++- .../buffer_cache/buffer_cache_base.h | 65 ++++ src/yuzu_video_core/engines/draw_manager.cpp | 13 +- src/yuzu_video_core/engines/draw_manager.h | 7 +- src/yuzu_video_core/engines/maxwell_3d.cpp | 18 + src/yuzu_video_core/engines/maxwell_3d.h | 3 + .../host1x/gpu_device_memory_manager.cpp | 11 +- src/yuzu_video_core/macro/macro_hle.cpp | 42 +- src/yuzu_video_core/memory_manager.cpp | 59 +++ src/yuzu_video_core/memory_manager.h | 16 + src/yuzu_video_core/rasterizer_interface.h | 5 +- .../renderer_opengl/gl_rasterizer.cpp | 3 +- .../renderer_opengl/gl_rasterizer.h | 2 +- .../renderer_vulkan/fixed_pipeline_state.cpp | 39 ++ .../renderer_vulkan/fixed_pipeline_state.h | 3 + .../renderer_vulkan/vk_pipeline_cache.cpp | 3 +- .../renderer_vulkan/vk_rasterizer.cpp | 13 +- .../renderer_vulkan/vk_rasterizer.h | 2 +- .../texture_cache/image_view_base.cpp | 18 +- 24 files changed, 692 insertions(+), 40 deletions(-) diff --git a/src/yuzu_shader_recompiler/backend/spirv/emit_spirv_special.cpp b/src/yuzu_shader_recompiler/backend/spirv/emit_spirv_special.cpp index 962b964e..63422633 100644 --- a/src/yuzu_shader_recompiler/backend/spirv/emit_spirv_special.cpp +++ b/src/yuzu_shader_recompiler/backend/spirv/emit_spirv_special.cpp @@ -138,7 +138,11 @@ void EmitEmitVertex(EmitContext& ctx, const IR::Value& stream) { ConvertDepthMode(ctx); } if (!ctx.profile.support_geometry_streams) { - throw NotImplementedException("Geometry streams"); + // The non-stream instructions are exactly stream 0 and need no GeometryStreams capability + if (!stream.IsImmediate() || stream.U32() != 0) { + throw NotImplementedException("Geometry streams"); + } + ctx.OpEmitVertex(); } else if (stream.IsImmediate()) { ctx.OpEmitStreamVertex(ctx.Def(stream)); } else { @@ -151,7 +155,10 @@ void EmitEmitVertex(EmitContext& ctx, const IR::Value& stream) { void EmitEndPrimitive(EmitContext& ctx, const IR::Value& stream) { if (!ctx.profile.support_geometry_streams) { - throw NotImplementedException("Geometry streams"); + if (!stream.IsImmediate() || stream.U32() != 0) { + throw NotImplementedException("Geometry streams"); + } + ctx.OpEndPrimitive(); } else if (stream.IsImmediate()) { ctx.OpEndStreamPrimitive(ctx.Def(stream)); } else { diff --git a/src/yuzu_shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/yuzu_shader_recompiler/backend/spirv/spirv_emit_context.cpp index 6e18f8f2..dd08033f 100644 --- a/src/yuzu_shader_recompiler/backend/spirv/spirv_emit_context.cpp +++ b/src/yuzu_shader_recompiler/backend/spirv/spirv_emit_context.cpp @@ -1742,12 +1742,20 @@ void EmitContext::DefineOutputs(const IR::Program& program) { break; case Stage::Fragment: for (u32 index = 0; index < 8; ++index) { - if (!info.stores_frag_color[index] && !profile.need_declared_frag_colors) { + // With dual source blending, color output 1 is the second source of render target 0 + const bool is_second_source{runtime_info.dual_source_blend && index == 1}; + if (!info.stores_frag_color[index] && !profile.need_declared_frag_colors && + !is_second_source) { continue; } const Id type{GetAttributeType(*this, runtime_info.color_output_types[index])}; frag_color[index] = DefineOutput(*this, type, std::nullopt); - Decorate(frag_color[index], spv::Decoration::Location, index); + if (is_second_source) { + Decorate(frag_color[index], spv::Decoration::Location, 0U); + Decorate(frag_color[index], spv::Decoration::Index, 1U); + } else { + Decorate(frag_color[index], spv::Decoration::Location, index); + } Name(frag_color[index], fmt::format("frag_color{}", index)); } if (info.stores_frag_depth) { diff --git a/src/yuzu_shader_recompiler/ir_opt/texture_pass.cpp b/src/yuzu_shader_recompiler/ir_opt/texture_pass.cpp index 611428db..9c83e4b2 100644 --- a/src/yuzu_shader_recompiler/ir_opt/texture_pass.cpp +++ b/src/yuzu_shader_recompiler/ir_opt/texture_pass.cpp @@ -463,13 +463,18 @@ std::optional TryGetConstBuffer(const IR::Inst* inst, Environme }; } +std::optional TrackBindless(Environment& env, IR::Block* block, IR::Inst& inst) { + std::optional track_addr{Track(inst.Arg(0), env)}; + if (!track_addr) { + track_addr = TrackLocalLoad(inst.Arg(0), block, env); + } + return track_addr; +} + TextureInst MakeInst(Environment& env, IR::Block* block, IR::Inst& inst) { ConstBufferAddr addr; if (IsBindless(inst)) { - std::optional track_addr{Track(inst.Arg(0), env)}; - if (!track_addr) { - track_addr = TrackLocalLoad(inst.Arg(0), block, env); - } + const std::optional track_addr{TrackBindless(env, block, inst)}; if (!track_addr) { throw NotImplementedException("Failed to track bindless texture constant buffer"); } diff --git a/src/yuzu_shader_recompiler/runtime_info.h b/src/yuzu_shader_recompiler/runtime_info.h index c71682ae..9a7af31b 100644 --- a/src/yuzu_shader_recompiler/runtime_info.h +++ b/src/yuzu_shader_recompiler/runtime_info.h @@ -113,6 +113,9 @@ struct RuntimeInfo { // Vulkan requires fragment outputs to match the attachment scalar type. // Float remains the default for other backends. std::array color_output_types{}; + + /// The first render target blends with a second source color taken from color output 1 + bool dual_source_blend{}; }; } // namespace Shader diff --git a/src/yuzu_video_core/buffer_cache/buffer_base.h b/src/yuzu_video_core/buffer_cache/buffer_base.h index 74262a53..af752574 100644 --- a/src/yuzu_video_core/buffer_cache/buffer_base.h +++ b/src/yuzu_video_core/buffer_cache/buffer_base.h @@ -38,6 +38,9 @@ public: static constexpr u64 BASE_PAGE_BITS = 16; static constexpr u64 BASE_PAGE_SIZE = 1ULL << BASE_PAGE_BITS; + /// Changes every time the host contents of the buffer change + u64 content_version{}; + explicit BufferBase(VAddr cpu_addr_, u64 size_bytes_) : cpu_addr{cpu_addr_}, size_bytes{size_bytes_} {} diff --git a/src/yuzu_video_core/buffer_cache/buffer_cache.h b/src/yuzu_video_core/buffer_cache/buffer_cache.h index 27b19622..9446762e 100644 --- a/src/yuzu_video_core/buffer_cache/buffer_cache.h +++ b/src/yuzu_video_core/buffer_cache/buffer_cache.h @@ -99,6 +99,7 @@ void BufferCache

::TickFrame() { } ++frame_tick; delayed_destruction_ring.Tick(); + RemoveUnusedSparseBuffers(); for (auto& buffer : async_buffers_death_ring) { runtime.FreeDeferredStagingBuffer(buffer); @@ -234,6 +235,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am src_buffer.MarkUsage(copy.src_offset, copy.size); dest_buffer.MarkUsage(copy.dst_offset, copy.size); runtime.CopyBuffer(dest_buffer, src_buffer, copies, true); + MarkBufferContentChanged(dest_buffer); if (has_new_downloads) { memory_tracker.MarkRegionAsGpuModified(*cpu_dest_address, amount); } @@ -262,6 +264,7 @@ bool BufferCache

::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) { Buffer& dest_buffer = slot_buffers[buffer]; const u32 offset = dest_buffer.Offset(*cpu_dst_address); runtime.ClearBuffer(dest_buffer, offset, size, value); + MarkBufferContentChanged(dest_buffer); dest_buffer.MarkUsage(offset, size); return true; } @@ -398,7 +401,15 @@ std::tuple BufferCache

::VertexBufferBinding(u3 template std::tuple BufferCache

::IndexBufferBinding() { const Binding& binding = channel_state->index_buffer; - if (binding.buffer_id == NULL_BUFFER_ID || binding.size == 0) { + if (binding.size == 0) { + return {nullptr, 0, 0}; + } + if (channel_state->is_index_buffer_sparse) { + const SparseBinding sparse = + SynchronizeSparseBuffer(channel_state->index_buffer_sparse_gpu_addr, binding.size); + return {&slot_buffers[sparse.buffer_id], sparse.offset, sparse.size}; + } + if (binding.buffer_id == NULL_BUFFER_ID) { return {nullptr, 0, 0}; } Buffer& buffer = slot_buffers[binding.buffer_id]; @@ -711,6 +722,22 @@ bool BufferCache

::IsRegionCpuModified(DAddr addr, size_t size) { template void BufferCache

::BindHostIndexBuffer() { + if (channel_state->is_index_buffer_sparse) { + const SparseBinding sparse = SynchronizeSparseBuffer( + channel_state->index_buffer_sparse_gpu_addr, channel_state->index_buffer.size); + Buffer& sparse_buffer = slot_buffers[sparse.buffer_id]; + const auto& sparse_draw = maxwell3d->draw_manager->GetDrawState(); + if constexpr (HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT) { + const u32 first_offset = sparse.offset + sparse_draw.index_buffer.first * + sparse_draw.index_buffer.FormatSizeInBytes(); + runtime.BindIndexBuffer(sparse_buffer, first_offset, sparse.size); + } else { + runtime.BindIndexBuffer(sparse_draw.topology, sparse_draw.index_buffer.format, + sparse_draw.index_buffer.first, sparse_draw.index_buffer.count, + sparse_buffer, sparse.offset, sparse.size); + } + return; + } Buffer& buffer = slot_buffers[channel_state->index_buffer.buffer_id]; TouchBuffer(buffer, channel_state->index_buffer.buffer_id); const u32 offset = buffer.Offset(channel_state->index_buffer.device_addr); @@ -891,6 +918,19 @@ void BufferCache

::BindHostGraphicsStorageBuffers(size_t stage) { u32 binding_index = 0; ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { const Binding& binding = channel_state->storage_buffers[stage][index]; + if (binding.is_sparse) { + const SparseBinding sparse = + SynchronizeSparseBuffer(binding.sparse_gpu_addr, binding.size); + Buffer& sparse_buffer = slot_buffers[sparse.buffer_id]; + if constexpr (NEEDS_BIND_STORAGE_INDEX) { + runtime.BindStorageBuffer(stage, binding_index, sparse_buffer, sparse.offset, + sparse.size, false); + ++binding_index; + } else { + runtime.BindStorageBuffer(sparse_buffer, sparse.offset, sparse.size, false); + } + return; + } Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); const u32 size = binding.size; @@ -917,6 +957,13 @@ template void BufferCache

::BindHostGraphicsTextureBuffers(size_t stage) { ForEachEnabledBit(channel_state->enabled_texture_buffers[stage], [&](u32 index) { const TextureBufferBinding& binding = channel_state->texture_buffers[stage][index]; + if (binding.is_sparse) { + const SparseBinding sparse = + SynchronizeSparseBuffer(binding.sparse_gpu_addr, binding.size); + runtime.BindTextureBuffer(slot_buffers[sparse.buffer_id], sparse.offset, sparse.size, + binding.format); + return; + } Buffer& buffer = slot_buffers[binding.buffer_id]; const u32 size = binding.size; SynchronizeBuffer(buffer, binding.device_addr, size); @@ -1003,6 +1050,19 @@ void BufferCache

::BindHostComputeStorageBuffers() { u32 binding_index = 0; ForEachEnabledBit(channel_state->enabled_compute_storage_buffers, [&](u32 index) { const Binding& binding = channel_state->compute_storage_buffers[index]; + if (binding.is_sparse) { + const SparseBinding sparse = + SynchronizeSparseBuffer(binding.sparse_gpu_addr, binding.size); + Buffer& sparse_buffer = slot_buffers[sparse.buffer_id]; + if constexpr (NEEDS_BIND_STORAGE_INDEX) { + runtime.BindComputeStorageBuffer(binding_index, sparse_buffer, sparse.offset, + sparse.size, false); + ++binding_index; + } else { + runtime.BindStorageBuffer(sparse_buffer, sparse.offset, sparse.size, false); + } + return; + } Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); const u32 size = binding.size; @@ -1030,6 +1090,13 @@ template void BufferCache

::BindHostComputeTextureBuffers() { ForEachEnabledBit(channel_state->enabled_compute_texture_buffers, [&](u32 index) { const TextureBufferBinding& binding = channel_state->compute_texture_buffers[index]; + if (binding.is_sparse) { + const SparseBinding sparse = + SynchronizeSparseBuffer(binding.sparse_gpu_addr, binding.size); + runtime.BindTextureBuffer(slot_buffers[sparse.buffer_id], sparse.offset, sparse.size, + binding.format); + return; + } Buffer& buffer = slot_buffers[binding.buffer_id]; const u32 size = binding.size; SynchronizeBuffer(buffer, binding.device_addr, size); @@ -1094,6 +1161,7 @@ void BufferCache

::UpdateIndexBuffer() { return; } flags[Dirty::IndexBuffer] = false; + channel_state->is_index_buffer_sparse = false; if (!draw_state.inline_index_draw_indexes.empty()) [[unlikely]] { auto inline_index_size = static_cast(draw_state.inline_index_draw_indexes.size()); u32 buffer_size = Common::AlignUp(inline_index_size, CACHING_PAGESIZE); @@ -1123,6 +1191,17 @@ void BufferCache

::UpdateIndexBuffer() { channel_state->index_buffer = NULL_BINDING; return; } + if (!gpu_memory->IsContinuousRange(gpu_addr_begin, size)) { + UpdateSparseBuffer(gpu_addr_begin, size); + channel_state->is_index_buffer_sparse = true; + channel_state->index_buffer_sparse_gpu_addr = gpu_addr_begin; + channel_state->index_buffer = Binding{ + .device_addr = *device_addr, + .size = size, + .buffer_id = BufferId{}, + }; + return; + } channel_state->index_buffer = Binding{ .device_addr = *device_addr, .size = size, @@ -1214,6 +1293,10 @@ void BufferCache

::UpdateStorageBuffers(size_t stage) { ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { // Resolve buffer Binding& binding = channel_state->storage_buffers[stage][index]; + if (binding.is_sparse) { + UpdateSparseBuffer(binding.sparse_gpu_addr, binding.size); + return; + } const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size); binding.buffer_id = buffer_id; }); @@ -1222,7 +1305,12 @@ void BufferCache

::UpdateStorageBuffers(size_t stage) { template void BufferCache

::UpdateTextureBuffers(size_t stage) { ForEachEnabledBit(channel_state->enabled_texture_buffers[stage], [&](u32 index) { - Binding& binding = channel_state->texture_buffers[stage][index]; + TextureBufferBinding& binding = channel_state->texture_buffers[stage][index]; + if (binding.is_sparse && ((channel_state->written_texture_buffers[stage] >> index) & 1) == 0) { + UpdateSparseBuffer(binding.sparse_gpu_addr, binding.size); + return; + } + binding.is_sparse = false; binding.buffer_id = FindBuffer(binding.device_addr, binding.size); }); } @@ -1278,6 +1366,10 @@ void BufferCache

::UpdateComputeStorageBuffers() { ForEachEnabledBit(channel_state->enabled_compute_storage_buffers, [&](u32 index) { // Resolve buffer Binding& binding = channel_state->compute_storage_buffers[index]; + if (binding.is_sparse) { + UpdateSparseBuffer(binding.sparse_gpu_addr, binding.size); + return; + } binding.buffer_id = FindBuffer(binding.device_addr, binding.size); }); } @@ -1285,7 +1377,12 @@ void BufferCache

::UpdateComputeStorageBuffers() { template void BufferCache

::UpdateComputeTextureBuffers() { ForEachEnabledBit(channel_state->enabled_compute_texture_buffers, [&](u32 index) { - Binding& binding = channel_state->compute_texture_buffers[index]; + TextureBufferBinding& binding = channel_state->compute_texture_buffers[index]; + if (binding.is_sparse && ((channel_state->written_compute_texture_buffers >> index) & 1) == 0) { + UpdateSparseBuffer(binding.sparse_gpu_addr, binding.size); + return; + } + binding.is_sparse = false; binding.buffer_id = FindBuffer(binding.device_addr, binding.size); }); } @@ -1295,6 +1392,7 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 memory_tracker.MarkRegionAsGpuModified(device_addr, size); gpu_modified_ranges.Add(device_addr, size); uncommitted_gpu_modified_ranges.Add(device_addr, size); + MarkBufferContentChanged(slot_buffers[buffer_id]); } template @@ -1423,6 +1521,7 @@ BufferId BufferCache

::CreateBuffer(DAddr device_addr, u32 wanted_size) { auto& new_buffer = slot_buffers[new_buffer_id]; const size_t size_bytes = new_buffer.SizeBytes(); runtime.ClearBuffer(new_buffer, 0, size_bytes, 0); + MarkBufferContentChanged(new_buffer); new_buffer.MarkUsage(0, size_bytes); for (const BufferId overlap_id : overlap.ids) { JoinOverlap(new_buffer_id, overlap_id, !overlap.has_stream_leap); @@ -1477,6 +1576,7 @@ void BufferCache

::TouchBuffer(Buffer& buffer, BufferId buffer_id) noexcept { template bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 size) { + // Keep ordinary uploads local; only sparse assembly needs the caller-visible copy list. boost::container::small_vector copies; u64 total_size_bytes = 0; u64 largest_copy = 0; @@ -1495,6 +1595,31 @@ bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 si } const std::span copies_span(copies.data(), copies.size()); UploadMemory(buffer, total_size_bytes, largest_copy, copies_span); + MarkBufferContentChanged(buffer); + return false; +} + +template +bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 size, + boost::container::small_vector& copies) { + u64 total_size_bytes = 0; + u64 largest_copy = 0; + DAddr buffer_start = buffer.CpuAddr(); + memory_tracker.ForEachUploadRange(device_addr, size, [&](u64 device_addr_out, u64 range_size) { + copies.push_back(BufferCopy{ + .src_offset = total_size_bytes, + .dst_offset = device_addr_out - buffer_start, + .size = range_size, + }); + total_size_bytes += range_size; + largest_copy = std::max(largest_copy, range_size); + }); + if (total_size_bytes == 0) { + return true; + } + const std::span copies_span(copies.data(), copies.size()); + UploadMemory(buffer, total_size_bytes, largest_copy, copies_span); + MarkBufferContentChanged(buffer); return false; } @@ -1596,6 +1721,7 @@ void BufferCache

::InlineMemoryImplementation(DAddr dest_address, size_t copy_ } else { buffer.ImmediateUpload(buffer.Offset(dest_address), inlined_buffer.first(copy_size)); } + MarkBufferContentChanged(buffer); } template @@ -1679,6 +1805,14 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { channel_state->index_buffer.buffer_id = BufferId{}; dirty_index = true; } + for (SparseBuffer& sparse : sparse_buffers) { + for (SparseBufferPart& part : sparse.parts) { + if (part.buffer_id == buffer_id) { + // It is found again before the part is read + part.buffer_id = BufferId{}; + } + } + } for (u32 index = 0; index < channel_state->vertex_buffers.size(); index++) { auto& binding = channel_state->vertex_buffers[index]; @@ -1726,7 +1860,7 @@ template Binding BufferCache

::StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, bool is_written) const { const GPUVAddr gpu_addr = gpu_memory->Read(ssbo_addr); - const auto size = [&]() { + const auto requested_size = [&]() { const bool is_nvn_cbuf = cbuf_index == 0; // The NVN driver buffer (index 0) is known to pack the SSBO address followed by its size. if (is_nvn_cbuf) { @@ -1741,6 +1875,9 @@ Binding BufferCache

::StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, const u32 memory_layout_size = static_cast(gpu_memory->GetMemoryLayoutSize(gpu_addr)); return std::min(memory_layout_size, static_cast(8_MiB)); }(); + // Guests can declare a size that runs past the memory backing the buffer. Bindings are linear + // in device memory, so stop where the mapping stops being contiguous. + const u32 size = static_cast(gpu_memory->MaxContinuousRange(gpu_addr, requested_size)); // Alignment only applies to the offset of the buffer const u32 alignment = runtime.GetStorageBufferAlignment(); const GPUVAddr aligned_gpu_addr = Common::AlignDown(gpu_addr, alignment); @@ -1757,14 +1894,231 @@ Binding BufferCache

::StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, // The end address used for size calculation does not need to be aligned const DAddr cpu_end = Common::AlignUp(*device_addr + size, Core::DEVICE_PAGESIZE); - const Binding binding{ + Binding binding{ .device_addr = *aligned_device_addr, .size = is_written ? aligned_size : static_cast(cpu_end - *aligned_device_addr), .buffer_id = BufferId{}, }; + if (size != requested_size && !is_written) { + // The buffer goes on in other parts of a sparse allocation, shaders address it from its + // start so it has to be assembled + if (const auto region = gpu_memory->GetSparseRegion(aligned_gpu_addr)) { + const u64 region_size = region->first + region->second - aligned_gpu_addr; + const u64 wanted_size = (gpu_addr - aligned_gpu_addr) + u64{requested_size}; + binding.size = static_cast(std::min(region_size, wanted_size)); + binding.sparse_gpu_addr = aligned_gpu_addr; + binding.is_sparse = true; + } + } return binding; } +template +typename BufferCache

::SparseBuffer& BufferCache

::FindSparseBuffer(GPUVAddr gpu_addr, + u32 size) { + const auto it = std::ranges::find_if(sparse_buffers, [&](const SparseBuffer& sparse) { + return sparse.gpu_memory == gpu_memory && sparse.gpu_addr <= gpu_addr && + gpu_addr + size <= sparse.gpu_addr + sparse.size; + }); + if (it != sparse_buffers.end()) { + return *it; + } + // Cover the whole sparse allocation, its ranges are bound with many sizes + GPUVAddr begin = gpu_addr; + u64 total_size = size; + if (const auto region = gpu_memory->GetSparseRegion(gpu_addr)) { + const GPUVAddr region_end = region->first + region->second; + if (gpu_addr + size <= region_end && region->second <= std::numeric_limits::max()) { + begin = region->first; + total_size = region->second; + } + } + return sparse_buffers.emplace_back(SparseBuffer{ + .gpu_memory = gpu_memory, + .gpu_addr = begin, + .size = static_cast(total_size), + .capacity = 0, + .buffer_id = {}, + .mapping_generation = ~u64{}, + .frame_tick = frame_tick, + .parts = {}, + }); +} + +template +void BufferCache

::ResizeSparseBuffer(SparseBuffer& sparse, u32 capacity) { + if (sparse.buffer_id) { + delayed_destruction_ring.Push(std::move(slot_buffers[sparse.buffer_id])); + slot_buffers.erase(sparse.buffer_id); + } + sparse.buffer_id = slot_buffers.insert(runtime, DAddr{}, capacity); + sparse.capacity = capacity; + // What is not mapped reads as zero + runtime.ClearBuffer(slot_buffers[sparse.buffer_id], 0, capacity, 0); + for (SparseBufferPart& part : sparse.parts) { + part.content_version = 0; + } +} + +template +void BufferCache

::UpdateSparseBuffer(GPUVAddr gpu_addr, u32 size) { + // Host memory is committed in steps as the guest maps more of the range + static constexpr u64 CAPACITY_STEP = 16_MiB; + + SparseBuffer& sparse = FindSparseBuffer(gpu_addr, size); + sparse.frame_tick = frame_tick; + + const u64 mapping_generation = gpu_memory->MappingGeneration(); + if (sparse.mapping_generation != mapping_generation) { + sparse.mapping_generation = mapping_generation; + const auto ranges = gpu_memory->GetSubmappedRange(sparse.gpu_addr, sparse.size); + std::vector parts; + parts.reserve(ranges.size()); + u64 extent = 0; + // GetSubmappedRange and parts are ordered by guest address. Match them in one pass. + auto old = sparse.parts.begin(); + const auto clear_old_part = [&](const SparseBufferPart& part) { + if (sparse.buffer_id && part.offset < sparse.capacity) { + runtime.ClearBuffer(slot_buffers[sparse.buffer_id], part.offset, + std::min(part.size, sparse.capacity - part.offset), 0); + } + }; + for (const auto& [part_addr, part_size] : ranges) { + const std::optional device_addr = gpu_memory->GpuToCpuAddress(part_addr); + if (!device_addr || part_size == 0) { + continue; + } + SparseBufferPart part{ + .device_addr = *device_addr, + .size = static_cast(part_size), + .offset = static_cast(part_addr - sparse.gpu_addr), + .buffer_id = {}, + .content_version = 0, + }; + while (old != sparse.parts.end() && old->offset < part.offset) { + clear_old_part(*old++); + } + if (old != sparse.parts.end() && old->offset == part.offset) { + if (old->device_addr == part.device_addr && old->size == part.size) { + part.content_version = old->content_version; + part.buffer_id = old->buffer_id; + } else { + clear_old_part(*old); + } + ++old; + } + extent = std::max(extent, u64{part.offset} + part.size); + parts.push_back(part); + } + while (old != sparse.parts.end()) { + clear_old_part(*old++); + } + sparse.parts = std::move(parts); + + const u64 capacity = std::min( + sparse.size, Common::AlignUp(std::max(extent, 1), CAPACITY_STEP)); + if (capacity > sparse.capacity) { + ResizeSparseBuffer(sparse, static_cast(capacity)); + } + } + for (SparseBufferPart& part : sparse.parts) { + if (!part.buffer_id) { + part.buffer_id = FindBuffer(part.device_addr, part.size); + } + } +} + +template +typename BufferCache

::SparseBinding BufferCache

::GetSparseBinding(GPUVAddr gpu_addr, + u32 size) { + const SparseBuffer& sparse = FindSparseBuffer(gpu_addr, size); + const u32 offset = static_cast(gpu_addr - sparse.gpu_addr); + ASSERT(offset < sparse.capacity); + return SparseBinding{ + .buffer_id = sparse.buffer_id, + .offset = offset, + .size = std::min(size, sparse.capacity - offset), + }; +} + +template +typename BufferCache

::SparseBinding BufferCache

::SynchronizeSparseBuffer(GPUVAddr gpu_addr, + u32 size) { + const auto& cached = FindSparseBuffer(gpu_addr, size); + if (!cached.buffer_id || cached.mapping_generation != gpu_memory->MappingGeneration()) { + // Index bindings can survive a mapping change without any register becoming dirty. + UpdateSparseBuffer(gpu_addr, size); + } + SparseBuffer& sparse = FindSparseBuffer(gpu_addr, size); + sparse.frame_tick = frame_tick; + // Finding the buffer of a part can delete the buffer of another one + while (std::ranges::any_of(sparse.parts, [](const SparseBufferPart& part) { + return !part.buffer_id; + })) { + for (SparseBufferPart& part : sparse.parts) { + if (!part.buffer_id) { + part.buffer_id = FindBuffer(part.device_addr, part.size); + } + } + } + Buffer& buffer = slot_buffers[sparse.buffer_id]; + boost::container::small_vector uploads; + boost::container::small_vector copies; + for (SparseBufferPart& part : sparse.parts) { + Buffer& source = slot_buffers[part.buffer_id]; + TouchBuffer(source, part.buffer_id); + // Anything but the uploads done here leaves the whole part to copy + const bool is_stale = source.content_version != part.content_version; + uploads.clear(); + SynchronizeBuffer(source, part.device_addr, part.size, uploads); + part.content_version = source.content_version; + + const u64 source_offset = source.Offset(part.device_addr); + copies.clear(); + if (is_stale) { + copies.push_back(BufferCopy{ + .src_offset = source_offset, + .dst_offset = part.offset, + .size = part.size, + }); + } else { + for (const BufferCopy& upload : uploads) { + const u64 begin = std::max(upload.dst_offset, source_offset); + const u64 end = + std::min(upload.dst_offset + upload.size, source_offset + part.size); + if (begin >= end) { + continue; + } + copies.push_back(BufferCopy{ + .src_offset = begin, + .dst_offset = part.offset + (begin - source_offset), + .size = end - begin, + }); + } + } + if (!copies.empty()) { + runtime.CopyBuffer(buffer, source, copies, true); + } + } + return GetSparseBinding(gpu_addr, size); +} + +template +void BufferCache

::RemoveUnusedSparseBuffers() { + static constexpr u64 TICKS_TO_REMOVE = 600; + + std::erase_if(sparse_buffers, [this](SparseBuffer& sparse) { + if (sparse.frame_tick + TICKS_TO_REMOVE > frame_tick) { + return false; + } + if (sparse.buffer_id) { + delayed_destruction_ring.Push(std::move(slot_buffers[sparse.buffer_id])); + slot_buffers.erase(sparse.buffer_id); + } + return true; + }); +} + template TextureBufferBinding BufferCache

::GetTextureBufferBinding(GPUVAddr gpu_addr, u32 size, PixelFormat format) { @@ -1780,6 +2134,10 @@ TextureBufferBinding BufferCache

::GetTextureBufferBinding(GPUVAddr gpu_addr, binding.size = size; binding.buffer_id = BufferId{}; binding.format = format; + if (!gpu_memory->IsContinuousRange(gpu_addr, size)) { + binding.sparse_gpu_addr = gpu_addr; + binding.is_sparse = true; + } } return binding; } diff --git a/src/yuzu_video_core/buffer_cache/buffer_cache_base.h b/src/yuzu_video_core/buffer_cache/buffer_cache_base.h index b12db317..fb31099f 100644 --- a/src/yuzu_video_core/buffer_cache/buffer_cache_base.h +++ b/src/yuzu_video_core/buffer_cache/buffer_cache_base.h @@ -75,6 +75,10 @@ struct Binding { DAddr device_addr{}; u32 size{}; BufferId buffer_id; + /// Guest address of a range that is not continuous in device memory, it is bound from the + /// buffer assembled from its parts + GPUVAddr sparse_gpu_addr{}; + bool is_sparse{}; }; struct TextureBufferBinding : Binding { @@ -105,6 +109,10 @@ public: BufferCacheChannelInfo& operator=(const BufferCacheChannelInfo&) = delete; Binding index_buffer; + /// Set when the index buffer is not continuous in device memory, it is then bound from the + /// buffer assembled from its parts + bool is_index_buffer_sparse = false; + GPUVAddr index_buffer_sparse_gpu_addr = 0; std::array vertex_buffers; std::array, NUM_STAGES> uniform_buffers; std::array, NUM_STAGES> storage_buffers; @@ -422,6 +430,10 @@ private: bool SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 size); + /// @param uploads Receives the ranges of the buffer that were uploaded + bool SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 size, + boost::container::small_vector& uploads); + void UploadMemory(Buffer& buffer, u64 total_size_bytes, u64 largest_copy, std::span copies); @@ -463,6 +475,59 @@ private: u32 last_index_count = 0; MemoryTracker memory_tracker; + + /// Part of a guest range that is continuous in device memory + struct SparseBufferPart { + DAddr device_addr{}; + u32 size{}; + u32 offset{}; + BufferId buffer_id; + u64 content_version{}; + }; + + /// Host buffer assembled from the parts of a guest range that is not continuous in device + /// memory. It is not registered, only the buffers of its parts track guest memory. + struct SparseBuffer { + const Tegra::MemoryManager* gpu_memory{}; + GPUVAddr gpu_addr{}; + u32 size{}; + /// Size of the host buffer, it stops after the last part + u32 capacity{}; + BufferId buffer_id; + /// Mapping generation of the address space the parts were found at + u64 mapping_generation{}; + u64 frame_tick{}; + std::vector parts; + }; + + struct SparseBinding { + BufferId buffer_id; + u32 offset{}; + u32 size{}; + }; + + [[nodiscard]] SparseBuffer& FindSparseBuffer(GPUVAddr gpu_addr, u32 size); + + /// Finds the parts of the buffer holding a guest range and their buffers + void UpdateSparseBuffer(GPUVAddr gpu_addr, u32 size); + + /// Where a guest range is in the buffer assembled for it + [[nodiscard]] SparseBinding GetSparseBinding(GPUVAddr gpu_addr, u32 size); + + /// Brings the buffer holding a guest range up to date with its parts + SparseBinding SynchronizeSparseBuffer(GPUVAddr gpu_addr, u32 size); + + void ResizeSparseBuffer(SparseBuffer& sparse, u32 capacity); + + void RemoveUnusedSparseBuffers(); + + void MarkBufferContentChanged(Buffer& buffer) noexcept { + buffer.content_version = ++content_version; + } + + std::vector sparse_buffers; + u64 content_version = 0; + Common::RangeSet uncommitted_gpu_modified_ranges; Common::RangeSet gpu_modified_ranges; std::deque> committed_gpu_modified_ranges; diff --git a/src/yuzu_video_core/engines/draw_manager.cpp b/src/yuzu_video_core/engines/draw_manager.cpp index b94100cf..6c8c7368 100644 --- a/src/yuzu_video_core/engines/draw_manager.cpp +++ b/src/yuzu_video_core/engines/draw_manager.cpp @@ -118,13 +118,13 @@ void DrawManager::DrawIndex(PrimitiveTopology topology, u32 index_first, u32 ind ProcessDraw(true, num_instances); } -void DrawManager::DrawArrayIndirect(PrimitiveTopology topology) { +bool DrawManager::DrawArrayIndirect(PrimitiveTopology topology) { draw_state.topology = topology; - ProcessDrawIndirect(); + return ProcessDrawIndirect(); } -void DrawManager::DrawIndexedIndirect(PrimitiveTopology topology, u32 index_first, +bool DrawManager::DrawIndexedIndirect(PrimitiveTopology topology, u32 index_first, u32 index_count) { const auto& regs{maxwell3d->regs}; draw_state.topology = topology; @@ -132,7 +132,7 @@ void DrawManager::DrawIndexedIndirect(PrimitiveTopology topology, u32 index_firs draw_state.index_buffer.first = index_first; draw_state.index_buffer.count = index_count; - ProcessDrawIndirect(); + return ProcessDrawIndirect(); } void DrawManager::SetInlineIndexBuffer(u32 index) { @@ -271,7 +271,7 @@ void DrawManager::ProcessDraw(bool draw_indexed, u32 instance_count) { } } -void DrawManager::ProcessDrawIndirect() { +bool DrawManager::ProcessDrawIndirect() { LOG_TRACE( HW_GPU, "called, topology={}, is_indexed={}, includes_count={}, buffer_size={}, max_draw_count={}", @@ -281,7 +281,8 @@ void DrawManager::ProcessDrawIndirect() { UpdateTopology(); if (maxwell3d->ShouldExecute()) { - maxwell3d->rasterizer->DrawIndirect(); + return maxwell3d->rasterizer->DrawIndirect(); } + return true; } } // namespace Tegra::Engines diff --git a/src/yuzu_video_core/engines/draw_manager.h b/src/yuzu_video_core/engines/draw_manager.h index 6506b7a6..31b80bb1 100644 --- a/src/yuzu_video_core/engines/draw_manager.h +++ b/src/yuzu_video_core/engines/draw_manager.h @@ -72,9 +72,10 @@ public: void DrawIndex(PrimitiveTopology topology, u32 index_first, u32 index_count, u32 base_index, u32 base_instance, u32 num_instances); - void DrawArrayIndirect(PrimitiveTopology topology); + /// Indirect draws return false when the host needs their parameters on the CPU + bool DrawArrayIndirect(PrimitiveTopology topology); - void DrawIndexedIndirect(PrimitiveTopology topology, u32 index_first, u32 index_count); + bool DrawIndexedIndirect(PrimitiveTopology topology, u32 index_first, u32 index_count); const State& GetDrawState() const { return draw_state; @@ -107,7 +108,7 @@ private: void ProcessDraw(bool draw_indexed, u32 instance_count); - void ProcessDrawIndirect(); + bool ProcessDrawIndirect(); Maxwell3D* maxwell3d{}; State draw_state{}; diff --git a/src/yuzu_video_core/engines/maxwell_3d.cpp b/src/yuzu_video_core/engines/maxwell_3d.cpp index c5d45e3c..fe83cd00 100644 --- a/src/yuzu_video_core/engines/maxwell_3d.cpp +++ b/src/yuzu_video_core/engines/maxwell_3d.cpp @@ -295,6 +295,21 @@ u32 Maxwell3D::GetMaxCurrentVertices() return num_vertices; } +size_t Maxwell3D::IndexBufferLayoutSize() +{ + const GPUVAddr start_address = regs.index_buffer.StartAddress(); + const GPUVAddr end_address = regs.index_buffer.EndAddress(); + const size_t byte_size = regs.index_buffer.FormatSizeInBytes(); + const size_t address_size = static_cast(end_address - start_address); + // The pages of a sparse allocation are mapped one part at a time, the indices can be anywhere + // in the allocation + if (const auto sparse = memory_manager.GetSparseRegion(start_address)) { + const size_t sparse_size = sparse->first + sparse->second - start_address; + return std::min(sparse_size, address_size) / byte_size; + } + return std::min(memory_manager.GetMemoryLayoutSize(start_address), address_size) / byte_size; +} + size_t Maxwell3D::EstimateIndexBufferSize() { GPUVAddr start_address = regs.index_buffer.StartAddress(); @@ -307,6 +322,9 @@ size_t Maxwell3D::EstimateIndexBufferSize() const size_t cap{GetMaxCurrentVertices() * 4 * byte_size}; const size_t lower_cap = std::min(static_cast(end_address - start_address), cap); + if (memory_manager.GetSparseRegion(start_address)) { + return IndexBufferLayoutSize(); + } return std::min( memory_manager.GetMemoryLayoutSize(start_address, byte_size * max_sizes[log2_byte_size]) / byte_size, diff --git a/src/yuzu_video_core/engines/maxwell_3d.h b/src/yuzu_video_core/engines/maxwell_3d.h index d979091f..c2f733c3 100644 --- a/src/yuzu_video_core/engines/maxwell_3d.h +++ b/src/yuzu_video_core/engines/maxwell_3d.h @@ -3121,6 +3121,9 @@ public: size_t EstimateIndexBufferSize(); + /// Number of indices the memory the index buffer starts in can hold + size_t IndexBufferLayoutSize(); + /// Handles a write to the CLEAR_BUFFERS register. void ProcessClearBuffers(u32 layer_count); diff --git a/src/yuzu_video_core/host1x/gpu_device_memory_manager.cpp b/src/yuzu_video_core/host1x/gpu_device_memory_manager.cpp index 035ee301..039b9b9e 100644 --- a/src/yuzu_video_core/host1x/gpu_device_memory_manager.cpp +++ b/src/yuzu_video_core/host1x/gpu_device_memory_manager.cpp @@ -247,7 +247,7 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size IMemory * process_memory = GetRegisteredProcess(asid); size_t start_page_d = address >> DEVICE_PAGEBITS; size_t num_pages = Common::AlignUp(size, DEVICE_PAGESIZE) >> DEVICE_PAGEBITS; - std::scoped_lock lk(mapping_guard); + std::unique_lock lk(mapping_guard); for (size_t i = 0; i < num_pages; i++) { const VAddr new_vaddress = virtual_address + i * DEVICE_PAGESIZE; @@ -279,6 +279,13 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size { TrackContinuityImpl(address, virtual_address, size, asid); } + lk.unlock(); + if (device_inter != nullptr) + { + // The caches may have synchronized these pages while nothing backed them. Their contents + // are unknown and the guest pages were never write protected, start over. + device_inter->InvalidateRegion(address, size); + } } template @@ -612,6 +619,8 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size if (vpage == 0) [[unlikely]] { + // Keep counting pages without a backing, they are balanced once the page is mapped + count.fetch_add(static_cast(delta), std::memory_order_release); release_pending(); continue; } diff --git a/src/yuzu_video_core/macro/macro_hle.cpp b/src/yuzu_video_core/macro/macro_hle.cpp index be201d2c..f84e53d3 100644 --- a/src/yuzu_video_core/macro/macro_hle.cpp +++ b/src/yuzu_video_core/macro/macro_hle.cpp @@ -3,6 +3,7 @@ #include #include +#include "nxemu-video/video_settings.h" #include "yuzu_common/yuzu_assert.h" #include "yuzu_common/scope_exit.h" #include "yuzu_video_core/dirty_flags.h" @@ -82,12 +83,15 @@ public: 0, 0x640, Maxwell3D::HLEReplacementAttributeType::BaseInstance); } - maxwell3d.draw_manager->DrawArrayIndirect(topology); + const bool is_drawn = maxwell3d.draw_manager->DrawArrayIndirect(topology); if constexpr (extended) { maxwell3d.engine_state = Maxwell3D::EngineHint::None; maxwell3d.replace_table.clear(); } + if (!is_drawn) { + Fallback(parameters); + } } private: @@ -171,7 +175,7 @@ public: params.max_draw_counts = 1; params.stride = 0; maxwell3d.dirty.flags[VideoCommon::Dirty::IndexBuffer] = true; - maxwell3d.draw_manager->DrawIndexedIndirect(topology, 0, estimate); + const bool is_drawn = maxwell3d.draw_manager->DrawIndexedIndirect(topology, 0, estimate); maxwell3d.regs.vertex_id_base = 0x0; maxwell3d.regs.global_base_vertex_index = 0x0; maxwell3d.regs.global_base_instance_index = 0x0; @@ -179,6 +183,9 @@ public: maxwell3d.engine_state = Maxwell3D::EngineHint::None; maxwell3d.replace_table.clear(); } + if (!is_drawn) { + Fallback(parameters); + } } private: @@ -255,7 +262,7 @@ public: const u32 indirect_words = 5 + padding; const u32 stride = indirect_words * sizeof(u32); const std::size_t draw_count = end_indirect - start_indirect; - const u32 estimate = static_cast(maxwell3d.EstimateIndexBufferSize()); + const u32 estimate = IndexCount(parameters, indirect_words, draw_count); maxwell3d.dirty.flags[VideoCommon::Dirty::IndexBuffer] = true; auto& params = maxwell3d.draw_manager->GetIndirectParams(); params.is_byte_count = false; @@ -274,12 +281,39 @@ public: 0, 0x644, Maxwell3D::HLEReplacementAttributeType::BaseInstance); maxwell3d.SetHLEReplacementAttributeType(0, 0x648, Maxwell3D::HLEReplacementAttributeType::DrawID); - maxwell3d.draw_manager->DrawIndexedIndirect(topology, 0, estimate); + const bool is_drawn = maxwell3d.draw_manager->DrawIndexedIndirect(topology, 0, estimate); maxwell3d.engine_state = Maxwell3D::EngineHint::None; maxwell3d.replace_table.clear(); + if (!is_drawn) { + Fallback(parameters); + } } private: + /// Number of indices the draws can read from the start of the index buffer + u32 IndexCount(const std::vector& parameters, u32 indirect_words, std::size_t draw_count) { + // The draws are parameters of the macro, when the guest wrote them they tell how far + // the indices go, the size of the vertex buffers tells nothing about it + static constexpr std::size_t FIRST_DRAW = 5; + u64 exact{}; + bool is_complete{true}; + for (std::size_t draw = 0; draw < draw_count; ++draw) { + const std::size_t base = FIRST_DRAW + draw * indirect_words; + if (base + 2 >= parameters.size()) { + is_complete = false; + break; + } + exact = std::max(exact, u64{parameters[base]} + parameters[base + 2]); + } + const u64 limit = maxwell3d.IndexBufferLayoutSize(); + const u64 clamped = std::min(exact, limit); + if (is_complete && Settings::IsGPULevelHigh() && !maxwell3d.AnyParametersDirty()) { + return static_cast(clamped); + } + // The draws may have been written by the GPU, the parameters can't be trusted + return static_cast(std::max(clamped, maxwell3d.EstimateIndexBufferSize())); + } + void Fallback(const std::vector& parameters) { SCOPE_EXIT { // Clean everything. diff --git a/src/yuzu_video_core/memory_manager.cpp b/src/yuzu_video_core/memory_manager.cpp index a7aaf378..3ebb31d0 100644 --- a/src/yuzu_video_core/memory_manager.cpp +++ b/src/yuzu_video_core/memory_manager.cpp @@ -265,6 +265,7 @@ void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface * rasterizer_) GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, DAddr dev_addr, std::size_t size, PTEKind kind, bool is_big_pages) { + mapping_generation.fetch_add(1, std::memory_order_relaxed); NotifyMappingChanged(gpu_addr, dev_addr, size, EntryType::Mapped, kind, is_big_pages); if (is_big_pages) [[likely]] { @@ -274,8 +275,44 @@ GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, DAddr dev_addr, std::size_t size, return PageTableOp(gpu_addr, dev_addr, size, kind); } +std::optional> MemoryManager::GetSparseRegion( + GPUVAddr gpu_addr) const +{ + if (!has_sparse_regions.load(std::memory_order_acquire)) { + return std::nullopt; + } + std::unique_lock lock(guard); + auto it = sparse_regions.upper_bound(gpu_addr); + if (it == sparse_regions.begin()) { + return std::nullopt; + } + --it; + if (gpu_addr >= it->second) { + return std::nullopt; + } + return std::make_pair(it->first, static_cast(it->second - it->first)); +} + GPUVAddr MemoryManager::MapSparse(GPUVAddr gpu_addr, std::size_t size, bool is_big_pages) { + { + std::unique_lock lock(guard); + GPUVAddr begin = gpu_addr; + GPUVAddr end = gpu_addr + size; + // Merge with the allocations this one overlaps + auto it = sparse_regions.upper_bound(begin); + if (it != sparse_regions.begin() && std::prev(it)->second > begin) { + --it; + } + while (it != sparse_regions.end() && it->first < end) { + begin = std::min(begin, it->first); + end = std::max(end, it->second); + it = sparse_regions.erase(it); + } + sparse_regions.emplace(begin, end); + has_sparse_regions.store(true, std::memory_order_release); + } + mapping_generation.fetch_add(1, std::memory_order_relaxed); NotifyMappingChanged(gpu_addr, 0, size, EntryType::Reserved, PTEKind::INVALID, is_big_pages); if (is_big_pages) [[likely]] { @@ -291,6 +328,28 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { return; } + { + std::unique_lock lock(guard); + const GPUVAddr unmap_end = gpu_addr + size; + auto it = sparse_regions.upper_bound(gpu_addr); + if (it != sparse_regions.begin() && std::prev(it)->second > gpu_addr) { + --it; + } + while (it != sparse_regions.end() && it->first < unmap_end) { + const GPUVAddr begin = it->first; + const GPUVAddr end = it->second; + it = sparse_regions.erase(it); + if (begin < gpu_addr) { + sparse_regions.emplace(begin, gpu_addr); + } + if (end > unmap_end) { + it = sparse_regions.emplace(unmap_end, end).first; + ++it; + } + } + has_sparse_regions.store(!sparse_regions.empty(), std::memory_order_release); + } + mapping_generation.fetch_add(1, std::memory_order_relaxed); NotifyMappingChanged(gpu_addr, 0, size, EntryType::Free, PTEKind::INVALID, false); GetSubmappedRangeImpl(gpu_addr, size, page_stash); diff --git a/src/yuzu_video_core/memory_manager.h b/src/yuzu_video_core/memory_manager.h index 359d5f06..cba29565 100644 --- a/src/yuzu_video_core/memory_manager.h +++ b/src/yuzu_video_core/memory_manager.h @@ -151,6 +151,17 @@ public: PTEKind GetPageKind(GPUVAddr gpu_addr) const; + /** + * Returns the sparse allocation an address belongs to, if any. + */ + [[nodiscard]] std::optional> GetSparseRegion( + GPUVAddr gpu_addr) const; + + /// Changes every time the address space is mapped or unmapped + [[nodiscard]] u64 MappingGeneration() const noexcept { + return mapping_generation.load(std::memory_order_relaxed); + } + size_t GetMemoryLayoutSize(GPUVAddr gpu_addr, size_t max_size = std::numeric_limits::max()) const; @@ -240,6 +251,11 @@ private: Common::MultiLevelPageTable page_table; Common::RangeMap kind_map; + /// Start and end of the sparse allocations, protected by guard + std::map sparse_regions; + // Publish catalog emptiness under guard so empty lookups can avoid the mutex. + std::atomic has_sparse_regions{}; + std::atomic mapping_generation{}; Common::VirtualBuffer big_page_table_dev; std::vector big_page_continuous; diff --git a/src/yuzu_video_core/rasterizer_interface.h b/src/yuzu_video_core/rasterizer_interface.h index 0e8e106a..6461ecf0 100644 --- a/src/yuzu_video_core/rasterizer_interface.h +++ b/src/yuzu_video_core/rasterizer_interface.h @@ -41,7 +41,10 @@ public: virtual void Draw(bool is_indexed, u32 instance_count) = 0; /// Dispatches an indirect draw invocation - virtual void DrawIndirect() {} + /// Returns false when the host needs the parameters of the draw on the CPU, nothing is drawn + virtual bool DrawIndirect() { + return true; + } /// Dispatches an draw texture invocation virtual void DrawTexture() = 0; diff --git a/src/yuzu_video_core/renderer_opengl/gl_rasterizer.cpp b/src/yuzu_video_core/renderer_opengl/gl_rasterizer.cpp index 85dd2a34..c49d3b80 100644 --- a/src/yuzu_video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/yuzu_video_core/renderer_opengl/gl_rasterizer.cpp @@ -299,7 +299,7 @@ void RasterizerOpenGL::Draw(bool is_indexed, u32 instance_count) { }); } -void RasterizerOpenGL::DrawIndirect() { +bool RasterizerOpenGL::DrawIndirect() { const auto& params = maxwell3d->draw_manager->GetIndirectParams(); buffer_cache.SetDrawIndirect(¶ms); PrepareDraw(params.is_indexed, [this, ¶ms](GLenum primitive_mode) { @@ -344,6 +344,7 @@ void RasterizerOpenGL::DrawIndirect() { } }); buffer_cache.SetDrawIndirect(nullptr); + return true; } void RasterizerOpenGL::DrawTexture() { diff --git a/src/yuzu_video_core/renderer_opengl/gl_rasterizer.h b/src/yuzu_video_core/renderer_opengl/gl_rasterizer.h index 1761d0a5..94a60f79 100644 --- a/src/yuzu_video_core/renderer_opengl/gl_rasterizer.h +++ b/src/yuzu_video_core/renderer_opengl/gl_rasterizer.h @@ -82,7 +82,7 @@ public: ~RasterizerOpenGL() override; void Draw(bool is_indexed, u32 instance_count) override; - void DrawIndirect() override; + bool DrawIndirect() override; void DrawTexture() override; void Clear(u32 layer_count) override; void DispatchCompute() override; diff --git a/src/yuzu_video_core/renderer_vulkan/fixed_pipeline_state.cpp b/src/yuzu_video_core/renderer_vulkan/fixed_pipeline_state.cpp index c6c3f312..5488ebc6 100644 --- a/src/yuzu_video_core/renderer_vulkan/fixed_pipeline_state.cpp +++ b/src/yuzu_video_core/renderer_vulkan/fixed_pipeline_state.cpp @@ -48,6 +48,37 @@ void RefreshXfbState(VideoCommon::TransformFeedbackState& state, const Maxwell& } } // Anonymous namespace +bool FixedPipelineState::UsesDualSourceBlend(const Maxwell& regs) noexcept { + const auto format = regs.rt[0].format; + if (!regs.blend.enable[0] || format == Tegra::RenderTargetFormat::NONE || + VideoCore::Surface::IsPixelFormatInteger( + VideoCore::Surface::PixelFormatFromRenderTargetFormat(format))) { + return false; + } + const auto is_second_source = [](Maxwell::Blend::Factor factor) { + using Factor = Maxwell::Blend::Factor; + switch (factor) { + case Factor::Source1Color_D3D: + case Factor::OneMinusSource1Color_D3D: + case Factor::Source1Alpha_D3D: + case Factor::OneMinusSource1Alpha_D3D: + case Factor::Source1Color_GL: + case Factor::OneMinusSource1Color_GL: + case Factor::Source1Alpha_GL: + case Factor::OneMinusSource1Alpha_GL: + return true; + default: + return false; + } + }; + const auto uses_second_source = [&](const auto& blend) { + return is_second_source(blend.color_source) || is_second_source(blend.color_dest) || + is_second_source(blend.alpha_source) || is_second_source(blend.alpha_dest); + }; + return regs.blend_per_target_enabled ? uses_second_source(regs.blend_per_target[0]) + : uses_second_source(regs.blend); +} + void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFeatures& features) { const Maxwell& regs = maxwell3d.regs; const auto topology_ = maxwell3d.draw_manager->GetDrawState().topology; @@ -70,7 +101,9 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe topology.Assign(topology_); msaa_mode.Assign(regs.anti_alias_samples_mode); + const bool cached_dual_source_blend = dual_source_blend != 0; raw2 = 0; + dual_source_blend.Assign(cached_dual_source_blend); vertex_stride_workaround.Assign(features.has_vertex_stride_workaround && !features.has_dynamic_vertex_input ? 1 : 0); @@ -177,6 +210,12 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe if (!extended_dynamic_state_3_enables) { dynamic_state.Refresh3(regs); } + // The shader output interface is static even when the blend equation is dynamic. + // Only this mode belongs in the key; ordinary blend factors remain dynamic. + if (!extended_dynamic_state_3_blend || color_formats_changed || + maxwell3d.dirty.flags[Dirty::Blending] || maxwell3d.dirty.flags[Dirty::BlendEnable]) { + dual_source_blend.Assign(UsesDualSourceBlend(regs)); + } if (xfb_enabled) { RefreshXfbState(xfb_state, regs); } diff --git a/src/yuzu_video_core/renderer_vulkan/fixed_pipeline_state.h b/src/yuzu_video_core/renderer_vulkan/fixed_pipeline_state.h index 46250dcc..da293253 100644 --- a/src/yuzu_video_core/renderer_vulkan/fixed_pipeline_state.h +++ b/src/yuzu_video_core/renderer_vulkan/fixed_pipeline_state.h @@ -49,6 +49,8 @@ struct FixedPipelineState { static u32 PackBlendEquation(Maxwell::Blend::Equation equation) noexcept; static Maxwell::Blend::Equation UnpackBlendEquation(u32 packed) noexcept; + static bool UsesDualSourceBlend(const Maxwell& regs) noexcept; + static u32 PackBlendFactor(Maxwell::Blend::Factor factor) noexcept; static Maxwell::Blend::Factor UnpackBlendFactor(u32 packed) noexcept; @@ -203,6 +205,7 @@ struct FixedPipelineState { }; union { u32 raw2; + BitField<0, 1, u32> dual_source_blend; BitField<1, 3, u32> alpha_test_func; BitField<4, 1, u32> early_z; BitField<5, 1, u32> depth_enabled; diff --git a/src/yuzu_video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/yuzu_video_core/renderer_vulkan/vk_pipeline_cache.cpp index dc806bc8..b371d234 100644 --- a/src/yuzu_video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/yuzu_video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -56,7 +56,7 @@ using VideoCommon::FileEnvironment; using VideoCommon::GenericEnvironment; using VideoCommon::GraphicsEnvironment; -constexpr u32 CACHE_VERSION = 14; +constexpr u32 CACHE_VERSION = 15; constexpr std::array VULKAN_CACHE_MAGIC_NUMBER{'y', 'u', 'z', 'u', 'v', 'k', 'c', 'h'}; template @@ -231,6 +231,7 @@ Shader::RuntimeInfo MakeRuntimeInfo(std::span program info.alpha_test_func = MaxwellToCompareFunction( key.state.UnpackComparisonOp(key.state.alpha_test_func.Value())); info.alpha_test_reference = Common::BitCast(key.state.alpha_test_ref); + info.dual_source_blend = key.state.dual_source_blend != 0; // Vulkan requires integer fragment outputs for integer color attachments. for (size_t index = 0; index < info.color_output_types.size(); ++index) { const auto format = diff --git a/src/yuzu_video_core/renderer_vulkan/vk_rasterizer.cpp b/src/yuzu_video_core/renderer_vulkan/vk_rasterizer.cpp index 54342315..5a5eb29c 100644 --- a/src/yuzu_video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/yuzu_video_core/renderer_vulkan/vk_rasterizer.cpp @@ -278,12 +278,20 @@ void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) { }); } -void RasterizerVulkan::DrawIndirect() { +bool RasterizerVulkan::DrawIndirect() { const auto& params = maxwell3d->draw_manager->GetIndirectParams(); + if (!device.IsGeometryShaderSupported() && !params.is_byte_count) { + // The stages running as compute need the parameters of the draw on the CPU + gpu_memory->FlushCaching(); + const GraphicsPipeline* const pipeline{pipeline_cache.CurrentGraphicsPipeline()}; + if (pipeline && pipeline->IsVtgAsCompute()) { + return false; + } + } buffer_cache.SetDrawIndirect(¶ms); PrepareDraw(params.is_indexed, [this, ¶ms](const GraphicsPipeline* pipeline) { if (pipeline->IsVtgAsCompute()) { - // The stages running as compute need the draw parameters on the CPU + // Only draws that take their count from transform feedback get here return; } const auto indirect_buffer = buffer_cache.GetDrawIndirectBuffer(); @@ -338,6 +346,7 @@ void RasterizerVulkan::DrawIndirect() { }); }); buffer_cache.SetDrawIndirect(nullptr); + return true; } void RasterizerVulkan::DrawTexture() { diff --git a/src/yuzu_video_core/renderer_vulkan/vk_rasterizer.h b/src/yuzu_video_core/renderer_vulkan/vk_rasterizer.h index b8d2dfc2..e52ef0c7 100644 --- a/src/yuzu_video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/yuzu_video_core/renderer_vulkan/vk_rasterizer.h @@ -83,7 +83,7 @@ public: ~RasterizerVulkan() override; void Draw(bool is_indexed, u32 instance_count) override; - void DrawIndirect() override; + bool DrawIndirect() override; void DrawTexture() override; void Clear(u32 layer_count) override; void DispatchCompute() override; diff --git a/src/yuzu_video_core/texture_cache/image_view_base.cpp b/src/yuzu_video_core/texture_cache/image_view_base.cpp index 68d992f1..53af06ae 100644 --- a/src/yuzu_video_core/texture_cache/image_view_base.cpp +++ b/src/yuzu_video_core/texture_cache/image_view_base.cpp @@ -16,15 +16,21 @@ namespace VideoCommon { ImageViewBase::ImageViewBase(const ImageViewInfo& info, const ImageInfo& image_info, ImageId image_id_, GPUVAddr addr) - : image_id{image_id_}, gpu_addr{addr}, format{info.format}, type{info.type}, range{info.range}, - size{ - .width = std::max(image_info.size.width >> range.base.level, 1u), - .height = std::max(image_info.size.height >> range.base.level, 1u), - .depth = std::max(image_info.size.depth >> range.base.level, 1u), - } { + : image_id{image_id_}, gpu_addr{addr}, format{info.format}, type{info.type}, + range{info.range}, + size{} { ASSERT_MSG(VideoCore::Surface::IsViewCompatible(image_info.format, info.format, false, true), "Image view format {} is incompatible with image format {}", info.format, image_info.format); + // Clamp before shifting so malformed internal views cannot cause an invalid shift. + range.base.level = std::clamp(range.base.level, 0, std::max(image_info.resources.levels - 1, 0)); + size = Extent3D{ + .width = std::max(image_info.size.width >> range.base.level, 1u), + .height = std::max(image_info.size.height >> range.base.level, 1u), + .depth = std::max(image_info.size.depth >> range.base.level, 1u), + }; + range.extent.levels = + std::clamp(range.extent.levels, 1, std::max(image_info.resources.levels - range.base.level, 1)); if (image_info.forced_flushed) { flags |= ImageViewFlagBits::PreemtiveDownload; }