Skip empty GPU cache categories during Vulkan flushes (#359)

src/yuzu_video_core/CMakeLists.txt

@@ -18,6 +18,7 @@ add_library(video_core STATIC
     buffer_cache/usage_tracker.h
     buffer_cache/word_manager.h
     cache_types.h
+    flush_region_cache_filter.h
     capture.h
     cdma_pusher.cpp
     cdma_pusher.h

src/yuzu_video_core/buffer_cache/buffer_cache.h

@@ -1411,6 +1411,7 @@ void BufferCache<P>::ChangeRegister(BufferId buffer_id) {
     Buffer& buffer = slot_buffers[buffer_id];
     const auto size = buffer.SizeBytes();
     if (insert) {
+        device_memory.RegisterFlushCacheRegion(buffer.CpuAddr(), size, CacheType::BufferCache);
         total_used_memory += Common::AlignUp(size, 1024);
         buffer.setLRUID(lru_cache.Insert(buffer_id, frame_tick));
     } else {

src/yuzu_video_core/device_memory_manager.h

@@ -15,6 +15,7 @@
 #include "yuzu_common/range_mutex.h"
 #include "yuzu_common/scratch_buffer.h"
 #include "yuzu_common/virtual_buffer.h"
+#include "yuzu_video_core/flush_region_cache_filter.h"
 
 #include <nxemu-module-spec/cpu.h>
 #include <nxemu-module-spec/operating_system.h>
@@ -111,6 +112,16 @@ public:
 
     void UpdatePagesCachedCount(DAddr addr, size_t size, s32 delta);
 
+    // Presence remains conservative across deletion, remapping and address reuse.
+    void RegisterFlushCacheRegion(DAddr addr, u64 size, VideoCommon::CacheType types) {
+        flush_cache_filter.Register(addr, size, types);
+    }
+
+    [[nodiscard]] VideoCommon::CacheType FilterFlushCaches(
+        DAddr addr, u64 size, VideoCommon::CacheType requested) const {
+        return flush_cache_filter.Filter(addr, size, requested);
+    }
+
     static constexpr size_t AS_BITS = Traits::device_virtual_bits;
 
 private:
@@ -208,6 +219,7 @@ private:
         (1ULL << (device_virtual_bits - page_bits)) / subentries;
     using CachedPages = std::array<CounterEntry, num_counter_entries>;
     std::unique_ptr<CachedPages> cached_pages;
+    VideoCommon::FlushRegionCacheFilter<Traits::device_virtual_bits> flush_cache_filter;
     Common::RangeMutex counter_guard;
     std::mutex mapping_guard;
 };

src/yuzu_video_core/flush_region_cache_filter.h

@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#pragma once
+
+#include <array>
+#include <atomic>
+#include "yuzu_video_core/cache_types.h"
+
+namespace VideoCommon {
+
+// Conservative presence filter, not a dirty-memory tracker. Bits are never cleared:
+// deletion, remapping and address reuse can only produce extra cache consultations.
+// Register each resource before publishing it in the cache's lookup tables.
+template <unsigned AddressBits = 34, unsigned PageBits = 16>
+class FlushRegionCacheFilter {
+    static_assert(PageBits < AddressBits && AddressBits < 64);
+    static constexpr u64 AddressLimit = u64{1} << AddressBits;
+    static constexpr size_t WordCount = ((AddressLimit >> PageBits) + 63) / 64;
+    static constexpr std::array Types{CacheType::TextureCache, CacheType::QueryCache,
+                                      CacheType::BufferCache};
+
+public:
+    void Register(DAddr addr, u64 size, CacheType types) noexcept {
+        if (size == 0) {
+            return;
+        }
+        if (addr >= AddressLimit || size > AddressLimit - addr) {
+            // An invalid/unsupported registration must not leave a false negative.
+            unbounded.fetch_or(static_cast<u32>(types), std::memory_order_release);
+            return;
+        }
+        const u64 first = addr >> PageBits;
+        const u64 last = (addr + size - 1) >> PageBits;
+        for (size_t cache = 0; cache < Types.size(); ++cache) {
+            if (False(types & Types[cache])) {
+                continue;
+            }
+            for (u64 page = first; page <= last;) {
+                const unsigned begin = static_cast<unsigned>(page & 63);
+                const u64 word = page >> 6;
+                const unsigned end = static_cast<unsigned>(
+                    (last >> 6) == word ? (last & 63) + 1 : 64);
+                const u64 mask = (~u64{0} << begin) &
+                                 (end == 64 ? ~u64{0} : (u64{1} << end) - 1);
+                pages[cache][word].fetch_or(mask, std::memory_order_release);
+                page = (word + 1) << 6;
+            }
+        }
+    }
+
+    [[nodiscard]] CacheType Filter(DAddr addr, u64 size, CacheType requested) const noexcept {
+        if (size == 0) {
+            return CacheType::None;
+        }
+        if (addr >= AddressLimit || size > AddressLimit - addr) {
+            return requested;
+        }
+        const auto fallback = static_cast<CacheType>(unbounded.load(std::memory_order_acquire));
+        const u64 first = addr >> PageBits;
+        const u64 last = (addr + size - 1) >> PageBits;
+        for (size_t cache = 0; cache < Types.size(); ++cache) {
+            if (False(requested & Types[cache]) || True(fallback & Types[cache])) {
+                continue;
+            }
+            bool present = false;
+            for (u64 page = first; page <= last; ++page) {
+                if ((pages[cache][page >> 6].load(std::memory_order_acquire) &
+                     (u64{1} << (page & 63))) != 0) {
+                    present = true;
+                    break;
+                }
+            }
+            if (!present) {
+                requested &= ~Types[cache];
+            }
+        }
+        return requested;
+    }
+
+private:
+    std::array<std::array<std::atomic<u64>, WordCount>, Types.size()> pages{};
+    std::atomic<u32> unbounded{};
+};
+
+} // namespace VideoCommon

src/yuzu_video_core/query_cache/query_cache.h

@@ -244,6 +244,8 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type
         return;
     }
     DAddr cpu_addr = *cpu_addr_opt;
+    impl->device_memory.RegisterFlushCacheRegion(cpu_addr, has_timestamp ? 16 : 4,
+                                                CacheType::QueryCache);
     const size_t new_query_id = streamer->WriteCounter(cpu_addr, has_timestamp, payload, subreport);
     auto* query = streamer->GetQuery(new_query_id);
     if (is_fence) {

src/yuzu_video_core/renderer_vulkan/vk_rasterizer.cpp

@@ -534,6 +534,13 @@ void RasterizerVulkan::FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType
     if (addr == 0 || size == 0) {
         return;
     }
+    // Registered pages retain the existing lock/download/fence path.
+    which = device_memory.FilterFlushCaches(addr, size, which);
+    if (False(which & (VideoCommon::CacheType::TextureCache |
+                       VideoCommon::CacheType::BufferCache |
+                       VideoCommon::CacheType::QueryCache))) {
+        return;
+    }
     if (True(which & VideoCommon::CacheType::TextureCache)) {
         std::scoped_lock lock{texture_cache.mutex};
         texture_cache.DownloadMemory(addr, size);

src/yuzu_video_core/texture_cache/texture_cache.h

@@ -2053,6 +2053,8 @@ void TextureCache<P>::RegisterImage(ImageId image_id) {
         (*channel_state->gpu_page_table)[page].push_back(image_id);
     });
     if (False(image.flags & ImageFlagBits::Sparse)) {
+        device_memory.RegisterFlushCacheRegion(image.cpu_addr, image.guest_size_bytes,
+                                              CacheType::TextureCache);
         auto map_id =
             slot_map_views.insert(image.gpu_addr, image.cpu_addr, image.guest_size_bytes, image_id);
         ForEachCPUPage(image.cpu_addr, image.guest_size_bytes,
@@ -2063,6 +2065,7 @@ void TextureCache<P>::RegisterImage(ImageId image_id) {
     boost::container::small_vector<ImageViewId, 16> sparse_maps;
     ForEachSparseSegment(
         image, [this, image_id, &sparse_maps](GPUVAddr gpu_addr, DAddr cpu_addr, size_t size) {
+            device_memory.RegisterFlushCacheRegion(cpu_addr, size, CacheType::TextureCache);
             auto map_id = slot_map_views.insert(gpu_addr, cpu_addr, size, image_id);
             ForEachCPUPage(cpu_addr, size,
                            [this, map_id](u64 page) { page_table[page].push_back(map_id); });

src/yuzu_video_core/yuzu_video_core.vcxproj

@@ -39,6 +39,7 @@
     <ClInclude Include="buffer_cache\usage_tracker.h" />
     <ClInclude Include="buffer_cache\word_manager.h" />
     <ClInclude Include="cache_types.h" />
+    <ClInclude Include="flush_region_cache_filter.h" />
     <ClInclude Include="capture.h" />
     <ClInclude Include="cdma_pusher.h" />
     <ClInclude Include="compatible_formats.h" />

src/yuzu_video_core/yuzu_video_core.vcxproj.filters

@@ -267,6 +267,9 @@
     <ClInclude Include="cache_types.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="flush_region_cache_filter.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="capture.h">
       <Filter>Header Files</Filter>
     </ClInclude>