diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index 73639a91a..8cdc11cb3 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -3,9 +3,11 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include +#include #include #include #include +#include #include namespace AgcDriver::Graphics { @@ -28,10 +30,12 @@ struct BufferAllocation { // requests of a MiB and more keep their exact size (the multi-MiB registered-range snapshots repeat // exactly, and rounding them would waste pinned host memory). Retained allocations are evicted least // recently used under a byte budget and a slot count, so a burst of small buffers between two large -// builds no longer sweeps the large ones out. Reuse is safe because a Buffer is only released once +// builds no longer sweeps the large ones out. Free slots are kept per size, usage and memory +// properties, so a lookup does not scan the other slots. Reuse is safe because a Buffer is only released once // the GPU work using it completed (kept until the batch fence, or after a CommandBatch wait). -// APS5_NO_BUFFER_CLASSES=1 matches exact sizes only, as before; APS5_BUFFER_POOL_SLOTS=64 restores -// the old slot count (the bound applies to each tier below, so twice that many slots in all). +// APS5_NO_BUFFER_CLASSES=1 matches exact sizes only, as before; APS5_BUFFER_POOL_SLOTS= sets the +// slot count (the bound applies to each tier below, and is held to a sixth of the device's +// maxMemoryAllocationCount so the three tiers keep half of it free). // // The size classes and the exact-size allocations are retained in two tiers with a budget each: // the multi-MiB texture scratch buffers that come back from a batch (GPU-direct uploads and @@ -60,10 +64,23 @@ private: BufferAllocation allocation; std::uint64_t lastUse; }; - // One retention tier: its slots, their bytes, the byte budget they are evicted under and its - // counters (APS5_PROFILE_DRAW, reported every 10 s from Take). + struct SlotKey { + std::size_t bytes; + VkBufferUsageFlags usage; + VkMemoryPropertyFlags properties; + bool operator==(const SlotKey&) const = default; + }; + struct SlotKeyHash { + std::size_t operator()(const SlotKey& key) const noexcept { + return std::hash{}(key.bytes) ^ (static_cast(key.usage) << 32u) ^ (static_cast(key.properties) << 48u); + } + }; + // One retention tier: its slots by key (each list oldest first), their count and bytes, the + // byte budget they are evicted under and its counters (APS5_PROFILE_DRAW, reported every 10 s + // from Take). struct Tier { - std::vector free; + std::unordered_map, SlotKeyHash> free; + std::size_t slots = 0; VkDeviceSize retainedBytes = 0; VkDeviceSize budget = 0; std::uint64_t hits = 0; @@ -76,12 +93,13 @@ private: // The device tier's budget (APS5_STAGING_POOL_MIB), read once. static VkDeviceSize DeviceBudget(); void destroy(const BufferAllocation& allocation) noexcept; - // Moves the tier's least recently used slot to `evicted`; the caller destroys those after - // releasing the mutex, so builds taking buffers on other threads do not wait behind the - // Vulkan destroy calls. Nothing changes when the vector cannot grow. + // Moves the tier's least recently used slot (the oldest front of its lists) to `evicted`; the + // caller destroys those after releasing the mutex, so builds taking buffers on other threads + // do not wait behind the Vulkan destroy calls. Nothing changes when the vector cannot grow. void evictOldest(Tier& tier, std::vector& evicted); // The retained-slot bound of each tier (APS5_BUFFER_POOL_SLOTS, default `defaultSlots`), read once. static std::size_t MaxSlots(); + std::size_t maxSlots; VkDevice device; PFN_vkUnmapMemory unmap; PFN_vkDestroyBuffer destroyBuffer; @@ -93,12 +111,12 @@ private: Tier deviceTier; std::uint64_t clock = 0; static constexpr VkDeviceSize budget = 512ull * 1024 * 1024; - // The small tier's own budget (512 slots of at most half a MiB each): pinned host memory the - // large tier's budget does not count. + // The small tier's own budget (slots of at most half a MiB each): pinned host memory the large + // tier's budget does not count. static constexpr VkDeviceSize smallBudget = 64ull * 1024 * 1024; // Requests of this size and more keep their exact size and go to the large tier. static constexpr std::size_t classLimit = std::size_t{1} << 20u; - static constexpr std::size_t defaultSlots = 512; + static constexpr std::size_t defaultSlots = 4096; }; std::shared_ptr GetBufferPool(const Context& context); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index a24303db3..cdd541e0f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -17,16 +17,18 @@ bool sharedTiers() { } -BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function("vkUnmapMemory")), destroyBuffer(context.Function("vkDestroyBuffer")), freeMemory(context.Function("vkFreeMemory")) { +BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function("vkUnmapMemory")), destroyBuffer(context.Function("vkDestroyBuffer")), freeMemory(context.Function("vkFreeMemory")), maxSlots(std::max(1, std::min(MaxSlots(), context.limits.maxMemoryAllocationCount / 6u))) { smallTier.budget = smallBudget; largeTier.budget = budget; deviceTier.budget = DeviceBudget(); } BufferPool::~BufferPool() { - for (const auto& slot : smallTier.free) destroy(slot.allocation); - for (const auto& slot : largeTier.free) destroy(slot.allocation); - for (const auto& slot : deviceTier.free) destroy(slot.allocation); + for (auto* tier : {&smallTier, &largeTier, &deviceTier}) { + for (const auto& [key, slots] : tier->free) { + for (const auto& slot : slots) destroy(slot.allocation); + } + } } VkDeviceSize BufferPool::DeviceBudget() { @@ -66,31 +68,32 @@ std::optional BufferPool::Take(std::size_t bytes, VkBufferUsag const auto now = std::chrono::steady_clock::now(); if (now - lastReport > std::chrono::seconds(10)) { lastReport = now; - std::fprintf(stderr, "[bufferpool] small: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB); large: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB)%s; device: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB of %llu)\n", static_cast(smallTier.hits), static_cast(smallTier.misses), static_cast(smallTier.evictions), smallTier.free.size(), smallTier.retainedBytes / 1048576.0, static_cast(largeTier.hits), static_cast(largeTier.misses), static_cast(largeTier.evictions), largeTier.free.size(), largeTier.retainedBytes / 1048576.0, sharedTiers() ? " (shared)" : "", static_cast(deviceTier.hits), static_cast(deviceTier.misses), static_cast(deviceTier.evictions), deviceTier.free.size(), deviceTier.retainedBytes / 1048576.0, static_cast(deviceTier.budget >> 20u)); + std::fprintf(stderr, "[bufferpool] small: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB); large: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB)%s; device: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB of %llu)\n", static_cast(smallTier.hits), static_cast(smallTier.misses), static_cast(smallTier.evictions), smallTier.slots, smallTier.retainedBytes / 1048576.0, static_cast(largeTier.hits), static_cast(largeTier.misses), static_cast(largeTier.evictions), largeTier.slots, largeTier.retainedBytes / 1048576.0, sharedTiers() ? " (shared)" : "", static_cast(deviceTier.hits), static_cast(deviceTier.misses), static_cast(deviceTier.evictions), deviceTier.slots, deviceTier.retainedBytes / 1048576.0, static_cast(deviceTier.budget >> 20u)); } } auto& tier = tierFor(capacity, properties); - for (auto it = tier.free.begin(); it != tier.free.end(); ++it) { - const auto& allocation = it->allocation; - if (allocation.bytes != capacity || allocation.usage != usage || allocation.properties != properties) continue; - auto result = allocation; - tier.retainedBytes -= allocation.allocationBytes; - *it = std::move(tier.free.back()); - tier.free.pop_back(); - ++tier.hits; - return result; + const auto found = tier.free.find({capacity, usage, properties}); + if (found == tier.free.end() || found->second.empty()) { + ++tier.misses; + return std::nullopt; } - ++tier.misses; - return std::nullopt; + auto result = found->second.back().allocation; + found->second.pop_back(); + if (found->second.empty()) tier.free.erase(found); + --tier.slots; + tier.retainedBytes -= result.allocationBytes; + ++tier.hits; + return result; } void BufferPool::evictOldest(Tier& tier, std::vector& evicted) { - const auto oldest = std::min_element(tier.free.begin(), tier.free.end(), [](const Slot& left, const Slot& right) { return left.lastUse < right.lastUse; }); + const auto oldest = std::min_element(tier.free.begin(), tier.free.end(), [](const auto& left, const auto& right) { return left.second.front().lastUse < right.second.front().lastUse; }); // First: a throw here leaves the slot retained and counted. - evicted.push_back(oldest->allocation); - tier.retainedBytes -= oldest->allocation.allocationBytes; - *oldest = std::move(tier.free.back()); - tier.free.pop_back(); + evicted.push_back(oldest->second.front().allocation); + tier.retainedBytes -= oldest->second.front().allocation.allocationBytes; + oldest->second.pop_front(); + if (oldest->second.empty()) tier.free.erase(oldest); + --tier.slots; ++tier.evictions; } @@ -115,9 +118,9 @@ void BufferPool::Put(const BufferAllocation& allocation) noexcept { if (allocation.allocationBytes > tier.budget) { evicted.push_back(allocation); } else { - const auto maxSlots = MaxSlots(); - while (!tier.free.empty() && (tier.retainedBytes + allocation.allocationBytes > tier.budget || tier.free.size() >= maxSlots)) evictOldest(tier, evicted); - tier.free.push_back({allocation, ++clock}); + while (tier.slots != 0 && (tier.retainedBytes + allocation.allocationBytes > tier.budget || tier.slots >= maxSlots)) evictOldest(tier, evicted); + tier.free[{allocation.bytes, allocation.usage, allocation.properties}].push_back({allocation, ++clock}); + ++tier.slots; tier.retainedBytes += allocation.allocationBytes; } } catch (...) {