From 40692dfb64a87d3214c04271f075feb4ef61754a Mon Sep 17 00:00:00 2001 From: nmzik Date: Tue, 13 Oct 2026 01:53:15 +0200 Subject: [PATCH] graphics: copy sparse buffer uploads through resident mappings --- .../host_gpu/renderer/cache/bufferCache.cpp | 8 +- src/kernel/memory.cpp | 6 +- src/kernel/memory.h | 1 + src/kernel/memoryAddressSpace.inc | 103 +++++++++++------- tests/ShaderRecompilerComputeTests.cpp | 41 +++++++ tests/VirtualMemoryAllocationTests.cpp | 55 +++++++++- 6 files changed, 172 insertions(+), 42 deletions(-) diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index da3c8d75..91376ea2 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -471,7 +471,9 @@ vk::Buffer BufferCache::UploadCopies(Buffer& buffer, std::span c vk::BufferUsageFlagBits::eTransferSrc, [&](uint8_t* mapped, uint64_t base_offset) { for (auto& copy: copies) { const auto address = buffer.CpuAddress() + copy.dstOffset; - std::memcpy(mapped + copy.srcOffset, reinterpret_cast(address), copy.size); + if (!LibKernel::Memory::CopySparseMemory(address, mapped + copy.srcOffset, copy.size)) { + EXIT("BufferCache: failed to read sparse guest buffer memory\n"); + } copy.srcOffset += base_offset; } }); @@ -493,7 +495,9 @@ std::pair BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t m_graphics.physical_device_properties.limits.minUniformBufferOffsetAlignment, 1); auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false); if (mapped != nullptr) { - std::memcpy(mapped, reinterpret_cast(vaddr), size); + if (!LibKernel::Memory::CopySparseMemory(vaddr, mapped, size)) { + EXIT("BufferCache: failed to read sparse guest buffer memory\n"); + } m_stream_buffer.Commit(); return {&m_stream_buffer, offset}; } diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index 87a9bf94..3eb86119 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -962,7 +962,11 @@ static bool IsInPrtAperture(uint64_t address, uint64_t size = 1) { } bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size) { - return g_guest_address_space->TryReadSparseBacking(vaddr, data, size); + return g_guest_address_space->CopySparseMemory(vaddr, data, size); +} + +bool CopySparseMemory(uint64_t vaddr, void* data, uint64_t size) { + return g_guest_address_space->CopySparseMemory(vaddr, data, size); } static bool SelfTestSub64SharedPlaceholderAlias() { diff --git a/src/kernel/memory.h b/src/kernel/memory.h index 4b82e41b..c3eb032a 100644 --- a/src/kernel/memory.h +++ b/src/kernel/memory.h @@ -116,6 +116,7 @@ bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_ bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size); bool TryReadBufferBacking(uint64_t vaddr, void* data, uint64_t size); bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size); +bool CopySparseMemory(uint64_t vaddr, void* data, uint64_t size); [[nodiscard]] uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size); void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept; void InvalidateMemory(uint64_t vaddr, uint64_t size); diff --git a/src/kernel/memoryAddressSpace.inc b/src/kernel/memoryAddressSpace.inc index 47e916f3..69bb6eb7 100644 --- a/src/kernel/memoryAddressSpace.inc +++ b/src/kernel/memoryAddressSpace.inc @@ -28,6 +28,35 @@ static bool CoalescePlaceholders(uint64_t vaddr, uint64_t size) { } #endif +template +static void CopySparseRange(const Regions& regions, uint64_t vaddr, void* data, uint64_t size, + Source&& source, Gap&& gap) { + const auto end = vaddr + size; + auto mapping = regions.lower_bound(vaddr); + if (mapping != regions.begin()) { + auto previous = std::prev(mapping); + if (previous->second.vaddr + previous->second.size > vaddr) { + mapping = previous; + } + } + + auto* destination = static_cast(data); + auto cursor = vaddr; + for (; mapping != regions.end() && mapping->second.vaddr < end; ++mapping) { + const auto& entry = mapping->second; + const auto start = std::max(vaddr, entry.vaddr); + const auto bytes = std::min(end, entry.vaddr + entry.size) - start; + if (cursor < start) { + gap(cursor, destination + (cursor - vaddr), start - cursor); + } + std::memcpy(destination + (start - vaddr), source(entry, start), bytes); + cursor = start + bytes; + } + if (cursor < end) { + gap(cursor, destination + (cursor - vaddr), end - cursor); + } +} + class GuestBackingStore { public: enum class FailureReason { @@ -247,7 +276,8 @@ public: return true; } - bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size) { + template + bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size, Gap&& gap) { if (!IsAvailable() || data == nullptr || size == 0 || UINT64_MAX - vaddr < size) { return false; } @@ -258,31 +288,11 @@ public: g_test_backing_read(vaddr, size); } #endif - const auto end = vaddr + size; - auto mapping = m_maps.lower_bound(vaddr); - if (mapping != m_maps.begin()) { - auto previous = std::prev(mapping); - if (previous->second.vaddr + previous->second.size > vaddr) { - mapping = previous; - } - } - - auto* destination = static_cast(data); - auto cursor = vaddr; - for (; mapping != m_maps.end() && mapping->second.vaddr < end; ++mapping) { - const auto& entry = mapping->second; - const auto start = std::max(vaddr, entry.vaddr); - const auto bytes = std::min(end, entry.vaddr + entry.size) - start; - const auto offset = entry.backing_offset + start - entry.vaddr; - if (cursor < start) { - std::memset(destination + (cursor - vaddr), 0, start - cursor); - } - std::memcpy(destination + (start - vaddr), m_backing_base + offset, bytes); - cursor = start + bytes; - } - if (cursor < end) { - std::memset(destination + (cursor - vaddr), 0, end - cursor); - } + CopySparseRange(m_maps, vaddr, data, size, + [this](const auto& entry, uint64_t start) { + return m_backing_base + (entry.backing_offset + start - entry.vaddr); + }, + gap); return true; } @@ -773,22 +783,41 @@ public: return m_backing->TryReadBacking(vaddr, data, size); } - bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size) { - if (!Owns(vaddr, size)) { + template + bool CopySparseMemory(uint64_t vaddr, void* data, uint64_t size) { + if (data == nullptr || !Owns(vaddr, size)) { return false; } std::shared_lock lock(m_mutex); - auto mapping = m_mapped.upper_bound(vaddr); - if (mapping != m_mapped.begin()) { - --mapping; - } - for (; mapping != m_mapped.end() && mapping->second.vaddr < vaddr + size; ++mapping) { - const auto& region = mapping->second; - if (region.type == MappingType::Private && region.vaddr + region.size > vaddr) { - return false; + if constexpr (!copy_private) { + auto mapping = m_mapped.upper_bound(vaddr); + if (mapping != m_mapped.begin()) { + --mapping; + } + for (; mapping != m_mapped.end() && mapping->second.vaddr < vaddr + size; ++mapping) { + const auto& region = mapping->second; + if (region.type == MappingType::Private && region.vaddr + region.size > vaddr) { + return false; + } } } - return m_backing->TryReadSparseBacking(vaddr, data, size); + return m_backing->TryReadSparseBacking( + vaddr, data, size, [&](uint64_t start, uint8_t* destination, uint64_t bytes) { + const auto zero = [](uint64_t, uint8_t* output, uint64_t length) { + std::memset(output, 0, length); + }; + if constexpr (copy_private) { + // Backing gaps can contain private mappings or reservations only. + CopySparseRange(m_mapped, start, destination, bytes, + [](const auto& region, uint64_t address) { + EXIT_IF(region.type != MappingType::Private); + return reinterpret_cast(address); + }, + zero); + } else { + zero(start, destination, bytes); + } + }); } bool BackingContains(uint64_t vaddr, uint64_t size) { return m_backing->Contains(vaddr, size); } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index fcaf9005..deeec598 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -5344,6 +5344,47 @@ public: ReadBuffer(name, readback, 8) == std::vector{0x00020001u, 0x00020001u, 0u, 0u, 0x00020101u, 0xabcdef01u, 0x00020103u, 0x00040101u}, "upload lost sparse holes, boundary bytes, compute access, or in-flight lifetime"); + // Cached uploads must read reserved holes without faulting back into the tracker, + // while retaining the private-memory sources accepted by ordinary buffer reads. + const auto private_base = LibKernel::Memory::AllocateRuntimeMemory( + 0, page, Common::VirtualMemory::Mode::ReadWrite, "private_buffer_upload"); + Require(name, "private source allocation", private_base != 0, "private allocation failed"); + auto *private_words = reinterpret_cast(private_base); + private_words[0] = 0x55667788u; + private_words[page / sizeof(u32) - 1u] = 0x99aabbccu; + context.MapMemory(private_base, page); + context.MapMemory(base + page, 2u * page); + const auto sparse_buffer = cache.ObtainBuffer(base, 2u * page, false); + const auto private_buffer = cache.ObtainBuffer(private_base, page, false); + const auto sparse_stream = cache.ObtainBuffer(base + 2u * page, sizeof(u32), false); + Require(name, "sparse stream source", + sparse_stream.first == &cache.GetUtilityBuffer(MemoryUsage::Stream), + "small sparse read did not exercise the stream path"); + const std::array sparse_copies{{ + {sparse_buffer.second, 0, sizeof(u32)}, + {sparse_buffer.second + page, sizeof(u32), sizeof(u32)}}}; + const std::array private_copies{{ + {private_buffer.second, 2u * sizeof(u32), sizeof(u32)}, + {private_buffer.second + page - sizeof(u32), 3u * sizeof(u32), sizeof(u32)}}}; + scheduler.Current().Handle().copyBuffer(sparse_buffer.first->Handle(), readback.buffer, + sparse_copies.size(), sparse_copies.data()); + scheduler.Current().Handle().copyBuffer(private_buffer.first->Handle(), readback.buffer, + private_copies.size(), private_copies.data()); + const vk::BufferCopy stream_copy{sparse_stream.second, 4u * sizeof(u32), sizeof(u32)}; + scheduler.Current().Handle().copyBuffer(sparse_stream.first->Handle(), readback.buffer, + 1, &stream_copy); + scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr); + scheduler.Finish(); + Require(name, "cached sparse and private upload", + ReadBuffer(name, readback, 5) == std::vector{0x00020001u, 0u, 0x55667788u, 0x99aabbccu, 0u} && + !cache.IsRegionGpuModified(base, 2u * page) && + !cache.IsRegionGpuModified(private_base, page), + "cached upload lost sparse/private data or changed read-only GPU ownership"); + context.UnmapMemory(private_base, page); + context.UnmapMemory(base + page, 2u * page); + Require(name, "private source release", LibKernel::Memory::FreeGuestMemory(private_base, page), + "private allocation release failed"); DestroyBuffer(&readback); for (const auto offset : mapped_offsets) context.UnmapMemory(base + offset, page); scheduler.Finish(); diff --git a/tests/VirtualMemoryAllocationTests.cpp b/tests/VirtualMemoryAllocationTests.cpp index 6ff8d20f..db066e01 100644 --- a/tests/VirtualMemoryAllocationTests.cpp +++ b/tests/VirtualMemoryAllocationTests.cpp @@ -619,7 +619,8 @@ void TestConcurrentBackingReads() { constexpr uint64_t original = 0x123456789abcdef0ull; for (const auto& readers: {std::pair {&TryReadBacking, &TryReadBacking}, std::pair {&TryReadBacking, &TryReadSparseBacking}, - std::pair {&TryReadSparseBacking, &TryReadSparseBacking}}) { + std::pair {&TryReadSparseBacking, &TryReadSparseBacking}, + std::pair {&CopySparseMemory, &CopySparseMemory}}) { const auto base = MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "concurrent_backing_reads"); std::memcpy(reinterpret_cast(base), &original, sizeof(original)); @@ -654,7 +655,7 @@ void TestBackingReadExcludesWritesAndUnmap() { const char* test = "BackingReadExcludesWritesAndUnmap"; constexpr uint64_t original = 0x123456789abcdef0ull; constexpr uint64_t replacement = 0xfedcba9876543210ull; - for (const auto read: {TryReadBacking, TryReadSparseBacking}) { + for (const auto read: {TryReadBacking, TryReadSparseBacking, CopySparseMemory}) { for (const bool unmap: {false, true}) { const auto base = MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_read_exclusion"); @@ -1152,6 +1153,54 @@ void TestRuntimeMemoryOwnerLifecycle() { std::printf("[host] %-48s ok\n", test); } +void TestCopySparseMemoryMappings() { + using namespace Libs::LibKernel::Memory; + const char* test = "CopySparseMemoryMappings"; + constexpr uint64_t size = SceKernelPageSize * 5; + const auto base = AllocateRuntimeMemory(0, size, Common::VirtualMemory::Mode::ReadWrite, + "sparse_copy_private"); + Check(test, base != 0, "private allocation failed"); + for (const auto page : {0u, 2u, 4u}) { + Check(test, FreeGuestMemory(base + page * SceKernelPageSize, SceKernelPageSize), + "private page release failed"); + } + void* shared = reinterpret_cast(base); + CheckOk(test, KernelMapNamedFlexibleMemory(&shared, SceKernelPageSize, SceKernelProtCpuRw, + SceKernelMapFixed, "sparse_copy_shared"), + "map shared first page"); + constexpr std::array values {0x3c, 0xa7, 0, 0x5d, 0}; + for (const auto page : {0u, 1u, 3u}) { + std::memset(reinterpret_cast(base + page * SceKernelPageSize), values[page], + SceKernelPageSize); + } + CheckOk(test, KernelMprotect(shared, SceKernelPageSize, 0), "protect shared first page"); + constexpr uint64_t prefix = 29, suffix = 17, copied = size - prefix - suffix; + std::vector bytes(copied + 2, 0x55); + Check(test, !TryReadSparseBacking(base + prefix, bytes.data() + 1, copied) && + std::all_of(bytes.begin(), bytes.end(), [](uint8_t value) { return value == 0x55; }), + "strict backing read accepted private memory or changed its destination"); + Check(test, CopySparseMemory(base + prefix, bytes.data() + 1, copied), + "sparse copy rejected mixed shared, private and unbacked pages"); + Check(test, bytes.front() == 0x55 && bytes.back() == 0x55, + "sparse copy overwrote destination canaries"); + for (uint64_t i = 0; i < copied; ++i) { + Check(test, bytes[i + 1] == values[(prefix + i) / SceKernelPageSize], + "sparse copy misplaced mapped data or an unbacked gap"); + } + uint64_t canary = UINT64_MAX; + Check(test, !CopySparseMemory(0x10000, &canary, sizeof(canary)) && + !CopySparseMemory(UINT64_MAX - 3, &canary, sizeof(canary)) && + !CopySparseMemory(base, &canary, 0) && + !CopySparseMemory(base, nullptr, sizeof(canary)) && canary == UINT64_MAX, + "invalid sparse copy accepted its range or changed its destination"); + CheckOk(test, KernelMunmap(base, SceKernelPageSize), "unmap shared first page"); + for (const auto page : {1u, 3u}) { + Check(test, FreeGuestMemory(base + page * SceKernelPageSize, SceKernelPageSize), + "private page cleanup failed"); + } + std::printf("[host] %-48s ok\n", test); +} + void TestFlexibleMapQueryAndWholeMunmap() { const char* test = "FlexibleMapQueryAndWholeMunmap"; const auto baseline = AvailableFlexibleMemory(test); @@ -4420,6 +4469,7 @@ int main(int argc, char** argv) { RunTest(TestConcurrentBackingReads); RunTest(TestBackingReadExcludesWritesAndUnmap); RunTest(TestSparseBackingReadPreservesResidency); + RunTest(TestCopySparseMemoryMappings); RunTest(TestSparseReadDuringDirectCommit); RunTest(TestDirectMapQueryOffsetAndPartialMunmap); RunTest(TestFlexibleMemoryUsesSharedBacking); @@ -4473,6 +4523,7 @@ int main(int argc, char** argv) { RunTest(TestProsperoArgumentAndInfoSizeContracts); RunTest(TestGuestAddressSpaceOwnsReservationsBeforeBacking); RunTest(TestSparseBackingReadPreservesResidency); + RunTest(TestCopySparseMemoryMappings); RunTest(TestSparseReadDuringDirectCommit); RunTest(TestConcurrentBackingReads); RunTest(TestBackingReadExcludesWritesAndUnmap);