shifathosn
/
KytyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

graphics: copy sparse buffer uploads through resident mappings

This commit is contained in:
nmzik 2026-10-13 01:53:15 +02:00
parent 9a3aca963a
commit 40692dfb64
6 changed files with 172 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/graphics/host_gpu/renderer/cache/bufferCache.cpp vendored
View File

@ -471,7 +471,9 @@ vk::Buffer BufferCache::UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> c
vk::BufferUsageFlagBits::eTransferSrc, [&](uint8_t* mapped, uint64_t base_offset) {
for (auto& copy: copies) {
const auto address = buffer.CpuAddress() + copy.dstOffset;
std::memcpy(mapped + copy.srcOffset, reinterpret_cast<const void*>(address), copy.size);
if (!LibKernel::Memory::CopySparseMemory(address, mapped + copy.srcOffset, copy.size)) {
EXIT("BufferCache: failed to read sparse guest buffer memory\n");
}
copy.srcOffset += base_offset;
}
});
@ -493,7 +495,9 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
m_graphics.physical_device_properties.limits.minUniformBufferOffsetAlignment, 1);
auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false);
if (mapped != nullptr) {
std::memcpy(mapped, reinterpret_cast<const void*>(vaddr), size);
if (!LibKernel::Memory::CopySparseMemory(vaddr, mapped, size)) {
EXIT("BufferCache: failed to read sparse guest buffer memory\n");
}
m_stream_buffer.Commit();
return {&m_stream_buffer, offset};
}

6
src/kernel/memory.cpp
View File

@ -962,7 +962,11 @@ static bool IsInPrtAperture(uint64_t address, uint64_t size = 1) {
}
bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size) {
return g_guest_address_space->TryReadSparseBacking(vaddr, data, size);
return g_guest_address_space->CopySparseMemory<false>(vaddr, data, size);
}
bool CopySparseMemory(uint64_t vaddr, void* data, uint64_t size) {
return g_guest_address_space->CopySparseMemory<true>(vaddr, data, size);
}
static bool SelfTestSub64SharedPlaceholderAlias() {

1
src/kernel/memory.h
View File

@ -116,6 +116,7 @@ bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_
bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
bool TryReadBufferBacking(uint64_t vaddr, void* data, uint64_t size);
bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size);
bool CopySparseMemory(uint64_t vaddr, void* data, uint64_t size);
[[nodiscard]] uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size);
void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept;
void InvalidateMemory(uint64_t vaddr, uint64_t size);

103
src/kernel/memoryAddressSpace.inc
View File

@ -28,6 +28,35 @@ static bool CoalescePlaceholders(uint64_t vaddr, uint64_t size) {
}
#endif
template <typename Regions, typename Source, typename Gap>
static void CopySparseRange(const Regions& regions, uint64_t vaddr, void* data, uint64_t size,
Source&& source, Gap&& gap) {
const auto end = vaddr + size;
auto mapping = regions.lower_bound(vaddr);
if (mapping != regions.begin()) {
auto previous = std::prev(mapping);
if (previous->second.vaddr + previous->second.size > vaddr) {
mapping = previous;
}
}
auto* destination = static_cast<uint8_t*>(data);
auto cursor = vaddr;
for (; mapping != regions.end() && mapping->second.vaddr < end; ++mapping) {
const auto& entry = mapping->second;
const auto start = std::max(vaddr, entry.vaddr);
const auto bytes = std::min(end, entry.vaddr + entry.size) - start;
if (cursor < start) {
gap(cursor, destination + (cursor - vaddr), start - cursor);
}
std::memcpy(destination + (start - vaddr), source(entry, start), bytes);
cursor = start + bytes;
}
if (cursor < end) {
gap(cursor, destination + (cursor - vaddr), end - cursor);
}
}
class GuestBackingStore {
public:
enum class FailureReason {
@ -247,7 +276,8 @@ public:
return true;
}
bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size) {
template <typename Gap>
bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size, Gap&& gap) {
if (!IsAvailable() || data == nullptr || size == 0 || UINT64_MAX - vaddr < size) {
return false;
}
@ -258,31 +288,11 @@ public:
g_test_backing_read(vaddr, size);
}
#endif
const auto end = vaddr + size;
auto mapping = m_maps.lower_bound(vaddr);
if (mapping != m_maps.begin()) {
auto previous = std::prev(mapping);
if (previous->second.vaddr + previous->second.size > vaddr) {
mapping = previous;
}
}
auto* destination = static_cast<uint8_t*>(data);
auto cursor = vaddr;
for (; mapping != m_maps.end() && mapping->second.vaddr < end; ++mapping) {
const auto& entry = mapping->second;
const auto start = std::max(vaddr, entry.vaddr);
const auto bytes = std::min(end, entry.vaddr + entry.size) - start;
const auto offset = entry.backing_offset + start - entry.vaddr;
if (cursor < start) {
std::memset(destination + (cursor - vaddr), 0, start - cursor);
}
std::memcpy(destination + (start - vaddr), m_backing_base + offset, bytes);
cursor = start + bytes;
}
if (cursor < end) {
std::memset(destination + (cursor - vaddr), 0, end - cursor);
}
CopySparseRange(m_maps, vaddr, data, size,
[this](const auto& entry, uint64_t start) {
return m_backing_base + (entry.backing_offset + start - entry.vaddr);
},
gap);
return true;
}
@ -773,22 +783,41 @@ public:
return m_backing->TryReadBacking(vaddr, data, size);
}
bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size) {
if (!Owns(vaddr, size)) {
template <bool copy_private>
bool CopySparseMemory(uint64_t vaddr, void* data, uint64_t size) {
if (data == nullptr || !Owns(vaddr, size)) {
return false;
}
std::shared_lock lock(m_mutex);
auto mapping = m_mapped.upper_bound(vaddr);
if (mapping != m_mapped.begin()) {
--mapping;
}
for (; mapping != m_mapped.end() && mapping->second.vaddr < vaddr + size; ++mapping) {
const auto& region = mapping->second;
if (region.type == MappingType::Private && region.vaddr + region.size > vaddr) {
return false;
if constexpr (!copy_private) {
auto mapping = m_mapped.upper_bound(vaddr);
if (mapping != m_mapped.begin()) {
--mapping;
}
for (; mapping != m_mapped.end() && mapping->second.vaddr < vaddr + size; ++mapping) {
const auto& region = mapping->second;
if (region.type == MappingType::Private && region.vaddr + region.size > vaddr) {
return false;
}
}
}
return m_backing->TryReadSparseBacking(vaddr, data, size);
return m_backing->TryReadSparseBacking(
vaddr, data, size, [&](uint64_t start, uint8_t* destination, uint64_t bytes) {
const auto zero = [](uint64_t, uint8_t* output, uint64_t length) {
std::memset(output, 0, length);
};
if constexpr (copy_private) {
// Backing gaps can contain private mappings or reservations only.
CopySparseRange(m_mapped, start, destination, bytes,
[](const auto& region, uint64_t address) {
EXIT_IF(region.type != MappingType::Private);
return reinterpret_cast<const void*>(address);
},
zero);
} else {
zero(start, destination, bytes);
}
});
}
bool BackingContains(uint64_t vaddr, uint64_t size) { return m_backing->Contains(vaddr, size); }

41
tests/ShaderRecompilerComputeTests.cpp
View File

@ -5344,6 +5344,47 @@ public:
ReadBuffer(name, readback, 8) == std::vector<u32>{0x00020001u, 0x00020001u, 0u, 0u,
0x00020101u, 0xabcdef01u, 0x00020103u, 0x00040101u},
"upload lost sparse holes, boundary bytes, compute access, or in-flight lifetime");
// Cached uploads must read reserved holes without faulting back into the tracker,
// while retaining the private-memory sources accepted by ordinary buffer reads.
const auto private_base = LibKernel::Memory::AllocateRuntimeMemory(
0, page, Common::VirtualMemory::Mode::ReadWrite, "private_buffer_upload");
Require(name, "private source allocation", private_base != 0, "private allocation failed");
auto *private_words = reinterpret_cast<u32 *>(private_base);
private_words[0] = 0x55667788u;
private_words[page / sizeof(u32) - 1u] = 0x99aabbccu;
context.MapMemory(private_base, page);
context.MapMemory(base + page, 2u * page);
const auto sparse_buffer = cache.ObtainBuffer(base, 2u * page, false);
const auto private_buffer = cache.ObtainBuffer(private_base, page, false);
const auto sparse_stream = cache.ObtainBuffer(base + 2u * page, sizeof(u32), false);
Require(name, "sparse stream source",
sparse_stream.first == &cache.GetUtilityBuffer(MemoryUsage::Stream),
"small sparse read did not exercise the stream path");
const std::array<vk::BufferCopy, 2> sparse_copies{{
{sparse_buffer.second, 0, sizeof(u32)},
{sparse_buffer.second + page, sizeof(u32), sizeof(u32)}}};
const std::array<vk::BufferCopy, 2> private_copies{{
{private_buffer.second, 2u * sizeof(u32), sizeof(u32)},
{private_buffer.second + page - sizeof(u32), 3u * sizeof(u32), sizeof(u32)}}};
scheduler.Current().Handle().copyBuffer(sparse_buffer.first->Handle(), readback.buffer,
sparse_copies.size(), sparse_copies.data());
scheduler.Current().Handle().copyBuffer(private_buffer.first->Handle(), readback.buffer,
private_copies.size(), private_copies.data());
const vk::BufferCopy stream_copy{sparse_stream.second, 4u * sizeof(u32), sizeof(u32)};
scheduler.Current().Handle().copyBuffer(sparse_stream.first->Handle(), readback.buffer,
1, &stream_copy);
scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr);
scheduler.Finish();
Require(name, "cached sparse and private upload",
ReadBuffer(name, readback, 5) == std::vector<u32>{0x00020001u, 0u, 0x55667788u, 0x99aabbccu, 0u} &&
!cache.IsRegionGpuModified(base, 2u * page) &&
!cache.IsRegionGpuModified(private_base, page),
"cached upload lost sparse/private data or changed read-only GPU ownership");
context.UnmapMemory(private_base, page);
context.UnmapMemory(base + page, 2u * page);
Require(name, "private source release", LibKernel::Memory::FreeGuestMemory(private_base, page),
"private allocation release failed");
DestroyBuffer(&readback);
for (const auto offset : mapped_offsets) context.UnmapMemory(base + offset, page);
scheduler.Finish();

55
tests/VirtualMemoryAllocationTests.cpp
View File

@ -619,7 +619,8 @@ void TestConcurrentBackingReads() {
constexpr uint64_t original = 0x123456789abcdef0ull;
for (const auto& readers: {std::pair {&TryReadBacking, &TryReadBacking},
std::pair {&TryReadBacking, &TryReadSparseBacking},
std::pair {&TryReadSparseBacking, &TryReadSparseBacking}}) {
std::pair {&TryReadSparseBacking, &TryReadSparseBacking},
std::pair {&CopySparseMemory, &CopySparseMemory}}) {
const auto base = MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw,
"concurrent_backing_reads");
std::memcpy(reinterpret_cast<void*>(base), &original, sizeof(original));
@ -654,7 +655,7 @@ void TestBackingReadExcludesWritesAndUnmap() {
const char* test = "BackingReadExcludesWritesAndUnmap";
constexpr uint64_t original = 0x123456789abcdef0ull;
constexpr uint64_t replacement = 0xfedcba9876543210ull;
for (const auto read: {TryReadBacking, TryReadSparseBacking}) {
for (const auto read: {TryReadBacking, TryReadSparseBacking, CopySparseMemory}) {
for (const bool unmap: {false, true}) {
const auto base = MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw,
"backing_read_exclusion");
@ -1152,6 +1153,54 @@ void TestRuntimeMemoryOwnerLifecycle() {
std::printf("[host] %-48s ok\n", test);
}
void TestCopySparseMemoryMappings() {
using namespace Libs::LibKernel::Memory;
const char* test = "CopySparseMemoryMappings";
constexpr uint64_t size = SceKernelPageSize * 5;
const auto base = AllocateRuntimeMemory(0, size, Common::VirtualMemory::Mode::ReadWrite,
"sparse_copy_private");
Check(test, base != 0, "private allocation failed");
for (const auto page : {0u, 2u, 4u}) {
Check(test, FreeGuestMemory(base + page * SceKernelPageSize, SceKernelPageSize),
"private page release failed");
}
void* shared = reinterpret_cast<void*>(base);
CheckOk(test, KernelMapNamedFlexibleMemory(&shared, SceKernelPageSize, SceKernelProtCpuRw,
SceKernelMapFixed, "sparse_copy_shared"),
"map shared first page");
constexpr std::array<uint8_t, 5> values {0x3c, 0xa7, 0, 0x5d, 0};
for (const auto page : {0u, 1u, 3u}) {
std::memset(reinterpret_cast<void*>(base + page * SceKernelPageSize), values[page],
SceKernelPageSize);
}
CheckOk(test, KernelMprotect(shared, SceKernelPageSize, 0), "protect shared first page");
constexpr uint64_t prefix = 29, suffix = 17, copied = size - prefix - suffix;
std::vector<uint8_t> bytes(copied + 2, 0x55);
Check(test, !TryReadSparseBacking(base + prefix, bytes.data() + 1, copied) &&
std::all_of(bytes.begin(), bytes.end(), [](uint8_t value) { return value == 0x55; }),
"strict backing read accepted private memory or changed its destination");
Check(test, CopySparseMemory(base + prefix, bytes.data() + 1, copied),
"sparse copy rejected mixed shared, private and unbacked pages");
Check(test, bytes.front() == 0x55 && bytes.back() == 0x55,
"sparse copy overwrote destination canaries");
for (uint64_t i = 0; i < copied; ++i) {
Check(test, bytes[i + 1] == values[(prefix + i) / SceKernelPageSize],
"sparse copy misplaced mapped data or an unbacked gap");
}
uint64_t canary = UINT64_MAX;
Check(test, !CopySparseMemory(0x10000, &canary, sizeof(canary)) &&
!CopySparseMemory(UINT64_MAX - 3, &canary, sizeof(canary)) &&
!CopySparseMemory(base, &canary, 0) &&
!CopySparseMemory(base, nullptr, sizeof(canary)) && canary == UINT64_MAX,
"invalid sparse copy accepted its range or changed its destination");
CheckOk(test, KernelMunmap(base, SceKernelPageSize), "unmap shared first page");
for (const auto page : {1u, 3u}) {
Check(test, FreeGuestMemory(base + page * SceKernelPageSize, SceKernelPageSize),
"private page cleanup failed");
}
std::printf("[host] %-48s ok\n", test);
}
void TestFlexibleMapQueryAndWholeMunmap() {
const char* test = "FlexibleMapQueryAndWholeMunmap";
const auto baseline = AvailableFlexibleMemory(test);
@ -4420,6 +4469,7 @@ int main(int argc, char** argv) {
RunTest(TestConcurrentBackingReads);
RunTest(TestBackingReadExcludesWritesAndUnmap);
RunTest(TestSparseBackingReadPreservesResidency);
RunTest(TestCopySparseMemoryMappings);
RunTest(TestSparseReadDuringDirectCommit);
RunTest(TestDirectMapQueryOffsetAndPartialMunmap);
RunTest(TestFlexibleMemoryUsesSharedBacking);
@ -4473,6 +4523,7 @@ int main(int argc, char** argv) {
RunTest(TestProsperoArgumentAndInfoSizeContracts);
RunTest(TestGuestAddressSpaceOwnsReservationsBeforeBacking);
RunTest(TestSparseBackingReadPreservesResidency);
RunTest(TestCopySparseMemoryMappings);
RunTest(TestSparseReadDuringDirectCommit);
RunTest(TestConcurrentBackingReads);
RunTest(TestBackingReadExcludesWritesAndUnmap);