1224 lines
47 KiB
C++
1224 lines
47 KiB
C++
#include "common/hostException.h"
|
|
#include "common/virtualMemory.h"
|
|
#include "graphics/host_gpu/memoryTracker.h"
|
|
#include "graphics/host_gpu/rangeSet.h"
|
|
|
|
#include <atomic>
|
|
#include <chrono>
|
|
#include <cstdint>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <mutex>
|
|
#include <semaphore>
|
|
#include <string>
|
|
#include <thread>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
|
#ifndef NOMINMAX
|
|
#define NOMINMAX
|
|
#endif
|
|
#include <windows.h>
|
|
#undef min
|
|
#undef max
|
|
#elif defined(__APPLE__)
|
|
#include <mach/mach.h>
|
|
#include <mach/mach_vm.h>
|
|
#include <mach-o/dyld.h>
|
|
#include <csignal>
|
|
#include <limits.h>
|
|
#include <map>
|
|
#include <sys/mman.h>
|
|
#include <sys/wait.h>
|
|
#include <unistd.h>
|
|
#else
|
|
#include <csignal>
|
|
#include <map>
|
|
#include <sys/mman.h>
|
|
#include <sys/wait.h>
|
|
#include <unistd.h>
|
|
#endif
|
|
|
|
namespace {
|
|
|
|
using Libs::Graphics::GuestRange;
|
|
using Libs::Graphics::MemoryTracker;
|
|
using Libs::Graphics::PageManager;
|
|
using Libs::Graphics::RangeSet;
|
|
using Libs::Graphics::TRACKER_ADDRESS_SIZE;
|
|
|
|
void Check(bool value, const char *text) {
|
|
if (!value) {
|
|
std::fprintf(stderr, "MemoryTrackerTests: failed: %s\n", text);
|
|
std::abort();
|
|
}
|
|
}
|
|
|
|
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
|
|
using DWORD = uint32_t;
|
|
constexpr uint32_t PAGE_NOACCESS = 1;
|
|
constexpr uint32_t PAGE_READONLY = 2;
|
|
constexpr uint32_t PAGE_READWRITE = 3;
|
|
constexpr uint32_t MEM_RESERVE = 0;
|
|
constexpr uint32_t MEM_COMMIT = 0;
|
|
constexpr uint32_t MEM_RELEASE = 0;
|
|
|
|
int ToHostProt(uint32_t protection) {
|
|
switch (protection) {
|
|
case PAGE_NOACCESS:
|
|
return PROT_NONE;
|
|
case PAGE_READONLY:
|
|
return PROT_READ;
|
|
default:
|
|
return PROT_READ | PROT_WRITE;
|
|
}
|
|
}
|
|
|
|
uint32_t Protection(const void *address) {
|
|
#if defined(__APPLE__)
|
|
mach_vm_address_t region_address =
|
|
reinterpret_cast<mach_vm_address_t>(address);
|
|
mach_vm_size_t region_size = 0;
|
|
vm_region_basic_info_data_64_t info{};
|
|
mach_msg_type_number_t info_count = VM_REGION_BASIC_INFO_COUNT_64;
|
|
mach_port_t object_name = MACH_PORT_NULL;
|
|
Check(mach_vm_region(mach_task_self(), ®ion_address, ®ion_size,
|
|
VM_REGION_BASIC_INFO_64,
|
|
reinterpret_cast<vm_region_info_t>(&info), &info_count,
|
|
&object_name) == KERN_SUCCESS,
|
|
"mach_vm_region failed");
|
|
if (object_name != MACH_PORT_NULL) {
|
|
mach_port_deallocate(mach_task_self(), object_name);
|
|
}
|
|
return (info.protection & VM_PROT_WRITE) != 0
|
|
? PAGE_READWRITE
|
|
: (info.protection & VM_PROT_READ) != 0 ? PAGE_READONLY
|
|
: PAGE_NOACCESS;
|
|
#else
|
|
const auto addr = reinterpret_cast<uintptr_t>(address);
|
|
std::FILE *maps = std::fopen("/proc/self/maps", "r");
|
|
Check(maps != nullptr, "open /proc/self/maps failed");
|
|
char line[512];
|
|
uint32_t result = 0;
|
|
while (std::fgets(line, sizeof(line), maps) != nullptr) {
|
|
unsigned long start = 0;
|
|
unsigned long end = 0;
|
|
char perms[8]{};
|
|
if (std::sscanf(line, "%lx-%lx %7s", &start, &end, perms) != 3) {
|
|
continue;
|
|
}
|
|
if (addr >= start && addr < end) {
|
|
result = perms[1] == 'w' ? PAGE_READWRITE
|
|
: perms[0] == 'r' ? PAGE_READONLY
|
|
: PAGE_NOACCESS;
|
|
break;
|
|
}
|
|
}
|
|
std::fclose(maps);
|
|
return result;
|
|
#endif
|
|
}
|
|
|
|
std::map<void *, size_t> &AllocationSizes() {
|
|
static std::map<void *, size_t> sizes;
|
|
return sizes;
|
|
}
|
|
|
|
void *VirtualAlloc(void *address, size_t size, DWORD, uint32_t protection) {
|
|
#if defined(__APPLE__)
|
|
mach_vm_address_t raw_address = reinterpret_cast<mach_vm_address_t>(address);
|
|
const auto flags = address != nullptr ? VM_FLAGS_FIXED : VM_FLAGS_ANYWHERE;
|
|
if (mach_vm_allocate(mach_task_self(), &raw_address, size, flags) !=
|
|
KERN_SUCCESS) {
|
|
return nullptr;
|
|
}
|
|
if (mach_vm_protect(mach_task_self(), raw_address, size, false,
|
|
static_cast<vm_prot_t>(ToHostProt(protection))) !=
|
|
KERN_SUCCESS) {
|
|
mach_vm_deallocate(mach_task_self(), raw_address, size);
|
|
return nullptr;
|
|
}
|
|
void *raw = reinterpret_cast<void *>(raw_address);
|
|
AllocationSizes()[raw] = size;
|
|
return raw;
|
|
#else
|
|
const int extra = address != nullptr ? MAP_FIXED_NOREPLACE : 0;
|
|
void *raw = ::mmap(address, size, ToHostProt(protection),
|
|
MAP_PRIVATE | MAP_ANONYMOUS | extra, -1, 0);
|
|
if (raw == MAP_FAILED) {
|
|
return nullptr;
|
|
}
|
|
AllocationSizes()[raw] = size;
|
|
return raw;
|
|
#endif
|
|
}
|
|
|
|
int VirtualFree(void *address, size_t, DWORD) {
|
|
auto &sizes = AllocationSizes();
|
|
auto it = sizes.find(address);
|
|
if (it == sizes.end()) {
|
|
return 0;
|
|
}
|
|
#if defined(__APPLE__)
|
|
const int ok = mach_vm_deallocate(mach_task_self(),
|
|
reinterpret_cast<mach_vm_address_t>(address),
|
|
it->second) == KERN_SUCCESS
|
|
? 1
|
|
: 0;
|
|
#else
|
|
const int ok = ::munmap(address, it->second) == 0 ? 1 : 0;
|
|
#endif
|
|
sizes.erase(it);
|
|
return ok;
|
|
}
|
|
|
|
int VirtualProtect(void *address, size_t size, uint32_t protection,
|
|
DWORD *old_protection) {
|
|
if (old_protection != nullptr) {
|
|
*old_protection = Protection(address);
|
|
}
|
|
#if defined(__APPLE__)
|
|
return mach_vm_protect(mach_task_self(),
|
|
reinterpret_cast<mach_vm_address_t>(address), size,
|
|
false, static_cast<vm_prot_t>(ToHostProt(protection))) ==
|
|
KERN_SUCCESS
|
|
? 1
|
|
: 0;
|
|
#else
|
|
return ::mprotect(address, size, ToHostProt(protection)) == 0 ? 1 : 0;
|
|
#endif
|
|
}
|
|
#else
|
|
uint32_t Protection(const void *address) {
|
|
MEMORY_BASIC_INFORMATION info{};
|
|
Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed");
|
|
return info.Protect;
|
|
}
|
|
#endif
|
|
|
|
bool IsWritable(const void *address) {
|
|
return Protection(address) == PAGE_READWRITE;
|
|
}
|
|
|
|
uint64_t g_protection_calls = 0;
|
|
|
|
struct ProtectionCall {
|
|
uint64_t address;
|
|
uint64_t size;
|
|
Common::VirtualMemory::Mode mode;
|
|
};
|
|
|
|
std::vector<ProtectionCall> g_protection_log;
|
|
std::mutex g_protection_log_mutex;
|
|
|
|
void ResetProtectionLog() {
|
|
g_protection_calls = 0;
|
|
g_protection_log.clear();
|
|
}
|
|
|
|
bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
|
|
Common::VirtualMemory::Mode mode) {
|
|
uint32_t protection = PAGE_NOACCESS;
|
|
if (mode == Common::VirtualMemory::Mode::Read) {
|
|
protection = PAGE_READONLY;
|
|
} else if (mode == Common::VirtualMemory::Mode::ReadWrite) {
|
|
protection = PAGE_READWRITE;
|
|
}
|
|
DWORD old_protection = 0;
|
|
{
|
|
std::lock_guard lock(g_protection_log_mutex);
|
|
g_protection_calls++;
|
|
g_protection_log.push_back({vaddr, size, mode});
|
|
}
|
|
return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
|
|
&old_protection) != 0;
|
|
}
|
|
|
|
struct TrackerHarness {
|
|
TrackerHarness() : tracker(page_manager) {}
|
|
|
|
PageManager page_manager;
|
|
MemoryTracker tracker;
|
|
};
|
|
|
|
uint8_t *AllocateFixedGuestRange(uint64_t size, uintptr_t offset) {
|
|
// Keep the mapping inside the tracker's guest address space and preserve the
|
|
// requested region alignment. A fixed address can be occupied by the host
|
|
// process (notably by the macOS runner's ASLR layout).
|
|
constexpr uintptr_t first_base = 0x0000000200000000ull;
|
|
constexpr uintptr_t stride = 0x0000000100000000ull;
|
|
for (uintptr_t attempt = 0; attempt < 256; attempt++) {
|
|
auto *wanted = reinterpret_cast<void *>(first_base + offset + attempt * stride);
|
|
auto *memory = static_cast<uint8_t *>(
|
|
VirtualAlloc(wanted, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
|
|
if (memory == wanted) {
|
|
return memory;
|
|
}
|
|
if (memory != nullptr) {
|
|
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0,
|
|
"release unexpected fixed allocation failed");
|
|
}
|
|
}
|
|
Check(false, "no free fixed guest address found");
|
|
return nullptr;
|
|
}
|
|
|
|
uint8_t *Allocate(PageManager &manager, uint64_t pages) {
|
|
const auto size = manager.GetPageSize() * pages;
|
|
return AllocateFixedGuestRange(size, 0x10000);
|
|
}
|
|
|
|
void Release(uint8_t *memory) {
|
|
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
|
|
}
|
|
|
|
void TestRangeSet() {
|
|
RangeSet ranges;
|
|
ranges.Add(0x1000, 0x80);
|
|
ranges.Add(0x1080, 0x80);
|
|
ranges.Add(0x1200, 0x40);
|
|
Check(ranges.Contains(0x1010, 0xe0) && !ranges.Contains(0x1010, 0x200),
|
|
"range set containment did not require full coverage");
|
|
Check(ranges.Intersects(0x0fff, 2) && ranges.Intersects(0x11ff, 2) &&
|
|
!ranges.Intersects(0x1100, 0x100) &&
|
|
!ranges.Intersects(0x1240, 1),
|
|
"range set intersection did not preserve half-open boundaries");
|
|
std::vector<std::pair<uint64_t, uint64_t>> intersections;
|
|
ranges.ForEachInRange(0x1070, 0x1b0, [&](uint64_t start, uint64_t end) {
|
|
intersections.emplace_back(start, end);
|
|
});
|
|
Check(intersections.size() == 2 && intersections[0].first == 0x1070 &&
|
|
intersections[0].second == 0x1100 &&
|
|
intersections[1].first == 0x1200 && intersections[1].second == 0x1220,
|
|
"range set did not merge and intersect exact byte ranges");
|
|
ranges.Subtract(0x1040, 0x1e0);
|
|
intersections.clear();
|
|
ranges.ForEachInRange(0x1000, 0x300, [&](uint64_t start, uint64_t end) {
|
|
intersections.emplace_back(start, end);
|
|
});
|
|
Check(intersections.size() == 2 && intersections[0].first == 0x1000 &&
|
|
intersections[0].second == 0x1040 &&
|
|
intersections[1].first == 0x1220 && intersections[1].second == 0x1240,
|
|
"range set subtraction did not preserve both exact tails");
|
|
}
|
|
|
|
void TestGuestRange() {
|
|
constexpr GuestRange empty{};
|
|
constexpr GuestRange first_byte{1, 1};
|
|
constexpr uint64_t extended_end = Libs::LibKernel::Memory::kExtendedMemoryBase +
|
|
Libs::LibKernel::Memory::kExtendedMemorySize;
|
|
constexpr GuestRange last_byte{extended_end - 1, 1};
|
|
|
|
static_assert(empty.Empty() && !empty.Valid() && empty.ValidOrEmpty());
|
|
static_assert(!first_byte.Empty() && first_byte.Valid() &&
|
|
first_byte.ValidOrEmpty() && first_byte.End() == 2);
|
|
static_assert(last_byte.Valid() && last_byte.End() == extended_end);
|
|
|
|
Check(!GuestRange{0, 1}.Empty() && !GuestRange{0, 1}.ValidOrEmpty(),
|
|
"zero-address nonempty guest range is rejected");
|
|
Check(!GuestRange{1, 0}.Empty() && !GuestRange{1, 0}.ValidOrEmpty(),
|
|
"nonzero-address empty guest range is rejected");
|
|
Check(GuestRange{Libs::LibKernel::Memory::kExtendedMemoryBase, 1}.Valid() &&
|
|
!GuestRange{Libs::Graphics::LOWER_ADDRESS_SIZE, 1}.Valid() &&
|
|
!GuestRange{Libs::LibKernel::Memory::kExtendedMemoryBase - 1, 2}.Valid() &&
|
|
!GuestRange{extended_end - 1, 2}.Valid(),
|
|
"extended range and gap boundaries are enforced");
|
|
Check(!GuestRange{TRACKER_ADDRESS_SIZE, 1}.Valid(),
|
|
"first address beyond the guest range is rejected");
|
|
Check(!GuestRange{TRACKER_ADDRESS_SIZE - 1, 2}.Valid(),
|
|
"guest range crossing the address-space end is rejected");
|
|
Check(!GuestRange{UINT64_MAX, 2}.Valid(), "wrapping guest range is rejected");
|
|
}
|
|
|
|
void TestQueriesDoNotRequireMappedOwnership() {
|
|
constexpr uint64_t address = 0x0000000203000000ull;
|
|
TrackerHarness harness;
|
|
const auto page_size = harness.page_manager.GetPageSize();
|
|
Check(harness.tracker.IsRegionCpuModified(address, page_size) &&
|
|
!harness.tracker.IsRegionGpuModified(address, page_size),
|
|
"unowned tracker range did not expose its initial CPU-dirty state");
|
|
}
|
|
|
|
void TestConcurrentRegionPublication() {
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto &page_manager = harness.page_manager;
|
|
const auto page_size = page_manager.GetPageSize();
|
|
auto *memory = Allocate(page_manager, 1);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
|
|
std::binary_semaphore start_first{0};
|
|
std::binary_semaphore start_second{0};
|
|
std::atomic_uint32_t cpu_dirty_results{0};
|
|
std::jthread first([&] {
|
|
start_first.acquire();
|
|
if (tracker.IsRegionCpuModified(address, page_size)) {
|
|
cpu_dirty_results.fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
});
|
|
std::jthread second([&] {
|
|
start_second.acquire();
|
|
if (tracker.IsRegionCpuModified(address, page_size)) {
|
|
cpu_dirty_results.fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
});
|
|
start_first.release();
|
|
start_second.release();
|
|
first.join();
|
|
second.join();
|
|
|
|
tracker.UntrackMemory(address, page_size);
|
|
Release(memory);
|
|
Check(cpu_dirty_results.load(std::memory_order_relaxed) == 2,
|
|
"concurrent region publication lost initial CPU ownership");
|
|
}
|
|
|
|
void TestCpuDirtyUpload() {
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto &page_manager = harness.page_manager;
|
|
const auto page_size = page_manager.GetPageSize();
|
|
auto *memory = Allocate(page_manager, 2);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
Check(tracker.IsRegionCpuModified(address + 16, 32),
|
|
"new region was not CPU dirty");
|
|
|
|
uint32_t ranges = 0;
|
|
bool uploaded = false;
|
|
tracker.ForEachUploadRange(
|
|
address + 16, 32, false,
|
|
[&](uint64_t upload_address, uint64_t upload_size) noexcept {
|
|
Check(upload_address == address && upload_size == page_size,
|
|
"upload range was not page aligned");
|
|
ranges++;
|
|
},
|
|
[&]() noexcept { uploaded = true; });
|
|
Check(ranges == 1 && uploaded &&
|
|
!tracker.IsRegionCpuModified(address, page_size) &&
|
|
Protection(memory) == PAGE_READONLY,
|
|
"upload did not clear CPU dirty state and arm protection");
|
|
|
|
tracker.MarkRegionAsCpuModified(address + 16, 32);
|
|
Check(tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
|
|
"explicit CPU dirtiness did not release write protection");
|
|
tracker.UntrackMemory(address, page_size * 2);
|
|
Release(memory);
|
|
}
|
|
|
|
void TestCleanUploadPreservesOwnership() {
|
|
constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
|
|
constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto *memory = AllocateFixedGuestRange(region_size * 2, region_size);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
tracker.ForEachUploadRange(address, region_size * 2, false,
|
|
[](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
|
|
for (const auto start : {address + 63 * page_size,
|
|
address + region_size - page_size}) {
|
|
const auto before = start - page_size;
|
|
const auto after = start + 2 * page_size;
|
|
tracker.MarkRegionAsCpuModified(before, page_size);
|
|
tracker.MarkRegionAsCpuModified(after, page_size);
|
|
uint32_t ranges = 0;
|
|
uint32_t completions = 0;
|
|
ResetProtectionLog();
|
|
tracker.ForEachUploadRange(
|
|
start, 2 * page_size, false,
|
|
[&](uint64_t, uint64_t) noexcept { ranges++; },
|
|
[&]() noexcept { completions++; });
|
|
Check(ranges == 0 && completions == 1 && g_protection_calls == 0 &&
|
|
!tracker.IsRegionCpuModified(start, 2 * page_size),
|
|
"clean read-only upload changed dirty state or protection");
|
|
|
|
tracker.ForEachUploadRange(
|
|
start, 2 * page_size, true,
|
|
[&](uint64_t, uint64_t) noexcept { ranges++; },
|
|
[&]() noexcept { completions++; });
|
|
Check(ranges == 0 && completions == 2 &&
|
|
tracker.IsRegionGpuModified(start, page_size) &&
|
|
tracker.IsRegionGpuModified(start + page_size, page_size) &&
|
|
Protection(reinterpret_cast<void *>(start)) == PAGE_NOACCESS &&
|
|
Protection(reinterpret_cast<void *>(start + page_size)) ==
|
|
PAGE_NOACCESS &&
|
|
tracker.IsRegionCpuModified(before, page_size) &&
|
|
tracker.IsRegionCpuModified(after, page_size) &&
|
|
!tracker.IsRegionGpuModified(before, page_size) &&
|
|
!tracker.IsRegionGpuModified(after, page_size) &&
|
|
IsWritable(reinterpret_cast<void *>(before)) &&
|
|
IsWritable(reinterpret_cast<void *>(after)),
|
|
"clean written upload lost GPU ownership or changed dirty neighbors");
|
|
tracker.UnmarkRegionAsGpuModified(start, 2 * page_size);
|
|
}
|
|
tracker.UntrackMemory(address, region_size * 2);
|
|
Release(memory);
|
|
}
|
|
|
|
void BenchmarkCleanUploads() {
|
|
constexpr uint64_t size = 256ull * 1024 * 1024;
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto *memory = AllocateFixedGuestRange(size, 0);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
tracker.ForEachUploadRange(address, size, false,
|
|
[](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
std::puts("chunk_bytes,sweeps,calls,elapsed_ns,ns_per_sweep,ns_per_call");
|
|
for (const uint64_t chunk : {size, Libs::Graphics::TRACKER_REGION_SIZE,
|
|
uint64_t{64 * 1024}}) {
|
|
uint64_t ranges = 0;
|
|
uint64_t completions = 0;
|
|
uint64_t sweeps = 0;
|
|
ResetProtectionLog();
|
|
const auto start = std::chrono::steady_clock::now();
|
|
std::chrono::nanoseconds elapsed{};
|
|
do {
|
|
for (uint64_t offset = 0; offset < size; offset += chunk) {
|
|
tracker.ForEachUploadRange(
|
|
address + offset, chunk, false,
|
|
[&](uint64_t, uint64_t) noexcept { ranges++; },
|
|
[&]() noexcept { completions++; });
|
|
}
|
|
sweeps++;
|
|
elapsed = std::chrono::steady_clock::now() - start;
|
|
} while (elapsed < std::chrono::milliseconds(250));
|
|
Check(ranges == 0 && completions == sweeps * (size / chunk) &&
|
|
g_protection_calls == 0,
|
|
"clean upload benchmark changed ranges, completion, or protection");
|
|
std::printf("%llu,%llu,%llu,%lld,%.2f,%.2f\n",
|
|
static_cast<unsigned long long>(chunk),
|
|
static_cast<unsigned long long>(sweeps),
|
|
static_cast<unsigned long long>(completions),
|
|
static_cast<long long>(elapsed.count()),
|
|
static_cast<double>(elapsed.count()) / sweeps,
|
|
static_cast<double>(elapsed.count()) / completions);
|
|
}
|
|
tracker.UntrackMemory(address, size);
|
|
Release(memory);
|
|
}
|
|
|
|
void TestRangeInvalidation() {
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
constexpr uint64_t size = Libs::Graphics::TRACKER_REGION_SIZE * 2;
|
|
auto *memory = AllocateFixedGuestRange(size, 0x1000000);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
|
|
tracker.ForEachUploadRange(
|
|
address, size, true, [](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
Check(tracker.IsRegionGpuModified(address, size) && !IsWritable(memory),
|
|
"range invalidation setup did not establish GPU ownership");
|
|
uint32_t flushes = 0;
|
|
tracker.InvalidateRegion(address + 16, size - 32, [&] {
|
|
flushes++;
|
|
tracker.ForEachDownloadRange<true>(address + 16, size - 32,
|
|
[](uint64_t, uint64_t) noexcept {});
|
|
tracker.MarkRegionAsCpuModified(address + 16, size - 32);
|
|
});
|
|
Check(flushes == 1 && !tracker.IsRegionGpuModified(address, size) &&
|
|
tracker.IsRegionCpuModified(address, size) && IsWritable(memory) &&
|
|
IsWritable(memory + size - 1),
|
|
"range invalidation did not batch ownership transfer across regions");
|
|
tracker.InvalidateRegion(address + 16, size - 32, [&] { flushes++; });
|
|
Check(flushes == 1,
|
|
"clean range invalidation unnecessarily requested a GPU flush");
|
|
tracker.UntrackMemory(address, size);
|
|
Release(memory);
|
|
}
|
|
|
|
void TestGpuReacquisitionAfterInvalidation() {
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto &page_manager = harness.page_manager;
|
|
const auto page_size = page_manager.GetPageSize();
|
|
auto *memory = Allocate(page_manager, 1);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
|
|
tracker.ForEachUploadRange(
|
|
address, page_size, true, [](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
Check(tracker.IsRegionGpuModified(address, page_size) &&
|
|
!tracker.IsRegionCpuModified(address, page_size),
|
|
"reacquisition setup did not establish GPU ownership");
|
|
|
|
uint32_t flushes = 0;
|
|
uint32_t uploads = 0;
|
|
std::binary_semaphore reacquire{0};
|
|
std::binary_semaphore reacquired{0};
|
|
std::jthread publisher([&] {
|
|
reacquire.acquire();
|
|
tracker.ForEachUploadRange(
|
|
address + 16, 32, true,
|
|
[&](uint64_t, uint64_t) noexcept { uploads++; }, []() noexcept {});
|
|
reacquired.release();
|
|
});
|
|
tracker.InvalidateRegion(address + 16, 32, [&] {
|
|
flushes++;
|
|
tracker.ForEachDownloadRange<true>(address + 16, 32,
|
|
[](uint64_t, uint64_t) noexcept {});
|
|
tracker.MarkRegionAsCpuModified(address + 16, 32);
|
|
reacquire.release();
|
|
reacquired.acquire();
|
|
});
|
|
publisher.join();
|
|
Check(flushes == 1 && uploads == 1 &&
|
|
tracker.IsRegionGpuModified(address, page_size) &&
|
|
!tracker.IsRegionCpuModified(address, page_size) &&
|
|
!IsWritable(memory),
|
|
"invalidation rejected a new generation of GPU ownership");
|
|
|
|
tracker.UnmarkRegionAsGpuModified(address, page_size);
|
|
tracker.MarkRegionAsCpuModified(address, page_size);
|
|
tracker.UntrackMemory(address, page_size);
|
|
Release(memory);
|
|
}
|
|
|
|
void TestGpuDirtyBits() {
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto &page_manager = harness.page_manager;
|
|
const auto page_size = page_manager.GetPageSize();
|
|
auto *memory = Allocate(page_manager, 2);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
|
|
tracker.ForEachUploadRange(
|
|
address, page_size, true, [](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
Check(tracker.IsRegionGpuModified(address, page_size) &&
|
|
!tracker.IsRegionGpuModified(address + page_size, page_size) &&
|
|
Protection(memory) == PAGE_NOACCESS,
|
|
"GPU dirty state escaped the requested range");
|
|
tracker.UnmarkRegionAsGpuModified(address, page_size);
|
|
Check(!tracker.IsRegionGpuModified(address, page_size) &&
|
|
Protection(memory) == PAGE_READONLY,
|
|
"GPU dirty state did not restore write-only tracking");
|
|
tracker.MarkRegionAsCpuModified(address, page_size);
|
|
tracker.UntrackMemory(address, page_size * 2);
|
|
Release(memory);
|
|
}
|
|
|
|
void TestExactDirtyIntervalsSharingTrackerPage() {
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto &page_manager = harness.page_manager;
|
|
const auto page_size = page_manager.GetPageSize();
|
|
auto *memory = Allocate(page_manager, 1);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
|
|
tracker.ForEachUploadRange(
|
|
address, page_size, false, [](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
RangeSet exact_dirty;
|
|
exact_dirty.Add(address + 64, 16);
|
|
exact_dirty.Add(address + 192, 32);
|
|
|
|
ResetProtectionLog();
|
|
tracker.MarkRegionAsGpuModified(address + 64, 16);
|
|
tracker.MarkRegionAsGpuModified(address + 192, 32);
|
|
Check(g_protection_calls == 1 &&
|
|
tracker.IsRegionGpuModified(address, page_size) &&
|
|
Protection(memory) == PAGE_NOACCESS,
|
|
"disjoint byte dirtiness duplicated the page watcher");
|
|
|
|
exact_dirty.Subtract(address + 64, 16);
|
|
if (!exact_dirty.Intersects(address, page_size)) {
|
|
tracker.UnmarkRegionAsGpuModified(address, page_size);
|
|
}
|
|
Check(g_protection_calls == 1 &&
|
|
tracker.IsRegionGpuModified(address, page_size) &&
|
|
Protection(memory) == PAGE_NOACCESS,
|
|
"draining one exact interval prematurely released its shared page");
|
|
|
|
exact_dirty.Subtract(address + 192, 32);
|
|
if (!exact_dirty.Intersects(address, page_size)) {
|
|
tracker.UnmarkRegionAsGpuModified(address, page_size);
|
|
}
|
|
Check(g_protection_calls == 2 &&
|
|
!tracker.IsRegionGpuModified(address, page_size) &&
|
|
Protection(memory) == PAGE_READONLY,
|
|
"draining the final exact interval did not release its tracker page");
|
|
|
|
tracker.UntrackMemory(address, page_size);
|
|
Release(memory);
|
|
}
|
|
|
|
void TestGpuDownloadProtectionMirrors() {
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto &page_manager = harness.page_manager;
|
|
const auto page_size = page_manager.GetPageSize();
|
|
auto *memory = Allocate(page_manager, 4);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
|
|
tracker.ForEachUploadRange(
|
|
address, page_size * 4, false, [](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
tracker.MarkRegionAsGpuModified(address + 16, 32);
|
|
tracker.MarkRegionAsGpuModified(address + page_size * 2 + 16, 32);
|
|
|
|
std::vector<std::pair<uint64_t, uint64_t>> visited;
|
|
ResetProtectionLog();
|
|
tracker.ForEachDownloadRange<false>(
|
|
address, page_size * 3,
|
|
[&](uint64_t range_address, uint64_t range_size) noexcept {
|
|
visited.push_back({range_address, range_size});
|
|
});
|
|
Check(visited.size() == 2 && visited[0].first == address &&
|
|
visited[0].second == page_size &&
|
|
visited[1].first == address + page_size * 2 &&
|
|
visited[1].second == page_size && g_protection_calls == 0 &&
|
|
tracker.IsRegionGpuModified(address, page_size * 3),
|
|
"non-clearing download changed protection or lost sparse ranges");
|
|
|
|
visited.clear();
|
|
bool protected_during_download = false;
|
|
tracker.ForEachDownloadRange<true>(
|
|
address + 16, 32,
|
|
[&](uint64_t range_address, uint64_t range_size) noexcept {
|
|
protected_during_download = Protection(memory) == PAGE_NOACCESS;
|
|
visited.push_back({range_address, range_size});
|
|
});
|
|
Check(protected_during_download && visited.size() == 1 &&
|
|
visited[0].first == address &&
|
|
visited[0].second == page_size && g_protection_log.size() == 1 &&
|
|
g_protection_log[0].address == address &&
|
|
g_protection_log[0].size == page_size &&
|
|
g_protection_log[0].mode == Common::VirtualMemory::Mode::Read &&
|
|
!tracker.IsRegionGpuModified(address, page_size) &&
|
|
tracker.IsRegionGpuModified(address + page_size * 2, page_size) &&
|
|
Protection(memory) == PAGE_READONLY &&
|
|
Protection(memory + page_size * 2) == PAGE_NOACCESS,
|
|
"partial download did not preserve the CPU/GPU protection mirrors");
|
|
|
|
visited.clear();
|
|
ResetProtectionLog();
|
|
tracker.ForEachDownloadRange<true>(
|
|
address + 16, 32,
|
|
[&](uint64_t range_address, uint64_t range_size) noexcept {
|
|
visited.push_back({range_address, range_size});
|
|
});
|
|
Check(visited.empty() && g_protection_calls == 0 &&
|
|
tracker.IsRegionGpuModified(address + page_size * 2, page_size),
|
|
"idempotent partial download disturbed another GPU-owned page");
|
|
|
|
tracker.UnmarkRegionAsGpuModified(address, page_size * 3);
|
|
Check(!tracker.IsRegionGpuModified(address, page_size * 3) &&
|
|
Protection(memory + page_size * 2) == PAGE_READONLY,
|
|
"broad final unmark did not restore write-only tracking");
|
|
ResetProtectionLog();
|
|
tracker.MarkRegionAsCpuModified(address + 16, 32);
|
|
Check(
|
|
g_protection_log.size() == 1 && g_protection_log[0].address == address &&
|
|
g_protection_log[0].size == page_size &&
|
|
g_protection_log[0].mode == Common::VirtualMemory::Mode::ReadWrite &&
|
|
IsWritable(memory) && !IsWritable(memory + page_size),
|
|
"CPU-dirty transition did not release only its write watcher");
|
|
|
|
tracker.UntrackMemory(address, page_size * 4);
|
|
Release(memory);
|
|
}
|
|
|
|
void TestCrossRegionUpload() {
|
|
constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
|
|
constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
|
|
auto *memory = AllocateFixedGuestRange(region_size * 2, 0x10000);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
const auto boundary = (address + region_size - 1) & ~(region_size - 1);
|
|
for (const bool is_written : {false, true}) {
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
uint32_t ranges = 0;
|
|
tracker.ForEachUploadRange(
|
|
boundary - page_size, page_size * 2, is_written,
|
|
[&](uint64_t start, uint64_t size) noexcept {
|
|
Check(start == boundary - page_size + ranges * page_size &&
|
|
size == page_size,
|
|
"cold upload did not visit both region boundaries exactly");
|
|
ranges++;
|
|
},
|
|
[&]() noexcept {
|
|
Check(ranges == 2 &&
|
|
Protection(reinterpret_cast<void *>(boundary - page_size)) ==
|
|
PAGE_READONLY &&
|
|
Protection(reinterpret_cast<void *>(boundary)) == PAGE_READONLY,
|
|
"cold upload completed before clearing and protecting both regions");
|
|
});
|
|
for (const auto page : {boundary - page_size, boundary}) {
|
|
Check(!tracker.IsRegionCpuModified(page, page_size) &&
|
|
tracker.IsRegionGpuModified(page, page_size) == is_written &&
|
|
Protection(reinterpret_cast<void *>(page)) ==
|
|
(is_written ? PAGE_NOACCESS : PAGE_READONLY),
|
|
"cross-region upload did not preserve final ownership and protection");
|
|
}
|
|
Check(IsWritable(reinterpret_cast<void *>(boundary - 2 * page_size)) &&
|
|
IsWritable(reinterpret_cast<void *>(boundary + page_size)),
|
|
"cross-region upload changed neighboring pages");
|
|
tracker.UnmarkRegionAsGpuModified(boundary - page_size, 2 * page_size);
|
|
tracker.UntrackMemory(address, region_size * 2);
|
|
}
|
|
Release(memory);
|
|
}
|
|
|
|
void TestConcurrentColdUploads() {
|
|
constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
|
|
constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto *memory = AllocateFixedGuestRange(region_size * 2, region_size);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
std::counting_semaphore<2> start{0};
|
|
std::counting_semaphore<2> upload_entered{0};
|
|
std::counting_semaphore<2> finish_upload{0};
|
|
std::vector<std::jthread> workers;
|
|
for (const auto page : {address, address + region_size}) {
|
|
workers.emplace_back([&, page] {
|
|
start.acquire();
|
|
uint32_t ranges = 0;
|
|
tracker.ForEachUploadRange(
|
|
page, page_size, true,
|
|
[&](uint64_t upload_address, uint64_t upload_size) noexcept {
|
|
Check(upload_address == page && upload_size == page_size,
|
|
"concurrent cold upload lost its dirty page");
|
|
ranges++;
|
|
},
|
|
[&]() noexcept {
|
|
Check(ranges == 1, "concurrent cold upload skipped its dirty page");
|
|
upload_entered.release();
|
|
finish_upload.acquire();
|
|
});
|
|
});
|
|
}
|
|
start.release(2);
|
|
const bool first_entered = upload_entered.try_acquire_for(std::chrono::seconds(5));
|
|
const bool second_entered = upload_entered.try_acquire_for(std::chrono::seconds(5));
|
|
finish_upload.release(2);
|
|
for (auto &worker : workers) {
|
|
worker.join();
|
|
}
|
|
Check(first_entered && second_entered,
|
|
"cold writable uploads serialized disjoint regions");
|
|
for (const auto page : {address, address + region_size}) {
|
|
Check(!tracker.IsRegionCpuModified(page, page_size) &&
|
|
tracker.IsRegionGpuModified(page, page_size) &&
|
|
Protection(reinterpret_cast<void *>(page)) == PAGE_NOACCESS,
|
|
"concurrent cold upload did not retain GPU ownership");
|
|
tracker.UnmarkRegionAsGpuModified(page, page_size);
|
|
}
|
|
tracker.UntrackMemory(address, region_size * 2);
|
|
Release(memory);
|
|
}
|
|
|
|
void TestUploadDoesNotSerializeDisjointRegion() {
|
|
constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
|
|
constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto &page_manager = harness.page_manager;
|
|
auto *memory = Allocate(page_manager, region_size * 2 / page_size);
|
|
const auto allocation_base = reinterpret_cast<uint64_t>(memory);
|
|
const auto second_region =
|
|
(allocation_base & ~(region_size - 1)) + region_size;
|
|
Check(second_region + page_size <= allocation_base + region_size * 2,
|
|
"test allocation does not span two tracker regions");
|
|
|
|
// Publish both managers before the concurrent section so this test measures
|
|
// tracker access serialization rather than manager allocation.
|
|
Check(tracker.IsRegionCpuModified(allocation_base, page_size) &&
|
|
tracker.IsRegionCpuModified(second_region, page_size),
|
|
"disjoint upload setup did not initialize both regions");
|
|
|
|
std::binary_semaphore upload_entered{0};
|
|
std::binary_semaphore finish_upload{0};
|
|
std::binary_semaphore query_finished{0};
|
|
std::atomic_bool query_result{false};
|
|
std::jthread uploader([&] {
|
|
tracker.ForEachUploadRange(
|
|
allocation_base, page_size, true, [](uint64_t, uint64_t) noexcept {},
|
|
[&]() noexcept {
|
|
upload_entered.release();
|
|
finish_upload.acquire();
|
|
});
|
|
});
|
|
upload_entered.acquire();
|
|
std::jthread query([&] {
|
|
query_result.store(tracker.IsRegionCpuModified(second_region, page_size),
|
|
std::memory_order_relaxed);
|
|
query_finished.release();
|
|
});
|
|
|
|
const bool completed_while_upload_blocked =
|
|
query_finished.try_acquire_for(std::chrono::seconds(5));
|
|
finish_upload.release();
|
|
uploader.join();
|
|
query.join();
|
|
|
|
tracker.UnmarkRegionAsGpuModified(allocation_base, page_size);
|
|
tracker.MarkRegionAsCpuModified(allocation_base, page_size);
|
|
tracker.UntrackMemory(allocation_base, region_size * 2);
|
|
Release(memory);
|
|
Check(completed_while_upload_blocked &&
|
|
query_result.load(std::memory_order_relaxed),
|
|
"upload callback serialized an unrelated tracker region");
|
|
}
|
|
|
|
void TestDownloadDoesNotSerializeDisjointRegion() {
|
|
constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
|
|
constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto &page_manager = harness.page_manager;
|
|
auto *memory = Allocate(page_manager, region_size * 2 / page_size);
|
|
const auto allocation_base = reinterpret_cast<uint64_t>(memory);
|
|
const auto second_region =
|
|
(allocation_base & ~(region_size - 1)) + region_size;
|
|
Check(second_region + page_size <= allocation_base + region_size * 2,
|
|
"test allocation does not span two tracker regions");
|
|
|
|
tracker.ForEachUploadRange(
|
|
allocation_base, page_size, true, [](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
tracker.ForEachUploadRange(
|
|
second_region, page_size, false, [](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
|
|
std::binary_semaphore download_entered{0};
|
|
std::binary_semaphore finish_download{0};
|
|
std::binary_semaphore mutation_finished{0};
|
|
std::jthread downloader([&] {
|
|
tracker.ForEachDownloadRange<false>(
|
|
allocation_base, second_region + page_size - allocation_base,
|
|
[&](uint64_t address, uint64_t) noexcept {
|
|
if (address == allocation_base) {
|
|
download_entered.release();
|
|
finish_download.acquire();
|
|
}
|
|
});
|
|
});
|
|
download_entered.acquire();
|
|
std::jthread mutation([&] {
|
|
tracker.MarkRegionAsGpuModified(second_region, page_size);
|
|
mutation_finished.release();
|
|
});
|
|
|
|
const bool completed_while_download_blocked =
|
|
mutation_finished.try_acquire_for(std::chrono::seconds(5));
|
|
finish_download.release();
|
|
downloader.join();
|
|
mutation.join();
|
|
|
|
const bool both_gpu_owned =
|
|
tracker.IsRegionGpuModified(allocation_base, page_size) &&
|
|
tracker.IsRegionGpuModified(second_region, page_size);
|
|
tracker.UnmarkRegionAsGpuModified(allocation_base, page_size);
|
|
tracker.UnmarkRegionAsGpuModified(second_region, page_size);
|
|
tracker.MarkRegionAsCpuModified(allocation_base, page_size);
|
|
tracker.MarkRegionAsCpuModified(second_region, page_size);
|
|
tracker.UntrackMemory(allocation_base, region_size * 2);
|
|
Release(memory);
|
|
Check(completed_while_download_blocked && both_gpu_owned,
|
|
"download callback serialized an unrelated tracker region");
|
|
}
|
|
|
|
void TestGpuUnmarkUsesRegionMask() {
|
|
constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
|
|
constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto &page_manager = harness.page_manager;
|
|
auto *memory = Allocate(page_manager, region_size * 2 / page_size);
|
|
const auto allocation_base = reinterpret_cast<uint64_t>(memory);
|
|
const auto region_base =
|
|
(allocation_base + region_size - 1) & ~(region_size - 1);
|
|
Check(region_base + region_size + page_size <=
|
|
allocation_base + region_size * 2,
|
|
"test allocation does not span two complete tracker regions");
|
|
|
|
const auto sparse_begin = region_base + page_size;
|
|
tracker.ForEachUploadRange(
|
|
sparse_begin, page_size * 3, false, [](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
tracker.MarkRegionAsGpuModified(sparse_begin, page_size);
|
|
tracker.MarkRegionAsGpuModified(sparse_begin + page_size * 2, page_size);
|
|
ResetProtectionLog();
|
|
tracker.UnmarkRegionAsGpuModified(sparse_begin, page_size * 3);
|
|
Check(
|
|
g_protection_calls == 1 && g_protection_log.size() == 1 &&
|
|
g_protection_log[0].address == sparse_begin &&
|
|
g_protection_log[0].size == page_size * 3 &&
|
|
g_protection_log[0].mode == Common::VirtualMemory::Mode::Read &&
|
|
!tracker.IsRegionGpuModified(sparse_begin, page_size * 3) &&
|
|
Protection(reinterpret_cast<void *>(sparse_begin)) == PAGE_READONLY &&
|
|
Protection(reinterpret_cast<void *>(sparse_begin + page_size)) ==
|
|
PAGE_READONLY &&
|
|
Protection(reinterpret_cast<void *>(sparse_begin + page_size * 2)) ==
|
|
PAGE_READONLY,
|
|
"GPU unmark did not coalesce a sparse 4 MiB region mask");
|
|
ResetProtectionLog();
|
|
tracker.UnmarkRegionAsGpuModified(sparse_begin, page_size * 3);
|
|
Check(g_protection_calls == 0,
|
|
"idempotent GPU unmark performed a protection call");
|
|
|
|
const auto boundary = region_base + region_size;
|
|
const auto cross_begin = boundary - page_size;
|
|
tracker.ForEachUploadRange(
|
|
cross_begin, page_size * 2, false, [](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
tracker.MarkRegionAsGpuModified(cross_begin, page_size * 2);
|
|
ResetProtectionLog();
|
|
tracker.UnmarkRegionAsGpuModified(cross_begin, page_size * 2);
|
|
Check(g_protection_calls == 2 && g_protection_log.size() == 2 &&
|
|
g_protection_log[0].address == cross_begin &&
|
|
g_protection_log[0].size == page_size &&
|
|
g_protection_log[0].mode == Common::VirtualMemory::Mode::Read &&
|
|
g_protection_log[1].address == boundary &&
|
|
g_protection_log[1].size == page_size &&
|
|
g_protection_log[1].mode == Common::VirtualMemory::Mode::Read &&
|
|
!tracker.IsRegionGpuModified(cross_begin, page_size * 2),
|
|
"cross-region GPU unmark did not use one update per 4 MiB region");
|
|
|
|
tracker.UntrackMemory(allocation_base, region_size * 2);
|
|
Release(memory);
|
|
}
|
|
|
|
void TestFullRegionGpuUnmarkBatching() {
|
|
constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
|
|
constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto &page_manager = harness.page_manager;
|
|
auto *memory = Allocate(page_manager, region_size * 2 / page_size);
|
|
const auto allocation_base = reinterpret_cast<uint64_t>(memory);
|
|
const auto region_base =
|
|
(allocation_base + region_size - 1) & ~(region_size - 1);
|
|
Check(region_base + region_size <= allocation_base + region_size * 2,
|
|
"test allocation does not contain a complete tracker region");
|
|
|
|
tracker.ForEachUploadRange(
|
|
region_base, region_size, false, [](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
tracker.MarkRegionAsGpuModified(region_base, region_size);
|
|
Check(tracker.IsRegionGpuModified(region_base, region_size) &&
|
|
Protection(reinterpret_cast<void *>(region_base)) ==
|
|
PAGE_NOACCESS &&
|
|
Protection(reinterpret_cast<void *>(region_base + region_size -
|
|
page_size)) == PAGE_NOACCESS,
|
|
"full-region setup did not establish GPU read protection");
|
|
|
|
ResetProtectionLog();
|
|
tracker.UnmarkRegionAsGpuModified(region_base, region_size);
|
|
Check(
|
|
g_protection_log.size() == 1 &&
|
|
g_protection_log[0].address == region_base &&
|
|
g_protection_log[0].size == region_size &&
|
|
g_protection_log[0].mode == Common::VirtualMemory::Mode::Read &&
|
|
!tracker.IsRegionGpuModified(region_base, region_size) &&
|
|
Protection(reinterpret_cast<void *>(region_base)) == PAGE_READONLY &&
|
|
Protection(reinterpret_cast<void *>(region_base + region_size -
|
|
page_size)) == PAGE_READONLY,
|
|
"full-region GPU unmark did not use one exact 4 MiB protection request");
|
|
|
|
tracker.UntrackMemory(allocation_base, region_size * 2);
|
|
Release(memory);
|
|
}
|
|
|
|
[[noreturn]] void RunDeathCase(const char *name) {
|
|
TrackerHarness harness;
|
|
auto &tracker = harness.tracker;
|
|
auto &page_manager = harness.page_manager;
|
|
const auto page_size = page_manager.GetPageSize();
|
|
auto *memory = Allocate(page_manager, 1);
|
|
const auto address = reinterpret_cast<uint64_t>(memory);
|
|
if (std::strcmp(name, "gpu-dirty-explicit-cpu") == 0) {
|
|
tracker.ForEachUploadRange(
|
|
address, page_size, true, [](uint64_t, uint64_t) noexcept {},
|
|
[]() noexcept {});
|
|
tracker.MarkRegionAsCpuModified(address, page_size);
|
|
} else if (std::strcmp(name, "reentrant-upload") == 0) {
|
|
tracker.ForEachUploadRange(
|
|
address, page_size, true, [](uint64_t, uint64_t) noexcept {},
|
|
[&]() noexcept {
|
|
(void)tracker.IsRegionCpuModified(address, page_size);
|
|
});
|
|
} else if (std::strcmp(name, "recursive-tracking-lock") == 0) {
|
|
Libs::Graphics::TrackingSpinLock lock;
|
|
lock.lock();
|
|
lock.lock();
|
|
} else if (std::strcmp(name, "non-owner-tracking-unlock") == 0) {
|
|
Libs::Graphics::TrackingSpinLock lock;
|
|
lock.lock();
|
|
std::thread worker([&] { lock.unlock(); });
|
|
worker.join();
|
|
}
|
|
std::_Exit(0x7f);
|
|
}
|
|
|
|
void CheckDeathCase(const char *name) {
|
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
|
char path[MAX_PATH]{};
|
|
Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
|
|
"GetModuleFileName failed");
|
|
std::string command = std::string("\"") + path + "\" --death " + name;
|
|
std::vector<char> mutable_command(command.begin(), command.end());
|
|
mutable_command.push_back('\0');
|
|
STARTUPINFOA startup{sizeof(startup)};
|
|
PROCESS_INFORMATION process{};
|
|
Check(CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE,
|
|
CREATE_NO_WINDOW, nullptr, nullptr, &startup,
|
|
&process) != 0,
|
|
"CreateProcess failed");
|
|
Check(WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0,
|
|
"MemoryTracker death test timed out");
|
|
DWORD exit_code = 0;
|
|
Check(
|
|
GetExitCodeProcess(process.hProcess, &exit_code) != 0 &&
|
|
(exit_code == 321 || exit_code == EXCEPTION_NONCONTINUABLE_EXCEPTION),
|
|
"MemoryTracker death path used the wrong exit");
|
|
CloseHandle(process.hThread);
|
|
CloseHandle(process.hProcess);
|
|
#else
|
|
#if defined(__APPLE__)
|
|
std::vector<char> path(PATH_MAX);
|
|
uint32_t path_size = static_cast<uint32_t>(path.size());
|
|
if (_NSGetExecutablePath(path.data(), &path_size) != 0) {
|
|
path.resize(path_size);
|
|
Check(_NSGetExecutablePath(path.data(), &path_size) == 0,
|
|
"_NSGetExecutablePath failed");
|
|
}
|
|
#endif
|
|
const pid_t pid = ::fork();
|
|
Check(pid >= 0, "fork failed");
|
|
if (pid == 0) {
|
|
#if defined(__APPLE__)
|
|
::execl(path.data(), "MemoryTrackerTests", "--death", name, nullptr);
|
|
#else
|
|
::execl("/proc/self/exe", "MemoryTrackerTests", "--death", name, nullptr);
|
|
#endif
|
|
std::_Exit(0x7e);
|
|
}
|
|
int status = 0;
|
|
Check(::waitpid(pid, &status, 0) == pid, "waitpid failed");
|
|
const bool fatal_exit =
|
|
WIFEXITED(status) && WEXITSTATUS(status) == (321 & 0xff);
|
|
Check(fatal_exit || WIFSIGNALED(status),
|
|
"MemoryTracker death path used the wrong exit");
|
|
#endif
|
|
}
|
|
|
|
void TestFatalPaths() {
|
|
for (const char *name : {"gpu-dirty-explicit-cpu", "reentrant-upload",
|
|
"recursive-tracking-lock", "non-owner-tracking-unlock"}) {
|
|
CheckDeathCase(name);
|
|
}
|
|
}
|
|
|
|
#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
|
|
void *g_fault_stack = nullptr;
|
|
constexpr size_t FAULT_STACK_SIZE = 64 * 1024;
|
|
volatile sig_atomic_t g_stack_faults = 0;
|
|
|
|
bool HandleStackFault(const Common::HostException::ExceptionInfo &info) {
|
|
using namespace Common::HostException;
|
|
stack_t active_stack{};
|
|
const auto fault_address = reinterpret_cast<uintptr_t>(g_fault_stack) +
|
|
FAULT_STACK_SIZE - sizeof(uintptr_t);
|
|
if (info.type != ExceptionType::AccessViolation ||
|
|
info.access_violation_type != AccessViolationType::Write ||
|
|
info.access_violation_vaddr != fault_address ||
|
|
::sigaltstack(nullptr, &active_stack) != 0 ||
|
|
(active_stack.ss_flags & SS_ONSTACK) == 0) {
|
|
std::_Exit(1);
|
|
}
|
|
g_stack_faults = 1;
|
|
return ::mprotect(g_fault_stack, FAULT_STACK_SIZE,
|
|
PROT_READ | PROT_WRITE) == 0;
|
|
}
|
|
|
|
// A stack write must fault before any signal frame can use the protected stack.
|
|
__attribute__((naked)) void WriteProtectedStack(void *) {
|
|
asm volatile("mov %rsp, %rax\n"
|
|
"mov %rdi, %rsp\n"
|
|
"push %rax\n"
|
|
"pop %rsp\n"
|
|
"ret\n");
|
|
}
|
|
|
|
void TestFaultOnProtectedStack() {
|
|
const pid_t pid = ::fork();
|
|
Check(pid >= 0, "stack fault fork failed");
|
|
if (pid == 0) {
|
|
std::thread worker([] {
|
|
Check(Common::HostException::InitializeThreadSignalStack(),
|
|
"initialize thread signal stack failed");
|
|
Check(Common::HostException::InstallHandler(HandleStackFault),
|
|
"install stack fault handler failed");
|
|
g_fault_stack = ::mmap(nullptr, FAULT_STACK_SIZE, PROT_READ,
|
|
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
|
Check(g_fault_stack != MAP_FAILED, "allocate protected stack failed");
|
|
WriteProtectedStack(static_cast<char *>(g_fault_stack) + FAULT_STACK_SIZE);
|
|
Check(g_stack_faults == 1, "protected stack write did not resume");
|
|
struct sigaction action{};
|
|
Check(::sigaction(SIGSEGV, nullptr, &action) == 0 &&
|
|
action.sa_handler != SIG_DFL,
|
|
"stack fault reset the process handler");
|
|
Check(::munmap(g_fault_stack, FAULT_STACK_SIZE) == 0,
|
|
"release protected stack failed");
|
|
});
|
|
worker.join();
|
|
std::_Exit(0);
|
|
}
|
|
int status = 0;
|
|
Check(::waitpid(pid, &status, 0) == pid && WIFEXITED(status) &&
|
|
WEXITSTATUS(status) == 0,
|
|
"fault on a protected stack did not recover");
|
|
}
|
|
#endif
|
|
|
|
} // namespace
|
|
|
|
namespace Libs::LibKernel::Memory {
|
|
|
|
bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size,
|
|
Common::VirtualMemory::Mode mode) {
|
|
return ProtectAddressSpace(vaddr, size, mode);
|
|
}
|
|
|
|
} // namespace Libs::LibKernel::Memory
|
|
|
|
int main(int argc, char **argv) {
|
|
if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
|
|
RunDeathCase(argv[2]);
|
|
}
|
|
if (argc == 2 && std::strcmp(argv[1], "--benchmark-clean-upload") == 0) {
|
|
BenchmarkCleanUploads();
|
|
return 0;
|
|
}
|
|
TestGuestRange();
|
|
TestRangeSet();
|
|
TestQueriesDoNotRequireMappedOwnership();
|
|
TestConcurrentRegionPublication();
|
|
TestCpuDirtyUpload();
|
|
TestCleanUploadPreservesOwnership();
|
|
TestRangeInvalidation();
|
|
TestGpuReacquisitionAfterInvalidation();
|
|
TestGpuDirtyBits();
|
|
TestExactDirtyIntervalsSharingTrackerPage();
|
|
TestGpuDownloadProtectionMirrors();
|
|
TestCrossRegionUpload();
|
|
TestConcurrentColdUploads();
|
|
TestUploadDoesNotSerializeDisjointRegion();
|
|
TestDownloadDoesNotSerializeDisjointRegion();
|
|
TestGpuUnmarkUsesRegionMask();
|
|
TestFullRegionGpuUnmarkBatching();
|
|
TestFatalPaths();
|
|
#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
|
|
TestFaultOnProtectedStack();
|
|
#endif
|
|
std::puts("MemoryTrackerTests: all cases passed");
|
|
return 0;
|
|
}
|