diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index fa5dc7530..6f0bc11d2 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -747,7 +747,7 @@ constexpr std::size_t WriteBlockBytes = 65536; constexpr std::size_t WritePageBytes = 4096; constexpr std::size_t WritePagesPerBlock = WriteBlockBytes / WritePageBytes; -enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow }; +enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow, Fresh }; #ifdef _WIN32 void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_t generation); @@ -939,7 +939,7 @@ struct WriteTracker { } void stamp(std::uint64_t block, std::uint32_t stampGeneration, StampKind kind) { - const bool cpu = kind != StampKind::Driver; + const bool cpu = kind == StampKind::Cpu || kind == StampKind::ImportWindow; const bool written = kind != StampKind::ImportWindow; #ifdef _WIN32 blocks[block] = stampGeneration; @@ -981,6 +981,7 @@ void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_ if (!tracker.watched || bytes == 0 || bytes > tracker.size || address < tracker.base || address - tracker.base > tracker.size - bytes) return; tracker.coverage.Restore(address, bytes, generation); ++tracker.generation; + tracker.noteCpuStore(address, address + bytes, tracker.generation); for (auto block = tracker.blockOf(address); block <= tracker.blockOf(address + bytes - 1); ++block) { tracker.driverPieces.erase(block); tracker.stamp(block, tracker.generation, StampKind::Cpu); @@ -1026,11 +1027,15 @@ bool sharedCollectMemo() { struct StampRuns { WriteTracker& tracker; StampKind kind; + std::size_t stamped = 0; }; void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) { - auto& [tracker, kind] = *static_cast(context); + auto& runs = *static_cast(context); + auto& tracker = runs.tracker; + const auto kind = runs.kind; if (end <= begin) return; + ++runs.stamped; const auto generation = tracker.generation.load(std::memory_order_relaxed); if (kind != StampKind::Driver) tracker.noteCpuStore(begin, end, generation); for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, kind); @@ -1038,9 +1043,10 @@ void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) { } #endif -bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind) { +bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind, bool armedOnly = false) { ++tracker.generation; #ifdef _WIN32 + static_cast(armedOnly); constexpr std::uint64_t page = 4096; // One resetting walk: the kernel reports and clears a page's dirty bit together, a write landing // after the walk passed a page is reported by the next walk, and a clean page is not touched by the @@ -1086,7 +1092,8 @@ bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, #else const auto dirtyRuns = collectDirtyRuns.load(std::memory_order_relaxed); StampRuns runs{tracker, kind}; - const bool complete = GuestWriteWatch::GuestWriteWatchCollect_nid_postfix(static_cast(first), static_cast(stop - first), &stampWrittenRun, &runs); + const auto collect = armedOnly ? &GuestWriteWatch::GuestWriteWatchCollectArmed_nid_postfix : &GuestWriteWatch::GuestWriteWatchCollect_nid_postfix; + const bool complete = collect(static_cast(first), static_cast(stop - first), &stampWrittenRun, &runs); if (collectDirtyRuns.load(std::memory_order_relaxed) != dirtyRuns) collectDirty.fetch_add(1, std::memory_order_relaxed); return complete; #endif @@ -1206,9 +1213,9 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function const auto first = address & ~(page - 1); const auto stop = (address + bytes + page - 1) & ~(page - 1); const bool covered = tracker.covers(first, static_cast(stop - first)); - const bool before = covered && walkWrites(tracker, first, stop, StampKind::Cpu); + const bool before = covered && walkWrites(tracker, first, stop, StampKind::Cpu, true); if (!import()) return false; - if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow)) unwatchLocked(tracker, first, static_cast(stop - first)); + if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow, true)) unwatchLocked(tracker, first, static_cast(stop - first)); return true; } @@ -1301,6 +1308,18 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) { if (bytes == 0) return 0; ++tracker.generation; if (!tracker.watched || !tracker.covers(address, bytes)) return 0; +#ifndef _WIN32 + { + const auto blockFirst = address & ~(std::uint64_t{WriteBlockBytes} - 1); + const auto blockStop = (address + bytes + WriteBlockBytes - 1) & ~(std::uint64_t{WriteBlockBytes} - 1); + StampRuns runs{tracker, StampKind::Fresh}; + if (!GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(static_cast(blockFirst), static_cast(blockStop - blockFirst), &stampWrittenRun, &runs)) { + unwatchLocked(tracker, blockFirst, static_cast(blockStop - blockFirst)); + return 0; + } + if (runs.stamped != 0) ++tracker.generation; + } +#endif const auto first = tracker.blockOf(address); const auto last = tracker.blockOf(address + bytes - 1); for (auto block = first; block <= last; ++block) { @@ -1320,15 +1339,15 @@ bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generati const auto last = tracker.blockOf(end - 1); for (auto block = first; block <= last; ++block) { if (tracker.writtenStampOf(block) <= generation) continue; - if (tracker.cpuStampOf(block) > generation) { + if (const auto found = tracker.cpuPages.find(block); found != tracker.cpuPages.end()) { const auto begin = tracker.blockBegin(block); const auto from = std::max(address, begin) - begin; const auto to = std::min(end, begin + WriteBlockBytes) - begin; - const auto found = tracker.cpuPages.find(block); - if (found == tracker.cpuPages.end()) return true; for (auto page = from / WritePageBytes; page <= (to - 1) / WritePageBytes; ++page) { if (found->second.pages[page] > generation) return true; } + } else if (tracker.cpuStampOf(block) > generation) { + return true; } const auto found = tracker.driverPieces.find(block); if (found == tracker.driverPieces.end()) return true; @@ -1359,8 +1378,18 @@ bool UnchangedSinceCollected(std::uint64_t address, std::size_t bytes, std::uint if (!tracker.watched || generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false; const auto first = tracker.blockOf(address); const auto last = tracker.blockOf(address + bytes - 1); + const auto end = address + bytes; for (auto block = first; block <= last; ++block) { - if (tracker.cpuStampOf(block) > generation) return false; + if (const auto found = tracker.cpuPages.find(block); found != tracker.cpuPages.end()) { + const auto begin = tracker.blockBegin(block); + const auto from = std::max(address, begin) - begin; + const auto to = std::min(end, begin + WriteBlockBytes) - begin; + for (auto page = from / WritePageBytes; page <= (to - 1) / WritePageBytes; ++page) { + if (found->second.pages[page] > generation) return false; + } + } else if (tracker.cpuStampOf(block) > generation) { + return false; + } } return true; } diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp index e73849699..fb2664979 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp @@ -3,6 +3,8 @@ #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestHeap.hpp" #include "prx/libc/include/GuestArena.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" #ifdef _WIN32 #ifndef NOMINMAX #define NOMINMAX @@ -20,6 +22,9 @@ #include #include +#ifndef _WIN32 +#include +#endif namespace { using namespace AgcDriver::Graphics; @@ -142,6 +147,123 @@ void importedHeapMirrorTests(const Context& context, const BdaTestAccess& access GuestArena::GuestArenaRelease_nid_postfix(block, bytes); } +void importedFreshTests(const Context& context) { +#ifndef _WIN32 + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { + std::cout << "write watch unavailable: imported never-collected pages not tested\n"; + return; + } + constexpr std::size_t blockBytes = 65536; + void* area = mmap(nullptr, 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(area != MAP_FAILED, "cannot map the imported block"); + const auto base = (reinterpret_cast(area) + blockBytes - 1) & ~(blockBytes - 1); + std::memset(reinterpret_cast(base), 0x55, blockBytes); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(base), blockBytes); + auto importing = context; + importing.hostImportAlignment = blockBytes; + const auto registry = [&](bool add) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, reinterpret_cast(base), blockBytes, true, true); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, reinterpret_cast(base)); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + registry(true); + if (HostImportFor(importing, base, blockBytes) == nullptr) { + std::cout << "the never-collected block was not imported: imported never-collected pages not tested\n"; + } else { + const auto gpuWrite = AgcDriver::GuestMemory::MarkWritten(base, 4); + Require(gpuWrite != 0, "a GPU write into an imported never-collected block was not stamped"); + reinterpret_cast(base)[5 * 4096 + 8] = 0x66; + static_cast(AgcDriver::GuestMemory::CollectWritesUncached(base, blockBytes)); + Require(AgcDriver::GuestMemory::WrittenSince(base + 5 * 4096, 4096, gpuWrite), "a CPU write to an imported page protected after the import was not seen"); + } + registry(false); + Require(HostImportFor(importing, base, blockBytes) == nullptr, "the import outlived its range"); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(base), blockBytes); + munmap(area, 2 * blockBytes); +#else + static_cast(context); +#endif +} + +void importWatchTests() { +#ifndef _WIN32 + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { + std::cout << "write watch unavailable: import watch not tested\n"; + return; + } + constexpr std::size_t bytes = 4 * 65536; + void* block = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(block != MAP_FAILED, "cannot map the import watch block"); + auto* guest = static_cast(block); + const auto address = reinterpret_cast(block); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes); + std::memset(block, 0x22, bytes); + const auto written = [&] { + std::size_t reported = 0; + Require(GuestWriteWatch::GuestWriteWatchCollect_nid_postfix(address, bytes, [](void* context, std::uintptr_t begin, std::uintptr_t end) { *static_cast(context) += end - begin; }, &reported), "the write watch did not collect the block"); + return reported; + }; + Require(AgcDriver::GuestMemory::ImportWatched(address, bytes, [] { return true; }), "importing a block nothing had collected failed"); + Require(written() == bytes, "an import write-protected pages nothing had collected"); + guest[3 * 4096] = 0x33; + Require(AgcDriver::GuestMemory::ImportWatched(address, bytes, [] { return true; }), "importing a collected block failed"); + Require(written() == 0, "an import did not take the writes to pages a collect had protected"); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, bytes); + munmap(block, bytes); + constexpr std::size_t blockBytes = 65536; + void* area = mmap(nullptr, bytes + 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(area != MAP_FAILED, "cannot map the GPU write block"); + const auto aligned = (reinterpret_cast(area) + blockBytes - 1) & ~(blockBytes - 1); + const auto imported = aligned + 4096; + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(imported), bytes); + Require(AgcDriver::GuestMemory::ImportWatched(imported, bytes, [] { return true; }), "importing a fresh range failed"); + const auto gpuWrite = AgcDriver::GuestMemory::MarkWritten(imported + 4096, 4096); + Require(gpuWrite != 0, "a GPU write into the imported range was not stamped"); + static_cast(AgcDriver::GuestMemory::CollectWritesUncached(imported, bytes)); + Require(!AgcDriver::GuestMemory::WrittenSince(imported + 4096, 4096, gpuWrite), "a collect after a GPU write into an uncollected import read as a CPU write over the GPU's results"); + reinterpret_cast(imported)[4096 + 100] = 0x44; + static_cast(AgcDriver::GuestMemory::CollectWritesUncached(imported, bytes)); + Require(AgcDriver::GuestMemory::WrittenSince(imported + 4096, 4096, gpuWrite), "a CPU write after a GPU write into an import was not seen"); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(imported), bytes); + munmap(area, bytes + 2 * blockBytes); + + void* labels = mmap(nullptr, 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(labels != MAP_FAILED, "cannot map the label block"); + const auto label = (reinterpret_cast(labels) + blockBytes - 1) & ~(blockBytes - 1); + auto* words = reinterpret_cast(label); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(label), blockBytes); + words[0] = 1; + static_cast(AgcDriver::GuestMemory::CollectWritesUncached(label, 4)); + const auto before = AgcDriver::GuestMemory::TrackerGeneration(); + Require(AgcDriver::GuestMemory::MarkWritten(label, 4) != 0, "a GPU write next to never-collected pages was not stamped"); + Require(AgcDriver::GuestMemory::UnchangedSinceCollected(label, 4, before), "a GPU write next to never-collected pages read as a CPU write of its block"); + Require(AgcDriver::GuestMemory::StoredOver(label + 8 * 4096, 4096, before), "a never-collected page a GPU write collected is not reported as possibly stored"); + const std::array since{before}; + std::array changed{}; + std::array cpu{}; + Require(AgcDriver::GuestMemory::ChangedBlocks(label, blockBytes, since, changed, cpu) && changed[0] == AgcDriver::GuestMemory::BlockWritten && cpu[0] == 0, "collecting never-collected pages at a GPU write stamped their block as written by the CPU"); + const auto now = AgcDriver::GuestMemory::TrackerGeneration(); + Require(AgcDriver::GuestMemory::MarkWritten(label, 4) == now + 1, "a GPU write into a block with nothing left to collect advanced the generation more than once"); + words[1] = 2; + Require(!AgcDriver::GuestMemory::UnchangedSinceCollected(label, 4, before), "a CPU write next to a GPU write was not seen"); + void* awaitedArea = mmap(nullptr, 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(awaitedArea != MAP_FAILED, "cannot map the awaited block"); + const auto awaitedBlock = (reinterpret_cast(awaitedArea) + blockBytes - 1) & ~(blockBytes - 1); + const auto awaited = awaitedBlock + 5 * 4096; + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(awaitedBlock), blockBytes); + static_cast(AgcDriver::GuestMemory::CollectWritesUncached(awaitedBlock, 4)); + const auto closed = AgcDriver::GuestMemory::TrackerGeneration(); + *reinterpret_cast(awaited) = 7; + Require(AgcDriver::GuestMemory::MarkWritten(awaitedBlock, 4) != 0, "a GPU write into the awaited block was not stamped"); + Require(!AgcDriver::GuestMemory::UnchangedSinceCollected(awaited, 4, closed), "a CPU store to a never-collected page that a GPU write elsewhere in its block collected was not seen"); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(awaitedBlock), blockBytes); + munmap(awaitedArea, 2 * blockBytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(label), blockBytes); + munmap(labels, 2 * blockBytes); +#endif +} + void heapMirrorTests(const Context& context, const BdaTestAccess& access) { if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) { std::cout << "guest arena unavailable or not write-watched: heap mirrors not tested\n"; @@ -570,6 +692,8 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { importCrossingTests(context, access); heapMirrorTests(context, access); importedHeapMirrorTests(context, access); + importWatchTests(); + importedFreshTests(context); Require(AddressCopyOverflow({{0x1000, 0x3000, 0x2000, "uncommitted pages"}}, 0x2000).empty(), "copies within the limit were refused"); const auto copies = AddressCopyOverflow({{0x1000, 0x2000, 0x1000, "not mirrored"}, {0x10000, 0x30000, 0x18000, "uncommitted pages"}}, 0x2000); Require(!copies.empty() && copies.find("0x10000+0x20000 (0.1 MiB committed, uncommitted pages)") < copies.find("0x1000+0x1000"), "the copy limit does not name the largest copy first"); diff --git a/core/libs/prx/libc/include/GuestWriteWatch.hpp b/core/libs/prx/libc/include/GuestWriteWatch.hpp index db526a514..95d5df1fc 100644 --- a/core/libs/prx/libc/include/GuestWriteWatch.hpp +++ b/core/libs/prx/libc/include/GuestWriteWatch.hpp @@ -17,6 +17,8 @@ void GuestWriteWatchBeginHostWrite_nid_postfix(const void* pointer, std::size_t void GuestWriteWatchEndHostWrite_nid_postfix(const void* pointer, std::size_t bytes); void GuestWriteWatchProtectionChanged_nid_postfix(const void* pointer, std::size_t bytes, int protection); int GuestWriteWatchProtection_nid_postfix(std::uintptr_t address, std::uintptr_t limit, std::uintptr_t* runEnd); +bool GuestWriteWatchCollectArmed_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context); +bool GuestWriteWatchCollectFresh_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context); } diff --git a/core/libs/prx/libc/src/GuestWriteWatch.cpp b/core/libs/prx/libc/src/GuestWriteWatch.cpp index 84bc2f1ea..0345de9a4 100644 --- a/core/libs/prx/libc/src/GuestWriteWatch.cpp +++ b/core/libs/prx/libc/src/GuestWriteWatch.cpp @@ -95,10 +95,56 @@ public: begin &= ~(PageBytes - 1); end = (end + PageBytes - 1) & ~(PageBytes - 1); if (end <= begin) return true; + if (!takeFresh(begin, end, true, written, context)) return false; + return scanWritten(begin, end, written, context); + } + + bool CollectFresh(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) { + if (!Available()) return false; + begin &= ~(PageBytes - 1); + end = (end + PageBytes - 1) & ~(PageBytes - 1); + if (end <= begin) return true; + { + std::shared_lock lock(_lock); + auto it = _fresh.upper_bound(begin); + if (it != _fresh.begin()) --it; + while (it != _fresh.end() && it->first < end && it->second <= begin) ++it; + if (it == _fresh.end() || it->first >= end) return true; + } + return takeFresh(begin, end, false, written, context); + } + + bool CollectArmed(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) { + if (!Available()) return false; + begin &= ~(PageBytes - 1); + end = (end + PageBytes - 1) & ~(PageBytes - 1); + if (end <= begin) return true; + std::vector> armed; + { + std::shared_lock lock(_lock); + if (!covers(begin, end)) return false; + auto cursor = begin; + auto it = _fresh.upper_bound(begin); + if (it != _fresh.begin()) --it; + for (; it != _fresh.end() && it->first < end; ++it) { + if (it->second <= cursor) continue; + if (it->first > cursor) armed.emplace_back(cursor, it->first); + cursor = std::max(cursor, it->second); + } + if (cursor < end) armed.emplace_back(cursor, end); + } + for (const auto& [from, to] : armed) { + if (!scanWritten(from, to, written, context)) return false; + } + return true; + } + +private: + bool takeFresh(std::uintptr_t begin, std::uintptr_t end, bool whole, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) { std::vector> fresh; { std::unique_lock lock(_lock); - if (!covers(begin, end)) return false; + if (whole && !covers(begin, end)) return false; auto it = _fresh.upper_bound(begin); if (it != _fresh.begin()) --it; for (; it != _fresh.end() && it->first < end; ++it) { @@ -106,7 +152,7 @@ public: const auto to = std::min(it->second, end); if (from < to) fresh.emplace_back(from, to); } - if (!fresh.empty()) remove(_fresh, begin, end); + for (const auto& [from, to] : fresh) remove(_fresh, from, to); } for (const auto& [from, to] : fresh) { written(context, from, to); @@ -117,9 +163,14 @@ public: remove(_fresh, left, right); insert(_fresh, left, right); } - written(context, begin, end); + if (whole) written(context, begin, end); + else for (const auto& [left, right] : fresh) written(context, left, right); return false; } + return true; + } + + bool scanWritten(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) const { std::array regions; auto cursor = begin; while (cursor < end) { @@ -644,4 +695,37 @@ int GuestWriteWatchProtection_nid_postfix(std::uintptr_t address, std::uintptr_t #endif } +bool GuestWriteWatchCollectFresh_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context) { +#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) + if (bytes == 0 || address + bytes < address) return false; + return Watch::Get().CollectFresh(address, address + bytes, written, context); +#elif defined(__APPLE__) + static_cast(written); + static_cast(context); + return bytes != 0 && address + bytes >= address; +#else + static_cast(address); + static_cast(bytes); + static_cast(written); + static_cast(context); + return false; +#endif +} + +bool GuestWriteWatchCollectArmed_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context) { +#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) + if (bytes == 0 || address + bytes < address) return false; + return Watch::Get().CollectArmed(address, address + bytes, written, context); +#elif defined(__APPLE__) + if (bytes == 0 || address + bytes < address) return false; + return Watch::Get().Collect(address, address + bytes, written, context); +#else + static_cast(address); + static_cast(bytes); + static_cast(written); + static_cast(context); + return false; +#endif +} + } diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index 4ce6ba220..cd82d5724 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -935,6 +935,62 @@ static bool Written(const void* base, std::size_t bytes, PageRuns expected) { return false; } +static bool CollectArmedRuns(const void* base, std::size_t offset, std::size_t bytes, PageRuns& runs) { + runs.clear(); + const auto address = reinterpret_cast(base); + std::pair context{address, &runs}; + return GuestWriteWatch::GuestWriteWatchCollectArmed_nid_postfix(address + offset, bytes, [](void* context, std::uintptr_t begin, std::uintptr_t end) { + auto& [origin, into] = *static_cast*>(context); + if (!into->empty() && into->back().second == (begin - origin) / 4096) into->back().second = (end - origin) / 4096; + else into->emplace_back((begin - origin) / 4096, (end - origin) / 4096); + }, &context); +} + +static void CheckWriteWatchArmedCollect() { + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) return; + constexpr std::size_t length = 0x100000; + constexpr std::size_t small = 4096; + void* mapping = nullptr; + Require(sceKernelMapFlexibleMemory(&mapping, length, 3, 0) == 0); + auto* bytes = static_cast(mapping); + PageRuns runs; + Require(CollectRuns(mapping, 0, length / 2, runs) && runs == PageRuns{{0, length / small / 2}}); + bytes[5 * small] = 1; + bytes[200 * small] = 1; + Require(CollectArmedRuns(mapping, 0, length, runs) && runs == PageRuns{{5, 6}}); + Require(Written(mapping, length, {{length / small / 2, length / small}})); + Require(Written(mapping, length, {})); + Require(sceKernelMunmap(mapping, length) == 0); +} + +static bool CollectFreshRuns(const void* base, std::size_t offset, std::size_t bytes, PageRuns& runs) { + runs.clear(); + const auto address = reinterpret_cast(base); + std::pair context{address, &runs}; + return GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(address + offset, bytes, [](void* context, std::uintptr_t begin, std::uintptr_t end) { + auto& [origin, into] = *static_cast*>(context); + if (!into->empty() && into->back().second == (begin - origin) / 4096) into->back().second = (end - origin) / 4096; + else into->emplace_back((begin - origin) / 4096, (end - origin) / 4096); + }, &context); +} + +static void CheckWriteWatchFreshCollect() { + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) return; + constexpr std::size_t length = 0x100000; + constexpr std::size_t small = 4096; + void* mapping = nullptr; + Require(sceKernelMapFlexibleMemory(&mapping, length, 3, 0) == 0); + auto* bytes = static_cast(mapping); + PageRuns runs; + Require(CollectRuns(mapping, 0, length / 2, runs) && runs == PageRuns{{0, length / small / 2}}); + bytes[5 * small] = 1; + Require(CollectFreshRuns(mapping, 0, length, runs) && runs == PageRuns{{length / small / 2, length / small}}); + Require(CollectFreshRuns(mapping, 0, length, runs) && runs.empty()); + bytes[200 * small] = 1; + Require(Written(mapping, length, {{5, 6}, {200, 201}})); + Require(sceKernelMunmap(mapping, length) == 0); +} + static void CheckWriteWatch() { if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { std::puts("write watch unavailable: not tested"); @@ -1274,6 +1330,8 @@ int main() { CheckFailedCollectKeepsWrites(); #if defined(__linux__) CheckWriteWatch(); + CheckWriteWatchArmedCollect(); + CheckWriteWatchFreshCollect(); CheckDirectMemoryWriteWatch(); CheckDirectMemoryBackingNeedsNoFilesystem(); CheckDirectMemorySharedBacking();