Merge pull request #1960 from Raghav314/fix/agc-import-leaves-pages-unarmed
fix(agc): host imports write-protect only pages a collect already protects
This commit is contained in:
Adrià
GitHub
commit
4fbf6ca5a3
|
|
@ -747,7 +747,7 @@ constexpr std::size_t WriteBlockBytes = 65536;
|
|||
constexpr std::size_t WritePageBytes = 4096;
|
||||
constexpr std::size_t WritePagesPerBlock = WriteBlockBytes / WritePageBytes;
|
||||
|
||||
enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow };
|
||||
enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow, Fresh };
|
||||
|
||||
#ifdef _WIN32
|
||||
void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_t generation);
|
||||
|
|
@ -939,7 +939,7 @@ struct WriteTracker {
|
|||
}
|
||||
|
||||
void stamp(std::uint64_t block, std::uint32_t stampGeneration, StampKind kind) {
|
||||
const bool cpu = kind != StampKind::Driver;
|
||||
const bool cpu = kind == StampKind::Cpu || kind == StampKind::ImportWindow;
|
||||
const bool written = kind != StampKind::ImportWindow;
|
||||
#ifdef _WIN32
|
||||
blocks[block] = stampGeneration;
|
||||
|
|
@ -981,6 +981,7 @@ void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_
|
|||
if (!tracker.watched || bytes == 0 || bytes > tracker.size || address < tracker.base || address - tracker.base > tracker.size - bytes) return;
|
||||
tracker.coverage.Restore(address, bytes, generation);
|
||||
++tracker.generation;
|
||||
tracker.noteCpuStore(address, address + bytes, tracker.generation);
|
||||
for (auto block = tracker.blockOf(address); block <= tracker.blockOf(address + bytes - 1); ++block) {
|
||||
tracker.driverPieces.erase(block);
|
||||
tracker.stamp(block, tracker.generation, StampKind::Cpu);
|
||||
|
|
@ -1026,11 +1027,15 @@ bool sharedCollectMemo() {
|
|||
struct StampRuns {
|
||||
WriteTracker& tracker;
|
||||
StampKind kind;
|
||||
std::size_t stamped = 0;
|
||||
};
|
||||
|
||||
void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) {
|
||||
auto& [tracker, kind] = *static_cast<StampRuns*>(context);
|
||||
auto& runs = *static_cast<StampRuns*>(context);
|
||||
auto& tracker = runs.tracker;
|
||||
const auto kind = runs.kind;
|
||||
if (end <= begin) return;
|
||||
++runs.stamped;
|
||||
const auto generation = tracker.generation.load(std::memory_order_relaxed);
|
||||
if (kind != StampKind::Driver) tracker.noteCpuStore(begin, end, generation);
|
||||
for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, kind);
|
||||
|
|
@ -1038,9 +1043,10 @@ void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) {
|
|||
}
|
||||
#endif
|
||||
|
||||
bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind) {
|
||||
bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind, bool armedOnly = false) {
|
||||
++tracker.generation;
|
||||
#ifdef _WIN32
|
||||
static_cast<void>(armedOnly);
|
||||
constexpr std::uint64_t page = 4096;
|
||||
// One resetting walk: the kernel reports and clears a page's dirty bit together, a write landing
|
||||
// after the walk passed a page is reported by the next walk, and a clean page is not touched by the
|
||||
|
|
@ -1086,7 +1092,8 @@ bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop,
|
|||
#else
|
||||
const auto dirtyRuns = collectDirtyRuns.load(std::memory_order_relaxed);
|
||||
StampRuns runs{tracker, kind};
|
||||
const bool complete = GuestWriteWatch::GuestWriteWatchCollect_nid_postfix(static_cast<std::uintptr_t>(first), static_cast<std::size_t>(stop - first), &stampWrittenRun, &runs);
|
||||
const auto collect = armedOnly ? &GuestWriteWatch::GuestWriteWatchCollectArmed_nid_postfix : &GuestWriteWatch::GuestWriteWatchCollect_nid_postfix;
|
||||
const bool complete = collect(static_cast<std::uintptr_t>(first), static_cast<std::size_t>(stop - first), &stampWrittenRun, &runs);
|
||||
if (collectDirtyRuns.load(std::memory_order_relaxed) != dirtyRuns) collectDirty.fetch_add(1, std::memory_order_relaxed);
|
||||
return complete;
|
||||
#endif
|
||||
|
|
@ -1206,9 +1213,9 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
|
|||
const auto first = address & ~(page - 1);
|
||||
const auto stop = (address + bytes + page - 1) & ~(page - 1);
|
||||
const bool covered = tracker.covers(first, static_cast<std::size_t>(stop - first));
|
||||
const bool before = covered && walkWrites(tracker, first, stop, StampKind::Cpu);
|
||||
const bool before = covered && walkWrites(tracker, first, stop, StampKind::Cpu, true);
|
||||
if (!import()) return false;
|
||||
if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow)) unwatchLocked(tracker, first, static_cast<std::size_t>(stop - first));
|
||||
if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow, true)) unwatchLocked(tracker, first, static_cast<std::size_t>(stop - first));
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -1301,6 +1308,18 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
|
|||
if (bytes == 0) return 0;
|
||||
++tracker.generation;
|
||||
if (!tracker.watched || !tracker.covers(address, bytes)) return 0;
|
||||
#ifndef _WIN32
|
||||
{
|
||||
const auto blockFirst = address & ~(std::uint64_t{WriteBlockBytes} - 1);
|
||||
const auto blockStop = (address + bytes + WriteBlockBytes - 1) & ~(std::uint64_t{WriteBlockBytes} - 1);
|
||||
StampRuns runs{tracker, StampKind::Fresh};
|
||||
if (!GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(static_cast<std::uintptr_t>(blockFirst), static_cast<std::size_t>(blockStop - blockFirst), &stampWrittenRun, &runs)) {
|
||||
unwatchLocked(tracker, blockFirst, static_cast<std::size_t>(blockStop - blockFirst));
|
||||
return 0;
|
||||
}
|
||||
if (runs.stamped != 0) ++tracker.generation;
|
||||
}
|
||||
#endif
|
||||
const auto first = tracker.blockOf(address);
|
||||
const auto last = tracker.blockOf(address + bytes - 1);
|
||||
for (auto block = first; block <= last; ++block) {
|
||||
|
|
@ -1320,15 +1339,15 @@ bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generati
|
|||
const auto last = tracker.blockOf(end - 1);
|
||||
for (auto block = first; block <= last; ++block) {
|
||||
if (tracker.writtenStampOf(block) <= generation) continue;
|
||||
if (tracker.cpuStampOf(block) > generation) {
|
||||
if (const auto found = tracker.cpuPages.find(block); found != tracker.cpuPages.end()) {
|
||||
const auto begin = tracker.blockBegin(block);
|
||||
const auto from = std::max(address, begin) - begin;
|
||||
const auto to = std::min<std::uint64_t>(end, begin + WriteBlockBytes) - begin;
|
||||
const auto found = tracker.cpuPages.find(block);
|
||||
if (found == tracker.cpuPages.end()) return true;
|
||||
for (auto page = from / WritePageBytes; page <= (to - 1) / WritePageBytes; ++page) {
|
||||
if (found->second.pages[page] > generation) return true;
|
||||
}
|
||||
} else if (tracker.cpuStampOf(block) > generation) {
|
||||
return true;
|
||||
}
|
||||
const auto found = tracker.driverPieces.find(block);
|
||||
if (found == tracker.driverPieces.end()) return true;
|
||||
|
|
@ -1359,8 +1378,18 @@ bool UnchangedSinceCollected(std::uint64_t address, std::size_t bytes, std::uint
|
|||
if (!tracker.watched || generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false;
|
||||
const auto first = tracker.blockOf(address);
|
||||
const auto last = tracker.blockOf(address + bytes - 1);
|
||||
const auto end = address + bytes;
|
||||
for (auto block = first; block <= last; ++block) {
|
||||
if (tracker.cpuStampOf(block) > generation) return false;
|
||||
if (const auto found = tracker.cpuPages.find(block); found != tracker.cpuPages.end()) {
|
||||
const auto begin = tracker.blockBegin(block);
|
||||
const auto from = std::max(address, begin) - begin;
|
||||
const auto to = std::min<std::uint64_t>(end, begin + WriteBlockBytes) - begin;
|
||||
for (auto page = from / WritePageBytes; page <= (to - 1) / WritePageBytes; ++page) {
|
||||
if (found->second.pages[page] > generation) return false;
|
||||
}
|
||||
} else if (tracker.cpuStampOf(block) > generation) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
#include "prx/libc/include/GuestAllocations.hpp"
|
||||
#include "prx/libc/include/GuestHeap.hpp"
|
||||
#include "prx/libc/include/GuestArena.hpp"
|
||||
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
|
||||
#include "prx/libc/include/GuestWriteWatch.hpp"
|
||||
#ifdef _WIN32
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
|
|
@ -20,6 +22,9 @@
|
|||
#include <new>
|
||||
#include <string>
|
||||
|
||||
#ifndef _WIN32
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
namespace {
|
||||
|
||||
using namespace AgcDriver::Graphics;
|
||||
|
|
@ -142,6 +147,123 @@ void importedHeapMirrorTests(const Context& context, const BdaTestAccess& access
|
|||
GuestArena::GuestArenaRelease_nid_postfix(block, bytes);
|
||||
}
|
||||
|
||||
void importedFreshTests(const Context& context) {
|
||||
#ifndef _WIN32
|
||||
if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) {
|
||||
std::cout << "write watch unavailable: imported never-collected pages not tested\n";
|
||||
return;
|
||||
}
|
||||
constexpr std::size_t blockBytes = 65536;
|
||||
void* area = mmap(nullptr, 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
Require(area != MAP_FAILED, "cannot map the imported block");
|
||||
const auto base = (reinterpret_cast<std::uintptr_t>(area) + blockBytes - 1) & ~(blockBytes - 1);
|
||||
std::memset(reinterpret_cast<void*>(base), 0x55, blockBytes);
|
||||
GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast<void*>(base), blockBytes);
|
||||
auto importing = context;
|
||||
importing.hostImportAlignment = blockBytes;
|
||||
const auto registry = [&](bool add) {
|
||||
auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
|
||||
if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, reinterpret_cast<void*>(base), blockBytes, true, true);
|
||||
else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, reinterpret_cast<void*>(base));
|
||||
GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
|
||||
};
|
||||
registry(true);
|
||||
if (HostImportFor(importing, base, blockBytes) == nullptr) {
|
||||
std::cout << "the never-collected block was not imported: imported never-collected pages not tested\n";
|
||||
} else {
|
||||
const auto gpuWrite = AgcDriver::GuestMemory::MarkWritten(base, 4);
|
||||
Require(gpuWrite != 0, "a GPU write into an imported never-collected block was not stamped");
|
||||
reinterpret_cast<volatile std::uint8_t*>(base)[5 * 4096 + 8] = 0x66;
|
||||
static_cast<void>(AgcDriver::GuestMemory::CollectWritesUncached(base, blockBytes));
|
||||
Require(AgcDriver::GuestMemory::WrittenSince(base + 5 * 4096, 4096, gpuWrite), "a CPU write to an imported page protected after the import was not seen");
|
||||
}
|
||||
registry(false);
|
||||
Require(HostImportFor(importing, base, blockBytes) == nullptr, "the import outlived its range");
|
||||
GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast<void*>(base), blockBytes);
|
||||
munmap(area, 2 * blockBytes);
|
||||
#else
|
||||
static_cast<void>(context);
|
||||
#endif
|
||||
}
|
||||
|
||||
void importWatchTests() {
|
||||
#ifndef _WIN32
|
||||
if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) {
|
||||
std::cout << "write watch unavailable: import watch not tested\n";
|
||||
return;
|
||||
}
|
||||
constexpr std::size_t bytes = 4 * 65536;
|
||||
void* block = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
Require(block != MAP_FAILED, "cannot map the import watch block");
|
||||
auto* guest = static_cast<std::uint8_t*>(block);
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(block);
|
||||
GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes);
|
||||
std::memset(block, 0x22, bytes);
|
||||
const auto written = [&] {
|
||||
std::size_t reported = 0;
|
||||
Require(GuestWriteWatch::GuestWriteWatchCollect_nid_postfix(address, bytes, [](void* context, std::uintptr_t begin, std::uintptr_t end) { *static_cast<std::size_t*>(context) += end - begin; }, &reported), "the write watch did not collect the block");
|
||||
return reported;
|
||||
};
|
||||
Require(AgcDriver::GuestMemory::ImportWatched(address, bytes, [] { return true; }), "importing a block nothing had collected failed");
|
||||
Require(written() == bytes, "an import write-protected pages nothing had collected");
|
||||
guest[3 * 4096] = 0x33;
|
||||
Require(AgcDriver::GuestMemory::ImportWatched(address, bytes, [] { return true; }), "importing a collected block failed");
|
||||
Require(written() == 0, "an import did not take the writes to pages a collect had protected");
|
||||
GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, bytes);
|
||||
munmap(block, bytes);
|
||||
constexpr std::size_t blockBytes = 65536;
|
||||
void* area = mmap(nullptr, bytes + 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
Require(area != MAP_FAILED, "cannot map the GPU write block");
|
||||
const auto aligned = (reinterpret_cast<std::uintptr_t>(area) + blockBytes - 1) & ~(blockBytes - 1);
|
||||
const auto imported = aligned + 4096;
|
||||
GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast<void*>(imported), bytes);
|
||||
Require(AgcDriver::GuestMemory::ImportWatched(imported, bytes, [] { return true; }), "importing a fresh range failed");
|
||||
const auto gpuWrite = AgcDriver::GuestMemory::MarkWritten(imported + 4096, 4096);
|
||||
Require(gpuWrite != 0, "a GPU write into the imported range was not stamped");
|
||||
static_cast<void>(AgcDriver::GuestMemory::CollectWritesUncached(imported, bytes));
|
||||
Require(!AgcDriver::GuestMemory::WrittenSince(imported + 4096, 4096, gpuWrite), "a collect after a GPU write into an uncollected import read as a CPU write over the GPU's results");
|
||||
reinterpret_cast<volatile std::uint8_t*>(imported)[4096 + 100] = 0x44;
|
||||
static_cast<void>(AgcDriver::GuestMemory::CollectWritesUncached(imported, bytes));
|
||||
Require(AgcDriver::GuestMemory::WrittenSince(imported + 4096, 4096, gpuWrite), "a CPU write after a GPU write into an import was not seen");
|
||||
GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast<void*>(imported), bytes);
|
||||
munmap(area, bytes + 2 * blockBytes);
|
||||
|
||||
void* labels = mmap(nullptr, 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
Require(labels != MAP_FAILED, "cannot map the label block");
|
||||
const auto label = (reinterpret_cast<std::uintptr_t>(labels) + blockBytes - 1) & ~(blockBytes - 1);
|
||||
auto* words = reinterpret_cast<volatile std::uint32_t*>(label);
|
||||
GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast<void*>(label), blockBytes);
|
||||
words[0] = 1;
|
||||
static_cast<void>(AgcDriver::GuestMemory::CollectWritesUncached(label, 4));
|
||||
const auto before = AgcDriver::GuestMemory::TrackerGeneration();
|
||||
Require(AgcDriver::GuestMemory::MarkWritten(label, 4) != 0, "a GPU write next to never-collected pages was not stamped");
|
||||
Require(AgcDriver::GuestMemory::UnchangedSinceCollected(label, 4, before), "a GPU write next to never-collected pages read as a CPU write of its block");
|
||||
Require(AgcDriver::GuestMemory::StoredOver(label + 8 * 4096, 4096, before), "a never-collected page a GPU write collected is not reported as possibly stored");
|
||||
const std::array<std::uint64_t, 1> since{before};
|
||||
std::array<std::uint8_t, 1> changed{};
|
||||
std::array<std::uint8_t, 1> cpu{};
|
||||
Require(AgcDriver::GuestMemory::ChangedBlocks(label, blockBytes, since, changed, cpu) && changed[0] == AgcDriver::GuestMemory::BlockWritten && cpu[0] == 0, "collecting never-collected pages at a GPU write stamped their block as written by the CPU");
|
||||
const auto now = AgcDriver::GuestMemory::TrackerGeneration();
|
||||
Require(AgcDriver::GuestMemory::MarkWritten(label, 4) == now + 1, "a GPU write into a block with nothing left to collect advanced the generation more than once");
|
||||
words[1] = 2;
|
||||
Require(!AgcDriver::GuestMemory::UnchangedSinceCollected(label, 4, before), "a CPU write next to a GPU write was not seen");
|
||||
void* awaitedArea = mmap(nullptr, 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
Require(awaitedArea != MAP_FAILED, "cannot map the awaited block");
|
||||
const auto awaitedBlock = (reinterpret_cast<std::uintptr_t>(awaitedArea) + blockBytes - 1) & ~(blockBytes - 1);
|
||||
const auto awaited = awaitedBlock + 5 * 4096;
|
||||
GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast<void*>(awaitedBlock), blockBytes);
|
||||
static_cast<void>(AgcDriver::GuestMemory::CollectWritesUncached(awaitedBlock, 4));
|
||||
const auto closed = AgcDriver::GuestMemory::TrackerGeneration();
|
||||
*reinterpret_cast<volatile std::uint32_t*>(awaited) = 7;
|
||||
Require(AgcDriver::GuestMemory::MarkWritten(awaitedBlock, 4) != 0, "a GPU write into the awaited block was not stamped");
|
||||
Require(!AgcDriver::GuestMemory::UnchangedSinceCollected(awaited, 4, closed), "a CPU store to a never-collected page that a GPU write elsewhere in its block collected was not seen");
|
||||
GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast<void*>(awaitedBlock), blockBytes);
|
||||
munmap(awaitedArea, 2 * blockBytes);
|
||||
GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast<void*>(label), blockBytes);
|
||||
munmap(labels, 2 * blockBytes);
|
||||
#endif
|
||||
}
|
||||
|
||||
void heapMirrorTests(const Context& context, const BdaTestAccess& access) {
|
||||
if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
|
||||
std::cout << "guest arena unavailable or not write-watched: heap mirrors not tested\n";
|
||||
|
|
@ -570,6 +692,8 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
|
|||
importCrossingTests(context, access);
|
||||
heapMirrorTests(context, access);
|
||||
importedHeapMirrorTests(context, access);
|
||||
importWatchTests();
|
||||
importedFreshTests(context);
|
||||
Require(AddressCopyOverflow({{0x1000, 0x3000, 0x2000, "uncommitted pages"}}, 0x2000).empty(), "copies within the limit were refused");
|
||||
const auto copies = AddressCopyOverflow({{0x1000, 0x2000, 0x1000, "not mirrored"}, {0x10000, 0x30000, 0x18000, "uncommitted pages"}}, 0x2000);
|
||||
Require(!copies.empty() && copies.find("0x10000+0x20000 (0.1 MiB committed, uncommitted pages)") < copies.find("0x1000+0x1000"), "the copy limit does not name the largest copy first");
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ void GuestWriteWatchBeginHostWrite_nid_postfix(const void* pointer, std::size_t
|
|||
void GuestWriteWatchEndHostWrite_nid_postfix(const void* pointer, std::size_t bytes);
|
||||
void GuestWriteWatchProtectionChanged_nid_postfix(const void* pointer, std::size_t bytes, int protection);
|
||||
int GuestWriteWatchProtection_nid_postfix(std::uintptr_t address, std::uintptr_t limit, std::uintptr_t* runEnd);
|
||||
bool GuestWriteWatchCollectArmed_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context);
|
||||
bool GuestWriteWatchCollectFresh_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context);
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -95,10 +95,56 @@ public:
|
|||
begin &= ~(PageBytes - 1);
|
||||
end = (end + PageBytes - 1) & ~(PageBytes - 1);
|
||||
if (end <= begin) return true;
|
||||
if (!takeFresh(begin, end, true, written, context)) return false;
|
||||
return scanWritten(begin, end, written, context);
|
||||
}
|
||||
|
||||
bool CollectFresh(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) {
|
||||
if (!Available()) return false;
|
||||
begin &= ~(PageBytes - 1);
|
||||
end = (end + PageBytes - 1) & ~(PageBytes - 1);
|
||||
if (end <= begin) return true;
|
||||
{
|
||||
std::shared_lock lock(_lock);
|
||||
auto it = _fresh.upper_bound(begin);
|
||||
if (it != _fresh.begin()) --it;
|
||||
while (it != _fresh.end() && it->first < end && it->second <= begin) ++it;
|
||||
if (it == _fresh.end() || it->first >= end) return true;
|
||||
}
|
||||
return takeFresh(begin, end, false, written, context);
|
||||
}
|
||||
|
||||
bool CollectArmed(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) {
|
||||
if (!Available()) return false;
|
||||
begin &= ~(PageBytes - 1);
|
||||
end = (end + PageBytes - 1) & ~(PageBytes - 1);
|
||||
if (end <= begin) return true;
|
||||
std::vector<std::pair<std::uintptr_t, std::uintptr_t>> armed;
|
||||
{
|
||||
std::shared_lock lock(_lock);
|
||||
if (!covers(begin, end)) return false;
|
||||
auto cursor = begin;
|
||||
auto it = _fresh.upper_bound(begin);
|
||||
if (it != _fresh.begin()) --it;
|
||||
for (; it != _fresh.end() && it->first < end; ++it) {
|
||||
if (it->second <= cursor) continue;
|
||||
if (it->first > cursor) armed.emplace_back(cursor, it->first);
|
||||
cursor = std::max(cursor, it->second);
|
||||
}
|
||||
if (cursor < end) armed.emplace_back(cursor, end);
|
||||
}
|
||||
for (const auto& [from, to] : armed) {
|
||||
if (!scanWritten(from, to, written, context)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
bool takeFresh(std::uintptr_t begin, std::uintptr_t end, bool whole, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) {
|
||||
std::vector<std::pair<std::uintptr_t, std::uintptr_t>> fresh;
|
||||
{
|
||||
std::unique_lock lock(_lock);
|
||||
if (!covers(begin, end)) return false;
|
||||
if (whole && !covers(begin, end)) return false;
|
||||
auto it = _fresh.upper_bound(begin);
|
||||
if (it != _fresh.begin()) --it;
|
||||
for (; it != _fresh.end() && it->first < end; ++it) {
|
||||
|
|
@ -106,7 +152,7 @@ public:
|
|||
const auto to = std::min(it->second, end);
|
||||
if (from < to) fresh.emplace_back(from, to);
|
||||
}
|
||||
if (!fresh.empty()) remove(_fresh, begin, end);
|
||||
for (const auto& [from, to] : fresh) remove(_fresh, from, to);
|
||||
}
|
||||
for (const auto& [from, to] : fresh) {
|
||||
written(context, from, to);
|
||||
|
|
@ -117,9 +163,14 @@ public:
|
|||
remove(_fresh, left, right);
|
||||
insert(_fresh, left, right);
|
||||
}
|
||||
written(context, begin, end);
|
||||
if (whole) written(context, begin, end);
|
||||
else for (const auto& [left, right] : fresh) written(context, left, right);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool scanWritten(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) const {
|
||||
std::array<page_region, 256> regions;
|
||||
auto cursor = begin;
|
||||
while (cursor < end) {
|
||||
|
|
@ -644,4 +695,37 @@ int GuestWriteWatchProtection_nid_postfix(std::uintptr_t address, std::uintptr_t
|
|||
#endif
|
||||
}
|
||||
|
||||
bool GuestWriteWatchCollectFresh_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context) {
|
||||
#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC)
|
||||
if (bytes == 0 || address + bytes < address) return false;
|
||||
return Watch::Get().CollectFresh(address, address + bytes, written, context);
|
||||
#elif defined(__APPLE__)
|
||||
static_cast<void>(written);
|
||||
static_cast<void>(context);
|
||||
return bytes != 0 && address + bytes >= address;
|
||||
#else
|
||||
static_cast<void>(address);
|
||||
static_cast<void>(bytes);
|
||||
static_cast<void>(written);
|
||||
static_cast<void>(context);
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool GuestWriteWatchCollectArmed_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context) {
|
||||
#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC)
|
||||
if (bytes == 0 || address + bytes < address) return false;
|
||||
return Watch::Get().CollectArmed(address, address + bytes, written, context);
|
||||
#elif defined(__APPLE__)
|
||||
if (bytes == 0 || address + bytes < address) return false;
|
||||
return Watch::Get().Collect(address, address + bytes, written, context);
|
||||
#else
|
||||
static_cast<void>(address);
|
||||
static_cast<void>(bytes);
|
||||
static_cast<void>(written);
|
||||
static_cast<void>(context);
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -935,6 +935,62 @@ static bool Written(const void* base, std::size_t bytes, PageRuns expected) {
|
|||
return false;
|
||||
}
|
||||
|
||||
static bool CollectArmedRuns(const void* base, std::size_t offset, std::size_t bytes, PageRuns& runs) {
|
||||
runs.clear();
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(base);
|
||||
std::pair<std::uintptr_t, PageRuns*> context{address, &runs};
|
||||
return GuestWriteWatch::GuestWriteWatchCollectArmed_nid_postfix(address + offset, bytes, [](void* context, std::uintptr_t begin, std::uintptr_t end) {
|
||||
auto& [origin, into] = *static_cast<std::pair<std::uintptr_t, PageRuns*>*>(context);
|
||||
if (!into->empty() && into->back().second == (begin - origin) / 4096) into->back().second = (end - origin) / 4096;
|
||||
else into->emplace_back((begin - origin) / 4096, (end - origin) / 4096);
|
||||
}, &context);
|
||||
}
|
||||
|
||||
static void CheckWriteWatchArmedCollect() {
|
||||
if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) return;
|
||||
constexpr std::size_t length = 0x100000;
|
||||
constexpr std::size_t small = 4096;
|
||||
void* mapping = nullptr;
|
||||
Require(sceKernelMapFlexibleMemory(&mapping, length, 3, 0) == 0);
|
||||
auto* bytes = static_cast<volatile unsigned char*>(mapping);
|
||||
PageRuns runs;
|
||||
Require(CollectRuns(mapping, 0, length / 2, runs) && runs == PageRuns{{0, length / small / 2}});
|
||||
bytes[5 * small] = 1;
|
||||
bytes[200 * small] = 1;
|
||||
Require(CollectArmedRuns(mapping, 0, length, runs) && runs == PageRuns{{5, 6}});
|
||||
Require(Written(mapping, length, {{length / small / 2, length / small}}));
|
||||
Require(Written(mapping, length, {}));
|
||||
Require(sceKernelMunmap(mapping, length) == 0);
|
||||
}
|
||||
|
||||
static bool CollectFreshRuns(const void* base, std::size_t offset, std::size_t bytes, PageRuns& runs) {
|
||||
runs.clear();
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(base);
|
||||
std::pair<std::uintptr_t, PageRuns*> context{address, &runs};
|
||||
return GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(address + offset, bytes, [](void* context, std::uintptr_t begin, std::uintptr_t end) {
|
||||
auto& [origin, into] = *static_cast<std::pair<std::uintptr_t, PageRuns*>*>(context);
|
||||
if (!into->empty() && into->back().second == (begin - origin) / 4096) into->back().second = (end - origin) / 4096;
|
||||
else into->emplace_back((begin - origin) / 4096, (end - origin) / 4096);
|
||||
}, &context);
|
||||
}
|
||||
|
||||
static void CheckWriteWatchFreshCollect() {
|
||||
if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) return;
|
||||
constexpr std::size_t length = 0x100000;
|
||||
constexpr std::size_t small = 4096;
|
||||
void* mapping = nullptr;
|
||||
Require(sceKernelMapFlexibleMemory(&mapping, length, 3, 0) == 0);
|
||||
auto* bytes = static_cast<volatile unsigned char*>(mapping);
|
||||
PageRuns runs;
|
||||
Require(CollectRuns(mapping, 0, length / 2, runs) && runs == PageRuns{{0, length / small / 2}});
|
||||
bytes[5 * small] = 1;
|
||||
Require(CollectFreshRuns(mapping, 0, length, runs) && runs == PageRuns{{length / small / 2, length / small}});
|
||||
Require(CollectFreshRuns(mapping, 0, length, runs) && runs.empty());
|
||||
bytes[200 * small] = 1;
|
||||
Require(Written(mapping, length, {{5, 6}, {200, 201}}));
|
||||
Require(sceKernelMunmap(mapping, length) == 0);
|
||||
}
|
||||
|
||||
static void CheckWriteWatch() {
|
||||
if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) {
|
||||
std::puts("write watch unavailable: not tested");
|
||||
|
|
@ -1274,6 +1330,8 @@ int main() {
|
|||
CheckFailedCollectKeepsWrites();
|
||||
#if defined(__linux__)
|
||||
CheckWriteWatch();
|
||||
CheckWriteWatchArmedCollect();
|
||||
CheckWriteWatchFreshCollect();
|
||||
CheckDirectMemoryWriteWatch();
|
||||
CheckDirectMemoryBackingNeedsNoFilesystem();
|
||||
CheckDirectMemorySharedBacking();
|
||||
|
|
|
|||
Loading…
Reference in New Issue