diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index c522b689c..d13886926 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,10 @@ void WriteChanged(std::uint64_t address, std::span current, std // resources sharing pages see each other's writes. GPU writes into host-imported memory bypass the // page tables and are reported with MarkWritten. Without write watching CollectWrites returns 0 and // UnchangedSince is always false, so callers fall back to comparing bytes. +bool WriteWatched(); +void Unwatch(std::uint64_t address, std::size_t bytes); +bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function& import); +bool Watched(std::uint64_t address, std::size_t bytes); std::uint64_t CollectWrites(std::uint64_t address, std::size_t bytes); bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t generation); // UnchangedSince for several ranges under one tracker lock: true only when every one holds. @@ -64,6 +69,7 @@ struct UnchangedQuery { std::size_t bytes; std::uint64_t generation; }; +bool WrittenSince(std::uint64_t address, std::size_t bytes, std::uint64_t generation); bool UnchangedSinceAll(std::span queries); // Returns the generation the blocks were stamped with (0 when the arena is not write-watched): // UnchangedSince(range, it) holds until the next store over the range. @@ -102,7 +108,10 @@ bool UnchangedSinceCollected(std::uint64_t address, std::size_t bytes, std::uint // UnchangedSinceCollected tells them apart from the driver's MarkWritten stamps, but without its // walk: the caller collected the range first). One tracker lock for the whole range. Outside the // watched arena, or for a generation of 0, a block reads as changed. -void ChangedBlocks(std::uint64_t address, std::size_t bytes, std::span generations, std::span changed, std::span cpu = {}); +constexpr std::uint8_t BlockUnchanged = 0; +constexpr std::uint8_t BlockWritten = 1; +constexpr std::uint8_t BlockMaybeWritten = 2; +bool ChangedBlocks(std::uint64_t address, std::size_t bytes, std::span generations, std::span changed, std::span cpu = {}); // Serializes device work: draws, dispatches, presentation and the deferred write-backs below. The // mutex is recursive; it is wrapped so every acquisition (std::lock_guard at any site) measures how diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index 354e53311..d8ca9bad0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -1,15 +1,18 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" #include #include #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -60,9 +63,13 @@ std::atomic forgetSerial{0}; std::atomic forgetBytes{0}; std::atomic collectMemoHits{0}; std::atomic collectEpochBumps{0}; +std::atomic unwatchSerial{0}; // Walks that reported at least one written page (the ones the probe pass used to double), and the // tracker mutex acquisitions that had to wait (APS5_PROFILE_DRAW; see lockTracker). std::atomic collectDirty{0}; +#ifndef _WIN32 +std::atomic collectDirtyRuns{0}; +#endif std::atomic trackerWaits{0}; std::atomic trackerAcquisitions{0}; std::uintptr_t PagesBase(); @@ -619,10 +626,13 @@ namespace { constexpr std::size_t WriteBlockBytes = 65536; +enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow }; + struct WriteTracker { std::mutex mutex; bool initialized = false; bool watched = false; +#ifdef _WIN32 std::uintptr_t base = 0; std::size_t size = 0; // Generation of the last collected write per 64 KiB block of the arena. @@ -631,6 +641,17 @@ struct WriteTracker { // late label rule asks whether the CPU touched a label's block, and the driver's own GPU label // records in the same block (the title's per-job slots are 0x20 apart) must not count. std::vector cpuBlocks; + std::vector writtenBlocks; +#else + static constexpr std::size_t LeafBlocks = std::size_t{1} << 16; + static constexpr std::size_t LeafCount = std::size_t{1} << 15; + struct Leaf { + std::array blocks{}; + std::array cpuBlocks{}; + std::array writtenBlocks{}; + }; + std::vector> leaves; +#endif // Atomic only so a per-thread memo hit can read it without the mutex (every mutation and every // stamp still happen under it): a hit must return the current value, including MarkWritten // bumps, or an image refreshed after a stamp would keep failing UnchangedSince until the next @@ -649,6 +670,7 @@ struct WriteTracker { std::uintptr_t begin = 0; std::uintptr_t end = 0; std::uint64_t epoch = 0; + std::uint64_t unwatched = 0; }; std::array memo{}; std::size_t nextMemo = 0; @@ -656,12 +678,77 @@ struct WriteTracker { void initialize() { if (initialized) return; initialized = true; +#ifdef _WIN32 GuestArena::GuestArenaRange_nid_postfix(&base, &size); watched = size != 0 && GuestArena::GuestArenaWriteWatched_nid_postfix(); if (!watched) return; blocks.assign(size / WriteBlockBytes + 1, 0); cpuBlocks.assign(size / WriteBlockBytes + 1, 0); + writtenBlocks.assign(size / WriteBlockBytes + 1, 0); pages.resize(1u << 16); +#else + watched = GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix(); + if (watched) leaves.resize(LeafCount); +#endif + } + + bool covers(std::uint64_t address, std::size_t bytes) const { +#ifdef _WIN32 + return address >= base && address - base <= size - bytes; +#else + return address + bytes <= LeafCount * LeafBlocks * WriteBlockBytes && GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(static_cast(address), bytes); +#endif + } + + std::uint64_t blockOf(std::uint64_t address) const { +#ifdef _WIN32 + return (address - base) / WriteBlockBytes; +#else + return address / WriteBlockBytes; +#endif + } + + std::uint32_t stampOf(std::uint64_t block) const { +#ifdef _WIN32 + return blocks[block]; +#else + const auto& leaf = leaves[block / LeafBlocks]; + return leaf != nullptr ? leaf->blocks[block % LeafBlocks] : 0; +#endif + } + + std::uint32_t cpuStampOf(std::uint64_t block) const { +#ifdef _WIN32 + return cpuBlocks[block]; +#else + const auto& leaf = leaves[block / LeafBlocks]; + return leaf != nullptr ? leaf->cpuBlocks[block % LeafBlocks] : 0; +#endif + } + + std::uint32_t writtenStampOf(std::uint64_t block) const { +#ifdef _WIN32 + return writtenBlocks[block]; +#else + const auto& leaf = leaves[block / LeafBlocks]; + return leaf != nullptr ? leaf->writtenBlocks[block % LeafBlocks] : 0; +#endif + } + + void stamp(std::uint64_t block, std::uint32_t stampGeneration, StampKind kind) { + const bool cpu = kind != StampKind::Driver; + const bool written = kind != StampKind::ImportWindow; +#ifdef _WIN32 + blocks[block] = stampGeneration; + if (cpu) cpuBlocks[block] = stampGeneration; + if (written) writtenBlocks[block] = stampGeneration; +#else + auto& leaf = leaves[block / LeafBlocks]; + if (leaf == nullptr) leaf = std::make_unique(); + leaf->blocks[block % LeafBlocks] = stampGeneration; + if (cpu) leaf->cpuBlocks[block % LeafBlocks] = stampGeneration; + if (written) leaf->writtenBlocks[block % LeafBlocks] = stampGeneration; +#endif } }; @@ -717,9 +804,69 @@ bool sharedCollectMemo() { return shared; } +#ifndef _WIN32 +struct StampRuns { + WriteTracker& tracker; + StampKind kind; +}; + +void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) { + auto& [tracker, kind] = *static_cast(context); + if (end <= begin) return; + const auto generation = tracker.generation.load(std::memory_order_relaxed); + for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, kind); + collectDirtyRuns.fetch_add(1, std::memory_order_relaxed); +} +#endif + +bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind) { + ++tracker.generation; +#ifdef _WIN32 + constexpr std::uint64_t page = 4096; + // One resetting walk: the kernel reports and clears a page's dirty bit together, a write landing + // after the walk passed a page is reported by the next walk, and a clean page is not touched by the + // reset, so a probe pass first (which doubled the walk of every dirty range) buys nothing. + // APS5_NO_SINGLE_PASS_COLLECT=1 restores the probe pass followed by a resetting pass when dirty. + static const bool singlePass = std::getenv("APS5_NO_SINGLE_PASS_COLLECT") == nullptr; + auto cursor = first; + bool dirty = false; + while (cursor < stop) { + std::size_t count = tracker.pages.size(); + DWORD granularity = 4096; + if (!GuestArena::GuestArenaCollectWrites_nid_postfix(cursor, static_cast(stop - cursor), tracker.pages.data(), &count, singlePass)) { + // Uncommitted pages inside the range make the call fail; such ranges are compared instead. + return false; + } + if (count != 0) dirty = true; + for (ULONG_PTR i = 0; i < count; ++i) tracker.stamp(tracker.blockOf(reinterpret_cast(tracker.pages[i])), tracker.generation, kind); + if (count < tracker.pages.size()) break; + cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); + } + if (dirty) collectDirty.fetch_add(1, std::memory_order_relaxed); + if (dirty && !singlePass) { + // The resetting pass reports pages again, so a write racing the first pass is stamped too. + cursor = first; + while (cursor < stop) { + std::size_t count = tracker.pages.size(); + DWORD granularity = 4096; + if (!GuestArena::GuestArenaCollectWrites_nid_postfix(cursor, static_cast(stop - cursor), tracker.pages.data(), &count, true)) return false; + for (ULONG_PTR i = 0; i < count; ++i) tracker.stamp(tracker.blockOf(reinterpret_cast(tracker.pages[i])), tracker.generation, kind); + if (count < tracker.pages.size()) break; + cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); + } + } + return true; +#else + const auto dirtyRuns = collectDirtyRuns.load(std::memory_order_relaxed); + StampRuns runs{tracker, kind}; + const bool complete = GuestWriteWatch::GuestWriteWatchCollect_nid_postfix(static_cast(first), static_cast(stop - first), &stampWrittenRun, &runs); + if (collectDirtyRuns.load(std::memory_order_relaxed) != dirtyRuns) collectDirty.fetch_add(1, std::memory_order_relaxed); + return complete; +#endif +} + std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoized) { auto& tracker = Tracker(); -#ifdef _WIN32 // Whole pages, so a page shared with the next range is collected with either. constexpr std::uint64_t page = 4096; const auto first = address & ~(page - 1); @@ -727,12 +874,13 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi // One epoch for the lookup and the entry made after the walk: an unbumped thread's fresh epoch // must not differ between them. const auto epoch = currentCollectEpoch(); + const auto unwatched = unwatchSerial.load(std::memory_order_acquire); const bool useMemo = memoized && collectMemoEnabled() && bytes != 0; if (useMemo && !sharedCollectMemo()) { // An entry exists only for a completed walk of an in-arena range, so the tracker is // initialized and watched; nothing below the lock needs asking. for (const auto& entry : threadCollectMemo.entries) { - if (entry.epoch == epoch && entry.begin <= first && stop <= entry.end) { + if (entry.epoch == epoch && entry.unwatched == unwatched && entry.begin <= first && stop <= entry.end) { // The current generation, not the memoized one: blocks stamped since (MarkWritten, an // overlapping collect) were written before this caller reads, and an older value would // make every later UnchangedSince fail until the epoch ends. @@ -741,72 +889,27 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi } } } -#endif const auto lock = lockTracker(tracker); tracker.initialize(); - if (!tracker.watched || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return 0; -#ifdef _WIN32 + if (!tracker.watched || bytes == 0 || bytes > std::numeric_limits::max() - address || !tracker.covers(address, bytes)) return 0; auto cursor = first; if (useMemo && sharedCollectMemo()) { for (const auto& entry : tracker.memo) { - if (entry.epoch == epoch && entry.begin <= cursor && stop <= entry.end) { + if (entry.epoch == epoch && entry.unwatched == unwatchSerial.load(std::memory_order_relaxed) && entry.begin <= cursor && stop <= entry.end) { collectMemoHits.fetch_add(1, std::memory_order_relaxed); return tracker.generation; } } } const TimedAccess timed(CounterCollect, bytes); - ++tracker.generation; - // One resetting walk: the kernel reports and clears a page's dirty bit together, a write landing - // after the walk passed a page is reported by the next walk, and a clean page is not touched by the - // reset, so a probe pass first (which doubled the walk of every dirty range) buys nothing. - // APS5_NO_SINGLE_PASS_COLLECT=1 restores the probe pass followed by a resetting pass when dirty. - static const bool singlePass = std::getenv("APS5_NO_SINGLE_PASS_COLLECT") == nullptr; - bool dirty = false; - while (cursor < stop) { - std::size_t count = tracker.pages.size(); - DWORD granularity = 4096; - if (!GuestArena::GuestArenaCollectWrites_nid_postfix(cursor, static_cast(stop - cursor), tracker.pages.data(), &count, singlePass)) { - // Uncommitted pages inside the range make the call fail; such ranges are compared instead. - return 0; - } - if (count != 0) dirty = true; - for (ULONG_PTR i = 0; i < count; ++i) { - const auto block = (reinterpret_cast(tracker.pages[i]) - tracker.base) / WriteBlockBytes; - tracker.blocks[block] = tracker.generation; - tracker.cpuBlocks[block] = tracker.generation; - } - if (count < tracker.pages.size()) break; - cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); - } - if (dirty) collectDirty.fetch_add(1, std::memory_order_relaxed); - if (dirty && !singlePass) { - // The resetting pass reports pages again, so a write racing the first pass is stamped too. - cursor = address & ~(page - 1); - while (cursor < stop) { - std::size_t count = tracker.pages.size(); - DWORD granularity = 4096; - if (!GuestArena::GuestArenaCollectWrites_nid_postfix(cursor, static_cast(stop - cursor), tracker.pages.data(), &count, true)) return 0; - for (ULONG_PTR i = 0; i < count; ++i) { - const auto block = (reinterpret_cast(tracker.pages[i]) - tracker.base) / WriteBlockBytes; - tracker.blocks[block] = tracker.generation; - tracker.cpuBlocks[block] = tracker.generation; - } - if (count < tracker.pages.size()) break; - cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); - } - } + if (!walkWrites(tracker, first, stop, StampKind::Cpu)) return 0; // Only a completed walk is remembered; a failed one (uncommitted pages) returned 0 above. if (collectMemoEnabled()) { - if (sharedCollectMemo()) tracker.memo[tracker.nextMemo++ % tracker.memo.size()] = {first, stop, epoch}; - else threadCollectMemo.entries[threadCollectMemo.next++ % threadCollectMemo.entries.size()] = {first, stop, epoch}; + const auto serial = unwatchSerial.load(std::memory_order_relaxed); + if (sharedCollectMemo()) tracker.memo[tracker.nextMemo++ % tracker.memo.size()] = {first, stop, epoch, serial}; + else threadCollectMemo.entries[threadCollectMemo.next++ % threadCollectMemo.entries.size()] = {first, stop, epoch, serial}; } return tracker.generation; -#else - static_cast(memoized); - static_cast(threadCollectMemo); - return 0; -#endif } } @@ -828,15 +931,70 @@ std::uint64_t CollectEpochBumps() { return collectEpochBumps.load(std::memory_order_relaxed); } +namespace { + +void unwatchLocked(const WriteTracker& tracker, std::uint64_t address, std::size_t bytes) { + if (!tracker.watched) return; +#ifdef _WIN32 + static_cast(address); + static_cast(bytes); +#else + if (GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(address), bytes)) unwatchSerial.fetch_add(1, std::memory_order_release); +#endif +} + +} + +void Unwatch(std::uint64_t address, std::size_t bytes) { + if (bytes == 0 || bytes > std::numeric_limits::max() - address) return; + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + tracker.initialize(); + unwatchLocked(tracker, address, bytes); +} + +bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function& import) { + if (bytes == 0 || bytes > std::numeric_limits::max() - address) return import(); + auto& tracker = Tracker(); + auto lock = lockTracker(tracker); + tracker.initialize(); + if (!tracker.watched) { + lock.unlock(); + return import(); + } + constexpr std::uint64_t page = 4096; + const auto first = address & ~(page - 1); + const auto stop = (address + bytes + page - 1) & ~(page - 1); + const bool covered = tracker.covers(first, static_cast(stop - first)); + const bool before = covered && walkWrites(tracker, first, stop, StampKind::Cpu); + if (!import()) return false; + if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow)) unwatchLocked(tracker, first, static_cast(stop - first)); + return true; +} + +bool WriteWatched() { + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + tracker.initialize(); + return tracker.watched; +} + +bool Watched(std::uint64_t address, std::size_t bytes) { + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + tracker.initialize(); + return tracker.watched && bytes != 0 && bytes <= std::numeric_limits::max() - address && tracker.covers(address, bytes); +} + bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t generation) { auto& tracker = Tracker(); const auto lock = lockTracker(tracker); tracker.initialize(); - if (!tracker.watched || generation == 0 || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return false; - const auto first = (address - tracker.base) / WriteBlockBytes; - const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + if (!tracker.watched || generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false; + const auto first = tracker.blockOf(address); + const auto last = tracker.blockOf(address + bytes - 1); for (auto block = first; block <= last; ++block) { - if (tracker.blocks[block] > generation) return false; + if (tracker.stampOf(block) > generation) return false; } return true; } @@ -847,11 +1005,11 @@ bool UnchangedSinceAll(std::span queries) { tracker.initialize(); if (!tracker.watched) return false; for (const auto& [address, bytes, generation] : queries) { - if (generation == 0 || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return false; - const auto first = (address - tracker.base) / WriteBlockBytes; - const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + if (generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false; + const auto first = tracker.blockOf(address); + const auto last = tracker.blockOf(address + bytes - 1); for (auto block = first; block <= last; ++block) { - if (tracker.blocks[block] > generation) return false; + if (tracker.stampOf(block) > generation) return false; } } return true; @@ -861,11 +1019,11 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) { auto& tracker = Tracker(); const auto lock = lockTracker(tracker); tracker.initialize(); - if (!tracker.watched || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return 0; - const auto first = (address - tracker.base) / WriteBlockBytes; - const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + if (!tracker.watched || bytes == 0 || !tracker.covers(address, bytes)) return 0; + const auto first = tracker.blockOf(address); + const auto last = tracker.blockOf(address + bytes - 1); ++tracker.generation; - for (auto block = first; block <= last; ++block) tracker.blocks[block] = tracker.generation; + for (auto block = first; block <= last; ++block) tracker.stamp(block, tracker.generation, StampKind::Driver); return tracker.generation; } @@ -881,31 +1039,45 @@ bool UnchangedSinceCollected(std::uint64_t address, std::size_t bytes, std::uint if (CollectWritesUncached(address, bytes) == 0) return false; auto& tracker = Tracker(); const auto lock = lockTracker(tracker); - if (!tracker.watched || generation == 0 || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return false; - const auto first = (address - tracker.base) / WriteBlockBytes; - const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + if (!tracker.watched || generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false; + const auto first = tracker.blockOf(address); + const auto last = tracker.blockOf(address + bytes - 1); for (auto block = first; block <= last; ++block) { - if (tracker.cpuBlocks[block] > generation) return false; + if (tracker.cpuStampOf(block) > generation) return false; } return true; } -void ChangedBlocks(std::uint64_t address, std::size_t bytes, std::span generations, std::span changed, std::span cpu) { - std::fill(changed.begin(), changed.end(), std::uint8_t{1}); +bool WrittenSince(std::uint64_t address, std::size_t bytes, std::uint64_t generation) { + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + tracker.initialize(); + if (!tracker.watched || generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false; + const auto first = tracker.blockOf(address); + const auto last = tracker.blockOf(address + bytes - 1); + for (auto block = first; block <= last; ++block) { + if (tracker.writtenStampOf(block) > generation) return true; + } + return false; +} + +bool ChangedBlocks(std::uint64_t address, std::size_t bytes, std::span generations, std::span changed, std::span cpu) { + std::fill(changed.begin(), changed.end(), BlockWritten); std::fill(cpu.begin(), cpu.end(), std::uint8_t{1}); auto& tracker = Tracker(); const auto lock = lockTracker(tracker); tracker.initialize(); - if (!tracker.watched || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return; - const auto first = (address - tracker.base) / WriteBlockBytes; - const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + if (!tracker.watched || bytes == 0 || !tracker.covers(address, bytes)) return false; + const auto first = tracker.blockOf(address); + const auto last = tracker.blockOf(address + bytes - 1); for (auto block = first; block <= last; ++block) { const auto k = block - first; if (k >= generations.size() || k >= changed.size()) break; const auto generation = generations[k]; - changed[k] = generation == 0 || tracker.blocks[block] > generation ? 1 : 0; - if (k < cpu.size()) cpu[k] = generation == 0 || tracker.cpuBlocks[block] > generation ? 1 : 0; + changed[k] = generation == 0 || tracker.writtenStampOf(block) > generation ? BlockWritten : tracker.stampOf(block) > generation ? BlockMaybeWritten : BlockUnchanged; + if (k < cpu.size()) cpu[k] = generation == 0 || tracker.cpuStampOf(block) > generation ? 1 : 0; } + return true; } namespace { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 662b45c2d..2509d238c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -881,6 +881,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; poolInfo.queueFamilyIndex = family; check(state->DeviceFunction("vkCreateCommandPool")(state->device, &poolInfo, nullptr, &state->pool), "vkCreateCommandPool"); + Graphics::PrepareImportWatch(graphicsContext()); state->bufferPool = std::make_shared(graphicsContext()); state->emptyBuffer = std::make_unique(graphicsContext(), Graphics::EmptyBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); state->pipelineCache = std::make_unique(graphicsContext(), state->properties); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index 1256823fe..c2e662d7d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -28,8 +28,23 @@ struct HostImport { VkDeviceAddress address; // Identity for the life of this import (see HostImportSerial); 0 until first asked for. std::uint64_t serial = 0; + bool unwatched = false; }; +enum class ImportWatch : std::uint8_t { Watch, Unwatch }; + +struct ImportProbe { + const char* failure = nullptr; + VkResult result = VK_SUCCESS; + std::uint32_t pages = 0; + std::uint32_t writtenAtImport = 0; + std::uint32_t writtenAfterSubmit = 0; +}; + +ImportProbe ProbeImportWriteProtection(const Context& context); +ImportWatch PrepareImportWatch(const Context& context); +void SetImportWatch(const Context& context, ImportWatch watch); + // The host import of the registered allocation containing [address, address + bytes), made on demand // (alignment and budget permitting), or null. Bytes at `address` are at `address - import->base` in // the import's buffer. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 8b8846c77..c46db689d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -6,8 +6,11 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" #include "prx/libc/include/CpuTopology.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" #ifdef _WIN32 #include +#else +#include #endif #include #include @@ -118,6 +121,8 @@ struct HostImports { // Bumped whenever an import is dropped, so HostImport pointers taken under the lock earlier can be // reused while it is unchanged. std::uint64_t epoch = 1; + VkDevice watchDevice = VK_NULL_HANDLE; + bool unwatchImports = false; }; HostImports& Imports() { @@ -157,6 +162,88 @@ void retireImport(const Context& context, HostImports& state, std::map const char* { + if (entry.buffer) context.Function("vkDestroyBuffer")(context.device, entry.buffer, nullptr); + if (entry.memory) context.Function("vkFreeMemory")(context.device, entry.memory, nullptr); + entry.buffer = VK_NULL_HANDLE; + entry.memory = VK_NULL_HANDLE; + failure = result; + return step; + }; + const VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, nullptr, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT}; + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external}; + info.size = bytes; + // INDIRECT_BUFFER: DISPATCH_INDIRECT group counts are read in place (VulkanDevice::DispatchIndirect). + info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + if (const auto result = context.Function("vkCreateBuffer")(context.device, &info, nullptr, &entry.buffer); result != VK_SUCCESS) return failed("vkCreateBuffer", result); + VkMemoryHostPointerPropertiesEXT pointer{VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT}; + if (const auto result = context.Function("vkGetMemoryHostPointerPropertiesEXT")(context.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, reinterpret_cast(base), &pointer); result != VK_SUCCESS) return failed("vkGetMemoryHostPointerPropertiesEXT", result); + VkMemoryRequirements requirements{}; + context.Function("vkGetBufferMemoryRequirements")(context.device, entry.buffer, &requirements); + const auto types = requirements.memoryTypeBits & pointer.memoryTypeBits; + if (types == 0) return failed("memory type selection", VK_ERROR_FORMAT_NOT_SUPPORTED); + const VkImportMemoryHostPointerInfoEXT import{VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, nullptr, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, reinterpret_cast(base)}; + const VkMemoryAllocateFlagsInfo flags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, &import, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0}; + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &flags}; + allocation.allocationSize = bytes; + allocation.memoryTypeIndex = static_cast(std::countr_zero(types)); + if (const auto result = context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &entry.memory); result != VK_SUCCESS) return failed("vkAllocateMemory", result); + if (const auto result = context.Function("vkBindBufferMemory")(context.device, entry.buffer, entry.memory, 0); result != VK_SUCCESS) return failed("vkBindBufferMemory", result); + const VkBufferDeviceAddressInfo addressInfo{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, entry.buffer}; + entry.address = context.Function("vkGetBufferDeviceAddressKHR")(context.device, &addressInfo); + if (entry.address == 0) return failed("vkGetBufferDeviceAddressKHR", VK_ERROR_UNKNOWN); + return nullptr; +} + +#ifndef _WIN32 +enum class ImportWatchRequest : std::uint8_t { Probe, Watch, Unwatch }; + +ImportWatchRequest importWatchRequest() { + static const auto request = [] { + const char* value = std::getenv("APS5_WRITE_WATCH_IMPORTS"); + if (value == nullptr || std::strcmp(value, "probe") == 0) return ImportWatchRequest::Probe; + if (std::strcmp(value, "watch") == 0) return ImportWatchRequest::Watch; + if (std::strcmp(value, "unwatch") == 0) return ImportWatchRequest::Unwatch; + throw std::runtime_error(std::string("APS5_WRITE_WATCH_IMPORTS=") + value + ": expected probe, watch or unwatch"); + }(); + return request; +} +#endif + +void decideImportWatch(const Context& context, HostImports& state) { + if (state.watchDevice == context.device) return; +#ifdef _WIN32 + state.watchDevice = context.device; + state.unwatchImports = false; +#else + const auto request = importWatchRequest(); + state.watchDevice = context.device; + state.unwatchImports = false; + if (context.hostImportAlignment == 0 || !GuestMemory::WriteWatched()) return; + if (request == ImportWatchRequest::Watch) { + std::fprintf(stderr, "[write-watch] host imports stay watched (APS5_WRITE_WATCH_IMPORTS=watch)\n"); + return; + } + if (request == ImportWatchRequest::Unwatch) { + state.unwatchImports = true; + std::fprintf(stderr, "[write-watch] host imports are compared, not watched (APS5_WRITE_WATCH_IMPORTS=unwatch)\n"); + return; + } + const auto probe = ProbeImportWriteProtection(context); + if (probe.failure != nullptr) { + state.unwatchImports = true; + std::fprintf(stderr, "[write-watch] host imports resolve write protection: unknown (probe failed at %s, %d); imported ranges are compared\n", probe.failure, static_cast(probe.result)); + return; + } + state.unwatchImports = probe.writtenAfterSubmit != 0; + std::fprintf(stderr, "[write-watch] host imports resolve write protection: %s (%u of %u scratch pages written after a GPU read, %u after the import); imported ranges %s\n", state.unwatchImports ? "yes" : "no", probe.writtenAfterSubmit, probe.pages, probe.writtenAtImport, state.unwatchImports ? "are compared" : "stay watched"); +#endif +} + const HostImport* importAllocation(const Context& context, HostImports& state, std::uint64_t base, std::uint64_t bytes, const GuestAllocations::Lease& lease) { if (const auto found = state.imports.find(base); found != state.imports.end()) { if (found->second.bytes == bytes) return &found->second; @@ -196,10 +283,17 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s cursor = reinterpret_cast(info.BaseAddress) + info.RegionSize; } #endif + decideImportWatch(context, state); + if (state.unwatchImports) GuestMemory::Unwatch(base, bytes); HostImport entry{base, bytes, VK_NULL_HANDLE, VK_NULL_HANDLE, 0}; - const auto fail = [&](const char* step, VkResult result) -> const HostImport* { - if (entry.buffer) context.Function("vkDestroyBuffer")(context.device, entry.buffer, nullptr); - if (entry.memory) context.Function("vkFreeMemory")(context.device, entry.memory, nullptr); + entry.unwatched = state.unwatchImports; + VkResult result = VK_SUCCESS; + const char* step = nullptr; + GuestMemory::ImportWatched(base, bytes, [&] { + step = createImport(context, entry, result); + return step == nullptr; + }); + if (step != nullptr) { state.failed.insert(base); std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx failed at %s (%d); falling back to copies\n", static_cast(base), static_cast(bytes), step, static_cast(result)); #ifdef _WIN32 @@ -213,30 +307,7 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s } #endif return nullptr; - }; - const VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, nullptr, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT}; - VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external}; - info.size = bytes; - // INDIRECT_BUFFER: DISPATCH_INDIRECT group counts are read in place (VulkanDevice::DispatchIndirect). - info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; - info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - if (const auto result = context.Function("vkCreateBuffer")(context.device, &info, nullptr, &entry.buffer); result != VK_SUCCESS) return fail("vkCreateBuffer", result); - VkMemoryHostPointerPropertiesEXT pointer{VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT}; - if (const auto result = context.Function("vkGetMemoryHostPointerPropertiesEXT")(context.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, reinterpret_cast(base), &pointer); result != VK_SUCCESS) return fail("vkGetMemoryHostPointerPropertiesEXT", result); - VkMemoryRequirements requirements{}; - context.Function("vkGetBufferMemoryRequirements")(context.device, entry.buffer, &requirements); - const auto types = requirements.memoryTypeBits & pointer.memoryTypeBits; - if (types == 0) return fail("memory type selection", VK_ERROR_FORMAT_NOT_SUPPORTED); - const VkImportMemoryHostPointerInfoEXT import{VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, nullptr, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, reinterpret_cast(base)}; - const VkMemoryAllocateFlagsInfo flags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, &import, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0}; - VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &flags}; - allocation.allocationSize = bytes; - allocation.memoryTypeIndex = static_cast(std::countr_zero(types)); - if (const auto result = context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &entry.memory); result != VK_SUCCESS) return fail("vkAllocateMemory", result); - if (const auto result = context.Function("vkBindBufferMemory")(context.device, entry.buffer, entry.memory, 0); result != VK_SUCCESS) return fail("vkBindBufferMemory", result); - const VkBufferDeviceAddressInfo addressInfo{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, entry.buffer}; - entry.address = context.Function("vkGetBufferDeviceAddressKHR")(context.device, &addressInfo); - if (entry.address == 0) return fail("vkGetBufferDeviceAddressKHR", VK_ERROR_UNKNOWN); + } static std::uint64_t importedBytes = 0; importedBytes += bytes; static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr; @@ -276,6 +347,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc for (auto it = state.imports.begin(); it != state.imports.end();) { const auto* range = leasedRangeAt(lease, it->first); if (range != nullptr && range->bytes == it->second.bytes) { + if (it->second.unwatched) GuestMemory::Unwatch(it->first, it->second.bytes); ++it; continue; } @@ -886,6 +958,114 @@ MirrorStats MirrorCounters() { return stats; } +ImportProbe ProbeImportWriteProtection(const Context& context) { + ImportProbe probe; +#ifdef _WIN32 + static_cast(context); + probe.failure = "the Linux write watch"; + return probe; +#else + if (context.hostImportAlignment == 0) { + probe.failure = "host import support"; + return probe; + } + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { + probe.failure = "the write watch"; + return probe; + } + constexpr std::uint64_t page = 4096; + const std::uint64_t alignment = std::max(context.hostImportAlignment, page); + const std::uint64_t bytes = (4 * page + alignment - 1) / alignment * alignment; + probe.pages = static_cast(bytes / page); + void* raw = mmap(nullptr, bytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) { + probe.failure = "mmap"; + return probe; + } + const auto base = (reinterpret_cast(raw) + alignment - 1) / alignment * alignment; + auto* scratch = reinterpret_cast(base); + for (std::uint64_t offset = 0; offset < bytes; offset += page) scratch[offset] = 1; + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(base), bytes); + HostImport import{base, bytes, VK_NULL_HANDLE, VK_NULL_HANDLE, 0}; + VkBuffer destination = VK_NULL_HANDLE; + VkDeviceMemory destinationMemory = VK_NULL_HANDLE; + VkCommandBuffer commands = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + bool submitted = false; + const auto collect = [&](std::uint32_t& pages) { + pages = 0; + return GuestWriteWatch::GuestWriteWatchCollect_nid_postfix(static_cast(base), static_cast(bytes), [](void* count, std::uintptr_t begin, std::uintptr_t end) { *static_cast(count) += static_cast((end - begin) / 4096); }, &pages); + }; + const auto run = [&]() -> const char* { + std::uint32_t quiet = 0; + if (!collect(quiet)) return "the first collect"; + if (!collect(quiet)) return "the second collect"; + if (quiet != 0) return "an unwritten scratch range"; + if (const char* step = createImport(context, import, probe.result)) return step; + if (!collect(probe.writtenAtImport)) return "the collect after the import"; + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + info.size = bytes; + info.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + if ((probe.result = context.Function("vkCreateBuffer")(context.device, &info, nullptr, &destination)) != VK_SUCCESS) return "vkCreateBuffer"; + VkMemoryRequirements requirements{}; + context.Function("vkGetBufferMemoryRequirements")(context.device, destination, &requirements); + if (requirements.memoryTypeBits == 0) return "the destination memory type"; + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = static_cast(std::countr_zero(requirements.memoryTypeBits)); + if ((probe.result = context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &destinationMemory)) != VK_SUCCESS) return "vkAllocateMemory"; + if ((probe.result = context.Function("vkBindBufferMemory")(context.device, destination, destinationMemory, 0)) != VK_SUCCESS) return "vkBindBufferMemory"; + VkCommandBufferAllocateInfo allocate{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + allocate.commandPool = context.pool; + allocate.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocate.commandBufferCount = 1; + if ((probe.result = context.Function("vkAllocateCommandBuffers")(context.device, &allocate, &commands)) != VK_SUCCESS) return "vkAllocateCommandBuffers"; + VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + if ((probe.result = context.Function("vkBeginCommandBuffer")(commands, &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer"; + const VkBufferCopy region{0, 0, bytes}; + context.Function("vkCmdCopyBuffer")(commands, import.buffer, destination, 1, ®ion); + if ((probe.result = context.Function("vkEndCommandBuffer")(commands)) != VK_SUCCESS) return "vkEndCommandBuffer"; + const VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + if ((probe.result = context.Function("vkCreateFence")(context.device, &fenceInfo, nullptr, &fence)) != VK_SUCCESS) return "vkCreateFence"; + VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + submit.commandBufferCount = 1; + submit.pCommandBuffers = &commands; + if ((probe.result = context.Function("vkQueueSubmit")(context.queue, 1, &submit, fence)) != VK_SUCCESS) return "vkQueueSubmit"; + submitted = true; + if ((probe.result = context.Function("vkWaitForFences")(context.device, 1, &fence, VK_TRUE, 10'000'000'000ull)) != VK_SUCCESS) return "vkWaitForFences"; + submitted = false; + if (!collect(probe.writtenAfterSubmit)) return "the collect after the submission"; + return nullptr; + }; + probe.failure = run(); + if (submitted) context.Function("vkQueueWaitIdle")(context.queue); + if (fence != VK_NULL_HANDLE) context.Function("vkDestroyFence")(context.device, fence, nullptr); + if (commands != VK_NULL_HANDLE) context.Function("vkFreeCommandBuffers")(context.device, context.pool, 1, &commands); + if (destination != VK_NULL_HANDLE) context.Function("vkDestroyBuffer")(context.device, destination, nullptr); + if (destinationMemory != VK_NULL_HANDLE) context.Function("vkFreeMemory")(context.device, destinationMemory, nullptr); + if (import.buffer != VK_NULL_HANDLE) destroyImport(context, import); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(base), bytes); + munmap(raw, bytes + alignment); + return probe; +#endif +} + +ImportWatch PrepareImportWatch(const Context& context) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + decideImportWatch(context, state); + return state.unwatchImports ? ImportWatch::Unwatch : ImportWatch::Watch; +} + +void SetImportWatch(const Context& context, ImportWatch watch) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + state.watchDevice = context.device; + state.unwatchImports = watch == ImportWatch::Unwatch; +} + const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) { if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; auto& state = Imports(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 4115fce75..ce85f2fe7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -951,10 +951,10 @@ bool StorageTexture::Refresh() { if (alias != nullptr && !(ProvedClearKeys(descriptor, guestBytes, keyProof) == DccKeys::Uncompressed && HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) != nullptr)) alias = nullptr; if (alias != nullptr) { std::vector aliasStamped(trackedLayers); - GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), alias->layerGeneration, aliasStamped); - for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { + if (!GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), alias->layerGeneration, aliasStamped)) alias = nullptr; + for (std::uint32_t unit = 0; alias != nullptr && unit < trackedLayers; ++unit) { if (!alias->layerPending[unit]) continue; - if (alias->layerGeneration[unit] == 0 || (aliasStamped[unit] == 0 && layerPending[unit])) { + if (alias->layerGeneration[unit] == 0 || aliasStamped[unit] == GuestMemory::BlockMaybeWritten || (aliasStamped[unit] == GuestMemory::BlockUnchanged && layerPending[unit])) { alias = nullptr; break; } @@ -978,6 +978,7 @@ bool StorageTexture::Refresh() { bool anyBorrowed = false; bool unchanged = true; bool stamped = true; + bool tracked = true; bool keysChanged = false; bool cpuWrote = false; bool direct = false; @@ -992,7 +993,7 @@ bool StorageTexture::Refresh() { blockGenerations(generations); stampedBlocks.assign(generations.size(), 0); cpuBlocks.assign(generations.size(), 0); - GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), generations, stampedBlocks, cpuBlocks); + tracked = GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), generations, stampedBlocks, cpuBlocks); }; const auto keysStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; keys = ProvedClearKeys(descriptor, guestBytes, keyProof); @@ -1029,7 +1030,7 @@ bool StorageTexture::Refresh() { // "cpu" names a CPU store in both models; a block without a generation (untracked) says // nothing about who wrote it. for (std::size_t k = 0; k < cpuBlocks.size(); ++k) { - if (cpuBlocks[k] != 0 && generations[k] != 0) cpuWrote = true; + if (tracked && cpuBlocks[k] != 0 && generations[k] != 0) cpuWrote = true; } // Unchanged bytes rescue an upload only when nothing else moved: with the keys changed // the same bytes read differently, and a changed unit holding pending results must take @@ -1052,10 +1053,10 @@ bool StorageTexture::Refresh() { // already holding them at the same alias version stay as they are. const bool sameBorrow = borrowedFrom.lock() == alias && borrowedVersion == alias->version && borrowedUnits.size() == trackedLayers; std::vector aliasStamped(trackedLayers); - GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), alias->layerGeneration, aliasStamped); + const bool aliasTracked = GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), alias->layerGeneration, aliasStamped); borrowed.assign(trackedLayers, false); for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { - if (!alias->layerPending[unit] || aliasStamped[unit] != 0 || alias->layerGeneration[unit] == 0 || layerPending[unit]) continue; + if (!aliasTracked || !alias->layerPending[unit] || aliasStamped[unit] != 0 || alias->layerGeneration[unit] == 0 || layerPending[unit]) continue; if (sameBorrow && borrowedUnits[unit]) continue; borrowed[unit] = true; anyBorrowed = true; @@ -1099,7 +1100,7 @@ bool StorageTexture::Refresh() { // unit's stamps say nothing, so it is stored. const auto droppable = [&](std::uint32_t unit) { if (keysChanged && IsDccClear(keys)) return true; - return layerGeneration[unit] != 0 && unit < stampedBlocks.size() && stampedBlocks[unit] != 0; + return tracked && layerGeneration[unit] != 0 && unit < stampedBlocks.size() && stampedBlocks[unit] == GuestMemory::BlockWritten; }; // A clear code -> uncompressed flip on an image with results pending: an unstamped pending // unit's results ARE the uncompressed texels (nothing wrote its memory since), so it keeps @@ -1118,7 +1119,7 @@ bool StorageTexture::Refresh() { } for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { if (!layerPending[layer]) continue; - if (keyFlip && layerGeneration[layer] != 0 && stampedBlocks[layer] == 0) { + if (keyFlip && layerGeneration[layer] != 0 && stampedBlocks[layer] != GuestMemory::BlockWritten) { changed[layer] = false; keyFlipKept.fetch_add(1, std::memory_order_relaxed); if (direct) continue; @@ -2081,7 +2082,7 @@ bool StorageTexture::blocksKept(std::span> for (auto at = begin & ~(block - 1); at < end; at += block) { const auto from = std::max(at, begin); const auto to = std::min(at + block, end); - if (GuestMemory::UnchangedSince(from, static_cast(to - from), texture->layerGeneration[layer])) return false; + if (!GuestMemory::WrittenSince(from, static_cast(to - from), texture->layerGeneration[layer])) return false; } } } @@ -2177,7 +2178,7 @@ bool StorageTexture::evictStale(std::uint64_t address, std::size_t bytes, std::s const auto end = begin + layerBytes(layer); for (auto at = begin & ~(block - 1); at < end; at += block) { const auto from = std::max(at, begin); - if (GuestMemory::UnchangedSince(from, static_cast(std::min(at + block, end) - from), layerGeneration[layer])) return false; + if (!GuestMemory::WrittenSince(from, static_cast(std::min(at + block, end) - from), layerGeneration[layer])) return false; } } // As the empty write-back leaves the layers: no results pending, the generation unchanged (the @@ -2881,12 +2882,12 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { for (auto at = layerBegin(layer) & ~(block - 1); at < end; at += block) generations[static_cast((at - spanBegin) / block)] = layerGeneration[layer]; } std::vector changedBlocks(spanBlocks); - GuestMemory::ChangedBlocks(firstStored, static_cast(lastStored - firstStored), generations, changedBlocks); + const bool tracked = GuestMemory::ChangedBlocks(firstStored, static_cast(lastStored - firstStored), generations, changedBlocks); for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { if (!layers[layer]) continue; const auto begin = layerBegin(layer); const auto end = begin + layerBytes(layer); - if (layerGeneration[layer] == 0) { + if (!tracked || layerGeneration[layer] == 0) { keep.emplace_back(begin, end); continue; } @@ -2894,7 +2895,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { const auto from = std::max(at, begin); const auto to = std::min(at + block, end); if (from >= to) continue; - if (changedBlocks[static_cast((at - spanBegin) / block)] != 0) { + if (changedBlocks[static_cast((at - spanBegin) / block)] == GuestMemory::BlockWritten) { skippedAny = true; skippedLayer[layer] = true; ++skipped; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp index c93e1a47d..efa6a44bf 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp @@ -4,7 +4,6 @@ #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" -#include "prx/libc/include/GuestArena.hpp" #include #include #include @@ -15,7 +14,6 @@ #include #include #include -#include namespace AgcDriver::Graphics { @@ -314,7 +312,7 @@ std::size_t publishUnits(const std::shared_ptr& shadow, std::uint64_ for (auto unit = first; unit <= last; ++unit) { const auto k = static_cast(unit - first); if (!selected[k] || !shadow->Live(unit)) continue; - if (changed[k] != 0) { + if (changed[k] == GuestMemory::BlockWritten) { // Newer bytes reached the import since the retile: the slab's are dead. shadow->generation[unit] = 0; (cpu[k] != 0 ? Stats().staleDroppedCpu : Stats().staleDroppedDriver).fetch_add(1, std::memory_order_relaxed); @@ -418,7 +416,7 @@ bool evictOne(Shadows& registry) { } bool UnitShadowEnabled() { - static const bool enabled = std::getenv("APS5_NO_UNIT_SHADOW") == nullptr && GuestArena::GuestArenaWriteWatched_nid_postfix(); + static const bool enabled = std::getenv("APS5_NO_UNIT_SHADOW") == nullptr && GuestMemory::WriteWatched(); return enabled; } @@ -519,13 +517,7 @@ VkDeviceSize SlabOffset(const HostImport& import, const ShadowSlab& slab, std::u std::optional ShadowDestinationFor(const Context& context, const HostImport& import, std::uint64_t begin, std::uint64_t end) { if (!UnitShadowEnabled() || end <= begin || begin < import.base || end > import.base + import.bytes) return std::nullopt; - static const auto arena = [] { - std::uintptr_t base = 0; - std::size_t size = 0; - GuestArena::GuestArenaRange_nid_postfix(&base, &size); - return std::pair{base, size}; - }(); - if (arena.second == 0 || begin < arena.first || end > arena.first + arena.second) return std::nullopt; + if (!GuestMemory::Watched(begin, static_cast(end - begin))) return std::nullopt; auto& registry = Registry(); std::shared_ptr shadow; std::size_t slabIndex = 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 2e14fc548..5c2317cc8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -12,13 +12,17 @@ #include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" #include #ifdef _WIN32 #ifndef NOMINMAX #define NOMINMAX #endif #include +#else +#include #endif +#include #include #include #include @@ -26,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +40,35 @@ namespace { using namespace AgcDriver::Graphics; using AgcDriver::GuestMemory::GpuMutex; +void* AllocateWatched(std::size_t bytes, std::size_t alignment) { + if (!AgcDriver::GuestMemory::WriteWatched()) return nullptr; +#ifdef _WIN32 + void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, alignment); + GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes); +#else + void* raw = mmap(nullptr, bytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) throw std::runtime_error("cannot map the watched block"); + const auto begin = reinterpret_cast(raw); + const auto aligned = (begin + alignment - 1) & ~(static_cast(alignment) - 1); + if (aligned != begin) munmap(raw, aligned - begin); + if (aligned + bytes != begin + bytes + alignment) munmap(reinterpret_cast(aligned + bytes), begin + alignment - aligned); + void* block = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes); +#endif + if (!AgcDriver::GuestMemory::Watched(reinterpret_cast(block), bytes)) throw std::runtime_error("the watched block is not watched"); + return block; +} + +void ReleaseWatched(void* block, std::size_t bytes) { +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(block, bytes); + GuestArena::GuestArenaRelease_nid_postfix(block, bytes); +#else + munmap(block, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, bytes); +#endif +} + // A compute-capable device with host imports (VK_EXT_external_memory_host) when the host offers // them, as the driver creates its own; the recorder's batches need a real queue and real fences. class Device { @@ -339,24 +373,17 @@ void lateLabelTests(Recorder& recorder) { void unchangedSinceTests() { using namespace AgcDriver::GuestMemory; static std::uint32_t outside[16]; - Require(!UnchangedSinceCollected(reinterpret_cast(outside), 4, TrackerGeneration()), "(6) a range outside the arena is unchanged"); - if (!GuestArena::GuestArenaAvailable_nid_postfix()) { - std::cout << "guest arena unavailable: UnchangedSinceCollected not tested inside it\n"; + Require(!UnchangedSinceCollected(reinterpret_cast(outside), 4, TrackerGeneration()), "(6) a range outside the watched memory is unchanged"); + Require(!Watched(reinterpret_cast(outside), 4) && CollectWrites(reinterpret_cast(outside), 4) == 0, "(6) a range outside the watched memory is watched"); + constexpr std::size_t bytes = 65536; + void* block = AllocateWatched(bytes, bytes); + if (block == nullptr) { + std::cout << "no write watching: UnchangedSinceCollected is always false\n"; return; } - constexpr std::size_t bytes = 65536; - void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, bytes); -#ifdef _WIN32 - GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes); -#endif auto* words = static_cast(block); const auto address = reinterpret_cast(block); words[0] = 0; - if (!GuestArena::GuestArenaWriteWatched_nid_postfix()) { - Require(!UnchangedSinceCollected(address, 4, TrackerGeneration()), "(6) an unwatched arena reports unchanged"); - std::cout << "arena not write-watched: UnchangedSinceCollected is always false\n"; - return; - } CollectWritesUncached(address, 4); const auto generation = TrackerGeneration(); Require(UnchangedSinceCollected(address, 4, generation), "(6) an untouched range is not unchanged"); @@ -374,10 +401,7 @@ void unchangedSinceTests() { words[1] = 2; CollectWritesUncached(address, 4); Require(!UnchangedSinceCollected(address, 4, generation3), "(6) a CPU write collected by another caller is not seen"); -#ifdef _WIN32 - GuestArena::GuestArenaReset_nid_postfix(block, bytes); -#endif - GuestArena::GuestArenaRelease_nid_postfix(block, bytes); + ReleaseWatched(block, bytes); } // (9) ProvedClearKeys: a surface's DCC keys are scanned once and answered from the proof after, @@ -386,23 +410,15 @@ void unchangedSinceTests() { // scans and no proof is stored. void keyProofTests(const Device& device, Recorder& recorder) { using namespace AgcDriver::GuestMemory; - if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) { - std::cout << "guest arena unavailable or not write-watched: key proofs not tested\n"; + constexpr std::size_t bytes = 65536; + void* block = AllocateWatched(bytes, bytes); + if (block == nullptr) { + std::cout << "no write watching: key proofs not tested\n"; return; } - constexpr std::size_t bytes = 65536; - void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, bytes); -#ifdef _WIN32 - GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes); -#endif struct Release { void* block; - ~Release() { -#ifdef _WIN32 - GuestArena::GuestArenaReset_nid_postfix(block, bytes); -#endif - GuestArena::GuestArenaRelease_nid_postfix(block, bytes); - } + ~Release() { ReleaseWatched(block, bytes); } } release{block}; constexpr std::size_t keyCount = 1024; constexpr std::uint64_t surfaceBytes = keyCount * 256; @@ -476,12 +492,9 @@ void closeRaceTests(const Device& device, Recorder& recorder) { Require(!Recorder::LookupLabel(0x80000, 4, 70, &refusal).has_value() && refusal == Refusal::Unclosed, "(8) a submitted batch's entry was closed"); device.WaitQueue(); recorder.Sync(); - if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) return; constexpr std::size_t bytes = 1u << 20; - void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, 65536); -#ifdef _WIN32 - GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes); -#endif + void* block = AllocateWatched(bytes, 65536); + if (block == nullptr) return; const auto base = reinterpret_cast(block); // Late in the range, so the walker is usually still ahead of the page when the store lands. const auto label = base + 4096 * 250; @@ -513,10 +526,7 @@ void closeRaceTests(const Device& device, Recorder& recorder) { walker.thread.join(); recorder.Sync(); Require(!Recorder::LookupLabel(label, 4, 5).has_value(), "(8) the entry outlived its batch"); -#ifdef _WIN32 - GuestArena::GuestArenaReset_nid_postfix(block, bytes); -#endif - GuestArena::GuestArenaRelease_nid_postfix(block, bytes); + ReleaseWatched(block, bytes); } // A resource build over host-imported guest memory notes its in-place reads when its writes are @@ -713,29 +723,21 @@ void unitShadowTests(const Device& device, Recorder& recorder) { const auto& context = device.GetContext(); if (!UnitShadowEnabled()) { Require(!AnyShadowedOverlaps(0x10000, 16) && PublishShadow(0x10000, 16, PublishScope::Whole, PublishReason::Hook) == 0, "unit shadows are off but not inert"); - std::cout << "unit shadows off (APS5_NO_UNIT_SHADOW, or an arena without write watching): primitives inert\n"; + std::cout << "unit shadows off (APS5_NO_UNIT_SHADOW, or no write watching): primitives inert\n"; return; } - if (context.hostImportAlignment == 0 || !GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) { - std::cout << "host imports or the write-watched arena unavailable: unit shadows not tested\n"; + if (context.hostImportAlignment == 0) { + std::cout << "host imports unavailable: unit shadows not tested\n"; return; } constexpr std::uint64_t unit = 65536; // Three slabs at the default 8 MiB: units 0..127, 128..255, 256..271. constexpr std::size_t bytes = (17u << 20u); - void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, 65536); - Require(block != nullptr, "cannot allocate the shadow test block"); -#ifdef _WIN32 - GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes); -#endif + void* block = AllocateWatched(bytes, 65536); + Require(block != nullptr, "unit shadows are on without write watching"); struct Release { void* block; - ~Release() { -#ifdef _WIN32 - GuestArena::GuestArenaReset_nid_postfix(block, bytes); -#endif - GuestArena::GuestArenaRelease_nid_postfix(block, bytes); - } + ~Release() { ReleaseWatched(block, bytes); } } release{block}; std::memset(block, 0x11, bytes); const auto address = reinterpret_cast(block); @@ -757,6 +759,10 @@ void unitShadowTests(const Device& device, Recorder& recorder) { std::cout << "host import of the shadow test block refused: unit shadows not tested\n"; return; } + if (!Watched(address, bytes)) { + std::cout << "host imports are compared, not watched: unit shadows not tested\n"; + return; + } Require(import->base == address && import->bytes == bytes, "the import does not cover the block"); CollectWritesUncached(address, bytes); const auto* words = static_cast(block); @@ -897,7 +903,7 @@ void unitShadowTests(const Device& device, Recorder& recorder) { else Require(words[128 * unit] == 0x77, "the eviction before the retire did not publish unit 128"); } -void storageRefreshTests(const Device& device, Recorder& recorder) { +void storageRefreshTests(const Device& device, Recorder& recorder, bool watched) { const auto& base = device.GetContext(); if (base.hostImportAlignment == 0) { std::cout << "host imports unavailable: storage refresh not tested\n"; @@ -907,11 +913,20 @@ void storageRefreshTests(const Device& device, Recorder& recorder) { constexpr std::size_t surfaceBytes = side * side * 4; constexpr std::size_t keyCount = surfaceBytes / 256; constexpr std::size_t bytes = surfaceBytes + 65536; + void* block = nullptr; + if (watched) { + block = AllocateWatched(bytes, 65536); + if (block == nullptr) { + std::cout << "no write watching: storage refresh in watched memory not tested\n"; + return; + } + } else { #ifdef _WIN32 - void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); #else - void* block = std::aligned_alloc(65536, bytes); + block = std::aligned_alloc(65536, bytes); #endif + } Require(block != nullptr, "cannot allocate the storage refresh block"); auto* texels = static_cast(block); auto* keys = texels + surfaceBytes; @@ -939,8 +954,10 @@ void storageRefreshTests(const Device& device, Recorder& recorder) { std::cout << "host import of the storage refresh block refused: storage refresh not tested\n"; return; } - Require(!ShadowDestinationFor(base, *import, address, address + 65536).has_value(), "a unit shadow was offered for memory the write tracker does not watch"); - if (!GuestArena::GuestArenaWriteWatched_nid_postfix()) Require(!UnitShadowEnabled(), "unit shadows are on without write watching"); + if (watched && !AgcDriver::GuestMemory::Watched(address, bytes)) std::cout << "host imports are compared, not watched: storage refresh in watched memory runs as unwatched\n"; + watched = watched && AgcDriver::GuestMemory::Watched(address, bytes); + Require(watched || !ShadowDestinationFor(base, *import, address, address + 65536).has_value(), "a unit shadow was offered for memory the write tracker does not watch"); + if (!AgcDriver::GuestMemory::WriteWatched()) Require(!UnitShadowEnabled(), "unit shadows are on without write watching"); TextureDetiler detiler(base); auto context = base; context.detiler = &detiler; @@ -989,26 +1006,396 @@ void storageRefreshTests(const Device& device, Recorder& recorder) { } return true; }; - const auto memoryHolds = [&](std::array texel) { + const auto memoryHolds = [&](std::array texel, std::size_t first = 0, std::uint8_t other = 0) { + std::vector read(surfaceBytes); + if (watched) AgcDriver::GuestMemory::Read(address, read); + else std::memcpy(read.data(), texels, surfaceBytes); for (std::size_t i = 0; i < surfaceBytes; ++i) { - if (texels[i] != texel[i % 4]) return false; + if (std::to_integer(read[i]) != (i < first ? other : texel[i % 4])) return false; } return true; }; + const auto holdsMixed = [&](std::array texel, std::size_t count, std::uint8_t other) { + Buffer readback(context, surfaceBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {side, side, 1}; + context.Function("vkCmdCopyImageToBuffer")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, readback.Handle(), 1, ©); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + const auto pixels = readback.Bytes(); + std::size_t others = 0; + for (std::size_t i = 0; i < pixels.size(); i += 4) { + bool isOther = true, isTexel = true; + for (std::size_t c = 0; c < 4; ++c) { + const auto value = std::to_integer(pixels[i + c]); + isOther = isOther && value == other; + isTexel = isTexel && value == texel[c]; + } + if (isOther) ++others; + else if (!isTexel) return false; + } + return others == count; + }; Require(holds({0, 0, 0, 0}), "a surface under 0000 clear keys was not cleared"); draw({{1.0f, 0.0f, 0.0f, 1.0f}}); image->Refresh(); Require(holds({255, 0, 0, 255}), "a refresh under the image's own clear keys dropped the draw's results"); Require(memoryHolds({255, 0, 0, 255}), "the refresh's store of the results did not reach guest memory"); + if (watched) { + draw({{0.0f, 0.0f, 1.0f, 1.0f}}); + std::memset(texels, 0x33, 65536); + image->Refresh(); + Require(holdsMixed({0, 0, 255, 255}, 65536 / 4, 0x33), "a CPU store into a unit with results pending was not seen by the refresh"); + Require(memoryHolds({0, 0, 255, 255}, 65536, 0x33), "guest memory lost the CPU store or the other units' results"); + image->Refresh(); + Require(holdsMixed({0, 0, 255, 255}, 65536 / 4, 0x33), "an unchanged surface changed at a refresh"); + } draw({{0.0f, 1.0f, 0.0f, 1.0f}}); std::memset(keys, 0x40, keyCount); image->Refresh(); Require(holds({0, 0, 0, 255}), "new 0001 clear keys did not clear the image"); - Require(memoryHolds({255, 0, 0, 255}), "results dead under new clear keys were stored"); + Require(watched ? memoryHolds({0, 0, 255, 255}, 65536, 0x33) : memoryHolds({255, 0, 0, 255}), "results dead under new clear keys were stored"); } recorder.Sync(); } +void importWatchTests(const Device& device) { + using namespace AgcDriver::GuestMemory; + const auto& context = device.GetContext(); +#ifdef _WIN32 + static_cast(context); + std::cout << "import watch decisions: Linux write watch only\n"; +#else + if (context.hostImportAlignment == 0 || !WriteWatched()) { + std::cout << "host imports or write watching unavailable: import watch decisions not tested\n"; + return; + } + const auto probe = ProbeImportWriteProtection(context); + Require(probe.failure == nullptr, std::string("(u) the import probe failed at ") + (probe.failure != nullptr ? probe.failure : "") + " (" + std::to_string(static_cast(probe.result)) + ")"); + std::cout << "import probe: " << probe.writtenAfterSubmit << " of " << probe.pages << " scratch pages written after a GPU read, " << probe.writtenAtImport << " after the import\n"; + const auto decided = PrepareImportWatch(context); + struct Restore { + const Context& context; + ImportWatch decided; + ~Restore() { SetImportWatch(context, decided); } + } restore{context, decided}; + constexpr std::size_t unit = 65536; + const auto remap = [](void* block, std::size_t bytes) { + munmap(block, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, bytes); + Require(mmap(block, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == block, "(u) cannot map the test block again"); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes); + }; + const auto registerRange = [](void* block, std::size_t bytes) { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true); + }; + const auto unregisterRange = [](void* block) { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + }; + SetImportWatch(context, ImportWatch::Unwatch); + { + constexpr std::size_t bytes = 3 * unit; + void* block = AllocateWatched(bytes, unit); + const auto address = reinterpret_cast(block); + std::memset(block, 0x11, bytes); + const auto before = CollectWrites(address, bytes); + Require(before != 0, "(u) a watched block is not collected"); + BumpCollectEpoch(); + Require(CollectWrites(address, 2 * unit) != 0, "(u) the memoized collect of a watched block failed"); + registerRange(block, 2 * unit); + const auto* import = HostImportFor(context, address, 2 * unit); + Require(import != nullptr && import->unwatched, "(u) an import made under the unwatch decision is not marked unwatched"); + Require(!Watched(address, 2 * unit) && !Watched(address + unit, 4096), "(u) an imported range stays watched"); + Require(CollectWrites(address, 2 * unit) == 0 && CollectWrites(address + 4096, 4096) == 0, "(u) a collect memoized before the import answers for the unwatched range"); + Require(CollectWrites(address + 2 * unit - 4096, 8192) == 0, "(u) a range partly imported is collected"); + Require(Watched(address + 2 * unit, unit), "(u) the unimported neighbour left the watch"); + const auto neighbour = CollectWrites(address + 2 * unit, unit); + Require(neighbour != 0, "(u) the unimported neighbour is not collected"); + static_cast(block)[4096] = 0x22; + Require(!UnchangedSince(address, 2 * unit, before) && !UnchangedSince(address, 4096, TrackerGeneration()) && !UnchangedSinceCollected(address, 4096, TrackerGeneration()), "(u) an unwatched range reports unchanged"); + const std::array generations{before, TrackerGeneration()}; + std::array changed{}; + std::array cpu{}; + Require(!ChangedBlocks(address, 2 * unit, generations, changed, cpu) && changed[0] != 0 && changed[1] != 0, "(u) the stamps of an unwatched range are offered as tracked"); + Require(MarkWritten(address, unit) == 0, "(u) a GPU write into an unwatched range claims a generation"); + Require(MarkWritten(address + 2 * unit, 4096) != 0 && !UnchangedSince(address + 2 * unit, unit, neighbour), "(u) a GPU write into the watched neighbour is not stamped"); + unregisterRange(block); + remap(block, 2 * unit); + Require(Watched(address, 2 * unit), "(u) a remapped range is not watched"); + registerRange(block, 2 * unit); + const auto* kept = HostImportFor(context, address, 2 * unit); + Require(kept != nullptr && kept->unwatched && !Watched(address, 2 * unit) && CollectWrites(address, 2 * unit) == 0, "(u) an import kept over a remap left the range watched"); + unregisterRange(block); + remap(block, 2 * unit); + registerRange(block, unit); + const auto* again = HostImportFor(context, address, unit); + Require(again != nullptr && again->unwatched && again->bytes == unit && !Watched(address, unit), "(u) a re-import after an unmap did not follow the decision"); + Require(Watched(address + unit, unit) && CollectWrites(address + unit, unit) != 0, "(u) the unregistered rest of a remapped range is not watched"); + unregisterRange(block); + Require(HostImportFor(context, address, unit) == nullptr, "(u) an unregistered range still imports"); + ReleaseWatched(block, bytes); + } + SetImportWatch(context, ImportWatch::Watch); + { + constexpr std::size_t bytes = unit; + void* block = AllocateWatched(bytes, unit); + const auto address = reinterpret_cast(block); + registerRange(block, bytes); + const auto* import = HostImportFor(context, address, bytes); + Require(import != nullptr && !import->unwatched && Watched(address, bytes), "(u) an import made under the watch decision left the watch"); + const auto generation = CollectWrites(address, bytes); + Require(generation != 0 && UnchangedSince(address, bytes, generation), "(u) a watched import is not collected"); + Require(MarkWritten(address, 4096) != 0 && !UnchangedSince(address, bytes, generation), "(u) a GPU write into a watched import is not stamped"); + unregisterRange(block); + HostImportFor(context, address, bytes); + ReleaseWatched(block, bytes); + } +#endif +} + +void staleGenerationTests(const Device& device, Recorder& recorder) { + const auto& base = device.GetContext(); +#ifdef _WIN32 + static_cast(recorder); + static_cast(base); + std::cout << "a range leaving the watch: Linux write watch only\n"; +#else + if (base.hostImportAlignment == 0) { + std::cout << "host imports unavailable: a range leaving the watch not tested\n"; + return; + } + constexpr std::uint32_t side = 256; + constexpr std::size_t surfaceBytes = side * side * 4; + constexpr std::size_t keyCount = surfaceBytes / 256; + constexpr std::size_t bytes = surfaceBytes + 65536; + void* block = AllocateWatched(bytes, 65536); + if (block == nullptr) { + std::cout << "no write watching: a range leaving the watch not tested\n"; + return; + } + struct Release { + void* block; + ~Release() { ReleaseWatched(block, bytes); } + } release{block}; + const auto decided = PrepareImportWatch(base); + SetImportWatch(base, ImportWatch::Unwatch); + struct Restore { + const Context& context; + ImportWatch decided; + ~Restore() { SetImportWatch(context, decided); } + } restore{base, decided}; + auto* texels = static_cast(block); + std::memset(texels, 0x55, surfaceBytes); + std::memset(texels + surfaceBytes, 0x00, keyCount); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + } + } unregister{base, block, address}; + TextureDetiler detiler(base); + auto context = base; + context.detiler = &detiler; + GuestTextureResource resource{}; + resource.baseAddress = address; + resource.width = side; + resource.height = side; + resource.mipCount = 1; + resource.tileMode = TextureTileMode::kR64KBX; + resource.dimension = TextureDimension::k2D; + resource.format = 56; + resource.dstSelX = 4; + resource.dstSelY = 5; + resource.dstSelZ = 6; + resource.dstSelW = 7; + resource.dccAddress = address + surfaceBytes; + { + auto image = std::make_shared(context, detiler, resource, 0); + Require(AgcDriver::GuestMemory::Watched(address, bytes), "(s) the storage image imported its memory before a generation was taken"); + const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + const auto commands = recorder.Commands(); + recorder.Keep(image); + const VkClearColorValue red{{1.0f, 0.0f, 0.0f, 1.0f}}; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &red, 1, &range); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + image->MarkDirty(); + const auto* import = HostImportFor(base, address, bytes); + if (import == nullptr) { + std::cout << "host import of the stale generation block refused: a range leaving the watch not tested\n"; + recorder.Sync(); + return; + } + Require(!AgcDriver::GuestMemory::Watched(address, bytes), "(s) the imported range stayed watched"); + image->Refresh(); + Buffer readback(context, surfaceBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const auto copyCommands = recorder.Commands(); + RecordMemoryBarrier(context, copyCommands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {side, side, 1}; + context.Function("vkCmdCopyImageToBuffer")(copyCommands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, readback.Handle(), 1, ©); + RecordMemoryBarrier(context, copyCommands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + const auto pixels = readback.Bytes(); + constexpr std::array expected{255, 0, 0, 255}; + for (std::size_t i = 0; i < pixels.size(); ++i) Require(std::to_integer(pixels[i]) == expected[i % 4], "(s) results pending when their memory left the watch were dropped"); + for (std::size_t i = 0; i < surfaceBytes; ++i) Require(texels[i] == expected[i % 4], "(s) results pending when their memory left the watch did not reach it"); + } + recorder.Sync(); +#endif +} + +void importWindowTests(const Device& device, Recorder& recorder) { + using namespace AgcDriver::GuestMemory; + const auto& base = device.GetContext(); +#ifdef _WIN32 + static_cast(recorder); + static_cast(base); + std::cout << "the import window: Linux write watch only\n"; +#else + if (base.hostImportAlignment == 0) { + std::cout << "host imports unavailable: the import window not tested\n"; + return; + } + constexpr std::uint32_t side = 256; + constexpr std::size_t surfaceBytes = side * side * 4; + constexpr std::size_t keyCount = surfaceBytes / 256; + constexpr std::size_t bytes = surfaceBytes + 65536; + void* block = AllocateWatched(bytes, 65536); + if (block == nullptr) { + std::cout << "no write watching: the import window not tested\n"; + return; + } + struct Release { + void* block; + ~Release() { ReleaseWatched(block, bytes); } + } release{block}; + const auto probe = ProbeImportWriteProtection(base); + Require(probe.failure == nullptr, "(w) the import probe failed"); + const bool importWrites = probe.writtenAtImport != 0; + const auto decided = PrepareImportWatch(base); + SetImportWatch(base, ImportWatch::Watch); + struct Restore { + const Context& context; + ImportWatch decided; + ~Restore() { SetImportWatch(context, decided); } + } restore{base, decided}; + auto* texels = static_cast(block); + std::memset(texels, 0x55, surfaceBytes); + std::memset(texels + surfaceBytes, 0x00, keyCount); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + } + } unregister{base, block, address}; + TextureDetiler detiler(base); + auto context = base; + context.detiler = &detiler; + GuestTextureResource resource{}; + resource.baseAddress = address; + resource.width = side; + resource.height = side; + resource.mipCount = 1; + resource.tileMode = TextureTileMode::kR64KBX; + resource.dimension = TextureDimension::k2D; + resource.format = 56; + resource.dstSelX = 4; + resource.dstSelY = 5; + resource.dstSelZ = 6; + resource.dstSelW = 7; + resource.dccAddress = address + surfaceBytes; + { + auto image = std::make_shared(context, detiler, resource, 0); + Require(!HostImportCovers(base, address, bytes), "(w) the storage image imported its memory before a generation was taken"); + const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + const auto commands = recorder.Commands(); + recorder.Keep(image); + const VkClearColorValue red{{1.0f, 0.0f, 0.0f, 1.0f}}; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &red, 1, &range); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + image->MarkDirty(); + const auto cached = CollectWritesUncached(address, surfaceBytes); + Require(cached != 0 && UnchangedSince(address, surfaceBytes, cached), "(w) the cache's generation is not current before the import"); + const auto* import = HostImportFor(base, address, bytes); + if (import == nullptr) { + std::cout << "host import of the import window block refused: the import window not tested\n"; + recorder.Sync(); + return; + } + Require(!import->unwatched && Watched(address, bytes), "(w) an import under the watch decision left the watch"); + CollectWritesUncached(address, surfaceBytes); + if (importWrites) { + Require(!UnchangedSince(address, surfaceBytes, cached), "(w) a cache over the pages the import reported sees no change"); + std::array changed{}; + const std::array generations{cached, cached, cached, cached}; + Require(ChangedBlocks(address, surfaceBytes, generations, changed) && std::all_of(changed.begin(), changed.end(), [](std::uint8_t value) { return value == BlockMaybeWritten; }), "(w) the pages the import reported read as written by the CPU"); + Require(!WrittenSince(address, surfaceBytes, cached), "(w) the import window reads as a CPU store"); + } else { + std::cout << "the import reports no pages here: only the pending results of the import window checked\n"; + } + const auto after = CollectWritesUncached(address, surfaceBytes); + Require(after != 0 && UnchangedSince(address, surfaceBytes, after), "(w) the imported range is not watched again after the import window"); + image->Refresh(); + Buffer readback(context, surfaceBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const auto copyCommands = recorder.Commands(); + RecordMemoryBarrier(context, copyCommands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {side, side, 1}; + context.Function("vkCmdCopyImageToBuffer")(copyCommands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, readback.Handle(), 1, ©); + RecordMemoryBarrier(context, copyCommands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + const auto pixels = readback.Bytes(); + constexpr std::array expected{255, 0, 0, 255}; + for (std::size_t i = 0; i < pixels.size(); ++i) Require(std::to_integer(pixels[i]) == expected[i % 4], "(w) results pending over the import window were dropped"); + std::vector memory(surfaceBytes); + Read(address, memory); + for (std::size_t i = 0; i < surfaceBytes; ++i) Require(std::to_integer(memory[i]) == expected[i % 4], "(w) results pending over the import window did not reach guest memory"); + const auto stored = CollectWritesUncached(address, surfaceBytes); + texels[65536 + 8] = 0x77; + CollectWritesUncached(address, surfaceBytes); + Require(WrittenSince(address + 65536, 65536, stored) && !WrittenSince(address, 65536, stored), "(w) a CPU store after the import window is not reported as one"); + } + recorder.Sync(); +#endif +} + // The data word positions of a dispatch-cache variant (Driver.cpp's data-only hits): leaves // located among the runs' words, aliased, unaligned, out-of-run and mismatched ones skipped. void dataWordPositionsTests() { @@ -1075,6 +1462,7 @@ int main() { try { Device device; std::lock_guard gpu(GpuMutex()); + std::cout << "host imports " << (PrepareImportWatch(device.GetContext()) == ImportWatch::Unwatch ? "are compared" : "stay watched") << '\n'; Recorder recorder(device.GetContext()); recorder.Activate(); readTrackingTests(device, recorder); @@ -1088,7 +1476,11 @@ int main() { resourceReadTests(device, recorder); storeRunTests(device, recorder); unitShadowTests(device, recorder); - storageRefreshTests(device, recorder); + storageRefreshTests(device, recorder, false); + storageRefreshTests(device, recorder, true); + importWatchTests(device); + staleGenerationTests(device, recorder); + importWindowTests(device, recorder); dataWordPositionsTests(); dataRefreshTests(device, recorder); std::cout << "Recorder read tracking and label tests passed\n"; diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index d8e52f150..3fbadd044 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -15,6 +15,7 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/StandardStreams.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestAllocations.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestArena.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestWriteWatch.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestHeap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/HeapDiagnostics.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Mspace.cpp diff --git a/core/libs/prx/libc/include/GuestWriteWatch.hpp b/core/libs/prx/libc/include/GuestWriteWatch.hpp new file mode 100644 index 000000000..4e5d98dd5 --- /dev/null +++ b/core/libs/prx/libc/include/GuestWriteWatch.hpp @@ -0,0 +1,21 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_GUESTWRITEWATCH_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_GUESTWRITEWATCH_HPP + +#include +#include + +namespace GuestWriteWatch { + +extern "C" { + +bool GuestWriteWatchAvailable_nid_postfix(); +void GuestWriteWatchRegister_nid_postfix(const void* pointer, std::size_t bytes); +bool GuestWriteWatchUnregister_nid_postfix(const void* pointer, std::size_t bytes); +bool GuestWriteWatchCovers_nid_postfix(std::uintptr_t address, std::size_t bytes); +bool GuestWriteWatchCollect_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context); + +} + +} + +#endif diff --git a/core/libs/prx/libc/src/GuestWriteWatch.cpp b/core/libs/prx/libc/src/GuestWriteWatch.cpp new file mode 100644 index 000000000..afde2dfc5 --- /dev/null +++ b/core/libs/prx/libc/src/GuestWriteWatch.cpp @@ -0,0 +1,309 @@ +#include "prx/libc/include/GuestWriteWatch.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +namespace GuestWriteWatch { +namespace { + +#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) +constexpr std::uintptr_t PageBytes = 4096; + +class Watch { +public: + static Watch& Get() { + static Watch watch; + return watch; + } + + bool Available() const { + return _pagemap >= 0; + } + + void Register(std::uintptr_t begin, std::uintptr_t end) { + if (!Available() || end <= begin) return; + std::unique_lock lock(_lock); + remove(_ranges, begin, end); + remove(_fresh, begin, end); + uffdio_register registration{}; + registration.range.start = begin; + registration.range.len = end - begin; + registration.mode = UFFDIO_REGISTER_MODE_WP; + if (ioctl(_uffd, UFFDIO_REGISTER, ®istration) != 0) { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[memory] write watch: cannot register 0x%llx+0x%llx (%s); the range stays unwatched\n", static_cast(begin), static_cast(end - begin), std::strerror(errno)); + } + return; + } + insert(_ranges, begin, end); + insert(_fresh, begin, end); + } + + bool Unregister(std::uintptr_t begin, std::uintptr_t end) { + if (!Available() || end <= begin) return false; + std::unique_lock lock(_lock); + const bool watched = remove(_ranges, begin, end); + remove(_fresh, begin, end); + return watched; + } + + bool Covers(std::uintptr_t begin, std::uintptr_t end) { + if (!Available()) return false; + std::shared_lock lock(_lock); + return covers(begin, end); + } + + bool Collect(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) { + if (!Available()) return false; + begin &= ~(PageBytes - 1); + end = (end + PageBytes - 1) & ~(PageBytes - 1); + if (end <= begin) return true; + std::vector> fresh; + { + std::unique_lock lock(_lock); + if (!covers(begin, end)) return false; + auto it = _fresh.upper_bound(begin); + if (it != _fresh.begin()) --it; + for (; it != _fresh.end() && it->first < end; ++it) { + const auto from = std::max(it->first, begin); + const auto to = std::min(it->second, end); + if (from < to) fresh.emplace_back(from, to); + } + if (!fresh.empty()) remove(_fresh, begin, end); + } + for (const auto& [from, to] : fresh) { + written(context, from, to); + if (protect(from, to)) continue; + std::unique_lock lock(_lock); + for (const auto& [left, right] : fresh) { + if (!covers(left, right)) continue; + remove(_fresh, left, right); + insert(_fresh, left, right); + } + written(context, begin, end); + return false; + } + std::array regions; + auto cursor = begin; + while (cursor < end) { + pm_scan_arg scan{}; + scan.size = sizeof(scan); + scan.flags = PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC; + scan.start = cursor; + scan.end = end; + scan.vec = reinterpret_cast(regions.data()); + scan.vec_len = regions.size(); + scan.category_mask = PAGE_IS_WRITTEN; + scan.return_mask = PAGE_IS_WRITTEN; + const auto count = ioctl(_pagemap, PAGEMAP_SCAN, &scan); + if (count < 0) { + written(context, cursor, end); + return false; + } + for (long i = 0; i < count; ++i) written(context, regions[i].start, regions[i].end); + if (scan.walk_end <= cursor || scan.walk_end >= end) break; + cursor = scan.walk_end; + } + return true; + } + +private: + Watch() { + if (std::getenv("APS5_NO_WRITE_WATCH") == nullptr) open(); + } + + void open() { + _uffd = static_cast(syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK)); + if (_uffd < 0 && errno == EPERM) _uffd = static_cast(syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY)); + if (_uffd < 0) return unavailable("userfaultfd"); + uffdio_api api{}; + api.api = UFFD_API; + api.features = UFFD_FEATURE_WP_ASYNC | UFFD_FEATURE_WP_UNPOPULATED; + if (ioctl(_uffd, UFFDIO_API, &api) != 0 || (api.features & UFFD_FEATURE_WP_ASYNC) == 0 || (api.features & UFFD_FEATURE_WP_UNPOPULATED) == 0) return unavailable("asynchronous userfaultfd write protection"); + const int pagemap = ::open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (pagemap < 0) return unavailable("/proc/self/pagemap"); + if (!probe(pagemap)) { + close(pagemap); + return unavailable("PAGEMAP_SCAN"); + } + _pagemap = pagemap; + } + + void unavailable(const char* what) { + const int error = errno; + if (_uffd >= 0) close(_uffd); + _uffd = -1; + std::fprintf(stderr, "[memory] write watch unavailable: %s failed (%s); guest memory is compared instead\n", what, std::strerror(error)); + } + + bool protect(std::uintptr_t begin, std::uintptr_t end) const { + uffdio_writeprotect protection{}; + protection.range.start = begin; + protection.range.len = end - begin; + protection.mode = UFFDIO_WRITEPROTECT_MODE_WP; + return ioctl(_uffd, UFFDIO_WRITEPROTECT, &protection) == 0; + } + + bool probe(int pagemap) { + constexpr std::uintptr_t probeBytes = 4 * PageBytes; + void* pages = mmap(nullptr, probeBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (pages == MAP_FAILED) return false; + const auto address = reinterpret_cast(pages); + auto* bytes = static_cast(pages); + bytes[0] = 1; + uffdio_register registration{}; + registration.range.start = address; + registration.range.len = probeBytes; + registration.mode = UFFDIO_REGISTER_MODE_WP; + bool working = ioctl(_uffd, UFFDIO_REGISTER, ®istration) == 0 && protect(address, address + probeBytes); + std::array regions{}; + const auto scan = [&]() -> long { + pm_scan_arg arguments{}; + arguments.size = sizeof(arguments); + arguments.flags = PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC; + arguments.start = address; + arguments.end = address + probeBytes; + arguments.vec = reinterpret_cast(regions.data()); + arguments.vec_len = regions.size(); + arguments.category_mask = PAGE_IS_WRITTEN; + arguments.return_mask = PAGE_IS_WRITTEN; + const auto count = ioctl(pagemap, PAGEMAP_SCAN, &arguments); + if (count < 0) return -1; + long pagesWritten = 0; + for (long i = 0; i < count; ++i) pagesWritten += static_cast((regions[i].end - regions[i].start) / PageBytes); + return pagesWritten; + }; + working = working && scan() == 0; + if (working) { + bytes[0] = 2; + bytes[2 * PageBytes] = 1; + working = scan() == 2 && scan() == 0; + } + munmap(pages, probeBytes); + return working; + } + + bool covers(std::uintptr_t begin, std::uintptr_t end) const { + auto next = _ranges.upper_bound(begin); + if (next == _ranges.begin()) return false; + return std::prev(next)->second >= end; + } + + static void insert(std::map& ranges, std::uintptr_t begin, std::uintptr_t end) { + auto next = ranges.upper_bound(begin); + if (next != ranges.begin() && std::prev(next)->second == begin) { + begin = std::prev(next)->first; + ranges.erase(std::prev(next)); + } + if (next != ranges.end() && next->first == end) { + end = next->second; + ranges.erase(next); + } + ranges.emplace(begin, end); + } + + static bool remove(std::map& ranges, std::uintptr_t begin, std::uintptr_t end) { + bool removed = false; + auto it = ranges.upper_bound(begin); + if (it != ranges.begin()) --it; + while (it != ranges.end() && it->first < end) { + const auto rangeBegin = it->first; + const auto rangeEnd = it->second; + if (rangeEnd <= begin) { + ++it; + continue; + } + removed = true; + it = ranges.erase(it); + if (rangeBegin < begin) ranges.emplace(rangeBegin, begin); + if (rangeEnd > end) ranges.emplace(end, rangeEnd); + } + return removed; + } + + int _uffd = -1; + int _pagemap = -1; + std::shared_mutex _lock; + std::map _ranges; + std::map _fresh; +}; + +const bool g_opened = (Watch::Get(), true); +#endif + +} + +bool GuestWriteWatchAvailable_nid_postfix() { +#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) + return Watch::Get().Available(); +#else + return false; +#endif +} + +void GuestWriteWatchRegister_nid_postfix(const void* pointer, std::size_t bytes) { +#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) + const auto begin = reinterpret_cast(pointer); + Watch::Get().Register(begin, begin + bytes); +#else + static_cast(pointer); + static_cast(bytes); +#endif +} + +bool GuestWriteWatchUnregister_nid_postfix(const void* pointer, std::size_t bytes) { +#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) + const auto begin = reinterpret_cast(pointer); + return Watch::Get().Unregister(begin, begin + bytes); +#else + static_cast(pointer); + static_cast(bytes); + return false; +#endif +} + +bool GuestWriteWatchCovers_nid_postfix(std::uintptr_t address, std::size_t bytes) { +#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) + return bytes != 0 && address + bytes > address && Watch::Get().Covers(address, address + bytes); +#else + static_cast(address); + static_cast(bytes); + return false; +#endif +} + +bool GuestWriteWatchCollect_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context) { +#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) + if (bytes == 0 || address + bytes < address) return false; + return Watch::Get().Collect(address, address + bytes, written, context); +#else + static_cast(address); + static_cast(bytes); + static_cast(written); + static_cast(context); + return false; +#endif +} + +} diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp index a1440a0f5..f9d251b39 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp @@ -1,6 +1,7 @@ #include "prx/libkernel/DirectMemory/DirectMemory.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" #include #include #include @@ -307,6 +308,25 @@ void ValidatePhysicalRange(std::uint64_t phys, std::size_t len) { } } +#if defined(__linux__) +void WatchMapping(std::uintptr_t address, std::size_t len) { + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(address), len); + const auto end = address + len; + for (auto added = g_directMappings.lower_bound(address); added != g_directMappings.end() && added->first < end; ++added) { + const auto addedPhys = added->second.phys; + const auto addedPhysEnd = addedPhys + (added->second.end - added->first); + for (const auto& [base, other] : g_directMappings) { + if ((base >= address && base < end) || other.backing != added->second.backing) continue; + const auto first = std::max(addedPhys, other.phys); + const auto last = std::min(addedPhysEnd, other.phys + (other.end - base)); + if (first >= last) continue; + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(added->first + (first - addedPhys)), last - first); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(base + (first - other.phys)), last - first); + } + } +} +#endif + void AddMapping(std::uintptr_t address, std::size_t len, std::uint64_t phys, int nativeProt) { ValidatePhysicalRange(phys, len); EraseMappings(address, address + len); @@ -322,6 +342,9 @@ void AddMapping(std::uintptr_t address, std::size_t len, std::uint64_t phys, int g_directMappings.emplace(address + offset, DirectMapping{address + offset + bytes, phys + offset, page.backing->MemoryType(), page.backing}); offset += bytes; } +#if defined(__linux__) + WatchMapping(address, len); +#endif } bool RemapFixedIntoRegistered(GuestAllocations::Mutation& mutation, void* addr, size_t len, int prot, int flags, int64_t physStart = -1) { @@ -343,6 +366,7 @@ bool RemapFixedIntoRegistered(GuestAllocations::Mutation& mutation, void* addr, if (nativeProtection != PROT_NONE) CommitArenaRange(addr, len, WinProtFromPosix(nativeProtection)); #else if (::mmap(addr, len, nativeProtection, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == MAP_FAILED) throw std::system_error(errno, std::generic_category(), "remap anonymous memory"); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(addr, len); #endif } }); @@ -352,13 +376,14 @@ bool RemapFixedIntoRegistered(GuestAllocations::Mutation& mutation, void* addr, void Unmap(void* addr, size_t len) { #if defined(__linux__) if (munmap(addr, len) != 0) throw std::system_error(errno, std::generic_category(), "munmap failed"); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(addr, len); #else if (KernelArena::Get().Contains(addr, len)) munmap(addr, len); else munmap_release(addr); #endif } -void* MapAligned(void* addr, size_t len, int prot, int flags, size_t alignment) { +void* MapPlaced(void* addr, size_t len, int prot, int flags, size_t alignment) { ValidateLength(len); alignment = ValidateAlignment(alignment); constexpr int GuestMapFixed = 0x10; @@ -450,6 +475,14 @@ void RecordProtection(const void* addr, size_t len, int prot) { if (prot >= 0) g_protections.emplace(start, ProtectedRange{start + len, prot}); } +void* MapAligned(void* addr, size_t len, int prot, int flags, size_t alignment) { + void* mapped = MapPlaced(addr, len, prot, flags, alignment); +#if defined(__linux__) + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(mapped, len); +#endif + return mapped; +} + void ValidateOutput(void** addr) { if (!addr) { // return SCE_KERNEL_ERROR_EINVAL; diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index 0d3233446..7449b08bd 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -2,6 +2,7 @@ #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestHeap.hpp" #include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" #include #include "SceTypes.hpp" #include @@ -10,6 +11,14 @@ #include #include #include +#include +#include +#include +#include +#if defined(__linux__) +#include +#include +#endif extern "C" { void* APS5_VABI mmap_nid_postfix(void*, std::size_t, int, int, int, std::int64_t) noexcept; @@ -18,19 +27,19 @@ int* APS5_VABI __error_nid_postfix(); int APS5_VABI sceKernelMapNamedFlexibleMemory(void**, std::size_t, int, int, const char*); int APS5_VABI sceKernelMapFlexibleMemory(void**, std::size_t, int, int); int APS5_VABI sceKernelMunmap(void*, std::size_t); +int APS5_VABI sceKernelMprotect(const void*, std::size_t, int); int APS5_VABI sceKernelVirtualQuery(const void*, int, VirtualQueryInfo*, std::uint64_t); int APS5_VABI sceKernelSetVirtualRangeName(const void*, std::uint64_t, const char*); int APS5_VABI sceKernelClearVirtualRangeName(const void*, std::uint64_t); int APS5_VABI sceKernelAllocateDirectMemory(std::int64_t, std::int64_t, std::size_t, std::size_t, int, std::int64_t*); int APS5_VABI sceKernelMapDirectMemory(void**, std::size_t, int, int, std::int64_t, std::size_t); -int APS5_VABI sceKernelMprotect(const void*, std::size_t, int); int APS5_VABI sceKernelReleaseDirectMemory(std::int64_t, std::size_t); int APS5_VABI sceKernelReserveVirtualRange(void**, std::size_t, int, std::size_t); } -static void Require(bool condition) { +static void Require(bool condition, std::source_location location = std::source_location::current()) { if (!condition) { - std::fputs("Guest memory check failed\n", stderr); + std::fprintf(stderr, "Guest memory check failed at %s:%u\n", location.file_name(), static_cast(location.line())); std::abort(); } } @@ -246,12 +255,168 @@ static void CheckSharedWriteTracking() { #endif } +#if defined(__linux__) +using PageRuns = std::vector>; + +static bool CollectRuns(const void* base, std::size_t offset, std::size_t bytes, PageRuns& runs) { + runs.clear(); + const auto address = reinterpret_cast(base); + std::pair context{address, &runs}; + return GuestWriteWatch::GuestWriteWatchCollect_nid_postfix(address + offset, bytes, [](void* context, std::uintptr_t begin, std::uintptr_t end) { + auto& [origin, into] = *static_cast*>(context); + if (!into->empty() && into->back().second == (begin - origin) / 4096) into->back().second = (end - origin) / 4096; + else into->emplace_back((begin - origin) / 4096, (end - origin) / 4096); + }, &context); +} + +static bool Written(const void* base, std::size_t bytes, PageRuns expected) { + PageRuns runs; + const bool complete = CollectRuns(base, 0, bytes, runs); + if (complete && runs == expected) return true; + std::fprintf(stderr, "write watch collect %s, written pages:", complete ? "complete" : "incomplete"); + for (const auto& [first, last] : runs) std::fprintf(stderr, " [%zu, %zu)", static_cast(first), static_cast(last)); + std::fputs("\n", stderr); + return false; +} + +static void CheckWriteWatch() { + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { + std::puts("write watch unavailable: not tested"); + return; + } + constexpr std::size_t length = 0x100000; + constexpr std::size_t small = 4096; + void* mapping = nullptr; + Require(sceKernelMapFlexibleMemory(&mapping, length, 3, 0) == 0); + auto* bytes = static_cast(mapping); + const auto address = reinterpret_cast(mapping); + Require(GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(address, length)); + Require(GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(address + small, small)); + Require(!GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(address, length + small)); + Require(!GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(reinterpret_cast(&length), sizeof(length))); + Require(Written(mapping, length, {{0, length / small}})); + Require(Written(mapping, length, {})); + bytes[5 * small + 17] = 1; + Require(Written(mapping, length, {{5, 6}})); + Require(Written(mapping, length, {})); + static_cast(bytes[10 * small]); + Require(Written(mapping, length, {})); + bytes[7 * small] = 1; + bytes[9 * small] = 1; + PageRuns runs; + Require(CollectRuns(mapping, 8 * small, small, runs) && runs.empty()); + Require(CollectRuns(mapping, 7 * small + 100, 1, runs) && runs == PageRuns{{7, 8}}); + Require(Written(mapping, length, {{9, 10}})); + int pipe[2]; + Require(::pipe(pipe) == 0); + Require(::write(pipe[1], "kernel", 6) == 6); + Require(::read(pipe[0], const_cast(bytes + 20 * small + 8), 6) == 6); + ::close(pipe[0]); + ::close(pipe[1]); + Require(bytes[20 * small + 8] == 'k' && Written(mapping, length, {{20, 21}})); + std::thread([&] { bytes[30 * small + 5] = 3; }).join(); + Require(Written(mapping, length, {{30, 31}})); + std::vector source(2 * small, 0xab); + std::memcpy(const_cast(bytes + 40 * small + 2048), source.data(), source.size()); + Require(Written(mapping, length, {{40, 43}})); + Require(sceKernelMprotect(mapping, length, 1) == 0); + Require(sceKernelMprotect(mapping, length, 3) == 0); + Require(Written(mapping, length, {})); + bytes[50 * small] = 1; + Require(Written(mapping, length, {{50, 51}})); + constexpr std::size_t guestPage = 0x4000; + auto* middle = const_cast(bytes + 4 * guestPage); + Require(sceKernelMunmap(middle, guestPage) == 0); + Require(!GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(address, length)); + Require(!CollectRuns(mapping, 0, length, runs) && runs.empty()); + void* fixed = middle; + Require(sceKernelMapFlexibleMemory(&fixed, guestPage, 3, 0x10) == 0 && fixed == middle); + Require(GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(address, length)); + Require(Written(mapping, length, {{16, 20}})); + Require(Written(mapping, length, {})); + middle[1] = 1; + Require(Written(mapping, length, {{16, 17}})); + Require(sceKernelMunmap(mapping, length) == 0); + Require(!GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(address, small)); + constexpr std::size_t tableSpan = 0x200000; + constexpr std::size_t spanned = 2 * tableSpan; + void* raw = mmap(nullptr, spanned + tableSpan, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED); + const auto rawAddress = reinterpret_cast(raw); + void* region = reinterpret_cast((rawAddress + tableSpan - 1) & ~(tableSpan - 1)); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(region, spanned); + Require(GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(reinterpret_cast(region), spanned)); + Require(Written(region, spanned, {{0, spanned / small}})); + Require(Written(region, spanned, {})); + static_cast(region)[tableSpan + 3 * small] = 1; + static_cast(region)[7 * small] = 1; + Require(Written(region, spanned, {{7, 8}, {tableSpan / small + 3, tableSpan / small + 4}})); + Require(Written(region, spanned, {})); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(region, spanned); + Require(munmap(raw, spanned + tableSpan) == 0); +} + +static void CheckDirectMemoryWriteWatch() { + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { + std::puts("write watch unavailable: direct memory not tested"); + return; + } + constexpr std::size_t page = 0x4000; + constexpr std::size_t small = 4096; + std::int64_t phys = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * 3, 0, 0, &phys) == 0); + void* first = nullptr; + Require(sceKernelMapDirectMemory(&first, page * 3, 3, 0, phys, 0) == 0); + const auto address = reinterpret_cast(first); + auto* bytes = static_cast(first); + Require(GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(address, page * 3)); + Require(Written(first, page * 3, {{0, page * 3 / small}})); + Require(Written(first, page * 3, {})); + bytes[page + 5] = 1; + Require(Written(first, page * 3, {{page / small, page / small + 1}})); + void* alias = nullptr; + Require(sceKernelMapDirectMemory(&alias, page, 3, 0, phys + page, 0) == 0); + const auto aliasAddress = reinterpret_cast(alias); + Require(!GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(aliasAddress, page)); + Require(!GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(address + page, page)); + Require(GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(address, page) && GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(address + page * 2, page)); + PageRuns runs; + Require(!CollectRuns(first, 0, page * 3, runs)); + bytes[0] = 2; + static_cast(alias)[7] = 3; + Require(bytes[page + 7] == 3); + Require(CollectRuns(first, 0, page, runs) && runs == PageRuns{{0, 1}}); + Require(CollectRuns(first, page * 2, page, runs) && runs.empty()); + Require(sceKernelMunmap(alias, page) == 0); + Require(sceKernelMunmap(first, page * 3) == 0); + Require(!GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(address, page)); + void* flexible = nullptr; + Require(sceKernelMapFlexibleMemory(&flexible, page * 3, 3, 0) == 0); + Require(Written(flexible, page * 3, {{0, page * 3 / small}})); + Require(Written(flexible, page * 3, {})); + void* fixed = static_cast(flexible) + page; + Require(sceKernelMapDirectMemory(&fixed, page, 3, 0x10, phys + page * 2, 0) == 0); + Require(fixed == static_cast(flexible) + page); + Require(GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(reinterpret_cast(flexible), page * 3)); + Require(Written(flexible, page * 3, {{page / small, page * 2 / small}})); + Require(Written(flexible, page * 3, {})); + static_cast(fixed)[9] = 4; + Require(Written(flexible, page * 3, {{page / small, page / small + 1}})); + Require(sceKernelMunmap(flexible, page * 3) == 0); + Require(sceKernelReleaseDirectMemory(phys, page * 3) == 0); +} +#endif + int main() { CheckNamedAndHintedMappings(); CheckDirectMemoryFollowsPhysicalPages(); CheckSharedDirectMemoryLifecycle(); CheckHeapAfterMappingReuse(); CheckSharedWriteTracking(); +#if defined(__linux__) + CheckWriteWatch(); + CheckDirectMemoryWriteWatch(); +#endif constexpr std::size_t page = 0x4000; const auto failed = reinterpret_cast(static_cast(-1)); const auto reject = [&](std::size_t length, int protection, int flags, int fd,