#include "prx/libc/include/GuestWriteWatch.hpp" #include #include #include #include #include #include #include #include #include #include #if defined(__linux__) #include #include #include #include #include #include #include #include #include #endif namespace GuestWriteWatch { namespace { #if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) constexpr std::uintptr_t PageBytes = 4096; class Watch { public: static Watch& Get() { static Watch watch; return watch; } bool Available() const { return _pagemap >= 0; } void Register(std::uintptr_t begin, std::uintptr_t end) { if (!Available() || end <= begin) return; std::unique_lock lock(_lock); remove(_ranges, begin, end); remove(_fresh, begin, end); uffdio_register registration{}; registration.range.start = begin; registration.range.len = end - begin; registration.mode = UFFDIO_REGISTER_MODE_WP; if (ioctl(_uffd, UFFDIO_REGISTER, ®istration) != 0) { static bool reported = false; if (!reported) { reported = true; std::fprintf(stderr, "[memory] write watch: cannot register 0x%llx+0x%llx (%s); the range stays unwatched\n", static_cast(begin), static_cast(end - begin), std::strerror(errno)); } return; } insert(_ranges, begin, end); insert(_fresh, begin, end); } bool Unregister(std::uintptr_t begin, std::uintptr_t end) { if (!Available() || end <= begin) return false; std::unique_lock lock(_lock); const bool watched = remove(_ranges, begin, end); remove(_fresh, begin, end); return watched; } bool Covers(std::uintptr_t begin, std::uintptr_t end) { if (!Available()) return false; std::shared_lock lock(_lock); return covers(begin, end); } bool Collect(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) { if (!Available()) return false; begin &= ~(PageBytes - 1); end = (end + PageBytes - 1) & ~(PageBytes - 1); if (end <= begin) return true; std::vector> fresh; { std::unique_lock lock(_lock); if (!covers(begin, end)) return false; auto it = _fresh.upper_bound(begin); if (it != _fresh.begin()) --it; for (; it != _fresh.end() && it->first < end; ++it) { const auto from = std::max(it->first, begin); const auto to = std::min(it->second, end); if (from < to) fresh.emplace_back(from, to); } if (!fresh.empty()) remove(_fresh, begin, end); } for (const auto& [from, to] : fresh) { written(context, from, to); if (protect(from, to)) continue; std::unique_lock lock(_lock); for (const auto& [left, right] : fresh) { if (!covers(left, right)) continue; remove(_fresh, left, right); insert(_fresh, left, right); } written(context, begin, end); return false; } std::array regions; auto cursor = begin; while (cursor < end) { pm_scan_arg scan{}; scan.size = sizeof(scan); scan.flags = PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC; scan.start = cursor; scan.end = end; scan.vec = reinterpret_cast(regions.data()); scan.vec_len = regions.size(); scan.category_mask = PAGE_IS_WRITTEN; scan.return_mask = PAGE_IS_WRITTEN; const auto count = ioctl(_pagemap, PAGEMAP_SCAN, &scan); if (count < 0) { written(context, cursor, end); return false; } for (long i = 0; i < count; ++i) written(context, regions[i].start, regions[i].end); if (scan.walk_end <= cursor || scan.walk_end >= end) break; cursor = scan.walk_end; } return true; } private: Watch() { if (std::getenv("APS5_NO_WRITE_WATCH") == nullptr) open(); } void open() { _uffd = static_cast(syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK)); if (_uffd < 0 && errno == EPERM) _uffd = static_cast(syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY)); if (_uffd < 0) return unavailable("userfaultfd"); uffdio_api api{}; api.api = UFFD_API; api.features = UFFD_FEATURE_WP_ASYNC | UFFD_FEATURE_WP_UNPOPULATED; if (ioctl(_uffd, UFFDIO_API, &api) != 0 || (api.features & UFFD_FEATURE_WP_ASYNC) == 0 || (api.features & UFFD_FEATURE_WP_UNPOPULATED) == 0) return unavailable("asynchronous userfaultfd write protection"); const int pagemap = ::open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); if (pagemap < 0) return unavailable("/proc/self/pagemap"); if (!probe(pagemap)) { close(pagemap); return unavailable("PAGEMAP_SCAN"); } _pagemap = pagemap; } void unavailable(const char* what) { const int error = errno; if (_uffd >= 0) close(_uffd); _uffd = -1; std::fprintf(stderr, "[memory] write watch unavailable: %s failed (%s); guest memory is compared instead\n", what, std::strerror(error)); } bool protect(std::uintptr_t begin, std::uintptr_t end) const { uffdio_writeprotect protection{}; protection.range.start = begin; protection.range.len = end - begin; protection.mode = UFFDIO_WRITEPROTECT_MODE_WP; return ioctl(_uffd, UFFDIO_WRITEPROTECT, &protection) == 0; } bool probe(int pagemap) { constexpr std::uintptr_t probeBytes = 4 * PageBytes; void* pages = mmap(nullptr, probeBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (pages == MAP_FAILED) return false; const auto address = reinterpret_cast(pages); auto* bytes = static_cast(pages); bytes[0] = 1; uffdio_register registration{}; registration.range.start = address; registration.range.len = probeBytes; registration.mode = UFFDIO_REGISTER_MODE_WP; bool working = ioctl(_uffd, UFFDIO_REGISTER, ®istration) == 0 && protect(address, address + probeBytes); std::array regions{}; const auto scan = [&]() -> long { pm_scan_arg arguments{}; arguments.size = sizeof(arguments); arguments.flags = PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC; arguments.start = address; arguments.end = address + probeBytes; arguments.vec = reinterpret_cast(regions.data()); arguments.vec_len = regions.size(); arguments.category_mask = PAGE_IS_WRITTEN; arguments.return_mask = PAGE_IS_WRITTEN; const auto count = ioctl(pagemap, PAGEMAP_SCAN, &arguments); if (count < 0) return -1; long pagesWritten = 0; for (long i = 0; i < count; ++i) pagesWritten += static_cast((regions[i].end - regions[i].start) / PageBytes); return pagesWritten; }; working = working && scan() == 0; if (working) { bytes[0] = 2; bytes[2 * PageBytes] = 1; working = scan() == 2 && scan() == 0; } munmap(pages, probeBytes); return working; } bool covers(std::uintptr_t begin, std::uintptr_t end) const { auto next = _ranges.upper_bound(begin); if (next == _ranges.begin()) return false; return std::prev(next)->second >= end; } static void insert(std::map& ranges, std::uintptr_t begin, std::uintptr_t end) { auto next = ranges.upper_bound(begin); if (next != ranges.begin() && std::prev(next)->second == begin) { begin = std::prev(next)->first; ranges.erase(std::prev(next)); } if (next != ranges.end() && next->first == end) { end = next->second; ranges.erase(next); } ranges.emplace(begin, end); } static bool remove(std::map& ranges, std::uintptr_t begin, std::uintptr_t end) { bool removed = false; auto it = ranges.upper_bound(begin); if (it != ranges.begin()) --it; while (it != ranges.end() && it->first < end) { const auto rangeBegin = it->first; const auto rangeEnd = it->second; if (rangeEnd <= begin) { ++it; continue; } removed = true; it = ranges.erase(it); if (rangeBegin < begin) ranges.emplace(rangeBegin, begin); if (rangeEnd > end) ranges.emplace(end, rangeEnd); } return removed; } int _uffd = -1; int _pagemap = -1; std::shared_mutex _lock; std::map _ranges; std::map _fresh; }; const bool g_opened = (Watch::Get(), true); #endif } bool GuestWriteWatchAvailable_nid_postfix() { #if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) return Watch::Get().Available(); #else return false; #endif } void GuestWriteWatchRegister_nid_postfix(const void* pointer, std::size_t bytes) { #if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) const auto begin = reinterpret_cast(pointer); Watch::Get().Register(begin, begin + bytes); #else static_cast(pointer); static_cast(bytes); #endif } bool GuestWriteWatchUnregister_nid_postfix(const void* pointer, std::size_t bytes) { #if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) const auto begin = reinterpret_cast(pointer); return Watch::Get().Unregister(begin, begin + bytes); #else static_cast(pointer); static_cast(bytes); return false; #endif } bool GuestWriteWatchCovers_nid_postfix(std::uintptr_t address, std::size_t bytes) { #if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) return bytes != 0 && address + bytes > address && Watch::Get().Covers(address, address + bytes); #else static_cast(address); static_cast(bytes); return false; #endif } bool GuestWriteWatchCollect_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context) { #if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) if (bytes == 0 || address + bytes < address) return false; return Watch::Get().Collect(address, address + bytes, written, context); #else static_cast(address); static_cast(bytes); static_cast(written); static_cast(context); return false; #endif } }