fix(libc): track writes to singly mapped direct memory without faults on Windows
A direct memory page mapped at one guest address now lives in private MEM_WRITE_WATCH memory at that address, and its writes are collected with GetWriteWatch like flexible memory: no fault is raised. A page mapped at a second address moves its whole resident run into the section, behind a read-only fence while it is copied, and every address becomes a view of it, tracked by write protection. Unmapped resident pages that are not blank are kept in a private reservation until they are mapped again, not in the section. Ported onto main's 4 KiB subpage tracking of shared views: resident runs keep a per-page collected bit and enter the fresh ranges, so Fresh and Armed collections select them like shared views; a demoted run carries its collected bits into the views and reports every page once. MapAlias throws through refuse() when a resident run extends outside the aliased range, and the caller keeps main's fallback. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1W4jdNSKTV7xwuS6xQPJV
This commit is contained in:
DotDebian
thibaut
parent
ce5c839533
commit
d9852df2a2
|
|
@ -985,6 +985,9 @@ if(BUILD_TESTING)
|
|||
if(WIN32)
|
||||
set_tests_properties(agc_host_import_unmap agc_parked_import_unmap agc_host_import_retirement agc_unit_shadow agc_descriptor_pool_recycling agc_draw_snapshot_cap agc_multisample_guest_access agc_draw_inputs_in_place PROPERTIES ENVIRONMENT_MODIFICATION
|
||||
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
|
||||
add_test(NAME agc_host_import_second_mapping COMMAND agc_driver_recorder_tests --host-import-second-mapping-only)
|
||||
set_tests_properties(agc_host_import_second_mapping PROPERTIES SKIP_RETURN_CODE 77 ENVIRONMENT_MODIFICATION
|
||||
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
|
|
|||
|
|
@ -1310,6 +1310,41 @@ void hostImportRetirementTests(const Device& device) {
|
|||
Require(preservedEveryRecorder, "unlocked mapping changed recorder batches");
|
||||
std::cout << "Host import retirement locking and GPU readback tests passed\n";
|
||||
}
|
||||
#ifdef _WIN32
|
||||
void secondMappingImportTests(const Device& device) {
|
||||
const auto& context = device.GetContext();
|
||||
if (context.hostImportAlignment == 0) {
|
||||
std::cout << "host imports unavailable: imports after a second mapping not tested\n";
|
||||
return;
|
||||
}
|
||||
constexpr std::size_t bytes = 65536;
|
||||
constexpr std::size_t page = 16384;
|
||||
struct Release {
|
||||
void* mapping = nullptr;
|
||||
void* alias = nullptr;
|
||||
std::int64_t physical = -1;
|
||||
~Release() {
|
||||
if (alias != nullptr) sceKernelMunmap(alias, page);
|
||||
if (mapping != nullptr) sceKernelMunmap(mapping, bytes);
|
||||
if (physical >= 0) sceKernelReleaseDirectMemory(physical, bytes);
|
||||
}
|
||||
} release;
|
||||
Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, bytes, bytes, 0, &release.physical) == 0, "allocate the second mapping test backing");
|
||||
Require(sceKernelMapDirectMemory(&release.mapping, bytes, 0x33, 0, release.physical, bytes) == 0, "map the second mapping test backing");
|
||||
const auto address = reinterpret_cast<std::uint64_t>(release.mapping);
|
||||
if (HostImportFor(context, address, bytes) == nullptr) {
|
||||
std::cout << "host import of singly mapped direct memory refused: imports after a second mapping not tested\n";
|
||||
return;
|
||||
}
|
||||
const auto first = HostImportSerial(context, address, bytes, true);
|
||||
Require(first != 0 && HostImportSerial(context, address, bytes, true) == first, "singly mapped direct memory lost its import");
|
||||
Require(sceKernelMapDirectMemory(&release.alias, page, 0x33, 0, release.physical, page) == 0, "map the first page a second time");
|
||||
Require(HostImportFor(context, address, bytes) != nullptr, "direct memory was not imported again after a second mapping of its first page");
|
||||
const auto second = HostImportSerial(context, address, bytes, true);
|
||||
Require(second != 0 && second != first, "the import of direct memory whose pages were replaced was kept");
|
||||
HostImportFor(context, address, bytes);
|
||||
}
|
||||
#endif
|
||||
|
||||
void movedMetadataTests(const Device& device, Recorder& recorder) {
|
||||
const auto& base = device.GetContext();
|
||||
|
|
@ -6436,6 +6471,12 @@ int main(int argc, char** argv) {
|
|||
return vertexTargetRangeTests(device, recorder, std::string_view(argv[1]) == "--vertex-target-range-imported") ? 0 : 77;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
if (argc == 2 && std::string_view(argv[1]) == "--host-import-second-mapping-only") {
|
||||
secondMappingImportTests(device);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
readTrackingTests(device, recorder);
|
||||
writeSettledTests(device, recorder);
|
||||
completionCountTests(device, recorder);
|
||||
|
|
@ -6466,6 +6507,9 @@ int main(int argc, char** argv) {
|
|||
storeRunTests(device, recorder);
|
||||
queuedWriteGroupTests();
|
||||
remappedImportTests(device);
|
||||
#ifdef _WIN32
|
||||
secondMappingImportTests(device);
|
||||
#endif
|
||||
movedMetadataTests(device, recorder);
|
||||
viewPastLastMipTests(device, recorder);
|
||||
keysFillTests(device, recorder);
|
||||
|
|
|
|||
|
|
@ -100,6 +100,9 @@ public:
|
|||
const auto section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE, 0, BlockBytes, nullptr);
|
||||
Require(section != nullptr, "cannot create shared test memory");
|
||||
GuestArena::GuestArenaMap_nid_postfix(block, BlockBytes, section, 0, PAGE_READWRITE);
|
||||
twin = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, 65536));
|
||||
Require(twin != nullptr, "cannot reserve the second mapping of shared test memory");
|
||||
GuestArena::GuestArenaMap_nid_postfix(twin, BlockBytes, section, 0, PAGE_READWRITE);
|
||||
CloseHandle(section);
|
||||
} else if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes);
|
||||
} else {
|
||||
|
|
@ -130,6 +133,10 @@ public:
|
|||
~GuestBlock() {
|
||||
GuestAllocations::Mutation().Remove(block);
|
||||
#ifdef _WIN32
|
||||
if (twin != nullptr) {
|
||||
GuestArena::GuestArenaReset_nid_postfix(twin, BlockBytes);
|
||||
GuestArena::GuestArenaRelease_nid_postfix(twin, BlockBytes);
|
||||
}
|
||||
if (watched) {
|
||||
GuestArena::GuestArenaReset_nid_postfix(block, BlockBytes);
|
||||
GuestArena::GuestArenaRelease_nid_postfix(block, BlockBytes);
|
||||
|
|
@ -154,6 +161,7 @@ public:
|
|||
private:
|
||||
bool watched;
|
||||
std::uint8_t* block = nullptr;
|
||||
std::uint8_t* twin = nullptr;
|
||||
};
|
||||
|
||||
std::array<std::uint32_t, 8> TextureDescriptor(const std::uint8_t* texels, const std::uint8_t* keys) {
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ void GuestArenaCommit_nid_postfix(void* pointer, std::size_t bytes, std::uint32_
|
|||
void GuestArenaReset_nid_postfix(void* pointer, std::size_t bytes);
|
||||
#ifdef _WIN32
|
||||
bool GuestArenaHandleWrite_nid_postfix(std::uintptr_t address);
|
||||
bool GuestArenaHandleRead_nid_postfix(std::uintptr_t address);
|
||||
void GuestArenaPinWritable_nid_postfix(const void* pointer, std::size_t bytes);
|
||||
void GuestArenaUnpinWritable_nid_postfix(const void* pointer, std::size_t bytes);
|
||||
bool GuestArenaProtection_nid_postfix(std::uintptr_t address, std::uint32_t* protection);
|
||||
|
|
@ -42,6 +43,8 @@ void GuestArenaSetPrivateMappingObserver_nid_postfix(void (*callback)(std::uintp
|
|||
void GuestArenaMap_nid_postfix(void* pointer, std::size_t bytes, void* section, std::uint64_t offset, std::uint32_t protection);
|
||||
void* GuestArenaMapAlias_nid_postfix(std::uintptr_t address, std::size_t bytes);
|
||||
void GuestArenaUnmapAlias_nid_postfix(void* alias);
|
||||
bool GuestArenaResidentSpan_nid_postfix(std::uintptr_t address, std::size_t bytes, std::uintptr_t* start, std::uintptr_t* end);
|
||||
void GuestArenaForgetSection_nid_postfix(void* section);
|
||||
#else
|
||||
void GuestArenaSetSharedBacking_nid_postfix(SharedBackingResolver resolver);
|
||||
void GuestArenaSetSharedBackingWriter_nid_no_patch(SharedBackingWriter writer);
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
#include <cstdio>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
|
|
@ -74,33 +75,67 @@ public:
|
|||
reset(reinterpret_cast<std::uintptr_t>(address), bytes);
|
||||
}
|
||||
|
||||
void Map(void* address, std::size_t bytes, HANDLE section, std::uint64_t offset, DWORD protection) {
|
||||
void Map(void* address, std::size_t bytes, HANDLE section, std::uint64_t offset, DWORD protection, bool watched) {
|
||||
std::lock_guard lock(mutex);
|
||||
auto cursor = reinterpret_cast<std::uintptr_t>(address);
|
||||
reset(cursor, bytes);
|
||||
watchResident = watched;
|
||||
HANDLE duplicate = nullptr;
|
||||
if (!DuplicateHandle(GetCurrentProcess(), section, GetCurrentProcess(), &duplicate, 0, FALSE, DUPLICATE_SAME_ACCESS)) fail("keep shared guest section");
|
||||
const auto owned = std::make_shared<Section>(duplicate);
|
||||
for (std::size_t done = 0; done < bytes; done += pageBytes) {
|
||||
split(cursor + done, pageBytes);
|
||||
void* page = reinterpret_cast<void*>(cursor + done);
|
||||
if (!map(section, GetCurrentProcess(), page, offset + done, pageBytes, MEM_REPLACE_PLACEHOLDER, PAGE_EXECUTE_READWRITE, nullptr, 0)) fail("map shared guest page");
|
||||
MEMORY_BASIC_INFORMATION memory{};
|
||||
if (VirtualQuery(page, &memory, sizeof(memory)) != sizeof(memory)) fail("query shared guest page");
|
||||
if (memory.State == MEM_RESERVE && !VirtualAlloc(page, pageBytes, MEM_COMMIT, PAGE_EXECUTE_READWRITE)) fail("commit shared guest page");
|
||||
DWORD previous;
|
||||
if (!VirtualProtect(page, pageBytes, protection, &previous)) fail("protect shared guest page");
|
||||
const auto key = std::make_pair(reinterpret_cast<std::uintptr_t>(section), offset + done);
|
||||
auto shared = physical[key].lock();
|
||||
const auto slotOf = [&](std::size_t done) -> Physical& { return physical[std::make_pair(reinterpret_cast<std::uintptr_t>(section), offset + done)]; };
|
||||
const auto pageOf = [](Physical& slot) {
|
||||
auto shared = slot.page.lock();
|
||||
if (!shared) {
|
||||
shared = std::make_shared<SharedPage>();
|
||||
physical[key] = shared;
|
||||
slot.page = shared;
|
||||
}
|
||||
return shared;
|
||||
};
|
||||
const auto single = [&](std::size_t done) {
|
||||
const auto& slot = slotOf(done);
|
||||
const auto shared = slot.page.lock();
|
||||
return !slot.stored && (!shared || shared->aliases.empty());
|
||||
};
|
||||
for (std::size_t done = 0; done < bytes;) {
|
||||
const auto base = cursor + done;
|
||||
if (single(done)) {
|
||||
auto runBytes = pageBytes;
|
||||
while (done + runBytes < bytes && (base + runBytes) % chunkBytes != 0 && single(done + runBytes)) runBytes += pageBytes;
|
||||
split(base, runBytes);
|
||||
commitResident(base, runBytes);
|
||||
residentRuns.emplace(base, Run{base + runBytes, {}, static_cast<std::uint32_t>(runBytes >> 12)});
|
||||
for (std::size_t at = 0; at < runBytes; at += pageBytes) {
|
||||
auto& slot = slotOf(done + at);
|
||||
const auto shared = pageOf(slot);
|
||||
if (slot.saved) {
|
||||
std::memcpy(reinterpret_cast<void*>(base + at), slot.saved->bytes + slot.savedOffset, pageBytes);
|
||||
slot.saved.reset();
|
||||
}
|
||||
shared->aliases.push_back(base + at);
|
||||
views.emplace(base + at, View{shared, protection, {}, 0, owned, offset + done + at, 0, 0, true, &slot});
|
||||
invalidate(*shared);
|
||||
}
|
||||
remember(freshRanges, base, base + runBytes);
|
||||
DWORD previous;
|
||||
if (protection != PAGE_READWRITE && !VirtualProtect(reinterpret_cast<void*>(base), runBytes, protection, &previous)) fail("protect resident guest pages");
|
||||
done += runBytes;
|
||||
continue;
|
||||
}
|
||||
auto& slot = slotOf(done);
|
||||
const auto shared = pageOf(slot);
|
||||
const auto aliases = shared->aliases;
|
||||
for (const auto alias : aliases) {
|
||||
if (views.at(alias).resident) demote(alias);
|
||||
}
|
||||
split(base, pageBytes);
|
||||
mapShared(base, section, offset + done, protection);
|
||||
slot.stored = true;
|
||||
shared->aliases.push_back(base);
|
||||
views.emplace(base, View{shared, protection, {}, 0, owned, offset + done, 0});
|
||||
views.emplace(base, View{shared, protection, {}, 0, owned, offset + done, 0, 0, false, &slot});
|
||||
remember(freshRanges, base, base + pageBytes);
|
||||
invalidate(*shared);
|
||||
done += pageBytes;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -143,6 +178,10 @@ public:
|
|||
const auto found = views.find(base);
|
||||
if (found == views.end() || !writable(found->second.protection)) return false;
|
||||
auto& view = found->second;
|
||||
if (view.resident) {
|
||||
const auto memory = query(address);
|
||||
return memory.State == MEM_COMMIT && writable(memory.Protect & 0xffu);
|
||||
}
|
||||
const auto written = subpagesOf(base, address, address + 1);
|
||||
invalidate(*view.page, written);
|
||||
protectSubpages(base, written, view.protection, "resume shared memory write");
|
||||
|
|
@ -150,6 +189,36 @@ public:
|
|||
return true;
|
||||
}
|
||||
|
||||
bool HandleRead(std::uintptr_t address) {
|
||||
std::lock_guard lock(mutex);
|
||||
if (!views.contains(address & ~(pageBytes - 1))) return false;
|
||||
const auto memory = query(address);
|
||||
return memory.State == MEM_COMMIT && (memory.Protect & (PAGE_NOACCESS | PAGE_GUARD)) == 0;
|
||||
}
|
||||
|
||||
bool ResidentSpan(std::uintptr_t address, std::size_t bytes, std::uintptr_t* start, std::uintptr_t* end) {
|
||||
std::lock_guard lock(mutex);
|
||||
auto run = residentRuns.upper_bound(address);
|
||||
if (run != residentRuns.begin() && std::prev(run)->second.end > address) --run;
|
||||
if (run == residentRuns.end() || run->first >= address + bytes) return false;
|
||||
*start = run->first;
|
||||
for (; run != residentRuns.end() && run->first < address + bytes; ++run) *end = run->second.end;
|
||||
return true;
|
||||
}
|
||||
|
||||
void ForgetSection(HANDLE section) {
|
||||
std::lock_guard lock(mutex);
|
||||
const auto key = reinterpret_cast<std::uintptr_t>(section);
|
||||
for (auto slot = physical.lower_bound(std::make_pair(key, std::uint64_t{0})); slot != physical.end() && slot->first.first == key;) {
|
||||
const auto shared = slot->second.page.lock();
|
||||
if (shared && !shared->aliases.empty()) {
|
||||
++slot;
|
||||
continue;
|
||||
}
|
||||
slot = physical.erase(slot);
|
||||
}
|
||||
}
|
||||
|
||||
bool BeginHostWrite(std::uintptr_t address, std::size_t bytes) {
|
||||
std::lock_guard lock(mutex);
|
||||
const auto first = views.lower_bound(address & ~(pageBytes - 1));
|
||||
|
|
@ -187,6 +256,16 @@ public:
|
|||
return std::runtime_error(text);
|
||||
};
|
||||
if (address % pageBytes != 0 || bytes % pageBytes != 0 || bytes == 0) throw refuse("the range is not made of whole shared pages");
|
||||
for (std::size_t done = 0; done < bytes; done += pageBytes) {
|
||||
const auto found = views.find(address + done);
|
||||
if (found == views.end() || !found->second.resident) continue;
|
||||
const auto run = std::prev(residentRuns.upper_bound(address + done));
|
||||
if (run->first < address || run->second.end > address + bytes) throw refuse("a resident run of direct memory extends outside the range");
|
||||
}
|
||||
for (std::size_t done = 0; done < bytes; done += pageBytes) {
|
||||
const auto found = views.find(address + done);
|
||||
if (found != views.end() && found->second.resident) demote(address + done);
|
||||
}
|
||||
auto view = views.find(address);
|
||||
if (view == views.end()) throw refuse("the range does not start at a shared view");
|
||||
const auto section = view->second.section;
|
||||
|
|
@ -244,6 +323,10 @@ public:
|
|||
const auto fresh = freshRanges.upper_bound(address);
|
||||
if ((fresh == freshRanges.begin() || std::prev(fresh)->second <= address) && (fresh == freshRanges.end() || fresh->first >= end)) return true;
|
||||
for (auto it = views.lower_bound(address & ~(pageBytes - 1)); it != views.end() && it->first < end; ++it) {
|
||||
if (it->second.resident) {
|
||||
if (!watchResident) return false;
|
||||
continue;
|
||||
}
|
||||
if (it->second.protection != PAGE_NOACCESS) continue;
|
||||
for (auto at = std::max(address, it->first); at < std::min(end, it->first + pageBytes); at += 4096) {
|
||||
if ((it->second.collected & (1u << ((at - it->first) / 4096))) == 0) return false;
|
||||
|
|
@ -261,7 +344,16 @@ public:
|
|||
}
|
||||
const auto base = cursor & ~(pageBytes - 1);
|
||||
const auto found = views.find(base);
|
||||
if (found != views.end()) {
|
||||
if (found != views.end() && found->second.resident) {
|
||||
if (!watchResident) return false;
|
||||
const auto available = capacity - *count;
|
||||
if (available == 0) return true;
|
||||
const auto run = std::prev(residentRuns.upper_bound(cursor));
|
||||
const auto stop = std::min({end, run->second.end, (cursor & ~std::uintptr_t{4095}) + available * 4096});
|
||||
collectResident(run->first, run->second, cursor, stop, pages, count, clear, collection);
|
||||
if (*count == capacity) return true;
|
||||
cursor = stop;
|
||||
} else if (found != views.end()) {
|
||||
auto& view = found->second;
|
||||
auto stop = std::min(std::min(end, base + pageBytes), nextClean == cleanRanges.end() ? end : nextClean->first);
|
||||
if (collection != Collection::All) {
|
||||
|
|
@ -309,6 +401,7 @@ public:
|
|||
std::uint8_t untracked = 0;
|
||||
for (const auto alias : view.page->aliases) {
|
||||
auto& other = views.at(alias);
|
||||
if (other.resident) continue;
|
||||
const auto arming = static_cast<std::uint8_t>(settled & ~other.armed);
|
||||
if (!writable(other.protection) || arming == 0) continue;
|
||||
if (other.hostWrites != 0) {
|
||||
|
|
@ -364,7 +457,7 @@ private:
|
|||
for (auto cursor = address; cursor < end;) {
|
||||
const auto base = cursor & ~(pageBytes - 1);
|
||||
if (const auto found = views.find(base); found != views.end()) {
|
||||
if (found->second.protection == PAGE_NOACCESS) return false;
|
||||
if (found->second.protection == PAGE_NOACCESS || (found->second.resident && !watchResident)) return false;
|
||||
cursor = std::min(end, base + pageBytes);
|
||||
continue;
|
||||
}
|
||||
|
|
@ -380,6 +473,7 @@ private:
|
|||
static_assert(subpageBytes == 4096, "Collect reports one page per subpage");
|
||||
static constexpr std::size_t subpages = pageBytes / subpageBytes;
|
||||
static constexpr std::uint8_t allSubpages = (1u << subpages) - 1;
|
||||
static constexpr std::size_t chunkBytes = 0x200000;
|
||||
struct SharedPage {
|
||||
std::array<std::uint64_t, subpages> generation = [] {
|
||||
std::array<std::uint64_t, subpages> initial{};
|
||||
|
|
@ -396,6 +490,19 @@ private:
|
|||
Section& operator=(const Section&) = delete;
|
||||
~Section() { CloseHandle(handle); }
|
||||
};
|
||||
struct Saved {
|
||||
unsigned char* bytes;
|
||||
explicit Saved(unsigned char* bytes) : bytes(bytes) {}
|
||||
Saved(const Saved&) = delete;
|
||||
Saved& operator=(const Saved&) = delete;
|
||||
~Saved() { VirtualFree(bytes, 0, MEM_RELEASE); }
|
||||
};
|
||||
struct Physical {
|
||||
std::weak_ptr<SharedPage> page;
|
||||
bool stored = false;
|
||||
std::shared_ptr<Saved> saved;
|
||||
std::size_t savedOffset = 0;
|
||||
};
|
||||
struct View {
|
||||
std::shared_ptr<SharedPage> page;
|
||||
DWORD protection;
|
||||
|
|
@ -405,9 +512,159 @@ private:
|
|||
std::uint64_t offset;
|
||||
std::uint32_t hostWrites;
|
||||
std::uint8_t collected = 0;
|
||||
bool resident = false;
|
||||
Physical* slot = nullptr;
|
||||
};
|
||||
struct Run {
|
||||
std::uintptr_t end;
|
||||
std::array<std::uint64_t, 8> collected;
|
||||
std::uint32_t fresh;
|
||||
};
|
||||
struct Window {
|
||||
void* view = nullptr;
|
||||
unsigned char* bytes = nullptr;
|
||||
};
|
||||
using Ranges = std::map<std::uintptr_t, std::uintptr_t>;
|
||||
|
||||
static bool runCollected(const Run& run, std::size_t bit) {
|
||||
return (run.collected[bit >> 6] >> (bit & 63) & 1) != 0;
|
||||
}
|
||||
|
||||
void collectResident(std::uintptr_t start, Run& run, std::uintptr_t from, std::uintptr_t stop, void** pages, std::size_t* count, bool clear, Collection collection) {
|
||||
const auto first = from & ~std::uintptr_t{4095};
|
||||
std::vector<std::uintptr_t> written;
|
||||
if (collection != Collection::Fresh) {
|
||||
std::vector<void*> found((stop - first + 4095) / 4096);
|
||||
ULONG_PTR reported = found.size();
|
||||
DWORD granularity = 0;
|
||||
if (GetWriteWatch(clear ? WRITE_WATCH_FLAG_RESET : 0, reinterpret_cast<void*>(first), stop - first, found.data(), &reported, &granularity) != 0) fail("collect resident guest writes");
|
||||
for (ULONG_PTR index = 0; index < reported; ++index) written.push_back(reinterpret_cast<std::uintptr_t>(found[index]));
|
||||
}
|
||||
auto next = written.begin();
|
||||
std::uintptr_t resetFrom = 0;
|
||||
const auto flushReset = [&](std::uintptr_t to) {
|
||||
if (resetFrom != 0 && ResetWriteWatch(reinterpret_cast<void*>(resetFrom), to - resetFrom) != 0) fail("reset resident guest writes");
|
||||
resetFrom = 0;
|
||||
};
|
||||
for (auto at = first; at < stop; at += 4096) {
|
||||
const auto bit = (at - start) >> 12;
|
||||
const bool fresh = !runCollected(run, bit);
|
||||
bool wrote = false;
|
||||
while (next != written.end() && *next < at) ++next;
|
||||
if (next != written.end() && *next == at) wrote = true;
|
||||
const bool report = collection == Collection::Fresh ? fresh : collection == Collection::Armed ? !fresh && wrote : fresh || wrote;
|
||||
if (report) pages[(*count)++] = reinterpret_cast<void*>(std::max(at, from));
|
||||
const bool settle = clear && fresh && collection != Collection::Armed;
|
||||
if (settle) {
|
||||
run.collected[bit >> 6] |= 1ull << (bit & 63);
|
||||
--run.fresh;
|
||||
forget(freshRanges, at, at + 4096);
|
||||
if (collection == Collection::Fresh && resetFrom == 0) resetFrom = at;
|
||||
} else if (collection == Collection::Fresh) {
|
||||
flushReset(at);
|
||||
}
|
||||
}
|
||||
if (collection == Collection::Fresh) flushReset(stop);
|
||||
}
|
||||
|
||||
Window open(HANDLE section, std::uint64_t offset, std::size_t bytes) {
|
||||
const auto lead = offset % granularity;
|
||||
Window window;
|
||||
window.view = map(section, GetCurrentProcess(), nullptr, offset - lead, lead + bytes, 0, PAGE_READWRITE, nullptr, 0);
|
||||
if (window.view == nullptr) fail("open a window on direct memory");
|
||||
window.bytes = static_cast<unsigned char*>(window.view) + lead;
|
||||
return window;
|
||||
}
|
||||
|
||||
void close(Window& window) {
|
||||
if (window.view != nullptr && !unmap(GetCurrentProcess(), window.view, 0)) fail("close a window on direct memory");
|
||||
window = {};
|
||||
}
|
||||
|
||||
void commitResident(std::uintptr_t base, std::size_t bytes) {
|
||||
const DWORD flags = MEM_RESERVE | MEM_COMMIT | MEM_REPLACE_PLACEHOLDER | (watchResident ? MEM_WRITE_WATCH : 0);
|
||||
if (!allocate(GetCurrentProcess(), reinterpret_cast<void*>(base), bytes, flags, PAGE_READWRITE, nullptr, 0)) fail("replace guest placeholder with resident direct memory");
|
||||
}
|
||||
|
||||
void mapShared(std::uintptr_t base, HANDLE section, std::uint64_t offset, DWORD protection) {
|
||||
void* page = reinterpret_cast<void*>(base);
|
||||
if (!map(section, GetCurrentProcess(), page, offset, pageBytes, MEM_REPLACE_PLACEHOLDER, PAGE_EXECUTE_READWRITE, nullptr, 0)) fail("map shared guest page");
|
||||
if (!VirtualAlloc(page, pageBytes, MEM_COMMIT, PAGE_EXECUTE_READWRITE)) fail("commit shared guest page");
|
||||
DWORD previous;
|
||||
if (!VirtualProtect(page, pageBytes, protection, &previous)) fail("protect shared guest page");
|
||||
}
|
||||
|
||||
static bool blank(std::uintptr_t page) {
|
||||
const auto* words = reinterpret_cast<const std::uint64_t*>(page);
|
||||
for (std::size_t i = 0; i < pageBytes / sizeof(std::uint64_t); ++i) {
|
||||
if (words[i] != 0) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void demote(std::uintptr_t address) {
|
||||
const auto run = std::prev(residentRuns.upper_bound(address));
|
||||
const auto start = run->first;
|
||||
const auto stop = run->second.end;
|
||||
DWORD previous;
|
||||
if (!VirtualProtect(reinterpret_cast<void*>(start), stop - start, PAGE_READONLY, &previous)) fail("fence resident guest pages");
|
||||
const auto section = views.at(start).section;
|
||||
auto window = open(section->handle, views.at(start).offset, stop - start);
|
||||
if (!VirtualAlloc(window.bytes, stop - start, MEM_COMMIT, PAGE_READWRITE)) fail("commit direct memory pages");
|
||||
std::memcpy(window.bytes, reinterpret_cast<const void*>(start), stop - start);
|
||||
close(window);
|
||||
if (!VirtualFree(reinterpret_cast<void*>(start), stop - start, MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) fail("release resident guest pages");
|
||||
for (auto page = start; page < stop; page += pageBytes) {
|
||||
auto& view = views.at(page);
|
||||
split(page, pageBytes);
|
||||
mapShared(page, view.section->handle, view.offset, view.protection);
|
||||
view.slot->stored = true;
|
||||
view.resident = false;
|
||||
view.seen.fill(0);
|
||||
view.armed = 0;
|
||||
view.collected = 0;
|
||||
for (std::size_t sub = 0; sub < subpages; ++sub) {
|
||||
if (runCollected(run->second, ((page - start) >> 12) + sub)) view.collected |= static_cast<std::uint8_t>(1u << sub);
|
||||
}
|
||||
invalidate(*view.page);
|
||||
}
|
||||
residentRuns.erase(run);
|
||||
}
|
||||
|
||||
void breakRun(std::uintptr_t at) {
|
||||
const auto next = residentRuns.upper_bound(at);
|
||||
if (next == residentRuns.begin()) return;
|
||||
const auto run = std::prev(next);
|
||||
if (at > run->first && at < run->second.end) demote(at);
|
||||
}
|
||||
|
||||
void evict(std::map<std::uintptr_t, Run>::iterator run) {
|
||||
const auto start = run->first;
|
||||
const auto stop = run->second.end;
|
||||
DWORD previous;
|
||||
if (!VirtualProtect(reinterpret_cast<void*>(start), stop - start, PAGE_READONLY, &previous)) fail("open resident guest pages");
|
||||
std::shared_ptr<Saved> saved;
|
||||
for (auto page = start; page < stop; page += pageBytes) {
|
||||
const auto found = views.find(page);
|
||||
auto& view = found->second;
|
||||
if (!blank(page)) {
|
||||
if (!saved) {
|
||||
auto* bytes = static_cast<unsigned char*>(VirtualAlloc(nullptr, stop - start, MEM_RESERVE, PAGE_READWRITE));
|
||||
if (bytes == nullptr) fail("reserve unmapped direct memory");
|
||||
saved = std::make_shared<Saved>(bytes);
|
||||
}
|
||||
if (!VirtualAlloc(saved->bytes + (page - start), pageBytes, MEM_COMMIT, PAGE_READWRITE)) fail("keep unmapped direct memory");
|
||||
std::memcpy(saved->bytes + (page - start), reinterpret_cast<const void*>(page), pageBytes);
|
||||
view.slot->saved = saved;
|
||||
view.slot->savedOffset = page - start;
|
||||
}
|
||||
std::erase(view.page->aliases, page);
|
||||
views.erase(found);
|
||||
}
|
||||
forget(freshRanges, start, stop);
|
||||
residentRuns.erase(run);
|
||||
}
|
||||
|
||||
static void forget(Ranges& ranges, std::uintptr_t start, std::uintptr_t end) {
|
||||
auto it = ranges.lower_bound(start);
|
||||
if (it != ranges.begin() && std::prev(it)->second > start) --it;
|
||||
|
|
@ -490,6 +747,9 @@ private:
|
|||
unmap = reinterpret_cast<UnmapFunction>(GetProcAddress(module, "UnmapViewOfFile2"));
|
||||
if (!allocate || !map || !unmap) throw std::runtime_error("Windows placeholder memory APIs are required");
|
||||
compare = reinterpret_cast<CompareFunction>(GetProcAddress(module, "CompareObjectHandles"));
|
||||
SYSTEM_INFO system{};
|
||||
GetSystemInfo(&system);
|
||||
granularity = system.dwAllocationGranularity;
|
||||
}
|
||||
|
||||
[[noreturn]] static void fail(const char* operation) {
|
||||
|
|
@ -518,6 +778,8 @@ private:
|
|||
void reset(std::uintptr_t address, std::size_t bytes) {
|
||||
const auto end = address + bytes;
|
||||
forgetClean(address, end);
|
||||
breakRun(address);
|
||||
breakRun(end);
|
||||
for (auto cursor = address; cursor < end;) {
|
||||
const auto memory = query(cursor);
|
||||
if (memory.State == MEM_RESERVE) {
|
||||
|
|
@ -541,6 +803,7 @@ private:
|
|||
forget(freshRanges, cursor, cursor + pageBytes);
|
||||
}
|
||||
} else if (memory.Type == MEM_PRIVATE) {
|
||||
if (const auto run = residentRuns.find(cursor); run != residentRuns.end()) evict(run);
|
||||
if (!VirtualFree(reinterpret_cast<void*>(cursor), allocationEnd - cursor, MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) fail("release private guest memory");
|
||||
} else {
|
||||
throw std::runtime_error("unsupported guest mapping type");
|
||||
|
|
@ -594,8 +857,11 @@ private:
|
|||
Ranges cleanRanges;
|
||||
Ranges freshRanges;
|
||||
std::map<std::uintptr_t, View> views;
|
||||
std::map<std::pair<std::uintptr_t, std::uint64_t>, std::weak_ptr<SharedPage>> physical;
|
||||
std::map<std::pair<std::uintptr_t, std::uint64_t>, Physical> physical;
|
||||
std::map<std::uintptr_t, Run> residentRuns;
|
||||
std::map<std::uintptr_t, std::size_t> splitAliases;
|
||||
std::size_t granularity = 0x10000;
|
||||
bool watchResident = false;
|
||||
std::mutex mutex;
|
||||
AllocateFunction allocate = nullptr;
|
||||
MapFunction map = nullptr;
|
||||
|
|
|
|||
|
|
@ -320,6 +320,10 @@ bool GuestArenaHandleWrite_nid_postfix(std::uintptr_t address) {
|
|||
return WindowsMappings::Get().HandleWrite(address);
|
||||
}
|
||||
|
||||
bool GuestArenaHandleRead_nid_postfix(std::uintptr_t address) {
|
||||
return WindowsMappings::Get().HandleRead(address);
|
||||
}
|
||||
|
||||
void GuestArenaPinWritable_nid_postfix(const void* pointer, std::size_t bytes) {
|
||||
if (!Arena::Get().Contains(pointer, bytes)) return;
|
||||
WindowsMappings::Get().Pin(reinterpret_cast<std::uintptr_t>(pointer), bytes);
|
||||
|
|
@ -374,7 +378,7 @@ void GuestArenaReset_nid_postfix(void* pointer, std::size_t bytes) {
|
|||
|
||||
void GuestArenaMap_nid_postfix(void* pointer, std::size_t bytes, void* section, std::uint64_t offset, std::uint32_t protection) {
|
||||
if (!Arena::Get().Contains(pointer, bytes)) throw OutsideArena("shared mapping", pointer, bytes);
|
||||
WindowsMappings::Get().Map(pointer, bytes, section, offset, protection);
|
||||
WindowsMappings::Get().Map(pointer, bytes, section, offset, protection, Arena::Get().WriteWatched());
|
||||
}
|
||||
|
||||
void* GuestArenaMapAlias_nid_postfix(std::uintptr_t address, std::size_t bytes) {
|
||||
|
|
@ -384,6 +388,14 @@ void* GuestArenaMapAlias_nid_postfix(std::uintptr_t address, std::size_t bytes)
|
|||
void GuestArenaUnmapAlias_nid_postfix(void* alias) {
|
||||
WindowsMappings::Get().UnmapAlias(alias);
|
||||
}
|
||||
|
||||
bool GuestArenaResidentSpan_nid_postfix(std::uintptr_t address, std::size_t bytes, std::uintptr_t* start, std::uintptr_t* end) {
|
||||
return WindowsMappings::Get().ResidentSpan(address, bytes, start, end);
|
||||
}
|
||||
|
||||
void GuestArenaForgetSection_nid_postfix(void* section) {
|
||||
WindowsMappings::Get().ForgetSection(section);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__linux__)
|
||||
|
|
|
|||
|
|
@ -246,6 +246,7 @@ LONG WINAPI ReportCrash(EXCEPTION_POINTERS* info) {
|
|||
static std::atomic<bool> reported{false};
|
||||
const auto* fault = info->ExceptionRecord;
|
||||
if (fault->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && fault->NumberParameters >= 2 && fault->ExceptionInformation[0] == 1 && GuestArena::GuestArenaHandleWrite_nid_postfix(fault->ExceptionInformation[1])) return EXCEPTION_CONTINUE_EXECUTION;
|
||||
if (fault->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && fault->NumberParameters >= 2 && fault->ExceptionInformation[0] == 0 && GuestArena::GuestArenaHandleRead_nid_postfix(fault->ExceptionInformation[1])) return EXCEPTION_CONTINUE_EXECUTION;
|
||||
if (HandleWatch(info)) return EXCEPTION_CONTINUE_EXECUTION;
|
||||
if (HandleSse4a(info)) return EXCEPTION_CONTINUE_EXECUTION;
|
||||
const auto* record = info->ExceptionRecord;
|
||||
|
|
|
|||
|
|
@ -302,6 +302,7 @@ public:
|
|||
|
||||
~PhysicalBacking() {
|
||||
#ifdef _WIN32
|
||||
GuestArena::GuestArenaForgetSection_nid_postfix(section);
|
||||
CloseHandle(section);
|
||||
#else
|
||||
if (hostWriteView != nullptr) ::munmap(hostWriteView, bytes);
|
||||
|
|
@ -552,11 +553,44 @@ void AddMapping(std::uintptr_t address, std::size_t len, std::uint64_t phys, int
|
|||
#endif
|
||||
}
|
||||
|
||||
void RequireResidentUnpinned(GuestAllocations::Mutation& mutation, std::uintptr_t address, std::size_t len) {
|
||||
#ifdef _WIN32
|
||||
std::uintptr_t start = 0;
|
||||
std::uintptr_t end = 0;
|
||||
if (GuestArena::GuestArenaResidentSpan_nid_postfix(address, len, &start, &end)) mutation.RequireUnpinned(reinterpret_cast<const void*>(start), end - start);
|
||||
#else
|
||||
static_cast<void>(mutation);
|
||||
static_cast<void>(address);
|
||||
static_cast<void>(len);
|
||||
#endif
|
||||
}
|
||||
|
||||
void RequireAliasesUnpinned(GuestAllocations::Mutation& mutation, std::uint64_t phys, std::size_t len) {
|
||||
#ifdef _WIN32
|
||||
std::vector<std::pair<std::uintptr_t, std::size_t>> aliases;
|
||||
{
|
||||
std::lock_guard lock(g_directLock);
|
||||
for (const auto& [base, mapping] : g_directMappings) {
|
||||
const auto first = std::max(phys, mapping.phys);
|
||||
const auto last = std::min(phys + len, mapping.phys + (mapping.end - base));
|
||||
if (first < last) aliases.emplace_back(base + (first - mapping.phys), last - first);
|
||||
}
|
||||
}
|
||||
for (const auto& [address, bytes] : aliases) RequireResidentUnpinned(mutation, address, bytes);
|
||||
#else
|
||||
static_cast<void>(mutation);
|
||||
static_cast<void>(phys);
|
||||
static_cast<void>(len);
|
||||
#endif
|
||||
}
|
||||
|
||||
bool RemapFixedIntoRegistered(GuestAllocations::Mutation& mutation, void* addr, size_t len, int prot, int flags, int64_t physStart = -1) {
|
||||
constexpr int GuestMapFixed = 0x10;
|
||||
constexpr int GuestMapNoOverwrite = 0x80;
|
||||
if (addr == nullptr || (flags & GuestMapFixed) == 0 || (flags & GuestMapNoOverwrite) != 0 || !mutation.Covers(addr, len)) return false;
|
||||
ValidateRange(addr, len, PS5_PAGE_SIZE);
|
||||
RequireResidentUnpinned(mutation, reinterpret_cast<std::uintptr_t>(addr), len);
|
||||
if (physStart >= 0) RequireAliasesUnpinned(mutation, static_cast<std::uint64_t>(physStart), len);
|
||||
Trace("remap fixed %p+0x%zx prot=0x%x phys=0x%llx", addr, len, prot, static_cast<long long>(physStart));
|
||||
const auto nativeProtection = LinuxProtFromSce(prot);
|
||||
mutation.Protect(addr, len, (prot & 3) != 0, (prot & 2) != 0, (prot & GuestProtGpuReadWrite) != 0, [&] {
|
||||
|
|
@ -798,6 +832,7 @@ bool Reserved(const void* addr, size_t len) {
|
|||
}
|
||||
|
||||
void UnmapRegistered(GuestAllocations::Mutation& mutation, void* addr, size_t len) {
|
||||
RequireResidentUnpinned(mutation, reinterpret_cast<std::uintptr_t>(addr), len);
|
||||
mutation.Unmap(addr, len, [&](const void* piece, std::size_t pieceBytes, const void* allocation, bool last) {
|
||||
auto* pieceAddress = const_cast<void*>(piece);
|
||||
std::lock_guard lock(g_directLock);
|
||||
|
|
@ -847,6 +882,7 @@ int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart,
|
|||
return 0;
|
||||
}
|
||||
ReplaceFixedOverlap(mutation, *addr, len, flags);
|
||||
RequireAliasesUnpinned(mutation, static_cast<std::uint64_t>(physStart), len);
|
||||
std::lock_guard lock(g_directLock);
|
||||
ValidatePhysicalRange(static_cast<std::uint64_t>(physStart), len);
|
||||
void* mapped = MapAligned(*addr, len, PROT_NONE, flags, alignment);
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@
|
|||
#ifdef _WIN32
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
#include <psapi.h>
|
||||
#endif
|
||||
|
||||
extern "C" {
|
||||
|
|
@ -1318,6 +1319,188 @@ static void CheckHeapAfterMappingReuse() {
|
|||
GuestHeap::GuestHeapFree_nid_postfix(pointer);
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
static int g_accessViolations = 0;
|
||||
|
||||
static LONG WINAPI CountAccessViolations(EXCEPTION_POINTERS* info) {
|
||||
if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) ++g_accessViolations;
|
||||
return EXCEPTION_CONTINUE_SEARCH;
|
||||
}
|
||||
#endif
|
||||
|
||||
static void CheckSingleDirectMappingTracksWritesWithoutFaults() {
|
||||
#ifdef _WIN32
|
||||
constexpr std::size_t page = 0x4000;
|
||||
std::int64_t phys = 0;
|
||||
Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * 4, 0, 0, &phys) == 0);
|
||||
void* mapped = nullptr;
|
||||
Require(sceKernelMapDirectMemory(&mapped, page * 4, 3, 0, phys, 0) == 0);
|
||||
const auto collect = [](void* address, std::size_t bytes) {
|
||||
std::array<void*, 32> pages{};
|
||||
std::size_t count = pages.size();
|
||||
Require(GuestArena::GuestArenaCollectWrites_nid_postfix(reinterpret_cast<std::uintptr_t>(address), bytes, pages.data(), &count, true));
|
||||
return count;
|
||||
};
|
||||
auto* bytes = static_cast<volatile unsigned char*>(mapped);
|
||||
Require(collect(mapped, page * 4) == 16);
|
||||
Require(collect(mapped, page * 4) == 0);
|
||||
void* const handler = AddVectoredExceptionHandler(1, CountAccessViolations);
|
||||
Require(handler != nullptr);
|
||||
bytes[page + 0x1000] = 7;
|
||||
bytes[page * 3] = 9;
|
||||
Require(RemoveVectoredExceptionHandler(handler) != 0);
|
||||
Require(g_accessViolations == 0);
|
||||
Require(collect(mapped, page * 4) == 2);
|
||||
Require(collect(mapped, page * 4) == 0);
|
||||
Require(sceKernelMunmap(static_cast<unsigned char*>(mapped) + page * 2, page) == 0);
|
||||
Require(bytes[page + 0x1000] == 7 && bytes[page * 3] == 9);
|
||||
bytes[0] = 5;
|
||||
Require(collect(mapped, page * 2) != 0);
|
||||
Require(collect(mapped, page * 2) == 0);
|
||||
void* second = nullptr;
|
||||
Require(sceKernelMapDirectMemory(&second, page, 3, 0, phys + page, 0) == 0);
|
||||
Require(static_cast<unsigned char*>(second)[0x1000] == 7);
|
||||
static_cast<volatile unsigned char*>(second)[0x1001] = 11;
|
||||
Require(bytes[page + 0x1001] == 11);
|
||||
Require(collect(mapped, page * 2) != 0);
|
||||
Require(sceKernelMunmap(second, page) == 0);
|
||||
Require(sceKernelMunmap(mapped, page * 2) == 0);
|
||||
Require(sceKernelMunmap(static_cast<unsigned char*>(mapped) + page * 3, page) == 0);
|
||||
void* again = nullptr;
|
||||
Require(sceKernelMapDirectMemory(&again, page * 4, 3, 0, phys, 0) == 0);
|
||||
const auto* data = static_cast<const unsigned char*>(again);
|
||||
Require(data[0] == 5 && data[page + 0x1000] == 7 && data[page + 0x1001] == 11 && data[page * 2] == 0 && data[page * 3] == 9);
|
||||
Require(sceKernelMunmap(again, page * 4) == 0);
|
||||
Require(sceKernelReleaseDirectMemory(phys, page * 4) == 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void CheckSecondDirectMappingWaitsForLeases() {
|
||||
#ifdef _WIN32
|
||||
constexpr std::size_t page = 0x4000;
|
||||
std::int64_t phys = 0;
|
||||
Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * 4, 0, 0, &phys) == 0);
|
||||
void* mapped = nullptr;
|
||||
Require(sceKernelMapDirectMemory(&mapped, page * 4, 3, 0, phys, 0) == 0);
|
||||
auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
|
||||
Require(std::count_if(lease.begin(), lease.end(), [&](const auto& range) { return range->address == reinterpret_cast<std::uintptr_t>(mapped); }) == 1);
|
||||
std::atomic<bool> released{false};
|
||||
std::thread holder([&] {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
released = true;
|
||||
lease.clear();
|
||||
});
|
||||
void* second = nullptr;
|
||||
Require(sceKernelMapDirectMemory(&second, page, 3, 0, phys + page, 0) == 0);
|
||||
Require(released);
|
||||
holder.join();
|
||||
static_cast<volatile unsigned char*>(second)[8] = 3;
|
||||
Require(static_cast<unsigned char*>(mapped)[page + 8] == 3);
|
||||
static_cast<volatile unsigned char*>(mapped)[page * 2 + 8] = 4;
|
||||
Require(sceKernelMunmap(second, page) == 0);
|
||||
Require(sceKernelMunmap(mapped, page * 4) == 0);
|
||||
Require(sceKernelReleaseDirectMemory(phys, page * 4) == 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void CheckBreakingAResidentRunKeepsConcurrentAccesses() {
|
||||
#ifdef _WIN32
|
||||
constexpr std::size_t page = 0x4000;
|
||||
constexpr std::size_t chunk = 0x200000;
|
||||
constexpr std::size_t chunks = 4;
|
||||
for (int round = 0; round < 20; ++round) {
|
||||
const bool unmap = round % 2 != 0;
|
||||
std::int64_t phys = 0;
|
||||
Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, chunk * chunks, 0, 0, &phys) == 0);
|
||||
void* mapped = nullptr;
|
||||
Require(sceKernelMapDirectMemory(&mapped, chunk * chunks, 3, 0, phys, 0) == 0);
|
||||
auto* memory = static_cast<volatile unsigned char*>(mapped);
|
||||
const auto value = [&](std::size_t at) { return static_cast<unsigned char>((at * 31 + static_cast<std::size_t>(round)) | 1); };
|
||||
const auto skipped = [&](std::size_t offset) { return unmap && offset >= page && offset < page * 2; };
|
||||
std::atomic<bool> go{false};
|
||||
std::atomic<bool> done{false};
|
||||
std::atomic<bool> consistent{true};
|
||||
std::thread writer([&] {
|
||||
while (!go) {}
|
||||
for (std::size_t offset = 0; offset < chunk; ++offset) {
|
||||
if (skipped(offset)) continue;
|
||||
for (std::size_t index = 0; index < chunks; ++index) memory[index * chunk + offset] = value(index * chunk + offset);
|
||||
}
|
||||
});
|
||||
std::thread reader([&] {
|
||||
while (!go) {}
|
||||
while (!done) {
|
||||
for (std::size_t offset = 0; offset < chunk && !done; offset += 61) {
|
||||
if (skipped(offset)) continue;
|
||||
for (std::size_t index = 0; index < chunks; ++index) {
|
||||
const unsigned char seen = memory[index * chunk + offset];
|
||||
if (seen != 0 && seen != value(index * chunk + offset)) consistent = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
std::array<void*, chunks> second{};
|
||||
go = true;
|
||||
for (std::size_t index = 0; index < chunks; ++index) {
|
||||
if (unmap) Require(sceKernelMunmap(static_cast<unsigned char*>(mapped) + index * chunk + page, page) == 0);
|
||||
else Require(sceKernelMapDirectMemory(&second[index], page, 3, 0, phys + static_cast<std::int64_t>(index * chunk + page), 0) == 0);
|
||||
}
|
||||
writer.join();
|
||||
done = true;
|
||||
reader.join();
|
||||
Require(consistent);
|
||||
for (std::size_t index = 0; index < chunks; ++index) {
|
||||
for (std::size_t offset = 0; offset < chunk; ++offset) {
|
||||
if (!skipped(offset)) Require(memory[index * chunk + offset] == value(index * chunk + offset));
|
||||
}
|
||||
if (unmap) continue;
|
||||
for (std::size_t offset = 0; offset < page; ++offset) Require(static_cast<unsigned char*>(second[index])[offset] == value(index * chunk + page + offset));
|
||||
Require(sceKernelMunmap(second[index], page) == 0);
|
||||
}
|
||||
for (std::size_t index = 0; index < chunks; ++index) {
|
||||
auto* base = static_cast<unsigned char*>(mapped) + index * chunk;
|
||||
if (!unmap) {
|
||||
Require(sceKernelMunmap(base, chunk) == 0);
|
||||
continue;
|
||||
}
|
||||
Require(sceKernelMunmap(base, page) == 0);
|
||||
Require(sceKernelMunmap(base + page * 2, chunk - page * 2) == 0);
|
||||
}
|
||||
Require(sceKernelReleaseDirectMemory(phys, chunk * chunks) == 0);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static void CheckRemappedDirectMemoryIsChargedOnce() {
|
||||
#ifdef _WIN32
|
||||
constexpr std::size_t bytes = 0x10000000;
|
||||
const auto committed = [] {
|
||||
PERFORMANCE_INFORMATION info{};
|
||||
info.cb = sizeof(info);
|
||||
Require(K32GetPerformanceInfo(&info, sizeof(info)) != 0);
|
||||
return static_cast<std::int64_t>(info.CommitTotal) * static_cast<std::int64_t>(info.PageSize);
|
||||
};
|
||||
bool once = false;
|
||||
for (int attempt = 0; attempt < 3 && !once; ++attempt) {
|
||||
const auto before = committed();
|
||||
std::int64_t phys = 0;
|
||||
Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, bytes, 0, 0, &phys) == 0);
|
||||
void* mapped = nullptr;
|
||||
Require(sceKernelMapDirectMemory(&mapped, bytes, 3, 0, phys, 0) == 0);
|
||||
std::memset(mapped, 0xa5, bytes);
|
||||
Require(sceKernelMunmap(mapped, bytes) == 0);
|
||||
void* again = nullptr;
|
||||
Require(sceKernelMapDirectMemory(&again, bytes, 3, 0, phys, 0) == 0);
|
||||
const auto* data = static_cast<const unsigned char*>(again);
|
||||
Require(data[0] == 0xa5 && data[bytes / 2] == 0xa5 && data[bytes - 1] == 0xa5);
|
||||
once = committed() - before < static_cast<std::int64_t>(bytes + bytes / 2);
|
||||
Require(sceKernelMunmap(again, bytes) == 0);
|
||||
Require(sceKernelReleaseDirectMemory(phys, bytes) == 0);
|
||||
}
|
||||
Require(once);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void CheckSharedWriteTracking() {
|
||||
#ifdef _WIN32
|
||||
constexpr std::size_t page = 0x4000;
|
||||
|
|
@ -1535,7 +1718,9 @@ static void CheckPinnedSharedPages() {
|
|||
std::int64_t phys = 0;
|
||||
Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * 3, 0, 0, &phys) == 0);
|
||||
void* mapped = nullptr;
|
||||
void* alias = nullptr;
|
||||
Require(sceKernelMapDirectMemory(&mapped, page * 3, 3, 0, phys, 0) == 0);
|
||||
Require(sceKernelMapDirectMemory(&alias, page * 3, 3, 0, phys, 0) == 0);
|
||||
auto* bytes = static_cast<volatile unsigned char*>(mapped);
|
||||
const auto collect = [&] {
|
||||
std::array<void*, 32> pages{};
|
||||
|
|
@ -1567,6 +1752,7 @@ static void CheckPinnedSharedPages() {
|
|||
Require(protection(page) == PAGE_READONLY);
|
||||
bytes[page + 8] = 11;
|
||||
Require(collect() == 1);
|
||||
Require(sceKernelMunmap(alias, page * 3) == 0);
|
||||
Require(sceKernelMunmap(mapped, page * 3) == 0);
|
||||
Require(sceKernelReleaseDirectMemory(phys, page * 3) == 0);
|
||||
#endif
|
||||
|
|
@ -2229,6 +2415,10 @@ int main() {
|
|||
CheckReadsIntoSharedWriteTracking();
|
||||
CheckPinnedSharedPages();
|
||||
CheckFailedCollectKeepsWrites();
|
||||
CheckSingleDirectMappingTracksWritesWithoutFaults();
|
||||
CheckSecondDirectMappingWaitsForLeases();
|
||||
CheckBreakingAResidentRunKeepsConcurrentAccesses();
|
||||
CheckRemappedDirectMemoryIsChargedOnce();
|
||||
#if defined(__linux__)
|
||||
CheckWriteWatch();
|
||||
CheckWriteWatchArmedCollect();
|
||||
|
|
|
|||
|
|
@ -32,8 +32,8 @@ struct Region {
|
|||
|
||||
Region() {
|
||||
Require(section && first && alias, "cannot allocate shared test memory");
|
||||
Mappings::Get().Map(first, Bytes, section, 0, PAGE_READWRITE);
|
||||
Mappings::Get().Map(alias, Bytes, section, 0, PAGE_READWRITE);
|
||||
Mappings::Get().Map(first, Bytes, section, 0, PAGE_READWRITE, true);
|
||||
Mappings::Get().Map(alias, Bytes, section, 0, PAGE_READWRITE, true);
|
||||
}
|
||||
~Region() {
|
||||
Mappings::Get().Reset(first, Bytes);
|
||||
|
|
@ -113,7 +113,7 @@ void CheckAliasesAndCapacity() {
|
|||
Require(region.Collect(Collection::Armed) == Pages(0, 1), "alias write was missed or fresh neighbors were consumed");
|
||||
Require(region.Collect(Collection::Fresh) == Pages(1, 16), "alias collection changed another view's history");
|
||||
Mappings::Get().Reset(region.first, Bytes);
|
||||
Mappings::Get().Map(region.first, Bytes, region.section, 0, PAGE_READWRITE);
|
||||
Mappings::Get().Map(region.first, Bytes, region.section, 0, PAGE_READWRITE, true);
|
||||
Require(region.Collect(Collection::Armed).empty(), "remapped address kept old collection history");
|
||||
Require(region.Collect(Collection::Fresh) == Pages(0, 16), "remapped memory was not fresh");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ Entries about one library or component are in [debt](debt/), one file per librar
|
|||
|
||||
### Direct memory
|
||||
|
||||
- Windows direct-memory backing reserves a paging-file section and commits each mapped guest page before setting its protection. Committed physical contents remain in the section across unmap/remap and are released when the backing and its remaining views are destroyed; commitment is not recovered merely by unmapping a page. Linux retains its existing backing. Peak gameplay memory and Windows versions other than the tested host remain unverified.
|
||||
- Windows direct-memory backing reserves a paging-file section. A [direct memory page](../../core/libs/prx/libc/include/WindowsMappings.hpp) mapped at one guest address is private write-watched memory at that address and is not committed in the section; a page mapped at a second address moves its whole resident run into the section (committed there) and every address becomes a view of it, tracked by write protection. An unmapped resident page that is not all zeros is kept in a private reservation until it is mapped again. Committed section contents remain across unmap/remap and are released when the backing and its remaining views are destroyed. A host import alias over part of a resident run is refused. Linux retains its existing backing. Peak gameplay memory and Windows versions other than the tested host remain unverified.
|
||||
|
||||
### Silent stubs
|
||||
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ Use the generated files printed as `Guest module:` for bundled title modules. `l
|
|||
|
||||
On Windows, a self-built `libs/` also needs `libgcc_s_seh-1.dll`, `libstdc++-6.dll` and `libwinpthread-1.dll` from the `mingw64/bin` directory of the toolchain that built the libraries; the release archives already contain them. The libraries are loaded without searching `PATH`, so a copy elsewhere on the system is not used, and a library that needs one of them fails with error 126 (the specified module could not be found) although its `.prx` file is present.
|
||||
|
||||
On Windows, direct memory (`sceKernelAllocateDirectMemory`, up to 13824 MiB per title) is committed in full when the title allocates it, not when its pages are first used. The system commit limit (installed memory plus page file size, the second value of Committed in Task Manager) must cover it together with all other committed memory. Otherwise the allocation throws `create direct memory backing of 0x<n> bytes (<m> MiB)` with the Windows error; enlarge the page file or close other applications.
|
||||
On Windows, direct memory (`sceKernelAllocateDirectMemory`, up to 13824 MiB per title) is committed when the title maps it, not when the title allocates it or first uses its pages. The system commit limit (installed memory plus page file size, the second value of Committed in Task Manager) must cover the mapped direct memory together with all other committed memory. Otherwise `sceKernelMapDirectMemory` throws `replace guest placeholder with resident direct memory` with the Windows error (`commit direct memory pages` when a page is mapped at a second address, `keep unmapped direct memory` when it is unmapped); enlarge the page file or close other applications.
|
||||
|
||||
On Linux, when the Vulkan driver imports dma-buf memory (not the NVIDIA proprietary driver), shared direct memory is imported through `/dev/udmabuf`, and the user who runs the game needs read-write access to it. Many distributions create it as `root:kvm` with mode `0660`: add the user to the `kvm` group and log in again (an ACL such as `setfacl -m u:$USER:rw /dev/udmabuf` lasts until the next reboot). Without access, startup prints `[gpu] open /dev/udmabuf: Permission denied`, these ranges are copied instead of imported, and GPU stores to them through FLAT/GLOBAL addresses fail with `BDA access failed`. Ranges above udmabuf's `size_limit_mb` (64 MiB by default) are copied as well.
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue