From 40df5282e66bec2bfadb3d6026efb4943338596c Mon Sep 17 00:00:00 2001 From: boykopovar Date: Fri, 25 Sep 2026 19:47:18 +0300 Subject: [PATCH] perf(agc): track guest memory ownership + optimize gpu synchronization --- .../Execution/include/GuestMemory.hpp | 1 + .../Execution/src/DeviceSynchronization.cpp | 17 ++ .../libSceAgcDriver/Execution/src/Driver.cpp | 3 +- .../Execution/src/GuestGpuRange.cpp | 18 ++ .../Execution/src/GuestMemory.cpp | 2 + .../Execution/src/VulkanDevice.cpp | 32 ++-- .../Graphics/include/DrawQueue.hpp | 5 + .../Graphics/include/GpuColorTransfer.hpp | 2 + .../Graphics/include/RenderCache.hpp | 12 +- .../Graphics/include/Resources.hpp | 1 + .../Graphics/src/ColorMemoryWriteback.cpp | 28 +++ .../Graphics/src/CommandCompletion.cpp | 15 ++ .../Graphics/src/DrawCompletion.cpp | 56 ++++++ .../Graphics/src/DrawQueue.cpp | 21 +-- .../Graphics/src/GpuColorTransfer.cpp | 10 - .../Graphics/src/RenderCache.cpp | 29 +-- .../Graphics/src/RenderMemoryOwnership.cpp | 62 +++++++ .../libSceAgcDriver/Graphics/src/State.cpp | 2 +- .../prx/libSceAgcDriver/PostCompile.cmake | 14 +- core/libs/prx/libc/CMakeLists.txt | 11 ++ .../prx/libc/include/GuestHeapStorage.hpp | 13 ++ .../prx/libc/include/GuestMemoryBacking.hpp | 18 ++ .../prx/libc/include/GuestMemoryTracking.hpp | 40 ++++ .../libc/include/MemoryBackingPlatform.hpp | 22 +++ .../libc/include/MemoryTrackingPlatform.hpp | 24 +++ core/libs/prx/libc/src/GuestAllocations.cpp | 8 +- core/libs/prx/libc/src/GuestHeap.cpp | 14 +- core/libs/prx/libc/src/GuestHeapStorage.cpp | 116 ++++++++++++ core/libs/prx/libc/src/GuestMemoryBacking.cpp | 96 ++++++++++ .../libs/prx/libc/src/GuestMemoryTracking.cpp | 173 ++++++++++++++++++ core/libs/prx/libc/src/MemoryBackingLinux.cpp | 70 +++++++ .../prx/libc/src/MemoryBackingWindows.cpp | 72 ++++++++ .../libs/prx/libc/src/MemoryTrackingLinux.cpp | 99 ++++++++++ .../prx/libc/src/MemoryTrackingWindows.cpp | 78 ++++++++ .../libkernel/DirectMemory/DirectMemory.cpp | 94 +--------- 35 files changed, 1105 insertions(+), 173 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/Execution/src/DeviceSynchronization.cpp create mode 100644 core/libs/prx/libSceAgcDriver/Execution/src/GuestGpuRange.cpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/ColorMemoryWriteback.cpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/CommandCompletion.cpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/DrawCompletion.cpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/src/RenderMemoryOwnership.cpp create mode 100644 core/libs/prx/libc/include/GuestHeapStorage.hpp create mode 100644 core/libs/prx/libc/include/GuestMemoryBacking.hpp create mode 100644 core/libs/prx/libc/include/GuestMemoryTracking.hpp create mode 100644 core/libs/prx/libc/include/MemoryBackingPlatform.hpp create mode 100644 core/libs/prx/libc/include/MemoryTrackingPlatform.hpp create mode 100644 core/libs/prx/libc/src/GuestHeapStorage.cpp create mode 100644 core/libs/prx/libc/src/GuestMemoryBacking.cpp create mode 100644 core/libs/prx/libc/src/GuestMemoryTracking.cpp create mode 100644 core/libs/prx/libc/src/MemoryBackingLinux.cpp create mode 100644 core/libs/prx/libc/src/MemoryBackingWindows.cpp create mode 100644 core/libs/prx/libc/src/MemoryTrackingLinux.cpp create mode 100644 core/libs/prx/libc/src/MemoryTrackingWindows.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index a849d8824..ab0b5fe4f 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -8,6 +8,7 @@ namespace AgcDriver::GuestMemory { void CheckRange(const void* pointer, std::size_t bytes, std::size_t alignment, bool writable = false); +void CheckGpuRange(const void* pointer, std::size_t bytes, std::size_t alignment, bool writable = false); void Read(std::uint64_t address, std::span destination, std::size_t alignment = 1); void Write(std::uint64_t address, std::span source, std::size_t alignment = 1); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/DeviceSynchronization.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/DeviceSynchronization.cpp new file mode 100644 index 000000000..fbefc93ec --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/DeviceSynchronization.cpp @@ -0,0 +1,17 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" +#include "prx/libc/include/GuestMemoryTracking.hpp" + +namespace AgcDriver { + +void VulkanDevice::AcquireGpuMemory() { + std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); + PerformanceTimer timing("Vulkan.AcquireGpuMemory"); + const auto context = graphicsContext(); + Graphics::Require(context.drawQueue != nullptr, "GPU memory barrier requires a draw queue"); + context.drawQueue->RecordMemoryBarrier(context); + timing.Mark("barrier_record"); +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp index 0e067bac3..efd60cf2e 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp @@ -9,6 +9,7 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "prx/libc/include/Shutdown.hpp" #include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" +#include "prx/libc/include/GuestMemoryTracking.hpp" #include #include #include @@ -279,7 +280,7 @@ private: std::map> shaders; std::map queues; std::map> outputs; - std::mutex gpuMutex; + std::recursive_mutex& gpuMutex = GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix(); std::shared_ptr device; std::uint64_t accepted = 0; std::uint64_t completed = 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestGpuRange.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestGpuRange.cpp new file mode 100644 index 000000000..b0da605d1 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestGpuRange.cpp @@ -0,0 +1,18 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" +#include "prx/libc/include/GuestMemoryTracking.hpp" +#include +#include + +namespace AgcDriver::GuestMemory { + +void CheckGpuRange(const void* pointer, std::size_t bytes, std::size_t alignment, bool writable) { + const auto address = reinterpret_cast(pointer); + if (alignment == 0 || address == 0 || address % alignment != 0 || bytes > std::numeric_limits::max() - address) throw std::invalid_argument("invalid guest GPU memory range"); + const MemoryAccessScope suspended(nullptr, nullptr); + GuestMemoryTracking::GuestMemoryTrackingValidate_nid_postfix(address, bytes, [writable](std::uint64_t first, std::size_t count) { + CheckRange(reinterpret_cast(first), count, 1, writable); + }); +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index 2e16944e6..52e4ccc09 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libc/include/GuestMemoryTracking.hpp" #include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include @@ -29,6 +30,7 @@ void CheckRange(const void* pointer, std::size_t bytes, std::size_t alignment, b require(address != 0 && address % alignment == 0, "null or misaligned address"); require(bytes <= std::numeric_limits::max() - address, "address range overflow"); MemoryAccessScope::Resolve(address, bytes, writable); + GuestMemoryTracking::GuestMemoryTrackingResolve_nid_postfix(address, bytes, writable); auto cursor = address; const auto end = address + bytes; #ifdef _WIN32 diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 398a8d8ec..6f28b5525 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -165,7 +165,10 @@ struct VulkanDevice::State { } ~State() { + std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); if (device != VK_NULL_HANDLE) { + if (drawQueue) drawQueue->Wait(); + if (renderCache) renderCache->Flush(); const auto idle = reinterpret_cast(deviceProc(device, "vkDeviceWaitIdle"))(device); if (idle != VK_SUCCESS && idle != VK_ERROR_DEVICE_LOST) std::terminate(); drawQueue.reset(); @@ -490,41 +493,26 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u VulkanDevice::~VulkanDevice() = default; void VulkanDevice::WaitIdle() { + std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); PerformanceTimer timing("Vulkan.WaitIdle"); state->drawQueue->Wait(); timing.Mark("draw_wait"); - state->renderCache->Flush(); - timing.Mark("render_cache_flush"); check(state->DeviceFunction("vkDeviceWaitIdle")(state->device), "vkDeviceWaitIdle"); timing.Mark("device_wait"); } void VulkanDevice::WaitDraws() { + std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); PerformanceTimer timing("Vulkan.WaitDraws"); state->drawQueue->Wait(); } -void VulkanDevice::AcquireGpuMemory() { - PerformanceTimer timing("Vulkan.AcquireGpuMemory"); - state->drawQueue->Wait(); - timing.Mark("draw_wait"); - const auto context = graphicsContext(); - Graphics::CommandBatch batch(context); - VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT; - barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; - context.Function("vkCmdPipelineBarrier")(batch.Handle(), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr); - batch.SubmitAndWait(); - timing.Mark("gpu_barrier"); - state->renderCache->InvalidateClean(); - timing.Mark("invalidate_clean"); -} - void* VulkanDevice::Window() const { return state->window; } void VulkanDevice::Resize(std::uint32_t width, std::uint32_t height) { + std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); require(state->swapchain != VK_NULL_HANDLE, "cannot resize an unavailable swapchain"); if (width == 0 || height == 0) { state->extent = {0, 0}; @@ -592,7 +580,10 @@ void VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { } void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaque, std::span pixels, const DisplayBuffer* display) { + std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); PerformanceTimer timing("Vulkan.Present"); + state->drawQueue->Wait(); + timing.Mark("draw_wait"); require(state->swapchain != VK_NULL_HANDLE, "device has no swapchain"); require(state->extent.width != 0 && state->extent.height != 0, "output window is minimized"); std::shared_ptr resident; @@ -746,6 +737,7 @@ Graphics::Context VulkanDevice::graphicsContext() const { } void VulkanDevice::ResolveMemory(std::uint64_t address, std::size_t bytes, bool writable) { + std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); state->drawQueue->Resolve(address, bytes); state->renderCache->Resolve(address, bytes, writable); } @@ -756,6 +748,7 @@ void VulkanDevice::Draw(const Graphics::State& graphics, const Pm4::DrawParamete } void VulkanDevice::EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots) { + std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); const GuestMemory::MemoryAccessScope memoryScope(this, [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { static_cast(context)->ResolveMemory(address, bytes, writable); }); @@ -764,6 +757,7 @@ void VulkanDevice::EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawP } void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots) { + std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); PerformanceTimer timing("Vulkan.Dispatch"); const GuestMemory::MemoryAccessScope memoryScope(this, [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { static_cast(context)->ResolveMemory(address, bytes, writable); @@ -782,6 +776,8 @@ void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std if (x > limit[0] || y > limit[1] || z > limit[2]) { throw std::runtime_error("Vulkan dispatch: workgroup count exceeds device limits"); } + state->drawQueue->Wait(); + timing.Mark("draw_wait"); const auto destroyModule = state->DeviceFunction("vkDestroyShaderModule"); const auto destroyLayout = state->DeviceFunction("vkDestroyPipelineLayout"); const auto destroyPipeline = state->DeviceFunction("vkDestroyPipeline"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp index 3f7081e9d..fc5a053ad 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp @@ -13,6 +13,9 @@ public: void Flush(); void Resolve(std::uint64_t address, std::size_t bytes); void Wait(); + void WaitGpu(); + void Collect(); + void RecordMemoryBarrier(const Context& context); private: struct Entry { @@ -22,7 +25,9 @@ private: struct Batch { std::vector entries; std::unique_ptr commands; + bool hasBarrier = false; }; + void retire(Batch batch); Batch recording; std::vector pending; std::vector> available; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp index 10dc5511f..72cbe172e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp @@ -16,11 +16,13 @@ public: void Detile(VkCommandBuffer commands, bool swapRedBlue = false); void Tile(VkCommandBuffer commands); void WriteBack(std::uint64_t address); + void WriteBackTracked(std::uint64_t address); bool MatchesGuest(std::uint64_t address); VkBuffer LinearBuffer() const; RenderTarget& Target(const ColorTarget& color, bool blending); private: + void writeBack(std::uint64_t address, bool tracked); void prepare(std::uint32_t width, std::uint32_t height, ColorTileMode mode); void convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue); void release() noexcept; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/RenderCache.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/RenderCache.hpp index 0b1af36bc..601c5cbaf 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/RenderCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/RenderCache.hpp @@ -2,6 +2,7 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RENDERCACHE_HPP #include "prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp" +#include "prx/libc/include/GuestMemoryTracking.hpp" #include namespace AgcDriver::Graphics { @@ -9,6 +10,7 @@ namespace AgcDriver::Graphics { class ResidentColor { public: ResidentColor(const Context& context, const ColorTarget& color); + ~ResidentColor(); void Begin(VkCommandBuffer commands); void Download(VkCommandBuffer commands); void Commit(); @@ -18,28 +20,30 @@ public: bool Valid() const { return valid; } bool Dirty() const { return dirty; } std::uint64_t Generation() const { return generation; } - bool MatchesGuest(); - void Invalidate() { valid = false; } + void Invalidate(); + void ReleaseMemory(); + bool SharesPages(const ColorTarget& other) const; private: + void resolveCpuAccess(GuestMemoryTracking::Access access); Context context; ColorTarget color; GpuColorTransfer transfer; VkImageLayout layout = VK_IMAGE_LAYOUT_UNDEFINED; bool valid = false; bool dirty = false; - bool hasSnapshot = false; std::uint64_t generation = 0; + std::unique_ptr memoryWatch; }; class RenderCache { public: explicit RenderCache(const Context& context) : context(context) {} + ~RenderCache(); std::shared_ptr Get(const ColorTarget& color, bool blending); std::shared_ptr Find(std::uint64_t address) const; void Resolve(std::uint64_t address, std::size_t bytes, bool writable); void Flush(); - void InvalidateClean(); private: Context context; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp index 841435d03..c205f5bb4 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp @@ -60,6 +60,7 @@ public: void SubmitAndWait(); void Submit(); void Wait(); + bool IsComplete(); void Reset(); private: diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorMemoryWriteback.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorMemoryWriteback.cpp new file mode 100644 index 000000000..8dbbf04a5 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorMemoryWriteback.cpp @@ -0,0 +1,28 @@ +#include "prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include "prx/libc/include/GuestMemoryBacking.hpp" + +namespace AgcDriver::Graphics { + +void GpuColorTransfer::WriteBack(std::uint64_t address) { + writeBack(address, false); +} + +void GpuColorTransfer::WriteBackTracked(std::uint64_t address) { + writeBack(address, true); +} + +void GpuColorTransfer::writeBack(std::uint64_t address, bool tracked) { + PerformanceTimer timing("ColorTransfer.WriteBack"); + Require(tiled != nullptr && readback != nullptr, "color transfer is not prepared for writeback"); + const ColorTargetLayout layout(width, height, mode); + Require(address != 0 && address % layout.Alignment() == 0, "misaligned color writeback address"); + timing.Mark("validate"); + readback->Invalidate(); + if (tracked) GuestMemoryBacking::GuestMemoryBackingWrite_nid_postfix(address, readback->Bytes().data(), readback->Bytes().size()); + else GuestMemory::Write(address, readback->Bytes(), layout.Alignment()); + timing.Mark("guest_write", layout.Bytes()); +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/CommandCompletion.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/CommandCompletion.cpp new file mode 100644 index 000000000..283c14f85 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/CommandCompletion.cpp @@ -0,0 +1,15 @@ +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" + +namespace AgcDriver::Graphics { + +bool CommandBatch::IsComplete() { + Require(submitted, "cannot query an unsubmitted command batch"); + if (!pending) return true; + const auto result = context.Function("vkGetFenceStatus")(context.device, fence); + if (result == VK_NOT_READY) return false; + Check(result, "vkGetFenceStatus graphics"); + pending = false; + return true; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DrawCompletion.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DrawCompletion.cpp new file mode 100644 index 000000000..460eecaf3 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DrawCompletion.cpp @@ -0,0 +1,56 @@ +#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" +#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" + +namespace AgcDriver::Graphics { + +void DrawQueue::retire(Batch batch) { + PerformanceTimer timing("Graphics.DrawQueue.Retire"); + const GuestMemory::MemoryAccessScope suspended(nullptr, nullptr); + Require(drawCount >= batch.entries.size(), "draw queue completion count underflow"); + drawCount -= batch.entries.size(); + for (auto& entry : batch.entries) entry.resources->WriteBack(); + timing.Mark("resources_writeback"); + batch.entries.clear(); + available.push_back(std::move(batch.commands)); + timing.Mark("resources_release"); +} + +void DrawQueue::Collect() { + while (!pending.empty() && pending.front().commands->IsComplete()) { + auto batch = std::move(pending.front()); + pending.erase(pending.begin()); + retire(std::move(batch)); + } +} + +void DrawQueue::WaitGpu() { + Flush(); + for (auto& batch : pending) batch.commands->Wait(); +} + +void DrawQueue::Wait() { + if (pending.empty() && !recording.commands) return; + PerformanceTimer timing("Graphics.DrawQueue.Wait"); + Flush(); + timing.Mark("submit"); + while (!pending.empty()) { + pending.front().commands->Wait(); + timing.Mark("fence_wait"); + auto batch = std::move(pending.front()); + pending.erase(pending.begin()); + retire(std::move(batch)); + timing.Mark("retire"); + } +} + +void DrawQueue::RecordMemoryBarrier(const Context& context) { + const auto commands = Begin(context); + VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr); + recording.hasBarrier = true; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DrawQueue.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DrawQueue.cpp index 67f39029f..722df65bf 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DrawQueue.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DrawQueue.cpp @@ -11,6 +11,7 @@ DrawQueue::~DrawQueue() { } VkCommandBuffer DrawQueue::Begin(const Context& context) { + Collect(); if (drawCount >= 64) Wait(); if (!recording.commands) { if (available.empty()) recording.commands = std::make_unique(context); @@ -32,7 +33,7 @@ void DrawQueue::Enqueue(std::shared_ptr resources, std::shared_ void DrawQueue::Flush() { if (!recording.commands) return; - Require(!recording.entries.empty(), "cannot submit an incomplete draw batch"); + Require(!recording.entries.empty() || recording.hasBarrier, "cannot submit an incomplete draw batch"); pending.push_back(std::move(recording)); recording = Batch{}; pending.back().commands->Submit(); @@ -43,23 +44,5 @@ void DrawQueue::Resolve(std::uint64_t address, std::size_t bytes) { if (std::any_of(recording.entries.begin(), recording.entries.end(), overlaps) || std::any_of(pending.begin(), pending.end(), [&](const auto& batch) { return std::any_of(batch.entries.begin(), batch.entries.end(), overlaps); })) Wait(); } -void DrawQueue::Wait() { - if (pending.empty() && !recording.commands) return; - PerformanceTimer timing("Graphics.DrawQueue.Wait"); - const GuestMemory::MemoryAccessScope suspended(nullptr, nullptr); - Flush(); - timing.Mark("submit"); - for (auto& batch : pending) { - batch.commands->Wait(); - timing.Mark("fence_wait"); - for (auto& entry : batch.entries) entry.resources->WriteBack(); - timing.Mark("resources_writeback"); - available.push_back(std::move(batch.commands)); - batch.entries.clear(); - timing.Mark("resources_release"); - } - pending.clear(); - drawCount = 0; -} } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp index 8f778b199..1f01f2768 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp @@ -149,16 +149,6 @@ void GpuColorTransfer::Tile(VkCommandBuffer commands) { barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &after, 0, nullptr, 0, nullptr); } -void GpuColorTransfer::WriteBack(std::uint64_t address) { - PerformanceTimer timing("ColorTransfer.WriteBack"); - Require(tiled != nullptr, "color transfer is not prepared for writeback"); - const ColorTargetLayout layout(width, height, mode); - timing.Mark("validate"); - readback->Invalidate(); - GuestMemory::Write(address, readback->Bytes(), layout.Alignment()); - timing.Mark("guest_write", layout.Bytes()); -} - VkBuffer GpuColorTransfer::LinearBuffer() const { Require(linear != nullptr, "color transfer is not prepared"); return linear->Handle(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/RenderCache.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/RenderCache.cpp index ac2134ce1..ef51eac56 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/RenderCache.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/RenderCache.cpp @@ -6,8 +6,6 @@ namespace AgcDriver::Graphics { -ResidentColor::ResidentColor(const Context& context, const ColorTarget& color) : context(context), color(color), transfer(context) {} - void ResidentColor::Transition(VkCommandBuffer commands, VkImageLayout next) { VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; barrier.srcAccessMask = layout == VK_IMAGE_LAYOUT_UNDEFINED ? 0u : VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; @@ -26,7 +24,9 @@ void ResidentColor::Begin(VkCommandBuffer commands) { PerformanceTimer timing("Graphics.ResidentColor.Begin"); Require(generation != std::numeric_limits::max(), "resident color generation overflow"); ++generation; - if (!valid && !MatchesGuest()) { + Require(memoryWatch != nullptr, "render target memory ownership was released"); + if (!valid) { + memoryWatch->Protect(GuestMemoryTracking::Protection::Read); const GuestMemory::MemoryAccessScope suspended(nullptr, nullptr); transfer.Upload(color.address, color.extent.width, color.extent.height, color.tileMode); transfer.Detile(commands); @@ -42,7 +42,7 @@ void ResidentColor::Begin(VkCommandBuffer commands) { Transition(commands, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); valid = true; dirty = true; - hasSnapshot = false; + memoryWatch->Protect(GuestMemoryTracking::Protection::None); } void ResidentColor::Download(VkCommandBuffer commands) { @@ -62,14 +62,9 @@ void ResidentColor::Download(VkCommandBuffer commands) { void ResidentColor::Commit() { if (!dirty) return; const GuestMemory::MemoryAccessScope suspended(nullptr, nullptr); - transfer.WriteBack(color.address); + transfer.WriteBackTracked(color.address); dirty = false; - hasSnapshot = true; -} - -bool ResidentColor::MatchesGuest() { - const GuestMemory::MemoryAccessScope suspended(nullptr, nullptr); - return hasSnapshot && transfer.MatchesGuest(color.address); + memoryWatch->Protect(GuestMemoryTracking::Protection::Read); } std::shared_ptr RenderCache::Get(const ColorTarget& color, bool blending) { @@ -82,16 +77,18 @@ std::shared_ptr RenderCache::Get(const ColorTarget& color, bool b } for (auto it = entries.begin(); it != entries.end();) { const auto& previous = it->second->Description(); - if (color.address >= previous.address + previous.bytes || previous.address >= color.address + color.bytes) { + if (!it->second->SharesPages(color)) { ++it; continue; } if (previous.address == color.address && previous.bytes == color.bytes && previous.extent.width == color.extent.width && previous.extent.height == color.extent.height && previous.format == color.format && previous.tileMode == color.tileMode) return it->second; Resolve(previous.address, previous.bytes, true); + it->second->ReleaseMemory(); it = entries.erase(it); } if (entries.size() >= 64) { Flush(); + for (const auto& [address, resident] : entries) resident->ReleaseMemory(); entries.clear(); } auto entry = std::make_shared(context, color); @@ -102,7 +99,7 @@ std::shared_ptr RenderCache::Get(const ColorTarget& color, bool b std::shared_ptr RenderCache::Find(std::uint64_t address) const { const auto it = entries.find(address); if (it != entries.end() && context.drawQueue) context.drawQueue->Resolve(address, it->second->Description().bytes); - return it != entries.end() && (it->second->Valid() || it->second->MatchesGuest()) ? it->second : nullptr; + return it != entries.end() && it->second->Valid() ? it->second : nullptr; } void RenderCache::Resolve(std::uint64_t address, std::size_t bytes, bool writable) { @@ -131,10 +128,4 @@ void RenderCache::Flush() { Resolve(0, std::numeric_limits::max(), true); } -void RenderCache::InvalidateClean() { - for (const auto& [address, entry] : entries) { - if (!entry->Dirty()) entry->Invalidate(); - } -} - } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/RenderMemoryOwnership.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/RenderMemoryOwnership.cpp new file mode 100644 index 000000000..f76dd122d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/RenderMemoryOwnership.cpp @@ -0,0 +1,62 @@ +#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include + +namespace AgcDriver::Graphics { + +ResidentColor::ResidentColor(const Context& context, const ColorTarget& color) : context(context), color(color), transfer(context) { + memoryWatch = std::make_unique(color.address, color.bytes, this, [](void* owner, GuestMemoryTracking::Access access) { + static_cast(owner)->resolveCpuAccess(access); + }); +} + +ResidentColor::~ResidentColor() = default; + +void ResidentColor::Invalidate() { + Require(!dirty, "cannot discard GPU-owned render target contents"); + if (memoryWatch) memoryWatch->Protect(GuestMemoryTracking::Protection::ReadWrite); + valid = false; +} + +void ResidentColor::ReleaseMemory() { + Require(!dirty, "cannot release GPU-owned render target memory"); + Invalidate(); + memoryWatch.reset(); +} + +bool ResidentColor::SharesPages(const ColorTarget& other) const { + const auto pageSize = GuestMemoryTracking::GuestMemoryTrackingPageSize_nid_postfix(); + Require(color.bytes != 0 && other.bytes != 0, "empty render target page range"); + Require(color.bytes <= std::numeric_limits::max() - color.address && other.bytes <= std::numeric_limits::max() - other.address, "render target page range overflow"); + const auto first = color.address / pageSize; + const auto last = (color.address + color.bytes - 1) / pageSize; + const auto otherFirst = other.address / pageSize; + const auto otherLast = (other.address + other.bytes - 1) / pageSize; + return first <= otherLast && otherFirst <= last; +} + +void ResidentColor::resolveCpuAccess(GuestMemoryTracking::Access access) { + PerformanceTimer timing("Graphics.RenderMemory.CpuAccess"); + Require(memoryWatch != nullptr && context.drawQueue != nullptr, "render target memory resolver is unavailable"); + if (dirty || access == GuestMemoryTracking::Access::Invalidate) { + context.drawQueue->WaitGpu(); + timing.Mark("draw_wait"); + } + if (dirty) { + CommandBatch batch(context); + Download(batch.Handle()); + batch.SubmitAndWait(); + timing.Mark("download_wait"); + Commit(); + timing.Mark("guest_writeback"); + } + if (access != GuestMemoryTracking::Access::Read) Invalidate(); + if (access == GuestMemoryTracking::Access::Invalidate) context.drawQueue->Wait(); +} + +RenderCache::~RenderCache() { + for (const auto& [address, entry] : entries) entry->ReleaseMemory(); +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp index c7b4efdde..c759c5dfb 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp @@ -208,7 +208,7 @@ State DecodeState(const QueueState& queue) { Require((high & ~0xffu) == 0, "invalid color address extension"); result.color.address = (static_cast(high) << 40u) | (static_cast(read(cx, 0x318)) << 8u); result.color.bytes = colorLayout.Bytes(); - GuestMemory::CheckRange(reinterpret_cast(result.color.address), result.color.bytes, colorLayout.Alignment(), true); + GuestMemory::CheckGpuRange(reinterpret_cast(result.color.address), result.color.bytes, colorLayout.Alignment(), true); result.color.format = swap == 0 ? (number == 0 ? VK_FORMAT_R8G8B8A8_UNORM : VK_FORMAT_R8G8B8A8_SRGB) : (number == 0 ? VK_FORMAT_B8G8R8A8_UNORM : VK_FORMAT_B8G8R8A8_SRGB); result.color.componentMapping = 0xe4u; result.renderExtent = result.color.extent; diff --git a/core/libs/prx/libSceAgcDriver/PostCompile.cmake b/core/libs/prx/libSceAgcDriver/PostCompile.cmake index 0232474ba..072eeed37 100644 --- a/core/libs/prx/libSceAgcDriver/PostCompile.cmake +++ b/core/libs/prx/libSceAgcDriver/PostCompile.cmake @@ -12,7 +12,13 @@ add_custom_command( foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests agc_driver_bda_device_tests) if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Graphics/src/BufferPool.cpp) + target_sources(${agcTarget} PRIVATE Graphics/src/BufferPool.cpp Graphics/src/CommandCompletion.cpp) + endif() +endforeach() + +foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests agc_driver_bda_device_tests) + if(TARGET ${agcTarget}) + target_sources(${agcTarget} PRIVATE Execution/src/GuestGpuRange.cpp) endif() endforeach() @@ -24,7 +30,7 @@ endforeach() foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests agc_driver_bda_device_tests) if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Graphics/src/GpuColorTransfer.cpp ${agcColorTransferHeader}) + target_sources(${agcTarget} PRIVATE Graphics/src/GpuColorTransfer.cpp Graphics/src/ColorMemoryWriteback.cpp ${agcColorTransferHeader}) endif() endforeach() @@ -37,12 +43,12 @@ endif() foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests) if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Graphics/src/RenderCache.cpp Graphics/src/DrawQueue.cpp Graphics/src/RenderTexture.cpp Graphics/src/DescriptorCache.cpp) + target_sources(${agcTarget} PRIVATE Graphics/src/RenderCache.cpp Graphics/src/RenderMemoryOwnership.cpp Graphics/src/DrawQueue.cpp Graphics/src/DrawCompletion.cpp Graphics/src/RenderTexture.cpp Graphics/src/DescriptorCache.cpp) endif() endforeach() foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test) if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Execution/src/PresentationImage.cpp Graphics/src/GraphicsPipelineCache.cpp) + target_sources(${agcTarget} PRIVATE Execution/src/PresentationImage.cpp Execution/src/DeviceSynchronization.cpp Graphics/src/GraphicsPipelineCache.cpp) endif() endforeach() diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index 6ae1d0f20..3851f8563 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -8,6 +8,9 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/Time.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/FileAndHeap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestAllocations.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestMemoryTracking.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestMemoryBacking.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestHeapStorage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/GuestHeap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/ApplicationHeap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/ApplicationProcessParameters.cpp @@ -20,6 +23,14 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/exception/Exports.cpp ) +if(WIN32) + target_sources(libc PRIVATE src/MemoryTrackingWindows.cpp src/MemoryBackingWindows.cpp) +elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64)$") + target_sources(libc PRIVATE src/MemoryTrackingLinux.cpp src/MemoryBackingLinux.cpp) +else() + message(FATAL_ERROR "Guest GPU memory tracking requires Windows or x86-64 Linux") +endif() + target_include_directories(libc PRIVATE ${LIBS_INCLUDE_DIR} ) diff --git a/core/libs/prx/libc/include/GuestHeapStorage.hpp b/core/libs/prx/libc/include/GuestHeapStorage.hpp new file mode 100644 index 000000000..22600050d --- /dev/null +++ b/core/libs/prx/libc/include/GuestHeapStorage.hpp @@ -0,0 +1,13 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_GUESTHEAPSTORAGE_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_GUESTHEAPSTORAGE_HPP + +#include + +namespace GuestHeapStorage { + +void* Allocate(std::size_t alignment, std::size_t bytes); +void Free(void* pointer); + +} + +#endif diff --git a/core/libs/prx/libc/include/GuestMemoryBacking.hpp b/core/libs/prx/libc/include/GuestMemoryBacking.hpp new file mode 100644 index 000000000..1ac03e17a --- /dev/null +++ b/core/libs/prx/libc/include/GuestMemoryBacking.hpp @@ -0,0 +1,18 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_GUESTMEMORYBACKING_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_GUESTMEMORYBACKING_HPP + +#include +#include + +namespace GuestMemoryBacking { + +extern "C" { +void* GuestMemoryBackingMap_nid_postfix(void* address, std::size_t bytes, std::size_t alignment, int protection); +void GuestMemoryBackingUnmap_nid_postfix(void* address, std::size_t bytes); +void GuestMemoryBackingRequire_nid_postfix(std::uint64_t address, std::size_t bytes); +void GuestMemoryBackingWrite_nid_postfix(std::uint64_t address, const void* source, std::size_t bytes); +} + +} + +#endif diff --git a/core/libs/prx/libc/include/GuestMemoryTracking.hpp b/core/libs/prx/libc/include/GuestMemoryTracking.hpp new file mode 100644 index 000000000..669543a67 --- /dev/null +++ b/core/libs/prx/libc/include/GuestMemoryTracking.hpp @@ -0,0 +1,40 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_GUESTMEMORYTRACKING_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_GUESTMEMORYTRACKING_HPP + +#include +#include +#include +#include + +namespace GuestMemoryTracking { + +enum class Access { Read, Write, Invalidate }; +enum class Protection { None, Read, ReadWrite }; +using Resolver = void (*)(void*, Access); + +extern "C" { +std::recursive_mutex& GuestMemoryTrackingMutex_nid_postfix(); +std::size_t GuestMemoryTrackingPageSize_nid_postfix(); +void* GuestMemoryTrackingCreate_nid_postfix(std::uint64_t address, std::size_t bytes, void* context, Resolver resolver); +void GuestMemoryTrackingDestroy_nid_postfix(void* handle) noexcept; +void GuestMemoryTrackingProtect_nid_postfix(void* handle, Protection protection); +void GuestMemoryTrackingResolve_nid_postfix(std::uint64_t address, std::size_t bytes, bool writable); +void GuestMemoryTrackingInvalidate_nid_postfix(std::uint64_t address, std::size_t bytes); +void GuestMemoryTrackingValidate_nid_postfix(std::uint64_t address, std::size_t bytes, const std::function& validate); +} + +class Watch { +public: + Watch(std::uint64_t address, std::size_t bytes, void* context, Resolver resolver) : handle(GuestMemoryTrackingCreate_nid_postfix(address, bytes, context, resolver)) {} + ~Watch() { GuestMemoryTrackingDestroy_nid_postfix(handle); } + Watch(const Watch&) = delete; + Watch& operator=(const Watch&) = delete; + void Protect(Protection protection) { GuestMemoryTrackingProtect_nid_postfix(handle, protection); } + +private: + void* handle; +}; + +} + +#endif diff --git a/core/libs/prx/libc/include/MemoryBackingPlatform.hpp b/core/libs/prx/libc/include/MemoryBackingPlatform.hpp new file mode 100644 index 000000000..952c8f6d1 --- /dev/null +++ b/core/libs/prx/libc/include/MemoryBackingPlatform.hpp @@ -0,0 +1,22 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_MEMORYBACKINGPLATFORM_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_MEMORYBACKINGPLATFORM_HPP + +#include +#include + +namespace GuestMemoryBacking::Platform { + +struct Mapping { + std::uint64_t address; + std::size_t bytes; + void* alias; + std::uintptr_t handle; +}; + +Mapping Map(void* address, std::size_t bytes, std::size_t alignment, int protection); +void Unmap(const Mapping& mapping); +void Deactivate(std::uint64_t address, std::size_t bytes); + +} + +#endif diff --git a/core/libs/prx/libc/include/MemoryTrackingPlatform.hpp b/core/libs/prx/libc/include/MemoryTrackingPlatform.hpp new file mode 100644 index 000000000..116581951 --- /dev/null +++ b/core/libs/prx/libc/include/MemoryTrackingPlatform.hpp @@ -0,0 +1,24 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_MEMORYTRACKINGPLATFORM_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_MEMORYTRACKINGPLATFORM_HPP + +#include "prx/libc/include/GuestMemoryTracking.hpp" +#include + +namespace GuestMemoryTracking::Platform { + +struct Region { + std::uint64_t address; + std::size_t bytes; + std::uint64_t protection; +}; + +using FaultHandler = bool (*)(std::uint64_t, bool); +std::size_t PageSize(); +void Install(FaultHandler handler); +std::vector Query(std::uint64_t address, std::size_t bytes); +void Protect(std::uint64_t address, std::size_t bytes, Protection protection); +void Restore(const std::vector& regions); + +} + +#endif diff --git a/core/libs/prx/libc/src/GuestAllocations.cpp b/core/libs/prx/libc/src/GuestAllocations.cpp index f89137008..bcac0d0cf 100644 --- a/core/libs/prx/libc/src/GuestAllocations.cpp +++ b/core/libs/prx/libc/src/GuestAllocations.cpp @@ -1,4 +1,5 @@ #include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestMemoryTracking.hpp" #include #include #include @@ -14,7 +15,7 @@ namespace GuestAllocations { namespace { struct Registry { - std::mutex mutex; + std::recursive_mutex& mutex = GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix(); std::map> ranges; bool mainImageRegistered = false; }; @@ -31,11 +32,11 @@ void require(bool condition, const char* reason) { } void* GuestAllocationsBegin_nid_postfix() { - return new std::unique_lock(registry().mutex); + return new std::unique_lock(registry().mutex); } void GuestAllocationsEnd_nid_postfix(void* mutation) noexcept { - delete static_cast*>(mutation); + delete static_cast*>(mutation); } #ifdef _WIN32 @@ -93,6 +94,7 @@ void GuestAllocationsAdd_nid_postfix(void*, void* pointer, std::size_t bytes, bo void GuestAllocationsRequireUnpinned_nid_postfix(void*, const void* pointer, std::size_t bytes) { const auto address = reinterpret_cast(pointer); require(bytes <= std::numeric_limits::max() - address, "guest allocation range overflow"); + GuestMemoryTracking::GuestMemoryTrackingInvalidate_nid_postfix(address, bytes); const auto end = address + bytes; for (const auto& [base, range] : registry().ranges) { if (base >= end && base != address) break; diff --git a/core/libs/prx/libc/src/GuestHeap.cpp b/core/libs/prx/libc/src/GuestHeap.cpp index 5a861a5d3..51e8699a8 100644 --- a/core/libs/prx/libc/src/GuestHeap.cpp +++ b/core/libs/prx/libc/src/GuestHeap.cpp @@ -1,5 +1,6 @@ #include "prx/libc/include/GuestHeap.hpp" #include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestHeapStorage.hpp" #include #include #include @@ -13,18 +14,11 @@ namespace { void* allocate(GuestAllocations::Mutation& mutation, std::size_t alignment, std::size_t bytes) { if (alignment == 0 || (alignment & (alignment - 1)) != 0) throw std::invalid_argument("invalid guest heap alignment"); alignment = std::max(alignment, alignof(void*)); - constexpr auto headerBytes = 2 * sizeof(void*); - if (alignment - 1 > std::numeric_limits::max() - headerBytes || bytes > std::numeric_limits::max() - headerBytes - (alignment - 1)) throw std::length_error("guest heap allocation overflow"); - void* raw = std::malloc(bytes + headerBytes + alignment - 1); - if (raw == nullptr) throw std::bad_alloc(); - const auto address = (reinterpret_cast(raw) + headerBytes + alignment - 1) & ~(static_cast(alignment) - 1); - auto* pointer = reinterpret_cast(address); - reinterpret_cast(pointer)[-2] = raw; - reinterpret_cast(pointer)[-1] = 1; + void* pointer = GuestHeapStorage::Allocate(alignment, bytes); try { mutation.Add(pointer, bytes, true, true); } catch (...) { - std::free(raw); + GuestHeapStorage::Free(pointer); throw; } return pointer; @@ -32,7 +26,7 @@ void* allocate(GuestAllocations::Mutation& mutation, std::size_t alignment, std: void free(GuestAllocations::Mutation& mutation, void* pointer) { mutation.Remove(pointer); - std::free(reinterpret_cast(pointer)[-2]); + GuestHeapStorage::Free(pointer); } } diff --git a/core/libs/prx/libc/src/GuestHeapStorage.cpp b/core/libs/prx/libc/src/GuestHeapStorage.cpp new file mode 100644 index 000000000..e1dc44e7f --- /dev/null +++ b/core/libs/prx/libc/src/GuestHeapStorage.cpp @@ -0,0 +1,116 @@ +#include "prx/libc/include/GuestHeapStorage.hpp" +#include "prx/libc/include/GuestMemoryBacking.hpp" +#include "prx/libc/include/GuestMemoryTracking.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace GuestHeapStorage { +namespace { + +struct Chunk { + void* address; + std::size_t bytes; + std::map free; +}; + +struct Allocation { + Chunk* chunk; + std::size_t bytes; +}; + +struct Arena { + std::vector> chunks; + std::map allocations; +}; + +Arena& arena() { + static auto* value = new Arena; + return *value; +} + +} + +void* Allocate(std::size_t alignment, std::size_t bytes) { + if (alignment == 0 || (alignment & (alignment - 1)) != 0) throw std::invalid_argument("invalid shared heap alignment"); + bytes = std::max(bytes, std::size_t{1}); + if (bytes > std::numeric_limits::max() - alignment) throw std::length_error("shared heap allocation overflow"); + std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); + auto& state = arena(); + for (std::size_t index = 0;; ++index) { + if (index == state.chunks.size()) { + constexpr std::size_t chunkSize = 64 * 1024 * 1024; + const auto granularity = std::max(alignment, std::size_t{65536}); + const auto requested = std::max(bytes, chunkSize); + if (requested > std::numeric_limits::max() - (granularity - 1)) throw std::length_error("shared heap chunk overflow"); + const auto size = (requested + granularity - 1) & ~(granularity - 1); + void* memory = GuestMemoryBacking::GuestMemoryBackingMap_nid_postfix(nullptr, size, granularity, 3); + try { + auto chunk = std::make_unique(); + chunk->address = memory; + chunk->bytes = size; + chunk->free.emplace(reinterpret_cast(memory), size); + state.chunks.push_back(std::move(chunk)); + } catch (...) { + GuestMemoryBacking::GuestMemoryBackingUnmap_nid_postfix(memory, size); + throw; + } + } + auto& chunk = *state.chunks[index]; + for (auto it = chunk.free.begin(); it != chunk.free.end(); ++it) { + const auto first = it->first; + const auto available = it->second; + if (first > std::numeric_limits::max() - (alignment - 1)) throw std::overflow_error("shared heap address overflow"); + const auto address = (first + alignment - 1) & ~(static_cast(alignment) - 1); + const auto prefix = address - first; + if (prefix > available || bytes > available - prefix) continue; + if (!state.allocations.emplace(address, Allocation{&chunk, bytes}).second) throw std::runtime_error("duplicate shared heap allocation"); + const auto suffix = available - prefix - bytes; + try { + if (suffix != 0) chunk.free.emplace(address + bytes, suffix); + } catch (...) { + state.allocations.erase(address); + throw; + } + if (prefix != 0) it->second = prefix; + else chunk.free.erase(it); + return reinterpret_cast(address); + } + } +} + +void Free(void* pointer) { + std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); + auto& allocations = arena().allocations; + const auto address = reinterpret_cast(pointer); + const auto found = allocations.find(address); + if (found == allocations.end()) throw std::invalid_argument("unknown shared heap allocation"); + auto* chunk = found->second.chunk; + auto& free = chunk->free; + auto [entry, inserted] = free.emplace(address, found->second.bytes); + if (!inserted) throw std::runtime_error("shared heap allocation was already freed"); + allocations.erase(found); + if (entry != free.begin()) { + auto previous = std::prev(entry); + if (previous->first + previous->second == entry->first) { + previous->second += entry->second; + free.erase(entry); + entry = previous; + } + } + const auto next = std::next(entry); + if (next != free.end() && entry->first + entry->second == next->first) { + entry->second += next->second; + free.erase(next); + } + if (free.size() == 1 && entry->first == reinterpret_cast(chunk->address) && entry->second == chunk->bytes) { + GuestMemoryBacking::GuestMemoryBackingUnmap_nid_postfix(chunk->address, chunk->bytes); + std::erase_if(arena().chunks, [chunk](const auto& value) { return value.get() == chunk; }); + } +} + +} diff --git a/core/libs/prx/libc/src/GuestMemoryBacking.cpp b/core/libs/prx/libc/src/GuestMemoryBacking.cpp new file mode 100644 index 000000000..f3facf723 --- /dev/null +++ b/core/libs/prx/libc/src/GuestMemoryBacking.cpp @@ -0,0 +1,96 @@ +#include "prx/libc/include/GuestMemoryBacking.hpp" +#include "prx/libc/include/MemoryBackingPlatform.hpp" +#include "prx/libc/include/GuestMemoryTracking.hpp" +#include +#include +#include +#include +#include +#include + +namespace GuestMemoryBacking { +namespace { + +struct Allocation { + Platform::Mapping mapping; + std::map ranges; +}; + +std::map& allocations() { + static auto* value = new std::map; + return *value; +} + +Allocation& find(std::uint64_t address, std::size_t bytes) { + if (address == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) throw std::invalid_argument("invalid guest backing range"); + auto found = allocations().upper_bound(address); + if (found == allocations().begin()) throw std::runtime_error("guest memory has no shared backing"); + auto& allocation = std::prev(found)->second; + auto range = allocation.ranges.upper_bound(address); + if (range == allocation.ranges.begin() || address + bytes > std::prev(range)->second) throw std::runtime_error("guest memory backing range is unmapped"); + return allocation; +} + +} + +void* GuestMemoryBackingMap_nid_postfix(void* address, std::size_t bytes, std::size_t alignment, int protection) { + const auto pageSize = GuestMemoryTracking::GuestMemoryTrackingPageSize_nid_postfix(); + if (bytes == 0 || bytes % pageSize != 0 || alignment < pageSize || (alignment & (alignment - 1)) != 0 || (protection & ~7) != 0) throw std::invalid_argument("invalid shared guest memory mapping"); + if (address != nullptr && reinterpret_cast(address) % alignment != 0) throw std::invalid_argument("misaligned fixed guest memory mapping"); + std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); + const auto mapping = Platform::Map(address, bytes, alignment, protection); + try { + if (mapping.bytes > std::numeric_limits::max() - mapping.address) throw std::overflow_error("guest backing mapping overflow"); + const auto next = allocations().lower_bound(mapping.address); + if (next != allocations().end() && next->first < mapping.address + bytes) throw std::runtime_error("overlapping guest backing mappings"); + if (next != allocations().begin()) { + const auto& previous = std::prev(next)->second.mapping; + if (previous.address + previous.bytes > mapping.address) throw std::runtime_error("guest mapping overlaps a retained backing reservation"); + } + Allocation allocation{mapping, {{mapping.address, mapping.address + bytes}}}; + if (!allocations().emplace(mapping.address, std::move(allocation)).second) throw std::runtime_error("duplicate guest backing mapping"); + } catch (...) { + Platform::Unmap(mapping); + throw; + } + return reinterpret_cast(mapping.address); +} + +void GuestMemoryBackingUnmap_nid_postfix(void* pointer, std::size_t bytes) { + const auto address = reinterpret_cast(pointer); + const auto pageSize = GuestMemoryTracking::GuestMemoryTrackingPageSize_nid_postfix(); + if (address % pageSize != 0 || bytes % pageSize != 0) throw std::invalid_argument("misaligned guest backing unmap"); + std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); + auto& allocation = find(address, bytes); + auto replacement = allocation.ranges; + const auto found = std::prev(replacement.upper_bound(address)); + const auto first = found->first; + const auto last = found->second; + replacement.erase(found); + if (first < address) replacement.emplace(first, address); + if (address + bytes < last) replacement.emplace(address + bytes, last); + GuestMemoryTracking::GuestMemoryTrackingInvalidate_nid_postfix(address, bytes); + if (replacement.empty()) { + const auto base = allocation.mapping.address; + Platform::Unmap(allocation.mapping); + allocations().erase(base); + } else { + Platform::Deactivate(address, bytes); + allocation.ranges.swap(replacement); + } +} + +void GuestMemoryBackingRequire_nid_postfix(std::uint64_t address, std::size_t bytes) { + std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); + static_cast(find(address, bytes)); +} + +void GuestMemoryBackingWrite_nid_postfix(std::uint64_t address, const void* source, std::size_t bytes) { + if (source == nullptr) throw std::invalid_argument("missing guest backing write source"); + std::lock_guard lock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); + auto& allocation = find(address, bytes); + auto* destination = static_cast(allocation.mapping.alias) + (address - allocation.mapping.address); + std::memcpy(destination, source, bytes); +} + +} diff --git a/core/libs/prx/libc/src/GuestMemoryTracking.cpp b/core/libs/prx/libc/src/GuestMemoryTracking.cpp new file mode 100644 index 000000000..d11f274a2 --- /dev/null +++ b/core/libs/prx/libc/src/GuestMemoryTracking.cpp @@ -0,0 +1,173 @@ +#include "prx/libc/include/GuestMemoryTracking.hpp" +#include "prx/libc/include/MemoryTrackingPlatform.hpp" +#include "prx/libc/include/GuestMemoryBacking.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace GuestMemoryTracking { +namespace { + +struct Entry { + std::uint64_t address; + std::size_t bytes; + void* context; + Resolver resolver; + Protection protection = Protection::ReadWrite; + std::vector original; + bool resolving = false; + bool active = false; +}; + +struct Registry { + std::recursive_mutex mutex; + std::map> entries; + bool installed = false; +}; + +Registry& registry() { + static auto* value = new Registry; + return *value; +} + +std::uint64_t checkedEnd(std::uint64_t address, std::size_t bytes) { + if (address == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) throw std::invalid_argument("invalid tracked guest memory range"); + return address + bytes; +} + +void resolve(const std::shared_ptr& entry, Access access) { + if (entry->resolving) throw std::runtime_error("recursive guest memory ownership resolution"); + entry->resolving = true; + try { + entry->resolver(entry->context, access); + if (entry->protection == Protection::None || (access != Access::Read && entry->protection != Protection::ReadWrite)) throw std::runtime_error("guest memory resolver did not release CPU access"); + entry->resolving = false; + } catch (...) { + entry->resolving = false; + throw; + } +} + +std::vector> overlapping(std::uint64_t address, std::size_t bytes) { + const auto end = checkedEnd(address, bytes); + auto& entries = registry().entries; + auto first = entries.upper_bound(address); + if (first != entries.begin()) --first; + std::vector> result; + for (auto it = first; it != entries.end() && it->first < end; ++it) { + if (it->first + it->second->bytes > address) result.push_back(it->second); + } + return result; +} + +bool fault(std::uint64_t address, bool writable) { + std::lock_guard lock(registry().mutex); + const auto entries = overlapping(address, 1); + if (entries.empty()) return false; + bool handled = false; + for (const auto& entry : entries) { + if (!entry->active) continue; + if (entry->protection == Protection::None || (writable && entry->protection == Protection::Read)) resolve(entry, writable ? Access::Write : Access::Read); + handled = true; + } + return handled; +} + +} + +std::recursive_mutex& GuestMemoryTrackingMutex_nid_postfix() { + return registry().mutex; +} + +std::size_t GuestMemoryTrackingPageSize_nid_postfix() { + return Platform::PageSize(); +} + +void* GuestMemoryTrackingCreate_nid_postfix(std::uint64_t address, std::size_t bytes, void* context, Resolver resolver) { + const auto end = checkedEnd(address, bytes); + const auto pageSize = Platform::PageSize(); + if (context == nullptr || resolver == nullptr) throw std::invalid_argument("missing guest memory ownership resolver"); + if (end > std::numeric_limits::max() - (pageSize - 1)) throw std::overflow_error("tracked guest memory page range overflow"); + const auto first = address - address % pageSize; + const auto last = (end + pageSize - 1) / pageSize * pageSize; + std::lock_guard lock(registry().mutex); + GuestMemoryBacking::GuestMemoryBackingRequire_nid_postfix(first, static_cast(last - first)); + if (!overlapping(first, static_cast(last - first)).empty()) throw std::runtime_error("overlapping guest memory ownership pages"); + static_cast(Platform::Query(first, static_cast(last - first))); + if (!registry().installed) { + Platform::Install(fault); + registry().installed = true; + } + auto entry = std::make_shared(); + entry->address = first; + entry->bytes = static_cast(last - first); + entry->context = context; + entry->resolver = resolver; + auto handle = std::make_unique>(entry); + registry().entries.emplace(first, std::move(entry)); + return handle.release(); +} + +void GuestMemoryTrackingDestroy_nid_postfix(void* handle) noexcept { + if (handle == nullptr) std::terminate(); + std::lock_guard lock(registry().mutex); + std::unique_ptr> owner(static_cast*>(handle)); + const auto& entry = **owner; + Platform::Restore(entry.original); + registry().entries.erase(entry.address); +} + +void GuestMemoryTrackingProtect_nid_postfix(void* handle, Protection protection) { + if (handle == nullptr) throw std::invalid_argument("missing guest memory watch"); + std::lock_guard lock(registry().mutex); + auto& entry = **static_cast*>(handle); + if (entry.protection == protection) return; + if (protection == Protection::ReadWrite) { + Platform::Restore(entry.original); + entry.original.clear(); + } else { + if (entry.original.empty()) entry.original = Platform::Query(entry.address, entry.bytes); + Platform::Protect(entry.address, entry.bytes, protection); + entry.active = true; + } + entry.protection = protection; +} + +void GuestMemoryTrackingResolve_nid_postfix(std::uint64_t address, std::size_t bytes, bool writable) { + if (bytes == 0) return; + std::lock_guard lock(registry().mutex); + for (const auto& entry : overlapping(address, bytes)) { + if (entry->protection == Protection::None || (writable && entry->protection == Protection::Read)) resolve(entry, writable ? Access::Write : Access::Read); + } +} + +void GuestMemoryTrackingInvalidate_nid_postfix(std::uint64_t address, std::size_t bytes) { + if (bytes == 0) return; + std::lock_guard lock(registry().mutex); + for (const auto& entry : overlapping(address, bytes)) { + resolve(entry, Access::Invalidate); + entry->active = false; + } +} + +void GuestMemoryTrackingValidate_nid_postfix(std::uint64_t address, std::size_t bytes, const std::function& validate) { + if (bytes == 0) return; + if (!validate) throw std::invalid_argument("missing native memory range validator"); + const auto end = checkedEnd(address, bytes); + std::lock_guard lock(registry().mutex); + for (const auto& entry : overlapping(address, bytes)) { + if (address < entry->address) validate(address, static_cast(entry->address - address)); + const auto first = std::max(address, entry->address); + const auto last = std::min(end, entry->address + entry->bytes); + if (entry->protection == Protection::ReadWrite) validate(first, static_cast(last - first)); + else if (entry->original.empty()) throw std::runtime_error("protected guest memory has no native permission record"); + address = last; + } + if (address < end) validate(address, static_cast(end - address)); +} + +} diff --git a/core/libs/prx/libc/src/MemoryBackingLinux.cpp b/core/libs/prx/libc/src/MemoryBackingLinux.cpp new file mode 100644 index 000000000..5c788aa74 --- /dev/null +++ b/core/libs/prx/libc/src/MemoryBackingLinux.cpp @@ -0,0 +1,70 @@ +#include "prx/libc/include/MemoryBackingPlatform.hpp" +#include +#include +#include +#include +#include +#include + +namespace GuestMemoryBacking::Platform { +namespace { + +void check(bool success, const char* operation) { + if (!success) throw std::system_error(errno, std::generic_category(), operation); +} + +} + +Mapping Map(void* address, std::size_t bytes, std::size_t alignment, int protection) { + if (bytes > static_cast(std::numeric_limits::max()) || bytes > std::numeric_limits::max() - alignment) throw std::overflow_error("guest backing size overflow"); + int descriptor = memfd_create("AnyPS5 guest memory", MFD_CLOEXEC); + check(descriptor >= 0, "memfd_create guest backing"); + void* alias = MAP_FAILED; + void* guest = MAP_FAILED; + void* reservation = MAP_FAILED; + std::size_t reservationBytes = 0; + try { + check(ftruncate(descriptor, static_cast(bytes)) == 0, "ftruncate guest backing"); + alias = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, descriptor, 0); + check(alias != MAP_FAILED, "mmap guest backing alias"); + if (address == nullptr) { + reservationBytes = bytes + alignment; + reservation = mmap(nullptr, reservationBytes, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + } else { + reservationBytes = bytes; + reservation = mmap(address, reservationBytes, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + } + check(reservation != MAP_FAILED, "mmap guest backing reservation"); + if (address != nullptr && reservation != address) throw std::runtime_error("fixed guest backing reservation address mismatch"); + const auto first = reinterpret_cast(reservation); + const auto aligned = (first + alignment - 1) & ~(static_cast(alignment) - 1); + guest = mmap(reinterpret_cast(aligned), bytes, protection, MAP_SHARED | MAP_FIXED, descriptor, 0); + check(guest != MAP_FAILED, "mmap guest backing view"); + const auto prefix = aligned - first; + const auto suffix = reservationBytes - prefix - bytes; + if (prefix != 0) check(munmap(reservation, prefix) == 0, "munmap guest backing prefix"); + if (suffix != 0) check(munmap(reinterpret_cast(aligned + bytes), suffix) == 0, "munmap guest backing suffix"); + reservation = MAP_FAILED; + const auto closeResult = close(descriptor); + descriptor = -1; + check(closeResult == 0, "close guest backing descriptor"); + return {aligned, bytes, alias, 0}; + } catch (...) { + if (reservation != MAP_FAILED) check(munmap(reservation, reservationBytes) == 0, "munmap failed guest reservation"); + else if (guest != MAP_FAILED) check(munmap(guest, bytes) == 0, "munmap failed guest view"); + if (alias != MAP_FAILED) check(munmap(alias, bytes) == 0, "munmap failed guest alias"); + if (descriptor >= 0) check(close(descriptor) == 0, "close failed guest backing"); + throw; + } +} + +void Unmap(const Mapping& mapping) { + check(munmap(reinterpret_cast(mapping.address), mapping.bytes) == 0, "munmap guest view"); + check(munmap(mapping.alias, mapping.bytes) == 0, "munmap guest alias"); +} + +void Deactivate(std::uint64_t address, std::size_t bytes) { + check(mprotect(reinterpret_cast(address), bytes, PROT_NONE) == 0, "mprotect guest backing unmap"); +} + +} diff --git a/core/libs/prx/libc/src/MemoryBackingWindows.cpp b/core/libs/prx/libc/src/MemoryBackingWindows.cpp new file mode 100644 index 000000000..8b0c96456 --- /dev/null +++ b/core/libs/prx/libc/src/MemoryBackingWindows.cpp @@ -0,0 +1,72 @@ +#include "prx/libc/include/MemoryBackingPlatform.hpp" +#include +#include +#include +#include +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include + +namespace GuestMemoryBacking::Platform { +namespace { + +DWORD nativeProtection(int protection) { + if ((protection & 4) != 0) return (protection & 2) != 0 ? PAGE_EXECUTE_READWRITE : (protection & 1) != 0 ? PAGE_EXECUTE_READ : PAGE_EXECUTE; + if ((protection & 2) != 0) return PAGE_READWRITE; + return (protection & 1) != 0 ? PAGE_READONLY : PAGE_NOACCESS; +} + +void check(bool success, const char* operation) { + if (!success) throw std::system_error(static_cast(GetLastError()), std::system_category(), operation); +} + +} + +Mapping Map(void* address, std::size_t bytes, std::size_t alignment, int protection) { + SYSTEM_INFO info{}; + GetSystemInfo(&info); + alignment = std::max(alignment, static_cast(info.dwAllocationGranularity)); + if (address != nullptr && reinterpret_cast(address) % info.dwAllocationGranularity != 0) throw std::invalid_argument("fixed guest view is not aligned to native allocation granularity"); + const auto size = static_cast(bytes); + HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE, static_cast(size >> 32u), static_cast(size), nullptr); + check(section != nullptr, "CreateFileMapping guest backing"); + void* alias = nullptr; + void* guest = nullptr; + try { + alias = MapViewOfFile(section, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, bytes); + check(alias != nullptr, "MapViewOfFile guest backing alias"); + if (address == nullptr && alignment > info.dwAllocationGranularity) { + if (bytes > std::numeric_limits::max() - alignment) throw std::overflow_error("aligned guest backing reservation overflow"); + void* reservation = VirtualAlloc(nullptr, bytes + alignment, MEM_RESERVE, PAGE_NOACCESS); + check(reservation != nullptr, "VirtualAlloc guest backing reservation"); + const auto first = reinterpret_cast(reservation); + address = reinterpret_cast((first + alignment - 1) & ~(static_cast(alignment) - 1)); + check(VirtualFree(reservation, 0, MEM_RELEASE) != FALSE, "VirtualFree guest backing reservation"); + } + guest = MapViewOfFileEx(section, FILE_MAP_READ | FILE_MAP_WRITE | FILE_MAP_EXECUTE, 0, 0, bytes, address); + check(guest != nullptr, "MapViewOfFileEx guest memory"); + if (reinterpret_cast(guest) % alignment != 0 || (address != nullptr && guest != address)) throw std::runtime_error("guest backing view address mismatch"); + DWORD previous = 0; + check(VirtualProtect(guest, bytes, nativeProtection(protection), &previous) != FALSE, "VirtualProtect guest backing view"); + return {reinterpret_cast(guest), bytes, alias, reinterpret_cast(section)}; + } catch (...) { + if (guest != nullptr) check(UnmapViewOfFile(guest) != FALSE, "UnmapViewOfFile failed guest view"); + if (alias != nullptr) check(UnmapViewOfFile(alias) != FALSE, "UnmapViewOfFile failed guest alias"); + check(CloseHandle(section) != FALSE, "CloseHandle failed guest backing"); + throw; + } +} + +void Unmap(const Mapping& mapping) { + check(UnmapViewOfFile(reinterpret_cast(mapping.address)) != FALSE, "UnmapViewOfFile guest view"); + check(UnmapViewOfFile(mapping.alias) != FALSE, "UnmapViewOfFile guest alias"); + check(CloseHandle(reinterpret_cast(mapping.handle)) != FALSE, "CloseHandle guest backing"); +} + +void Deactivate(std::uint64_t address, std::size_t bytes) { + DWORD previous = 0; + check(VirtualProtect(reinterpret_cast(address), bytes, PAGE_NOACCESS, &previous) != FALSE, "VirtualProtect guest backing unmap"); +} + +} diff --git a/core/libs/prx/libc/src/MemoryTrackingLinux.cpp b/core/libs/prx/libc/src/MemoryTrackingLinux.cpp new file mode 100644 index 000000000..54b9ca0dd --- /dev/null +++ b/core/libs/prx/libc/src/MemoryTrackingLinux.cpp @@ -0,0 +1,99 @@ +#include "prx/libc/include/MemoryTrackingPlatform.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace GuestMemoryTracking::Platform { +namespace { + +FaultHandler faultHandler = nullptr; +struct sigaction previousAction{}; + +void handleFault(int signal, siginfo_t* info, void* context) { + const auto* native = static_cast(context); + const auto error = native->uc_mcontext.gregs[REG_ERR]; + if (info->si_code == SEGV_ACCERR && (error & 16) == 0) { + try { + if (faultHandler(reinterpret_cast(info->si_addr), (error & 2) != 0)) return; + } catch (...) { + std::terminate(); + } + } + if (previousAction.sa_handler == SIG_DFL || previousAction.sa_handler == SIG_IGN) { + if (sigaction(signal, &previousAction, nullptr) != 0) std::terminate(); + if (raise(signal) != 0) std::terminate(); + return; + } + if ((previousAction.sa_flags & SA_SIGINFO) != 0) previousAction.sa_sigaction(signal, info, context); + else previousAction.sa_handler(signal); +} + +void protect(std::uint64_t address, std::size_t bytes, int protection) { + if (mprotect(reinterpret_cast(address), bytes, protection) != 0) throw std::system_error(errno, std::generic_category(), "guest memory tracking mprotect failed"); +} + +} + +std::size_t PageSize() { + static const auto size = [] { + const auto result = sysconf(_SC_PAGESIZE); + if (result <= 0) throw std::runtime_error("invalid native page size"); + return static_cast(result); + }(); + return size; +} + +void Install(FaultHandler handler) { + if (handler == nullptr || faultHandler != nullptr) throw std::runtime_error("invalid guest memory fault handler installation"); + faultHandler = handler; + struct sigaction action{}; + action.sa_flags = SA_SIGINFO; + action.sa_sigaction = handleFault; + if (sigemptyset(&action.sa_mask) != 0 || sigaction(SIGSEGV, &action, &previousAction) != 0) { + const auto error = errno; + faultHandler = nullptr; + throw std::system_error(error, std::generic_category(), "guest memory fault handler installation failed"); + } +} + +std::vector Query(std::uint64_t address, std::size_t bytes) { + std::ifstream maps("/proc/self/maps"); + if (!maps) throw std::runtime_error("cannot query tracked guest memory mappings"); + std::vector regions; + const auto end = address + bytes; + std::string line; + while (address < end && std::getline(maps, line)) { + std::istringstream input(line); + std::uint64_t first = 0; + std::uint64_t last = 0; + char separator = 0; + std::string permissions; + if (!(input >> std::hex >> first >> separator >> last >> permissions) || separator != '-' || permissions.size() < 3 || first >= last) throw std::runtime_error("invalid tracked guest memory mapping"); + if (last <= address) continue; + if (first > address || permissions[0] != 'r' || permissions[1] != 'w' || permissions[2] != '-') throw std::runtime_error("tracked render memory must be mapped, writable and non-executable"); + const auto next = std::min(end, last); + regions.push_back({address, static_cast(next - address), static_cast(PROT_READ | PROT_WRITE | (permissions[2] == 'x' ? PROT_EXEC : 0))}); + address = next; + } + if (address != end) throw std::runtime_error("tracked guest memory range is not mapped"); + return regions; +} + +void Protect(std::uint64_t address, std::size_t bytes, Protection protection) { + if (protection != Protection::None && protection != Protection::Read) throw std::invalid_argument("invalid tracked page protection"); + protect(address, bytes, protection == Protection::None ? PROT_NONE : PROT_READ); +} + +void Restore(const std::vector& regions) { + for (const auto& region : regions) protect(region.address, region.bytes, static_cast(region.protection)); +} + +} diff --git a/core/libs/prx/libc/src/MemoryTrackingWindows.cpp b/core/libs/prx/libc/src/MemoryTrackingWindows.cpp new file mode 100644 index 000000000..d504f0fd6 --- /dev/null +++ b/core/libs/prx/libc/src/MemoryTrackingWindows.cpp @@ -0,0 +1,78 @@ +#include "prx/libc/include/MemoryTrackingPlatform.hpp" +#include +#include +#include +#include +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include + +namespace GuestMemoryTracking::Platform { +namespace { + +FaultHandler faultHandler = nullptr; + +LONG CALLBACK handleException(EXCEPTION_POINTERS* exception) { + const auto* record = exception->ExceptionRecord; + if (record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || record->NumberParameters < 2 || record->ExceptionInformation[0] > 1) return EXCEPTION_CONTINUE_SEARCH; + try { + if (faultHandler(record->ExceptionInformation[1], record->ExceptionInformation[0] == 1)) return EXCEPTION_CONTINUE_EXECUTION; + } catch (...) { + std::terminate(); + } + return EXCEPTION_CONTINUE_SEARCH; +} + +void protect(std::uint64_t address, std::size_t bytes, DWORD protection) { + DWORD previous = 0; + if (!VirtualProtect(reinterpret_cast(address), bytes, protection, &previous)) throw std::system_error(static_cast(GetLastError()), std::system_category(), "guest memory tracking VirtualProtect failed"); +} + +} + +std::size_t PageSize() { + static const auto size = [] { + SYSTEM_INFO info{}; + GetSystemInfo(&info); + if (info.dwPageSize == 0) throw std::runtime_error("invalid native page size"); + return static_cast(info.dwPageSize); + }(); + return size; +} + +void Install(FaultHandler handler) { + if (handler == nullptr || faultHandler != nullptr) throw std::runtime_error("invalid guest memory fault handler installation"); + faultHandler = handler; + if (AddVectoredExceptionHandler(1, handleException) == nullptr) { + const auto error = GetLastError(); + faultHandler = nullptr; + throw std::system_error(static_cast(error), std::system_category(), "guest memory fault handler installation failed"); + } +} + +std::vector Query(std::uint64_t address, std::size_t bytes) { + std::vector regions; + const auto end = address + bytes; + while (address < end) { + MEMORY_BASIC_INFORMATION info{}; + if (VirtualQuery(reinterpret_cast(address), &info, sizeof(info)) != sizeof(info)) throw std::system_error(static_cast(GetLastError()), std::system_category(), "tracked guest memory query failed"); + if (info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) throw std::runtime_error("tracked render memory must be committed, writable and non-executable"); + const auto next = std::min(end, reinterpret_cast(info.BaseAddress) + info.RegionSize); + if (next <= address) throw std::runtime_error("invalid tracked guest memory mapping"); + regions.push_back({address, static_cast(next - address), info.Protect}); + address = next; + } + return regions; +} + +void Protect(std::uint64_t address, std::size_t bytes, Protection protection) { + if (protection != Protection::None && protection != Protection::Read) throw std::invalid_argument("invalid tracked page protection"); + protect(address, bytes, protection == Protection::None ? PAGE_NOACCESS : PAGE_READONLY); +} + +void Restore(const std::vector& regions) { + for (const auto& region : regions) protect(region.address, region.bytes, static_cast(region.protection)); +} + +} diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp index 321fb2828..69be5eb95 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp @@ -1,5 +1,6 @@ -#include "DirectMemory.hpp" +#include "prx/libkernel/DirectMemory/DirectMemory.hpp" #include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestMemoryBacking.hpp" #include #include #include @@ -14,10 +15,6 @@ static constexpr int PROT_NONE = 0; static constexpr int PROT_READ = 1; static constexpr int PROT_WRITE = 2; static constexpr int PROT_EXEC = 4; -static constexpr int MAP_PRIVATE = 0x02; -static constexpr int MAP_ANONYMOUS = 0x20; -static constexpr int MAP_FIXED = 0x10; -static void* const MAP_FAILED = reinterpret_cast(-1); static DWORD WinProtFromPosix(int prot) { if (prot == PROT_NONE) return PAGE_NOACCESS; @@ -28,30 +25,6 @@ static DWORD WinProtFromPosix(int prot) { return PAGE_READONLY; } -static void* mmap(void* addr, size_t len, int prot, int flags, int, int) { - DWORD winProt = WinProtFromPosix(prot); - if (flags & MAP_FIXED) { - void* result = VirtualAlloc(addr, len, MEM_RESERVE | MEM_COMMIT, winProt); - if (!result) throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualAlloc fixed failed"); - return result; - } - void* result = VirtualAlloc(nullptr, len, MEM_RESERVE | MEM_COMMIT, winProt); - if (!result) throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualAlloc failed"); - return result; -} - -static int munmap(void* addr, size_t len) { - if (!VirtualFree(addr, len, MEM_DECOMMIT)) - throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualFree decommit failed"); - return 0; -} - -static int munmap_release(void* addr) { - if (!VirtualFree(addr, 0, MEM_RELEASE)) - throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualFree release failed"); - return 0; -} - static int mprotect(void* addr, size_t len, int prot) { DWORD old; if (!VirtualProtect(addr, len, WinProtFromPosix(prot), &old)) @@ -100,60 +73,18 @@ int LinuxProtFromSce(int prot) { } void Unmap(void* addr, size_t len) { -#if defined(__linux__) - if (munmap(addr, len) != 0) throw std::system_error(errno, std::generic_category(), "munmap failed"); -#else - munmap_release(addr); -#endif + GuestMemoryBacking::GuestMemoryBackingUnmap_nid_postfix(addr, len); } void* MapAligned(void* addr, size_t len, int prot, int flags, size_t alignment) { ValidateLength(len); alignment = ValidateAlignment(alignment); - constexpr int GuestMapFixed = 0x10; - constexpr int GuestMapNoCoalesce = 0x400000; - if ((flags & ~(GuestMapFixed | GuestMapNoCoalesce)) != 0) { - // return SCE_KERNEL_ERROR_EINVAL; - throw std::invalid_argument("Unsupported memory mapping flags"); - } - if ((flags & GuestMapFixed) != 0) { - ValidateRange(addr, len, alignment); - void* result = mmap(addr, len, prot, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); - if (result == MAP_FAILED) { - // return SCE_KERNEL_ERROR_ENOMEM; - throw std::system_error(errno, std::generic_category(), "Fixed mmap failed"); - } - return result; - } - if (addr) { - // return SCE_KERNEL_ERROR_EINVAL; - throw std::invalid_argument("Non-fixed mapping address hints are not implemented"); - } - if (len > std::numeric_limits::max() - alignment) { - // return SCE_KERNEL_ERROR_EINVAL; - throw std::overflow_error("Aligned mapping size overflow"); - } - const size_t allocLen = len + alignment; - void* result = mmap(nullptr, allocLen, prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (result == MAP_FAILED) { - // return SCE_KERNEL_ERROR_ENOMEM; - throw std::system_error(errno, std::generic_category(), "Aligned mmap failed"); - } - const auto raw = reinterpret_cast(result); - const size_t prefix = (alignment - (raw & (alignment - 1))) & (alignment - 1); - void* aligned = reinterpret_cast(raw + prefix); - const size_t suffix = allocLen - prefix - len; - if (prefix != 0 && munmap(result, prefix) != 0) { - const int error = errno; - Unmap(result, allocLen); - throw std::system_error(error, std::generic_category(), "Mapping prefix munmap failed"); - } - if (suffix != 0 && munmap(reinterpret_cast(raw + prefix + len), suffix) != 0) { - const int error = errno; - Unmap(aligned, allocLen - prefix); - throw std::system_error(error, std::generic_category(), "Mapping suffix munmap failed"); - } - return aligned; + constexpr int guestMapFixed = 0x10; + constexpr int guestMapNoCoalesce = 0x400000; + if ((flags & ~(guestMapFixed | guestMapNoCoalesce)) != 0) throw std::invalid_argument("Unsupported memory mapping flags"); + if ((flags & guestMapFixed) != 0) ValidateRange(addr, len, alignment); + else if (addr != nullptr) throw std::invalid_argument("Non-fixed mapping address hints are not implemented"); + return GuestMemoryBacking::GuestMemoryBackingMap_nid_postfix(addr, len, alignment, prot); } void ValidateOutput(void** addr) { @@ -228,13 +159,8 @@ int DoMprotect(const void* addr, size_t len, int prot) { int DoMunmap(void* addr, size_t len) { if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0 || !addr) return SCE_KERNEL_ERROR_EINVAL; GuestAllocations::Mutation mutation; - mutation.Unmap(addr, len, [&](const void* allocation, bool last) { -#if defined(__linux__) + mutation.Unmap(addr, len, [&](const void*, bool) { Unmap(addr, len); -#else - if (last) munmap_release(const_cast(allocation)); - else munmap(addr, len); -#endif }); return 0; }