diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index b01612cb..fb7c38b1 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -359,12 +359,6 @@ public: EditUnlocked(start, size, [protection](Range* r) { r->protection = protection; }); } - void SetMemoryType(uint64_t start, uint64_t size, int memory_type) { - Common::LockGuard lock(m_mutex); - - EditUnlocked(start, size, [memory_type](Range* r) { r->memory_type = memory_type; }); - } - bool Query(uint64_t addr, int flags, Range* out) { EXIT_IF(out == nullptr); @@ -702,7 +696,7 @@ public: bool Release(uint64_t start, size_t len, uint64_t* vaddr, uint64_t* size, GpuAccessMode* gpu_mode); bool Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int prot, VirtualMemory::Mode mode, - GpuAccessMode gpu_mode); + GpuAccessMode gpu_mode, const char* name, int memory_type); bool Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode, uint64_t* host_vaddr_to_release = nullptr); bool Find(uint64_t vaddr, uint64_t* base_addr, size_t* len, int* prot, @@ -715,7 +709,6 @@ public: void ProtectMapping(uint64_t vaddr, uint64_t size, int prot, VirtualMemory::Mode mode, GpuAccessMode gpu_mode); void SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name); - void SetVirtualRangeMemoryType(uint64_t vaddr, uint64_t len, int memory_type); [[nodiscard]] Common::Mutex& GetMutex() { return m_mutex; } [[nodiscard]] const std::map& GetPhysicalBlocks() const { @@ -1368,7 +1361,8 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64 } bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int prot, - VirtualMemory::Mode mode, GpuAccessMode gpu_mode) { + VirtualMemory::Mode mode, GpuAccessMode gpu_mode, const char* name, + int memory_type) { Common::LockGuard lock(m_mutex); if (len == 0 || UINT64_MAX - phys_addr < len) { @@ -1413,6 +1407,8 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro mapping.prot = prot; mapping.mode = mode; mapping.gpu_mode = gpu_mode; + mapping.memory_type = memory_type; + CopyVirtualRangeName(mapping.name, name); m_mappings.push_back(mapping); if (has_automatic) { RemoveFreeRange(m_automatic_free, phys_addr, len); @@ -2040,16 +2036,6 @@ void FlexibleMemory::SetVirtualRangeName(uint64_t vaddr, uint64_t len, const cha } } -void PhysicalMemory::SetVirtualRangeMemoryType(uint64_t vaddr, uint64_t len, int memory_type) { - Common::LockGuard lock(m_mutex); - - for (auto& b: m_mappings) { - if (VirtualRangesOverlap(vaddr, len, b.map_vaddr, b.map_size)) { - b.memory_type = memory_type; - } - } -} - uint64_t FlexibleMemory::Available() { Common::LockGuard lock(m_mutex); @@ -2268,12 +2254,9 @@ static bool UnmapPooledBackingTransactional(const std::vector memory_operation_lock(g_memory_operation_mutex); - +// The ABI entry points and batch operations hold g_memory_operation_mutex. +static int MapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags, + const char* name) { EXIT_NOT_IMPLEMENTED(addr_in_out == nullptr); constexpr size_t PAGE_SIZE = 0x4000; @@ -2398,8 +2381,17 @@ int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t le return OK; } +int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, + int flags, const char* name) { + PRINT_NAME(); + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + return MapFlexibleMemory(addr_in_out, len, prot, flags, name); +} + int KYTY_SYSV_ABI KernelMapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags) { - return KernelMapNamedFlexibleMemory(addr_in_out, len, prot, flags, ""); + PRINT_NAME(); + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + return MapFlexibleMemory(addr_in_out, len, prot, flags, ""); } int KYTY_SYSV_ABI KernelSetPrtAperture(int index, void* addr, size_t len) { @@ -2607,11 +2599,8 @@ static int UnmapMemoryRange(uint64_t vaddr, size_t len) { return OK; } -int KYTY_SYSV_ABI KernelMunmap(uint64_t vaddr, size_t len) { - PRINT_NAME(); - - std::lock_guard memory_operation_lock(g_memory_operation_mutex); - +// The ABI entry point and batch operations hold g_memory_operation_mutex. +static int UnmapMemory(uint64_t vaddr, size_t len) { LOGF("\t start = 0x%016" PRIx64 "\n" "\t len = 0x%016" PRIx64 "\n", vaddr, len); @@ -2627,6 +2616,12 @@ int KYTY_SYSV_ABI KernelMunmap(uint64_t vaddr, size_t len) { return UnmapMemoryRange(vaddr, len); } +int KYTY_SYSV_ABI KernelMunmap(uint64_t vaddr, size_t len) { + PRINT_NAME(); + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + return UnmapMemory(vaddr, len); +} + size_t KYTY_SYSV_ABI KernelGetDirectMemorySize() { PRINT_NAME(); @@ -2909,7 +2904,8 @@ static int ReleaseDirectMemoryInternal(int64_t start, size_t len) { for (const auto& removed: metadata_unmapped) { EXIT_IF(!g_physical_memory->Map(removed.map_vaddr, removed.start_addr, removed.map_size, removed.prot, removed.mode, - removed.gpu_mode)); + removed.gpu_mode, removed.name, + removed.memory_type)); } restore_owner_aliases(owner_unmapped); restore_gpu_aliases(); @@ -2980,12 +2976,10 @@ int KYTY_SYSV_ABI KernelCheckedReleaseDirectMemory(int64_t start, size_t len) { return result == KERNEL_ERROR_EACCES ? KERNEL_ERROR_ENOENT : result; } -int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int flags, - int64_t direct_memory_start, size_t alignment) { - PRINT_NAME(); - - std::lock_guard memory_operation_lock(g_memory_operation_mutex); - +// The ABI entry points and batch operations hold g_memory_operation_mutex. +static int MapDirectMemory(void** addr, size_t len, int prot, int flags, + int64_t direct_memory_start, size_t alignment, const char* name, + int type = -1) { if (addr == nullptr) { return KERNEL_ERROR_EFAULT; } @@ -3036,10 +3030,8 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f old.start_addr, old.mode)); } EXIT_IF(!g_physical_memory->Map(old.map_vaddr, old.start_addr, old.map_size, - old.prot, old.mode, old.gpu_mode)); - g_physical_memory->SetVirtualRangeName(old.map_vaddr, old.map_size, old.name); - g_physical_memory->SetVirtualRangeMemoryType(old.map_vaddr, old.map_size, - old.memory_type); + old.prot, old.mode, old.gpu_mode, old.name, + old.memory_type)); } MapGpuRange(in_addr, len); }; @@ -3168,7 +3160,11 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f return KERNEL_ERROR_ENOMEM; } - if (!g_physical_memory->Map(out_addr, direct_memory_start, len, prot, mode, gpu_mode)) { + PhysicalMemory::AllocatedBlock mapped_block {}; + g_physical_memory->Find(direct_memory_start, false, &mapped_block); + const int memory_type = type == -1 ? mapped_block.memory_type : type; + if (!g_physical_memory->Map(out_addr, direct_memory_start, len, prot, mode, gpu_mode, name, + memory_type)) { LOGF_COLOR(Log::Color::Red, "\t [Fail]\n"); if (replaced_mappings.empty()) { EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len)); @@ -3178,18 +3174,16 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f return KERNEL_ERROR_EBUSY; } - PhysicalMemory::AllocatedBlock mapped_block {}; - g_physical_memory->Find(direct_memory_start, false, &mapped_block); const bool published = !replaced_mappings.empty() || reserved_target ? g_virtual_ranges->ReplaceSpan( out_addr, len, replaced_mappings.empty() ? VirtualRangeType::Reserved : VirtualRangeType::Direct, direct_memory_start, - prot, mapped_block.memory_type, VirtualRangeType::Direct, "") + prot, memory_type, VirtualRangeType::Direct, name) : g_virtual_ranges->Add(out_addr, len, direct_memory_start, prot, - mapped_block.memory_type, - VirtualRangeType::Direct, ""); + memory_type, + VirtualRangeType::Direct, name); if (!published) { GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess; EXIT_IF(!g_physical_memory->Unmap(out_addr, len, &rollback_gpu_mode)); @@ -3212,22 +3206,18 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f return OK; } +int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int flags, + int64_t direct_memory_start, size_t alignment) { + PRINT_NAME(); + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + return MapDirectMemory(addr, len, prot, flags, direct_memory_start, alignment, ""); +} + int KYTY_SYSV_ABI KernelMapDirectMemory2(void** addr, size_t len, int type, int prot, int flags, int64_t direct_memory_start, size_t alignment) { PRINT_NAME(); - std::lock_guard memory_operation_lock(g_memory_operation_mutex); - - LOGF("\t type = %d\n", type); - - auto ret = KernelMapDirectMemory(addr, len, prot, flags, direct_memory_start, alignment); - if (ret == OK && addr != nullptr && *addr != nullptr) { - const auto out_addr = reinterpret_cast(*addr); - g_physical_memory->SetVirtualRangeMemoryType(out_addr, len, type); - g_virtual_ranges->SetMemoryType(out_addr, len, type); - } - - return ret; + return MapDirectMemory(addr, len, prot, flags, direct_memory_start, alignment, "", type); } int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot) { @@ -3246,11 +3236,11 @@ int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot) { uint64_t mapped = 0; for (const auto& [offset, length]: ranges) { void* address = reinterpret_cast(vaddr + mapped); - const int result = KernelMapDirectMemory2(&address, length, type, prot, 0x10, - static_cast(offset), page_size); + const int result = MapDirectMemory(&address, length, prot, 0x10, + static_cast(offset), page_size, "", type); if (result != OK) { if (mapped != 0) { - EXIT_IF(KernelMunmap(vaddr, mapped) != OK); + EXIT_IF(UnmapMemory(vaddr, mapped) != OK); } g_physical_memory->RestoreAutomatic(ranges); return result; @@ -3277,13 +3267,7 @@ int KYTY_SYSV_ABI KernelMapNamedDirectMemory(void** addr, size_t len, int prot, return KERNEL_ERROR_ENAMETOOLONG; } - auto ret = KernelMapDirectMemory(addr, len, prot, flags, direct_memory_start, alignment); - if (ret == OK && addr != nullptr) { - g_physical_memory->SetVirtualRangeName(reinterpret_cast(*addr), len, name); - g_virtual_ranges->Rename(reinterpret_cast(*addr), len, name); - } - - return ret; + return MapDirectMemory(addr, len, prot, flags, direct_memory_start, alignment, name); } int KYTY_SYSV_ABI KernelIsAddressSanitizerEnabled() { @@ -3389,7 +3373,8 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size) { if (chunk.backend_unmapped) { backend_restored = g_physical_memory->Map( chunk.range.start, chunk.range.offset, chunk.range.size, - chunk.range.protection, chunk.mode, chunk.gpu_mode); + chunk.range.protection, chunk.mode, chunk.gpu_mode, chunk.range.name, + chunk.range.memory_type); } } else if (chunk.range.type == VirtualRangeType::Flexible && chunk.backend_unmapped) { host_restored = backend_restored = @@ -3905,11 +3890,8 @@ bool FreeGuestMemory(uint64_t vaddr, uint64_t size) { return FreeGuestMemoryOwner(vaddr, mapped_size); } -int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot) { - PRINT_NAME(); - - std::lock_guard memory_operation_lock(g_memory_operation_mutex); - +// The ABI entry points and batch operations hold g_memory_operation_mutex. +static int ProtectMemory(const void* addr, size_t len, int prot) { auto vaddr = reinterpret_cast(addr); LOGF("\t addr = 0x%016" PRIx64 "\n" @@ -3967,19 +3949,17 @@ int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot) { return OK; } +int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot) { + PRINT_NAME(); + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + return ProtectMemory(addr, len, prot); +} + int KYTY_SYSV_ABI KernelMtypeprotect(const void* addr, size_t len, int type, int prot) { PRINT_NAME(); - std::lock_guard memory_operation_lock(g_memory_operation_mutex); - - LOGF("\t addr = 0x%016" PRIx64 "\n" - "\t len = 0x%016" PRIx64 "\n" - "\t type = 0x%08" PRIx32 "\n" - "\t prot = 0x%08" PRIx32 "\n", - reinterpret_cast(addr), static_cast(len), static_cast(type), - static_cast(prot)); - - return KernelMprotect(addr, len, prot); + LOGF("\t type = %d\n", type); + return ProtectMemory(addr, len, prot); } int KYTY_SYSV_ABI KernelBatchMap2(KernelBatchMapEntry* entries, int num_entries, @@ -4026,23 +4006,19 @@ int KYTY_SYSV_ABI KernelBatchMap2(KernelBatchMapEntry* entries, int num_entries, int result = OK; switch (entry->operation) { case MAP_OP_MAP_DIRECT: - result = KernelMapNamedDirectMemory(&entry->start, entry->length, entry->protection, - flags, static_cast(entry->offset), 0, - "anon"); + result = MapDirectMemory(&entry->start, entry->length, entry->protection, flags, + static_cast(entry->offset), 0, "anon"); break; case MAP_OP_UNMAP: - result = KernelMunmap(reinterpret_cast(entry->start), entry->length); + result = UnmapMemory(reinterpret_cast(entry->start), entry->length); break; case MAP_OP_PROTECT: - result = KernelMprotect(entry->start, entry->length, entry->protection); + case MAP_OP_TYPE_PROTECT: + result = ProtectMemory(entry->start, entry->length, entry->protection); break; case MAP_OP_MAP_FLEXIBLE: - result = KernelMapNamedFlexibleMemory(&entry->start, entry->length, - entry->protection, flags, "anon"); - break; - case MAP_OP_TYPE_PROTECT: - result = - KernelMtypeprotect(entry->start, entry->length, entry->type, entry->protection); + result = MapFlexibleMemory(&entry->start, entry->length, entry->protection, flags, + "anon"); break; default: result = KERNEL_ERROR_EINVAL; break; } diff --git a/tests/VirtualMemoryAllocationTests.cpp b/tests/VirtualMemoryAllocationTests.cpp index 74d78cca..bdf9e4b0 100644 --- a/tests/VirtualMemoryAllocationTests.cpp +++ b/tests/VirtualMemoryAllocationTests.cpp @@ -1370,6 +1370,109 @@ void TestWindowsBackingViewPermissions() { } #endif +void TestDirectMappingNamesTypesAndValidation() { + using namespace Libs::LibKernel::Memory; + const char* test = "DirectMappingNamesTypesAndValidation"; + constexpr auto page = SceKernelPageSize; + int64_t physical = -1; + CheckOk(test, KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), page * 2, page, + SceKernelMtypeC, &physical), + "KernelAllocateDirectMemory"); + void* named = nullptr; + const std::string long_name(KERNEL_MAXIMUM_NAME_LENGTH, 'x'); + Check(test, + KernelMapNamedDirectMemory(nullptr, 0, SceKernelProtCpuExec, 0, -1, 0, long_name.c_str()) == + Libs::LibKernel::KERNEL_ERROR_ENAMETOOLONG, + "named-map validation no longer checks the name first"); + Check(test, KernelMapNamedDirectMemory(&named, page, SceKernelProtCpuRw, 0, physical, page, + nullptr) == Libs::LibKernel::KERNEL_ERROR_EFAULT && !named, + "null-name failure changed the output"); + CheckOk(test, KernelReserveVirtualRange(&named, page * 2, 0, page), "KernelReserveVirtualRange"); + const auto base = reinterpret_cast(named); + const auto name = long_name.substr(1); + CheckOk(test, KernelMapNamedDirectMemory(&named, page, SceKernelProtCpuRw, SceKernelMapFixed, + physical, page, name.c_str()), + "KernelMapNamedDirectMemory"); + void* second = reinterpret_cast(base + page); + CheckOk(test, KernelMapNamedDirectMemory(&second, page, SceKernelProtCpuRw, SceKernelMapFixed, + physical + page, page, "second"), + "KernelMapNamedDirectMemory(adjacent)"); + ExpectRange(test, Query(test, base), base, base + page, SceKernelProtCpuRw, + 0, 1, 0, 1, name.c_str(), physical); + ExpectRange(test, Query(test, base + page), base + page, base + page * 2, SceKernelProtCpuRw, + 0, 1, 0, 1, "second", physical + page); + void* typed = nullptr; + constexpr int type = 2; + CheckOk(test, KernelMapDirectMemory2(&typed, page, type, SceKernelProtCpuRw, 0, physical, page), + "KernelMapDirectMemory2"); + Check(test, Query(test, reinterpret_cast(typed)).memory_type == type && + Query(test, base).memory_type == SceKernelMtypeC, + "typed alias changed another mapping's type"); + *static_cast(typed) = 0x12345678; + Check(test, *static_cast(named) == 0x12345678, + "typed mapping stopped aliasing its backing"); + void* unchanged_type = nullptr; + CheckOk(test, KernelMapDirectMemory2(&unchanged_type, page, -1, SceKernelProtCpuRw, 0, + physical, page), + "KernelMapDirectMemory2(unchanged type)"); + Check(test, Query(test, reinterpret_cast(unchanged_type)).memory_type == SceKernelMtypeC, + "type -1 did not preserve the allocation's memory type"); + void* plain = nullptr; + CheckOk(test, KernelMapDirectMemory(&plain, page, SceKernelProtCpuRw, 0, physical, page), + "KernelMapDirectMemory"); + const auto plain_info = Query(test, reinterpret_cast(plain)); + Check(test, plain_info.memory_type == SceKernelMtypeC && plain_info.name[0] == '\0', + "plain mapping inherited another alias's type or name"); + CheckOk(test, KernelReleaseDirectMemory(physical, page * 2), "KernelReleaseDirectMemory"); + ExpectUnmapped(test, base); + ExpectUnmapped(test, reinterpret_cast(typed)); + ExpectUnmapped(test, reinterpret_cast(plain)); + ExpectUnmapped(test, reinterpret_cast(unchanged_type)); + std::printf("[host] %-48s ok\n", test); +} + +void TestBatchMappingOperationsAndPartialFailure() { + using namespace Libs::LibKernel::Memory; + const char* test = "BatchMappingOperationsAndPartialFailure"; + constexpr auto page = SceKernelPageSize; + int64_t physical = -1; + CheckOk(test, KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), page, page, + SceKernelMtypeC, &physical), + "KernelAllocateDirectMemory"); + void* reservation = nullptr; + CheckOk(test, KernelReserveVirtualRange(&reservation, page * 2, 0, page), "KernelReserveVirtualRange"); + const auto base = reinterpret_cast(reservation); + void* flexible = reinterpret_cast(base + page); + std::array entries {{ + {reservation, static_cast(physical), page, SceKernelProtCpuRw, 0, 0, 0}, + {flexible, 0, page, SceKernelProtCpuRw, 0, 0, 3}, + {reservation, 0, page, SceKernelProtCpuRead, 0, 0, 2}, + {flexible, 0, page, SceKernelProtCpuRead, 2, 0, 4}, + }}; + int processed = -1; + CheckOk(test, KernelBatchMap(entries.data(), entries.size(), &processed), "KernelBatchMap"); + Check(test, processed == entries.size(), "batch did not process every operation"); + ExpectRange(test, Query(test, base), base, base + page, SceKernelProtCpuRead, + 0, 1, 0, 1, "anon", physical); + ExpectRange(test, Query(test, base + page), base + page, base + page * 2, SceKernelProtCpuRead, + 1, 0, 0, 1, "anon"); + std::array partial {{ + {reservation, 0, page, 0, 0, 0, 1}, + {nullptr, static_cast(physical), page, SceKernelProtCpuRw, 0, 0, 0}, + {flexible, 0, page, 0, 0, 0, 1}, + }}; + Check(test, KernelBatchMap(partial.data(), partial.size(), &processed) == + Libs::LibKernel::KERNEL_ERROR_EINVAL && processed == 1, + "failed fixed mapping did not preserve the completed prefix count"); + ExpectUnmapped(test, base); + Check(test, Query(test, base + page).is_flexible == 1, "batch processed an entry after failure"); + CheckOk(test, KernelBatchMap2(&partial.back(), 1, &processed, 0), "KernelBatchMap2(unmap)"); + Check(test, processed == 1, "batch unmap count is wrong"); + ExpectUnmapped(test, base + page); + CheckOk(test, KernelReleaseDirectMemory(physical, page), "KernelReleaseDirectMemory"); + std::printf("[host] %-48s ok\n", test); +} + void TestDirectMapValidationBeforeOwnerMutation() { const char* test = "DirectMapValidationBeforeOwnerMutation"; int64_t invalid = -1; @@ -4267,6 +4370,8 @@ int main(int argc, char** argv) { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS RunTest(TestWindowsBackingViewPermissions); #endif + RunTest(TestDirectMappingNamesTypesAndValidation); + RunTest(TestBatchMappingOperationsAndPartialFailure); RunTest(TestDirectMapValidationBeforeOwnerMutation); RunTest(TestDirectReleaseRollbackRestoresOwnerMapping); RunTest(TestDirectReleaseContracts);