diff --git a/src/graphics/host_gpu/regionDefinitions.h b/src/graphics/host_gpu/regionDefinitions.h index aa6c4e92..b6bd162e 100644 --- a/src/graphics/host_gpu/regionDefinitions.h +++ b/src/graphics/host_gpu/regionDefinitions.h @@ -3,6 +3,7 @@ #include "common/bitArray.h" #include "common/common.h" +#include "kernel/memory.h" #include @@ -10,7 +11,8 @@ namespace Libs::Graphics { constexpr uint64_t TRACKER_PAGE_SIZE = 4ull * 1024ull; constexpr uint64_t TRACKER_REGION_SIZE = 4ull * 1024ull * 1024ull; -constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 40u; +constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 44u; +constexpr uint64_t LOWER_ADDRESS_SIZE = 1ull << 40u; constexpr size_t TRACKER_REGION_PAGES = TRACKER_REGION_SIZE / TRACKER_PAGE_SIZE; struct GuestRange { @@ -19,8 +21,11 @@ struct GuestRange { [[nodiscard]] constexpr bool Empty() const noexcept { return address == 0 && size == 0; } [[nodiscard]] constexpr bool Valid() const noexcept { - return address != 0 && size != 0 && address < TRACKER_ADDRESS_SIZE && - size <= TRACKER_ADDRESS_SIZE - address; + if (address == 0 || size == 0) return false; + if (address < LOWER_ADDRESS_SIZE) return size <= LOWER_ADDRESS_SIZE - address; + constexpr auto base = LibKernel::Memory::kExtendedMemoryBase; + constexpr auto length = LibKernel::Memory::kExtendedMemorySize; + return address >= base && address - base < length && size <= length - (address - base); } [[nodiscard]] constexpr bool ValidOrEmpty() const noexcept { return Empty() || Valid(); } [[nodiscard]] constexpr uint64_t End() const noexcept { return address + size; } diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index fe9f3b5f..b1a89163 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -55,7 +55,8 @@ template void BufferCache::ChangeRegister(BufferId id) { auto& buffer = m_slot_buffers[id]; PageTable::PageRange pages {}; - EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages)); + EXIT_IF(!(GuestRange {buffer.CpuAddress(), buffer.Size()}.Valid()) || + !PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages)); for (size_t page = pages.first; page < pages.last_exclusive; ++page) { if constexpr (insert) { m_page_table[page] = id; @@ -64,6 +65,7 @@ void BufferCache::ChangeRegister(BufferId id) { } } const auto size_pages = pages.last_exclusive - pages.first; + const auto table_offset = PageIndex(buffer.CpuAddress()) * sizeof(vk::DeviceAddress); if constexpr (insert) { const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id); (void)it; @@ -75,7 +77,7 @@ void BufferCache::ChangeRegister(BufferId id) { for (uint64_t i = 0; i < size_pages; ++i) { addresses.push_back(buffer.BufferDeviceAddress() + (i << CACHING_PAGEBITS)); } - WriteDataBuffer(m_bda_pagetable_buffer, pages.first * sizeof(vk::DeviceAddress), + WriteDataBuffer(m_bda_pagetable_buffer, table_offset, addresses.data(), addresses.size() * sizeof(vk::DeviceAddress)); } else { const auto found = m_buffers.find(buffer.CpuAddress()); @@ -84,7 +86,7 @@ void BufferCache::ChangeRegister(BufferId id) { EXIT_IF(buffer.Size() > m_total_used_memory); m_total_used_memory -= buffer.Size(); m_lru_cache.Free(buffer.lru_id); - m_bda_pagetable_buffer.Fill(pages.first * sizeof(vk::DeviceAddress), + m_bda_pagetable_buffer.Fill(table_offset, size_pages * sizeof(vk::DeviceAddress), 0); buffer.is_deleted = true; } @@ -312,10 +314,14 @@ BufferCache::OverlapResult BufferCache::ResolveOverlaps(uint64_t vaddr, uint64_t // Reserve space in the incoming stream's direction of growth. // The old buffer extending left of the request predicts growth to the right, and vice versa. if (expands_left) { - end += std::min(StreamLeapSize, PageTable::kAddressSpaceSize - end); + end += std::min(StreamLeapSize, (vaddr < LOWER_ADDRESS_SIZE ? LOWER_ADDRESS_SIZE + : LibKernel::Memory::kExtendedMemoryBase + + LibKernel::Memory::kExtendedMemorySize) - end); } if (expands_right) { - const auto minimum = CACHING_PAGESIZE * 2; + const auto minimum = vaddr < LOWER_ADDRESS_SIZE + ? CACHING_PAGESIZE * 2 + : LibKernel::Memory::kExtendedMemoryBase; if (begin > minimum) { begin -= std::min(StreamLeapSize, begin - minimum); } diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.h b/src/graphics/host_gpu/renderer/cache/bufferCache.h index 1ca6cae9..aa969a9d 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.h +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.h @@ -29,10 +29,22 @@ class BufferCache { public: static constexpr uint32_t CACHING_PAGEBITS = 14; static constexpr uint64_t CACHING_PAGESIZE = uint64_t {1} << CACHING_PAGEBITS; - static constexpr uint64_t CACHING_NUMPAGES = uint64_t {1} << (40 - CACHING_PAGEBITS); + static constexpr uint64_t CACHING_NUMPAGES = (LOWER_ADDRESS_SIZE + LibKernel::Memory::kExtendedMemorySize) >> CACHING_PAGEBITS; static constexpr uint64_t BDA_PAGETABLE_SIZE = CACHING_NUMPAGES * sizeof(vk::DeviceAddress); + static constexpr uint64_t PageIndex(uint64_t address) { + return (address < LOWER_ADDRESS_SIZE + ? address + : address - LibKernel::Memory::kExtendedMemoryBase + LOWER_ADDRESS_SIZE) >> + CACHING_PAGEBITS; + } + static constexpr uint64_t GuestAddress(uint64_t offset) { + return offset < LOWER_ADDRESS_SIZE + ? offset + : offset - LOWER_ADDRESS_SIZE + LibKernel::Memory::kExtendedMemoryBase; + } + BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager, TextureCache& texture_cache); ~BufferCache(); @@ -88,7 +100,7 @@ private: bool has_stream_leap; }; - using PageTable = MultiLevelPageTable; + using PageTable = MultiLevelPageTable; static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits)); void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size); void TouchBuffer(const Buffer& buffer); diff --git a/src/graphics/host_gpu/renderer/cache/faultManager.cpp b/src/graphics/host_gpu/renderer/cache/faultManager.cpp index cc9172e4..6720ceca 100644 --- a/src/graphics/host_gpu/renderer/cache/faultManager.cpp +++ b/src/graphics/host_gpu/renderer/cache/faultManager.cpp @@ -134,8 +134,9 @@ void FaultManager::ProcessFaultBuffer() { const auto* faults = std::bit_cast(mapped); const auto count = static_cast(faults[0]); for (uint32_t index = 1; index <= count; ++index) { - fault_ranges.Add(faults[index], BufferCache::CACHING_PAGESIZE); - LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", faults[index]); + const auto address = BufferCache::GuestAddress(faults[index]); + fault_ranges.Add(address, BufferCache::CACHING_PAGESIZE); + LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", address); } fault_ranges.ForEach([this](uint64_t start, uint64_t end) { EXIT_IF(end - start > std::numeric_limits::max()); diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.h b/src/graphics/host_gpu/renderer/cache/textureCache.h index 1eecd6af..66c5f908 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.h +++ b/src/graphics/host_gpu/renderer/cache/textureCache.h @@ -91,9 +91,9 @@ private: }; using ImageIds = InlinePageOwnerList; - using ImagePageTable = MultiLevelPageTable; + using ImagePageTable = MultiLevelPageTable; - // Callers have validated the nonempty 40-bit range with TryGetPageRange. + // Callers have validated the nonempty 44-bit range with TryGetPageRange. template static void ForEachPage(uint64_t address, size_t size, Func&& func) { using FuncReturn = typename std::invoke_result::type; diff --git a/src/graphics/host_gpu/renderer/image/image.cpp b/src/graphics/host_gpu/renderer/image/image.cpp index 062552d0..1cd45106 100644 --- a/src/graphics/host_gpu/renderer/image/image.cpp +++ b/src/graphics/host_gpu/renderer/image/image.cpp @@ -628,10 +628,8 @@ void Validate(const ImageInfo& info) { break; case ImageMetadataKind::Dcc: case ImageMetadataKind::Cmask: - if (info.metadata.range.address == 0 || - info.metadata.range.address >= TRACKER_ADDRESS_SIZE || - (info.metadata.range.size != 0 && - info.metadata.range.size > TRACKER_ADDRESS_SIZE - info.metadata.range.address) || + if (!GuestRange {info.metadata.range.address, + std::max(info.metadata.range.size, 1)}.Valid() || info.metadata.compression == VideoOutCompression::Unsupported) { EXIT("invalid color metadata\n"); } diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index cf13cbb9..09a592ba 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -196,8 +196,8 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, bo constexpr uint32_t htile_control = 0x00280000u; const uint32_t expected_control = htile_control | (descriptor.MsaaDepth() ? (1u << 10u) : 0u); const auto metadata_addr = descriptor.MetaAddr() << 8u; - return metadata_control == expected_control && metadata_addr != 0 && - metadata_addr < TRACKER_ADDRESS_SIZE && (metadata_addr & 0x7fffu) == 0 && + return metadata_control == expected_control && GuestRange {metadata_addr, 1}.Valid() && + (metadata_addr & 0x7fffu) == 0 && descriptor.TileMode() == Prospero::TileMode::kDepth; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 3d21e0f4..d16cf087 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -975,7 +975,14 @@ void DefineGetBdaPointer(EmitterState& state) { state.builder.AddFunction(spv::OpFunctionParameter, type, address); EmitLabel(state, entry_label); - const auto page64 = Binary(state, spv::OpShiftRightLogical, type, address, + const auto extended = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address, + ConstantDeviceAddress(state, LibKernel::Memory::kExtendedMemoryBase)); + const auto packed = Select(state, type, extended, + Binary(state, spv::OpISub, type, address, + ConstantDeviceAddress(state, + LibKernel::Memory::kExtendedMemoryBase - LOWER_ADDRESS_SIZE)), + address); + const auto page64 = Binary(state, spv::OpShiftRightLogical, type, packed, ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS)); const auto page = Unary(state, spv::OpUConvert, TypeU32(state), page64); const auto entry_pointer = state.builder.AllocateId(); diff --git a/tests/ImagePageTableTests.cpp b/tests/ImagePageTableTests.cpp index 940310bb..44f82745 100644 --- a/tests/ImagePageTableTests.cpp +++ b/tests/ImagePageTableTests.cpp @@ -9,9 +9,9 @@ namespace { using Owners = std::vector; -using Table = Libs::Graphics::MultiLevelPageTable; +using Table = Libs::Graphics::MultiLevelPageTable; using PageOwners = Libs::Graphics::InlinePageOwnerList; -using OwnerTable = Libs::Graphics::MultiLevelPageTable; +using OwnerTable = Libs::Graphics::MultiLevelPageTable; void Check(bool value, const char* text) { if (!value) { diff --git a/tests/MemoryTrackerTests.cpp b/tests/MemoryTrackerTests.cpp index fb48d1d5..67572717 100644 --- a/tests/MemoryTrackerTests.cpp +++ b/tests/MemoryTrackerTests.cpp @@ -219,17 +219,24 @@ void TestRangeSet() { void TestGuestRange() { constexpr GuestRange empty{}; constexpr GuestRange first_byte{1, 1}; - constexpr GuestRange last_byte{TRACKER_ADDRESS_SIZE - 1, 1}; + constexpr uint64_t extended_end = Libs::LibKernel::Memory::kExtendedMemoryBase + + Libs::LibKernel::Memory::kExtendedMemorySize; + constexpr GuestRange last_byte{extended_end - 1, 1}; static_assert(empty.Empty() && !empty.Valid() && empty.ValidOrEmpty()); static_assert(!first_byte.Empty() && first_byte.Valid() && first_byte.ValidOrEmpty() && first_byte.End() == 2); - static_assert(last_byte.Valid() && last_byte.End() == TRACKER_ADDRESS_SIZE); + static_assert(last_byte.Valid() && last_byte.End() == extended_end); Check(!GuestRange{0, 1}.Empty() && !GuestRange{0, 1}.ValidOrEmpty(), "zero-address nonempty guest range is rejected"); Check(!GuestRange{1, 0}.Empty() && !GuestRange{1, 0}.ValidOrEmpty(), "nonzero-address empty guest range is rejected"); + Check(GuestRange{Libs::LibKernel::Memory::kExtendedMemoryBase, 1}.Valid() && + !GuestRange{Libs::Graphics::LOWER_ADDRESS_SIZE, 1}.Valid() && + !GuestRange{Libs::LibKernel::Memory::kExtendedMemoryBase - 1, 2}.Valid() && + !GuestRange{extended_end - 1, 2}.Valid(), + "extended range and gap boundaries are enforced"); Check(!GuestRange{TRACKER_ADDRESS_SIZE, 1}.Valid(), "first address beyond the guest range is rejected"); Check(!GuestRange{TRACKER_ADDRESS_SIZE - 1, 2}.Valid(), diff --git a/tests/PageManagerTests.cpp b/tests/PageManagerTests.cpp index 6c36c65c..85aa3bcc 100644 --- a/tests/PageManagerTests.cpp +++ b/tests/PageManagerTests.cpp @@ -136,8 +136,8 @@ bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, &old_protection) != 0; } -uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) { - constexpr uintptr_t test_address = 0x0000000200010000ull; +uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE, + uintptr_t test_address = 0x0000000200010000ull) { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS auto *memory = static_cast( VirtualAlloc(reinterpret_cast(test_address), size, @@ -155,11 +155,11 @@ uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) { return memory; } -void TestWatchAndUnwatch() { +void TestWatchAndUnwatch(uint64_t base = 0x0000000200010000ull) { g_protection_calls = 0; PageManager manager; const auto page_size = manager.GetPageSize(); - auto *memory = Allocate(page_size * 2); + auto *memory = Allocate(page_size * 2, PAGE_READWRITE, base); const auto address = reinterpret_cast(memory); manager.UpdatePageWatchers(address, page_size); @@ -580,6 +580,7 @@ int main(int argc, char **argv) { RunDeathCase(argv[2]); } TestWatchAndUnwatch(); + TestWatchAndUnwatch(Libs::LibKernel::Memory::kExtendedMemoryBase); TestSharedWatcherCounts(); TestCrossRegionRange(); TestBatchedWatcherRanges(); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index bd6a4fa0..c8e11a22 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -208,8 +208,8 @@ struct TextureCacheTestAccess { static_assert(std::same_as>); static_assert(TextureCache::ImagePageTable::kPageBits == 20); - static_assert(TextureCache::ImagePageTable::kAddressSpaceBits == 40); - static_assert(TextureCache::ImagePageTable::kFirstLevelBits == 10); + static_assert(TextureCache::ImagePageTable::kAddressSpaceBits == 44); + static_assert(TextureCache::ImagePageTable::kFirstLevelBits == 14); static std::unique_lock Lock(TextureCache &cache) { return std::unique_lock(cache.m_lock); @@ -13694,7 +13694,7 @@ public: (bytes + BufferCache::CACHING_PAGESIZE - 1) / BufferCache::CACHING_PAGESIZE; auto address = buffer.device_address + backing - page_offset; - auto page = guest_base >> BufferCache::CACHING_PAGEBITS; + auto page = BufferCache::PageIndex(guest_base); for (uint64_t mapped = 0; mapped < pages; mapped++) { cmd.updateBuffer(m_bda_pagetable_buffer.buffer, (page + mapped) * sizeof(vk::DeviceAddress), @@ -25957,6 +25957,11 @@ TestCase FlatVirtualAddressRebasesGuestAllocation() { using O = ShaderOpcode; constexpr uint64_t GuestBase = 0x0000000110000000ull; + constexpr uint64_t ExtendedBase = Libs::LibKernel::Memory::kExtendedMemoryBase + GuestBase; + static_assert(BufferCache::PageIndex(ExtendedBase) == + ((Libs::Graphics::LOWER_ADDRESS_SIZE + GuestBase) >> BufferCache::CACHING_PAGEBITS)); + static_assert(BufferCache::GuestAddress(BufferCache::PageIndex(ExtendedBase) << + BufferCache::CACHING_PAGEBITS) == ExtendedBase); std::vector code; AppendVMovLiteral(&code, 20, static_cast(GuestBase + 4u)); AppendVMovLiteral(&code, 21, static_cast(GuestBase >> 32u)); @@ -25967,17 +25972,22 @@ TestCase FlatVirtualAddressRebasesGuestAllocation() { AppendVMovLiteral(&code, 21, static_cast((GuestBase >> 32u) + 1u)); code.push_back(EncodeFlat0(0x0c, 0, 0)); code.push_back(EncodeFlat1(2, 0x7d, 0, 20)); + AppendVMovLiteral(&code, 20, static_cast(ExtendedBase + 4u)); + AppendVMovLiteral(&code, 21, static_cast(ExtendedBase >> 32u)); + code.push_back(EncodeFlat0(0x0c, 0, 0)); + code.push_back(EncodeFlat1(3, 0x7d, 0, 20)); AppendStoreVgpr(&code, 0, 0); AppendStoreVgpr(&code, 1, 1); AppendStoreVgpr(&code, 2, 2); + AppendStoreVgpr(&code, 3, 3); AppendEnd(&code); TestCase test; test.name = "FlatVirtualAddressRebasesGuestAllocation"; test.code = std::move(code); test.initial = {0xfeedfaceu, 0xcafebabeu, 0, 0x12345678u}; - test.expected = {0x12345678u, 0x12345678u, 0}; - test.bda_mappings = {{GuestBase, 8}}; + test.expected = {0x12345678u, 0x12345678u, 0, 0xcafebabeu}; + test.bda_mappings = {{GuestBase, 8}, {ExtendedBase, 0}}; test.opcodes = {O::V_MOV_B32, O::FLAT_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; return test;