diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp index f38b2a916..2fdf3e9bd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp @@ -1952,6 +1952,11 @@ private: // Debug aid (APS5_TRACE_DCC_KEYS=1): the title's fills of a surface's DCC metadata. static const bool traceKeys = std::getenv("APS5_TRACE_DCC_KEYS") != nullptr; if (traceKeys && coverage.cover == Graphics::StorageTexture::FillCover::Keys) std::fprintf(stderr, "[dcc-keys] title fills keys 0x%llx+0x%zx with %08x %08x %08x %08x (queue 0x%x)\n", static_cast(base), bytes, pattern[0], pattern[1], pattern[2], pattern[3], queueId); + const bool uniformKeysFill = coverage.cover == Graphics::StorageTexture::FillCover::Keys && std::all_of(pattern.begin(), pattern.end(), [&](std::uint32_t word) { return word == (pattern[0] & 0xffu) * 0x01010101u; }); + if (uniformKeysFill) { + Graphics::StorageTexture::NoteKeysFill(base, bytes, static_cast(pattern[0])); + Graphics::StorageTexture::ClearByKeysFill(base, bytes, static_cast(pattern[0])); + } // Results pending in images wholly inside the range are dead whichever way the fill is // done (it overwrites every byte of them): dropped rather than stored first. const auto discarded = fillClearEnabled() ? Graphics::StorageTexture::DiscardPendingInside(base, bytes) : 0u; @@ -1983,6 +1988,19 @@ private: phase(FillFlush); const bool stored = cleared || localDevice->FillBuffer(base, bytes, pattern); phase(FillDevice); + if (uniformKeysFill && stored && !cleared) { + Graphics::DccKeys filled = Graphics::DccKeys::Mixed; + switch (pattern[0] & 0xffu) { + case 0x00: filled = Graphics::DccKeys::Clear0000; break; + case 0x40: filled = Graphics::DccKeys::Clear0001; break; + case 0x80: filled = Graphics::DccKeys::Clear1110; break; + case 0xc0: filled = Graphics::DccKeys::Clear1111; break; + case 0x20: filled = Graphics::DccKeys::ClearRegister; break; + case 0xff: filled = Graphics::DccKeys::Uncompressed; break; + default: break; + } + if (filled != Graphics::DccKeys::Mixed) Graphics::NoteKeysFillOnGpu(base, bytes, filled); + } if (profile && stored && !cleared) { const bool uniform = pattern[0] == pattern[1] && pattern[1] == pattern[2] && pattern[2] == pattern[3]; ++(uniform ? uniformFills : patternFills); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp index ccd64b143..edaeb4532 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp @@ -71,6 +71,7 @@ struct DccKeyProofCounts { DccKeyProofCounts KeyProofCounts(); // A surface's texels as a read sees them: the guest bytes, or the clear value of fast-cleared keys. void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span bytes); +void NoteKeysFillOnGpu(std::uint64_t begin, std::size_t count, DccKeys keys); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index 6fc5ea467..697d5c330 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -214,6 +214,8 @@ public: std::size_t inside = 0; }; static FillCoverage ClassifyFill(std::uint64_t address, std::size_t bytes); + static std::size_t NoteKeysFill(std::uint64_t address, std::size_t bytes, std::uint8_t key); + static std::size_t ClearByKeysFill(std::uint64_t address, std::size_t bytes, std::uint8_t key); // Results pending in images lying wholly inside the range are dead (a fill overwrites every // byte of them): they are dropped instead of stored. Returns how many images were. static std::size_t DiscardPendingInside(std::uint64_t address, std::size_t bytes); @@ -256,6 +258,7 @@ public: // (ProvedClearKeys): Refresh's own key rule for the fast revalidation. Under // GuestMemory::GpuMutex only, as Refresh is; never from a build's stage A. DccKeys UploadedKeys() const { return uploadedKeys; } + DccKeys FilledKeys() const { return filledKeys; } DccKeyProof& KeyProof() const { return keyProof; } // Brings the image up to date with guest memory before another use; returns whether its content // was still current (nothing uploaded). @@ -361,6 +364,7 @@ private: bool skippedResultsInside(std::uint64_t address, std::size_t bytes) const; std::uint64_t borrowUnits(StorageTexture& source, const std::vector& units); void forgetBorrowed(std::uint32_t first, std::uint32_t count); + bool clearByKeysFill(DccKeys keys, std::uint8_t key); bool overlaps(std::uint64_t address, std::size_t bytes) const; bool pendingUnitInside(std::uint64_t address, std::size_t bytes) const; VkImageView createView(std::uint32_t mip) const; @@ -377,6 +381,7 @@ private: std::vector original; // DCC keys the image content was uploaded under: a fast-cleared surface starts as its clear value. DccKeys uploadedKeys = DccKeys::Uncompressed; + DccKeys filledKeys = DccKeys::Uncompressed; mutable DccKeyProof keyProof; // Write generation `original` is known current at (the oldest of layerGeneration). std::uint64_t generation = 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp index a7ba917ba..144b6f3aa 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp @@ -128,6 +128,7 @@ void CountStore(std::atomic& counter) { struct GpuKeyStore { std::uint64_t begin; std::uint64_t end; + DccKeys keys = DccKeys::Uncompressed; }; struct KeyStoreMemo { @@ -157,7 +158,6 @@ void ForgetStores(std::uint64_t begin, std::uint64_t end) { std::erase_if(memo.entries, [&](const GpuKeyStore& store) { return begin < store.end && store.begin < end; }); } -// Uncompressed when a GPU store still pending covers the whole range, else nullopt (the bytes decide). std::optional MemoizedKeys(std::uint64_t begin, std::uint64_t end) { auto& memo = Memo(); std::lock_guard lock(memo.mutex); @@ -181,7 +181,7 @@ std::optional MemoizedKeys(std::uint64_t begin, std::uint64_t end) { return std::nullopt; } if (ScanProfileEnabled()) Scans().memoHits.fetch_add(1, std::memory_order_relaxed); - return DccKeys::Uncompressed; + return it->keys; } return std::nullopt; } @@ -248,6 +248,20 @@ void StoreUncompressedOnCpu(std::uint64_t begin, std::size_t count) { CountStore(Scans().cpuStores); } +} + +void NoteKeysFillOnGpu(std::uint64_t begin, std::size_t count, DccKeys keys) { + static const bool disabled = std::getenv("APS5_NO_KEYS_FILL_MEMO") != nullptr; + if (disabled || count == 0) return; + auto& memo = Memo(); + std::lock_guard lock(memo.mutex); + const auto end = begin + count; + std::erase_if(memo.entries, [&](const GpuKeyStore& store) { return begin < store.end && store.begin < end; }); + if (MemoRecorder(memo) != nullptr) memo.entries.push_back({begin, end, keys}); +} + +namespace { + // Whether the keys already read as uncompressed: scans the same bytes as ReadDccKeys, so it counts // as a scan too. bool AlreadyUncompressed(std::uint64_t metaAddress, std::size_t count) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index c4b2930f9..048283b38 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -299,13 +299,29 @@ std::shared_ptr cachedTexture(const Context& context, std::span keys; + bool clearThroughKeys = false; + if (source != nullptr && resource.dccAddress != 0 && IsDccClear(source->FilledKeys())) { + keys = scanKeys(); + if (*keys == source->FilledKeys()) { + std::array probe{}; + if (!FillDccClear(ResolveTextureFormat(resource.format), *keys, resource.dccAlphaOnMsb, probe)) { + char text[256]; + std::snprintf(text, sizeof(text), "AGC graphics: sampled texture 0x%llx (%ux%u format %u, dcc 0x%llx) reads %s DCC keys filled over its pending image, a clear value the format has no encoding for", static_cast(resource.baseAddress), resource.width, resource.height, resource.format, static_cast(resource.dccAddress), DccKeysName(*keys)); + throw std::runtime_error(text); + } + static const bool traceKeys = std::getenv("APS5_TRACE_DCC_KEYS") != nullptr; + if (traceKeys) std::fprintf(stderr, "[dcc-keys] sampled 0x%llx through keys 0x%llx reads the %s fill, not the pending image\n", static_cast(address), static_cast(resource.dccAddress), DccKeysName(*keys)); + source.reset(); + clearThroughKeys = true; + } + } // Otherwise a surface in host-imported memory is viewed through its cached storage image (made // here when there is none): its refresh after a CPU or GPU write is a GPU-direct detile from the // import, recorded behind the producer, so no bytes are read or compared on the CPU and nothing // waits for the producer. A fast-cleared surface (keys) is viewed only through an image whose own // descriptor carries the DCC address (below); it stays a snapshot otherwise, its texels not read. - std::optional keys; - if (source == nullptr && SampledFromStorageEligible(context, resource, guestBytes)) { + if (source == nullptr && !clearThroughKeys && SampledFromStorageEligible(context, resource, guestBytes)) { keys = scanKeys(); if (*keys == DccKeys::Uncompressed) { source = sampledStorageSource(context, resource, guestBytes); @@ -337,7 +353,7 @@ std::shared_ptr cachedTexture(const Context& context, std::span* layers) { refreshGeneration(); }; uploadedKeys = ProvedClearKeys(descriptor, guestBytes, keyProof); + filledKeys = DccKeys::Uncompressed; VkClearColorValue clearValue{}; if (IsDccClear(uploadedKeys) && ClearColorFor(storageFormat, uploadedKeys, clearValue)) { // A fast-cleared surface is cleared on the GPU; its texel memory is neither read nor filled. @@ -2328,7 +2329,7 @@ StorageTexture::FillCoverage StorageTexture::ClassifyFill(std::uint64_t address, } else if (overlapping.empty()) { constexpr std::uint64_t keyBytes = 256; for (const auto* texture : live.textures) { - if (!texture->released && texture->descriptor.dccAddress == address && texture->guestBytes / keyBytes == bytes) coverage.cover = FillCover::Keys; + if (!texture->released && texture->descriptor.dccAddress == address && texture->guestBytes / keyBytes != 0 && bytes >= texture->guestBytes / keyBytes) coverage.cover = FillCover::Keys; } } else if (overlapping.size() > 1) { coverage.cover = FillCover::Several; @@ -2358,6 +2359,123 @@ StorageTexture::FillCoverage StorageTexture::ClassifyFill(std::uint64_t address, return coverage; } +std::size_t StorageTexture::NoteKeysFill(std::uint64_t address, std::size_t bytes, std::uint8_t key) { + static const bool refillDisabled = std::getenv("APS5_NO_KEYS_REFILL") != nullptr; + DccKeys keys = DccKeys::Mixed; + switch (key) { + case 0x00: keys = DccKeys::Clear0000; break; + case 0x40: keys = DccKeys::Clear0001; break; + case 0x80: keys = DccKeys::Clear1110; break; + case 0xc0: keys = DccKeys::Clear1111; break; + case 0x20: keys = DccKeys::ClearRegister; break; + case 0xff: keys = DccKeys::Uncompressed; break; + default: return 0; + } + constexpr std::uint64_t keyBytes = 256; + static const bool traceKeys = std::getenv("APS5_TRACE_DCC_KEYS") != nullptr; + auto& live = Live(); + std::lock_guard lock(live.mutex); + std::size_t covered = 0; + for (auto* texture : live.textures) { + if (texture->released || texture->descriptor.dccAddress != address || texture->guestBytes / keyBytes == 0 || bytes < texture->guestBytes / keyBytes) continue; + texture->filledKeys = keys; + ++covered; + if (traceKeys && IsDccClear(keys)) std::fprintf(stderr, "[dcc-keys] %s key fill over 0x%llx+0x%llx (uploaded %s, dirty %d)\n", DccKeysName(keys), static_cast(texture->descriptor.baseAddress), static_cast(texture->guestBytes), DccKeysName(texture->uploadedKeys), texture->dirty ? 1 : 0); + if (refillDisabled || !IsDccClear(keys) || texture->uploadedKeys != keys) continue; + texture->uploadedKeys = DccKeys::Uncompressed; + } + return covered; +} + +std::size_t StorageTexture::ClearByKeysFill(std::uint64_t address, std::size_t bytes, std::uint8_t key) { + static const bool enabled = [] { const char* text = std::getenv("APS5_KEYS_FILL_CLEAR"); return text != nullptr && std::strcmp(text, "1") == 0; }(); + if (!enabled) return 0; + DccKeys keys = DccKeys::Mixed; + switch (key) { + case 0x00: keys = DccKeys::Clear0000; break; + case 0x40: keys = DccKeys::Clear0001; break; + case 0x80: keys = DccKeys::Clear1110; break; + case 0xc0: keys = DccKeys::Clear1111; break; + case 0x20: keys = DccKeys::ClearRegister; break; + default: return 0; + } + constexpr std::uint64_t keyBytes = 256; + std::vector> covered; + { + auto& live = Live(); + std::lock_guard lock(live.mutex); + for (auto* texture : live.textures) { + if (texture->released || texture->descriptor.dccAddress != address || texture->guestBytes / keyBytes == 0 || bytes < texture->guestBytes / keyBytes) continue; + if (auto shared = texture->weak_from_this().lock()) covered.push_back(std::move(shared)); + } + } + std::size_t cleared = 0; + for (const auto& texture : covered) { + if (texture->clearByKeysFill(keys, key)) ++cleared; + } + return cleared; +} + +bool StorageTexture::clearByKeysFill(DccKeys keys, std::uint8_t key) { + static const bool traceKeys = std::getenv("APS5_TRACE_DCC_KEYS") != nullptr; + VkClearColorValue clearValue{}; + auto* recorder = Recorder::Active(); + const char* refusal = !ClearColorFor(storageFormat, keys, clearValue) ? (keys == DccKeys::ClearRegister ? "clear-register code" : "no clear value in the storage format") : recorder == nullptr ? "no recorder" : nullptr; + if (refusal != nullptr) { + static std::atomic reported{false}; + if (!reported.exchange(true)) std::fprintf(stderr, "[dcc-keys] key fill 0x%02x (%s) over surface 0x%llx+0x%llx (guest format %u, vk format %d, keys 0x%llx) not applied at once: %s\n", key, DccKeysName(keys), static_cast(descriptor.baseAddress), static_cast(guestBytes), descriptor.format, static_cast(storageFormat), static_cast(descriptor.dccAddress), refusal); + return false; + } + std::size_t droppedUnits = 0; + const bool wasDirty = dirty; + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + droppedUnits = static_cast(std::count(layerPending.begin(), layerPending.end(), true)); + layerPending.assign(trackedLayers, false); + if (dirty) { + dirty = false; + std::erase(pending.textures, this); + BumpPendingSerial(); + } + } + const auto commands = recorder->Commands(); + const auto timing = recorder->BeginGpuTiming(Recorder::CommandClass::DccClear); + if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + Recorder::CountBarriers(Recorder::CommandClass::DccClear, 2); + if (Recorder::BarrierValidate()) { + const std::pair clearedImage{image, true}; + recorder->NoteAccess(Recorder::CommandClass::DccClear, Recorder::Access{{}, {}, std::span(&clearedImage, 1), VK_PIPELINE_STAGE_TRANSFER_BIT}); + } + VkImageMemoryBarrier toClear{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toClear.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + toClear.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toClear.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toClear.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toClear.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toClear.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toClear.image = image; + toClear.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toClear); + context.Function("vkCmdClearColorImage")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &clearValue, 1, &toClear.subresourceRange); + VkImageMemoryBarrier toGeneral = toClear; + toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); + recorder->EndGpuTiming(timing, guestBytes); + originalValid = false; + uploadedKeys = keys; + keyProof = {}; + forgetBorrowed(0, trackedLayers); + layerGeneration.assign(trackedLayers, GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes))); + refreshGeneration(); + ++version; + if (traceKeys) std::fprintf(stderr, "[dcc-keys] key fill %s clears 0x%llx+0x%llx at once (keys 0x%llx, %zu pending units dropped, dirty %d)\n", DccKeysName(keys), static_cast(descriptor.baseAddress), static_cast(guestBytes), static_cast(descriptor.dccAddress), droppedUnits, wasDirty ? 1 : 0); + return true; +} + bool StorageTexture::FillClear(std::span pattern, std::uint32_t layer, const char*& refusal) { // Debug aid (APS5_TRACE_FILL_COVER=1): the first layer covers refused, with the surface, the // pattern and the generation the refusal was memoized at (ClassifyFill's layerRefusedGeneration). @@ -2960,7 +3078,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { if (profile) Profile().storageGpu += timer.lap(); originalValid = false; traceKeyStore("block write-back", descriptor, guestBytes); - MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + if (!IsDccClear(filledKeys)) MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); uploadedKeys = DccKeys::Uncompressed; for (const auto& [from, to] : keep) GuestMemory::MarkWritten(from, static_cast(to - from)); settle(true); @@ -3055,7 +3173,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { // metadata is host-imported (no CPU wait for the title's key-writing kernels), else a CPU // store (APS5_CPU_DCC_KEYS=1 keeps the CPU store; see DccMetadata.hpp). traceKeyStore("layer write-back", descriptor, guestBytes); - MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + if (!IsDccClear(filledKeys)) MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); uploadedKeys = DccKeys::Uncompressed; for (const auto& [from, to] : keep) GuestMemory::MarkWritten(from, static_cast(to - from)); // The walk covers this surface's pages only: an adjacent image's later CPU write is stamped @@ -3163,7 +3281,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { // The texels now hold the whole image, so later reads must see them rather than a fast clear // (the keys may be host-imported although the texels were not: then a recorded fill, else a CPU store). traceKeyStore("cpu write-back", descriptor, guestBytes); - MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + if (!IsDccClear(filledKeys)) MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); uploadedKeys = DccKeys::Uncompressed; // The store above is the only write to these pages, so `original` is current at a fresh // generation, unless blocks were kept for the CPU: then the image is stale there.