diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index d21498788..f1ef0b0da 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -308,6 +308,27 @@ private: // The windows moving the surface-relative runs: whole tile blocks of thin tiled mips, a tail // mip whole (its block holds every tail level), in slice order. std::vector sliceWindows(std::span> runs) const; + // The bytes of the surface-relative runs no element of the surface holds (the padding of partly + // covered tile blocks and of linear rows, tail blocks, the bytes between mips; every byte of a + // thick surface), ascending: a write-back keeps the guest's bytes there. + std::vector> uncoveredBytes(std::span> runs) const; + // The seed of a write-back's tiled scratch where its copies would carry bytes no element holds: + // those bytes' current contents (a fresh unit shadow's, else the import's), copied into the + // scratch before the retile, so the copies out of the scratch store them unchanged. `copied` + // lists the surface-relative [begin, end) ranges the write-back copies out of the scratch, each + // with the scratch offset of its begin. + struct CopiedBytes { + std::uint64_t begin; + std::uint64_t end; + std::uint64_t scratch; + }; + struct PaddingSeeds { + std::vector>> copies; + std::vector> slabs; + // Guest ranges read from the import. + std::vector> importReads; + }; + PaddingSeeds paddingSeeds(const HostImport& import, std::vector copied) const; // The direct upload and GPU-direct write-back of the runs through their windows (block units); // both return the bytes moved. The upload reads each unit from its unit shadow while fresh, // else from the import (`discard`: the image has no layout yet, a whole-surface first upload). diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp index 691baa8aa..adc0bc343 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace AgcDriver::Graphics { @@ -34,6 +35,11 @@ std::array ThickBlockExtent(TextureTileMode tileMode, std::uin // (the detiler's blockExtent). A mip's tiled bytes are its blocks in row-major order, blocksPerRow per // row (TileMipLayout), except the tail mips, which share one block. Throws for linear tiling. std::array ThinBlockLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement); +// The bytes of a thin mip (mip-relative [begin, end), ascending and merged) that hold one of its +// elements each: its whole tile blocks (every element of the block inside the mip), or its rows +// without the pitch padding when linear. A tail mip has none (its block also holds the other tail +// levels and their padding). A retile of the mip writes every byte of these and no other. +std::vector> CoveredMipBytes(TextureTileMode tileMode, std::uint32_t bytesPerElement, const TileMipLayout& mip); // 3D surface. Each slab holds the whole mip chain, smallest level first. Depth slice z of a level is detiled // from slab z / blockDepth (slabBytes apart) with the swizzle's slice input set to z, into linear diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp index b69b285fc..7748a0e9d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp @@ -100,7 +100,8 @@ bool AnyShadowedOverlaps(std::span // from one buffer (a slab while the unit is fresh, else the import) at `offset` (byte x of the run // is at offset + (x - begin)), split at unit freshness and slab boundaries with one tracker query // over the span. A tail block (surface-relative, every unit of it) must have one source: a mixed -// one is published (TailMip) and read from the import. +// one is published (TailMip) and read from the import. `countReads` false: not counted as detiled +// bytes (a write-back reading the current bytes of its padding). struct ShadowRun { std::uint64_t begin; std::uint64_t end; @@ -109,7 +110,7 @@ struct ShadowRun { bool shadow; std::shared_ptr slab; }; -std::vector ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span> runs, std::span> tailBlocks); +std::vector ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span> runs, std::span> tailBlocks, bool countReads = true); // The slab destination of a retile copy [begin, end) (guest addresses inside one slab: the caller // splits at SlabBoundary), making the slab under the budget; nullopt when refused (the caller writes diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index ea69132a0..777529fa5 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -1617,6 +1617,83 @@ std::vector StorageTexture::sliceWindows(std::span< return windows; } +std::vector> StorageTexture::uncoveredBytes(std::span> runs) const { + std::vector> covered; + if (!geometry.thick) { + const auto elementBytes = BytesPerElement(descriptor.format); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (std::uint32_t level = 0; level < descriptor.mipCount; ++level) { + if (!geometry.HasLayer(level, layer)) continue; + const auto sliceBegin = geometry.GuestLayerOffset(layer) + mips[level].tiledOffset; + for (const auto& [begin, end] : CoveredMipBytes(descriptor.tileMode, elementBytes, mips[level])) covered.emplace_back(sliceBegin + begin, sliceBegin + end); + } + } + std::sort(covered.begin(), covered.end()); + } + std::vector> sorted(runs.begin(), runs.end()); + std::sort(sorted.begin(), sorted.end()); + std::vector> uncovered; + const auto add = [&](std::uint64_t begin, std::uint64_t end) { + if (begin >= end) return; + if (!uncovered.empty() && uncovered.back().second >= begin) uncovered.back().second = std::max(uncovered.back().second, end); + else uncovered.emplace_back(begin, end); + }; + std::size_t next = 0; + for (const auto& [runBegin, runEnd] : sorted) { + while (next < covered.size() && covered[next].second <= runBegin) ++next; + auto at = runBegin; + for (auto i = next; i < covered.size() && covered[i].first < runEnd; ++i) { + add(at, std::min(covered[i].first, runEnd)); + at = std::max(at, covered[i].second); + } + add(at, runEnd); + } + return uncovered; +} + +StorageTexture::PaddingSeeds StorageTexture::paddingSeeds(const HostImport& import, std::vector copied) const { + PaddingSeeds seeds; + std::sort(copied.begin(), copied.end(), [](const CopiedBytes& a, const CopiedBytes& b) { return a.begin < b.begin; }); + std::vector> runs; + for (const auto& range : copied) runs.emplace_back(range.begin, range.end); + const auto uncovered = uncoveredBytes(runs); + if (uncovered.empty()) return seeds; + std::vector pieces; + std::vector> pieceRuns; + std::size_t next = 0; + for (const auto& [begin, end] : uncovered) { + while (next < copied.size() && copied[next].end <= begin) ++next; + for (auto i = next; i < copied.size() && copied[i].begin < end; ++i) { + const auto from = std::max(begin, copied[i].begin); + const auto to = std::min(end, copied[i].end); + if (from >= to) continue; + pieces.push_back({from, to, copied[i].scratch + (from - copied[i].begin)}); + pieceRuns.emplace_back(from, to); + } + } + if (pieces.empty()) return seeds; + const auto sources = ShadowSources(context, import, descriptor.baseAddress, pieceRuns, {}, false); + std::size_t first = 0; + for (const auto& source : sources) { + while (first < pieces.size() && pieces[first].end <= source.begin) ++first; + for (auto i = first; i < pieces.size() && pieces[i].begin < source.end; ++i) { + const auto from = std::max(source.begin, pieces[i].begin); + const auto to = std::min(source.end, pieces[i].end); + if (from >= to) continue; + auto target = std::find_if(seeds.copies.begin(), seeds.copies.end(), [&](const auto& entry) { return entry.first == source.buffer; }); + if (target == seeds.copies.end()) target = seeds.copies.insert(seeds.copies.end(), {source.buffer, {}}); + target->second.push_back({source.offset + (from - source.begin), pieces[i].scratch + (from - pieces[i].begin), to - from}); + if (!source.shadow) { + const auto guestBegin = descriptor.baseAddress + from; + if (!seeds.importReads.empty() && seeds.importReads.back().second == guestBegin) seeds.importReads.back().second = descriptor.baseAddress + to; + else seeds.importReads.emplace_back(guestBegin, descriptor.baseAddress + to); + } + } + if (source.shadow && std::find(seeds.slabs.begin(), seeds.slabs.end(), source.slab) == seeds.slabs.end()) seeds.slabs.push_back(source.slab); + } + return seeds; +} + std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span> runs, bool discard) { const auto elementBytes = BytesPerElement(descriptor.format); // Each run's pieces by source (a unit shadow's slab while fresh, else the import), the tail @@ -1808,6 +1885,17 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp } } std::sort(seeds.begin(), seeds.end(), [](const ShadowedRange& a, const ShadowedRange& b) { return a.begin < b.begin; }); + std::vector copied; + for (std::size_t i = 0; i < windows.size(); ++i) { + const auto& window = windows[i]; + if (i != 0 && scratchPositions[i] == scratchPositions[i - 1]) continue; + for (const auto& [from, to] : runs) { + const auto begin = std::max(from, window.tiledBegin); + const auto end = std::min(to, window.tiledEnd); + if (begin < end) copied.push_back({begin, end, scratchPositions[i] + (begin - window.tiledBegin)}); + } + } + const auto padding = paddingSeeds(import, std::move(copied)); // A queued label or key store inside a seeded unit lands before the seed copies the import. auto flushBegin = firstStored; auto flushEnd = lastStored; @@ -1816,7 +1904,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp flushEnd = std::max(flushEnd, seed.end); } auto linear = std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); - auto tiledScratch = std::make_shared(context, static_cast(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + auto tiledScratch = std::make_shared(context, static_cast(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); std::unique_ptr batch; @@ -1831,12 +1919,14 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp recorder->Keep(linear); recorder->Keep(tiledScratch); for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab); + for (const auto& slab : padding.slabs) recorder->Keep(slab); if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4); if (Recorder::BarrierValidate()) { const std::pair read{image, false}; std::vector> seedReads; for (const auto& seed : seeds) seedReads.emplace_back(seed.begin, seed.end); + seedReads.insert(seedReads.end(), padding.importReads.begin(), padding.importReads.end()); recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{seedReads, imported, std::span(&read, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT}); } } else { @@ -1853,7 +1943,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp toSource.image = image; toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); - if (!seeds.empty()) { + if (!seeds.empty() || !padding.copies.empty()) { // The seeded units' import bytes (every earlier writer of them, host stores included) // precede the seed copies; the previous command's trailing barrier may have covered that. constexpr VkAccessFlags transferAccess = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; @@ -1869,6 +1959,9 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, import.buffer, slab->buffer, 1, &seed); NoteShadowSeed(begin, end); } + // The padding's current bytes, under the retile that follows (the import-ready barrier + // orders the scratch writes). + for (const auto& [source, copies] : padding.copies) context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast(copies.size()), copies.data()); } std::vector regions; for (std::size_t i = 0; i < windows.size(); ++i) { @@ -3224,7 +3317,10 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { // untouched blocks are copied into the imported bytes in place, recorded behind the work that // produced the image; nothing crosses to the CPU. auto linear = std::make_shared(context, static_cast(sliceLinearBytes * arrayLayers), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); - auto tiledScratch = std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + auto tiledScratch = std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + std::vector copied; + for (const auto& [from, to] : keep) copied.push_back({from - descriptor.baseAddress, to - descriptor.baseAddress, from - descriptor.baseAddress}); + const auto padding = paddingSeeds(*import, std::move(copied)); detiler.BeginBatch(); auto* recorder = Recorder::Active(); std::unique_ptr batch; @@ -3239,12 +3335,13 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); recorder->Keep(linear); recorder->Keep(tiledScratch); + for (const auto& slab : padding.slabs) recorder->Keep(slab); // The image itself must outlive the recorded retile: the cache may evict it right after. if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); - Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4); + Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, padding.copies.empty() ? 4 : 5); if (Recorder::BarrierValidate()) { const std::pair read{image, false}; - recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{{}, keep, std::span(&read, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT}); + recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{padding.importReads, keep, std::span(&read, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT}); } } else { batch = std::make_unique(context); @@ -3264,6 +3361,13 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { toSource.image = image; toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); + if (!padding.copies.empty()) { + // The current bytes of the copied ranges no element holds go under the retile (every + // earlier writer of the import, host stores included, first; the import-ready barrier + // below orders the scratch writes). + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + for (const auto& [source, copies] : padding.copies) context.Function("vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast(copies.size()), copies.data()); + } const auto regions = CopyRegions(storedLayers); context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp index 0de68a1d5..14e44f047 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp @@ -255,6 +255,30 @@ std::array ThinBlockLayout(TextureTileMode tileMode, std::uint return {block.blockSize, block.blockWidth, block.blockHeight}; } +std::vector> CoveredMipBytes(TextureTileMode tileMode, std::uint32_t bytesPerElement, const TileMipLayout& mip) { + std::vector> covered; + if (mip.tail || mip.width == 0 || mip.height == 0) return covered; + const auto add = [&](std::uint64_t begin, std::uint64_t end) { + end = std::min(end, mip.tiledSize); + if (begin >= end) return; + if (!covered.empty() && covered.back().second == begin) covered.back().second = end; + else covered.emplace_back(begin, end); + }; + if (tileMode == TextureTileMode::kLinear) { + const auto rowBytes = static_cast(mip.width) * bytesPerElement; + for (std::uint32_t row = 0; row < mip.height; ++row) add(static_cast(row) * mip.pitchBytes, static_cast(row) * mip.pitchBytes + rowBytes); + return covered; + } + const auto block = ThinBlockLayout(tileMode, bytesPerElement); + const auto wholeColumns = std::min(mip.width / block[1], mip.blocksPerRow); + const auto wholeRows = mip.height / block[2]; + for (std::uint32_t row = 0; row < wholeRows && wholeColumns != 0; ++row) { + const auto first = static_cast(row) * mip.blocksPerRow; + add(first * block[0], (first + wholeColumns) * block[0]); + } + return covered; +} + ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t depth, std::uint32_t mipCount) { Require(width != 0 && height != 0 && depth != 0, "cannot compute layout for a zero-sized 3D texture"); Require(mipCount != 0 && mipCount <= 16u, "3D texture mip count is out of range"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp index efa6a44bf..f15aaf502 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp @@ -592,12 +592,12 @@ std::optional ShadowDestinationFor(const Context& context, co return destination; } -std::vector ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span> runs, std::span> tailBlocks) { +std::vector ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span> runs, std::span> tailBlocks, bool countReads) { std::vector result; if (runs.empty()) return result; const auto importPiece = [&](std::uint64_t begin, std::uint64_t end) { result.push_back({begin, end, import.buffer, surfaceBase + begin - import.base, false, nullptr}); - Stats().detiledImportBytes.fetch_add(end - begin, std::memory_order_relaxed); + if (countReads) Stats().detiledImportBytes.fetch_add(end - begin, std::memory_order_relaxed); }; std::shared_ptr shadow; if (UnitShadowEnabled()) { @@ -608,7 +608,7 @@ std::vector ShadowSources(const Context& context, const HostImport& i } if (shadow == nullptr) { for (const auto& [begin, end] : runs) importPiece(begin, end); - Stats().detiledImport.fetch_add(1, std::memory_order_relaxed); + if (countReads) Stats().detiledImport.fetch_add(1, std::memory_order_relaxed); return result; } const auto first = shadow->UnitOf(surfaceBase + runs.front().first); @@ -663,6 +663,7 @@ std::vector ShadowSources(const Context& context, const HostImport& i (isFresh ? shadowBytes : importBytes) += pieceEnd - pieceBegin; } } + if (!countReads) shadowBytes = importBytes = 0; if (shadowBytes != 0) { Stats().detiledShadow.fetch_add(1, std::memory_order_relaxed); Stats().detiledShadowBytes.fetch_add(shadowBytes, std::memory_order_relaxed); diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 732e684a1..8e07fcb18 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -1700,6 +1700,143 @@ void staleGenerationTests(const Device& device, Recorder& recorder) { #endif } +// A write-back stores the texels of an image and leaves every other byte of the surface's memory +// as it was: the padding of the tile blocks the surface covers only partly (its last block column +// and row), the tail block of a mip chain, the bytes between the mips and the pitch padding of a +// linear surface. Another surface over the memory, or the CPU, reads those bytes. Each case clears +// every mip of an image over memory holding a pattern no texel equals, stores it and reads the +// memory back (watched memory through the flush hook, after a store into the import's unit shadow): +// each 4-byte word is the clear texel or the pattern, and the texels are exactly the surface's. +// The 64 KiB-aligned tiled cases store through block windows, the others whole layers. +void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watched) { + const auto& base = device.GetContext(); + if (base.hostImportAlignment == 0) { + std::cout << "host imports unavailable: write-back padding not tested\n"; + return; + } + struct Case { + const char* name; + TextureTileMode tileMode; + std::uint32_t width; + std::uint32_t height; + std::uint32_t mipCount; + std::uint64_t offset; + }; + constexpr Case cases[] = { + {"64 KiB R_X 200x150", TextureTileMode::kR64KBX, 200, 150, 1, 0}, + {"64 KiB R_X 200x150, 3 mips", TextureTileMode::kR64KBX, 200, 150, 3, 0}, + {"64 KiB R_X 520x260, 6 mips", TextureTileMode::kR64KBX, 520, 260, 6, 0}, + {"64 KiB R_X 200x150 off a 64 KiB boundary, 3 mips", TextureTileMode::kR64KBX, 200, 150, 3, 4096}, + {"4 KiB S 100x70, 2 mips", TextureTileMode::kStandard4KB, 100, 70, 2, 0}, + {"linear 200x150", TextureTileMode::kLinear, 200, 150, 1, 0}, + }; + const auto pattern = [](std::size_t i) { return static_cast(i * 131u + 7u); }; + constexpr std::array red{255, 0, 0, 255}; + std::string failures; + for (const auto& test : cases) { + const std::string what = std::string(watched ? "(watched) " : "") + test.name; + GuestTextureResource resource{}; + resource.width = test.width; + resource.height = test.height; + resource.mipCount = test.mipCount; + resource.tileMode = test.tileMode; + resource.dimension = TextureDimension::k2D; + resource.format = 56; + resource.dstSelX = 4; + resource.dstSelY = 5; + resource.dstSelZ = 6; + resource.dstSelW = 7; + const auto geometry = DescribeSurface(resource); + const auto surfaceBytes = static_cast(geometry.guestBytes); + std::size_t texelCount = 0; + for (const auto& mip : geometry.mips) texelCount += static_cast(mip.width) * mip.height; + // The surface, then a 64 KiB unit past it that no store may reach. + const auto bytes = (static_cast(test.offset) + surfaceBytes + 65535) / 65536 * 65536 + 65536; + void* block = nullptr; + if (watched) { + block = AllocateWatched(bytes, 65536); + if (block == nullptr) { + std::cout << "no write watching: write-back padding in watched memory not tested\n"; + return; + } + } else { +#ifdef _WIN32 + block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + block = std::aligned_alloc(65536, bytes); +#endif + } + Require(block != nullptr, "cannot allocate the write-back padding block"); + auto* memory = static_cast(block); + for (std::size_t i = 0; i < bytes; ++i) memory[i] = pattern(i); + const auto blockAddress = reinterpret_cast(block); + const auto address = blockAddress + test.offset; + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + std::size_t bytes; + bool watched; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + if (watched) ReleaseWatched(block, bytes); + } + } unregister{base, block, blockAddress, bytes, watched}; + if (HostImportFor(base, blockAddress, bytes) == nullptr) { + std::cout << "host import of the write-back padding block refused: write-back padding not tested\n"; + return; + } + TextureDetiler detiler(base); + auto context = base; + context.detiler = &detiler; + resource.baseAddress = address; + { + auto image = std::make_shared(context, detiler, resource, 0); + const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, test.mipCount, 0, 1}; + const auto commands = recorder.Commands(); + recorder.Keep(image); + const VkClearColorValue value{{1.0f, 0.0f, 0.0f, 1.0f}}; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &value, 1, &range); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + image->MarkDirty(); + image->WriteBack(); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + std::vector read(bytes); + if (watched) AgcDriver::GuestMemory::Read(blockAddress, read); + else std::memcpy(read.data(), memory, bytes); + std::size_t texels = 0, garbage = 0, outside = 0; + for (std::size_t i = 0; i < bytes; i += 4) { + const bool inside = i >= test.offset && i < test.offset + surfaceBytes; + bool isTexel = inside, isPattern = true; + for (std::size_t c = 0; c < 4; ++c) { + const auto byte = std::to_integer(read[i + c]); + isTexel = isTexel && byte == red[c]; + isPattern = isPattern && byte == pattern(i + c); + } + if (isTexel) ++texels; + else if (!isPattern) ++(inside ? garbage : outside); + } + if (garbage != 0 || outside != 0 || texels != texelCount) { + failures += what + ": the write-back stored " + std::to_string(texels) + " texels of " + std::to_string(texelCount) + " and changed " + std::to_string(garbage) + " words no texel holds (" + std::to_string(outside) + " past the surface)\n"; + } + } + recorder.Sync(); + } + if (!failures.empty()) throw std::runtime_error("write-back padding:\n" + failures); + std::cout << "write-back padding" << (watched ? " (watched)" : "") << ": ok\n"; +} + void importWindowTests(const Device& device, Recorder& recorder) { using namespace AgcDriver::GuestMemory; const auto& base = device.GetContext(); @@ -2459,6 +2596,8 @@ int main() { unitShadowTests(device, recorder); storageRefreshTests(device, recorder, false); storageRefreshTests(device, recorder, true); + writeBackPaddingTests(device, recorder, false); + writeBackPaddingTests(device, recorder, true); importWatchTests(device); staleGenerationTests(device, recorder); importWindowTests(device, recorder); diff --git a/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp index 37b3b1a95..38324bf69 100644 --- a/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp @@ -1,7 +1,9 @@ #include "GraphicsTests.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include #include +#include #include #include #include @@ -22,6 +24,16 @@ void reject(TAction action, std::string_view reason) { throw std::runtime_error(std::string("expected texture tiling rejection: ") + std::string(reason)); } +std::uint64_t equationOffset(const TextureSwizzleEquation& equation, std::uint32_t x, std::uint32_t y, std::uint32_t z) { + std::uint64_t offset = 0; + for (std::uint32_t bit = 0; bit < 16; ++bit) { + const auto mask = equation.bits[bit]; + const auto selected = (x & (mask & 0xfffu)) ^ ((y << 12) & (mask & 0xfff000u)) ^ ((z << 24) & (mask & 0xff000000u)); + offset |= static_cast(std::popcount(selected) & 1u) << bit; + } + return offset; +} + } void RunTextureTilingTests() { @@ -149,6 +161,59 @@ void RunTextureTilingTests() { reject([&] { DescribeSurface(volume); }, "3D texture mip tails are not implemented"); } + { + // CoveredMipBytes against the XOR address equations: every element slot of a covered range + // holds exactly one element of the mip, and every tile block left out holds a slot no + // element does (the bytes a write-back must keep). + constexpr std::array modes{TextureTileMode::kZ64KBX, TextureTileMode::kS64KBX, TextureTileMode::kD64KBX, TextureTileMode::kR64KBX}; + constexpr std::array, 6> sizes{{{200, 150}, {1920, 1080}, {2432, 1368}, {960, 540}, {256, 128}, {300, 1}}}; + for (const auto mode : modes) { + for (std::uint32_t bytesPerElement = 1; bytesPerElement <= 16; bytesPerElement *= 2) { + const auto* equation = FindTextureSwizzleEquation(XorSwizzleMode(mode), bytesPerElement); + Require(equation != nullptr, "missing XOR swizzle equation"); + const auto block = ThinBlockLayout(mode, bytesPerElement); + for (const auto& [width, height] : sizes) { + if (static_cast(width) * height > (1u << 20) && mode != TextureTileMode::kR64KBX) continue; + const auto what = "XOR mode " + std::to_string(XorSwizzleMode(mode)) + ", " + std::to_string(bytesPerElement) + " bytes, " + std::to_string(width) + "x" + std::to_string(height); + for (const auto& mip : ComputeElementMipLayout(mode, bytesPerElement, width, height, 3)) { + const auto covered = CoveredMipBytes(mode, bytesPerElement, mip); + if (mip.tail) { + Require(covered.empty(), what + ": a tail mip has covered bytes"); + continue; + } + std::vector held(static_cast(mip.tiledSize / bytesPerElement), 0); + for (std::uint32_t y = 0; y < mip.height; ++y) { + for (std::uint32_t x = 0; x < mip.width; ++x) { + const auto offset = (static_cast(y / block[2]) * mip.blocksPerRow + x / block[1]) * block[0] + equationOffset(*equation, x, y, 0); + Require(offset % bytesPerElement == 0 && offset < mip.tiledSize, what + ": an element lies outside the mip"); + held[static_cast(offset / bytesPerElement)] += 1; + } + } + std::vector inCovered(static_cast(mip.tiledSize / block[0]), false); + std::uint64_t previous = 0; + for (const auto& [begin, end] : covered) { + Require(begin >= previous && begin < end && end <= mip.tiledSize && begin % block[0] == 0 && end % block[0] == 0, what + ": covered ranges are not ascending whole blocks"); + previous = end; + for (auto slot = begin / bytesPerElement; slot < end / bytesPerElement; ++slot) Require(held[static_cast(slot)] == 1, what + ": a covered byte holds no element or several"); + for (auto at = begin; at < end; at += block[0]) inCovered[static_cast(at / block[0])] = true; + } + for (std::size_t index = 0; index < inCovered.size(); ++index) { + if (inCovered[index]) continue; + const auto first = index * block[0] / bytesPerElement; + const auto last = (index + 1) * block[0] / bytesPerElement; + Require(std::any_of(held.begin() + static_cast(first), held.begin() + static_cast(last), [](std::uint8_t count) { return count == 0; }), what + ": a block every byte of which holds an element is left out"); + } + } + } + } + } + const auto linear = ComputeElementMipLayout(TextureTileMode::kLinear, 4, 200, 3, 1).front(); + const auto rows = CoveredMipBytes(TextureTileMode::kLinear, 4, linear); + Require(linear.pitchBytes > 800 && rows.size() == 3 && rows[1].first == linear.pitchBytes && rows[1].second == linear.pitchBytes + 800, "linear covered bytes are not the rows without their pitch padding"); + const auto dense = ComputeElementMipLayout(TextureTileMode::kLinear, 4, 256, 3, 1).front(); + Require(dense.pitchBytes == 1024 && CoveredMipBytes(TextureTileMode::kLinear, 4, dense) == std::vector>{{0, 3072}}, "linear rows without padding are not one range"); + } + reject([] { ComputeSurfaceSize({}, 1); }, "empty mip chain"); reject([] { ComputeSurfaceSize(ComputeMipLayout(TextureTileMode::kLinear, 1, 4, 4, 1), 0); }, "zero array layers"); }