Merge pull request #619 from oneandonlydean/fix/agc-writeback-block-padding
fix(agc): keep the guest's bytes in the padding a storage write-back stores
This commit is contained in:
Dean
GitHub
commit
b0c2e05ba0
|
|
@ -308,6 +308,27 @@ private:
|
|||
// The windows moving the surface-relative runs: whole tile blocks of thin tiled mips, a tail
|
||||
// mip whole (its block holds every tail level), in slice order.
|
||||
std::vector<SliceWindow> sliceWindows(std::span<const std::pair<std::uint64_t, std::uint64_t>> runs) const;
|
||||
// The bytes of the surface-relative runs no element of the surface holds (the padding of partly
|
||||
// covered tile blocks and of linear rows, tail blocks, the bytes between mips; every byte of a
|
||||
// thick surface), ascending: a write-back keeps the guest's bytes there.
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> uncoveredBytes(std::span<const std::pair<std::uint64_t, std::uint64_t>> runs) const;
|
||||
// The seed of a write-back's tiled scratch where its copies would carry bytes no element holds:
|
||||
// those bytes' current contents (a fresh unit shadow's, else the import's), copied into the
|
||||
// scratch before the retile, so the copies out of the scratch store them unchanged. `copied`
|
||||
// lists the surface-relative [begin, end) ranges the write-back copies out of the scratch, each
|
||||
// with the scratch offset of its begin.
|
||||
struct CopiedBytes {
|
||||
std::uint64_t begin;
|
||||
std::uint64_t end;
|
||||
std::uint64_t scratch;
|
||||
};
|
||||
struct PaddingSeeds {
|
||||
std::vector<std::pair<VkBuffer, std::vector<VkBufferCopy>>> copies;
|
||||
std::vector<std::shared_ptr<ShadowSlab>> slabs;
|
||||
// Guest ranges read from the import.
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> importReads;
|
||||
};
|
||||
PaddingSeeds paddingSeeds(const HostImport& import, std::vector<CopiedBytes> copied) const;
|
||||
// The direct upload and GPU-direct write-back of the runs through their windows (block units);
|
||||
// both return the bytes moved. The upload reads each unit from its unit shadow while fresh,
|
||||
// else from the import (`discard`: the image has no layout yet, a whole-surface first upload).
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace AgcDriver::Graphics {
|
||||
|
|
@ -34,6 +35,11 @@ std::array<std::uint32_t, 3> ThickBlockExtent(TextureTileMode tileMode, std::uin
|
|||
// (the detiler's blockExtent). A mip's tiled bytes are its blocks in row-major order, blocksPerRow per
|
||||
// row (TileMipLayout), except the tail mips, which share one block. Throws for linear tiling.
|
||||
std::array<std::uint32_t, 3> ThinBlockLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement);
|
||||
// The bytes of a thin mip (mip-relative [begin, end), ascending and merged) that hold one of its
|
||||
// elements each: its whole tile blocks (every element of the block inside the mip), or its rows
|
||||
// without the pitch padding when linear. A tail mip has none (its block also holds the other tail
|
||||
// levels and their padding). A retile of the mip writes every byte of these and no other.
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> CoveredMipBytes(TextureTileMode tileMode, std::uint32_t bytesPerElement, const TileMipLayout& mip);
|
||||
|
||||
// 3D surface. Each slab holds the whole mip chain, smallest level first. Depth slice z of a level is detiled
|
||||
// from slab z / blockDepth (slabBytes apart) with the swizzle's slice input set to z, into linear
|
||||
|
|
|
|||
|
|
@ -100,7 +100,8 @@ bool AnyShadowedOverlaps(std::span<const std::pair<std::uint64_t, std::uint64_t>
|
|||
// from one buffer (a slab while the unit is fresh, else the import) at `offset` (byte x of the run
|
||||
// is at offset + (x - begin)), split at unit freshness and slab boundaries with one tracker query
|
||||
// over the span. A tail block (surface-relative, every unit of it) must have one source: a mixed
|
||||
// one is published (TailMip) and read from the import.
|
||||
// one is published (TailMip) and read from the import. `countReads` false: not counted as detiled
|
||||
// bytes (a write-back reading the current bytes of its padding).
|
||||
struct ShadowRun {
|
||||
std::uint64_t begin;
|
||||
std::uint64_t end;
|
||||
|
|
@ -109,7 +110,7 @@ struct ShadowRun {
|
|||
bool shadow;
|
||||
std::shared_ptr<ShadowSlab> slab;
|
||||
};
|
||||
std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, std::span<const std::pair<std::uint64_t, std::uint64_t>> tailBlocks);
|
||||
std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, std::span<const std::pair<std::uint64_t, std::uint64_t>> tailBlocks, bool countReads = true);
|
||||
|
||||
// The slab destination of a retile copy [begin, end) (guest addresses inside one slab: the caller
|
||||
// splits at SlabBoundary), making the slab under the budget; nullopt when refused (the caller writes
|
||||
|
|
|
|||
|
|
@ -1617,6 +1617,83 @@ std::vector<StorageTexture::SliceWindow> StorageTexture::sliceWindows(std::span<
|
|||
return windows;
|
||||
}
|
||||
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> StorageTexture::uncoveredBytes(std::span<const std::pair<std::uint64_t, std::uint64_t>> runs) const {
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> covered;
|
||||
if (!geometry.thick) {
|
||||
const auto elementBytes = BytesPerElement(descriptor.format);
|
||||
for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) {
|
||||
for (std::uint32_t level = 0; level < descriptor.mipCount; ++level) {
|
||||
if (!geometry.HasLayer(level, layer)) continue;
|
||||
const auto sliceBegin = geometry.GuestLayerOffset(layer) + mips[level].tiledOffset;
|
||||
for (const auto& [begin, end] : CoveredMipBytes(descriptor.tileMode, elementBytes, mips[level])) covered.emplace_back(sliceBegin + begin, sliceBegin + end);
|
||||
}
|
||||
}
|
||||
std::sort(covered.begin(), covered.end());
|
||||
}
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> sorted(runs.begin(), runs.end());
|
||||
std::sort(sorted.begin(), sorted.end());
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> uncovered;
|
||||
const auto add = [&](std::uint64_t begin, std::uint64_t end) {
|
||||
if (begin >= end) return;
|
||||
if (!uncovered.empty() && uncovered.back().second >= begin) uncovered.back().second = std::max(uncovered.back().second, end);
|
||||
else uncovered.emplace_back(begin, end);
|
||||
};
|
||||
std::size_t next = 0;
|
||||
for (const auto& [runBegin, runEnd] : sorted) {
|
||||
while (next < covered.size() && covered[next].second <= runBegin) ++next;
|
||||
auto at = runBegin;
|
||||
for (auto i = next; i < covered.size() && covered[i].first < runEnd; ++i) {
|
||||
add(at, std::min(covered[i].first, runEnd));
|
||||
at = std::max(at, covered[i].second);
|
||||
}
|
||||
add(at, runEnd);
|
||||
}
|
||||
return uncovered;
|
||||
}
|
||||
|
||||
StorageTexture::PaddingSeeds StorageTexture::paddingSeeds(const HostImport& import, std::vector<CopiedBytes> copied) const {
|
||||
PaddingSeeds seeds;
|
||||
std::sort(copied.begin(), copied.end(), [](const CopiedBytes& a, const CopiedBytes& b) { return a.begin < b.begin; });
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> runs;
|
||||
for (const auto& range : copied) runs.emplace_back(range.begin, range.end);
|
||||
const auto uncovered = uncoveredBytes(runs);
|
||||
if (uncovered.empty()) return seeds;
|
||||
std::vector<CopiedBytes> pieces;
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> pieceRuns;
|
||||
std::size_t next = 0;
|
||||
for (const auto& [begin, end] : uncovered) {
|
||||
while (next < copied.size() && copied[next].end <= begin) ++next;
|
||||
for (auto i = next; i < copied.size() && copied[i].begin < end; ++i) {
|
||||
const auto from = std::max(begin, copied[i].begin);
|
||||
const auto to = std::min(end, copied[i].end);
|
||||
if (from >= to) continue;
|
||||
pieces.push_back({from, to, copied[i].scratch + (from - copied[i].begin)});
|
||||
pieceRuns.emplace_back(from, to);
|
||||
}
|
||||
}
|
||||
if (pieces.empty()) return seeds;
|
||||
const auto sources = ShadowSources(context, import, descriptor.baseAddress, pieceRuns, {}, false);
|
||||
std::size_t first = 0;
|
||||
for (const auto& source : sources) {
|
||||
while (first < pieces.size() && pieces[first].end <= source.begin) ++first;
|
||||
for (auto i = first; i < pieces.size() && pieces[i].begin < source.end; ++i) {
|
||||
const auto from = std::max(source.begin, pieces[i].begin);
|
||||
const auto to = std::min(source.end, pieces[i].end);
|
||||
if (from >= to) continue;
|
||||
auto target = std::find_if(seeds.copies.begin(), seeds.copies.end(), [&](const auto& entry) { return entry.first == source.buffer; });
|
||||
if (target == seeds.copies.end()) target = seeds.copies.insert(seeds.copies.end(), {source.buffer, {}});
|
||||
target->second.push_back({source.offset + (from - source.begin), pieces[i].scratch + (from - pieces[i].begin), to - from});
|
||||
if (!source.shadow) {
|
||||
const auto guestBegin = descriptor.baseAddress + from;
|
||||
if (!seeds.importReads.empty() && seeds.importReads.back().second == guestBegin) seeds.importReads.back().second = descriptor.baseAddress + to;
|
||||
else seeds.importReads.emplace_back(guestBegin, descriptor.baseAddress + to);
|
||||
}
|
||||
}
|
||||
if (source.shadow && std::find(seeds.slabs.begin(), seeds.slabs.end(), source.slab) == seeds.slabs.end()) seeds.slabs.push_back(source.slab);
|
||||
}
|
||||
return seeds;
|
||||
}
|
||||
|
||||
std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, bool discard) {
|
||||
const auto elementBytes = BytesPerElement(descriptor.format);
|
||||
// Each run's pieces by source (a unit shadow's slab while fresh, else the import), the tail
|
||||
|
|
@ -1808,6 +1885,17 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
|
|||
}
|
||||
}
|
||||
std::sort(seeds.begin(), seeds.end(), [](const ShadowedRange& a, const ShadowedRange& b) { return a.begin < b.begin; });
|
||||
std::vector<CopiedBytes> copied;
|
||||
for (std::size_t i = 0; i < windows.size(); ++i) {
|
||||
const auto& window = windows[i];
|
||||
if (i != 0 && scratchPositions[i] == scratchPositions[i - 1]) continue;
|
||||
for (const auto& [from, to] : runs) {
|
||||
const auto begin = std::max(from, window.tiledBegin);
|
||||
const auto end = std::min(to, window.tiledEnd);
|
||||
if (begin < end) copied.push_back({begin, end, scratchPositions[i] + (begin - window.tiledBegin)});
|
||||
}
|
||||
}
|
||||
const auto padding = paddingSeeds(import, std::move(copied));
|
||||
// A queued label or key store inside a seeded unit lands before the seed copies the import.
|
||||
auto flushBegin = firstStored;
|
||||
auto flushEnd = lastStored;
|
||||
|
|
@ -1816,7 +1904,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
|
|||
flushEnd = std::max(flushEnd, seed.end);
|
||||
}
|
||||
auto linear = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
|
||||
auto tiledScratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
|
||||
auto tiledScratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
|
||||
detiler.BeginBatch();
|
||||
auto* recorder = Recorder::Active();
|
||||
std::unique_ptr<CommandBatch> batch;
|
||||
|
|
@ -1831,12 +1919,14 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
|
|||
recorder->Keep(linear);
|
||||
recorder->Keep(tiledScratch);
|
||||
for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab);
|
||||
for (const auto& slab : padding.slabs) recorder->Keep(slab);
|
||||
if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self));
|
||||
Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4);
|
||||
if (Recorder::BarrierValidate()) {
|
||||
const std::pair<VkImage, bool> read{image, false};
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> seedReads;
|
||||
for (const auto& seed : seeds) seedReads.emplace_back(seed.begin, seed.end);
|
||||
seedReads.insert(seedReads.end(), padding.importReads.begin(), padding.importReads.end());
|
||||
recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{seedReads, imported, std::span(&read, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT});
|
||||
}
|
||||
} else {
|
||||
|
|
@ -1853,7 +1943,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
|
|||
toSource.image = image;
|
||||
toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers};
|
||||
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource);
|
||||
if (!seeds.empty()) {
|
||||
if (!seeds.empty() || !padding.copies.empty()) {
|
||||
// The seeded units' import bytes (every earlier writer of them, host stores included)
|
||||
// precede the seed copies; the previous command's trailing barrier may have covered that.
|
||||
constexpr VkAccessFlags transferAccess = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
|
||||
|
|
@ -1869,6 +1959,9 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
|
|||
context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, import.buffer, slab->buffer, 1, &seed);
|
||||
NoteShadowSeed(begin, end);
|
||||
}
|
||||
// The padding's current bytes, under the retile that follows (the import-ready barrier
|
||||
// orders the scratch writes).
|
||||
for (const auto& [source, copies] : padding.copies) context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast<std::uint32_t>(copies.size()), copies.data());
|
||||
}
|
||||
std::vector<VkBufferImageCopy> regions;
|
||||
for (std::size_t i = 0; i < windows.size(); ++i) {
|
||||
|
|
@ -3224,7 +3317,10 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
|
|||
// untouched blocks are copied into the imported bytes in place, recorded behind the work that
|
||||
// produced the image; nothing crosses to the CPU.
|
||||
auto linear = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(sliceLinearBytes * arrayLayers), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
|
||||
auto tiledScratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
|
||||
auto tiledScratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
|
||||
std::vector<CopiedBytes> copied;
|
||||
for (const auto& [from, to] : keep) copied.push_back({from - descriptor.baseAddress, to - descriptor.baseAddress, from - descriptor.baseAddress});
|
||||
const auto padding = paddingSeeds(*import, std::move(copied));
|
||||
detiler.BeginBatch();
|
||||
auto* recorder = Recorder::Active();
|
||||
std::unique_ptr<CommandBatch> batch;
|
||||
|
|
@ -3239,12 +3335,13 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
|
|||
timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack);
|
||||
recorder->Keep(linear);
|
||||
recorder->Keep(tiledScratch);
|
||||
for (const auto& slab : padding.slabs) recorder->Keep(slab);
|
||||
// The image itself must outlive the recorded retile: the cache may evict it right after.
|
||||
if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self));
|
||||
Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4);
|
||||
Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, padding.copies.empty() ? 4 : 5);
|
||||
if (Recorder::BarrierValidate()) {
|
||||
const std::pair<VkImage, bool> read{image, false};
|
||||
recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{{}, keep, std::span(&read, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT});
|
||||
recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{padding.importReads, keep, std::span(&read, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT});
|
||||
}
|
||||
} else {
|
||||
batch = std::make_unique<CommandBatch>(context);
|
||||
|
|
@ -3264,6 +3361,13 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
|
|||
toSource.image = image;
|
||||
toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers};
|
||||
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource);
|
||||
if (!padding.copies.empty()) {
|
||||
// The current bytes of the copied ranges no element holds go under the retile (every
|
||||
// earlier writer of the import, host stores included, first; the import-ready barrier
|
||||
// below orders the scratch writes).
|
||||
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
|
||||
for (const auto& [source, copies] : padding.copies) context.Function<PFN_vkCmdCopyBuffer>("vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast<std::uint32_t>(copies.size()), copies.data());
|
||||
}
|
||||
const auto regions = CopyRegions(storedLayers);
|
||||
context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast<std::uint32_t>(regions.size()), regions.data());
|
||||
const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
|
||||
|
|
|
|||
|
|
@ -255,6 +255,30 @@ std::array<std::uint32_t, 3> ThinBlockLayout(TextureTileMode tileMode, std::uint
|
|||
return {block.blockSize, block.blockWidth, block.blockHeight};
|
||||
}
|
||||
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> CoveredMipBytes(TextureTileMode tileMode, std::uint32_t bytesPerElement, const TileMipLayout& mip) {
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> covered;
|
||||
if (mip.tail || mip.width == 0 || mip.height == 0) return covered;
|
||||
const auto add = [&](std::uint64_t begin, std::uint64_t end) {
|
||||
end = std::min(end, mip.tiledSize);
|
||||
if (begin >= end) return;
|
||||
if (!covered.empty() && covered.back().second == begin) covered.back().second = end;
|
||||
else covered.emplace_back(begin, end);
|
||||
};
|
||||
if (tileMode == TextureTileMode::kLinear) {
|
||||
const auto rowBytes = static_cast<std::uint64_t>(mip.width) * bytesPerElement;
|
||||
for (std::uint32_t row = 0; row < mip.height; ++row) add(static_cast<std::uint64_t>(row) * mip.pitchBytes, static_cast<std::uint64_t>(row) * mip.pitchBytes + rowBytes);
|
||||
return covered;
|
||||
}
|
||||
const auto block = ThinBlockLayout(tileMode, bytesPerElement);
|
||||
const auto wholeColumns = std::min(mip.width / block[1], mip.blocksPerRow);
|
||||
const auto wholeRows = mip.height / block[2];
|
||||
for (std::uint32_t row = 0; row < wholeRows && wholeColumns != 0; ++row) {
|
||||
const auto first = static_cast<std::uint64_t>(row) * mip.blocksPerRow;
|
||||
add(first * block[0], (first + wholeColumns) * block[0]);
|
||||
}
|
||||
return covered;
|
||||
}
|
||||
|
||||
ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t depth, std::uint32_t mipCount) {
|
||||
Require(width != 0 && height != 0 && depth != 0, "cannot compute layout for a zero-sized 3D texture");
|
||||
Require(mipCount != 0 && mipCount <= 16u, "3D texture mip count is out of range");
|
||||
|
|
|
|||
|
|
@ -592,12 +592,12 @@ std::optional<ShadowDestination> ShadowDestinationFor(const Context& context, co
|
|||
return destination;
|
||||
}
|
||||
|
||||
std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, std::span<const std::pair<std::uint64_t, std::uint64_t>> tailBlocks) {
|
||||
std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& import, std::uint64_t surfaceBase, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, std::span<const std::pair<std::uint64_t, std::uint64_t>> tailBlocks, bool countReads) {
|
||||
std::vector<ShadowRun> result;
|
||||
if (runs.empty()) return result;
|
||||
const auto importPiece = [&](std::uint64_t begin, std::uint64_t end) {
|
||||
result.push_back({begin, end, import.buffer, surfaceBase + begin - import.base, false, nullptr});
|
||||
Stats().detiledImportBytes.fetch_add(end - begin, std::memory_order_relaxed);
|
||||
if (countReads) Stats().detiledImportBytes.fetch_add(end - begin, std::memory_order_relaxed);
|
||||
};
|
||||
std::shared_ptr<UnitShadow> shadow;
|
||||
if (UnitShadowEnabled()) {
|
||||
|
|
@ -608,7 +608,7 @@ std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& i
|
|||
}
|
||||
if (shadow == nullptr) {
|
||||
for (const auto& [begin, end] : runs) importPiece(begin, end);
|
||||
Stats().detiledImport.fetch_add(1, std::memory_order_relaxed);
|
||||
if (countReads) Stats().detiledImport.fetch_add(1, std::memory_order_relaxed);
|
||||
return result;
|
||||
}
|
||||
const auto first = shadow->UnitOf(surfaceBase + runs.front().first);
|
||||
|
|
@ -663,6 +663,7 @@ std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& i
|
|||
(isFresh ? shadowBytes : importBytes) += pieceEnd - pieceBegin;
|
||||
}
|
||||
}
|
||||
if (!countReads) shadowBytes = importBytes = 0;
|
||||
if (shadowBytes != 0) {
|
||||
Stats().detiledShadow.fetch_add(1, std::memory_order_relaxed);
|
||||
Stats().detiledShadowBytes.fetch_add(shadowBytes, std::memory_order_relaxed);
|
||||
|
|
|
|||
|
|
@ -1700,6 +1700,143 @@ void staleGenerationTests(const Device& device, Recorder& recorder) {
|
|||
#endif
|
||||
}
|
||||
|
||||
// A write-back stores the texels of an image and leaves every other byte of the surface's memory
|
||||
// as it was: the padding of the tile blocks the surface covers only partly (its last block column
|
||||
// and row), the tail block of a mip chain, the bytes between the mips and the pitch padding of a
|
||||
// linear surface. Another surface over the memory, or the CPU, reads those bytes. Each case clears
|
||||
// every mip of an image over memory holding a pattern no texel equals, stores it and reads the
|
||||
// memory back (watched memory through the flush hook, after a store into the import's unit shadow):
|
||||
// each 4-byte word is the clear texel or the pattern, and the texels are exactly the surface's.
|
||||
// The 64 KiB-aligned tiled cases store through block windows, the others whole layers.
|
||||
void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watched) {
|
||||
const auto& base = device.GetContext();
|
||||
if (base.hostImportAlignment == 0) {
|
||||
std::cout << "host imports unavailable: write-back padding not tested\n";
|
||||
return;
|
||||
}
|
||||
struct Case {
|
||||
const char* name;
|
||||
TextureTileMode tileMode;
|
||||
std::uint32_t width;
|
||||
std::uint32_t height;
|
||||
std::uint32_t mipCount;
|
||||
std::uint64_t offset;
|
||||
};
|
||||
constexpr Case cases[] = {
|
||||
{"64 KiB R_X 200x150", TextureTileMode::kR64KBX, 200, 150, 1, 0},
|
||||
{"64 KiB R_X 200x150, 3 mips", TextureTileMode::kR64KBX, 200, 150, 3, 0},
|
||||
{"64 KiB R_X 520x260, 6 mips", TextureTileMode::kR64KBX, 520, 260, 6, 0},
|
||||
{"64 KiB R_X 200x150 off a 64 KiB boundary, 3 mips", TextureTileMode::kR64KBX, 200, 150, 3, 4096},
|
||||
{"4 KiB S 100x70, 2 mips", TextureTileMode::kStandard4KB, 100, 70, 2, 0},
|
||||
{"linear 200x150", TextureTileMode::kLinear, 200, 150, 1, 0},
|
||||
};
|
||||
const auto pattern = [](std::size_t i) { return static_cast<std::uint8_t>(i * 131u + 7u); };
|
||||
constexpr std::array<std::uint8_t, 4> red{255, 0, 0, 255};
|
||||
std::string failures;
|
||||
for (const auto& test : cases) {
|
||||
const std::string what = std::string(watched ? "(watched) " : "") + test.name;
|
||||
GuestTextureResource resource{};
|
||||
resource.width = test.width;
|
||||
resource.height = test.height;
|
||||
resource.mipCount = test.mipCount;
|
||||
resource.tileMode = test.tileMode;
|
||||
resource.dimension = TextureDimension::k2D;
|
||||
resource.format = 56;
|
||||
resource.dstSelX = 4;
|
||||
resource.dstSelY = 5;
|
||||
resource.dstSelZ = 6;
|
||||
resource.dstSelW = 7;
|
||||
const auto geometry = DescribeSurface(resource);
|
||||
const auto surfaceBytes = static_cast<std::size_t>(geometry.guestBytes);
|
||||
std::size_t texelCount = 0;
|
||||
for (const auto& mip : geometry.mips) texelCount += static_cast<std::size_t>(mip.width) * mip.height;
|
||||
// The surface, then a 64 KiB unit past it that no store may reach.
|
||||
const auto bytes = (static_cast<std::size_t>(test.offset) + surfaceBytes + 65535) / 65536 * 65536 + 65536;
|
||||
void* block = nullptr;
|
||||
if (watched) {
|
||||
block = AllocateWatched(bytes, 65536);
|
||||
if (block == nullptr) {
|
||||
std::cout << "no write watching: write-back padding in watched memory not tested\n";
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
||||
#else
|
||||
block = std::aligned_alloc(65536, bytes);
|
||||
#endif
|
||||
}
|
||||
Require(block != nullptr, "cannot allocate the write-back padding block");
|
||||
auto* memory = static_cast<std::uint8_t*>(block);
|
||||
for (std::size_t i = 0; i < bytes; ++i) memory[i] = pattern(i);
|
||||
const auto blockAddress = reinterpret_cast<std::uint64_t>(block);
|
||||
const auto address = blockAddress + test.offset;
|
||||
{
|
||||
GuestAllocations::Mutation mutation;
|
||||
mutation.Add(block, bytes, true, true);
|
||||
}
|
||||
struct Unregister {
|
||||
const Context& context;
|
||||
void* block;
|
||||
std::uint64_t address;
|
||||
std::size_t bytes;
|
||||
bool watched;
|
||||
~Unregister() {
|
||||
{
|
||||
GuestAllocations::Mutation mutation;
|
||||
mutation.Remove(block);
|
||||
}
|
||||
HostImportFor(context, address, bytes);
|
||||
if (watched) ReleaseWatched(block, bytes);
|
||||
}
|
||||
} unregister{base, block, blockAddress, bytes, watched};
|
||||
if (HostImportFor(base, blockAddress, bytes) == nullptr) {
|
||||
std::cout << "host import of the write-back padding block refused: write-back padding not tested\n";
|
||||
return;
|
||||
}
|
||||
TextureDetiler detiler(base);
|
||||
auto context = base;
|
||||
context.detiler = &detiler;
|
||||
resource.baseAddress = address;
|
||||
{
|
||||
auto image = std::make_shared<StorageTexture>(context, detiler, resource, 0);
|
||||
const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, test.mipCount, 0, 1};
|
||||
const auto commands = recorder.Commands();
|
||||
recorder.Keep(image);
|
||||
const VkClearColorValue value{{1.0f, 0.0f, 0.0f, 1.0f}};
|
||||
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
context.Function<PFN_vkCmdClearColorImage>("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &value, 1, &range);
|
||||
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
image->MarkDirty();
|
||||
image->WriteBack();
|
||||
recorder.Submit();
|
||||
device.WaitQueue();
|
||||
recorder.Sync();
|
||||
std::vector<std::byte> read(bytes);
|
||||
if (watched) AgcDriver::GuestMemory::Read(blockAddress, read);
|
||||
else std::memcpy(read.data(), memory, bytes);
|
||||
std::size_t texels = 0, garbage = 0, outside = 0;
|
||||
for (std::size_t i = 0; i < bytes; i += 4) {
|
||||
const bool inside = i >= test.offset && i < test.offset + surfaceBytes;
|
||||
bool isTexel = inside, isPattern = true;
|
||||
for (std::size_t c = 0; c < 4; ++c) {
|
||||
const auto byte = std::to_integer<std::uint8_t>(read[i + c]);
|
||||
isTexel = isTexel && byte == red[c];
|
||||
isPattern = isPattern && byte == pattern(i + c);
|
||||
}
|
||||
if (isTexel) ++texels;
|
||||
else if (!isPattern) ++(inside ? garbage : outside);
|
||||
}
|
||||
if (garbage != 0 || outside != 0 || texels != texelCount) {
|
||||
failures += what + ": the write-back stored " + std::to_string(texels) + " texels of " + std::to_string(texelCount) + " and changed " + std::to_string(garbage) + " words no texel holds (" + std::to_string(outside) + " past the surface)\n";
|
||||
}
|
||||
}
|
||||
recorder.Sync();
|
||||
}
|
||||
if (!failures.empty()) throw std::runtime_error("write-back padding:\n" + failures);
|
||||
std::cout << "write-back padding" << (watched ? " (watched)" : "") << ": ok\n";
|
||||
}
|
||||
|
||||
void importWindowTests(const Device& device, Recorder& recorder) {
|
||||
using namespace AgcDriver::GuestMemory;
|
||||
const auto& base = device.GetContext();
|
||||
|
|
@ -2459,6 +2596,8 @@ int main() {
|
|||
unitShadowTests(device, recorder);
|
||||
storageRefreshTests(device, recorder, false);
|
||||
storageRefreshTests(device, recorder, true);
|
||||
writeBackPaddingTests(device, recorder, false);
|
||||
writeBackPaddingTests(device, recorder, true);
|
||||
importWatchTests(device);
|
||||
staleGenerationTests(device, recorder);
|
||||
importWindowTests(device, recorder);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
#include "GraphicsTests.hpp"
|
||||
#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp"
|
||||
#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <bit>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
|
|
@ -22,6 +24,16 @@ void reject(TAction action, std::string_view reason) {
|
|||
throw std::runtime_error(std::string("expected texture tiling rejection: ") + std::string(reason));
|
||||
}
|
||||
|
||||
std::uint64_t equationOffset(const TextureSwizzleEquation& equation, std::uint32_t x, std::uint32_t y, std::uint32_t z) {
|
||||
std::uint64_t offset = 0;
|
||||
for (std::uint32_t bit = 0; bit < 16; ++bit) {
|
||||
const auto mask = equation.bits[bit];
|
||||
const auto selected = (x & (mask & 0xfffu)) ^ ((y << 12) & (mask & 0xfff000u)) ^ ((z << 24) & (mask & 0xff000000u));
|
||||
offset |= static_cast<std::uint64_t>(std::popcount(selected) & 1u) << bit;
|
||||
}
|
||||
return offset;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void RunTextureTilingTests() {
|
||||
|
|
@ -149,6 +161,59 @@ void RunTextureTilingTests() {
|
|||
reject([&] { DescribeSurface(volume); }, "3D texture mip tails are not implemented");
|
||||
}
|
||||
|
||||
{
|
||||
// CoveredMipBytes against the XOR address equations: every element slot of a covered range
|
||||
// holds exactly one element of the mip, and every tile block left out holds a slot no
|
||||
// element does (the bytes a write-back must keep).
|
||||
constexpr std::array<TextureTileMode, 4> modes{TextureTileMode::kZ64KBX, TextureTileMode::kS64KBX, TextureTileMode::kD64KBX, TextureTileMode::kR64KBX};
|
||||
constexpr std::array<std::pair<std::uint32_t, std::uint32_t>, 6> sizes{{{200, 150}, {1920, 1080}, {2432, 1368}, {960, 540}, {256, 128}, {300, 1}}};
|
||||
for (const auto mode : modes) {
|
||||
for (std::uint32_t bytesPerElement = 1; bytesPerElement <= 16; bytesPerElement *= 2) {
|
||||
const auto* equation = FindTextureSwizzleEquation(XorSwizzleMode(mode), bytesPerElement);
|
||||
Require(equation != nullptr, "missing XOR swizzle equation");
|
||||
const auto block = ThinBlockLayout(mode, bytesPerElement);
|
||||
for (const auto& [width, height] : sizes) {
|
||||
if (static_cast<std::uint64_t>(width) * height > (1u << 20) && mode != TextureTileMode::kR64KBX) continue;
|
||||
const auto what = "XOR mode " + std::to_string(XorSwizzleMode(mode)) + ", " + std::to_string(bytesPerElement) + " bytes, " + std::to_string(width) + "x" + std::to_string(height);
|
||||
for (const auto& mip : ComputeElementMipLayout(mode, bytesPerElement, width, height, 3)) {
|
||||
const auto covered = CoveredMipBytes(mode, bytesPerElement, mip);
|
||||
if (mip.tail) {
|
||||
Require(covered.empty(), what + ": a tail mip has covered bytes");
|
||||
continue;
|
||||
}
|
||||
std::vector<std::uint8_t> held(static_cast<std::size_t>(mip.tiledSize / bytesPerElement), 0);
|
||||
for (std::uint32_t y = 0; y < mip.height; ++y) {
|
||||
for (std::uint32_t x = 0; x < mip.width; ++x) {
|
||||
const auto offset = (static_cast<std::uint64_t>(y / block[2]) * mip.blocksPerRow + x / block[1]) * block[0] + equationOffset(*equation, x, y, 0);
|
||||
Require(offset % bytesPerElement == 0 && offset < mip.tiledSize, what + ": an element lies outside the mip");
|
||||
held[static_cast<std::size_t>(offset / bytesPerElement)] += 1;
|
||||
}
|
||||
}
|
||||
std::vector<bool> inCovered(static_cast<std::size_t>(mip.tiledSize / block[0]), false);
|
||||
std::uint64_t previous = 0;
|
||||
for (const auto& [begin, end] : covered) {
|
||||
Require(begin >= previous && begin < end && end <= mip.tiledSize && begin % block[0] == 0 && end % block[0] == 0, what + ": covered ranges are not ascending whole blocks");
|
||||
previous = end;
|
||||
for (auto slot = begin / bytesPerElement; slot < end / bytesPerElement; ++slot) Require(held[static_cast<std::size_t>(slot)] == 1, what + ": a covered byte holds no element or several");
|
||||
for (auto at = begin; at < end; at += block[0]) inCovered[static_cast<std::size_t>(at / block[0])] = true;
|
||||
}
|
||||
for (std::size_t index = 0; index < inCovered.size(); ++index) {
|
||||
if (inCovered[index]) continue;
|
||||
const auto first = index * block[0] / bytesPerElement;
|
||||
const auto last = (index + 1) * block[0] / bytesPerElement;
|
||||
Require(std::any_of(held.begin() + static_cast<std::ptrdiff_t>(first), held.begin() + static_cast<std::ptrdiff_t>(last), [](std::uint8_t count) { return count == 0; }), what + ": a block every byte of which holds an element is left out");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const auto linear = ComputeElementMipLayout(TextureTileMode::kLinear, 4, 200, 3, 1).front();
|
||||
const auto rows = CoveredMipBytes(TextureTileMode::kLinear, 4, linear);
|
||||
Require(linear.pitchBytes > 800 && rows.size() == 3 && rows[1].first == linear.pitchBytes && rows[1].second == linear.pitchBytes + 800, "linear covered bytes are not the rows without their pitch padding");
|
||||
const auto dense = ComputeElementMipLayout(TextureTileMode::kLinear, 4, 256, 3, 1).front();
|
||||
Require(dense.pitchBytes == 1024 && CoveredMipBytes(TextureTileMode::kLinear, 4, dense) == std::vector<std::pair<std::uint64_t, std::uint64_t>>{{0, 3072}}, "linear rows without padding are not one range");
|
||||
}
|
||||
|
||||
reject([] { ComputeSurfaceSize({}, 1); }, "empty mip chain");
|
||||
reject([] { ComputeSurfaceSize(ComputeMipLayout(TextureTileMode::kLinear, 1, 4, 4, 1), 0); }, "zero array layers");
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue