perf(agc): index the pending storage images by guest range

Every guest memory read checks the pending storage images for results it
must store first (FlushPending, overlappingPending and the overlap test of
shader resources), each by a scan of the whole registry, and each scan read
the thread-local `refreshing` once per image (an emutls call with MinGW).
The registry now keeps a sorted, merged index of the ranges its images
cover, rebuilt on the first query after a change: a query that overlaps
none of them returns without a scan. The scans read `refreshing` once.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michon 2026-10-02 21:01:26 +02:00
parent 313de1c5a3
commit 85c8beea48
1 changed files with 58 additions and 10 deletions

View File

@ -708,10 +708,53 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler,
namespace {
class PendingList {
public:
using iterator = std::vector<StorageTexture*>::iterator;
iterator begin() { return textures.begin(); }
iterator end() { return textures.end(); }
void push_back(StorageTexture* texture) {
textures.push_back(texture);
++version;
}
iterator erase(iterator it) {
++version;
return textures.erase(it);
}
void remove(const StorageTexture* texture) {
if (std::erase(textures, texture) != 0) ++version;
}
bool MayOverlap(std::uint64_t address, std::size_t bytes) {
if (textures.empty() || bytes == 0) return false;
if (indexed != version) rebuild();
const auto end = bytes > std::numeric_limits<std::uint64_t>::max() - address ? std::numeric_limits<std::uint64_t>::max() : address + bytes;
const auto it = std::upper_bound(ranges.begin(), ranges.end(), address, [](std::uint64_t value, const std::pair<std::uint64_t, std::uint64_t>& range) { return value < range.second; });
return it != ranges.end() && it->first < end;
}
private:
void rebuild() {
ranges.clear();
for (const auto* texture : textures) ranges.emplace_back(texture->Descriptor().baseAddress, texture->Descriptor().baseAddress + texture->GuestBytes());
std::sort(ranges.begin(), ranges.end());
std::size_t merged = 0;
for (const auto& range : ranges) {
if (merged != 0 && range.first <= ranges[merged - 1].second) ranges[merged - 1].second = std::max(ranges[merged - 1].second, range.second);
else ranges[merged++] = range;
}
ranges.resize(merged);
indexed = version;
}
std::vector<StorageTexture*> textures;
std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges;
std::uint64_t version = 0;
std::uint64_t indexed = ~std::uint64_t{0};
};
// Storage images whose results have not reached guest memory yet.
struct PendingWrites {
std::mutex mutex;
std::vector<StorageTexture*> textures;
PendingList textures;
// Images taken out of `textures` by a FlushPending still storing them (see adjacentPendingUnchanged).
std::vector<StorageTexture*> flushing;
};
@ -1925,7 +1968,7 @@ void StorageTexture::reconcilePending() {
if (any == dirty) return;
dirty = any;
if (any) pending.textures.push_back(this);
else std::erase(pending.textures, this);
else pending.textures.remove(this);
BumpPendingSerial();
}
@ -1940,7 +1983,7 @@ void StorageTexture::Flush() {
std::lock_guard lock(pending.mutex);
if (!dirty) return;
dirty = false;
std::erase(pending.textures, this);
pending.textures.remove(this);
BumpPendingSerial();
}
std::lock_guard gpu(GuestMemory::GpuMutex());
@ -1972,9 +2015,10 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons
{
auto& pending = Pending();
std::lock_guard lock(pending.mutex);
for (auto it = pending.textures.begin(); it != pending.textures.end();) {
const auto* exempt = refreshing;
for (auto it = pending.textures.MayOverlap(address, bytes) ? pending.textures.begin() : pending.textures.end(); it != pending.textures.end();) {
auto* texture = *it;
if (texture != except && texture != refreshing && texture->overlaps(address, bytes)) {
if (texture != except && texture != exempt && texture->overlaps(address, bytes)) {
if (!texture->pendingUnitInside(address, bytes)) {
pretestSkipped.fetch_add(1, std::memory_order_relaxed);
++it;
@ -2063,6 +2107,7 @@ std::shared_ptr<StorageTexture> StorageTexture::FindPending(std::uint64_t addres
bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const StorageTexture* except) {
auto& pending = Pending();
std::lock_guard lock(pending.mutex);
if (!pending.textures.MayOverlap(address, bytes)) return false;
for (const auto* texture : pending.textures) {
// A free function (declared in ShaderResources.hpp): the overlap is computed from the public
// surface description rather than the private helper.
@ -2104,8 +2149,10 @@ std::vector<std::shared_ptr<StorageTexture>> StorageTexture::overlappingPending(
std::vector<std::shared_ptr<StorageTexture>> overlapping;
auto& pending = Pending();
std::lock_guard lock(pending.mutex);
if (!pending.textures.MayOverlap(address, bytes)) return overlapping;
const auto* exempt = refreshing;
for (auto* texture : pending.textures) {
if (texture == refreshing || !texture->overlaps(address, bytes) || !texture->pendingUnitInside(address, bytes)) continue;
if (texture == exempt || !texture->overlaps(address, bytes) || !texture->pendingUnitInside(address, bytes)) continue;
if (auto alive = texture->weak_from_this().lock()) overlapping.push_back(std::move(alive));
}
return overlapping;
@ -2280,10 +2327,11 @@ std::size_t StorageTexture::DiscardPendingInside(std::uint64_t address, std::siz
std::lock_guard lock(pending.mutex);
std::size_t discarded = 0;
const auto end = address + bytes;
const auto* exempt = refreshing;
for (auto it = pending.textures.begin(); it != pending.textures.end();) {
auto* texture = *it;
const auto begin = texture->descriptor.baseAddress;
if (texture != refreshing && begin >= address && begin + texture->guestBytes <= end) {
if (texture != exempt && begin >= address && begin + texture->guestBytes <= end) {
// The content no longer matches guest memory anywhere: the next use re-uploads once the
// fill stamped the range, never from a matching `original` (the fill may not have
// touched the bytes a CPU-path upload copied).
@ -2292,7 +2340,7 @@ std::size_t StorageTexture::DiscardPendingInside(std::uint64_t address, std::siz
texture->originalValid = false;
it = pending.textures.erase(it);
++discarded;
} else if (texture != refreshing && texture->blockUnits && texture->overlaps(address, bytes)) {
} else if (texture != exempt && texture->blockUnits && texture->overlaps(address, bytes)) {
// Its units wholly inside the range are dead too (re-uploaded from the range's bytes at
// the next use, never from a matching `original`: the dropped units hold results the
// bytes never received); the others stay pending.
@ -2478,7 +2526,7 @@ bool StorageTexture::clearByKeysFill(DccKeys keys, std::uint8_t key) {
layerPending.assign(trackedLayers, false);
if (dirty) {
dirty = false;
std::erase(pending.textures, this);
pending.textures.remove(this);
BumpPendingSerial();
}
}
@ -3350,7 +3398,7 @@ StorageTexture::~StorageTexture() {
std::lock_guard lock(pending.mutex);
if (dirty) {
dirty = false;
std::erase(pending.textures, this);
pending.textures.remove(this);
BumpPendingSerial();
static std::atomic<int> reports{0};
if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[gpu] storage image 0x%llx destroyed with GPU results pending\n", static_cast<unsigned long long>(descriptor.baseAddress));