[GpuMemory] Answer CPU-dirty range queries without the tracker lock
MayHaveCpuDirtyPages reads the per-block summary and the region masks without taking the region lock; a true answer still goes through the locked path. Regions also bump a CPU-dirty epoch whenever a page becomes CPU-dirty.
This commit is contained in:
parent
9fa858a82a
commit
defa313acc
|
|
@ -12,6 +12,11 @@ public sealed class CpuDirtySummary
|
|||
{
|
||||
private const int WordBits = 64;
|
||||
private readonly long[] _words = new long[(TrackerLayout.BlockCount + WordBits - 1) / WordBits];
|
||||
private long _epoch;
|
||||
|
||||
public long Epoch => Volatile.Read(ref _epoch);
|
||||
|
||||
public void NoteDirtied() => Interlocked.Increment(ref _epoch);
|
||||
|
||||
public void Set(ulong block, bool dirty)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -35,6 +35,34 @@ public sealed class GuestPageTracker
|
|||
});
|
||||
}
|
||||
|
||||
public bool MayHaveCpuDirtyPages(ulong vaddr, ulong size)
|
||||
{
|
||||
if (IsKnownCpuClean(vaddr, size))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var remaining = size;
|
||||
var index = vaddr / BlockBytes;
|
||||
var offset = vaddr % BlockBytes;
|
||||
while (remaining != 0)
|
||||
{
|
||||
var bytes = Math.Min(BlockBytes - offset, remaining);
|
||||
if (Volatile.Read(ref _regions[index]) is not { } region || region.IsModified(WriteOrigin.Cpu, offset, bytes))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
remaining -= bytes;
|
||||
offset = 0;
|
||||
index++;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public long CpuDirtyEpoch => _cpuDirtySummary.Epoch;
|
||||
|
||||
public bool HasGpuDirtyPages(ulong vaddr, ulong size)
|
||||
{
|
||||
RejectUploadCallbackReentry();
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ public sealed class TrackedRegion
|
|||
_writable.Fill();
|
||||
_readable.Fill();
|
||||
_summary?.Set(BaseAddress / BlockBytes, dirty: true);
|
||||
_summary?.NoteDirtied();
|
||||
}
|
||||
|
||||
public RegionLock Lock { get; } = new();
|
||||
|
|
@ -212,6 +213,11 @@ public sealed class TrackedRegion
|
|||
private void UpdateCpuProtection(bool track)
|
||||
{
|
||||
_summary?.Set(BaseAddress / BlockBytes, _cpuDirty.Any);
|
||||
if (_summary is not null && (_cpuDirty & ~_writable).Any)
|
||||
{
|
||||
_summary.NoteDirtied();
|
||||
}
|
||||
|
||||
var mask = _cpuDirty ^ _writable;
|
||||
_writable = _cpuDirty;
|
||||
if (mask.None)
|
||||
|
|
|
|||
|
|
@ -153,6 +153,25 @@ public sealed class GuestPageTrackerTests : IDisposable
|
|||
Release(address, Page * 2);
|
||||
}
|
||||
|
||||
[NativePageProtectionFact]
|
||||
public void LockFreeCpuDirtyQueryAgreesWithTheLockedQuery()
|
||||
{
|
||||
var address = Allocate(2);
|
||||
Assert.True(_tracker.MayHaveCpuDirtyPages(address, Page * 2));
|
||||
|
||||
_tracker.ForEachUploadRange(address, Page * 2, false, NoRange, NoUpload, preserveCpuWriteHotPages: false);
|
||||
Assert.False(_tracker.HasCpuDirtyPages(address, Page * 2));
|
||||
Assert.False(_tracker.MayHaveCpuDirtyPages(address, Page * 2));
|
||||
|
||||
_tracker.MarkCpuDirtyPages(address + Page + 16, 32);
|
||||
Assert.False(_tracker.MayHaveCpuDirtyPages(address, Page));
|
||||
Assert.True(_tracker.MayHaveCpuDirtyPages(address, Page * 2));
|
||||
Assert.True(_tracker.MayHaveCpuDirtyPages(address + Page + 64, 4));
|
||||
|
||||
_tracker.UntrackMemory(address, Page * 2);
|
||||
Release(address, Page * 2);
|
||||
}
|
||||
|
||||
// The lock-free block summary must agree with the precise page masks through every
|
||||
// CPU-dirty transition: upload, explicit mark, write-fault invalidation and GPU writes.
|
||||
[NativePageProtectionFact]
|
||||
|
|
|
|||
Loading…
Reference in New Issue